diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..cab4b3c7cab634c56f79e835a1f81de804ead504 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +*.axmodel filter=lfs diff=lfs merge=lfs -text +*.png filter=lfs diff=lfs merge=lfs -text +*.jpg filter=lfs diff=lfs merge=lfs -text diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..9d62893a933c319ab6307194c6604bc443ebb4e3 --- /dev/null +++ b/.gitignore @@ -0,0 +1,171 @@ +# Byte-compiled / optimized / DLL files +output* +logs* +taming* +samples* +datasets* +asset* +_* +logs* +__pycache__/ +*.py[cod] +*$py.class +scripts_demo* +compiled_* +onnx-* + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/VideoX-Fun/.gitignore b/VideoX-Fun/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..9d62893a933c319ab6307194c6604bc443ebb4e3 --- /dev/null +++ b/VideoX-Fun/.gitignore @@ -0,0 +1,171 @@ +# Byte-compiled / optimized / DLL files +output* +logs* +taming* +samples* +datasets* +asset* +_* +logs* +__pycache__/ +*.py[cod] +*$py.class +scripts_demo* +compiled_* +onnx-* + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/VideoX-Fun/Dockerfile.ds b/VideoX-Fun/Dockerfile.ds new file mode 100644 index 0000000000000000000000000000000000000000..a7e186db0303ed39c0cc85e4dae6759acada60f6 --- /dev/null +++ b/VideoX-Fun/Dockerfile.ds @@ -0,0 +1,52 @@ +FROM nvidia/cuda:12.1.0-cudnn8-devel-ubuntu22.04 +ENV DEBIAN_FRONTEND noninteractive + +RUN rm -r /etc/apt/sources.list.d/ + +RUN apt-get update -y && apt-get install -y \ + libgl1 libglib2.0-0 google-perftools \ + sudo wget git git-lfs vim tig pkg-config libcairo2-dev \ + aria2 telnet curl net-tools iputils-ping jq \ + python3-pip python-is-python3 python3.10-venv tzdata lsof zip tmux +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository ppa:ubuntuhandbook1/ffmpeg6 && \ + apt-get update && \ + apt-get install -y ffmpeg + +RUN pip3 install --upgrade pip -i https://mirrors.aliyun.com/pypi/simple/ + +# add all extensions +RUN pip install wandb tqdm GitPython==3.1.32 Pillow==9.5.0 setuptools --upgrade -i https://mirrors.aliyun.com/pypi/simple/ + +RUN pip install torch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 --index-url https://download.pytorch.org/whl/cu118 +RUN pip install xformers==0.0.27.post2 --index-url https://download.pytorch.org/whl/cu118 + +# install vllm (video-caption) +RUN pip install vllm==0.6.3 + +# install requirements (video-caption) +WORKDIR /root/ +COPY easyanimate/video_caption/requirements.txt /root/requirements-video_caption.txt +RUN pip install -r /root/requirements-video_caption.txt +RUN rm /root/requirements-video_caption.txt + +RUN pip install -U http://eas-data.oss-cn-shanghai.aliyuncs.com/sdk/allspark-0.15-py2.py3-none-any.whl +RUN pip install -e git+https://github.com/CompVis/taming-transformers.git@master#egg=taming-transformers +RUN pip install came-pytorch deepspeed pytorch_lightning==1.9.4 func_timeout -i https://mirrors.aliyun.com/pypi/simple/ + +# install requirements +RUN pip install bitsandbytes mamba-ssm causal-conv1d>=1.4.0 -i https://mirrors.aliyun.com/pypi/simple/ +RUN pip install ipykernel -i https://mirrors.aliyun.com/pypi/simple/ +COPY ./requirements.txt /root/requirements.txt +RUN pip install -r /root/requirements.txt -i https://mirrors.aliyun.com/pypi/simple/ +RUN rm -rf /root/requirements.txt + +# install package patches (video-caption) +COPY easyanimate/video_caption/package_patches/easyocr_detection_patched.py /usr/local/lib/python3.10/dist-packages/easyocr/detection.py +COPY easyanimate/video_caption/package_patches/vila_siglip_encoder_patched.py /usr/local/lib/python3.10/dist-packages/llava/model/multimodal_encoder/siglip_encoder.py + +ENV PYTHONUNBUFFERED 1 +ENV NVIDIA_DISABLE_REQUIRE 1 + +WORKDIR /root/ \ No newline at end of file diff --git a/VideoX-Fun/LICENSE b/VideoX-Fun/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..261eeb9e9f8b2b4b0d119366dda99c6fd7d35c64 --- /dev/null +++ b/VideoX-Fun/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/VideoX-Fun/README.md b/VideoX-Fun/README.md new file mode 100644 index 0000000000000000000000000000000000000000..bbc547c42af48aa914ea0f04fb2e0cefe216364f --- /dev/null +++ b/VideoX-Fun/README.md @@ -0,0 +1,697 @@ +# VideoX-Fun + +😊 Welcome! + +CogVideoX-Fun: +[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-yellow)](https://huggingface.co/spaces/alibaba-pai/CogVideoX-Fun-5b) + +Wan-Fun: +[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-yellow)](https://huggingface.co/spaces/alibaba-pai/Wan2.1-Fun-1.3B-InP) + +English | [简体中文](./README_zh-CN.md) | [日本語](./README_ja-JP.md) + +# Table of Contents +- [Table of Contents](#table-of-contents) +- [Introduction](#introduction) +- [Quick Start](#quick-start) +- [Video Result](#video-result) +- [How to use](#how-to-use) +- [Model zoo](#model-zoo) +- [Reference](#reference) +- [License](#license) + +# Introduction +VideoX-Fun is a video generation pipeline that can be used to generate AI images and videos, as well as to train baseline and Lora models for Diffusion Transformer. We support direct prediction from pre-trained baseline models to generate videos with different resolutions, durations, and FPS. Additionally, we also support users in training their own baseline and Lora models to perform specific style transformations. + +We will support quick pull-ups from different platforms, refer to [Quick Start](#quick-start). + +What's New: +- Added support for Wan 2.2 series models, Wan-VACE control model, Fantasy Talking digital human model, Qwen-Image, Flux image generation models, and more. [2025.10.16] +- Update Wan2.1-Fun-V1.1: Support for 14B and 1.3B model Control + Reference Image models, support for camera control, and the Inpaint model has been retrained for improved performance. [2025.04.25] +- Update Wan2.1-Fun-V1.0: Support I2V and Control models for 14B and 1.3B models, with support for start and end frame prediction. [2025.03.26] +- Update CogVideoX-Fun-V1.5: Upload I2V model and related training/prediction code. [2024.12.16] +- Reward Lora Support: Train Lora using reward backpropagation techniques to optimize generated videos, making them better aligned with human preferences. [More Information](scripts/README_TRAIN_REWARD.md). New version of the control model supports various control conditions such as Canny, Depth, Pose, MLSD, etc. [2024.11.21] +- Diffusers Support: CogVideoX-Fun Control is now supported in diffusers. Thanks to [a-r-r-o-w](https://github.com/a-r-r-o-w) for contributing support in this [PR](https://github.com/huggingface/diffusers/pull/9671). Check out the [documentation](https://huggingface.co/docs/diffusers/main/en/api/pipelines/cogvideox) for more details. [2024.10.16] +- Update CogVideoX-Fun-V1.1: Retrain i2v model, add Noise to increase the motion amplitude of the video. Upload control model training code and Control model. [2024.09.29] +- Update CogVideoX-Fun-V1.0: Initial code release! Now supports Windows and Linux. Supports video generation at arbitrary resolutions from 256x256x49 to 1024x1024x49 for 2B and 5B models. [2024.09.18] + +Function： +- [Data Preprocessing](#data-preprocess) +- [Train DiT](#dit-train) +- [Video Generation](#video-gen) + +Our UI interface is as follows: +![ui](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/ui.jpg) + +# Quick Start +### 1. Cloud usage: AliyunDSW/Docker +#### a. From AliyunDSW +DSW has free GPU time, which can be applied once by a user and is valid for 3 months after applying. + +Aliyun provide free GPU time in [Freetier](https://free.aliyun.com/?product=9602825&crowd=enterprise&spm=5176.28055625.J_5831864660.1.e939154aRgha4e&scm=20140722.M_9974135.P_110.MO_1806-ID_9974135-MID_9974135-CID_30683-ST_8512-V_1), get it and use in Aliyun PAI-DSW to start CogVideoX-Fun within 5min! + +[![DSW Notebook](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/asset/dsw.png)](https://gallery.pai-ml.com/#/preview/deepLearning/cv/cogvideox_fun) + +#### b. From ComfyUI +Our ComfyUI is as follows, please refer to [ComfyUI README](comfyui/README.md) for details. +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/cogvideoxfunv1_workflow_i2v.jpg) + +#### c. From docker +If you are using docker, please make sure that the graphics card driver and CUDA environment have been installed correctly in your machine. + +Then execute the following commands in this way: + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# clone code +git clone https://github.com/aigc-apps/VideoX-Fun.git + +# enter VideoX-Fun's dir +cd VideoX-Fun + +# download weights +mkdir models/Diffusion_Transformer +mkdir models/Personalized_Model + +# Please use the hugginface link or modelscope link to download the model. +# CogVideoX-Fun +# https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-InP +# https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-InP + +# Wan +# https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-InP +# https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP +``` + +### 2. Local install: Environment Check/Downloading/Installation +#### a. Environment Check +We have verified this repo execution on the following environment: + +The detailed of Windows: +- OS: Windows 10 +- python: python3.10 & python3.11 +- pytorch: torch2.2.0 +- CUDA: 11.8 & 12.1 +- CUDNN: 8+ +- GPU： Nvidia-3060 12G & Nvidia-3090 24G + +The detailed of Linux: +- OS: Ubuntu 20.04, CentOS +- python: python3.10 & python3.11 +- pytorch: torch2.2.0 +- CUDA: 11.8 & 12.1 +- CUDNN: 8+ +- GPU：Nvidia-V100 16G & Nvidia-A10 24G & Nvidia-A100 40G & Nvidia-A100 80G + +We need about 60GB available on disk (for saving weights), please check! + +#### b. Weights +We'd better place the [weights](#model-zoo) along the specified path: + +**Via ComfyUI**: +Put the models into the ComfyUI weights folder `ComfyUI/models/Fun_Models/`: +``` +📦 ComfyUI/ +├── 📂 models/ +│ └── 📂 Fun_Models/ +│ ├── 📂 CogVideoX-Fun-V1.1-2b-InP/ +│ ├── 📂 CogVideoX-Fun-V1.1-5b-InP/ +│ ├── 📂 Wan2.1-Fun-14B-InP +│ └── 📂 Wan2.1-Fun-1.3B-InP/ +``` + +**Run its own python file or UI interface**: +``` +📦 models/ +├── 📂 Diffusion_Transformer/ +│ ├── 📂 CogVideoX-Fun-V1.1-2b-InP/ +│ ├── 📂 CogVideoX-Fun-V1.1-5b-InP/ +│ ├── 📂 Wan2.1-Fun-14B-InP +│ └── 📂 Wan2.1-Fun-1.3B-InP/ +├── 📂 Personalized_Model/ +│ └── your trained trainformer model / your trained lora model (for UI load) +``` + +# Video Result + +### Wan2.1-Fun-V1.1-14B-InP && Wan2.1-Fun-V1.1-1.3B-InP + + + + + + + + +
+ + + + + + + +
+ + + + + + + + +
+ + + + + + + +
+ +### Wan2.1-Fun-V1.1-14B-Control && Wan2.1-Fun-V1.1-1.3B-Control + +Generic Control Video + Reference Image: + + + + + + + + + + + + +
+ Reference Image + + Control Video + + Wan2.1-Fun-V1.1-14B-Control + + Wan2.1-Fun-V1.1-1.3B-Control +
+ + + + + + + +
+ + +Generic Control Video (Canny, Pose, Depth, etc.) and Trajectory Control: + + + + + + +
+ + + + + +
+ + + + + + + + + + + +
+ + + + + +
+ + + + + +
+ +### Wan2.1-Fun-V1.1-14B-Control-Camera && Wan2.1-Fun-V1.1-1.3B-Control-Camera + + + + + + + + + + + + + + + + + + + +
+ Pan Up + + Pan Left + + Pan Right +
+ + + + + +
+ Pan Down + + Pan Up + Pan Left + + Pan Up + Pan Right +
+ + + + + +
+ +### CogVideoX-Fun-V1.1-5B + +Resolution-1024 + + + + + + + + +
+ + + + + + + +
+ + +Resolution-768 + + + + + + + + +
+ + + + + + + +
+ +Resolution-512 + + + + + + + + +
+ + + + + + + +
+ +### CogVideoX-Fun-V1.1-5B-Control + + + + + + + + + + + + + + + + +
+ + + + + +
+ A young woman with beautiful clear eyes and blonde hair, wearing white clothes and twisting her body, with the camera focused on her face. High quality, masterpiece, best quality, high resolution, ultra-fine, dreamlike. + + A young woman with beautiful clear eyes and blonde hair, wearing white clothes and twisting her body, with the camera focused on her face. High quality, masterpiece, best quality, high resolution, ultra-fine, dreamlike. + + A young bear. +
+ + + + + +
+ +# How to Use + +

1. Generation

+ +#### a. GPU Memory Optimization +Since Wan2.1 has a very large number of parameters, we need to consider memory optimization strategies to adapt to consumer-grade GPUs. We provide `GPU_memory_mode` for each prediction file, allowing you to choose between `model_cpu_offload`, `model_cpu_offload_and_qfloat8`, and `sequential_cpu_offload`. This solution is also applicable to CogVideoX-Fun generation. + +- `model_cpu_offload`: The entire model is moved to the CPU after use, saving some GPU memory. +- `model_cpu_offload_and_qfloat8`: The entire model is moved to the CPU after use, and the transformer model is quantized to float8, saving more GPU memory. +- `sequential_cpu_offload`: Each layer of the model is moved to the CPU after use. It is slower but saves a significant amount of GPU memory. + +`qfloat8` may slightly reduce model performance but saves more GPU memory. If you have sufficient GPU memory, it is recommended to use `model_cpu_offload`. + +#### b. Using ComfyUI +For details, refer to [ComfyUI README](comfyui/README.md). + +#### c. Running Python Files + +##### i. Single-GPU Inference: + +- **Step 1**: Download the corresponding [weights](#model-zoo) and place them in the `models` folder. +- **Step 2**: Use different files for prediction based on the weights and prediction goals. This library currently supports CogVideoX-Fun, Wan2.1, and Wan2.1-Fun. Different models are distinguished by folder names under the `examples` folder, and their supported features vary. Use them accordingly. Below is an example using CogVideoX-Fun: + - **Text-to-Video**: + - Modify `prompt`, `neg_prompt`, `guidance_scale`, and `seed` in the file `examples/cogvideox_fun/predict_t2v.py`. + - Run the file `examples/cogvideox_fun/predict_t2v.py` and wait for the results. The generated videos will be saved in the folder `samples/cogvideox-fun-videos`. + - **Image-to-Video**: + - Modify `validation_image_start`, `validation_image_end`, `prompt`, `neg_prompt`, `guidance_scale`, and `seed` in the file `examples/cogvideox_fun/predict_i2v.py`. + - `validation_image_start` is the starting image of the video, and `validation_image_end` is the ending image of the video. + - Run the file `examples/cogvideox_fun/predict_i2v.py` and wait for the results. The generated videos will be saved in the folder `samples/cogvideox-fun-videos_i2v`. + - **Video-to-Video**: + - Modify `validation_video`, `validation_image_end`, `prompt`, `neg_prompt`, `guidance_scale`, and `seed` in the file `examples/cogvideox_fun/predict_v2v.py`. + - `validation_video` is the reference video for video-to-video generation. You can use the following demo video: [Demo Video](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/play_guitar.mp4). + - Run the file `examples/cogvideox_fun/predict_v2v.py` and wait for the results. The generated videos will be saved in the folder `samples/cogvideox-fun-videos_v2v`. + - **Controlled Video Generation (Canny, Pose, Depth, etc.)**: + - Modify `control_video`, `validation_image_end`, `prompt`, `neg_prompt`, `guidance_scale`, and `seed` in the file `examples/cogvideox_fun/predict_v2v_control.py`. + - `control_video` is the control video extracted using operators such as Canny, Pose, or Depth. You can use the following demo video: [Demo Video](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/pose.mp4). + - Run the file `examples/cogvideox_fun/predict_v2v_control.py` and wait for the results. The generated videos will be saved in the folder `samples/cogvideox-fun-videos_v2v_control`. +- **Step 3**: If you want to integrate other backbones or Loras trained by yourself, modify `lora_path` and relevant paths in `examples/{model_name}/predict_t2v.py` or `examples/{model_name}/predict_i2v.py` as needed. + +##### ii. Multi-GPU Inference: +When using multi-GPU inference, please make sure to install the xfuser. We recommend installing xfuser==0.4.2 and yunchang==0.6.2. +``` +pip install xfuser==0.4.2 --progress-bar off -i https://mirrors.aliyun.com/pypi/simple/ +pip install yunchang==0.6.2 --progress-bar off -i https://mirrors.aliyun.com/pypi/simple/ +``` + +Please ensure that the product of `ulysses_degree` and `ring_degree` equals the number of GPUs being used. For example, if you are using 8 GPUs, you can set `ulysses_degree=2` and `ring_degree=4`, or alternatively `ulysses_degree=4` and `ring_degree=2`. + +- `ulysses_degree` performs parallelization after splitting across the heads. +- `ring_degree` performs parallelization after splitting across the sequence. + +Compared to `ulysses_degree`, `ring_degree` incurs higher communication costs. Therefore, when setting these parameters, you should take into account both the sequence length and the number of heads in the model. + +Let’s take 8-GPU parallel inference as an example: + +- **For Wan2.1-Fun-V1.1-14B-InP**, which has 40 heads, `ulysses_degree` should be set to a divisor of 40 (e.g., 2, 4, 8, etc.). Thus, when using 8 GPUs for parallel inference, you can set `ulysses_degree=8` and `ring_degree=1`. + +- **For Wan2.1-Fun-V1.1-1.3B-InP**, which has 12 heads, `ulysses_degree` should be set to a divisor of 12 (e.g., 2, 4, etc.). Thus, when using 8 GPUs for parallel inference, you can set `ulysses_degree=4` and `ring_degree=2`. + +After setting the parameters, run the following command for parallel inference: + +```sh +torchrun --nproc-per-node=8 examples/wan2.1_fun/predict_t2v.py +``` + +#### d. Using the Web UI +The web UI supports text-to-video, image-to-video, video-to-video, and controlled video generation (Canny, Pose, Depth, etc.). This library currently supports CogVideoX-Fun, Wan2.1, and Wan2.1-Fun. Different models are distinguished by folder names under the `examples` folder, and their supported features vary. Use them accordingly. Below is an example using CogVideoX-Fun: + +- **Step 1**: Download the corresponding [weights](#model-zoo) and place them in the `models` folder. +- **Step 2**: Run the file `examples/cogvideox_fun/app.py` to access the Gradio interface. +- **Step 3**: Select the generation model on the page, fill in `prompt`, `neg_prompt`, `guidance_scale`, and `seed`, click "Generate," and wait for the results. The generated videos will be saved in the `sample` folder. + +### 2. Model Training +A complete model training pipeline should include data preprocessing and Video DiT training. The training process for different models is similar, and the data formats are also similar: + +

a. data preprocessing

+ +We have provided a simple demo of training the Lora model through image data, which can be found in the [wiki](https://github.com/aigc-apps/CogVideoX-Fun/wiki/Training-Lora) for details. + +A complete data preprocessing link for long video segmentation, cleaning, and description can refer to [README](cogvideox/video_caption/README.md) in the video captions section. + +If you want to train a text to image and video generation model. You need to arrange the dataset in this format. + +``` +📦 project/ +├── 📂 datasets/ +│ ├── 📂 internal_datasets/ +│ ├── 📂 train/ +│ │ ├── 📄 00000001.mp4 +│ │ ├── 📄 00000002.jpg +│ │ └── 📄 ..... +│ └── 📄 json_of_internal_datasets.json +``` + +The json_of_internal_datasets.json is a standard JSON file. The file_path in the json can to be set as relative path, as shown in below: +```json +[ + { + "file_path": "train/00000001.mp4", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` + +You can also set the path as absolute path as follow: +```json +[ + { + "file_path": "/mnt/data/videos/00000001.mp4", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "/mnt/data/train/00000001.jpg", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` + +

b. Video DiT training

+ +If the data format is relative path during data preprocessing, please set ```scripts/{model_name}/train.sh``` as follow. +``` +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/json_of_internal_datasets.json" +``` + +If the data format is absolute path during data preprocessing, please set ```scripts/train.sh``` as follow. +``` +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/json_of_internal_datasets.json" +``` + +Then, we run scripts/train.sh. +```sh +sh scripts/train.sh +``` + +For details on some parameter settings: +Wan2.1-Fun can be found in [Readme Train](scripts/wan2.1_fun/README_TRAIN.md) and [Readme Lora](scripts/wan2.1_fun/README_TRAIN_LORA.md). +Wan2.1 can be found in [Readme Train](scripts/wan2.1/README_TRAIN.md) and [Readme Lora](scripts/wan2.1/README_TRAIN_LORA.md). +CogVideoX-Fun can be found in [Readme Train](scripts/cogvideox_fun/README_TRAIN.md) and [Readme Lora](scripts/cogvideox_fun/README_TRAIN_LORA.md). + + +# Model zoo +## 1. Wan2.2-Fun + +| Name | Storage Size | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| Wan2.2-Fun-A14B-InP | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP) | Wan2.2-Fun-14B text-to-video generation weights, trained at multiple resolutions, supports start-end image prediction. | +| Wan2.2-Fun-A14B-Control | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control)| Wan2.2-Fun-14B video control weights, supporting various control conditions such as Canny, Depth, Pose, MLSD, etc., and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction at 81 frames, trained at 16 frames per second, with multilingual prediction support. | +| Wan2.2-Fun-A14B-Control-Camera | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control-Camera)| Wan2.2-Fun-14B camera lens control weights. Supports multi-resolution (512, 768, 1024) video prediction, trained with 81 frames at 16 FPS, supports multilingual prediction. | +| Wan2.2-VACE-Fun-A14B | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B) | Control weights for Wan2.2 trained using the VACE scheme (based on the base model Wan2.2-T2V-A14B), supporting various control conditions such as Canny, Depth, Pose, MLSD, trajectory control, etc. It supports video generation by specifying the subject. It supports multi-resolution (512, 768, 1024) video prediction, and is trained with 81 frames at 16 FPS. It also supports multi-language prediction. | +| Wan2.2-Fun-5B-InP | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-InP) | Wan2.2-Fun-5B text-to-video weights trained at 121 frames, 24 FPS, supporting first/last frame prediction. | +| Wan2.2-Fun-5B-Control | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-Control)| Wan2.2-Fun-5B video control weights, supporting control conditions like Canny, Depth, Pose, MLSD, and trajectory control. Trained at 121 frames, 24 FPS, with multilingual prediction support. | +| Wan2.2-Fun-5B-Control-Camera | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-Control-Camera)| Wan2.2-Fun-5B camera lens control weights. Trained at 121 frames, 24 FPS, with multilingual prediction support. | + + +## 2. Wan2.2 + +| Name | Hugging Face | Model Scope | Description | +|--|--|--|--| +| Wan2.2-TI2V-5B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | Wan2.2-5B Text-to-Video Weights | +| Wan2.2-T2V-14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | Wan2.2-14B Text-to-Video Weights | +| Wan2.2-I2V-A14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | Wan2.2-I2V-A14B Image-to-Video Weights | + +## 3. Wan2.1-Fun + +V1.1: +| Name | Storage Size | Hugging Face | Model Scope | Description | +|------|--------------|--------------|-------------|-------------| +| Wan2.1-Fun-V1.1-1.3B-InP | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-InP) | Wan2.1-Fun-V1.1-1.3B text-to-video generation weights, trained at multiple resolutions, supports start-end image prediction. | +| Wan2.1-Fun-V1.1-14B-InP | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP) | Wan2.1-Fun-V1.1-14B text-to-video generation weights, trained at multiple resolutions, supports start-end image prediction. | +| Wan2.1-Fun-V1.1-1.3B-Control | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control) | Wan2.1-Fun-V1.1-1.3B video control weights support various control conditions such as Canny, Depth, Pose, MLSD, etc., supports reference image + control condition-based control, and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction, trained with 81 frames at 16 FPS, supports multilingual prediction. | +| Wan2.1-Fun-V1.1-14B-Control | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control) | Wan2.1-Fun-V1.1-14B video control weights support various control conditions such as Canny, Depth, Pose, MLSD, etc., supports reference image + control condition-based control, and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction, trained with 81 frames at 16 FPS, supports multilingual prediction. | +| Wan2.1-Fun-V1.1-1.3B-Control-Camera | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera) | Wan2.1-Fun-V1.1-1.3B camera lens control weights. Supports multi-resolution (512, 768, 1024) video prediction, trained with 81 frames at 16 FPS, supports multilingual prediction. | +| Wan2.1-Fun-V1.1-14B-Control-Camera | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control-Camera) | Wan2.1-Fun-V1.1-14B camera lens control weights. Supports multi-resolution (512, 768, 1024) video prediction, trained with 81 frames at 16 FPS, supports multilingual prediction. | + +V1.0: +| Name | Storage Space | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| Wan2.1-Fun-1.3B-InP | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-InP) | Wan2.1-Fun-1.3B text-to-video weights, trained at multiple resolutions, supporting start and end frame prediction. | +| Wan2.1-Fun-14B-InP | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-InP) | Wan2.1-Fun-14B text-to-video weights, trained at multiple resolutions, supporting start and end frame prediction. | +| Wan2.1-Fun-1.3B-Control | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control) | Wan2.1-Fun-1.3B video control weights, supporting various control conditions such as Canny, Depth, Pose, MLSD, etc., and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction at 81 frames, trained at 16 frames per second, with multilingual prediction support. | +| Wan2.1-Fun-14B-Control | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control) | Wan2.1-Fun-14B video control weights, supporting various control conditions such as Canny, Depth, Pose, MLSD, etc., and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction at 81 frames, trained at 16 frames per second, with multilingual prediction support. | + +## 4. Wan2.1 + +| Name | Hugging Face | Model Scope | Description | +|--|--|--|--| +| Wan2.1-T2V-1.3B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B) | Wanxiang 2.1-1.3B text-to-video weights | +| Wan2.1-T2V-14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-T2V-14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B) | Wanxiang 2.1-14B text-to-video weights | +| Wan2.1-I2V-14B-480P | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-480P) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P) | Wanxiang 2.1-14B-480P image-to-video weights | +| Wan2.1-I2V-14B-720P| [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-720P) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P) | Wanxiang 2.1-14B-720P image-to-video weights | + +## 5. FantasyTalking + +| Name | Storage | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| Wan2.1-I2V-14B-720P | - | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-720P) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P) | Wan 2.1-14B-720P image-to-video model weights | +| Wav2Vec | - | [🤗Link](https://huggingface.co/facebook/wav2vec2-base-960h) | [😄Link](https://modelscope.cn/models/AI-ModelScope/wav2vec2-base-960h) | Wav2Vec model; place inside the Wan2.1-I2V-14B-720P folder and rename to `audio_encoder` | +| FantasyTalking model | - | [🤗Link](https://huggingface.co/acvlab/FantasyTalking/) | [😄Link](https://www.modelscope.cn/models/amap_cvlab/FantasyTalking/) | Official audio-conditioned weights | + +## 6. Qwen-Image + +| Name | Storage | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| Qwen-Image | [🤗Link](https://huggingface.co/Qwen/Qwen-Image) | [😄Link](https://modelscope.cn/models/Qwen/Qwen-Image) | Official Qwen-Image weights | +| Qwen-Image-Edit | [🤗Link](https://huggingface.co/Qwen/Qwen-Image-Edit) | [😄Link](https://modelscope.cn/models/Qwen/Qwen-Image-Edit) | Official Qwen-Image-Edit weights | +| Qwen-Image-Edit-2509 | [🤗Link](https://huggingface.co/Qwen/Qwen-Image-Edit-2509) | [😄Link](https://modelscope.cn/models/Qwen/Qwen-Image-Edit-2509) | Official Qwen-Image-Edit-2509 weights | + +## 7. Z-Image + +| Name | Storage | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| Z-Image-Turbo | [🤗Link](https://huggingface.co/Tongyi-MAI/Z-Image-Turbo) | [😄Link](https://www.modelscope.cn/models/Tongyi-MAI/Z-Image-Turbo) | Official weights for Z-Image-Turbo | + +## 8. Z-Image-Fun + +| Name | Storage | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| Z-Image-Turbo-Fun-Controlnet-Union | - | [🤗Link](https://huggingface.co/alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union) | [😄Link](https://modelscope.cn/models/PAI/Z-Image-Turbo-Fun-Controlnet-Union) | ControlNet weights for Z-Image-Turbo, supporting multiple control conditions such as Canny, Depth, Pose, MLSD, etc. | + +## 9. Flux + +| Name | Storage | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| FLUX.1-dev | [🤗Link](https://huggingface.co/black-forest-labs/FLUX.1-dev) | [😄Link](https://www.modelscope.cn/models/black-forest-labs/FLUX.1-dev) | Official FLUX.1-dev weights | +| FLUX.2-dev | [🤗Link](https://huggingface.co/black-forest-labs/FLUX.2-dev) | [😄Link](https://www.modelscope.cn/models/black-forest-labs/FLUX.2-dev) | Official FLUX.2-dev weights | + +## 10. Flux-Fun + +| Name | Storage | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| Flux.2-dev-Fun-Controlnet-Union | - | [🤗Link](https://huggingface.co/alibaba-pai/FLUX.2-dev-Fun-Controlnet-Union) | [😄Link](https://modelscope.cn/models/PAI/FLUX.2-dev-Fun-Controlnet-Union) | Flux.2-dev control weights, supporting various control conditions such as Canny, Depth, Pose, MLSD, etc. | + +## 11. HunyuanVideo + +| Name | Storage | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| HunyuanVideo | [🤗Link](https://huggingface.co/hunyuanvideo-community/HunyuanVideo) | - | HunyuanVideo-diffusers weights | +| HunyuanVideo-I2V | [🤗Link](https://huggingface.co/hunyuanvideo-community/HunyuanVideo-I2V) | - | HunyuanVideo-I2V-diffusers weights | + +## 12. CogVideoX-Fun + +V1.5: + +| Name | Storage Space | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| CogVideoX-Fun-V1.5-5b-InP | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.5-5b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.5-5b-InP) | Our official graph-generated video model is capable of predicting videos at multiple resolutions (512, 768, 1024) and has been trained on 85 frames at a rate of 8 frames per second. | +| CogVideoX-Fun-V1.5-Reward-LoRAs | - | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.5-Reward-LoRAs) | The official reward backpropagation technology model optimizes the videos generated by CogVideoX-Fun-V1.5 to better match human preferences. ｜ + +V1.1: + +| Name | Storage Space | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| CogVideoX-Fun-V1.1-2b-InP | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-InP) | Our official graph-generated video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. | +| CogVideoX-Fun-V1.1-5b-InP | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-InP) | Our official graph-generated video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. Noise has been added to the reference image, and the amplitude of motion is greater compared to V1.0. | +| CogVideoX-Fun-V1.1-2b-Pose | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-Pose) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-Pose) | Our official pose-control video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second.| +| CogVideoX-Fun-V1.1-2b-Control | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-Control) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-Control) | Our official control video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. Supporting various control conditions such as Canny, Depth, Pose, MLSD, etc.| +| CogVideoX-Fun-V1.1-5b-Pose | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-Pose) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-Pose) | Our official pose-control video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second.| +| CogVideoX-Fun-V1.1-5b-Control | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-Control) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-Control) | Our official control video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. Supporting various control conditions such as Canny, Depth, Pose, MLSD, etc.| +| CogVideoX-Fun-V1.1-Reward-LoRAs | - | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-Reward-LoRAs) | The official reward backpropagation technology model optimizes the videos generated by CogVideoX-Fun-V1.1 to better match human preferences. ｜ + +
+ (Obsolete) V1.0: + +| Name | Storage Space | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| CogVideoX-Fun-2b-InP | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-2b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-2b-InP) | Our official graph-generated video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. | +| CogVideoX-Fun-5b-InP | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-5b-InP)| [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-5b-InP)| Our official graph-generated video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. | +
+ +# Reference +- CogVideo: https://github.com/THUDM/CogVideo/ +- EasyAnimate: https://github.com/aigc-apps/EasyAnimate +- Wan2.1: https://github.com/Wan-Video/Wan2.1/ +- Wan2.2: https://github.com/Wan-Video/Wan2.2/ +- Diffusers: https://github.com/huggingface/diffusers +- Qwen-Image: https://github.com/QwenLM/Qwen-Image +- Self-Forcing: https://github.com/guandeh17/Self-Forcing +- Flux: https://github.com/black-forest-labs/flux +- Flux2: https://github.com/black-forest-labs/flux2 +- HunyuanVideo: https://github.com/Tencent-Hunyuan/HunyuanVideo +- ComfyUI-KJNodes: https://github.com/kijai/ComfyUI-KJNodes +- ComfyUI-EasyAnimateWrapper: https://github.com/kijai/ComfyUI-EasyAnimateWrapper +- ComfyUI-CameraCtrl-Wrapper: https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper +- CameraCtrl: https://github.com/hehao13/CameraCtrl + +# License +This project is licensed under the [Apache License (Version 2.0)](https://github.com/modelscope/modelscope/blob/master/LICENSE). + +The CogVideoX-2B model (including its corresponding Transformers module and VAE module) is released under the [Apache 2.0 License](LICENSE). + +The CogVideoX-5B model (Transformers module) is released under the [CogVideoX LICENSE](https://huggingface.co/THUDM/CogVideoX-5b/blob/main/LICENSE). diff --git a/VideoX-Fun/README_ja-JP.md b/VideoX-Fun/README_ja-JP.md new file mode 100644 index 0000000000000000000000000000000000000000..9e7a8ff6063839c4cd09c748546b2dff82ef5f9b --- /dev/null +++ b/VideoX-Fun/README_ja-JP.md @@ -0,0 +1,697 @@ +# VideoX-Fun + +😊 ようこそ！ + +CogVideoX-Fun: +[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-yellow)](https://huggingface.co/spaces/alibaba-pai/CogVideoX-Fun-5b) + +Wan-Fun: +[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-yellow)](https://huggingface.co/spaces/alibaba-pai/Wan2.1-Fun-1.3B-InP) + +[English](./README.md) | [简体中文](./README_zh-CN.md) | 日本語 + +# 目次 +- [目次](#目次) +- [紹介](#紹介) +- [クイックスタート](#クイックスタート) +- [ビデオ結果](#ビデオ結果) +- [使用方法](#使用方法) +- [モデルの場所](#モデルの場所) +- [参考文献](#参考文献) +- [ライセンス](#ライセンス) + +# 紹介 +VideoX-Funはビデオ生成のパイプラインであり、AI画像やビデオの生成、Diffusion TransformerのベースラインモデルとLoraモデルのトレーニングに使用できます。我々は、すでに学習済みのベースラインモデルから直接予測を行い、異なる解像度、秒数、FPSのビデオを生成することをサポートしています。また、ユーザーが独自のベースラインモデルやLoraモデルをトレーニングし、特定のスタイル変換を行うこともサポートしています。 + +異なるプラットフォームからのクイックスタートをサポートします。詳細は[クイックスタート](#クイックスタート)を参照してください。 + +新機能： +- Wan 2.2シリーズモデル、Wan-VACE制御モデル、Fantasy Talkingデジタルヒューマンモデル、Qwen-Image、Flux画像生成モデルなどのサポートを追加しました。[2025.10.16] +- Wan2.1-Fun-V1.1バージョンを更新：14Bと1.3BモデルのControl＋参照画像モデルをサポート、カメラ制御にも対応。さらに、Inpaintモデルを再訓練し、性能が向上しました。[2025.04.25] +- Wan2.1-Fun-V1.0の更新：14Bおよび1.3BのI2V（画像からビデオ）モデルとControlモデルをサポートし、開始フレームと終了フレームの予測に対応。[2025.03.26] +- CogVideoX-Fun-V1.5の更新：I2Vモデルと関連するトレーニング・予測コードをアップロード。[2024.12.16] +- 報酬Loraのサポート：報酬逆伝播技術を使用してLoraをトレーニングし、生成された動画を最適化し、人間の好みによりよく一致させる。[詳細情報](scripts/README_TRAIN_REWARD.md)。新しいバージョンの制御モデルでは、Canny、Depth、Pose、MLSDなどの異なる制御条件に対応。[2024.11.21] +- diffusersのサポート：CogVideoX-Fun Controlがdiffusersでサポートされるようになりました。[a-r-r-o-w](https://github.com/a-r-r-o-w)がこの[PR](https://github.com/huggingface/diffusers/pull/9671)でサポートを提供してくれたことに感謝します。詳細は[ドキュメント](https://huggingface.co/docs/diffusers/main/en/api/pipelines/cogvideox)をご覧ください。[2024.10.16] +- CogVideoX-Fun-V1.1の更新：i2vモデルを再トレーニングし、Noiseを追加して動画の動きの範囲を拡大。制御モデルのトレーニングコードとControlモデルをアップロード。[2024.09.29] +- CogVideoX-Fun-V1.0の更新：コードを作成！WindowsとLinuxに対応しました。2Bおよび5Bモデルでの最大256x256x49から1024x1024x49までの任意の解像度の動画生成をサポート。[2024.09.18] + +機能： +- [データ前処理](#data-preprocess) +- [DiTのトレーニング](#dit-train) +- [ビデオ生成](#video-gen) + +私たちのUIインターフェースは次のとおりです： +![ui](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/ui.jpg) + +# クイックスタート +### 1. クラウド使用: AliyunDSW/Docker +#### a. AliyunDSWから +DSWには無料のGPU時間があり、ユーザーは一度申請でき、申請後3か月間有効です。 + +Aliyunは[Freetier](https://free.aliyun.com/?product=9602825&crowd=enterprise&spm=5176.28055625.J_5831864660.1.e939154aRgha4e&scm=20140722.M_9974135.P_110.MO_1806-ID_9974135-MID_9974135-CID_30683-ST_8512-V_1)で無料のGPU時間を提供しています。取得してAliyun PAI-DSWで使用し、5分以内にCogVideoX-Funを開始できます！ + +[![DSW Notebook](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/asset/dsw.png)](https://gallery.pai-ml.com/#/preview/deepLearning/cv/cogvideox_fun) + +#### b. ComfyUIから +私たちのComfyUIは次のとおりです。詳細は[ComfyUI README](comfyui/README.md)を参照してください。 +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/cogvideoxfunv1_workflow_i2v.jpg) + +#### c. Dockerから +Dockerを使用する場合、マシンにグラフィックスカードドライバとCUDA環境が正しくインストールされていることを確認してください。 + +次のコマンドをこの方法で実行します： + +``` +# イメージをプル +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# イメージに入る +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# コードをクローン +git clone https://github.com/aigc-apps/VideoX-Fun.git + +# VideoX-Funのディレクトリに入る +cd VideoX-Fun + +# 重みをダウンロード +mkdir models/Diffusion_Transformer +mkdir models/Personalized_Model + +# Please use the hugginface link or modelscope link to download the model. +# CogVideoX-Fun +# https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-InP +# https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-InP + +# Wan +# https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-InP +# https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP +``` + +### 2. ローカルインストール: 環境チェック/ダウンロード/インストール +#### a. 環境チェック +以下の環境でこのライブラリの実行を確認しています： + +Windowsの詳細： +- OS: Windows 10 +- python: python3.10 & python3.11 +- pytorch: torch2.2.0 +- CUDA: 11.8 & 12.1 +- CUDNN: 8+ +- GPU： Nvidia-3060 12G & Nvidia-3090 24G + +Linuxの詳細： +- OS: Ubuntu 20.04, CentOS +- python: python3.10 & python3.11 +- pytorch: torch2.2.0 +- CUDA: 11.8 & 12.1 +- CUDNN: 8+ +- GPU：Nvidia-V100 16G & Nvidia-A10 24G & Nvidia-A100 40G & Nvidia-A100 80G + +重みを保存するために約60GBのディスクスペースが必要です。確認してください！ + +#### b. 重み +[重み](#model-zoo)を指定されたパスに配置することをお勧めします： + +**ComfyUIを通じて**: +モデルをComfyUIの重みフォルダ `ComfyUI/models/Fun_Models/` に入れます： +``` +📦 ComfyUI/ +├── 📂 models/ +│ └── 📂 Fun_Models/ +│ ├── 📂 CogVideoX-Fun-V1.1-2b-InP/ +│ ├── 📂 CogVideoX-Fun-V1.1-5b-InP/ +│ ├── 📂 Wan2.1-Fun-V1.1-14B-InP +│ └── 📂 Wan2.1-Fun-V1.1-1.3B-InP/ +``` + +**独自のpythonファイルまたはUIインターフェースを実行**: +``` +📦 models/ +├── 📂 Diffusion_Transformer/ +│ ├── 📂 CogVideoX-Fun-V1.1-2b-InP/ +│ ├── 📂 CogVideoX-Fun-V1.1-5b-InP/ +│ ├── 📂 Wan2.1-Fun-V1.1-14B-InP +│ └── 📂 Wan2.1-Fun-V1.1-1.3B-InP/ +├── 📂 Personalized_Model/ +│ └── あなたのトレーニング済みのトランスフォーマーモデル / あなたのトレーニング済みのLoraモデル（UIロード用） +``` + +# ビデオ結果 + +### Wan2.1-Fun-V1.1-14B-InP && Wan2.1-Fun-V1.1-1.3B-InP + + + + + + + + +
+ + + + + + + +
+ + + + + + + + +
+ + + + + + + +
+ +### Wan2.1-Fun-V1.1-14B-Control && Wan2.1-Fun-V1.1-1.3B-Control + +Generic Control Video + Reference Image: + + + + + + + + + + + + +
+ Reference Image + + Control Video + + Wan2.1-Fun-V1.1-14B-Control + + Wan2.1-Fun-V1.1-1.3B-Control +
+ + + + + + + +
+ + +Generic Control Video (Canny, Pose, Depth, etc.) and Trajectory Control: + + + + + + +
+ + + + + +
+ + + + + + + + + + + +
+ + + + + +
+ + + + + +
+ +### Wan2.1-Fun-V1.1-14B-Control-Camera && Wan2.1-Fun-V1.1-1.3B-Control-Camera + + + + + + + + + + + + + + + + + + + +
+ Pan Up + + Pan Left + + Pan Right +
+ + + + + +
+ Pan Down + + Pan Up + Pan Left + + Pan Up + Pan Right +
+ + + + + +
+ +### CogVideoX-Fun-V1.1-5B + +解像度-1024 + + + + + + + + +
+ + + + + + + +
+ + +解像度-768 + + + + + + + + +
+ + + + + + + +
+ +解像度-512 + + + + + + + + +
+ + + + + + + +
+ +### CogVideoX-Fun-V1.1-5B-Control + + + + + + + + + + + + + + + + +
+ + + + + +
+ 美しい澄んだ目と金髪の若い女性が白い服を着て体をひねり、カメラは彼女の顔に焦点を合わせています。高品質、傑作、最高品質、高解像度、超微細、夢のような。 + + 美しい澄んだ目と金髪の若い女性が白い服を着て体をひねり、カメラは彼女の顔に焦点を合わせています。高品質、傑作、最高品質、高解像度、超微細、夢のような。 + + 若いクマ。 +
+ + + + + +
+ +# 使い方 + +

1. 生成

+ +#### a. GPUメモリ節約方法 +Wan2.1のパラメータが非常に大きいため、GPUメモリを節約し、コンシューマー向けGPUに適応させる必要があります。各予測ファイルには`GPU_memory_mode`を提供しており、`model_cpu_offload`、`model_cpu_offload_and_qfloat8`、`sequential_cpu_offload`の中から選択できます。この方法はCogVideoX-Funの生成にも適用されます。 + +- `model_cpu_offload`: モデル全体が使用後にCPUに移動し、一部のGPUメモリを節約します。 +- `model_cpu_offload_and_qfloat8`: モデル全体が使用後にCPUに移動し、Transformerモデルに対してfloat8の量子化を行い、より多くのGPUメモリを節約します。 +- `sequential_cpu_offload`: モデルの各層が使用後にCPUに移動します。速度は遅くなりますが、大量のGPUメモリを節約します。 + +`qfloat8`はモデルの性能を部分的に低下させる可能性がありますが、より多くのGPUメモリを節約できます。十分なGPUメモリがある場合は、`model_cpu_offload`の使用をお勧めします。 + +#### b. ComfyUIを使用する +詳細は[ComfyUI README](comfyui/README.md)をご覧ください。 + +#### c. Pythonファイルを実行する + +##### i. 単一GPUでの推論: + +- ステップ1: 対応する[重み](#model-zoo)をダウンロードし、`models`フォルダに配置します。 +- ステップ2: 異なる重みと予測目標に基づいて、異なるファイルを使用して予測を行います。現在、このライブラリはCogVideoX-Fun、Wan2.1、およびWan2.1-Funをサポートしています。`examples`フォルダ内のフォルダ名で区別され、異なるモデルがサポートする機能が異なりますので、状況に応じて区別してください。以下はCogVideoX-Funを例として説明します。 + - テキストからビデオ: + - `examples/cogvideox_fun/predict_t2v.py`ファイルで`prompt`、`neg_prompt`、`guidance_scale`、`seed`を変更します。 + - 次に、`examples/cogvideox_fun/predict_t2v.py`ファイルを実行し、結果が生成されるのを待ちます。結果は`samples/cogvideox-fun-videos`フォルダに保存されます。 + - 画像からビデオ: + - `examples/cogvideox_fun/predict_i2v.py`ファイルで`validation_image_start`、`validation_image_end`、`prompt`、`neg_prompt`、`guidance_scale`、`seed`を変更します。 + - `validation_image_start`はビデオの開始画像、`validation_image_end`はビデオの終了画像です。 + - 次に、`examples/cogvideox_fun/predict_i2v.py`ファイルを実行し、結果が生成されるのを待ちます。結果は`samples/cogvideox-fun-videos_i2v`フォルダに保存されます。 + - ビデオからビデオ: + - `examples/cogvideox_fun/predict_v2v.py`ファイルで`validation_video`、`validation_image_end`、`prompt`、`neg_prompt`、`guidance_scale`、`seed`を変更します。 + - `validation_video`はビデオ生成のための参照ビデオです。以下のデモビデオを使用して実行できます：[デモビデオ](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/play_guitar.mp4) + - 次に、`examples/cogvideox_fun/predict_v2v.py`ファイルを実行し、結果が生成されるのを待ちます。結果は`samples/cogvideox-fun-videos_v2v`フォルダに保存されます。 + - 通常の制御付きビデオ生成（Canny、Pose、Depthなど）: + - `examples/cogvideox_fun/predict_v2v_control.py`ファイルで`control_video`、`validation_image_end`、`prompt`、`neg_prompt`、`guidance_scale`、`seed`を変更します。 + - `control_video`は、Canny、Pose、Depthなどの演算子で抽出された制御用ビデオです。以下のデモビデオを使用して実行できます：[デモビデオ](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/pose.mp4) + - 次に、`examples/cogvideox_fun/predict_v2v_control.py`ファイルを実行し、結果が生成されるのを待ちます。結果は`samples/cogvideox-fun-videos_v2v_control`フォルダに保存されます。 +- ステップ3: 自分でトレーニングした他のバックボーンやLoraを組み合わせたい場合は、必要に応じて`examples/{model_name}/predict_t2v.py`や`examples/{model_name}/predict_i2v.py`、`lora_path`を修正します。 + +##### ii. 複数GPUでの推論: +多カードでの推論を行う際は、xfuserリポジトリのインストールに注意してください。xfuser==0.4.2 と yunchang==0.6.2 のインストールが推奨されます。 +``` +pip install xfuser==0.4.2 --progress-bar off -i https://mirrors.aliyun.com/pypi/simple/ +pip install yunchang==0.6.2 --progress-bar off -i https://mirrors.aliyun.com/pypi/simple/ +``` + +`ulysses_degree` と `ring_degree` の積が使用する GPU 数と一致することを確認してください。たとえば、8つのGPUを使用する場合、`ulysses_degree=2` と `ring_degree=4`、または `ulysses_degree=4` と `ring_degree=2` を設定することができます。 + +- `ulysses_degree` はヘッド（head）に分割した後の並列化を行います。 +- `ring_degree` はシーケンスに分割した後の並列化を行います。 + +`ring_degree` は `ulysses_degree` よりも通信コストが高いため、これらのパラメータを設定する際には、シーケンス長とモデルのヘッド数を考慮する必要があります。 + +8GPUでの並列推論を例に挙げます： + +- **Wan2.1-Fun-V1.1-14B-InP** はヘッド数が40あります。この場合、`ulysses_degree` は40で割り切れる値（例：2, 4, 8など）に設定する必要があります。したがって、8GPUを使用して並列推論を行う場合、`ulysses_degree=8` と `ring_degree=1` を設定できます。 + +- **Wan2.1-Fun-V1.1-1.3B-InP** はヘッド数が12あります。この場合、`ulysses_degree` は12で割り切れる値（例：2, 4など）に設定する必要があります。したがって、8GPUを使用して並列推論を行う場合、`ulysses_degree=4` と `ring_degree=2` を設定できます。 + +パラメータの設定が完了したら、以下のコマンドで並列推論を実行してください： + +```sh +torchrun --nproc-per-node=8 examples/wan2.1_fun/predict_t2v.py +``` + +#### d. UIインターフェースを使用する + +WebUIは、テキストからビデオ、画像からビデオ、ビデオからビデオ、および通常の制御付きビデオ生成（Canny、Pose、Depthなど）をサポートします。現在、このライブラリはCogVideoX-Fun、Wan2.1、およびWan2.1-Funをサポートしており、`examples`フォルダ内のフォルダ名で区別されています。異なるモデルがサポートする機能が異なるため、状況に応じて区別してください。以下はCogVideoX-Funを例として説明します。 + +- ステップ1: 対応する[重み](#model-zoo)をダウンロードし、`models`フォルダに配置します。 +- ステップ2: `examples/cogvideox_fun/app.py`ファイルを実行し、Gradioページに入ります。 +- ステップ3: ページ上で生成モデルを選択し、`prompt`、`neg_prompt`、`guidance_scale`、`seed`などを入力し、「生成」をクリックして結果が生成されるのを待ちます。結果は`sample`フォルダに保存されます。 + +### 2. モデルのトレーニング +完全なモデルトレーニングの流れには、データの前処理とVideo DiTのトレーニングが含まれるべきです。異なるモデルのトレーニングプロセスは類似しており、データ形式も類似しています： + +

a. データ前処理

+ +画像データを使用してLoraモデルをトレーニングする簡単なデモを提供しました。詳細は[wiki](https://github.com/aigc-apps/CogVideoX-Fun/wiki/Training-Lora)をご覧ください。 + +長いビデオのセグメンテーション、クリーニング、説明のための完全なデータ前処理リンクは、ビデオキャプションセクションの[README](cogvideox/video_caption/README.md)を参照してください。 + +テキストから画像およびビデオ生成モデルをトレーニングしたい場合。この形式でデータセットを配置する必要があります。 + +``` +📦 project/ +├── 📂 datasets/ +│ ├── 📂 internal_datasets/ +│ ├── 📂 train/ +│ │ ├── 📄 00000001.mp4 +│ │ ├── 📄 00000002.jpg +│ │ └── 📄 ..... +│ └── 📄 json_of_internal_datasets.json +``` + +json_of_internal_datasets.jsonは標準のJSONファイルです。json内のfile_pathは相対パスとして設定できます。以下のように： +```json +[ + { + "file_path": "train/00000001.mp4", + "text": "スーツとサングラスを着た若い男性のグループが街の通りを歩いている。", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "text": "スーツとサングラスを着た若い男性のグループが街の通りを歩いている。", + "type": "image" + }, + ..... +] +``` + +次のように絶対パスとして設定することもできます： +```json +[ + { + "file_path": "/mnt/data/videos/00000001.mp4", + "text": "スーツとサングラスを着た若い男性のグループが街の通りを歩いている。", + "type": "video" + }, + { + "file_path": "/mnt/data/train/00000001.jpg", + "text": "スーツとサングラスを着た若い男性のグループが街の通りを歩いている。", + "type": "image" + }, + ..... +] +``` + +

b. Video DiTトレーニング

+ +データ前処理時にデータ形式が相対パスの場合、```scripts/{model_name}/train.sh```を次のように設定します。 +``` +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/json_of_internal_datasets.json" +``` + +データ形式が絶対パスの場合、```scripts/train.sh```を次のように設定します。 +``` +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/json_of_internal_datasets.json" +``` + +次に、scripts/train.shを実行します。 +```sh +sh scripts/train.sh +``` +いくつかのパラメータ設定の詳細について： +Wan2.1-Funは[Readme Train](scripts/wan2.1_fun/README_TRAIN.md)と[Readme Lora](scripts/wan2.1_fun/README_TRAIN_LORA.md)を参照してください。 +Wan2.1は[Readme Train](scripts/wan2.1/README_TRAIN.md)と[Readme Lora](scripts/wan2.1/README_TRAIN_LORA.md)を参照してください。 +CogVideoX-Funは[Readme Train](scripts/cogvideox_fun/README_TRAIN.md)と[Readme Lora](scripts/cogvideox_fun/README_TRAIN_LORA.md)を参照してください。 + +# モデルの場所 + +## 1. Wan2.2-Fun + +| 名前 | ストレージ容量 | Hugging Face | Model Scope | 説明 | +|------|----------------|------------|-------------|------| +| Wan2.2-Fun-A14B-InP | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP) | Wan2.2-Fun-14Bのテキスト・画像から動画を生成するモデルの重み。複数の解像度で学習されており、動画の最初と最後のフレームの予測をサポートしています。 | +| Wan2.2-Fun-A14B-Control | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control) | Wan2.2-Fun-14Bの動画制御用重み。Canny、Depth、Pose、MLSDなどのさまざまな制御条件に対応しており、軌跡制御もサポートしています。512、768、1024の複数解像度での動画生成が可能で、81フレーム、16fpsで学習されています。多言語対応の予測もサポートしています。 | +| Wan2.2-Fun-A14B-Contro-Camera | 64.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-Control-Camera) | [😄リンク](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control-Camera)| Wan2.2-Fun-14Bのカメラレンズ制御重み。512、768、1024のマルチ解像度での動画予測をサポートし、81フレーム、毎秒16フレームで訓練されています。多言語予測に対応しています。 | +| Wan2.2-VACE-Fun-A14B | 64.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B) | [😄リンク](https://modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B) | VACE方式でトレーニングされたWan2.2の制御ウェイト（ベースモデルはWan2.2-T2V-A14B）。Canny、Depth、Pose、MLSD、軌道制御などの異なる制御条件をサポートします。対象を指定して動画生成が可能です。多解像度（512、768、1024）の動画予測をサポートし、81フレームで16FPSでトレーニングされています。多言語予測にも対応しています。 | +| Wan2.2-Fun-5B-InP | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-InP) | Wan2.2-Fun-5B テキストから動画生成用の重み。121フレーム、24 FPSで学習され、先頭/末尾フレーム予測をサポート。 | +| Wan2.2-Fun-5B-Control | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-Control)| Wan2.2-Fun-5B 動画制御用重み。Canny、Depth、Pose、MLSDなどの制御条件や軌道制御をサポート。121フレーム、24 FPSで学習され、多言語予測に対応。 | +| Wan2.2-Fun-5B-Control-Camera | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-Control-Camera)| Wan2.2-Fun-5B カメラレンズ制御用重み。121フレーム、24 FPSで学習され、多言語予測に対応。 | + +## 2. Wan2.2 + +| モデル名 | Hugging Face | Model Scope | 説明 | +|--|--|--|--| +| Wan2.2-TI2V-5B | [🤗リンク](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄リンク](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | 万象2.2-5B テキストから動画生成重み | +| Wan2.2-T2V-A14B | [🤗リンク](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄リンク](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | 万象2.2-14B テキストから動画生成重み | +| Wan2.2-I2V-A14B | [🤗リンク](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄リンク](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | 万象2.2-14B 画像から動画生成重み | + +## 3. Wan2.1-Fun + +V1.1: +| 名称 | ストレージ容量 | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| Wan2.1-Fun-V1.1-1.3B-InP | 19.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-InP) | [😄リンク](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-InP) | Wan2.1-Fun-V1.1-1.3Bのテキスト・画像から動画生成の重み。マルチ解像度で訓練され、最初と最後の画像予測をサポートします。 | +| Wan2.1-Fun-V1.1-14B-InP | 47.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-InP) | [😄リンク](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP) | Wan2.1-Fun-V1.1-14Bのテキスト・画像から動画生成の重み。マルチ解像度で訓練され、最初と最後の画像予測をサポートします。 | +| Wan2.1-Fun-V1.1-1.3B-Control | 19.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-Control) | [😄リンク](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control)| Wan2.1-Fun-V1.1-1.3Bのビデオ制御重み。Canny、Depth、Pose、MLSDなどの異なる制御条件に対応し、参照画像＋制御条件を使用した制御や軌跡制御をサポートします。512、768、1024のマルチ解像度での動画予測をサポートし、81フレーム、毎秒16フレームで訓練されています。多言語予測に対応しています。 | +| Wan2.1-Fun-V1.1-14B-Control | 47.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-Control) | [😄リンク](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control)| Wan2.1-Fun-V1.1-14Bのビデオ制御重み。Canny、Depth、Pose、MLSDなどの異なる制御条件に対応し、参照画像＋制御条件を使用した制御や軌跡制御をサポートします。512、768、1024のマルチ解像度での動画予測をサポートし、81フレーム、毎秒16フレームで訓練されています。多言語予測に対応しています。 | +| Wan2.1-Fun-V1.1-1.3B-Control-Camera | 19.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-Control-Camera) | [😄リンク](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera)| Wan2.1-Fun-V1.1-1.3Bのカメラレンズ制御重み。512、768、1024のマルチ解像度での動画予測をサポートし、81フレーム、毎秒16フレームで訓練されています。多言語予測に対応しています。 | +| Wan2.1-Fun-V1.1-14B-Control-Camera | 47.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-Control-Camera) | [😄リンク](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control-Camera)| Wan2.1-Fun-V1.1-14Bのカメラレンズ制御重み。512、768、1024のマルチ解像度での動画予測をサポートし、81フレーム、毎秒16フレームで訓練されています。多言語予測に対応しています。 | + + +V1.0: +| 名称 | ストレージ容量 | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| Wan2.1-Fun-1.3B-InP | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-InP) | Wan2.1-Fun-1.3Bのテキスト・画像から動画生成する重み。マルチ解像度で学習され、開始・終了画像予測をサポート。 | +| Wan2.1-Fun-14B-InP | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-InP) | Wan2.1-Fun-14Bのテキスト・画像から動画生成する重み。マルチ解像度で学習され、開始・終了画像予測をサポート。 | +| Wan2.1-Fun-1.3B-Control | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control) | Wan2.1-Fun-1.3Bのビデオ制御ウェイト。Canny、Depth、Pose、MLSDなどの異なる制御条件をサポートし、トラジェクトリ制御も利用可能。512、768、1024のマルチ解像度でのビデオ予測をサポートし、81フレーム（1秒間に16フレーム）でトレーニング済みで、多言語予測にも対応しています。 | +| Wan2.1-Fun-14B-Control | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control) | Wan2.1-Fun-14Bのビデオ制御ウェイト。Canny、Depth、Pose、MLSDなどの異なる制御条件をサポートし、トラジェクトリ制御も利用可能。512、768、1024のマルチ解像度でのビデオ予測をサポートし、81フレーム（1秒間に16フレーム）でトレーニング済みで、多言語予測にも対応しています。 | + +## 4. Wan2.1 + +| 名称 | Hugging Face | Model Scope | 説明 | +|--|--|--|--| +| Wan2.1-T2V-1.3B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B) | 万象2.1-1.3Bのテキストから動画生成する重み | +| Wan2.1-T2V-14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-T2V-14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B) | 万象2.1-14Bのテキストから動画生成する重み | +| Wan2.1-I2V-14B-480P | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-480P) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P) | 万象2.1-14B-480Pの画像から動画生成する重み | +| Wan2.1-I2V-14B-720P| [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-720P) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P) | 万象2.1-14B-720Pの画像から動画生成する重み | + +## 5. FantasyTalking + +| 名称 | ストレージ | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| Wan2.1-I2V-14B-720P | - | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-720P) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P) | 万象2.1-14B-720P 画像→動画モデルの重み | +| Wav2Vec | - | [🤗Link](https://huggingface.co/facebook/wav2vec2-base-960h) | [😄Link](https://modelscope.cn/models/AI-ModelScope/wav2vec2-base-960h) | Wav2Vecモデル。Wan2.1-I2V-14B-720Pフォルダ内に配置し、`audio_encoder` という名前に変更してください | +| FantasyTalking model | - | [🤗Link](https://huggingface.co/acvlab/FantasyTalking/) | [😄Link](https://www.modelscope.cn/models/amap_cvlab/FantasyTalking/) | 公式Audio Condition重み | + +## 6. Qwen-Image + +| 名称 | ストレージ | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| Qwen-Image | [🤗Link](https://huggingface.co/Qwen/Qwen-Image) | [😄Link](https://modelscope.cn/models/Qwen/Qwen-Image) | Qwen-Image 公式重み | +| Qwen-Image-Edit | [🤗Link](https://huggingface.co/Qwen/Qwen-Image-Edit) | [😄Link](https://modelscope.cn/models/Qwen/Qwen-Image-Edit) | Qwen-Image-Edit 公式重み | +| Qwen-Image-Edit-2509 | [🤗Link](https://huggingface.co/Qwen/Qwen-Image-Edit-2509) | [😄Link](https://modelscope.cn/models/Qwen/Qwen-Image-Edit-2509) | Qwen-Image-Edit-2509 公式重み | + +## 7. Z-Image + +| 名称 | ストレージ | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| Z-Image-Turbo | [🤗リンク](https://huggingface.co/Tongyi-MAI/Z-Image-Turbo) | [😄リンク](https://www.modelscope.cn/models/Tongyi-MAI/Z-Image-Turbo) | Z-Image-Turboの公式重み | + +## 8. Z-Image-Fun + +| 名称 | ストレージ | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| Z-Image-Turbo-Fun-Controlnet-Union | - | [🤗リンク](https://huggingface.co/alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union) | [😄リンク](https://modelscope.cn/models/PAI/Z-Image-Turbo-Fun-Controlnet-Union) | Z-Image-Turbo用のControlNet重み。Canny、Depth、Pose、MLSDなど複数の制御条件をサポート。 | + +## 9. Flux + +| 名称 | ストレージ | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| FLUX.1-dev | [🤗Link](https://huggingface.co/black-forest-labs/FLUX.1-dev) | [😄Link](https://www.modelscope.cn/models/black-forest-labs/FLUX.1-dev)| FLUX.1-dev 公式重み | +| FLUX.2-dev | [🤗Link](https://huggingface.co/black-forest-labs/FLUX.2-dev) | [😄Link](https://www.modelscope.cn/models/black-forest-labs/FLUX.2-dev) | FLUX.2-dev 公式重み | + +## 10. Flux-Fun + +| 名前 | ストレージ | Hugging Face | ModelScope | 説明 | +|--|--|--|--|--| +| Flux.2-dev-Fun-Controlnet-Union | - | [🤗リンク](https://huggingface.co/alibaba-pai/FLUX.2-dev-Fun-Controlnet-Union) | [😄リンク](https://modelscope.cn/models/PAI/FLUX.2-dev-Fun-Controlnet-Union) | Flux.2-dev 用の ControlNet 重みで、Canny、Depth、Pose、MLSD など様々な制御条件をサポートします。 | + +## 11. HunyuanVideo + +| 名称 | ストレージ | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| HunyuanVideo | [🤗Link](https://huggingface.co/hunyuanvideo-community/HunyuanVideo) | - | HunyuanVideo-diffusers 公式重み | +| HunyuanVideo-I2V | [🤗Link](https://huggingface.co/hunyuanvideo-community/HunyuanVideo-I2V) | - | HunyuanVideo-I2V-diffusers 公式重み | + +## 12. CogVideoX-Fun + +V1.5: + +| 名称 | ストレージスペース | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| CogVideoX-Fun-V1.5-5b-InP | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.5-5b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.5-5b-InP) | 公式のグラフ生成ビデオモデルは、複数の解像度（512、768、1024）でビデオを予測できます。85フレーム、8フレーム/秒でトレーニングされています。 | +| CogVideoX-Fun-V1.5-Reward-LoRAs | - | [🤗リンク](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs) | [😄リンク](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.5-Reward-LoRAs) | 公式の報酬逆伝播技術モデルで、CogVideoX-Fun-V1.5が生成するビデオを最適化し、人間の嗜好によりよく合うようにする。 | + +V1.1: + +| 名称 | ストレージスペース | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| CogVideoX-Fun-V1.1-2b-InP | 13.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-InP) | [😄リンク](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-InP) | 公式のグラフ生成ビデオモデルは、複数の解像度（512、768、1024、1280）でビデオを予測できます。49フレーム、8フレーム/秒でトレーニングされています。参照画像にノイズが追加され、V1.0と比較して動きの幅が広がっています。 | +| CogVideoX-Fun-V1.1-5b-InP | 20.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-InP) | [😄リンク](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-InP) | 公式のグラフ生成ビデオモデルは、複数の解像度（512、768、1024、1280）でビデオを予測できます。49フレーム、8フレーム/秒でトレーニングされています。参照画像にノイズが追加され、V1.0と比較して動きの幅が広がっています。 | +| CogVideoX-Fun-V1.1-2b-Pose | 13.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-Pose) | [😄リンク](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-Pose) | 公式のポーズコントロールビデオモデルは、複数の解像度（512、768、1024、1280）でビデオを予測できます。49フレーム、8フレーム/秒でトレーニングされています。| +| CogVideoX-Fun-V1.1-2b-Control | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-Control) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-Control) | 公式のコントロールビデオモデルは、複数の解像度（512、768、1024、1280）でビデオを予測できます。49フレーム、8フレーム/秒でトレーニングされています。Canny、Depth、Pose、MLSDなどのさまざまなコントロール条件をサポートします。| +| CogVideoX-Fun-V1.1-5b-Pose | 20.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-Pose) | [😄リンク](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-Pose) | 公式のポーズコントロールビデオモデルは、複数の解像度（512、768、1024、1280）でビデオを予測できます。49フレーム、8フレーム/秒でトレーニングされています。| +| CogVideoX-Fun-V1.1-5b-Control | 20.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-Control) | [😄リンク](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-Control) | 公式のコントロールビデオモデルは、複数の解像度（512、768、1024、1280）でビデオを予測できます。49フレーム、8フレーム/秒でトレーニングされています。Canny、Depth、Pose、MLSDなどのさまざまなコントロール条件をサポートします。| +| CogVideoX-Fun-V1.1-Reward-LoRAs | - | [🤗リンク](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs) | [😄リンク](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.5-Reward-LoRAs) | 公式の報酬逆伝播技術モデルで、CogVideoX-Fun-V1.1が生成するビデオを最適化し、人間の嗜好によりよく合うようにする。 | + +
+ (Obsolete) V1.0: + +| 名称 | ストレージスペース | Hugging Face | Model Scope | 説明 | +|--|--|--|--|--| +| CogVideoX-Fun-2b-InP | 13.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/CogVideoX-Fun-2b-InP) | [😄リンク](https://modelscope.cn/models/PAI/CogVideoX-Fun-2b-InP) | 公式のグラフ生成ビデオモデルは、複数の解像度（512、768、1024、1280）でビデオを予測できます。49フレーム、8フレーム/秒でトレーニングされています。 | +| CogVideoX-Fun-5b-InP | 20.0 GB | [🤗リンク](https://huggingface.co/alibaba-pai/CogVideoX-Fun-5b-InP)| [😄リンク](https://modelscope.cn/models/PAI/CogVideoX-Fun-5b-InP)| 公式のグラフ生成ビデオモデルは、複数の解像度（512、768、1024、1280）でビデオを予測できます。49フレーム、8フレーム/秒でトレーニングされています。| +
+ +# 参考文献 +- CogVideo: https://github.com/THUDM/CogVideo/ +- EasyAnimate: https://github.com/aigc-apps/EasyAnimate +- Wan2.1: https://github.com/Wan-Video/Wan2.1/ +- Wan2.2: https://github.com/Wan-Video/Wan2.2/ +- Diffusers: https://github.com/huggingface/diffusers +- Qwen-Image: https://github.com/QwenLM/Qwen-Image +- Self-Forcing: https://github.com/guandeh17/Self-Forcing +- Flux: https://github.com/black-forest-labs/flux +- Flux2: https://github.com/black-forest-labs/flux2 +- HunyuanVideo: https://github.com/Tencent-Hunyuan/HunyuanVideo +- ComfyUI-KJNodes: https://github.com/kijai/ComfyUI-KJNodes +- ComfyUI-EasyAnimateWrapper: https://github.com/kijai/ComfyUI-EasyAnimateWrapper +- ComfyUI-CameraCtrl-Wrapper: https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper +- CameraCtrl: https://github.com/hehao13/CameraCtrl + +# ライセンス +このプロジェクトは[Apache License (Version 2.0)](https://github.com/modelscope/modelscope/blob/master/LICENSE)の下でライセンスされています。 + +CogVideoX-2Bモデル（対応するTransformersモジュール、VAEモジュールを含む）は、[Apache 2.0ライセンス](LICENSE)の下でリリースされています。 + +CogVideoX-5Bモデル（Transformersモジュール）は、[CogVideoXライセンス](https://huggingface.co/THUDM/CogVideoX-5b/blob/main/LICENSE)の下でリリースされています。 diff --git a/VideoX-Fun/README_zh-CN.md b/VideoX-Fun/README_zh-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..4c2ee7514d84b2deb6a3e4d3f5feef15fba12288 --- /dev/null +++ b/VideoX-Fun/README_zh-CN.md @@ -0,0 +1,687 @@ +# VideoX-Fun + +😊 Welcome! + +CogVideoX-Fun: +[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-yellow)](https://huggingface.co/spaces/alibaba-pai/CogVideoX-Fun-5b) + +Wan-Fun: +[![Hugging Face Spaces](https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-Spaces-yellow)](https://huggingface.co/spaces/alibaba-pai/Wan2.1-Fun-1.3B-InP) + +[English](./README.md) | 简体中文 | [日本語](./README_ja-JP.md) + +# 目录 +- [目录](#目录) +- [简介](#简介) +- [快速启动](#快速启动) +- [视频作品](#视频作品) +- [如何使用](#如何使用) +- [模型地址](#模型地址) +- [参考文献](#参考文献) +- [许可证](#许可证) + +# 简介 +VideoX-Fun是一个视频生成的pipeline，可用于生成AI图片与视频、训练Diffusion Transformer的基线模型与Lora模型，我们支持从已经训练好的基线模型直接进行预测，生成不同分辨率，不同秒数、不同FPS的视频，也支持用户训练自己的基线模型与Lora模型，进行一定的风格变换。 + +我们会逐渐支持从不同平台快速启动，请参阅 [快速启动](#快速启动)。 + +新特性： +- 更新支持Wan2.2系列模型、Wan-VACE控制模型、支持Fantasy Talking数字人模型、Qwen-Image和Flux图片生成模型等。[2025.10.16]。 +- 更新Wan2.1-Fun-V1.1版本：支持14B与1.3B模型Control+参考图模型，支持镜头控制，另外Inpaint模型重新训练，性能更佳。[2025.04.25] +- 更新Wan2.1-Fun-V1.0版本：支持14B与1.3B模型的I2V和Control模型，支持首尾图预测。[2025.03.26] +- 更新CogVideoX-Fun-V1.5版本：上传I2V模型与相关训练预测代码。[2024.12.16] +- 奖励Lora支持：通过奖励反向传播技术训练Lora，以优化生成的视频，使其更好地与人类偏好保持一致，[更多信息](scripts/README_TRAIN_REWARD.md)。新版本的控制模型，支持不同的控制条件，如Canny、Depth、Pose、MLSD等。[2024.11.21] +- diffusers支持：CogVideoX-Fun Control现在在diffusers中得到了支持。感谢 [a-r-r-o-w](https://github.com/a-r-r-o-w)在这个 [PR](https://github.com/huggingface/diffusers/pull/9671)中贡献了支持。查看[文档](https://huggingface.co/docs/diffusers/main/en/api/pipelines/cogvideox)以了解更多信息。[2024.10.16] +- 更新CogVideoX-Fun-V1.1版本：重新训练i2v模型，添加Noise，使得视频的运动幅度更大。上传控制模型训练代码与Control模型。[2024.09.29] +- 更新CogVideoX-Fun-V1.0版本：创建代码！现在支持 Windows 和 Linux。支持2b与5b最大256x256x49到1024x1024x49的任意分辨率的视频生成。[2024.09.18] + +功能概览： +- [数据预处理](#data-preprocess) +- [训练DiT](#dit-train) +- [模型生成](#video-gen) + +我们的ui界面如下: +![ui](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/ui.jpg) + +# 快速启动 +### 1. 云使用: AliyunDSW/Docker +#### a. 通过阿里云 DSW +DSW 有免费 GPU 时间，用户可申请一次，申请后3个月内有效。 + +阿里云在[Freetier](https://free.aliyun.com/?product=9602825&crowd=enterprise&spm=5176.28055625.J_5831864660.1.e939154aRgha4e&scm=20140722.M_9974135.P_110.MO_1806-ID_9974135-MID_9974135-CID_30683-ST_8512-V_1)提供免费GPU时间，获取并在阿里云PAI-DSW中使用，5分钟内即可启动CogVideoX-Fun。 + +[![DSW Notebook](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/asset/dsw.png)](https://gallery.pai-ml.com/#/preview/deepLearning/cv/cogvideox_fun) + +#### b. 通过ComfyUI +我们的ComfyUI界面如下，具体查看[ComfyUI README](comfyui/README.md)。 +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/cogvideoxfunv1_workflow_i2v.jpg) + +#### c. 通过docker +使用docker的情况下，请保证机器中已经正确安装显卡驱动与CUDA环境，然后以此执行以下命令： + +``` +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:cogvideox_fun + +# clone code +git clone https://github.com/aigc-apps/VideoX-Fun.git + +# enter VideoX-Fun's dir +cd VideoX-Fun + +# download weights +mkdir models/Diffusion_Transformer +mkdir models/Personalized_Model + +# Please use the hugginface link or modelscope link to download the model. +# CogVideoX-Fun +# https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-InP +# https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-InP + +# Wan +# https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-InP +# https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP +``` + +### 2. 本地安装: 环境检查/下载/安装 +#### a. 环境检查 +我们已验证该库可在以下环境中执行： + +Windows 的详细信息： +- 操作系统 Windows 10 +- python: python3.10 & python3.11 +- pytorch: torch2.2.0 +- CUDA: 11.8 & 12.1 +- CUDNN: 8+ +- GPU： Nvidia-3060 12G & Nvidia-3090 24G + +Linux 的详细信息： +- 操作系统 Ubuntu 20.04, CentOS +- python: python3.10 & python3.11 +- pytorch: torch2.2.0 +- CUDA: 11.8 & 12.1 +- CUDNN: 8+ +- GPU：Nvidia-V100 16G & Nvidia-A10 24G & Nvidia-A100 40G & Nvidia-A100 80G + +我们需要大约 60GB 的可用磁盘空间，请检查！ + +#### b. 权重放置 +我们最好将[权重](#model-zoo)按照指定路径进行放置： + +**通过comfyui**： +将模型放入Comfyui的权重文件夹`ComfyUI/models/Fun_Models/`： +``` +📦 ComfyUI/ +├── 📂 models/ +│ └── 📂 Fun_Models/ +│ ├── 📂 CogVideoX-Fun-V1.1-2b-InP/ +│ ├── 📂 CogVideoX-Fun-V1.1-5b-InP/ +│ ├── 📂 Wan2.1-Fun-V1.1-14B-InP +│ └── 📂 Wan2.1-Fun-V1.1-1.3B-InP/ +``` + +**运行自身的python文件或ui界面**: +``` +📦 models/ +├── 📂 Diffusion_Transformer/ +│ ├── 📂 CogVideoX-Fun-V1.1-2b-InP/ +│ ├── 📂 CogVideoX-Fun-V1.1-5b-InP/ +│ ├── 📂 Wan2.1-Fun-V1.1-14B-InP +│ └── 📂 Wan2.1-Fun-V1.1-1.3B-InP/ +├── 📂 Personalized_Model/ +│ └── your trained trainformer model / your trained lora model (for UI load) +``` + +# 视频作品 + +### Wan2.1-Fun-V1.1-14B-InP && Wan2.1-Fun-V1.1-1.3B-InP + + + + + + + + +
+ + + + + + + +
+ + + + + + + + +
+ + + + + + + +
+ +### Wan2.1-Fun-V1.1-14B-Control && Wan2.1-Fun-V1.1-1.3B-Control + +Generic Control Video + Reference Image: + + + + + + + + + + + + +
+ Reference Image + + Control Video + + Wan2.1-Fun-V1.1-14B-Control + + Wan2.1-Fun-V1.1-1.3B-Control +
+ + + + + + + +
+ + +Generic Control Video (Canny, Pose, Depth, etc.) and Trajectory Control: + + + + + + +
+ + + + + +
+ + + + + + + + + + + +
+ + + + + +
+ + + + + +
+ +### Wan2.1-Fun-V1.1-14B-Control-Camera && Wan2.1-Fun-V1.1-1.3B-Control-Camera + + + + + + + + + + + + + + + + + + + +
+ Pan Up + + Pan Left + + Pan Right +
+ + + + + +
+ Pan Down + + Pan Up + Pan Left + + Pan Up + Pan Right +
+ + + + + +
+ +### CogVideoX-Fun-V1.1-5B + +Resolution-1024 + + + + + + + + +
+ + + + + + + +
+ + +Resolution-768 + + + + + + + + +
+ + + + + + + +
+ +Resolution-512 + + + + + + + + +
+ + + + + + + +
+ +### CogVideoX-Fun-V1.1-5B-Control + + + + + + + + + + + + + + + + +
+ + + + + +
+ A young woman with beautiful clear eyes and blonde hair, wearing white clothes and twisting her body, with the camera focused on her face. High quality, masterpiece, best quality, high resolution, ultra-fine, dreamlike. + + A young woman with beautiful clear eyes and blonde hair, wearing white clothes and twisting her body, with the camera focused on her face. High quality, masterpiece, best quality, high resolution, ultra-fine, dreamlike. + + A young bear. +
+ + + + + +
+ +# 如何使用 + +

1. 生成

+ +#### a、显存节省方案 +由于Wan2.1的参数非常大，我们需要考虑显存节省方案，以节省显存适应消费级显卡。我们给每个预测文件都提供了GPU_memory_mode，可以在model_cpu_offload，model_cpu_offload_and_qfloat8，sequential_cpu_offload中进行选择。该方案同样适用于CogVideoX-Fun的生成。 + +- model_cpu_offload代表整个模型在使用后会进入cpu，可以节省部分显存。 +- model_cpu_offload_and_qfloat8代表整个模型在使用后会进入cpu，并且对transformer模型进行了float8的量化，可以节省更多的显存。 +- sequential_cpu_offload代表模型的每一层在使用后会进入cpu，速度较慢，节省大量显存。 + +qfloat8会部分降低模型的性能，但可以节省更多的显存。如果显存足够，推荐使用model_cpu_offload。 + +#### b、通过comfyui +具体查看[ComfyUI README](comfyui/README.md)。 + +#### c、运行python文件 + +##### i、单卡运行: + +- 步骤1：下载对应[权重](#model-zoo)放入models文件夹。 +- 步骤2：根据不同的权重与预测目标使用不同的文件进行预测。当前该库支持CogVideoX-Fun、Wan2.1和Wan2.1-Fun，在examples文件夹下用文件夹名以区分，不同模型支持的功能不同，请视具体情况予以区分。以CogVideoX-Fun为例。 + - 文生视频： + - 使用examples/cogvideox_fun/predict_t2v.py文件中修改prompt、neg_prompt、guidance_scale和seed。 + - 而后运行examples/cogvideox_fun/predict_t2v.py文件，等待生成结果，结果保存在samples/cogvideox-fun-videos文件夹中。 + - 图生视频： + - 使用examples/cogvideox_fun/predict_i2v.py文件中修改validation_image_start、validation_image_end、prompt、neg_prompt、guidance_scale和seed。 + - validation_image_start是视频的开始图片，validation_image_end是视频的结尾图片。 + - 而后运行examples/cogvideox_fun/predict_i2v.py文件，等待生成结果，结果保存在samples/cogvideox-fun-videos_i2v文件夹中。 + - 视频生视频： + - 使用examples/cogvideox_fun/predict_v2v.py文件中修改validation_video、validation_image_end、prompt、neg_prompt、guidance_scale和seed。 + - validation_video是视频生视频的参考视频。您可以使用以下视频运行演示：[演示视频](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/play_guitar.mp4) + - 而后运行examples/cogvideox_fun/predict_v2v.py文件，等待生成结果，结果保存在samples/cogvideox-fun-videos_v2v文件夹中。 + - 普通控制生视频（Canny、Pose、Depth等）： + - 使用examples/cogvideox_fun/predict_v2v_control.py文件中修改control_video、validation_image_end、prompt、neg_prompt、guidance_scale和seed。 + - control_video是控制生视频的控制视频，是使用Canny、Pose、Depth等算子提取后的视频。您可以使用以下视频运行演示：[演示视频](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/pose.mp4) + - 而后运行examples/cogvideox_fun/predict_v2v_control.py文件，等待生成结果，结果保存在samples/cogvideox-fun-videos_v2v_control文件夹中。 +- 步骤3：如果想结合自己训练的其他backbone与Lora，则看情况修改examples/{model_name}/predict_t2v.py中的examples/{model_name}/predict_i2v.py和lora_path。 + +##### ii、多卡运行: +在使用多卡预测时请注意安装xfuser仓库，推荐安装xfuser==0.4.2和yunchang==0.6.2。 +``` +pip install xfuser==0.4.2 --progress-bar off -i https://mirrors.aliyun.com/pypi/simple/ +pip install yunchang==0.6.2 --progress-bar off -i https://mirrors.aliyun.com/pypi/simple/ +``` + +请确保ulysses_degree和ring_degree的乘积等于使用的GPU数量。例如，如果您使用8个GPU，则可以设置ulysses_degree=2和ring_degree=4，也可以设置ulysses_degree=4和ring_degree=2。 + +ulysses_degree是在head进行切分后并行生成，ring_degree是在sequence上进行切分后并行生成。ring_degree相比ulysses_degree有更大的通信成本，在设置参数时需要结合序列长度和模型的head数进行设置。 + +以8卡并行预测为例。 +- 以Wan2.1-Fun-V1.1-14B-InP为例，其head数为40，ulysses_degree需要设置为其可以整除的数如2、4、8等。因此在使用8卡并行预测时，可以设置ulysses_degree=8和ring_degree=1. +- 以Wan2.1-Fun-V1.1-1.3B-InP为例，其head数为12，ulysses_degree需要设置为其可以整除的数如2、4等。因此在使用8卡并行预测时，可以设置ulysses_degree=4和ring_degree=2. + +设置完成后，使用如下指令进行并行预测： +```sh +torchrun --nproc-per-node=8 examples/wan2.1_fun/predict_t2v.py +``` + +#### d、通过ui界面 + +webui支持文生视频、图生视频、视频生视频和普通控制生视频（Canny、Pose、Depth等）。当前该库支持CogVideoX-Fun、Wan2.1和Wan2.1-Fun，在examples文件夹下用文件夹名以区分，不同模型支持的功能不同，请视具体情况予以区分。以CogVideoX-Fun为例。 + +- 步骤1：下载对应[权重](#model-zoo)放入models文件夹。 +- 步骤2：运行examples/cogvideox_fun/app.py文件，进入gradio页面。 +- 步骤3：根据页面选择生成模型，填入prompt、neg_prompt、guidance_scale和seed等，点击生成，等待生成结果，结果保存在sample文件夹中。 + +### 2. 模型训练 +一个完整的模型训练链路应该包括数据预处理和Video DiT训练。不同模型的训练流程类似，数据格式也类似： + +

a.数据预处理

+我们给出了一个简单的demo通过图片数据训练lora模型，详情可以查看[wiki](https://github.com/aigc-apps/CogVideoX-Fun/wiki/Training-Lora)。 + +一个完整的长视频切分、清洗、描述的数据预处理链路可以参考video caption部分的[README](cogvideox/video_caption/README.md)进行。 + +如果期望训练一个文生图视频的生成模型，您需要以这种格式排列数据集。 +``` +📦 project/ +├── 📂 datasets/ +│ ├── 📂 internal_datasets/ +│ ├── 📂 train/ +│ │ ├── 📄 00000001.mp4 +│ │ ├── 📄 00000002.jpg +│ │ └── 📄 ..... +│ └── 📄 json_of_internal_datasets.json +``` + +json_of_internal_datasets.json是一个标准的json文件。json中的file_path可以被设置为相对路径，如下所示： +```json +[ + { + "file_path": "train/00000001.mp4", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` + +你也可以将路径设置为绝对路径： +```json +[ + { + "file_path": "/mnt/data/videos/00000001.mp4", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "/mnt/data/train/00000001.jpg", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` +

b. Video DiT训练

+ +如果数据预处理时，数据的格式为相对路径，则进入scripts/{model_name}/train.sh进行如下设置。 +``` +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/json_of_internal_datasets.json" +``` + +如果数据的格式为绝对路径，则进入scripts/train.sh进行如下设置。 +``` +export DATASET_NAME="" +export DATASET_META_NAME="/mnt/data/json_of_internal_datasets.json" +``` + +最后运行scripts/train.sh。 +```sh +sh scripts/train.sh +``` + +关于一些参数的设置细节： +Wan2.1-Fun可以查看[Readme Train](scripts/wan2.1_fun/README_TRAIN.md)与[Readme Lora](scripts/wan2.1_fun/README_TRAIN_LORA.md)。 +Wan2.1可以查看[Readme Train](scripts/wan2.1/README_TRAIN.md)与[Readme Lora](scripts/wan2.1/README_TRAIN_LORA.md)。 +CogVideoX-Fun可以查看[Readme Train](scripts/cogvideox_fun/README_TRAIN.md)与[Readme Lora](scripts/cogvideox_fun/README_TRAIN_LORA.md)。 + + +# 模型地址 +## 1.Wan2.2-Fun + +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| Wan2.2-Fun-A14B-InP | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP) | Wan2.2-Fun-14B文图生视频权重，以多分辨率训练，支持首尾图预测。 | +| Wan2.2-Fun-A14B-Control | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control)| Wan2.2-Fun-14B视频控制权重，支持不同的控制条件，如Canny、Depth、Pose、MLSD等，同时支持使用轨迹控制。支持多分辨率（512，768，1024）的视频预测，，以81帧、每秒16帧进行训练，支持多语言预测 | +| Wan2.2-Fun-A14B-Control-Camera | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control-Camera)| Wan2.2-Fun-14B相机镜头控制权重。支持多分辨率（512，768，1024）的视频预测，，以81帧、每秒16帧进行训练，支持多语言预测 | +| Wan2.2-VACE-Fun-A14B | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-VACE-Fun-A14B) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-VACE-Fun-A14B)| 以VACE方案训练的Wan2.2控制权重，基础模型为Wan2.2-T2V-A14B，支持不同的控制条件，如Canny、Depth、Pose、MLSD、轨迹控制等。支持通过主体指定生视频。支持多分辨率（512，768，1024）的视频预测，支持多分辨率（512，768，1024）的视频预测，以81帧、每秒16帧进行训练，支持多语言预测 | +| Wan2.2-Fun-5B-InP | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-InP) | Wan2.2-Fun-5B文图生视频权重，以121帧、每秒24帧进行训练支持首尾图预测。 | +| Wan2.2-Fun-5B-Control | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-Control)| Wan2.2-Fun-5B视频控制权重，支持不同的控制条件，如Canny、Depth、Pose、MLSD等，同时支持使用轨迹控制。以121帧、每秒24帧进行训练，支持多语言预测 | +| Wan2.2-Fun-5B-Control-Camera | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-Control-Camera)| Wan2.2-Fun-5B相机镜头控制权重。以121帧、每秒24帧进行训练，支持多语言预测 | + +## 2. Wan2.2 + +| 名称 | Hugging Face | Model Scope | 描述 | +|--|--|--|--| +| Wan2.2-TI2V-5B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | 万象2.2-5B文生视频权重 | +| Wan2.2-T2V-A14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | 万象2.2-14B文生视频权重 | +| Wan2.2-I2V-A14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | 万象2.2-14B图生视频权重 | + +## 3. Wan2.1-Fun + +V1.1: +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| Wan2.1-Fun-V1.1-1.3B-InP | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-InP) | Wan2.1-Fun-V1.1-1.3B文图生视频权重，以多分辨率训练，支持首尾图预测。 | +| Wan2.1-Fun-V1.1-14B-InP | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP) | Wan2.1-Fun-V1.1-14B文图生视频权重，以多分辨率训练，支持首尾图预测。 | +| Wan2.1-Fun-V1.1-1.3B-Control | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control)| Wan2.1-Fun-V1.1-1.3B视频控制权重支持不同的控制条件，如Canny、Depth、Pose、MLSD等，支持参考图 + 控制条件进行控制，支持使用轨迹控制。支持多分辨率（512，768，1024）的视频预测，，以81帧、每秒16帧进行训练，支持多语言预测 | +| Wan2.1-Fun-V1.1-14B-Control | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control)| Wan2.1-Fun-V1.1-14B视视频控制权重支持不同的控制条件，如Canny、Depth、Pose、MLSD等，支持参考图 + 控制条件进行控制，支持使用轨迹控制。支持多分辨率（512，768，1024）的视频预测，，以81帧、每秒16帧进行训练，支持多语言预测 | +| Wan2.1-Fun-V1.1-1.3B-Control-Camera | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera)| Wan2.1-Fun-V1.1-1.3B相机镜头控制权重。支持多分辨率（512，768，1024）的视频预测，，以81帧、每秒16帧进行训练，支持多语言预测 | +| Wan2.1-Fun-V1.1-14B-Control-Camera | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control-Camera)| Wan2.1-Fun-V1.1-14B相机镜头控制权重。支持多分辨率（512，768，1024）的视频预测，，以81帧、每秒16帧进行训练，支持多语言预测 | + +V1.0: +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| Wan2.1-Fun-1.3B-InP | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-InP) | Wan2.1-Fun-1.3B文图生视频权重，以多分辨率训练，支持首尾图预测。 | +| Wan2.1-Fun-14B-InP | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-InP) | Wan2.1-Fun-14B文图生视频权重，以多分辨率训练，支持首尾图预测。 | +| Wan2.1-Fun-1.3B-Control | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control)| Wan2.1-Fun-1.3B视频控制权重，支持不同的控制条件，如Canny、Depth、Pose、MLSD等，同时支持使用轨迹控制。支持多分辨率（512，768，1024）的视频预测，，以81帧、每秒16帧进行训练，支持多语言预测 | +| Wan2.1-Fun-14B-Control | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control)| Wan2.1-Fun-14B视频控制权重，支持不同的控制条件，如Canny、Depth、Pose、MLSD等，同时支持使用轨迹控制。支持多分辨率（512，768，1024）的视频预测，，以81帧、每秒16帧进行训练，支持多语言预测 | + +## 4. Wan2.1 + +| 名称 | Hugging Face | Model Scope | 描述 | +|--|--|--|--| +| Wan2.1-T2V-1.3B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-T2V-1.3B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B) | 万象2.1-1.3B文生视频权重 | +| Wan2.1-T2V-14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-T2V-14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B) | 万象2.1-14B文生视频权重 | +| Wan2.1-I2V-14B-480P | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-480P) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P) | 万象2.1-14B-480P图生视频权重 | +| Wan2.1-I2V-14B-720P| [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-720P) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P) | 万象2.1-14B-720P图生视频权重 | + +## 5. FantasyTalking + +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| Wan2.1-I2V-14B-720P | - | [🤗Link](https://huggingface.co/Wan-AI/Wan2.1-I2V-14B-720P) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P) | 万象2.1-14B-720P图生视频权重 | +| Wav2Vec | - | [🤗Link](https://huggingface.co/facebook/wav2vec2-base-960h) | [😄Link](https://modelscope.cn/models/AI-ModelScope/wav2vec2-base-960h) | Wav2Vec模型，请放在Wan2.1-I2V-14B-720P文件夹下，命名为audio_encoder | +| FantasyTalking model | - | [🤗Link](https://huggingface.co/acvlab/FantasyTalking/) | [😄Link](https://www.modelscope.cn/models/amap_cvlab/FantasyTalking/) | 官方Audio Condition的权重。 | + +## 6. Qwen-Image + +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| Qwen-Image | [🤗Link](https://huggingface.co/Qwen/Qwen-Image) | [😄Link](https://modelscope.cn/models/Qwen/Qwen-Image) | Qwen-Image官方权重 | +| Qwen-Image-Edit | [🤗Link](https://huggingface.co/Qwen/Qwen-Image-Edit) | [😄Link](https://modelscope.cn/models/Qwen/Qwen-Image-Edit) | Qwen-Image-Edit官方权重 | +| Qwen-Image-Edit-2509 | [🤗Link](https://huggingface.co/Qwen/Qwen-Image-Edit-2509) | [😄Link](https://modelscope.cn/models/Qwen/Qwen-Image-Edit-2509) | Qwen-Image-Edit-2509官方权重 | + +## 7. Z-Image + +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| Z-Image-Turbo | [🤗Link](https://huggingface.co/Tongyi-MAI/Z-Image-Turbo) | [😄Link](https://www.modelscope.cn/models/Tongyi-MAI/Z-Image-Turbo) | Z-Image-Turbo官方权重 | + +## 8. Z-Image-Fun + +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| Z-Image-Turbo-Fun-Controlnet-Union | - | [🤗链接](https://huggingface.co/alibaba-pai/Z-Image-Turbo-Fun-Controlnet-Union) | [😄链接](https://modelscope.cn/models/PAI/Z-Image-Turbo-Fun-Controlnet-Union) | Z-Image-Turbo 的 ControlNet 权重，支持 Canny、Depth、Pose、MLSD 等多种控制条件。 | + +## 9. Flux + +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| FLUX.1-dev | [🤗Link](https://huggingface.co/black-forest-labs/FLUX.1-dev) | [😄Link](https://www.modelscope.cn/models/black-forest-labs/FLUX.1-dev) | FLUX.1-dev官方权重 | +| FLUX.2-dev | [🤗Link](https://huggingface.co/black-forest-labs/FLUX.2-dev) | [😄Link](https://www.modelscope.cn/models/black-forest-labs/FLUX.2-dev) | FLUX.2-dev官方权重 | + +## 10. Flux-Fun + +| 名称 | 存储 | Hugging Face | 魔搭社区（ModelScope） | 描述 | +|--|--|--|--|--| +| Flux.2-dev-Fun-Controlnet-Union | - | [🤗链接](https://huggingface.co/alibaba-pai/FLUX.2-dev-Fun-Controlnet-Union) | [😄链接](https://modelscope.cn/models/PAI/FLUX.2-dev-Fun-Controlnet-Union) | Flux.2-dev 的 ControlNet 权重，支持 Canny、Depth、Pose、MLSD 等多种控制条件。 | + +## 11. HunyuanVideo + +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| HunyuanVideo | [🤗Link](https://huggingface.co/hunyuanvideo-community/HunyuanVideo) | - | HunyuanVideo-diffusers权重 | +| HunyuanVideo-I2V | [🤗Link](https://huggingface.co/hunyuanvideo-community/HunyuanVideo-I2V) | - | HunyuanVideo-I2V-diffusers权重 | + +## 12. CogVideoX-Fun + +V1.5: + +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| CogVideoX-Fun-V1.5-5b-InP | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.5-5b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.5-5b-InP) | 官方的图生视频权重。支持多分辨率（512，768，1024）的视频预测，以85帧、每秒8帧进行训练 | +| CogVideoX-Fun-V1.5-Reward-LoRAs | - | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.5-Reward-LoRAs) | 官方的奖励反向传播技术模型，优化CogVideoX-Fun-V1.5生成的视频，使其更好地符合人类偏好。 | + + +V1.1: + +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| CogVideoX-Fun-V1.1-2b-InP | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-InP) | 官方的图生视频权重。支持多分辨率（512，768，1024，1280）的视频预测，以49帧、每秒8帧进行训练 | +| CogVideoX-Fun-V1.1-5b-InP | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-InP) | 官方的图生视频权重。添加了Noise，运动幅度相比于V1.0更大。支持多分辨率（512，768，1024，1280）的视频预测，以49帧、每秒8帧进行训练 | +| CogVideoX-Fun-V1.1-2b-Pose | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-Pose) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-Pose) | 官方的姿态控制生视频权重。支持多分辨率（512，768，1024，1280）的视频预测，以49帧、每秒8帧进行训练 | +| CogVideoX-Fun-V1.1-2b-Control | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-Control) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-Control) | 官方的控制生视频权重。支持多分辨率（512，768，1024，1280）的视频预测，以49帧、每秒8帧进行训练。支持不同的控制条件，如Canny、Depth、Pose、MLSD等 | +| CogVideoX-Fun-V1.1-5b-Pose | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-Pose) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-Pose) | 官方的姿态控制生视频权重。支持多分辨率（512，768，1024，1280）的视频预测，以49帧、每秒8帧进行训练 | +| CogVideoX-Fun-V1.1-5b-Control | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-Control) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-Control) | 官方的控制生视频权重。支持多分辨率（512，768，1024，1280）的视频预测，以49帧、每秒8帧进行训练。支持不同的控制条件，如Canny、Depth、Pose、MLSD等 | +| CogVideoX-Fun-V1.1-Reward-LoRAs | - | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-Reward-LoRAs) | 官方的奖励反向传播技术模型，优化CogVideoX-Fun-V1.1生成的视频，使其更好地符合人类偏好。 | + +
+ (Obsolete) V1.0: + +| 名称 | 存储空间 | Hugging Face | Model Scope | 描述 | +|--|--|--|--|--| +| CogVideoX-Fun-2b-InP | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-2b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-2b-InP) | 官方的图生视频权重。支持多分辨率（512，768，1024，1280）的视频预测，以49帧、每秒8帧进行训练 | +| CogVideoX-Fun-5b-InP | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-5b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-5b-InP) | 官方的图生视频权重。支持多分辨率（512，768，1024，1280）的视频预测，以49帧、每秒8帧进行训练 | +
+ +# 参考文献 +- CogVideo: https://github.com/THUDM/CogVideo/ +- EasyAnimate: https://github.com/aigc-apps/EasyAnimate +- Wan2.1: https://github.com/Wan-Video/Wan2.1/ +- Wan2.2: https://github.com/Wan-Video/Wan2.2/ +- Diffusers: https://github.com/huggingface/diffusers +- Qwen-Image: https://github.com/QwenLM/Qwen-Image +- Self-Forcing: https://github.com/guandeh17/Self-Forcing +- Flux: https://github.com/black-forest-labs/flux +- Flux2: https://github.com/black-forest-labs/flux2 +- HunyuanVideo: https://github.com/Tencent-Hunyuan/HunyuanVideo +- ComfyUI-KJNodes: https://github.com/kijai/ComfyUI-KJNodes +- ComfyUI-EasyAnimateWrapper: https://github.com/kijai/ComfyUI-EasyAnimateWrapper +- ComfyUI-CameraCtrl-Wrapper: https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper +- CameraCtrl: https://github.com/hehao13/CameraCtrl + +# 许可证 +本项目采用 [Apache License (Version 2.0)](https://github.com/modelscope/modelscope/blob/master/LICENSE). + +CogVideoX-2B 模型 (包括其对应的Transformers模块，VAE模块) 根据 [Apache 2.0 协议](LICENSE) 许可证发布。 + +CogVideoX-5B 模型（Transformer 模块）在[CogVideoX许可证](https://huggingface.co/THUDM/CogVideoX-5b/blob/main/LICENSE)下发布. diff --git a/VideoX-Fun/build_context.json b/VideoX-Fun/build_context.json new file mode 100644 index 0000000000000000000000000000000000000000..37f35bfcb7eec13f2644597b6eaec7debb5bc648 --- /dev/null +++ b/VideoX-Fun/build_context.json @@ -0,0 +1,63 @@ +{ + "toolchain_version": "5.1-patch1", + "toolchain_commit": "5c5e711b", + "target_platform": "AX650A", + "cfg": { + "input": "", + "output_dir": "", + "output_name": "", + "work_dir": "", + "model_type": "ONNX", + "target_hardware": "AX650", + "npu_mode": "NPU1", + "input_shapes": "", + "input_processors": [], + "output_processors": [], + "const_processors": [], + "op_processors": [], + "quant_op_processors": [], + "custom_ops": [] + }, + "axmodel_extra": { + "version": "", + "tensor_extras": [], + "subgraphs": [], + "hardware_type": "AX650" + }, + "build_start": 1768291836.7460434, + "build_time": 16.673909187316895, + "input_model_size": 0, + "macs": 0, + "compiled_model_size": 0, + "input_model": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun", + "work_dir": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun", + "output_dir": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun", + "output_model": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun", + "npu_mode": 0, + "npu_mode_str": "", + "model_type": 0, + "model_type_str": "ONNX", + "quant_dir": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun", + "onnx_model_check": false, + "frontend_dir": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun", + "random_seed": 1504902737, + "stride_ios": [], + "input_tensor_infos": {}, + "compiler_dir": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun", + "compiler_extra_input_shapes": [], + "compiler_static_batchs": [], + "compiler_max_dynamic_batch": 0, + "compiler_batch_sizes": [], + "enable_compiler_check": false, + "dump_npu_case": false, + "dump_npu_trace": false, + "gen_neusight": false, + "compress_mcode": true, + "code": "UnknownError", + "error_msg": "OSError: [Errno 28] No space left on device", + "axe_code": "NotRunning", + "axe_error_msg": "", + "io_info": "", + "subgraphs": [], + "custom_ops": [] +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/README.md b/VideoX-Fun/comfyui/README.md new file mode 100644 index 0000000000000000000000000000000000000000..741c939784915fbf523530957466dd07f40d6576 --- /dev/null +++ b/VideoX-Fun/comfyui/README.md @@ -0,0 +1,281 @@ +# ComfyUI VideoX-Fun +Easily use VideoX-Fun and Wan2.1-Fun inside ComfyUI! + +- [Installation](#1-installation) +- [Node types](#node-types) +- [Example workflows](#example-workflows) + +## Installation +### 1. ComfyUI Installation + +#### Option 1: Install via ComfyUI Manager +![](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/comfyui_manage.jpg) + +#### Option 2: Install manually +The VideoX-Fun repository needs to be placed at `ComfyUI/custom_nodes/VideoX-Fun/`. + +``` +cd ComfyUI/custom_nodes/ + +# Git clone the cogvideox_fun itself +git clone https://github.com/aigc-apps/VideoX-Fun.git + +# Git clone the video outout node +git clone https://github.com/Kosinkadink/ComfyUI-VideoHelperSuite.git + +# Git clone the KJ Nodes +git clone https://github.com/kijai/ComfyUI-KJNodes.git + +cd VideoX-Fun/ +python install.py +``` + +### 2. Download models +#### i、Full loading +Download full model into `ComfyUI/models/Fun_Models/`. + +#### ii、Chunked loading +Put the transformer model weights to the `ComfyUI/models/diffusion_models/`. +Put the text encoer model weights to the `ComfyUI/models/text_encoders/`. +Put the clip vision model weights to the `ComfyUI/models/clip_vision/`. +Put the vae model weights to the `ComfyUI/models/vae/`. +Put the tokenizer files to the `ComfyUI/models/Fun_Models/` (For example: `ComfyUI/models/Fun_Models/umt5-xxl`). + +### 3. (Optional) Download preprocess weights into `ComfyUI/custom_nodes/Fun_Models/Third_Party/`. +Except for the fun models' weights, if you want to use the control preprocess nodes, you can download the preprocess weights to `ComfyUI/custom_nodes/Fun_Models/Third_Party/`. + +``` +remote_onnx_det = "https://huggingface.co/yzd-v/DWPose/resolve/main/yolox_l.onnx" +remote_onnx_pose = "https://huggingface.co/yzd-v/DWPose/resolve/main/dw-ll_ucoco_384.onnx" +remote_zoe= "https://huggingface.co/lllyasviel/Annotators/resolve/main/ZoeD_M12_N.pt" +``` +#### i. Wan2.2-Fun +| Name | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| Wan2.2-Fun-A14B-InP | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-InP) | Wan2.2-Fun-14B text-to-video generation weights, trained at multiple resolutions, supports start-end image prediction. | +| Wan2.2-Fun-A14B-Control | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control)| Wan2.2-Fun-14B video control weights, supporting various control conditions such as Canny, Depth, Pose, MLSD, etc., and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction at 81 frames, trained at 16 frames per second, with multilingual prediction support. | +| Wan2.2-Fun-A14B-Control-Camera | 64.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-A14B-Control-Camera)| Wan2.2-Fun-14B camera lens control weights. Supports multi-resolution (512, 768, 1024) video prediction, trained with 81 frames at 16 FPS, supports multilingual prediction. | +| Wan2.2-Fun-5B-InP | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-InP) | Wan2.2-Fun-5B text-to-video weights trained at 121 frames, 24 FPS, supporting first/last frame prediction. | +| Wan2.2-Fun-5B-Control | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-Control)| Wan2.2-Fun-5B video control weights, supporting control conditions like Canny, Depth, Pose, MLSD, and trajectory control. Trained at 121 frames, 24 FPS, with multilingual prediction support. | +| Wan2.2-Fun-5B-Control-Camera | 23.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-5B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.2-Fun-5B-Control-Camera)| Wan2.2-Fun-5B camera lens control weights. Trained at 121 frames, 24 FPS, with multilingual prediction support. | + +#### ii. Wan2.2 + +| Name | Hugging Face | Model Scope | Description | +|--|--|--|--| +| Wan2.2-TI2V-5B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-TI2V-5B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-TI2V-5B) | Wan2.2-5B Text-to-Video Weights | +| Wan2.2-T2V-14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-T2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-T2V-A14B) | Wan2.2-14B Text-to-Video Weights | +| Wan2.2-I2V-A14B | [🤗Link](https://huggingface.co/Wan-AI/Wan2.2-I2V-A14B) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.2-I2V-A14B) | Wan2.2-I2V-A14B Image-to-Video Weights | + +#### iii. Wan2.1-Fun + +V1.1: +| Name | Storage Size | Hugging Face | Model Scope | Description | +|------|--------------|--------------|-------------|-------------| +| Wan2.1-Fun-V1.1-1.3B-InP | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-InP) | Wan2.1-Fun-V1.1-1.3B text-to-video generation weights, trained at multiple resolutions, supports start-end image prediction. | +| Wan2.1-Fun-V1.1-14B-InP | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-InP) | Wan2.1-Fun-V1.1-14B text-to-video generation weights, trained at multiple resolutions, supports start-end image prediction. | +| Wan2.1-Fun-V1.1-1.3B-Control | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control) | Wan2.1-Fun-V1.1-1.3B video control weights support various control conditions such as Canny, Depth, Pose, MLSD, etc., supports reference image + control condition-based control, and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction, trained with 81 frames at 16 FPS, supports multilingual prediction. | +| Wan2.1-Fun-V1.1-14B-Control | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control) | Wan2.1-Fun-V1.1-14B video control weights support various control conditions such as Canny, Depth, Pose, MLSD, etc., supports reference image + control condition-based control, and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction, trained with 81 frames at 16 FPS, supports multilingual prediction. | +| Wan2.1-Fun-V1.1-1.3B-Control-Camera | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-1.3B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-1.3B-Control-Camera) | Wan2.1-Fun-V1.1-1.3B camera lens control weights. Supports multi-resolution (512, 768, 1024) video prediction, trained with 81 frames at 16 FPS, supports multilingual prediction. | +| Wan2.1-Fun-V1.1-14B-Control-Camera | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-V1.1-14B-Control-Camera) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-V1.1-14B-Control-Camera) | Wan2.1-Fun-V1.1-14B camera lens control weights. Supports multi-resolution (512, 768, 1024) video prediction, trained with 81 frames at 16 FPS, supports multilingual prediction. | + +V1.0: +| Name | Storage Space | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| Wan2.1-Fun-1.3B-InP | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-InP) | Wan2.1-Fun-1.3B text-to-video weights, trained at multiple resolutions, supporting start and end frame prediction. | +| Wan2.1-Fun-14B-InP | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-InP) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-InP) | Wan2.1-Fun-14B text-to-video weights, trained at multiple resolutions, supporting start and end frame prediction. | +| Wan2.1-Fun-1.3B-Control | 19.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-1.3B-Control) | Wan2.1-Fun-1.3B video control weights, supporting various control conditions such as Canny, Depth, Pose, MLSD, etc., and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction at 81 frames, trained at 16 frames per second, with multilingual prediction support. | +| Wan2.1-Fun-14B-Control | 47.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-Control) | [😄Link](https://modelscope.cn/models/PAI/Wan2.1-Fun-14B-Control) | Wan2.1-Fun-14B video control weights, supporting various control conditions such as Canny, Depth, Pose, MLSD, etc., and trajectory control. Supports multi-resolution (512, 768, 1024) video prediction at 81 frames, trained at 16 frames per second, with multilingual prediction support. | + +#### iv. Wan2.1 + +| Name | Hugging Face | Model Scope | Description | +|--|--|--|--| +| Wan2.1-T2V-1.3B | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-InP) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-T2V-1.3B) | Wanxiang 2.1-1.3B text-to-video weights | +| Wan2.1-T2V-14B | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-InP) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-T2V-14B) | Wanxiang 2.1-14B text-to-video weights | +| Wan2.1-I2V-14B-480P | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-InP) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-480P) | Wanxiang 2.1-14B-480P image-to-video weights | +| Wan2.1-I2V-14B-720P| [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-InP) | [😄Link](https://www.modelscope.cn/models/Wan-AI/Wan2.1-I2V-14B-720P) | Wanxiang 2.1-14B-720P image-to-video weights | + +#### v. CogVideoX-Fun + +V1.5: + +| Name | Storage Space | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| CogVideoX-Fun-V1.5-5b-InP | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.5-5b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.5-5b-InP) | Our official graph-generated video model is capable of predicting videos at multiple resolutions (512, 768, 1024) and has been trained on 85 frames at a rate of 8 frames per second. | +| CogVideoX-Fun-V1.5-Reward-LoRAs | - | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.5-Reward-LoRAs) | The official reward backpropagation technology model optimizes the videos generated by CogVideoX-Fun-V1.5 to better match human preferences. ｜ + +V1.1: + +| Name | Storage Space | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| CogVideoX-Fun-V1.1-2b-InP | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-InP) | Our official graph-generated video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. | +| CogVideoX-Fun-V1.1-5b-InP | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-InP) | Our official graph-generated video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. Noise has been added to the reference image, and the amplitude of motion is greater compared to V1.0. | +| CogVideoX-Fun-V1.1-2b-Pose | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-Pose) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-Pose) | Our official pose-control video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second.| +| CogVideoX-Fun-V1.1-2b-Control | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-2b-Control) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-2b-Control) | Our official control video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. Supporting various control conditions such as Canny, Depth, Pose, MLSD, etc.| +| CogVideoX-Fun-V1.1-5b-Pose | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-Pose) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-Pose) | Our official pose-control video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second.| +| CogVideoX-Fun-V1.1-5b-Control | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-5b-Control) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-5b-Control) | Our official control video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. Supporting various control conditions such as Canny, Depth, Pose, MLSD, etc.| +| CogVideoX-Fun-V1.1-Reward-LoRAs | - | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-V1.1-Reward-LoRAs) | The official reward backpropagation technology model optimizes the videos generated by CogVideoX-Fun-V1.1 to better match human preferences. ｜ + +
+ (Obsolete) V1.0: + +| Name | Storage Space | Hugging Face | Model Scope | Description | +|--|--|--|--|--| +| CogVideoX-Fun-2b-InP | 13.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-2b-InP) | [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-2b-InP) | Our official graph-generated video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. | +| CogVideoX-Fun-5b-InP | 20.0 GB | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-5b-InP)| [😄Link](https://modelscope.cn/models/PAI/CogVideoX-Fun-5b-InP)| Our official graph-generated video model is capable of predicting videos at multiple resolutions (512, 768, 1024, 1280) and has been trained on 49 frames at a rate of 8 frames per second. | +
+ +### 3. (Optional) Download Lora models into `ComfyUI/models/loras/fun_models/` +If you want to use lora in CogVideoX-Fun, please put the lora to `ComfyUI/models/loras/fun_models/`. + +## Node types +### 1. Wan-Fun +- **LoadWanFunModel** + - Loads the Wan-Fun Model. +- **LoadWanFunLora** + - Write the prompt for Wan-Fun model +- **WanFunInpaintSampler** + - Wan-Fun Sampler for Image to Video +- **WanFunT2VSampler** + - Wan-Fun Sampler for Text to Video + +### 2. Wan +- **LoadWanModel** + - Loads the Wan-Fun Model. +- **LoadWanLora** + - Write the prompt for Wan-Fun model +- **WanI2VSampler** + - Wan-Fun Sampler for Image to Video +- **WanT2VSampler** + - Wan-Fun Sampler for Text to Video + +### 3. CogVideoX-Fun +- **LoadCogVideoXFunModel** + - Loads the CogVideoX-Fun model +- **FunTextBox** + - Write the prompt for CogVideoX-Fun model +- **CogVideoXFunInpaintSampler** + - CogVideoX-Fun Sampler for Image to Video +- **CogVideoXFunT2VSampler** + - CogVideoX-Fun Sampler for Text to Video +- **CogVideoXFunV2VSampler** + - CogVideoX-Fun Sampler for Video to Video + +## Example workflows +### 1. Wan-Fun +#### i. Image to video generation +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_i2v.json) for wan-fun. + +Our ui is shown as follow: +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_i2v.jpg) + +You can run the demo using following photo: +![demo image](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/firework.png) + +#### ii. Text to video generation +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_t2v.json) for wan-fun. + +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_t2v.jpg) + +### iii. Trajectory Control Video Generation +Our user interface is shown as follows, this is the [json](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_control_trajectory.json): + +![Workflow Diagram](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_control_trajectory.jpg) + +You can run a demo using the following photo: + +![Demo Image](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/asset/v5.1/dog.png) + +### iv. Control Video Generation +Our user interface is shown as follows, this is the [json](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_v2v_control.json): + +To facilitate usage, we have added several JSON configurations that automatically process input videos into the necessary control videos. These include [canny processing](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_v2v_control_canny.json), [pose processing](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_v2v_control_pose.json), and [depth processing](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_v2v_control_depth.json). + +![Workflow Diagram](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_v2v_control.jpg) + +You can run a demo using the following video: + +[Demo Video](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/pose.mp4) + +### v. Control + Ref Video Generation +Our user interface is shown as follows, this is the [json](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_v2v_control_ref.json): + +To facilitate usage, we have added several JSON configurations that automatically process input videos into the necessary control videos. These include [pose processing](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_v2v_control_pose_ref.json), and [depth processing](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_v2v_control_depth_ref.json). + +![Workflow Diagram](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_v2v_control_ref.jpg) + +You can run a demo using the following video: + +[Demo Image](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/6.png) + +[Demo Video](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/pose.mp4) + +### vi. Camera Control Video Generation +Our user interface is shown as follows, this is the [json](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_control_camera.json): + +![Workflow Diagram](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan_fun/asset/v1.1/wan2.1_fun_workflow_control_camera.jpg) + +You can run a demo using the following photo: + +![Demo Image](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/firework.png) + +### 2. Wan +#### i. Image to video generation +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan/asset/v1.0/wan2.1_workflow_i2v.json) for wan-fun. + +Our ui is shown as follow: +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan/asset/v1.0/wan2.1_workflow_i2v.jpg) + +You can run the demo using following photo: +![demo image](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/firework.png) + +#### ii. Text to video generation +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan/asset/v1.0/wan2.1_workflow_t2v.json) for wan-fun. + +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/wan/asset/v1.0/wan2.1_workflow_t2v.jpg) + +### 3. CogVideoX-Fun +#### i. Video to video generation +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.5/cogvideoxfunv1.5_workflow_v2v.json) for v1.5. + +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/cogvideoxfunv1.1_workflow_v2v.json) for v1.1. + +Our ui is shown as follow: +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/cogvideoxfunv1.1_workflow_v2v.jpg) + +You can run the demo using following video: +[demo video](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/play_guitar.mp4) + +#### ii. Image to video generation +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.5/cogvideoxfunv1.5_workflow_i2v.json) for v1.5. + +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/cogvideoxfunv1.1_workflow_i2v.json) for v1.1. + +Our ui is shown as follow: +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/cogvideoxfunv1.1_workflow_i2v.jpg) + +You can run the demo using following photo: +![demo image](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1/firework.png) + +#### iii. Text to video generation +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.5/cogvideoxfunv1.5_workflow_t2v.json) for v1.5. + +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/cogvideoxfunv1.1_workflow_t2v.json) for v1.1. + +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/cogvideoxfunv1.1_workflow_t2v.jpg) + +#### iv. Control video generation +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/cogvideoxfunv1.1_workflow_v2v_control.json) for v1.1. + +Our ui is shown as follow: +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/cogvideoxfunv1.1_workflow_v2v_control.jpg) + +You can run the demo using following video: +[demo video](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/pose.mp4) + +#### v. Lora usage. +[Download link](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/cogvideoxfunv1.1_workflow_t2v_lora.json) for v1.1. + +Our ui is shown as follow: +![workflow graph](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/cogvideoxfunv1.1_workflow_t2v_lora.jpg) \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/dwpose_utils/onnxdet.py b/VideoX-Fun/comfyui/annotator/dwpose_utils/onnxdet.py new file mode 100644 index 0000000000000000000000000000000000000000..0c210a5307bb571410efaf1bbcce43f1d213cf3c --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/dwpose_utils/onnxdet.py @@ -0,0 +1,128 @@ +import cv2 +import numpy as np + +def nms(boxes, scores, nms_thr): + """Single class NMS implemented in Numpy.""" + x1 = boxes[:, 0] + y1 = boxes[:, 1] + x2 = boxes[:, 2] + y2 = boxes[:, 3] + + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + + keep = [] + while order.size > 0: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + + w = np.maximum(0.0, xx2 - xx1 + 1) + h = np.maximum(0.0, yy2 - yy1 + 1) + inter = w * h + ovr = inter / (areas[i] + areas[order[1:]] - inter) + + inds = np.where(ovr <= nms_thr)[0] + order = order[inds + 1] + + return keep + +def multiclass_nms(boxes, scores, nms_thr, score_thr): + """Multiclass NMS implemented in Numpy. Class-aware version.""" + final_dets = [] + num_classes = scores.shape[1] + for cls_ind in range(num_classes): + cls_scores = scores[:, cls_ind] + valid_score_mask = cls_scores > score_thr + if valid_score_mask.sum() == 0: + continue + else: + valid_scores = cls_scores[valid_score_mask] + valid_boxes = boxes[valid_score_mask] + keep = nms(valid_boxes, valid_scores, nms_thr) + if len(keep) > 0: + cls_inds = np.ones((len(keep), 1)) * cls_ind + dets = np.concatenate( + [valid_boxes[keep], valid_scores[keep, None], cls_inds], 1 + ) + final_dets.append(dets) + if len(final_dets) == 0: + return None + return np.concatenate(final_dets, 0) + +def demo_postprocess(outputs, img_size, p6=False): + grids = [] + expanded_strides = [] + strides = [8, 16, 32] if not p6 else [8, 16, 32, 64] + + hsizes = [img_size[0] // stride for stride in strides] + wsizes = [img_size[1] // stride for stride in strides] + + for hsize, wsize, stride in zip(hsizes, wsizes, strides): + xv, yv = np.meshgrid(np.arange(wsize), np.arange(hsize)) + grid = np.stack((xv, yv), 2).reshape(1, -1, 2) + grids.append(grid) + shape = grid.shape[:2] + expanded_strides.append(np.full((*shape, 1), stride)) + + grids = np.concatenate(grids, 1) + expanded_strides = np.concatenate(expanded_strides, 1) + outputs[..., :2] = (outputs[..., :2] + grids) * expanded_strides + outputs[..., 2:4] = np.exp(outputs[..., 2:4]) * expanded_strides + + return outputs + +def preprocess(img, input_size, swap=(2, 0, 1)): + if len(img.shape) == 3: + padded_img = np.ones((input_size[0], input_size[1], 3), dtype=np.uint8) * 114 + else: + padded_img = np.ones(input_size, dtype=np.uint8) * 114 + + r = min(input_size[0] / img.shape[0], input_size[1] / img.shape[1]) + resized_img = cv2.resize( + img, + (int(img.shape[1] * r), int(img.shape[0] * r)), + interpolation=cv2.INTER_LINEAR, + ).astype(np.uint8) + padded_img[: int(img.shape[0] * r), : int(img.shape[1] * r)] = resized_img + + padded_img = padded_img.transpose(swap) + padded_img = np.ascontiguousarray(padded_img, dtype=np.float32) + return padded_img, r + +def inference_detector(session, oriImg, detect_classes=[0]): + input_shape = (640,640) + img, ratio = preprocess(oriImg, input_shape) + + input = img[None, :, :, :] + if "InferenceSession" in type(session).__name__: + input_name = session.get_inputs()[0].name + output = session.run(None, {input_name: input}) + else: + outNames = session.getUnconnectedOutLayersNames() + session.setInput(input) + output = session.forward(outNames) + + predictions = demo_postprocess(output[0], input_shape)[0] + + boxes = predictions[:, :4] + scores = predictions[:, 4:5] * predictions[:, 5:] + + boxes_xyxy = np.ones_like(boxes) + boxes_xyxy[:, 0] = boxes[:, 0] - boxes[:, 2]/2. + boxes_xyxy[:, 1] = boxes[:, 1] - boxes[:, 3]/2. + boxes_xyxy[:, 2] = boxes[:, 0] + boxes[:, 2]/2. + boxes_xyxy[:, 3] = boxes[:, 1] + boxes[:, 3]/2. + boxes_xyxy /= ratio + dets = multiclass_nms(boxes_xyxy, scores, nms_thr=0.45, score_thr=0.1) + if dets is None: + return None + final_boxes, final_scores, final_cls_inds = dets[:, :4], dets[:, 4], dets[:, 5] + isscore = final_scores>0.3 + iscat = np.isin(final_cls_inds, detect_classes) + isbbox = [ i and j for (i, j) in zip(isscore, iscat)] + final_boxes = final_boxes[isbbox] + return final_boxes diff --git a/VideoX-Fun/comfyui/annotator/dwpose_utils/onnxpose.py b/VideoX-Fun/comfyui/annotator/dwpose_utils/onnxpose.py new file mode 100644 index 0000000000000000000000000000000000000000..529404ca82e49fe67b6f3a57609bd41b9022a94c --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/dwpose_utils/onnxpose.py @@ -0,0 +1,364 @@ +from typing import List, Tuple + +import cv2 +import numpy as np + +def preprocess( + img: np.ndarray, out_bbox, input_size: Tuple[int, int] = (192, 256) +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Do preprocessing for DWPose model inference. + + Args: + img (np.ndarray): Input image in shape. + input_size (tuple): Input image size in shape (w, h). + + Returns: + tuple: + - resized_img (np.ndarray): Preprocessed image. + - center (np.ndarray): Center of image. + - scale (np.ndarray): Scale of image. + """ + # get shape of image + img_shape = img.shape[:2] + out_img, out_center, out_scale = [], [], [] + if len(out_bbox) == 0: + out_bbox = [[0, 0, img_shape[1], img_shape[0]]] + for i in range(len(out_bbox)): + x0 = out_bbox[i][0] + y0 = out_bbox[i][1] + x1 = out_bbox[i][2] + y1 = out_bbox[i][3] + bbox = np.array([x0, y0, x1, y1]) + + # get center and scale + center, scale = bbox_xyxy2cs(bbox, padding=1.25) + + # do affine transformation + resized_img, scale = top_down_affine(input_size, scale, center, img) + + # normalize image + mean = np.array([123.675, 116.28, 103.53]) + std = np.array([58.395, 57.12, 57.375]) + resized_img = (resized_img - mean) / std + + out_img.append(resized_img) + out_center.append(center) + out_scale.append(scale) + + return out_img, out_center, out_scale + + +def inference(sess, img): + """Inference DWPose model. + + Args: + sess : ONNXRuntime session. + img : Input image in shape. + + Returns: + outputs : Output of DWPose model. + """ + all_out = [] + # build input + input = np.stack(img, axis=0).transpose(0, 3, 1, 2) + input = input.astype(np.float32) + if "InferenceSession" in type(sess).__name__: + input_name = sess.get_inputs()[0].name + all_outputs = sess.run(None, {input_name: input}) + for batch_idx in range(len(all_outputs[0])): + outputs = [all_outputs[i][batch_idx:batch_idx+1,...] for i in range(len(all_outputs))] + all_out.append(outputs) + return all_out + + for i in range(len(img)): + + input = img[i].transpose(2, 0, 1) + input = input[None, :, :, :] + + outNames = sess.getUnconnectedOutLayersNames() + sess.setInput(input) + outputs = sess.forward(outNames) + all_out.append(outputs) + + return all_out + + +def postprocess(outputs: List[np.ndarray], + model_input_size: Tuple[int, int], + center: Tuple[int, int], + scale: Tuple[int, int], + simcc_split_ratio: float = 2.0 + ) -> Tuple[np.ndarray, np.ndarray]: + """Postprocess for DWPose model output. + + Args: + outputs (np.ndarray): Output of RTMPose model. + model_input_size (tuple): RTMPose model Input image size. + center (tuple): Center of bbox in shape (x, y). + scale (tuple): Scale of bbox in shape (w, h). + simcc_split_ratio (float): Split ratio of simcc. + + Returns: + tuple: + - keypoints (np.ndarray): Rescaled keypoints. + - scores (np.ndarray): Model predict scores. + """ + all_key = [] + all_score = [] + for i in range(len(outputs)): + # use simcc to decode + simcc_x, simcc_y = outputs[i] + keypoints, scores = decode(simcc_x, simcc_y, simcc_split_ratio) + + # rescale keypoints + keypoints = keypoints / model_input_size * scale[i] + center[i] - scale[i] / 2 + all_key.append(keypoints[0]) + all_score.append(scores[0]) + + return np.array(all_key), np.array(all_score) + + +def bbox_xyxy2cs(bbox: np.ndarray, + padding: float = 1.) -> Tuple[np.ndarray, np.ndarray]: + """Transform the bbox format from (x,y,w,h) into (center, scale) + + Args: + bbox (ndarray): Bounding box(es) in shape (4,) or (n, 4), formatted + as (left, top, right, bottom) + padding (float): BBox padding factor that will be multilied to scale. + Default: 1.0 + + Returns: + tuple: A tuple containing center and scale. + - np.ndarray[float32]: Center (x, y) of the bbox in shape (2,) or + (n, 2) + - np.ndarray[float32]: Scale (w, h) of the bbox in shape (2,) or + (n, 2) + """ + # convert single bbox from (4, ) to (1, 4) + dim = bbox.ndim + if dim == 1: + bbox = bbox[None, :] + + # get bbox center and scale + x1, y1, x2, y2 = np.hsplit(bbox, [1, 2, 3]) + center = np.hstack([x1 + x2, y1 + y2]) * 0.5 + scale = np.hstack([x2 - x1, y2 - y1]) * padding + + if dim == 1: + center = center[0] + scale = scale[0] + + return center, scale + + +def _fix_aspect_ratio(bbox_scale: np.ndarray, + aspect_ratio: float) -> np.ndarray: + """Extend the scale to match the given aspect ratio. + + Args: + scale (np.ndarray): The image scale (w, h) in shape (2, ) + aspect_ratio (float): The ratio of ``w/h`` + + Returns: + np.ndarray: The reshaped image scale in (2, ) + """ + w, h = np.hsplit(bbox_scale, [1]) + bbox_scale = np.where(w > h * aspect_ratio, + np.hstack([w, w / aspect_ratio]), + np.hstack([h * aspect_ratio, h])) + return bbox_scale + + +def _rotate_point(pt: np.ndarray, angle_rad: float) -> np.ndarray: + """Rotate a point by an angle. + + Args: + pt (np.ndarray): 2D point coordinates (x, y) in shape (2, ) + angle_rad (float): rotation angle in radian + + Returns: + np.ndarray: Rotated point in shape (2, ) + """ + sn, cs = np.sin(angle_rad), np.cos(angle_rad) + rot_mat = np.array([[cs, -sn], [sn, cs]]) + return rot_mat @ pt + + +def _get_3rd_point(a: np.ndarray, b: np.ndarray) -> np.ndarray: + """To calculate the affine matrix, three pairs of points are required. This + function is used to get the 3rd point, given 2D points a & b. + + The 3rd point is defined by rotating vector `a - b` by 90 degrees + anticlockwise, using b as the rotation center. + + Args: + a (np.ndarray): The 1st point (x,y) in shape (2, ) + b (np.ndarray): The 2nd point (x,y) in shape (2, ) + + Returns: + np.ndarray: The 3rd point. + """ + direction = a - b + c = b + np.r_[-direction[1], direction[0]] + return c + + +def get_warp_matrix(center: np.ndarray, + scale: np.ndarray, + rot: float, + output_size: Tuple[int, int], + shift: Tuple[float, float] = (0., 0.), + inv: bool = False) -> np.ndarray: + """Calculate the affine transformation matrix that can warp the bbox area + in the input image to the output size. + + Args: + center (np.ndarray[2, ]): Center of the bounding box (x, y). + scale (np.ndarray[2, ]): Scale of the bounding box + wrt [width, height]. + rot (float): Rotation angle (degree). + output_size (np.ndarray[2, ] | list(2,)): Size of the + destination heatmaps. + shift (0-100%): Shift translation ratio wrt the width/height. + Default (0., 0.). + inv (bool): Option to inverse the affine transform direction. + (inv=False: src->dst or inv=True: dst->src) + + Returns: + np.ndarray: A 2x3 transformation matrix + """ + shift = np.array(shift) + src_w = scale[0] + dst_w = output_size[0] + dst_h = output_size[1] + + # compute transformation matrix + rot_rad = np.deg2rad(rot) + src_dir = _rotate_point(np.array([0., src_w * -0.5]), rot_rad) + dst_dir = np.array([0., dst_w * -0.5]) + + # get four corners of the src rectangle in the original image + src = np.zeros((3, 2), dtype=np.float32) + src[0, :] = center + scale * shift + src[1, :] = center + src_dir + scale * shift + src[2, :] = _get_3rd_point(src[0, :], src[1, :]) + + # get four corners of the dst rectangle in the input image + dst = np.zeros((3, 2), dtype=np.float32) + dst[0, :] = [dst_w * 0.5, dst_h * 0.5] + dst[1, :] = np.array([dst_w * 0.5, dst_h * 0.5]) + dst_dir + dst[2, :] = _get_3rd_point(dst[0, :], dst[1, :]) + + if inv: + warp_mat = cv2.getAffineTransform(np.float32(dst), np.float32(src)) + else: + warp_mat = cv2.getAffineTransform(np.float32(src), np.float32(dst)) + + return warp_mat + + +def top_down_affine(input_size: dict, bbox_scale: dict, bbox_center: dict, + img: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Get the bbox image as the model input by affine transform. + + Args: + input_size (dict): The input size of the model. + bbox_scale (dict): The bbox scale of the img. + bbox_center (dict): The bbox center of the img. + img (np.ndarray): The original image. + + Returns: + tuple: A tuple containing center and scale. + - np.ndarray[float32]: img after affine transform. + - np.ndarray[float32]: bbox scale after affine transform. + """ + w, h = input_size + warp_size = (int(w), int(h)) + + # reshape bbox to fixed aspect ratio + bbox_scale = _fix_aspect_ratio(bbox_scale, aspect_ratio=w / h) + + # get the affine matrix + center = bbox_center + scale = bbox_scale + rot = 0 + warp_mat = get_warp_matrix(center, scale, rot, output_size=(w, h)) + + # do affine transform + img = cv2.warpAffine(img, warp_mat, warp_size, flags=cv2.INTER_LINEAR) + + return img, bbox_scale + + +def get_simcc_maximum(simcc_x: np.ndarray, + simcc_y: np.ndarray) -> Tuple[np.ndarray, np.ndarray]: + """Get maximum response location and value from simcc representations. + + Note: + instance number: N + num_keypoints: K + heatmap height: H + heatmap width: W + + Args: + simcc_x (np.ndarray): x-axis SimCC in shape (K, Wx) or (N, K, Wx) + simcc_y (np.ndarray): y-axis SimCC in shape (K, Wy) or (N, K, Wy) + + Returns: + tuple: + - locs (np.ndarray): locations of maximum heatmap responses in shape + (K, 2) or (N, K, 2) + - vals (np.ndarray): values of maximum heatmap responses in shape + (K,) or (N, K) + """ + N, K, Wx = simcc_x.shape + simcc_x = simcc_x.reshape(N * K, -1) + simcc_y = simcc_y.reshape(N * K, -1) + + # get maximum value locations + x_locs = np.argmax(simcc_x, axis=1) + y_locs = np.argmax(simcc_y, axis=1) + locs = np.stack((x_locs, y_locs), axis=-1).astype(np.float32) + max_val_x = np.amax(simcc_x, axis=1) + max_val_y = np.amax(simcc_y, axis=1) + + # get maximum value across x and y axis + mask = max_val_x > max_val_y + max_val_x[mask] = max_val_y[mask] + vals = max_val_x + locs[vals <= 0.] = -1 + + # reshape + locs = locs.reshape(N, K, 2) + vals = vals.reshape(N, K) + + return locs, vals + + +def decode(simcc_x: np.ndarray, simcc_y: np.ndarray, + simcc_split_ratio) -> Tuple[np.ndarray, np.ndarray]: + """Modulate simcc distribution with Gaussian. + + Args: + simcc_x (np.ndarray[K, Wx]): model predicted simcc in x. + simcc_y (np.ndarray[K, Wy]): model predicted simcc in y. + simcc_split_ratio (int): The split ratio of simcc. + + Returns: + tuple: A tuple containing center and scale. + - np.ndarray[float32]: keypoints in shape (K, 2) or (n, K, 2) + - np.ndarray[float32]: scores in shape (K,) or (n, K) + """ + keypoints, scores = get_simcc_maximum(simcc_x, simcc_y) + keypoints /= simcc_split_ratio + + return keypoints, scores + + +def inference_pose(session, out_bbox, oriImg, model_input_size: Tuple[int, int]= (288, 384) ): + resized_img, center, scale = preprocess(oriImg, out_bbox, model_input_size) + outputs = inference(session, resized_img) + keypoints, scores = postprocess(outputs, model_input_size, center, scale) + + return keypoints, scores \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/dwpose_utils/util.py b/VideoX-Fun/comfyui/annotator/dwpose_utils/util.py new file mode 100644 index 0000000000000000000000000000000000000000..1c296984d8b1560ab63fd13b5a7ff31a13c71596 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/dwpose_utils/util.py @@ -0,0 +1,359 @@ +import math +import numpy as np +import matplotlib +import cv2 + +eps = 0.01 + +def smart_resize(x, s): + Ht, Wt = s + if x.ndim == 2: + Ho, Wo = x.shape + Co = 1 + else: + Ho, Wo, Co = x.shape + if Co == 3 or Co == 1: + k = float(Ht + Wt) / float(Ho + Wo) + return cv2.resize(x, (int(Wt), int(Ht)), interpolation=cv2.INTER_AREA if k < 1 else cv2.INTER_LANCZOS4) + else: + return np.stack([smart_resize(x[:, :, i], s) for i in range(Co)], axis=2) + +def smart_resize_k(x, fx, fy): + if x.ndim == 2: + Ho, Wo = x.shape + Co = 1 + else: + Ho, Wo, Co = x.shape + Ht, Wt = Ho * fy, Wo * fx + if Co == 3 or Co == 1: + k = float(Ht + Wt) / float(Ho + Wo) + return cv2.resize(x, (int(Wt), int(Ht)), interpolation=cv2.INTER_AREA if k < 1 else cv2.INTER_LANCZOS4) + else: + return np.stack([smart_resize_k(x[:, :, i], fx, fy) for i in range(Co)], axis=2) + +def padRightDownCorner(img, stride, padValue): + h = img.shape[0] + w = img.shape[1] + + pad = 4 * [None] + pad[0] = 0 # up + pad[1] = 0 # left + pad[2] = 0 if (h % stride == 0) else stride - (h % stride) # down + pad[3] = 0 if (w % stride == 0) else stride - (w % stride) # right + + img_padded = img + pad_up = np.tile(img_padded[0:1, :, :]*0 + padValue, (pad[0], 1, 1)) + img_padded = np.concatenate((pad_up, img_padded), axis=0) + pad_left = np.tile(img_padded[:, 0:1, :]*0 + padValue, (1, pad[1], 1)) + img_padded = np.concatenate((pad_left, img_padded), axis=1) + pad_down = np.tile(img_padded[-2:-1, :, :]*0 + padValue, (pad[2], 1, 1)) + img_padded = np.concatenate((img_padded, pad_down), axis=0) + pad_right = np.tile(img_padded[:, -2:-1, :]*0 + padValue, (1, pad[3], 1)) + img_padded = np.concatenate((img_padded, pad_right), axis=1) + + return img_padded, pad + +def transfer(model, model_weights): + transfered_model_weights = {} + for weights_name in model.state_dict().keys(): + transfered_model_weights[weights_name] = model_weights['.'.join(weights_name.split('.')[1:])] + return transfered_model_weights + +def is_normalized(keypoints) -> bool: + point_normalized = [ + 0 <= abs(k.x) <= 1 and 0 <= abs(k.y) <= 1 + for k in keypoints + if k is not None + ] + if not point_normalized: + return False + return all(point_normalized) + +def draw_bodypose(canvas: np.ndarray, keypoints) -> np.ndarray: + """ + Draw keypoints and limbs representing body pose on a given canvas. + + Args: + canvas (np.ndarray): A 3D numpy array representing the canvas (image) on which to draw the body pose. + keypoints (List[Keypoint]): A list of Keypoint objects representing the body keypoints to be drawn. + + Returns: + np.ndarray: A 3D numpy array representing the modified canvas with the drawn body pose. + + Note: + The function expects the x and y coordinates of the keypoints to be normalized between 0 and 1. + """ + if not is_normalized(keypoints): + H, W = 1.0, 1.0 + else: + H, W, _ = canvas.shape + + stickwidth = 4 + + limbSeq = [ + [2, 3], [2, 6], [3, 4], [4, 5], + [6, 7], [7, 8], [2, 9], [9, 10], + [10, 11], [2, 12], [12, 13], [13, 14], + [2, 1], [1, 15], [15, 17], [1, 16], + [16, 18], + ] + + colors = [[255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], [85, 255, 0], [0, 255, 0], \ + [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], \ + [170, 0, 255], [255, 0, 255], [255, 0, 170], [255, 0, 85]] + + for (k1_index, k2_index), color in zip(limbSeq, colors): + keypoint1 = keypoints[k1_index - 1] + keypoint2 = keypoints[k2_index - 1] + + if keypoint1 is None or keypoint2 is None: + continue + + Y = np.array([keypoint1.x, keypoint2.x]) * float(W) + X = np.array([keypoint1.y, keypoint2.y]) * float(H) + mX = np.mean(X) + mY = np.mean(Y) + length = ((X[0] - X[1]) ** 2 + (Y[0] - Y[1]) ** 2) ** 0.5 + angle = math.degrees(math.atan2(X[0] - X[1], Y[0] - Y[1])) + polygon = cv2.ellipse2Poly((int(mY), int(mX)), (int(length / 2), stickwidth), int(angle), 0, 360, 1) + cv2.fillConvexPoly(canvas, polygon, [int(float(c) * 0.6) for c in color]) + + for keypoint, color in zip(keypoints, colors): + if keypoint is None: + continue + + x, y = keypoint.x, keypoint.y + x = int(x * W) + y = int(y * H) + cv2.circle(canvas, (int(x), int(y)), 4, color, thickness=-1) + + return canvas + +def draw_handpose(canvas: np.ndarray, keypoints) -> np.ndarray: + """ + Draw keypoints and connections representing hand pose on a given canvas. + + Args: + canvas (np.ndarray): A 3D numpy array representing the canvas (image) on which to draw the hand pose. + keypoints (List[Keypoint]| None): A list of Keypoint objects representing the hand keypoints to be drawn + or None if no keypoints are present. + + Returns: + np.ndarray: A 3D numpy array representing the modified canvas with the drawn hand pose. + + Note: + The function expects the x and y coordinates of the keypoints to be normalized between 0 and 1. + """ + if not keypoints: + return canvas + + if not is_normalized(keypoints): + H, W = 1.0, 1.0 + else: + H, W, _ = canvas.shape + + edges = [[0, 1], [1, 2], [2, 3], [3, 4], [0, 5], [5, 6], [6, 7], [7, 8], [0, 9], [9, 10], \ + [10, 11], [11, 12], [0, 13], [13, 14], [14, 15], [15, 16], [0, 17], [17, 18], [18, 19], [19, 20]] + + for ie, (e1, e2) in enumerate(edges): + k1 = keypoints[e1] + k2 = keypoints[e2] + if k1 is None or k2 is None: + continue + + x1 = int(k1.x * W) + y1 = int(k1.y * H) + x2 = int(k2.x * W) + y2 = int(k2.y * H) + if x1 > eps and y1 > eps and x2 > eps and y2 > eps: + cv2.line(canvas, (x1, y1), (x2, y2), matplotlib.colors.hsv_to_rgb([ie / float(len(edges)), 1.0, 1.0]) * 255, thickness=2) + + for keypoint in keypoints: + if keypoint is None: + continue + + x, y = keypoint.x, keypoint.y + x = int(x * W) + y = int(y * H) + if x > eps and y > eps: + cv2.circle(canvas, (x, y), 4, (0, 0, 255), thickness=-1) + return canvas + + +def draw_facepose(canvas: np.ndarray, keypoints) -> np.ndarray: + """ + Draw keypoints representing face pose on a given canvas. + + Args: + canvas (np.ndarray): A 3D numpy array representing the canvas (image) on which to draw the face pose. + keypoints (List[Keypoint]| None): A list of Keypoint objects representing the face keypoints to be drawn + or None if no keypoints are present. + + Returns: + np.ndarray: A 3D numpy array representing the modified canvas with the drawn face pose. + + Note: + The function expects the x and y coordinates of the keypoints to be normalized between 0 and 1. + """ + if not keypoints: + return canvas + + if not is_normalized(keypoints): + H, W = 1.0, 1.0 + else: + H, W, _ = canvas.shape + + for keypoint in keypoints: + if keypoint is None: + continue + + x, y = keypoint.x, keypoint.y + x = int(x * W) + y = int(y * H) + if x > eps and y > eps: + cv2.circle(canvas, (x, y), 3, (255, 255, 255), thickness=-1) + return canvas + +# detect hand according to body pose keypoints +# please refer to https://github.com/CMU-Perceptual-Computing-Lab/openpose/blob/master/src/openpose/hand/handDetector.cpp +def handDetect(candidate, subset, oriImg): + # right hand: wrist 4, elbow 3, shoulder 2 + # left hand: wrist 7, elbow 6, shoulder 5 + ratioWristElbow = 0.33 + detect_result = [] + image_height, image_width = oriImg.shape[0:2] + for person in subset.astype(int): + # if any of three not detected + has_left = np.sum(person[[5, 6, 7]] == -1) == 0 + has_right = np.sum(person[[2, 3, 4]] == -1) == 0 + if not (has_left or has_right): + continue + hands = [] + #left hand + if has_left: + left_shoulder_index, left_elbow_index, left_wrist_index = person[[5, 6, 7]] + x1, y1 = candidate[left_shoulder_index][:2] + x2, y2 = candidate[left_elbow_index][:2] + x3, y3 = candidate[left_wrist_index][:2] + hands.append([x1, y1, x2, y2, x3, y3, True]) + # right hand + if has_right: + right_shoulder_index, right_elbow_index, right_wrist_index = person[[2, 3, 4]] + x1, y1 = candidate[right_shoulder_index][:2] + x2, y2 = candidate[right_elbow_index][:2] + x3, y3 = candidate[right_wrist_index][:2] + hands.append([x1, y1, x2, y2, x3, y3, False]) + + for x1, y1, x2, y2, x3, y3, is_left in hands: + # pos_hand = pos_wrist + ratio * (pos_wrist - pos_elbox) = (1 + ratio) * pos_wrist - ratio * pos_elbox + # handRectangle.x = posePtr[wrist*3] + ratioWristElbow * (posePtr[wrist*3] - posePtr[elbow*3]); + # handRectangle.y = posePtr[wrist*3+1] + ratioWristElbow * (posePtr[wrist*3+1] - posePtr[elbow*3+1]); + # const auto distanceWristElbow = getDistance(poseKeypoints, person, wrist, elbow); + # const auto distanceElbowShoulder = getDistance(poseKeypoints, person, elbow, shoulder); + # handRectangle.width = 1.5f * fastMax(distanceWristElbow, 0.9f * distanceElbowShoulder); + x = x3 + ratioWristElbow * (x3 - x2) + y = y3 + ratioWristElbow * (y3 - y2) + distanceWristElbow = math.sqrt((x3 - x2) ** 2 + (y3 - y2) ** 2) + distanceElbowShoulder = math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2) + width = 1.5 * max(distanceWristElbow, 0.9 * distanceElbowShoulder) + # x-y refers to the center --> offset to topLeft point + # handRectangle.x -= handRectangle.width / 2.f; + # handRectangle.y -= handRectangle.height / 2.f; + x -= width / 2 + y -= width / 2 # width = height + # overflow the image + if x < 0: x = 0 + if y < 0: y = 0 + width1 = width + width2 = width + if x + width > image_width: width1 = image_width - x + if y + width > image_height: width2 = image_height - y + width = min(width1, width2) + # the max hand box value is 20 pixels + if width >= 20: + detect_result.append([int(x), int(y), int(width), is_left]) + + ''' + return value: [[x, y, w, True if left hand else False]]. + width=height since the network require squared input. + x, y is the coordinate of top left + ''' + return detect_result + +# Written by Lvmin +def faceDetect(candidate, subset, oriImg): + # left right eye ear 14 15 16 17 + detect_result = [] + image_height, image_width = oriImg.shape[0:2] + for person in subset.astype(int): + has_head = person[0] > -1 + if not has_head: + continue + + has_left_eye = person[14] > -1 + has_right_eye = person[15] > -1 + has_left_ear = person[16] > -1 + has_right_ear = person[17] > -1 + + if not (has_left_eye or has_right_eye or has_left_ear or has_right_ear): + continue + + head, left_eye, right_eye, left_ear, right_ear = person[[0, 14, 15, 16, 17]] + + width = 0.0 + x0, y0 = candidate[head][:2] + + if has_left_eye: + x1, y1 = candidate[left_eye][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 3.0) + + if has_right_eye: + x1, y1 = candidate[right_eye][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 3.0) + + if has_left_ear: + x1, y1 = candidate[left_ear][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 1.5) + + if has_right_ear: + x1, y1 = candidate[right_ear][:2] + d = max(abs(x0 - x1), abs(y0 - y1)) + width = max(width, d * 1.5) + + x, y = x0, y0 + + x -= width + y -= width + + if x < 0: + x = 0 + + if y < 0: + y = 0 + + width1 = width * 2 + width2 = width * 2 + + if x + width > image_width: + width1 = image_width - x + + if y + width > image_height: + width2 = image_height - y + + width = min(width1, width2) + + if width >= 20: + detect_result.append([int(x), int(y), int(width)]) + + return detect_result + +# get max index of 2d array +def npmax(array): + arrayindex = array.argmax(1) + arrayvalue = array.max(1) + i = arrayvalue.argmax() + j = arrayindex[i] + return i, j diff --git a/VideoX-Fun/comfyui/annotator/dwpose_utils/wholebody.py b/VideoX-Fun/comfyui/annotator/dwpose_utils/wholebody.py new file mode 100644 index 0000000000000000000000000000000000000000..3c1d88c43afb66826a5bb7d62584004163574f8c --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/dwpose_utils/wholebody.py @@ -0,0 +1,129 @@ +import cv2 +import numpy as np + +import onnxruntime as ort +from .onnxdet import inference_detector +from .onnxpose import inference_pose +from typing import NamedTuple, List, Optional, Union + + +class Keypoint(NamedTuple): + x: float + y: float + score: float = 1.0 + id: int = -1 + + +class BodyResult(NamedTuple): + # Note: Using `Optional` instead of `|` operator as the ladder is a Python + # 3.10 feature. + # Annotator code should be Python 3.8 Compatible, as controlnet repo uses + # Python 3.8 environment. + # https://github.com/lllyasviel/ControlNet/blob/d3284fcd0972c510635a4f5abe2eeb71dc0de524/environment.yaml#L6 + keypoints: List[Optional[Keypoint]] + total_score: float = 0.0 + total_parts: int = 0 + + +HandResult = List[Keypoint] +FaceResult = List[Keypoint] +AnimalPoseResult = List[Keypoint] + + +class HumanPoseResult(NamedTuple): + body: BodyResult + left_hand: Optional[HandResult] + right_hand: Optional[HandResult] + face: Optional[FaceResult] + + +class Wholebody: + def __init__(self, onnx_det: str, onnx_pose: str): + # Always loads to CPU to avoid building OpenCV. + device = 'cpu' + backend = cv2.dnn.DNN_BACKEND_OPENCV if device == 'cpu' else cv2.dnn.DNN_BACKEND_CUDA + # You need to manually build OpenCV through cmake to work with your GPU. + providers = cv2.dnn.DNN_TARGET_CPU if device == 'cpu' else cv2.dnn.DNN_TARGET_CUDA + + self.session_det = cv2.dnn.readNetFromONNX(onnx_det) + self.session_det.setPreferableBackend(backend) + self.session_det.setPreferableTarget(providers) + + self.session_pose = cv2.dnn.readNetFromONNX(onnx_pose) + self.session_pose.setPreferableBackend(backend) + self.session_pose.setPreferableTarget(providers) + + def __call__(self, oriImg): + det_result = inference_detector(self.session_det, oriImg) + if det_result is None: + return None + + keypoints, scores = inference_pose(self.session_pose, det_result, oriImg) + + keypoints_info = np.concatenate( + (keypoints, scores[..., None]), axis=-1) + # compute neck joint + neck = np.mean(keypoints_info[:, [5, 6]], axis=1) + # neck score when visualizing pred + neck[:, 2:4] = np.logical_and( + keypoints_info[:, 5, 2:4] > 0.3, + keypoints_info[:, 6, 2:4] > 0.3).astype(int) + new_keypoints_info = np.insert( + keypoints_info, 17, neck, axis=1) + mmpose_idx = [ + 17, 6, 8, 10, 7, 9, 12, 14, 16, 13, 15, 2, 1, 4, 3 + ] + openpose_idx = [ + 1, 2, 3, 4, 6, 7, 8, 9, 10, 12, 13, 14, 15, 16, 17 + ] + new_keypoints_info[:, openpose_idx] = \ + new_keypoints_info[:, mmpose_idx] + keypoints_info = new_keypoints_info + + return keypoints_info + + @staticmethod + def format_result(keypoints_info: Optional[np.ndarray]) -> List[HumanPoseResult]: + def format_keypoint_part( + part: np.ndarray, + ) -> Optional[List[Optional[Keypoint]]]: + keypoints = [ + Keypoint(x, y, score, i) if score >= 0.3 else None + for i, (x, y, score) in enumerate(part) + ] + return ( + None if all(keypoint is None for keypoint in keypoints) else keypoints + ) + + def total_score(keypoints: Optional[List[Optional[Keypoint]]]) -> float: + return ( + sum(keypoint.score for keypoint in keypoints if keypoint is not None) + if keypoints is not None + else 0.0 + ) + + pose_results = [] + if keypoints_info is None: + return pose_results + + for instance in keypoints_info: + body_keypoints = format_keypoint_part(instance[:18]) or ([None] * 18) + left_hand = format_keypoint_part(instance[92:113]) + right_hand = format_keypoint_part(instance[113:134]) + face = format_keypoint_part(instance[24:92]) + + # Openpose face consists of 70 points in total, while DWPose only + # provides 68 points. Padding the last 2 points. + if face is not None: + # left eye + face.append(body_keypoints[14]) + # right eye + face.append(body_keypoints[15]) + + body = BodyResult( + body_keypoints, total_score(body_keypoints), len(body_keypoints) + ) + pose_results.append(HumanPoseResult(body, left_hand, right_hand, face)) + + return pose_results + diff --git a/VideoX-Fun/comfyui/annotator/nodes.py b/VideoX-Fun/comfyui/annotator/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..1fcd9cec5b9493e117d09392491c09b52a2e82fa --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/nodes.py @@ -0,0 +1,274 @@ +# This folder is modified from the https://github.com/Mikubill/sd-webui-controlnet +import os + +import cv2 +import folder_paths +import numpy as np +import torch +from einops import rearrange + +from .dwpose_utils import DWposeDetector +from .zoe.zoedepth.models.zoedepth.zoedepth_v1 import ZoeDepth +from .zoe.zoedepth.utils.config import get_config + +remote_onnx_det = "https://huggingface.co/yzd-v/DWPose/resolve/main/yolox_l.onnx" +remote_onnx_pose = "https://huggingface.co/yzd-v/DWPose/resolve/main/dw-ll_ucoco_384.onnx" +remote_zoe= "https://huggingface.co/lllyasviel/Annotators/resolve/main/ZoeD_M12_N.pt" + +def read_video(video_path): + cap = cv2.VideoCapture(video_path) + frames = [] + while cap.isOpened(): + ret, frame = cap.read() + if not ret: + break + frame = cv2.cvtColor(frame, cv2.COLOR_RGB2BGR) + frames.append(frame) + cap.release() + return frames + +def HWC3(x): + assert x.dtype == np.uint8 + if x.ndim == 2: + x = x[:, :, None] + assert x.ndim == 3 + H, W, C = x.shape + assert C == 1 or C == 3 or C == 4 + if C == 3: + return x + if C == 1: + return np.concatenate([x, x, x], axis=2) + if C == 4: + color = x[:, :, 0:3].astype(np.float32) + alpha = x[:, :, 3:4].astype(np.float32) / 255.0 + y = color * alpha + 255.0 * (1.0 - alpha) + y = y.clip(0, 255).astype(np.uint8) + return y + +def pad64(x): + return int(np.ceil(float(x) / 64.0) * 64 - x) + +def safer_memory(x): + # Fix many MAC/AMD problems + return np.ascontiguousarray(x.copy()).copy() + +def resize_image_with_pad(input_image, resolution, skip_hwc3=False): + if skip_hwc3: + img = input_image + else: + img = HWC3(input_image) + H_raw, W_raw, _ = img.shape + k = float(resolution) / float(min(H_raw, W_raw)) + interpolation = cv2.INTER_CUBIC if k > 1 else cv2.INTER_AREA + H_target = int(np.round(float(H_raw) * k)) + W_target = int(np.round(float(W_raw) * k)) + img = cv2.resize(img, (W_target, H_target), interpolation=interpolation) + H_pad, W_pad = pad64(H_target), pad64(W_target) + img_padded = np.pad(img, [[0, H_pad], [0, W_pad], [0, 0]], mode='edge') + + def remove_pad(x): + return safer_memory(x[:H_target, :W_target]) + + return safer_memory(img_padded), remove_pad + +def load_file_from_url( + url: str, + model_dir: str, + progress: bool = True, + file_name: str | None = None, + hash_prefix: str | None = None, +) -> str: + """Download a file from `url` into `model_dir`, using the file present if possible. + + Returns the path to the downloaded file. + """ + from urllib.parse import urlparse + os.makedirs(model_dir, exist_ok=True) + if not file_name: + parts = urlparse(url) + file_name = os.path.basename(parts.path) + cached_file = os.path.abspath(os.path.join(model_dir, file_name)) + if not os.path.exists(cached_file): + print(f'Downloading: "{url}" to {cached_file}\n') + from torch.hub import download_url_to_file + download_url_to_file(url, cached_file, progress=progress, hash_prefix=hash_prefix) + return cached_file + +class VideoToCanny: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "input_video": ("IMAGE",), + "low_threshold": ("INT", {"default": 100, "min": 0, "max": 255, "step": 1}), + "high_threshold": ("INT", {"default": 200, "min": 0, "max": 255, "step": 1}), + "video_length": ( + "INT", {"default": 81, "min": 1, "max": 81, "step": 4} + ), + } + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, input_video, low_threshold, high_threshold, video_length): + def extract_canny_frames(frames): + canny_frames = [] + for frame in frames: + gray = cv2.cvtColor(frame, cv2.COLOR_RGB2GRAY) + edges = cv2.Canny(gray, low_threshold, high_threshold) + edges_colored = cv2.cvtColor(edges, cv2.COLOR_GRAY2RGB) + canny_frames.append(edges_colored) + return canny_frames + + if type(input_video) is str: + video_frames = read_video(input_video) + else: + video_frames = np.array(input_video * 255, np.uint8)[:video_length] + output_video = extract_canny_frames(video_frames) + output_video = torch.from_numpy(np.array(output_video)) / 255 + return (output_video,) + +class VideoToDepth: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "input_video": ("IMAGE",), + "video_length": ( + "INT", {"default": 81, "min": 1, "max": 81, "step": 4} + ), + } + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + + def process_frame(self, model, image, device, weight_dtype): + with torch.no_grad(): + image, remove_pad = resize_image_with_pad(image, 512) + image_depth = image + with torch.no_grad(): + image_depth = torch.from_numpy(image_depth).to(device, weight_dtype) + image_depth = image_depth / 255.0 + image_depth = rearrange(image_depth, 'h w c -> 1 c h w') + depth = model.infer(image_depth) + + depth = depth[0, 0].cpu().numpy() + + vmin = np.percentile(depth, 2) + vmax = np.percentile(depth, 85) + + depth -= vmin + depth /= vmax - vmin + depth = 1.0 - depth + depth_image = (depth * 255.0).clip(0, 255).astype(np.uint8) + image = remove_pad(depth_image) + image = HWC3(image) + return image + + def process(self, input_video, video_length): + model = ZoeDepth.build_from_config(get_config("zoedepth", "infer")) + + # Detect model is existing or not + possible_folders = ["CogVideoX_Fun/Third_Party", "Fun_Models/Third_Party", "VideoX_Fun/Third_Party"] # Possible folder names to check + + # Check if the model exists in any of the possible folders within folder_paths.models_dir + zoe_model_path = "ZoeD_M12_N.pt" + for folder in possible_folders: + candidate_path = os.path.join(folder_paths.models_dir, folder, zoe_model_path) + if os.path.exists(candidate_path): + zoe_model_path = candidate_path + break + if not os.path.exists(zoe_model_path): + load_file_from_url(remote_zoe, model_dir=os.path.join(folder_paths.models_dir, "Fun_Models/Third_Party")) + zoe_model_path = os.path.join(folder_paths.models_dir, "Fun_Models/Third_Party", zoe_model_path) + + model.load_state_dict( + torch.load(zoe_model_path, map_location="cpu")['model'], + strict=False + ) + if torch.cuda.is_available(): + device = "cuda" + weight_dtype = torch.float32 + else: + device = "cpu" + weight_dtype = torch.float32 + model = model.to(device=device, dtype=weight_dtype).eval().requires_grad_(False) + + if isinstance(input_video, str): + video_frames = read_video(input_video) + else: + video_frames = np.array(input_video * 255, np.uint8)[:video_length] + + output_video = [self.process_frame(model, frame, device, weight_dtype) for frame in video_frames] + output_video = torch.from_numpy(np.array(output_video)) / 255 + + return (output_video,) + + +class VideoToPose: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "input_video": ("IMAGE",), + "video_length": ( + "INT", {"default": 81, "min": 1, "max": 81, "step": 4} + ), + } + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES = ("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process_frame(self, model, image): + with torch.no_grad(): + image, remove_pad = resize_image_with_pad(image, 512) + pose_image = model(image) + image = remove_pad(pose_image) + image = HWC3(image) + return image + + def process(self, input_video, video_length): + # Detect model is existing or not + possible_folders = ["CogVideoX_Fun/Third_Party", "Fun_Models/Third_Party", "VideoX_Fun/Third_Party"] # Possible folder names to check + + # Check if the model exists in any of the possible folders within folder_paths.models_dir + onnx_det = "yolox_l.onnx" + for folder in possible_folders: + candidate_path = os.path.join(folder_paths.models_dir, folder, onnx_det) + if os.path.exists(candidate_path): + onnx_det = candidate_path + break + if not os.path.exists(onnx_det): + load_file_from_url(remote_onnx_det, os.path.join(folder_paths.models_dir, "Fun_Models/Third_Party")) + onnx_det = os.path.join(folder_paths.models_dir, "Fun_Models/Third_Party", onnx_det) + + onnx_pose = "dw-ll_ucoco_384.onnx" + for folder in possible_folders: + candidate_path = os.path.join(folder_paths.models_dir, folder, onnx_pose) + if os.path.exists(candidate_path): + onnx_pose = candidate_path + break + if not os.path.exists(onnx_pose): + load_file_from_url(remote_onnx_pose, os.path.join(folder_paths.models_dir, "Fun_Models/Third_Party")) + onnx_pose = os.path.join(folder_paths.models_dir, "Fun_Models/Third_Party", onnx_pose) + + model = DWposeDetector(onnx_det, onnx_pose) + + if isinstance(input_video, str): + video_frames = read_video(input_video) + else: + video_frames = np.array(input_video * 255, np.uint8)[:video_length] + + output_video = [self.process_frame(model, frame) for frame in video_frames] + output_video = torch.from_numpy(np.array(output_video)) / 255 + return (output_video,) \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/LICENSE b/VideoX-Fun/comfyui/annotator/zoe/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..7a1e90d007836c327846ce8e5151013b115042ab --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2022 Intelligent Systems Lab Org + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas.py new file mode 100644 index 0000000000000000000000000000000000000000..380c34974ad2528be9ed7d8ad0f5e9e264ecac8d --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas.py @@ -0,0 +1,379 @@ +# MIT License +import os + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn +import numpy as np +from torchvision.transforms import Normalize + + +def denormalize(x): + """Reverses the imagenet normalization applied to the input. + + Args: + x (torch.Tensor - shape(N,3,H,W)): input tensor + + Returns: + torch.Tensor - shape(N,3,H,W): Denormalized input + """ + mean = torch.Tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1).to(x.device) + std = torch.Tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1).to(x.device) + return x * std + mean + +def get_activation(name, bank): + def hook(model, input, output): + bank[name] = output + return hook + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + ): + """Init. + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + print("Params passed to Resize transform:") + print("\twidth: ", width) + print("\theight: ", height) + print("\tresize_target: ", resize_target) + print("\tkeep_aspect_ratio: ", keep_aspect_ratio) + print("\tensure_multiple_of: ", ensure_multiple_of) + print("\tresize_method: ", resize_method) + + self.__width = width + self.__height = height + + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) + * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) + * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented" + ) + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, min_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, min_val=self.__width + ) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, max_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, max_val=self.__width + ) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def __call__(self, x): + width, height = self.get_size(*x.shape[-2:][::-1]) + return nn.functional.interpolate(x, (int(height), int(width)), mode='bilinear', align_corners=True) + +class PrepForMidas(object): + def __init__(self, resize_mode="minimal", keep_aspect_ratio=True, img_size=384, do_resize=True): + if isinstance(img_size, int): + img_size = (img_size, img_size) + net_h, net_w = img_size + self.normalization = Normalize( + mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + self.resizer = Resize(net_w, net_h, keep_aspect_ratio=keep_aspect_ratio, ensure_multiple_of=32, resize_method=resize_mode) \ + if do_resize else nn.Identity() + + def __call__(self, x): + return self.normalization(self.resizer(x)) + + +class MidasCore(nn.Module): + def __init__(self, midas, trainable=False, fetch_features=True, layer_names=('out_conv', 'l4_rn', 'r4', 'r3', 'r2', 'r1'), freeze_bn=False, keep_aspect_ratio=True, + img_size=384, **kwargs): + """Midas Base model used for multi-scale feature extraction. + + Args: + midas (torch.nn.Module): Midas model. + trainable (bool, optional): Train midas model. Defaults to False. + fetch_features (bool, optional): Extract multi-scale features. Defaults to True. + layer_names (tuple, optional): Layers used for feature extraction. Order = (head output features, last layer features, ...decoder features). Defaults to ('out_conv', 'l4_rn', 'r4', 'r3', 'r2', 'r1'). + freeze_bn (bool, optional): Freeze BatchNorm. Generally results in better finetuning performance. Defaults to False. + keep_aspect_ratio (bool, optional): Keep the aspect ratio of input images while resizing. Defaults to True. + img_size (int, tuple, optional): Input resolution. Defaults to 384. + """ + super().__init__() + self.core = midas + self.output_channels = None + self.core_out = {} + self.trainable = trainable + self.fetch_features = fetch_features + # midas.scratch.output_conv = nn.Identity() + self.handles = [] + # self.layer_names = ['out_conv','l4_rn', 'r4', 'r3', 'r2', 'r1'] + self.layer_names = layer_names + + self.set_trainable(trainable) + self.set_fetch_features(fetch_features) + + self.prep = PrepForMidas(keep_aspect_ratio=keep_aspect_ratio, + img_size=img_size, do_resize=kwargs.get('do_resize', True)) + + if freeze_bn: + self.freeze_bn() + + def set_trainable(self, trainable): + self.trainable = trainable + if trainable: + self.unfreeze() + else: + self.freeze() + return self + + def set_fetch_features(self, fetch_features): + self.fetch_features = fetch_features + if fetch_features: + if len(self.handles) == 0: + self.attach_hooks(self.core) + else: + self.remove_hooks() + return self + + def freeze(self): + for p in self.parameters(): + p.requires_grad = False + self.trainable = False + return self + + def unfreeze(self): + for p in self.parameters(): + p.requires_grad = True + self.trainable = True + return self + + def freeze_bn(self): + for m in self.modules(): + if isinstance(m, nn.BatchNorm2d): + m.eval() + return self + + def forward(self, x, denorm=False, return_rel_depth=False): + with torch.no_grad(): + if denorm: + x = denormalize(x) + x = self.prep(x) + # print("Shape after prep: ", x.shape) + + with torch.set_grad_enabled(self.trainable): + + # print("Input size to Midascore", x.shape) + rel_depth = self.core(x) + # print("Output from midas shape", rel_depth.shape) + if not self.fetch_features: + return rel_depth + out = [self.core_out[k] for k in self.layer_names] + + if return_rel_depth: + return rel_depth, out + return out + + def get_rel_pos_params(self): + for name, p in self.core.pretrained.named_parameters(): + if "relative_position" in name: + yield p + + def get_enc_params_except_rel_pos(self): + for name, p in self.core.pretrained.named_parameters(): + if "relative_position" not in name: + yield p + + def freeze_encoder(self, freeze_rel_pos=False): + if freeze_rel_pos: + for p in self.core.pretrained.parameters(): + p.requires_grad = False + else: + for p in self.get_enc_params_except_rel_pos(): + p.requires_grad = False + return self + + def attach_hooks(self, midas): + if len(self.handles) > 0: + self.remove_hooks() + if "out_conv" in self.layer_names: + self.handles.append(list(midas.scratch.output_conv.children())[ + 3].register_forward_hook(get_activation("out_conv", self.core_out))) + if "r4" in self.layer_names: + self.handles.append(midas.scratch.refinenet4.register_forward_hook( + get_activation("r4", self.core_out))) + if "r3" in self.layer_names: + self.handles.append(midas.scratch.refinenet3.register_forward_hook( + get_activation("r3", self.core_out))) + if "r2" in self.layer_names: + self.handles.append(midas.scratch.refinenet2.register_forward_hook( + get_activation("r2", self.core_out))) + if "r1" in self.layer_names: + self.handles.append(midas.scratch.refinenet1.register_forward_hook( + get_activation("r1", self.core_out))) + if "l4_rn" in self.layer_names: + self.handles.append(midas.scratch.layer4_rn.register_forward_hook( + get_activation("l4_rn", self.core_out))) + + return self + + def remove_hooks(self): + for h in self.handles: + h.remove() + return self + + def __del__(self): + self.remove_hooks() + + def set_output_channels(self, model_type): + self.output_channels = MIDAS_SETTINGS[model_type] + + @staticmethod + def build(midas_model_type="DPT_BEiT_L_384", train_midas=False, use_pretrained_midas=True, fetch_features=False, freeze_bn=True, force_keep_ar=False, force_reload=False, **kwargs): + if midas_model_type not in MIDAS_SETTINGS: + raise ValueError( + f"Invalid model type: {midas_model_type}. Must be one of {list(MIDAS_SETTINGS.keys())}") + if "img_size" in kwargs: + kwargs = MidasCore.parse_img_size(kwargs) + img_size = kwargs.pop("img_size", [384, 384]) + print("img_size", img_size) + midas_path = os.path.join(os.path.dirname(__file__), 'midas_repo') + midas = torch.hub.load(midas_path, midas_model_type, + pretrained=use_pretrained_midas, force_reload=force_reload, source='local') + kwargs.update({'keep_aspect_ratio': force_keep_ar}) + midas_core = MidasCore(midas, trainable=train_midas, fetch_features=fetch_features, + freeze_bn=freeze_bn, img_size=img_size, **kwargs) + midas_core.set_output_channels(midas_model_type) + return midas_core + + @staticmethod + def build_from_config(config): + return MidasCore.build(**config) + + @staticmethod + def parse_img_size(config): + assert 'img_size' in config + if isinstance(config['img_size'], str): + assert "," in config['img_size'], "img_size should be a string with comma separated img_size=H,W" + config['img_size'] = list(map(int, config['img_size'].split(","))) + assert len( + config['img_size']) == 2, "img_size should be a string with comma separated img_size=H,W" + elif isinstance(config['img_size'], int): + config['img_size'] = [config['img_size'], config['img_size']] + else: + assert isinstance(config['img_size'], list) and len( + config['img_size']) == 2, "img_size should be a list of H,W" + return config + + +nchannels2models = { + tuple([256]*5): ["DPT_BEiT_L_384", "DPT_BEiT_L_512", "DPT_BEiT_B_384", "DPT_SwinV2_L_384", "DPT_SwinV2_B_384", "DPT_SwinV2_T_256", "DPT_Large", "DPT_Hybrid"], + (512, 256, 128, 64, 64): ["MiDaS_small"] +} + +# Model name to number of output channels +MIDAS_SETTINGS = {m: k for k, v in nchannels2models.items() + for m in v + } diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/.gitignore b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..9b4486124f6db846f8746400e0826d8b37ce4a7f --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/.gitignore @@ -0,0 +1,110 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +*.png +*.pfm +*.jpg +*.jpeg +*.pt \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/Dockerfile b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..93164297a40e68cf479afc3b8e064468903f8bf1 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/Dockerfile @@ -0,0 +1,29 @@ +# enables cuda support in docker +FROM nvidia/cuda:10.2-cudnn7-runtime-ubuntu18.04 + +# install python 3.6, pip and requirements for opencv-python +# (see https://github.com/NVIDIA/nvidia-docker/issues/864) +RUN apt-get update && apt-get -y install \ + python3 \ + python3-pip \ + libsm6 \ + libxext6 \ + libxrender-dev \ + curl \ + && rm -rf /var/lib/apt/lists/* + +# install python dependencies +RUN pip3 install --upgrade pip +RUN pip3 install torch~=1.8 torchvision opencv-python-headless~=3.4 timm + +# copy inference code +WORKDIR /opt/MiDaS +COPY ./midas ./midas +COPY ./*.py ./ + +# download model weights so the docker image can be used offline +RUN cd weights && {curl -OL https://github.com/isl-org/MiDaS/releases/download/v3/dpt_hybrid_384.pt; cd -; } +RUN python3 run.py --model_type dpt_hybrid; exit 0 + +# entrypoint (dont forget to mount input and output directories) +CMD python3 run.py --model_type dpt_hybrid diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/LICENSE b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..0365733785a449c285c6ac704ef443f385fe798c --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2019 Intel ISL (Intel Intelligent Systems Lab) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/README.md b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..44d485c553211625a4f9303c9b6fb186d4aa56bb --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/README.md @@ -0,0 +1,259 @@ +## Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer + +This repository contains code to compute depth from a single image. It accompanies our [paper](https://arxiv.org/abs/1907.01341v3): + +>Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer +René Ranftl, Katrin Lasinger, David Hafner, Konrad Schindler, Vladlen Koltun + + +and our [preprint](https://arxiv.org/abs/2103.13413): + +> Vision Transformers for Dense Prediction +> René Ranftl, Alexey Bochkovskiy, Vladlen Koltun + + +MiDaS was trained on up to 12 datasets (ReDWeb, DIML, Movies, MegaDepth, WSVD, TartanAir, HRWSI, ApolloScape, BlendedMVS, IRS, KITTI, NYU Depth V2) with +multi-objective optimization. +The original model that was trained on 5 datasets (`MIX 5` in the paper) can be found [here](https://github.com/isl-org/MiDaS/releases/tag/v2). +The figure below shows an overview of the different MiDaS models; the bubble size scales with number of parameters. + +![](figures/Improvement_vs_FPS.png) + +### Setup + +1) Pick one or more models and download the corresponding weights to the `weights` folder: + +MiDaS 3.1 +- For highest quality: [dpt_beit_large_512](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt) +- For moderately less quality, but better speed-performance trade-off: [dpt_swin2_large_384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_large_384.pt) +- For embedded devices: [dpt_swin2_tiny_256](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_tiny_256.pt), [dpt_levit_224](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_levit_224.pt) +- For inference on Intel CPUs, OpenVINO may be used for the small legacy model: openvino_midas_v21_small [.xml](https://github.com/isl-org/MiDaS/releases/download/v3_1/openvino_midas_v21_small_256.xml), [.bin](https://github.com/isl-org/MiDaS/releases/download/v3_1/openvino_midas_v21_small_256.bin) + +MiDaS 3.0: Legacy transformer models [dpt_large_384](https://github.com/isl-org/MiDaS/releases/download/v3/dpt_large_384.pt) and [dpt_hybrid_384](https://github.com/isl-org/MiDaS/releases/download/v3/dpt_hybrid_384.pt) + +MiDaS 2.1: Legacy convolutional models [midas_v21_384](https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_384.pt) and [midas_v21_small_256](https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_small_256.pt) + +1) Set up dependencies: + + ```shell + conda env create -f environment.yaml + conda activate midas-py310 + ``` + +#### optional + +For the Next-ViT model, execute + +```shell +git submodule add https://github.com/isl-org/Next-ViT midas/external/next_vit +``` + +For the OpenVINO model, install + +```shell +pip install openvino +``` + +### Usage + +1) Place one or more input images in the folder `input`. + +2) Run the model with + + ```shell + python run.py --model_type --input_path input --output_path output + ``` + where `````` is chosen from [dpt_beit_large_512](#model_type), [dpt_beit_large_384](#model_type), + [dpt_beit_base_384](#model_type), [dpt_swin2_large_384](#model_type), [dpt_swin2_base_384](#model_type), + [dpt_swin2_tiny_256](#model_type), [dpt_swin_large_384](#model_type), [dpt_next_vit_large_384](#model_type), + [dpt_levit_224](#model_type), [dpt_large_384](#model_type), [dpt_hybrid_384](#model_type), + [midas_v21_384](#model_type), [midas_v21_small_256](#model_type), [openvino_midas_v21_small_256](#model_type). + +3) The resulting depth maps are written to the `output` folder. + +#### optional + +1) By default, the inference resizes the height of input images to the size of a model to fit into the encoder. This + size is given by the numbers in the model names of the [accuracy table](#accuracy). Some models do not only support a single + inference height but a range of different heights. Feel free to explore different heights by appending the extra + command line argument `--height`. Unsupported height values will throw an error. Note that using this argument may + decrease the model accuracy. +2) By default, the inference keeps the aspect ratio of input images when feeding them into the encoder if this is + supported by a model (all models except for Swin, Swin2, LeViT). In order to resize to a square resolution, + disregarding the aspect ratio while preserving the height, use the command line argument `--square`. + +#### via Camera + + If you want the input images to be grabbed from the camera and shown in a window, leave the input and output paths + away and choose a model type as shown above: + + ```shell + python run.py --model_type --side + ``` + + The argument `--side` is optional and causes both the input RGB image and the output depth map to be shown + side-by-side for comparison. + +#### via Docker + +1) Make sure you have installed Docker and the + [NVIDIA Docker runtime](https://github.com/NVIDIA/nvidia-docker/wiki/Installation-\(Native-GPU-Support\)). + +2) Build the Docker image: + + ```shell + docker build -t midas . + ``` + +3) Run inference: + + ```shell + docker run --rm --gpus all -v $PWD/input:/opt/MiDaS/input -v $PWD/output:/opt/MiDaS/output -v $PWD/weights:/opt/MiDaS/weights midas + ``` + + This command passes through all of your NVIDIA GPUs to the container, mounts the + `input` and `output` directories and then runs the inference. + +#### via PyTorch Hub + +The pretrained model is also available on [PyTorch Hub](https://pytorch.org/hub/intelisl_midas_v2/) + +#### via TensorFlow or ONNX + +See [README](https://github.com/isl-org/MiDaS/tree/master/tf) in the `tf` subdirectory. + +Currently only supports MiDaS v2.1. + + +#### via Mobile (iOS / Android) + +See [README](https://github.com/isl-org/MiDaS/tree/master/mobile) in the `mobile` subdirectory. + +#### via ROS1 (Robot Operating System) + +See [README](https://github.com/isl-org/MiDaS/tree/master/ros) in the `ros` subdirectory. + +Currently only supports MiDaS v2.1. DPT-based models to be added. + + +### Accuracy + +We provide a **zero-shot error** $\epsilon_d$ which is evaluated for 6 different datasets +(see [paper](https://arxiv.org/abs/1907.01341v3)). **Lower error values are better**. +$\color{green}{\textsf{Overall model quality is represented by the improvement}}$ ([Imp.](#improvement)) with respect to +MiDaS 3.0 DPTL-384. The models are grouped by the height used for inference, whereas the square training resolution is given by +the numbers in the model names. The table also shows the **number of parameters** (in millions) and the +**frames per second** for inference at the training resolution (for GPU RTX 3090): + +| MiDaS Model | DIW
WHDR | Eth3d
AbsRel | Sintel
AbsRel | TUM
δ1 | KITTI
δ1 | NYUv2
δ1 | $\color{green}{\textsf{Imp.}}$
% | Par.
M | FPS
  | +|-----------------------------------------------------------------------------------------------------------------------|-------------------------:|-----------------------------:|------------------------------:|-------------------------:|-------------------------:|-------------------------:|-------------------------------------------------:|----------------------:|--------------------------:| +| **Inference height 512** | | | | | | | | | | +| [v3.1 BEiTL-512](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt) | 0.1137 | 0.0659 | 0.2366 | **6.13** | 11.56* | **1.86*** | $\color{green}{\textsf{19}}$ | **345** | **5.7** | +| [v3.1 BEiTL-512](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt)$\tiny{\square}$ | **0.1121** | **0.0614** | **0.2090** | 6.46 | **5.00*** | 1.90* | $\color{green}{\textsf{34}}$ | **345** | **5.7** | +| | | | | | | | | | | +| **Inference height 384** | | | | | | | | | | +| [v3.1 BEiTL-512](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt) | 0.1245 | 0.0681 | **0.2176** | **6.13** | 6.28* | **2.16*** | $\color{green}{\textsf{28}}$ | 345 | 12 | +| [v3.1 Swin2L-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_large_384.pt)$\tiny{\square}$ | 0.1106 | 0.0732 | 0.2442 | 8.87 | **5.84*** | 2.92* | $\color{green}{\textsf{22}}$ | 213 | 41 | +| [v3.1 Swin2B-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_base_384.pt)$\tiny{\square}$ | 0.1095 | 0.0790 | 0.2404 | 8.93 | 5.97* | 3.28* | $\color{green}{\textsf{22}}$ | 102 | 39 | +| [v3.1 SwinL-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin_large_384.pt)$\tiny{\square}$ | 0.1126 | 0.0853 | 0.2428 | 8.74 | 6.60* | 3.34* | $\color{green}{\textsf{17}}$ | 213 | 49 | +| [v3.1 BEiTL-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_384.pt) | 0.1239 | **0.0667** | 0.2545 | 7.17 | 9.84* | 2.21* | $\color{green}{\textsf{17}}$ | 344 | 13 | +| [v3.1 Next-ViTL-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_next_vit_large_384.pt) | **0.1031** | 0.0954 | 0.2295 | 9.21 | 6.89* | 3.47* | $\color{green}{\textsf{16}}$ | **72** | 30 | +| [v3.1 BEiTB-384](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_base_384.pt) | 0.1159 | 0.0967 | 0.2901 | 9.88 | 26.60* | 3.91* | $\color{green}{\textsf{-31}}$ | 112 | 31 | +| [v3.0 DPTL-384](https://github.com/isl-org/MiDaS/releases/download/v3/dpt_large_384.pt) | 0.1082 | 0.0888 | 0.2697 | 9.97 | 8.46 | 8.32 | $\color{green}{\textsf{0}}$ | 344 | **61** | +| [v3.0 DPTH-384](https://github.com/isl-org/MiDaS/releases/download/v3/dpt_hybrid_384.pt) | 0.1106 | 0.0934 | 0.2741 | 10.89 | 11.56 | 8.69 | $\color{green}{\textsf{-10}}$ | 123 | 50 | +| [v2.1 Large384](https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_384.pt) | 0.1295 | 0.1155 | 0.3285 | 12.51 | 16.08 | 8.71 | $\color{green}{\textsf{-32}}$ | 105 | 47 | +| | | | | | | | | | | +| **Inference height 256** | | | | | | | | | | +| [v3.1 Swin2T-256](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_tiny_256.pt)$\tiny{\square}$ | **0.1211** | **0.1106** | **0.2868** | **13.43** | **10.13*** | **5.55*** | $\color{green}{\textsf{-11}}$ | 42 | 64 | +| [v2.1 Small256](https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_small_256.pt) | 0.1344 | 0.1344 | 0.3370 | 14.53 | 29.27 | 13.43 | $\color{green}{\textsf{-76}}$ | **21** | **90** | +| | | | | | | | | | | +| **Inference height 224** | | | | | | | | | | +| [v3.1 LeViT224](https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_levit_224.pt)$\tiny{\square}$ | **0.1314** | **0.1206** | **0.3148** | **18.21** | **15.27*** | **8.64*** | $\color{green}{\textsf{-40}}$ | **51** | **73** | + +* No zero-shot error, because models are also trained on KITTI and NYU Depth V2\ +$\square$ Validation performed at **square resolution**, either because the transformer encoder backbone of a model +does not support non-square resolutions (Swin, Swin2, LeViT) or for comparison with these models. All other +validations keep the aspect ratio. A difference in resolution limits the comparability of the zero-shot error and the +improvement, because these quantities are averages over the pixels of an image and do not take into account the +advantage of more details due to a higher resolution.\ +Best values per column and same validation height in bold + +#### Improvement + +The improvement in the above table is defined as the relative zero-shot error with respect to MiDaS v3.0 +DPTL-384 and averaging over the datasets. So, if $\epsilon_d$ is the zero-shot error for dataset $d$, then +the $\color{green}{\textsf{improvement}}$ is given by $100(1-(1/6)\sum_d\epsilon_d/\epsilon_{d,\rm{DPT_{L-384}}})$%. + +Note that the improvements of 10% for MiDaS v2.0 → v2.1 and 21% for MiDaS v2.1 → v3.0 are not visible from the +improvement column (Imp.) in the table but would require an evaluation with respect to MiDaS v2.1 Large384 +and v2.0 Large384 respectively instead of v3.0 DPTL-384. + +### Depth map comparison + +Zoom in for better visibility +![](figures/Comparison.png) + +### Speed on Camera Feed + +Test configuration +- Windows 10 +- 11th Gen Intel Core i7-1185G7 3.00GHz +- 16GB RAM +- Camera resolution 640x480 +- openvino_midas_v21_small_256 + +Speed: 22 FPS + +### Changelog + +* [Dec 2022] Released MiDaS v3.1: + - New models based on 5 different types of transformers ([BEiT](https://arxiv.org/pdf/2106.08254.pdf), [Swin2](https://arxiv.org/pdf/2111.09883.pdf), [Swin](https://arxiv.org/pdf/2103.14030.pdf), [Next-ViT](https://arxiv.org/pdf/2207.05501.pdf), [LeViT](https://arxiv.org/pdf/2104.01136.pdf)) + - Training datasets extended from 10 to 12, including also KITTI and NYU Depth V2 using [BTS](https://github.com/cleinc/bts) split + - Best model, BEiTLarge 512, with resolution 512x512, is on average about [28% more accurate](#Accuracy) than MiDaS v3.0 + - Integrated live depth estimation from camera feed +* [Sep 2021] Integrated to [Huggingface Spaces](https://huggingface.co/spaces) with [Gradio](https://github.com/gradio-app/gradio). See [Gradio Web Demo](https://huggingface.co/spaces/akhaliq/DPT-Large). +* [Apr 2021] Released MiDaS v3.0: + - New models based on [Dense Prediction Transformers](https://arxiv.org/abs/2103.13413) are on average [21% more accurate](#Accuracy) than MiDaS v2.1 + - Additional models can be found [here](https://github.com/isl-org/DPT) +* [Nov 2020] Released MiDaS v2.1: + - New model that was trained on 10 datasets and is on average about [10% more accurate](#Accuracy) than [MiDaS v2.0](https://github.com/isl-org/MiDaS/releases/tag/v2) + - New light-weight model that achieves [real-time performance](https://github.com/isl-org/MiDaS/tree/master/mobile) on mobile platforms. + - Sample applications for [iOS](https://github.com/isl-org/MiDaS/tree/master/mobile/ios) and [Android](https://github.com/isl-org/MiDaS/tree/master/mobile/android) + - [ROS package](https://github.com/isl-org/MiDaS/tree/master/ros) for easy deployment on robots +* [Jul 2020] Added TensorFlow and ONNX code. Added [online demo](http://35.202.76.57/). +* [Dec 2019] Released new version of MiDaS - the new model is significantly more accurate and robust +* [Jul 2019] Initial release of MiDaS ([Link](https://github.com/isl-org/MiDaS/releases/tag/v1)) + +### Citation + +Please cite our paper if you use this code or any of the models: +``` +@ARTICLE {Ranftl2022, + author = "Ren\'{e} Ranftl and Katrin Lasinger and David Hafner and Konrad Schindler and Vladlen Koltun", + title = "Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-Shot Cross-Dataset Transfer", + journal = "IEEE Transactions on Pattern Analysis and Machine Intelligence", + year = "2022", + volume = "44", + number = "3" +} +``` + +If you use a DPT-based model, please also cite: + +``` +@article{Ranftl2021, + author = {Ren\'{e} Ranftl and Alexey Bochkovskiy and Vladlen Koltun}, + title = {Vision Transformers for Dense Prediction}, + journal = {ICCV}, + year = {2021}, +} +``` + +### Acknowledgements + +Our work builds on and uses code from [timm](https://github.com/rwightman/pytorch-image-models) and [Next-ViT](https://github.com/bytedance/Next-ViT). +We'd like to thank the authors for making these libraries available. + +### License + +MIT License diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/environment.yaml b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..e20cfd31c52ecbdaba67382878fe281faf456beb --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/environment.yaml @@ -0,0 +1,16 @@ +name: midas-py310 +channels: + - pytorch + - defaults +dependencies: + - nvidia::cudatoolkit=11.7 + - python=3.10.8 + - pytorch::pytorch=1.13.0 + - torchvision=0.14.0 + - pip=22.3.1 + - numpy=1.23.4 + - pip: + - opencv-python==4.6.0.66 + - imutils==0.5.4 + - timm==0.6.12 + - einops==0.6.0 \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/hubconf.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/hubconf.py new file mode 100644 index 0000000000000000000000000000000000000000..37c4cbd644c5a0533a7203f2115875be8e596ec1 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/hubconf.py @@ -0,0 +1,435 @@ +dependencies = ["torch"] + +import torch + +from midas.dpt_depth import DPTDepthModel +from midas.midas_net import MidasNet +from midas.midas_net_custom import MidasNet_small + +def DPT_BEiT_L_512(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_BEiT_L_512 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="beitl16_512", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_512.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_BEiT_L_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_BEiT_L_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="beitl16_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_large_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_BEiT_B_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_BEiT_B_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="beitb16_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_beit_base_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_SwinV2_L_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_SwinV2_L_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="swin2l24_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_large_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_SwinV2_B_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_SwinV2_B_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="swin2b24_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_base_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_SwinV2_T_256(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_SwinV2_T_256 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="swin2t16_256", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin2_tiny_256.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_Swin_L_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_Swin_L_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="swinl12_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_swin_large_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_Next_ViT_L_384(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_Next_ViT_L_384 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="next_vit_large_6m", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_next_vit_large_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_LeViT_224(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT_LeViT_224 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="levit_384", + non_negative=True, + head_features_1=64, + head_features_2=8, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3_1/dpt_levit_224.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_Large(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT-Large model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="vitl16_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3/dpt_large_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def DPT_Hybrid(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS DPT-Hybrid model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = DPTDepthModel( + path=None, + backbone="vitb_rn50_384", + non_negative=True, + ) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v3/dpt_hybrid_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def MiDaS(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS v2.1 model for monocular depth estimation + pretrained (bool): load pretrained weights into model + """ + + model = MidasNet() + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_384.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + +def MiDaS_small(pretrained=True, **kwargs): + """ # This docstring shows up in hub.help() + MiDaS v2.1 small model for monocular depth estimation on resource-constrained devices + pretrained (bool): load pretrained weights into model + """ + + model = MidasNet_small(None, features=64, backbone="efficientnet_lite3", exportable=True, non_negative=True, blocks={'expand': True}) + + if pretrained: + checkpoint = ( + "https://github.com/isl-org/MiDaS/releases/download/v2_1/midas_v21_small_256.pt" + ) + state_dict = torch.hub.load_state_dict_from_url( + checkpoint, map_location=torch.device('cpu'), progress=True, check_hash=True + ) + model.load_state_dict(state_dict) + + return model + + +def transforms(): + import cv2 + from torchvision.transforms import Compose + from midas.transforms import Resize, NormalizeImage, PrepareForNet + from midas import transforms + + transforms.default_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 384, + 384, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method="upper_bound", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.small_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 256, + 256, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method="upper_bound", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.dpt_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 384, + 384, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.beit512_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 512, + 512, + resize_target=None, + keep_aspect_ratio=True, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.swin384_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 384, + 384, + resize_target=None, + keep_aspect_ratio=False, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.swin256_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 256, + 256, + resize_target=None, + keep_aspect_ratio=False, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + transforms.levit_transform = Compose( + [ + lambda img: {"image": img / 255.0}, + Resize( + 224, + 224, + resize_target=None, + keep_aspect_ratio=False, + ensure_multiple_of=32, + resize_method="minimal", + image_interpolation_method=cv2.INTER_CUBIC, + ), + NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + PrepareForNet(), + lambda sample: torch.from_numpy(sample["image"]).unsqueeze(0), + ] + ) + + return transforms diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/input/.placeholder b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/input/.placeholder new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/beit.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/beit.py new file mode 100644 index 0000000000000000000000000000000000000000..aadc019b6a8b0a1f84614c7da6f2267884eb60dd --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/beit.py @@ -0,0 +1,198 @@ +import timm +import torch +import types + +import numpy as np +import torch.nn.functional as F + +from .utils import forward_adapted_unflatten, make_backbone_default +from timm.models.beit import gen_relative_position_index +from torch.utils.checkpoint import checkpoint +from typing import Optional + + +def forward_beit(pretrained, x): + return forward_adapted_unflatten(pretrained, x, "forward_features") + + +def patch_embed_forward(self, x): + """ + Modification of timm.models.layers.patch_embed.py: PatchEmbed.forward to support arbitrary window sizes. + """ + x = self.proj(x) + if self.flatten: + x = x.flatten(2).transpose(1, 2) + x = self.norm(x) + return x + + +def _get_rel_pos_bias(self, window_size): + """ + Modification of timm.models.beit.py: Attention._get_rel_pos_bias to support arbitrary window sizes. + """ + old_height = 2 * self.window_size[0] - 1 + old_width = 2 * self.window_size[1] - 1 + + new_height = 2 * window_size[0] - 1 + new_width = 2 * window_size[1] - 1 + + old_relative_position_bias_table = self.relative_position_bias_table + + old_num_relative_distance = self.num_relative_distance + new_num_relative_distance = new_height * new_width + 3 + + old_sub_table = old_relative_position_bias_table[:old_num_relative_distance - 3] + + old_sub_table = old_sub_table.reshape(1, old_width, old_height, -1).permute(0, 3, 1, 2) + new_sub_table = F.interpolate(old_sub_table, size=(int(new_height), int(new_width)), mode="bilinear") + new_sub_table = new_sub_table.permute(0, 2, 3, 1).reshape(new_num_relative_distance - 3, -1) + + new_relative_position_bias_table = torch.cat( + [new_sub_table, old_relative_position_bias_table[old_num_relative_distance - 3:]]) + + key = str(window_size[1]) + "," + str(window_size[0]) + if key not in self.relative_position_indices.keys(): + self.relative_position_indices[key] = gen_relative_position_index(window_size) + + relative_position_bias = new_relative_position_bias_table[ + self.relative_position_indices[key].view(-1)].view( + window_size[0] * window_size[1] + 1, + window_size[0] * window_size[1] + 1, -1) # Wh*Ww,Wh*Ww,nH + relative_position_bias = relative_position_bias.permute(2, 0, 1).contiguous() # nH, Wh*Ww, Wh*Ww + return relative_position_bias.unsqueeze(0) + + +def attention_forward(self, x, resolution, shared_rel_pos_bias: Optional[torch.Tensor] = None): + """ + Modification of timm.models.beit.py: Attention.forward to support arbitrary window sizes. + """ + B, N, C = x.shape + + qkv_bias = torch.cat((self.q_bias, self.k_bias, self.v_bias)) if self.q_bias is not None else None + qkv = F.linear(input=x, weight=self.qkv.weight, bias=qkv_bias) + qkv = qkv.reshape(B, N, 3, self.num_heads, -1).permute(2, 0, 3, 1, 4) + q, k, v = qkv.unbind(0) # make torchscript happy (cannot use tensor as tuple) + + q = q * self.scale + attn = (q @ k.transpose(-2, -1)) + + if self.relative_position_bias_table is not None: + window_size = tuple(np.array(resolution) // 16) + attn = attn + self._get_rel_pos_bias(window_size) + if shared_rel_pos_bias is not None: + attn = attn + shared_rel_pos_bias + + attn = attn.softmax(dim=-1) + attn = self.attn_drop(attn) + + x = (attn @ v).transpose(1, 2).reshape(B, N, -1) + x = self.proj(x) + x = self.proj_drop(x) + return x + + +def block_forward(self, x, resolution, shared_rel_pos_bias: Optional[torch.Tensor] = None): + """ + Modification of timm.models.beit.py: Block.forward to support arbitrary window sizes. + """ + if hasattr(self, 'drop_path1') and not hasattr(self, 'drop_path'): + self.drop_path = self.drop_path1 + if self.gamma_1 is None: + x = x + self.drop_path(self.attn(self.norm1(x), resolution, shared_rel_pos_bias=shared_rel_pos_bias)) + x = x + self.drop_path(self.mlp(self.norm2(x))) + else: + x = x + self.drop_path(self.gamma_1 * self.attn(self.norm1(x), resolution, + shared_rel_pos_bias=shared_rel_pos_bias)) + x = x + self.drop_path(self.gamma_2 * self.mlp(self.norm2(x))) + return x + + +def beit_forward_features(self, x): + """ + Modification of timm.models.beit.py: Beit.forward_features to support arbitrary window sizes. + """ + resolution = x.shape[2:] + + x = self.patch_embed(x) + x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1) + if self.pos_embed is not None: + x = x + self.pos_embed + x = self.pos_drop(x) + + rel_pos_bias = self.rel_pos_bias() if self.rel_pos_bias is not None else None + for blk in self.blocks: + if self.grad_checkpointing and not torch.jit.is_scripting(): + x = checkpoint(blk, x, shared_rel_pos_bias=rel_pos_bias) + else: + x = blk(x, resolution, shared_rel_pos_bias=rel_pos_bias) + x = self.norm(x) + return x + + +def _make_beit_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[0, 4, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, + start_index_readout=1, +): + backbone = make_backbone_default(model, features, size, hooks, vit_features, use_readout, start_index, + start_index_readout) + + backbone.model.patch_embed.forward = types.MethodType(patch_embed_forward, backbone.model.patch_embed) + backbone.model.forward_features = types.MethodType(beit_forward_features, backbone.model) + + for block in backbone.model.blocks: + attn = block.attn + attn._get_rel_pos_bias = types.MethodType(_get_rel_pos_bias, attn) + attn.forward = types.MethodType(attention_forward, attn) + attn.relative_position_indices = {} + + block.forward = types.MethodType(block_forward, block) + + return backbone + + +def _make_pretrained_beitl16_512(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("beit_large_patch16_512", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks is None else hooks + + features = [256, 512, 1024, 1024] + + return _make_beit_backbone( + model, + features=features, + size=[512, 512], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + ) + + +def _make_pretrained_beitl16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("beit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks is None else hooks + return _make_beit_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + ) + + +def _make_pretrained_beitb16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("beit_base_patch16_384", pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks is None else hooks + return _make_beit_backbone( + model, + features=[96, 192, 384, 768], + hooks=hooks, + use_readout=use_readout, + ) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/levit.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/levit.py new file mode 100644 index 0000000000000000000000000000000000000000..e6a7770fb76f085a0f3b6015902797c5805bba01 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/levit.py @@ -0,0 +1,106 @@ +import timm +import torch +import torch.nn as nn +import numpy as np + +from .utils import activations, get_activation, Transpose + + +def forward_levit(pretrained, x): + pretrained.model.forward_features(x) + + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + + layer_1 = pretrained.act_postprocess1(layer_1) + layer_2 = pretrained.act_postprocess2(layer_2) + layer_3 = pretrained.act_postprocess3(layer_3) + + return layer_1, layer_2, layer_3 + + +def _make_levit_backbone( + model, + hooks=[3, 11, 21], + patch_grid=[14, 14] +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + + pretrained.activations = activations + + patch_grid_size = np.array(patch_grid, dtype=int) + + pretrained.act_postprocess1 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size(patch_grid_size.tolist())) + ) + pretrained.act_postprocess2 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size((np.ceil(patch_grid_size / 2).astype(int)).tolist())) + ) + pretrained.act_postprocess3 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size((np.ceil(patch_grid_size / 4).astype(int)).tolist())) + ) + + return pretrained + + +class ConvTransposeNorm(nn.Sequential): + """ + Modification of + https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/levit.py: ConvNorm + such that ConvTranspose2d is used instead of Conv2d. + """ + + def __init__( + self, in_chs, out_chs, kernel_size=1, stride=1, pad=0, dilation=1, + groups=1, bn_weight_init=1): + super().__init__() + self.add_module('c', + nn.ConvTranspose2d(in_chs, out_chs, kernel_size, stride, pad, dilation, groups, bias=False)) + self.add_module('bn', nn.BatchNorm2d(out_chs)) + + nn.init.constant_(self.bn.weight, bn_weight_init) + + @torch.no_grad() + def fuse(self): + c, bn = self._modules.values() + w = bn.weight / (bn.running_var + bn.eps) ** 0.5 + w = c.weight * w[:, None, None, None] + b = bn.bias - bn.running_mean * bn.weight / (bn.running_var + bn.eps) ** 0.5 + m = nn.ConvTranspose2d( + w.size(1), w.size(0), w.shape[2:], stride=self.c.stride, + padding=self.c.padding, dilation=self.c.dilation, groups=self.c.groups) + m.weight.data.copy_(w) + m.bias.data.copy_(b) + return m + + +def stem_b4_transpose(in_chs, out_chs, activation): + """ + Modification of + https://github.com/rwightman/pytorch-image-models/blob/master/timm/models/levit.py: stem_b16 + such that ConvTranspose2d is used instead of Conv2d and stem is also reduced to the half. + """ + return nn.Sequential( + ConvTransposeNorm(in_chs, out_chs, 3, 2, 1), + activation(), + ConvTransposeNorm(out_chs, out_chs // 2, 3, 2, 1), + activation()) + + +def _make_pretrained_levit_384(pretrained, hooks=None): + model = timm.create_model("levit_384", pretrained=pretrained) + + hooks = [3, 11, 21] if hooks == None else hooks + return _make_levit_backbone( + model, + hooks=hooks + ) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/next_vit.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/next_vit.py new file mode 100644 index 0000000000000000000000000000000000000000..773acfb1f67537edf56f4f67b686d8f9b174658d --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/next_vit.py @@ -0,0 +1,39 @@ +import timm + +import torch.nn as nn + +from pathlib import Path +from .utils import activations, forward_default, get_activation + +from ..external.next_vit.classification.nextvit import * + + +def forward_next_vit(pretrained, x): + return forward_default(pretrained, x, "forward") + + +def _make_next_vit_backbone( + model, + hooks=[2, 6, 36, 39], +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.features[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.features[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.features[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.features[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + return pretrained + + +def _make_pretrained_next_vit_large_6m(hooks=None): + model = timm.create_model("nextvit_large") + + hooks = [2, 6, 36, 39] if hooks == None else hooks + return _make_next_vit_backbone( + model, + hooks=hooks, + ) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin.py new file mode 100644 index 0000000000000000000000000000000000000000..7477c70cb0109c875ce58ba9b2dbd42e12b2cbe1 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin.py @@ -0,0 +1,13 @@ +import timm + +from .swin_common import _make_swin_backbone + + +def _make_pretrained_swinl12_384(pretrained, hooks=None): + model = timm.create_model("swin_large_patch4_window12_384", pretrained=pretrained) + + hooks = [1, 1, 17, 1] if hooks == None else hooks + return _make_swin_backbone( + model, + hooks=hooks + ) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin2.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin2.py new file mode 100644 index 0000000000000000000000000000000000000000..9b0ab3ccb244fc2d71cb01b96a2d6f0544a041ca --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin2.py @@ -0,0 +1,34 @@ +import timm + +from .swin_common import _make_swin_backbone + + +def _make_pretrained_swin2l24_384(pretrained, hooks=None): + model = timm.create_model("swinv2_large_window12to24_192to384_22kft1k", pretrained=pretrained) + + hooks = [1, 1, 17, 1] if hooks == None else hooks + return _make_swin_backbone( + model, + hooks=hooks + ) + + +def _make_pretrained_swin2b24_384(pretrained, hooks=None): + model = timm.create_model("swinv2_base_window12to24_192to384_22kft1k", pretrained=pretrained) + + hooks = [1, 1, 17, 1] if hooks == None else hooks + return _make_swin_backbone( + model, + hooks=hooks + ) + + +def _make_pretrained_swin2t16_256(pretrained, hooks=None): + model = timm.create_model("swinv2_tiny_window16_256", pretrained=pretrained) + + hooks = [1, 1, 5, 1] if hooks == None else hooks + return _make_swin_backbone( + model, + hooks=hooks, + patch_grid=[64, 64] + ) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin_common.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin_common.py new file mode 100644 index 0000000000000000000000000000000000000000..e61c3633fafcd496505ce209f8152f0aeee0572c --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/swin_common.py @@ -0,0 +1,52 @@ +import torch + +import torch.nn as nn +import numpy as np + +from .utils import activations, forward_default, get_activation, Transpose + + +def forward_swin(pretrained, x): + return forward_default(pretrained, x) + + +def _make_swin_backbone( + model, + hooks=[1, 1, 17, 1], + patch_grid=[96, 96] +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.layers[0].blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.layers[1].blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.layers[2].blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.layers[3].blocks[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + if hasattr(model, "patch_grid"): + used_patch_grid = model.patch_grid + else: + used_patch_grid = patch_grid + + patch_grid_size = np.array(used_patch_grid, dtype=int) + + pretrained.act_postprocess1 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size(patch_grid_size.tolist())) + ) + pretrained.act_postprocess2 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size((patch_grid_size // 2).tolist())) + ) + pretrained.act_postprocess3 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size((patch_grid_size // 4).tolist())) + ) + pretrained.act_postprocess4 = nn.Sequential( + Transpose(1, 2), + nn.Unflatten(2, torch.Size((patch_grid_size // 8).tolist())) + ) + + return pretrained diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/utils.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b334c3c55d5ee1da23d481de8b7555c5e9093754 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/utils.py @@ -0,0 +1,249 @@ +import torch + +import torch.nn as nn + + +class Slice(nn.Module): + def __init__(self, start_index=1): + super(Slice, self).__init__() + self.start_index = start_index + + def forward(self, x): + return x[:, self.start_index:] + + +class AddReadout(nn.Module): + def __init__(self, start_index=1): + super(AddReadout, self).__init__() + self.start_index = start_index + + def forward(self, x): + if self.start_index == 2: + readout = (x[:, 0] + x[:, 1]) / 2 + else: + readout = x[:, 0] + return x[:, self.start_index:] + readout.unsqueeze(1) + + +class ProjectReadout(nn.Module): + def __init__(self, in_features, start_index=1): + super(ProjectReadout, self).__init__() + self.start_index = start_index + + self.project = nn.Sequential(nn.Linear(2 * in_features, in_features), nn.GELU()) + + def forward(self, x): + readout = x[:, 0].unsqueeze(1).expand_as(x[:, self.start_index:]) + features = torch.cat((x[:, self.start_index:], readout), -1) + + return self.project(features) + + +class Transpose(nn.Module): + def __init__(self, dim0, dim1): + super(Transpose, self).__init__() + self.dim0 = dim0 + self.dim1 = dim1 + + def forward(self, x): + x = x.transpose(self.dim0, self.dim1) + return x + + +activations = {} + + +def get_activation(name): + def hook(model, input, output): + activations[name] = output + + return hook + + +def forward_default(pretrained, x, function_name="forward_features"): + exec(f"pretrained.model.{function_name}(x)") + + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + layer_4 = pretrained.activations["4"] + + if hasattr(pretrained, "act_postprocess1"): + layer_1 = pretrained.act_postprocess1(layer_1) + if hasattr(pretrained, "act_postprocess2"): + layer_2 = pretrained.act_postprocess2(layer_2) + if hasattr(pretrained, "act_postprocess3"): + layer_3 = pretrained.act_postprocess3(layer_3) + if hasattr(pretrained, "act_postprocess4"): + layer_4 = pretrained.act_postprocess4(layer_4) + + return layer_1, layer_2, layer_3, layer_4 + + +def forward_adapted_unflatten(pretrained, x, function_name="forward_features"): + b, c, h, w = x.shape + + exec(f"glob = pretrained.model.{function_name}(x)") + + layer_1 = pretrained.activations["1"] + layer_2 = pretrained.activations["2"] + layer_3 = pretrained.activations["3"] + layer_4 = pretrained.activations["4"] + + layer_1 = pretrained.act_postprocess1[0:2](layer_1) + layer_2 = pretrained.act_postprocess2[0:2](layer_2) + layer_3 = pretrained.act_postprocess3[0:2](layer_3) + layer_4 = pretrained.act_postprocess4[0:2](layer_4) + + unflatten = nn.Sequential( + nn.Unflatten( + 2, + torch.Size( + [ + h // pretrained.model.patch_size[1], + w // pretrained.model.patch_size[0], + ] + ), + ) + ) + + if layer_1.ndim == 3: + layer_1 = unflatten(layer_1) + if layer_2.ndim == 3: + layer_2 = unflatten(layer_2) + if layer_3.ndim == 3: + layer_3 = unflatten(layer_3) + if layer_4.ndim == 3: + layer_4 = unflatten(layer_4) + + layer_1 = pretrained.act_postprocess1[3: len(pretrained.act_postprocess1)](layer_1) + layer_2 = pretrained.act_postprocess2[3: len(pretrained.act_postprocess2)](layer_2) + layer_3 = pretrained.act_postprocess3[3: len(pretrained.act_postprocess3)](layer_3) + layer_4 = pretrained.act_postprocess4[3: len(pretrained.act_postprocess4)](layer_4) + + return layer_1, layer_2, layer_3, layer_4 + + +def get_readout_oper(vit_features, features, use_readout, start_index=1): + if use_readout == "ignore": + readout_oper = [Slice(start_index)] * len(features) + elif use_readout == "add": + readout_oper = [AddReadout(start_index)] * len(features) + elif use_readout == "project": + readout_oper = [ + ProjectReadout(vit_features, start_index) for out_feat in features + ] + else: + assert ( + False + ), "wrong operation for readout token, use_readout can be 'ignore', 'add', or 'project'" + + return readout_oper + + +def make_backbone_default( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, + start_index_readout=1, +): + pretrained = nn.Module() + + pretrained.model = model + pretrained.model.blocks[hooks[0]].register_forward_hook(get_activation("1")) + pretrained.model.blocks[hooks[1]].register_forward_hook(get_activation("2")) + pretrained.model.blocks[hooks[2]].register_forward_hook(get_activation("3")) + pretrained.model.blocks[hooks[3]].register_forward_hook(get_activation("4")) + + pretrained.activations = activations + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index_readout) + + # 32, 48, 136, 384 + pretrained.act_postprocess1 = nn.Sequential( + readout_oper[0], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[0], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[0], + out_channels=features[0], + kernel_size=4, + stride=4, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess2 = nn.Sequential( + readout_oper[1], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[1], + kernel_size=1, + stride=1, + padding=0, + ), + nn.ConvTranspose2d( + in_channels=features[1], + out_channels=features[1], + kernel_size=2, + stride=2, + padding=0, + bias=True, + dilation=1, + groups=1, + ), + ) + + pretrained.act_postprocess3 = nn.Sequential( + readout_oper[2], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[2], + kernel_size=1, + stride=1, + padding=0, + ), + ) + + pretrained.act_postprocess4 = nn.Sequential( + readout_oper[3], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[3], + kernel_size=1, + stride=1, + padding=0, + ), + nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ), + ) + + pretrained.model.start_index = start_index + pretrained.model.patch_size = [16, 16] + + return pretrained diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/vit.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/vit.py new file mode 100644 index 0000000000000000000000000000000000000000..27e103c6e383da4fa6ea10e2ec5e19b486be0193 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/backbones/vit.py @@ -0,0 +1,221 @@ +import torch +import torch.nn as nn +import timm +import types +import math +import torch.nn.functional as F + +from .utils import (activations, forward_adapted_unflatten, get_activation, get_readout_oper, + make_backbone_default, Transpose) + + +def forward_vit(pretrained, x): + return forward_adapted_unflatten(pretrained, x, "forward_flex") + + +def _resize_pos_embed(self, posemb, gs_h, gs_w): + posemb_tok, posemb_grid = ( + posemb[:, : self.start_index], + posemb[0, self.start_index:], + ) + + gs_old = int(math.sqrt(len(posemb_grid))) + + posemb_grid = posemb_grid.reshape(1, gs_old, gs_old, -1).permute(0, 3, 1, 2) + posemb_grid = F.interpolate(posemb_grid, size=(gs_h, gs_w), mode="bilinear") + posemb_grid = posemb_grid.permute(0, 2, 3, 1).reshape(1, gs_h * gs_w, -1) + + posemb = torch.cat([posemb_tok, posemb_grid], dim=1) + + return posemb + + +def forward_flex(self, x): + b, c, h, w = x.shape + + pos_embed = self._resize_pos_embed( + self.pos_embed, h // self.patch_size[1], w // self.patch_size[0] + ) + + B = x.shape[0] + + if hasattr(self.patch_embed, "backbone"): + x = self.patch_embed.backbone(x) + if isinstance(x, (list, tuple)): + x = x[-1] # last feature if backbone outputs list/tuple of features + + x = self.patch_embed.proj(x).flatten(2).transpose(1, 2) + + if getattr(self, "dist_token", None) is not None: + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + dist_token = self.dist_token.expand(B, -1, -1) + x = torch.cat((cls_tokens, dist_token, x), dim=1) + else: + if self.no_embed_class: + x = x + pos_embed + cls_tokens = self.cls_token.expand( + B, -1, -1 + ) # stole cls_tokens impl from Phil Wang, thanks + x = torch.cat((cls_tokens, x), dim=1) + + if not self.no_embed_class: + x = x + pos_embed + x = self.pos_drop(x) + + for blk in self.blocks: + x = blk(x) + + x = self.norm(x) + + return x + + +def _make_vit_b16_backbone( + model, + features=[96, 192, 384, 768], + size=[384, 384], + hooks=[2, 5, 8, 11], + vit_features=768, + use_readout="ignore", + start_index=1, + start_index_readout=1, +): + pretrained = make_backbone_default(model, features, size, hooks, vit_features, use_readout, start_index, + start_index_readout) + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained + + +def _make_pretrained_vitl16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("vit_large_patch16_384", pretrained=pretrained) + + hooks = [5, 11, 17, 23] if hooks == None else hooks + return _make_vit_b16_backbone( + model, + features=[256, 512, 1024, 1024], + hooks=hooks, + vit_features=1024, + use_readout=use_readout, + ) + + +def _make_pretrained_vitb16_384(pretrained, use_readout="ignore", hooks=None): + model = timm.create_model("vit_base_patch16_384", pretrained=pretrained) + + hooks = [2, 5, 8, 11] if hooks == None else hooks + return _make_vit_b16_backbone( + model, features=[96, 192, 384, 768], hooks=hooks, use_readout=use_readout + ) + + +def _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=[0, 1, 8, 11], + vit_features=768, + patch_size=[16, 16], + number_stages=2, + use_vit_only=False, + use_readout="ignore", + start_index=1, +): + pretrained = nn.Module() + + pretrained.model = model + + used_number_stages = 0 if use_vit_only else number_stages + for s in range(used_number_stages): + pretrained.model.patch_embed.backbone.stages[s].register_forward_hook( + get_activation(str(s + 1)) + ) + for s in range(used_number_stages, 4): + pretrained.model.blocks[hooks[s]].register_forward_hook(get_activation(str(s + 1))) + + pretrained.activations = activations + + readout_oper = get_readout_oper(vit_features, features, use_readout, start_index) + + for s in range(used_number_stages): + value = nn.Sequential(nn.Identity(), nn.Identity(), nn.Identity()) + exec(f"pretrained.act_postprocess{s + 1}=value") + for s in range(used_number_stages, 4): + if s < number_stages: + final_layer = nn.ConvTranspose2d( + in_channels=features[s], + out_channels=features[s], + kernel_size=4 // (2 ** s), + stride=4 // (2 ** s), + padding=0, + bias=True, + dilation=1, + groups=1, + ) + elif s > number_stages: + final_layer = nn.Conv2d( + in_channels=features[3], + out_channels=features[3], + kernel_size=3, + stride=2, + padding=1, + ) + else: + final_layer = None + + layers = [ + readout_oper[s], + Transpose(1, 2), + nn.Unflatten(2, torch.Size([size[0] // 16, size[1] // 16])), + nn.Conv2d( + in_channels=vit_features, + out_channels=features[s], + kernel_size=1, + stride=1, + padding=0, + ), + ] + if final_layer is not None: + layers.append(final_layer) + + value = nn.Sequential(*layers) + exec(f"pretrained.act_postprocess{s + 1}=value") + + pretrained.model.start_index = start_index + pretrained.model.patch_size = patch_size + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model.forward_flex = types.MethodType(forward_flex, pretrained.model) + + # We inject this function into the VisionTransformer instances so that + # we can use it with interpolated position embeddings without modifying the library source. + pretrained.model._resize_pos_embed = types.MethodType( + _resize_pos_embed, pretrained.model + ) + + return pretrained + + +def _make_pretrained_vitb_rn50_384( + pretrained, use_readout="ignore", hooks=None, use_vit_only=False +): + model = timm.create_model("vit_base_resnet50_384", pretrained=pretrained) + + hooks = [0, 1, 8, 11] if hooks == None else hooks + return _make_vit_b_rn50_backbone( + model, + features=[256, 512, 768, 768], + size=[384, 384], + hooks=hooks, + use_vit_only=use_vit_only, + use_readout=use_readout, + ) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/base_model.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/base_model.py new file mode 100644 index 0000000000000000000000000000000000000000..27fe93ec5bfcfae3f8e78392ca617cdb6adc9b11 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/base_model.py @@ -0,0 +1,16 @@ +import torch + + +class BaseModel(torch.nn.Module): + def load(self, path): + """Load model from file. + + Args: + path (str): file path + """ + parameters = torch.load(path, map_location=torch.device('cpu')) + + if "optimizer" in parameters: + parameters = parameters["model"] + + self.load_state_dict(parameters) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/blocks.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/blocks.py new file mode 100644 index 0000000000000000000000000000000000000000..7f48f075c3ad3f62d0f303937c410982f1491929 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/blocks.py @@ -0,0 +1,439 @@ +import torch +import torch.nn as nn + +from .backbones.beit import ( + _make_pretrained_beitl16_512, + _make_pretrained_beitl16_384, + _make_pretrained_beitb16_384, + forward_beit, +) +from .backbones.swin_common import ( + forward_swin, +) +from .backbones.swin2 import ( + _make_pretrained_swin2l24_384, + _make_pretrained_swin2b24_384, + _make_pretrained_swin2t16_256, +) +from .backbones.swin import ( + _make_pretrained_swinl12_384, +) +from .backbones.levit import ( + _make_pretrained_levit_384, + forward_levit, +) +from .backbones.vit import ( + _make_pretrained_vitb_rn50_384, + _make_pretrained_vitl16_384, + _make_pretrained_vitb16_384, + forward_vit, +) + +def _make_encoder(backbone, features, use_pretrained, groups=1, expand=False, exportable=True, hooks=None, + use_vit_only=False, use_readout="ignore", in_features=[96, 256, 512, 1024]): + if backbone == "beitl16_512": + pretrained = _make_pretrained_beitl16_512( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, expand=expand + ) # BEiT_512-L (backbone) + elif backbone == "beitl16_384": + pretrained = _make_pretrained_beitl16_384( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, expand=expand + ) # BEiT_384-L (backbone) + elif backbone == "beitb16_384": + pretrained = _make_pretrained_beitb16_384( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [96, 192, 384, 768], features, groups=groups, expand=expand + ) # BEiT_384-B (backbone) + elif backbone == "swin2l24_384": + pretrained = _make_pretrained_swin2l24_384( + use_pretrained, hooks=hooks + ) + scratch = _make_scratch( + [192, 384, 768, 1536], features, groups=groups, expand=expand + ) # Swin2-L/12to24 (backbone) + elif backbone == "swin2b24_384": + pretrained = _make_pretrained_swin2b24_384( + use_pretrained, hooks=hooks + ) + scratch = _make_scratch( + [128, 256, 512, 1024], features, groups=groups, expand=expand + ) # Swin2-B/12to24 (backbone) + elif backbone == "swin2t16_256": + pretrained = _make_pretrained_swin2t16_256( + use_pretrained, hooks=hooks + ) + scratch = _make_scratch( + [96, 192, 384, 768], features, groups=groups, expand=expand + ) # Swin2-T/16 (backbone) + elif backbone == "swinl12_384": + pretrained = _make_pretrained_swinl12_384( + use_pretrained, hooks=hooks + ) + scratch = _make_scratch( + [192, 384, 768, 1536], features, groups=groups, expand=expand + ) # Swin-L/12 (backbone) + elif backbone == "next_vit_large_6m": + from .backbones.next_vit import _make_pretrained_next_vit_large_6m + pretrained = _make_pretrained_next_vit_large_6m(hooks=hooks) + scratch = _make_scratch( + in_features, features, groups=groups, expand=expand + ) # Next-ViT-L on ImageNet-1K-6M (backbone) + elif backbone == "levit_384": + pretrained = _make_pretrained_levit_384( + use_pretrained, hooks=hooks + ) + scratch = _make_scratch( + [384, 512, 768], features, groups=groups, expand=expand + ) # LeViT 384 (backbone) + elif backbone == "vitl16_384": + pretrained = _make_pretrained_vitl16_384( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [256, 512, 1024, 1024], features, groups=groups, expand=expand + ) # ViT-L/16 - 85.0% Top1 (backbone) + elif backbone == "vitb_rn50_384": + pretrained = _make_pretrained_vitb_rn50_384( + use_pretrained, + hooks=hooks, + use_vit_only=use_vit_only, + use_readout=use_readout, + ) + scratch = _make_scratch( + [256, 512, 768, 768], features, groups=groups, expand=expand + ) # ViT-H/16 - 85.0% Top1 (backbone) + elif backbone == "vitb16_384": + pretrained = _make_pretrained_vitb16_384( + use_pretrained, hooks=hooks, use_readout=use_readout + ) + scratch = _make_scratch( + [96, 192, 384, 768], features, groups=groups, expand=expand + ) # ViT-B/16 - 84.6% Top1 (backbone) + elif backbone == "resnext101_wsl": + pretrained = _make_pretrained_resnext101_wsl(use_pretrained) + scratch = _make_scratch([256, 512, 1024, 2048], features, groups=groups, expand=expand) # efficientnet_lite3 + elif backbone == "efficientnet_lite3": + pretrained = _make_pretrained_efficientnet_lite3(use_pretrained, exportable=exportable) + scratch = _make_scratch([32, 48, 136, 384], features, groups=groups, expand=expand) # efficientnet_lite3 + else: + print(f"Backbone '{backbone}' not implemented") + assert False + + return pretrained, scratch + + +def _make_scratch(in_shape, out_shape, groups=1, expand=False): + scratch = nn.Module() + + out_shape1 = out_shape + out_shape2 = out_shape + out_shape3 = out_shape + if len(in_shape) >= 4: + out_shape4 = out_shape + + if expand: + out_shape1 = out_shape + out_shape2 = out_shape*2 + out_shape3 = out_shape*4 + if len(in_shape) >= 4: + out_shape4 = out_shape*8 + + scratch.layer1_rn = nn.Conv2d( + in_shape[0], out_shape1, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer2_rn = nn.Conv2d( + in_shape[1], out_shape2, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + scratch.layer3_rn = nn.Conv2d( + in_shape[2], out_shape3, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + if len(in_shape) >= 4: + scratch.layer4_rn = nn.Conv2d( + in_shape[3], out_shape4, kernel_size=3, stride=1, padding=1, bias=False, groups=groups + ) + + return scratch + + +def _make_pretrained_efficientnet_lite3(use_pretrained, exportable=False): + efficientnet = torch.hub.load( + "rwightman/gen-efficientnet-pytorch", + "tf_efficientnet_lite3", + pretrained=use_pretrained, + exportable=exportable + ) + return _make_efficientnet_backbone(efficientnet) + + +def _make_efficientnet_backbone(effnet): + pretrained = nn.Module() + + pretrained.layer1 = nn.Sequential( + effnet.conv_stem, effnet.bn1, effnet.act1, *effnet.blocks[0:2] + ) + pretrained.layer2 = nn.Sequential(*effnet.blocks[2:3]) + pretrained.layer3 = nn.Sequential(*effnet.blocks[3:5]) + pretrained.layer4 = nn.Sequential(*effnet.blocks[5:9]) + + return pretrained + + +def _make_resnet_backbone(resnet): + pretrained = nn.Module() + pretrained.layer1 = nn.Sequential( + resnet.conv1, resnet.bn1, resnet.relu, resnet.maxpool, resnet.layer1 + ) + + pretrained.layer2 = resnet.layer2 + pretrained.layer3 = resnet.layer3 + pretrained.layer4 = resnet.layer4 + + return pretrained + + +def _make_pretrained_resnext101_wsl(use_pretrained): + resnet = torch.hub.load("facebookresearch/WSL-Images", "resnext101_32x8d_wsl") + return _make_resnet_backbone(resnet) + + + +class Interpolate(nn.Module): + """Interpolation module. + """ + + def __init__(self, scale_factor, mode, align_corners=False): + """Init. + + Args: + scale_factor (float): scaling + mode (str): interpolation mode + """ + super(Interpolate, self).__init__() + + self.interp = nn.functional.interpolate + self.scale_factor = scale_factor + self.mode = mode + self.align_corners = align_corners + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: interpolated data + """ + + x = self.interp( + x, scale_factor=self.scale_factor, mode=self.mode, align_corners=self.align_corners + ) + + return x + + +class ResidualConvUnit(nn.Module): + """Residual convolution module. + """ + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True + ) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True + ) + + self.relu = nn.ReLU(inplace=True) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + out = self.relu(x) + out = self.conv1(out) + out = self.relu(out) + out = self.conv2(out) + + return out + x + + +class FeatureFusionBlock(nn.Module): + """Feature fusion block. + """ + + def __init__(self, features): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock, self).__init__() + + self.resConfUnit1 = ResidualConvUnit(features) + self.resConfUnit2 = ResidualConvUnit(features) + + def forward(self, *xs): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + output += self.resConfUnit1(xs[1]) + + output = self.resConfUnit2(output) + + output = nn.functional.interpolate( + output, scale_factor=2, mode="bilinear", align_corners=True + ) + + return output + + + + +class ResidualConvUnit_custom(nn.Module): + """Residual convolution module. + """ + + def __init__(self, features, activation, bn): + """Init. + + Args: + features (int): number of features + """ + super().__init__() + + self.bn = bn + + self.groups=1 + + self.conv1 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups + ) + + self.conv2 = nn.Conv2d( + features, features, kernel_size=3, stride=1, padding=1, bias=True, groups=self.groups + ) + + if self.bn==True: + self.bn1 = nn.BatchNorm2d(features) + self.bn2 = nn.BatchNorm2d(features) + + self.activation = activation + + self.skip_add = nn.quantized.FloatFunctional() + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input + + Returns: + tensor: output + """ + + out = self.activation(x) + out = self.conv1(out) + if self.bn==True: + out = self.bn1(out) + + out = self.activation(out) + out = self.conv2(out) + if self.bn==True: + out = self.bn2(out) + + if self.groups > 1: + out = self.conv_merge(out) + + return self.skip_add.add(out, x) + + # return out + x + + +class FeatureFusionBlock_custom(nn.Module): + """Feature fusion block. + """ + + def __init__(self, features, activation, deconv=False, bn=False, expand=False, align_corners=True, size=None): + """Init. + + Args: + features (int): number of features + """ + super(FeatureFusionBlock_custom, self).__init__() + + self.deconv = deconv + self.align_corners = align_corners + + self.groups=1 + + self.expand = expand + out_features = features + if self.expand==True: + out_features = features//2 + + self.out_conv = nn.Conv2d(features, out_features, kernel_size=1, stride=1, padding=0, bias=True, groups=1) + + self.resConfUnit1 = ResidualConvUnit_custom(features, activation, bn) + self.resConfUnit2 = ResidualConvUnit_custom(features, activation, bn) + + self.skip_add = nn.quantized.FloatFunctional() + + self.size=size + + def forward(self, *xs, size=None): + """Forward pass. + + Returns: + tensor: output + """ + output = xs[0] + + if len(xs) == 2: + res = self.resConfUnit1(xs[1]) + output = self.skip_add.add(output, res) + # output += res + + output = self.resConfUnit2(output) + + if (size is None) and (self.size is None): + modifier = {"scale_factor": 2} + elif size is None: + modifier = {"size": self.size} + else: + modifier = {"size": size} + + output = nn.functional.interpolate( + output, **modifier, mode="bilinear", align_corners=self.align_corners + ) + + output = self.out_conv(output) + + return output + diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/dpt_depth.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/dpt_depth.py new file mode 100644 index 0000000000000000000000000000000000000000..0382e7303b3920db8c28bf84e0e289862fcb8f26 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/dpt_depth.py @@ -0,0 +1,166 @@ +import torch +import torch.nn as nn + +from .base_model import BaseModel +from .blocks import ( + FeatureFusionBlock_custom, + Interpolate, + _make_encoder, + forward_beit, + forward_swin, + forward_levit, + forward_vit, +) +from .backbones.levit import stem_b4_transpose +from timm.models.layers import get_act_layer + + +def _make_fusion_block(features, use_bn, size = None): + return FeatureFusionBlock_custom( + features, + nn.ReLU(False), + deconv=False, + bn=use_bn, + expand=False, + align_corners=True, + size=size, + ) + + +class DPT(BaseModel): + def __init__( + self, + head, + features=256, + backbone="vitb_rn50_384", + readout="project", + channels_last=False, + use_bn=False, + **kwargs + ): + + super(DPT, self).__init__() + + self.channels_last = channels_last + + # For the Swin, Swin 2, LeViT and Next-ViT Transformers, the hierarchical architectures prevent setting the + # hooks freely. Instead, the hooks have to be chosen according to the ranges specified in the comments. + hooks = { + "beitl16_512": [5, 11, 17, 23], + "beitl16_384": [5, 11, 17, 23], + "beitb16_384": [2, 5, 8, 11], + "swin2l24_384": [1, 1, 17, 1], # Allowed ranges: [0, 1], [0, 1], [ 0, 17], [ 0, 1] + "swin2b24_384": [1, 1, 17, 1], # [0, 1], [0, 1], [ 0, 17], [ 0, 1] + "swin2t16_256": [1, 1, 5, 1], # [0, 1], [0, 1], [ 0, 5], [ 0, 1] + "swinl12_384": [1, 1, 17, 1], # [0, 1], [0, 1], [ 0, 17], [ 0, 1] + "next_vit_large_6m": [2, 6, 36, 39], # [0, 2], [3, 6], [ 7, 36], [37, 39] + "levit_384": [3, 11, 21], # [0, 3], [6, 11], [14, 21] + "vitb_rn50_384": [0, 1, 8, 11], + "vitb16_384": [2, 5, 8, 11], + "vitl16_384": [5, 11, 17, 23], + }[backbone] + + if "next_vit" in backbone: + in_features = { + "next_vit_large_6m": [96, 256, 512, 1024], + }[backbone] + else: + in_features = None + + # Instantiate backbone and reassemble blocks + self.pretrained, self.scratch = _make_encoder( + backbone, + features, + False, # Set to true of you want to train from scratch, uses ImageNet weights + groups=1, + expand=False, + exportable=False, + hooks=hooks, + use_readout=readout, + in_features=in_features, + ) + + self.number_layers = len(hooks) if hooks is not None else 4 + size_refinenet3 = None + self.scratch.stem_transpose = None + + if "beit" in backbone: + self.forward_transformer = forward_beit + elif "swin" in backbone: + self.forward_transformer = forward_swin + elif "next_vit" in backbone: + from .backbones.next_vit import forward_next_vit + self.forward_transformer = forward_next_vit + elif "levit" in backbone: + self.forward_transformer = forward_levit + size_refinenet3 = 7 + self.scratch.stem_transpose = stem_b4_transpose(256, 128, get_act_layer("hard_swish")) + else: + self.forward_transformer = forward_vit + + self.scratch.refinenet1 = _make_fusion_block(features, use_bn) + self.scratch.refinenet2 = _make_fusion_block(features, use_bn) + self.scratch.refinenet3 = _make_fusion_block(features, use_bn, size_refinenet3) + if self.number_layers >= 4: + self.scratch.refinenet4 = _make_fusion_block(features, use_bn) + + self.scratch.output_conv = head + + + def forward(self, x): + if self.channels_last == True: + x.contiguous(memory_format=torch.channels_last) + + layers = self.forward_transformer(self.pretrained, x) + if self.number_layers == 3: + layer_1, layer_2, layer_3 = layers + else: + layer_1, layer_2, layer_3, layer_4 = layers + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + if self.number_layers >= 4: + layer_4_rn = self.scratch.layer4_rn(layer_4) + + if self.number_layers == 3: + path_3 = self.scratch.refinenet3(layer_3_rn, size=layer_2_rn.shape[2:]) + else: + path_4 = self.scratch.refinenet4(layer_4_rn, size=layer_3_rn.shape[2:]) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn, size=layer_2_rn.shape[2:]) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn, size=layer_1_rn.shape[2:]) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + if self.scratch.stem_transpose is not None: + path_1 = self.scratch.stem_transpose(path_1) + + out = self.scratch.output_conv(path_1) + + return out + + +class DPTDepthModel(DPT): + def __init__(self, path=None, non_negative=True, **kwargs): + features = kwargs["features"] if "features" in kwargs else 256 + head_features_1 = kwargs["head_features_1"] if "head_features_1" in kwargs else features + head_features_2 = kwargs["head_features_2"] if "head_features_2" in kwargs else 32 + kwargs.pop("head_features_1", None) + kwargs.pop("head_features_2", None) + + head = nn.Sequential( + nn.Conv2d(head_features_1, head_features_1 // 2, kernel_size=3, stride=1, padding=1), + Interpolate(scale_factor=2, mode="bilinear", align_corners=True), + nn.Conv2d(head_features_1 // 2, head_features_2, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(head_features_2, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + nn.Identity(), + ) + + super().__init__(head, **kwargs) + + if path is not None: + self.load(path) + + def forward(self, x): + return super().forward(x).squeeze(dim=1) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net.py new file mode 100644 index 0000000000000000000000000000000000000000..e10487c8fd4d3b50a4d9cf9bb72ce0a4734894c2 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net.py @@ -0,0 +1,76 @@ +"""MidashNet: Network for monocular depth estimation trained by mixing several datasets. +This file contains code that is adapted from +https://github.com/thomasjpfan/pytorch_refinenet/blob/master/pytorch_refinenet/refinenet/refinenet_4cascade.py +""" +import torch +import torch.nn as nn + +from .base_model import BaseModel +from .blocks import FeatureFusionBlock, Interpolate, _make_encoder + + +class MidasNet(BaseModel): + """Network for monocular depth estimation. + """ + + def __init__(self, path=None, features=256, non_negative=True): + """Init. + + Args: + path (str, optional): Path to saved model. Defaults to None. + features (int, optional): Number of features. Defaults to 256. + backbone (str, optional): Backbone network for encoder. Defaults to resnet50 + """ + print("Loading weights: ", path) + + super(MidasNet, self).__init__() + + use_pretrained = False if path is None else True + + self.pretrained, self.scratch = _make_encoder(backbone="resnext101_wsl", features=features, use_pretrained=use_pretrained) + + self.scratch.refinenet4 = FeatureFusionBlock(features) + self.scratch.refinenet3 = FeatureFusionBlock(features) + self.scratch.refinenet2 = FeatureFusionBlock(features) + self.scratch.refinenet1 = FeatureFusionBlock(features) + + self.scratch.output_conv = nn.Sequential( + nn.Conv2d(features, 128, kernel_size=3, stride=1, padding=1), + Interpolate(scale_factor=2, mode="bilinear"), + nn.Conv2d(128, 32, kernel_size=3, stride=1, padding=1), + nn.ReLU(True), + nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + ) + + if path: + self.load(path) + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input data (image) + + Returns: + tensor: depth + """ + + layer_1 = self.pretrained.layer1(x) + layer_2 = self.pretrained.layer2(layer_1) + layer_3 = self.pretrained.layer3(layer_2) + layer_4 = self.pretrained.layer4(layer_3) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv(path_1) + + return torch.squeeze(out, dim=1) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net_custom.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net_custom.py new file mode 100644 index 0000000000000000000000000000000000000000..3a8df446c55f057deb6db828db276ae9c0e2308d --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/midas_net_custom.py @@ -0,0 +1,128 @@ +"""MidashNet: Network for monocular depth estimation trained by mixing several datasets. +This file contains code that is adapted from +https://github.com/thomasjpfan/pytorch_refinenet/blob/master/pytorch_refinenet/refinenet/refinenet_4cascade.py +""" +import torch +import torch.nn as nn + +from .base_model import BaseModel +from .blocks import FeatureFusionBlock, FeatureFusionBlock_custom, Interpolate, _make_encoder + + +class MidasNet_small(BaseModel): + """Network for monocular depth estimation. + """ + + def __init__(self, path=None, features=64, backbone="efficientnet_lite3", non_negative=True, exportable=True, channels_last=False, align_corners=True, + blocks={'expand': True}): + """Init. + + Args: + path (str, optional): Path to saved model. Defaults to None. + features (int, optional): Number of features. Defaults to 256. + backbone (str, optional): Backbone network for encoder. Defaults to resnet50 + """ + print("Loading weights: ", path) + + super(MidasNet_small, self).__init__() + + use_pretrained = False if path else True + + self.channels_last = channels_last + self.blocks = blocks + self.backbone = backbone + + self.groups = 1 + + features1=features + features2=features + features3=features + features4=features + self.expand = False + if "expand" in self.blocks and self.blocks['expand'] == True: + self.expand = True + features1=features + features2=features*2 + features3=features*4 + features4=features*8 + + self.pretrained, self.scratch = _make_encoder(self.backbone, features, use_pretrained, groups=self.groups, expand=self.expand, exportable=exportable) + + self.scratch.activation = nn.ReLU(False) + + self.scratch.refinenet4 = FeatureFusionBlock_custom(features4, self.scratch.activation, deconv=False, bn=False, expand=self.expand, align_corners=align_corners) + self.scratch.refinenet3 = FeatureFusionBlock_custom(features3, self.scratch.activation, deconv=False, bn=False, expand=self.expand, align_corners=align_corners) + self.scratch.refinenet2 = FeatureFusionBlock_custom(features2, self.scratch.activation, deconv=False, bn=False, expand=self.expand, align_corners=align_corners) + self.scratch.refinenet1 = FeatureFusionBlock_custom(features1, self.scratch.activation, deconv=False, bn=False, align_corners=align_corners) + + + self.scratch.output_conv = nn.Sequential( + nn.Conv2d(features, features//2, kernel_size=3, stride=1, padding=1, groups=self.groups), + Interpolate(scale_factor=2, mode="bilinear"), + nn.Conv2d(features//2, 32, kernel_size=3, stride=1, padding=1), + self.scratch.activation, + nn.Conv2d(32, 1, kernel_size=1, stride=1, padding=0), + nn.ReLU(True) if non_negative else nn.Identity(), + nn.Identity(), + ) + + if path: + self.load(path) + + + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input data (image) + + Returns: + tensor: depth + """ + if self.channels_last==True: + print("self.channels_last = ", self.channels_last) + x.contiguous(memory_format=torch.channels_last) + + + layer_1 = self.pretrained.layer1(x) + layer_2 = self.pretrained.layer2(layer_1) + layer_3 = self.pretrained.layer3(layer_2) + layer_4 = self.pretrained.layer4(layer_3) + + layer_1_rn = self.scratch.layer1_rn(layer_1) + layer_2_rn = self.scratch.layer2_rn(layer_2) + layer_3_rn = self.scratch.layer3_rn(layer_3) + layer_4_rn = self.scratch.layer4_rn(layer_4) + + + path_4 = self.scratch.refinenet4(layer_4_rn) + path_3 = self.scratch.refinenet3(path_4, layer_3_rn) + path_2 = self.scratch.refinenet2(path_3, layer_2_rn) + path_1 = self.scratch.refinenet1(path_2, layer_1_rn) + + out = self.scratch.output_conv(path_1) + + return torch.squeeze(out, dim=1) + + + +def fuse_model(m): + prev_previous_type = nn.Identity() + prev_previous_name = '' + previous_type = nn.Identity() + previous_name = '' + for name, module in m.named_modules(): + if prev_previous_type == nn.Conv2d and previous_type == nn.BatchNorm2d and type(module) == nn.ReLU: + # print("FUSED ", prev_previous_name, previous_name, name) + torch.quantization.fuse_modules(m, [prev_previous_name, previous_name, name], inplace=True) + elif prev_previous_type == nn.Conv2d and previous_type == nn.BatchNorm2d: + # print("FUSED ", prev_previous_name, previous_name) + torch.quantization.fuse_modules(m, [prev_previous_name, previous_name], inplace=True) + # elif previous_type == nn.Conv2d and type(module) == nn.ReLU: + # print("FUSED ", previous_name, name) + # torch.quantization.fuse_modules(m, [previous_name, name], inplace=True) + + prev_previous_type = previous_type + prev_previous_name = previous_name + previous_type = type(module) + previous_name = name \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/model_loader.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/model_loader.py new file mode 100644 index 0000000000000000000000000000000000000000..6c6db0d7d421cebba21da140eeb6eb09b8906196 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/model_loader.py @@ -0,0 +1,242 @@ +import cv2 +import torch + +from midas.dpt_depth import DPTDepthModel +from midas.midas_net import MidasNet +from midas.midas_net_custom import MidasNet_small +from midas.transforms import Resize, NormalizeImage, PrepareForNet + +from torchvision.transforms import Compose + +default_models = { + "dpt_beit_large_512": "weights/dpt_beit_large_512.pt", + "dpt_beit_large_384": "weights/dpt_beit_large_384.pt", + "dpt_beit_base_384": "weights/dpt_beit_base_384.pt", + "dpt_swin2_large_384": "weights/dpt_swin2_large_384.pt", + "dpt_swin2_base_384": "weights/dpt_swin2_base_384.pt", + "dpt_swin2_tiny_256": "weights/dpt_swin2_tiny_256.pt", + "dpt_swin_large_384": "weights/dpt_swin_large_384.pt", + "dpt_next_vit_large_384": "weights/dpt_next_vit_large_384.pt", + "dpt_levit_224": "weights/dpt_levit_224.pt", + "dpt_large_384": "weights/dpt_large_384.pt", + "dpt_hybrid_384": "weights/dpt_hybrid_384.pt", + "midas_v21_384": "weights/midas_v21_384.pt", + "midas_v21_small_256": "weights/midas_v21_small_256.pt", + "openvino_midas_v21_small_256": "weights/openvino_midas_v21_small_256.xml", +} + + +def load_model(device, model_path, model_type="dpt_large_384", optimize=True, height=None, square=False): + """Load the specified network. + + Args: + device (device): the torch device used + model_path (str): path to saved model + model_type (str): the type of the model to be loaded + optimize (bool): optimize the model to half-integer on CUDA? + height (int): inference encoder image height + square (bool): resize to a square resolution? + + Returns: + The loaded network, the transform which prepares images as input to the network and the dimensions of the + network input + """ + if "openvino" in model_type: + from openvino.runtime import Core + + keep_aspect_ratio = not square + + if model_type == "dpt_beit_large_512": + model = DPTDepthModel( + path=model_path, + backbone="beitl16_512", + non_negative=True, + ) + net_w, net_h = 512, 512 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_beit_large_384": + model = DPTDepthModel( + path=model_path, + backbone="beitl16_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_beit_base_384": + model = DPTDepthModel( + path=model_path, + backbone="beitb16_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_swin2_large_384": + model = DPTDepthModel( + path=model_path, + backbone="swin2l24_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + keep_aspect_ratio = False + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_swin2_base_384": + model = DPTDepthModel( + path=model_path, + backbone="swin2b24_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + keep_aspect_ratio = False + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_swin2_tiny_256": + model = DPTDepthModel( + path=model_path, + backbone="swin2t16_256", + non_negative=True, + ) + net_w, net_h = 256, 256 + keep_aspect_ratio = False + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_swin_large_384": + model = DPTDepthModel( + path=model_path, + backbone="swinl12_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + keep_aspect_ratio = False + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_next_vit_large_384": + model = DPTDepthModel( + path=model_path, + backbone="next_vit_large_6m", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + # We change the notation from dpt_levit_224 (MiDaS notation) to levit_384 (timm notation) here, where the 224 refers + # to the resolution 224x224 used by LeViT and 384 is the first entry of the embed_dim, see _cfg and model_cfgs of + # https://github.com/rwightman/pytorch-image-models/blob/main/timm/models/levit.py + # (commit id: 927f031293a30afb940fff0bee34b85d9c059b0e) + elif model_type == "dpt_levit_224": + model = DPTDepthModel( + path=model_path, + backbone="levit_384", + non_negative=True, + head_features_1=64, + head_features_2=8, + ) + net_w, net_h = 224, 224 + keep_aspect_ratio = False + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_large_384": + model = DPTDepthModel( + path=model_path, + backbone="vitl16_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "dpt_hybrid_384": + model = DPTDepthModel( + path=model_path, + backbone="vitb_rn50_384", + non_negative=True, + ) + net_w, net_h = 384, 384 + resize_mode = "minimal" + normalization = NormalizeImage(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]) + + elif model_type == "midas_v21_384": + model = MidasNet(model_path, non_negative=True) + net_w, net_h = 384, 384 + resize_mode = "upper_bound" + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + + elif model_type == "midas_v21_small_256": + model = MidasNet_small(model_path, features=64, backbone="efficientnet_lite3", exportable=True, + non_negative=True, blocks={'expand': True}) + net_w, net_h = 256, 256 + resize_mode = "upper_bound" + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + + elif model_type == "openvino_midas_v21_small_256": + ie = Core() + uncompiled_model = ie.read_model(model=model_path) + model = ie.compile_model(uncompiled_model, "CPU") + net_w, net_h = 256, 256 + resize_mode = "upper_bound" + normalization = NormalizeImage( + mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225] + ) + + else: + print(f"model_type '{model_type}' not implemented, use: --model_type large") + assert False + + if not "openvino" in model_type: + print("Model loaded, number of parameters = {:.0f}M".format(sum(p.numel() for p in model.parameters()) / 1e6)) + else: + print("Model loaded, optimized with OpenVINO") + + if "openvino" in model_type: + keep_aspect_ratio = False + + if height is not None: + net_w, net_h = height, height + + transform = Compose( + [ + Resize( + net_w, + net_h, + resize_target=None, + keep_aspect_ratio=keep_aspect_ratio, + ensure_multiple_of=32, + resize_method=resize_mode, + image_interpolation_method=cv2.INTER_CUBIC, + ), + normalization, + PrepareForNet(), + ] + ) + + if not "openvino" in model_type: + model.eval() + + if optimize and (device == torch.device("cuda")): + if not "openvino" in model_type: + model = model.to(memory_format=torch.channels_last) + model = model.half() + else: + print("Error: OpenVINO models are already optimized. No optimization to half-float possible.") + exit() + + if not "openvino" in model_type: + model.to(device) + + return model, transform, net_w, net_h diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/transforms.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..37d68afcebf67defadd7dfb2ff5494f257876575 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/midas/transforms.py @@ -0,0 +1,234 @@ +import numpy as np +import cv2 +import math + + +def apply_min_size(sample, size, image_interpolation_method=cv2.INTER_AREA): + """Rezise the sample to ensure the given size. Keeps aspect ratio. + + Args: + sample (dict): sample + size (tuple): image size + + Returns: + tuple: new size + """ + shape = list(sample["disparity"].shape) + + if shape[0] >= size[0] and shape[1] >= size[1]: + return sample + + scale = [0, 0] + scale[0] = size[0] / shape[0] + scale[1] = size[1] / shape[1] + + scale = max(scale) + + shape[0] = math.ceil(scale * shape[0]) + shape[1] = math.ceil(scale * shape[1]) + + # resize + sample["image"] = cv2.resize( + sample["image"], tuple(shape[::-1]), interpolation=image_interpolation_method + ) + + sample["disparity"] = cv2.resize( + sample["disparity"], tuple(shape[::-1]), interpolation=cv2.INTER_NEAREST + ) + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + tuple(shape[::-1]), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return tuple(shape) + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + image_interpolation_method=cv2.INTER_AREA, + ): + """Init. + + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + self.__width = width + self.__height = height + + self.__resize_target = resize_target + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + self.__image_interpolation_method = image_interpolation_method + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented" + ) + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, min_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, min_val=self.__width + ) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, max_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, max_val=self.__width + ) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError(f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def __call__(self, sample): + width, height = self.get_size( + sample["image"].shape[1], sample["image"].shape[0] + ) + + # resize sample + sample["image"] = cv2.resize( + sample["image"], + (width, height), + interpolation=self.__image_interpolation_method, + ) + + if self.__resize_target: + if "disparity" in sample: + sample["disparity"] = cv2.resize( + sample["disparity"], + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + + if "depth" in sample: + sample["depth"] = cv2.resize( + sample["depth"], (width, height), interpolation=cv2.INTER_NEAREST + ) + + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return sample + + +class NormalizeImage(object): + """Normlize image by given mean and std. + """ + + def __init__(self, mean, std): + self.__mean = mean + self.__std = std + + def __call__(self, sample): + sample["image"] = (sample["image"] - self.__mean) / self.__std + + return sample + + +class PrepareForNet(object): + """Prepare sample for usage as network input. + """ + + def __init__(self): + pass + + def __call__(self, sample): + image = np.transpose(sample["image"], (2, 0, 1)) + sample["image"] = np.ascontiguousarray(image).astype(np.float32) + + if "mask" in sample: + sample["mask"] = sample["mask"].astype(np.float32) + sample["mask"] = np.ascontiguousarray(sample["mask"]) + + if "disparity" in sample: + disparity = sample["disparity"].astype(np.float32) + sample["disparity"] = np.ascontiguousarray(disparity) + + if "depth" in sample: + depth = sample["depth"].astype(np.float32) + sample["depth"] = np.ascontiguousarray(depth) + + return sample diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/LICENSE b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..00ac5912e5fb9d000475d6dd6a0eec15f98b3b70 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Alexey + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/README.md b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/README.md new file mode 100644 index 0000000000000000000000000000000000000000..ed0c7e8465d69c21928e86996b5222b2d3fbfd5d --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/README.md @@ -0,0 +1,131 @@ +# MiDaS for ROS1 by using LibTorch in C++ + +### Requirements + +- Ubuntu 17.10 / 18.04 / 20.04, Debian Stretch +- ROS Melodic for Ubuntu (17.10 / 18.04) / Debian Stretch, ROS Noetic for Ubuntu 20.04 +- C++11 +- LibTorch >= 1.6 + +## Quick Start with a MiDaS Example + +MiDaS is a neural network to compute depth from a single image. + +* input from `image_topic`: `sensor_msgs/Image` - `RGB8` image with any shape +* output to `midas_topic`: `sensor_msgs/Image` - `TYPE_32FC1` inverse relative depth maps in range [0 - 255] with original size and channels=1 + +### Install Dependecies + +* install ROS Melodic for Ubuntu 17.10 / 18.04: +```bash +wget https://raw.githubusercontent.com/isl-org/MiDaS/master/ros/additions/install_ros_melodic_ubuntu_17_18.sh +./install_ros_melodic_ubuntu_17_18.sh +``` + +or Noetic for Ubuntu 20.04: + +```bash +wget https://raw.githubusercontent.com/isl-org/MiDaS/master/ros/additions/install_ros_noetic_ubuntu_20.sh +./install_ros_noetic_ubuntu_20.sh +``` + + +* install LibTorch 1.7 with CUDA 11.0: + +On **Jetson (ARM)**: +```bash +wget https://nvidia.box.com/shared/static/wa34qwrwtk9njtyarwt5nvo6imenfy26.whl -O torch-1.7.0-cp36-cp36m-linux_aarch64.whl +sudo apt-get install python3-pip libopenblas-base libopenmpi-dev +pip3 install Cython +pip3 install numpy torch-1.7.0-cp36-cp36m-linux_aarch64.whl +``` +Or compile LibTorch from source: https://github.com/pytorch/pytorch#from-source + +On **Linux (x86_64)**: +```bash +cd ~/ +wget https://download.pytorch.org/libtorch/cu110/libtorch-cxx11-abi-shared-with-deps-1.7.0%2Bcu110.zip +unzip libtorch-cxx11-abi-shared-with-deps-1.7.0+cu110.zip +``` + +* create symlink for OpenCV: + +```bash +sudo ln -s /usr/include/opencv4 /usr/include/opencv +``` + +* download and install MiDaS: + +```bash +source ~/.bashrc +cd ~/ +mkdir catkin_ws +cd catkin_ws +git clone https://github.com/isl-org/MiDaS +mkdir src +cp -r MiDaS/ros/* src + +chmod +x src/additions/*.sh +chmod +x src/*.sh +chmod +x src/midas_cpp/scripts/*.py +cp src/additions/do_catkin_make.sh ./do_catkin_make.sh +./do_catkin_make.sh +./src/additions/downloads.sh +``` + +### Usage + +* run only `midas` node: `~/catkin_ws/src/launch_midas_cpp.sh` + +#### Test + +* Test - capture video and show result in the window: + * place any `test.mp4` video file to the directory `~/catkin_ws/src/` + * run `midas` node: `~/catkin_ws/src/launch_midas_cpp.sh` + * run test nodes in another terminal: `cd ~/catkin_ws/src && ./run_talker_listener_test.sh` and wait 30 seconds + + (to use Python 2, run command `sed -i 's/python3/python2/' ~/catkin_ws/src/midas_cpp/scripts/*.py` ) + +## Mobile version of MiDaS - Monocular Depth Estimation + +### Accuracy + +* MiDaS v2 small - ResNet50 default-decoder 384x384 +* MiDaS v2.1 small - EfficientNet-Lite3 small-decoder 256x256 + +**Zero-shot error** (the lower - the better): + +| Model | DIW WHDR | Eth3d AbsRel | Sintel AbsRel | Kitti δ>1.25 | NyuDepthV2 δ>1.25 | TUM δ>1.25 | +|---|---|---|---|---|---|---| +| MiDaS v2 small 384x384 | **0.1248** | 0.1550 | **0.3300** | **21.81** | 15.73 | 17.00 | +| MiDaS v2.1 small 256x256 | 0.1344 | **0.1344** | 0.3370 | 29.27 | **13.43** | **14.53** | +| Relative improvement, % | -8 % | **+13 %** | -2 % | -34 % | **+15 %** | **+15 %** | + +None of Train/Valid/Test subsets of datasets (DIW, Eth3d, Sintel, Kitti, NyuDepthV2, TUM) were not involved in Training or Fine Tuning. + +### Inference speed (FPS) on nVidia GPU + +Inference speed excluding pre and post processing, batch=1, **Frames Per Second** (the higher - the better): + +| Model | Jetson Nano, FPS | RTX 2080Ti, FPS | +|---|---|---| +| MiDaS v2 small 384x384 | 1.6 | 117 | +| MiDaS v2.1 small 256x256 | 8.1 | 232 | +| SpeedUp, X times | **5x** | **2x** | + +### Citation + +This repository contains code to compute depth from a single image. It accompanies our [paper](https://arxiv.org/abs/1907.01341v3): + +>Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer +René Ranftl, Katrin Lasinger, David Hafner, Konrad Schindler, Vladlen Koltun + +Please cite our paper if you use this code or any of the models: +``` +@article{Ranftl2020, + author = {Ren\'{e} Ranftl and Katrin Lasinger and David Hafner and Konrad Schindler and Vladlen Koltun}, + title = {Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer}, + journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, + year = {2020}, +} +``` diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/do_catkin_make.sh b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/do_catkin_make.sh new file mode 100644 index 0000000000000000000000000000000000000000..291354217538efebf3464a77573f17961e81399d --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/do_catkin_make.sh @@ -0,0 +1,5 @@ +mkdir src +catkin_make +source devel/setup.bash +echo $ROS_PACKAGE_PATH +chmod +x ./devel/setup.bash diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/downloads.sh b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/downloads.sh new file mode 100644 index 0000000000000000000000000000000000000000..6d41fb4d621e71075e70d716d060f78a465ce943 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/downloads.sh @@ -0,0 +1,5 @@ +mkdir ~/.ros +wget https://github.com/isl-org/MiDaS/releases/download/v2_1/model-small-traced.pt +cp ./model-small-traced.pt ~/.ros/model-small-traced.pt + + diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_melodic_ubuntu_17_18.sh b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_melodic_ubuntu_17_18.sh new file mode 100644 index 0000000000000000000000000000000000000000..15630cf4ee85a415d932dc6ed3da7e7da8dad6a9 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_melodic_ubuntu_17_18.sh @@ -0,0 +1,34 @@ +#@title { display-mode: "code" } + +#from http://wiki.ros.org/indigo/Installation/Ubuntu + +#1.2 Setup sources.list +sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' + +# 1.3 Setup keys +sudo apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 +sudo apt-key adv --keyserver 'hkp://ha.pool.sks-keyservers.net:80' --recv-key 421C365BD9FF1F717815A3895523BAEEB01FA116 + +curl -sSL 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xC1CF6E31E6BADE8868B172B4F42ED6FBAB17C654' | sudo apt-key add - + +# 1.4 Installation +sudo apt-get update +sudo apt-get upgrade + +# Desktop-Full Install: +sudo apt-get install ros-melodic-desktop-full + +printf "\nsource /opt/ros/melodic/setup.bash\n" >> ~/.bashrc + +# 1.5 Initialize rosdep +sudo rosdep init +rosdep update + + +# 1.7 Getting rosinstall (python) +sudo apt-get install python-rosinstall +sudo apt-get install python-catkin-tools +sudo apt-get install python-rospy +sudo apt-get install python-rosdep +sudo apt-get install python-roscd +sudo apt-get install python-pip \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_noetic_ubuntu_20.sh b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_noetic_ubuntu_20.sh new file mode 100644 index 0000000000000000000000000000000000000000..4c62e758a8320ed71a60a1b9b591f0dfc21f04f4 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/install_ros_noetic_ubuntu_20.sh @@ -0,0 +1,33 @@ +#@title { display-mode: "code" } + +#from http://wiki.ros.org/indigo/Installation/Ubuntu + +#1.2 Setup sources.list +sudo sh -c 'echo "deb http://packages.ros.org/ros/ubuntu $(lsb_release -sc) main" > /etc/apt/sources.list.d/ros-latest.list' + +# 1.3 Setup keys +sudo apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-key C1CF6E31E6BADE8868B172B4F42ED6FBAB17C654 + +curl -sSL 'http://keyserver.ubuntu.com/pks/lookup?op=get&search=0xC1CF6E31E6BADE8868B172B4F42ED6FBAB17C654' | sudo apt-key add - + +# 1.4 Installation +sudo apt-get update +sudo apt-get upgrade + +# Desktop-Full Install: +sudo apt-get install ros-noetic-desktop-full + +printf "\nsource /opt/ros/noetic/setup.bash\n" >> ~/.bashrc + +# 1.5 Initialize rosdep +sudo rosdep init +rosdep update + + +# 1.7 Getting rosinstall (python) +sudo apt-get install python3-rosinstall +sudo apt-get install python3-catkin-tools +sudo apt-get install python3-rospy +sudo apt-get install python3-rosdep +sudo apt-get install python3-roscd +sudo apt-get install python3-pip \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/make_package_cpp.sh b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/make_package_cpp.sh new file mode 100644 index 0000000000000000000000000000000000000000..82781ac1ecbdde527feb7dc3f5cd7d97a0ea1886 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/additions/make_package_cpp.sh @@ -0,0 +1,16 @@ +cd ~/catkin_ws/src +catkin_create_pkg midas_cpp std_msgs roscpp cv_bridge sensor_msgs image_transport +cd ~/catkin_ws +catkin_make + +chmod +x ~/catkin_ws/devel/setup.bash +printf "\nsource ~/catkin_ws/devel/setup.bash" >> ~/.bashrc +source ~/catkin_ws/devel/setup.bash + + +sudo rosdep init +rosdep update +#rospack depends1 midas_cpp +roscd midas_cpp +#cat package.xml +#rospack depends midas_cpp \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/launch_midas_cpp.sh b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/launch_midas_cpp.sh new file mode 100644 index 0000000000000000000000000000000000000000..63e138549417f49173107ccbb1cce40f8fd46532 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/launch_midas_cpp.sh @@ -0,0 +1,2 @@ +source ~/catkin_ws/devel/setup.bash +roslaunch midas_cpp midas_cpp.launch model_name:="model-small-traced.pt" input_topic:="image_topic" output_topic:="midas_topic" out_orig_size:="true" \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/CMakeLists.txt b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..63ae893bff0e5bb113cfd9a5c24186189dcb8b9a --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/CMakeLists.txt @@ -0,0 +1,189 @@ +cmake_minimum_required(VERSION 3.0.2) +project(midas_cpp) + +## Compile as C++11, supported in ROS Kinetic and newer +# add_compile_options(-std=c++11) + +## Find catkin macros and libraries +## if COMPONENTS list like find_package(catkin REQUIRED COMPONENTS xyz) +## is used, also find other catkin packages +find_package(catkin REQUIRED COMPONENTS + cv_bridge + image_transport + roscpp + rospy + sensor_msgs + std_msgs +) + +## System dependencies are found with CMake's conventions +# find_package(Boost REQUIRED COMPONENTS system) + +list(APPEND CMAKE_PREFIX_PATH "~/libtorch") +list(APPEND CMAKE_PREFIX_PATH "/usr/local/lib/python3.6/dist-packages/torch/lib") +list(APPEND CMAKE_PREFIX_PATH "/usr/local/lib/python2.7/dist-packages/torch/lib") + +if(NOT EXISTS "~/libtorch") + if (EXISTS "/usr/local/lib/python3.6/dist-packages/torch") + include_directories(/usr/local/include) + include_directories(/usr/local/lib/python3.6/dist-packages/torch/include/torch/csrc/api/include) + include_directories(/usr/local/lib/python3.6/dist-packages/torch/include) + + link_directories(/usr/local/lib) + link_directories(/usr/local/lib/python3.6/dist-packages/torch/lib) + + set(CMAKE_PREFIX_PATH /usr/local/lib/python3.6/dist-packages/torch) + set(Boost_USE_MULTITHREADED ON) + set(Torch_DIR /usr/local/lib/python3.6/dist-packages/torch) + + elseif (EXISTS "/usr/local/lib/python2.7/dist-packages/torch") + + include_directories(/usr/local/include) + include_directories(/usr/local/lib/python2.7/dist-packages/torch/include/torch/csrc/api/include) + include_directories(/usr/local/lib/python2.7/dist-packages/torch/include) + + link_directories(/usr/local/lib) + link_directories(/usr/local/lib/python2.7/dist-packages/torch/lib) + + set(CMAKE_PREFIX_PATH /usr/local/lib/python2.7/dist-packages/torch) + set(Boost_USE_MULTITHREADED ON) + set(Torch_DIR /usr/local/lib/python2.7/dist-packages/torch) + endif() +endif() + + + +find_package(Torch REQUIRED) +find_package(OpenCV REQUIRED) +include_directories( ${OpenCV_INCLUDE_DIRS} ) + +add_executable(midas_cpp src/main.cpp) +target_link_libraries(midas_cpp "${TORCH_LIBRARIES}" "${OpenCV_LIBS} ${catkin_LIBRARIES}") +set_property(TARGET midas_cpp PROPERTY CXX_STANDARD 14) + + + +################################### +## catkin specific configuration ## +################################### +## The catkin_package macro generates cmake config files for your package +## Declare things to be passed to dependent projects +## INCLUDE_DIRS: uncomment this if your package contains header files +## LIBRARIES: libraries you create in this project that dependent projects also need +## CATKIN_DEPENDS: catkin_packages dependent projects also need +## DEPENDS: system dependencies of this project that dependent projects also need +catkin_package( +# INCLUDE_DIRS include +# LIBRARIES midas_cpp +# CATKIN_DEPENDS cv_bridge image_transport roscpp sensor_msgs std_msgs +# DEPENDS system_lib +) + +########### +## Build ## +########### + +## Specify additional locations of header files +## Your package locations should be listed before other locations +include_directories( +# include + ${catkin_INCLUDE_DIRS} +) + +## Declare a C++ library +# add_library(${PROJECT_NAME} +# src/${PROJECT_NAME}/midas_cpp.cpp +# ) + +## Add cmake target dependencies of the library +## as an example, code may need to be generated before libraries +## either from message generation or dynamic reconfigure +# add_dependencies(${PROJECT_NAME} ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) + +## Declare a C++ executable +## With catkin_make all packages are built within a single CMake context +## The recommended prefix ensures that target names across packages don't collide +# add_executable(${PROJECT_NAME}_node src/midas_cpp_node.cpp) + +## Rename C++ executable without prefix +## The above recommended prefix causes long target names, the following renames the +## target back to the shorter version for ease of user use +## e.g. "rosrun someones_pkg node" instead of "rosrun someones_pkg someones_pkg_node" +# set_target_properties(${PROJECT_NAME}_node PROPERTIES OUTPUT_NAME node PREFIX "") + +## Add cmake target dependencies of the executable +## same as for the library above +# add_dependencies(${PROJECT_NAME}_node ${${PROJECT_NAME}_EXPORTED_TARGETS} ${catkin_EXPORTED_TARGETS}) + +## Specify libraries to link a library or executable target against +# target_link_libraries(${PROJECT_NAME}_node +# ${catkin_LIBRARIES} +# ) + +############# +## Install ## +############# + +# all install targets should use catkin DESTINATION variables +# See http://ros.org/doc/api/catkin/html/adv_user_guide/variables.html + +## Mark executable scripts (Python etc.) for installation +## in contrast to setup.py, you can choose the destination +# catkin_install_python(PROGRAMS +# scripts/my_python_script +# DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} +# ) + +## Mark executables for installation +## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_executables.html +# install(TARGETS ${PROJECT_NAME}_node +# RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} +# ) + +## Mark libraries for installation +## See http://docs.ros.org/melodic/api/catkin/html/howto/format1/building_libraries.html +# install(TARGETS ${PROJECT_NAME} +# ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} +# LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} +# RUNTIME DESTINATION ${CATKIN_GLOBAL_BIN_DESTINATION} +# ) + +## Mark cpp header files for installation +# install(DIRECTORY include/${PROJECT_NAME}/ +# DESTINATION ${CATKIN_PACKAGE_INCLUDE_DESTINATION} +# FILES_MATCHING PATTERN "*.h" +# PATTERN ".svn" EXCLUDE +# ) + +## Mark other files for installation (e.g. launch and bag files, etc.) +# install(FILES +# # myfile1 +# # myfile2 +# DESTINATION ${CATKIN_PACKAGE_SHARE_DESTINATION} +# ) + +############# +## Testing ## +############# + +## Add gtest based cpp test target and link libraries +# catkin_add_gtest(${PROJECT_NAME}-test test/test_midas_cpp.cpp) +# if(TARGET ${PROJECT_NAME}-test) +# target_link_libraries(${PROJECT_NAME}-test ${PROJECT_NAME}) +# endif() + +## Add folders to be run by python nosetests +# catkin_add_nosetests(test) + +install(TARGETS ${PROJECT_NAME} + ARCHIVE DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} + LIBRARY DESTINATION ${CATKIN_PACKAGE_LIB_DESTINATION} + RUNTIME DESTINATION ${CATKIN_PACKAGE_BIN_DESTINATION} +) + +add_custom_command( + TARGET midas_cpp POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${CMAKE_CURRENT_BINARY_DIR}/midas_cpp + ${CMAKE_SOURCE_DIR}/midas_cpp +) \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_cpp.launch b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_cpp.launch new file mode 100644 index 0000000000000000000000000000000000000000..326b7dfb19e928a9199a633c45e630dd2ef22bc0 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_cpp.launch @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_talker_listener.launch b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_talker_listener.launch new file mode 100644 index 0000000000000000000000000000000000000000..88d10c59b53b7031ef5c35424317b86991f2af8a --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/launch/midas_talker_listener.launch @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/package.xml b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/package.xml new file mode 100644 index 0000000000000000000000000000000000000000..6a0d109e65dab0b7652bba809074ae012a175b7e --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/package.xml @@ -0,0 +1,77 @@ + + + midas_cpp + 0.1.0 + The midas_cpp package + + Alexey Bochkovskiy + MIT + https://github.com/isl-org/MiDaS/tree/master/ros + + + + + + + TODO + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + catkin + cv_bridge + image_transport + roscpp + rospy + sensor_msgs + std_msgs + cv_bridge + image_transport + roscpp + rospy + sensor_msgs + std_msgs + cv_bridge + image_transport + roscpp + rospy + sensor_msgs + std_msgs + + + + + + + + diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener.py new file mode 100644 index 0000000000000000000000000000000000000000..7ad3f437dedd3759dc6d288d18499c93581bca26 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +from __future__ import print_function + +import roslib +#roslib.load_manifest('my_package') +import sys +import rospy +import cv2 +import numpy as np +from std_msgs.msg import String +from sensor_msgs.msg import Image +from cv_bridge import CvBridge, CvBridgeError + +class video_show: + + def __init__(self): + self.show_output = rospy.get_param('~show_output', True) + self.save_output = rospy.get_param('~save_output', False) + self.output_video_file = rospy.get_param('~output_video_file','result.mp4') + # rospy.loginfo(f"Listener - params: show_output={self.show_output}, save_output={self.save_output}, output_video_file={self.output_video_file}") + + self.bridge = CvBridge() + self.image_sub = rospy.Subscriber("midas_topic", Image, self.callback) + + def callback(self, data): + try: + cv_image = self.bridge.imgmsg_to_cv2(data) + except CvBridgeError as e: + print(e) + return + + if cv_image.size == 0: + return + + rospy.loginfo("Listener: Received new frame") + cv_image = cv_image.astype("uint8") + + if self.show_output==True: + cv2.imshow("video_show", cv_image) + cv2.waitKey(10) + + if self.save_output==True: + if self.video_writer_init==False: + fourcc = cv2.VideoWriter_fourcc(*'XVID') + self.out = cv2.VideoWriter(self.output_video_file, fourcc, 25, (cv_image.shape[1], cv_image.shape[0])) + + self.out.write(cv_image) + + + +def main(args): + rospy.init_node('listener', anonymous=True) + ic = video_show() + try: + rospy.spin() + except KeyboardInterrupt: + print("Shutting down") + cv2.destroyAllWindows() + +if __name__ == '__main__': + main(sys.argv) \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener_original.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener_original.py new file mode 100644 index 0000000000000000000000000000000000000000..59ec1be8f8c35559ca8938fb99ac81a14a494a4a --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/listener_original.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +from __future__ import print_function + +import roslib +#roslib.load_manifest('my_package') +import sys +import rospy +import cv2 +import numpy as np +from std_msgs.msg import String +from sensor_msgs.msg import Image +from cv_bridge import CvBridge, CvBridgeError + +class video_show: + + def __init__(self): + self.show_output = rospy.get_param('~show_output', True) + self.save_output = rospy.get_param('~save_output', False) + self.output_video_file = rospy.get_param('~output_video_file','result.mp4') + # rospy.loginfo(f"Listener original - params: show_output={self.show_output}, save_output={self.save_output}, output_video_file={self.output_video_file}") + + self.bridge = CvBridge() + self.image_sub = rospy.Subscriber("image_topic", Image, self.callback) + + def callback(self, data): + try: + cv_image = self.bridge.imgmsg_to_cv2(data) + except CvBridgeError as e: + print(e) + return + + if cv_image.size == 0: + return + + rospy.loginfo("Listener_original: Received new frame") + cv_image = cv_image.astype("uint8") + + if self.show_output==True: + cv2.imshow("video_show_orig", cv_image) + cv2.waitKey(10) + + if self.save_output==True: + if self.video_writer_init==False: + fourcc = cv2.VideoWriter_fourcc(*'XVID') + self.out = cv2.VideoWriter(self.output_video_file, fourcc, 25, (cv_image.shape[1], cv_image.shape[0])) + + self.out.write(cv_image) + + + +def main(args): + rospy.init_node('listener_original', anonymous=True) + ic = video_show() + try: + rospy.spin() + except KeyboardInterrupt: + print("Shutting down") + cv2.destroyAllWindows() + +if __name__ == '__main__': + main(sys.argv) \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/talker.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/talker.py new file mode 100644 index 0000000000000000000000000000000000000000..ddbeabf25ad41f5a765bdfa6cfaa3102331d1d7c --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/scripts/talker.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + + +import roslib +#roslib.load_manifest('my_package') +import sys +import rospy +import cv2 +from std_msgs.msg import String +from sensor_msgs.msg import Image +from cv_bridge import CvBridge, CvBridgeError + + +def talker(): + rospy.init_node('talker', anonymous=True) + + use_camera = rospy.get_param('~use_camera', False) + input_video_file = rospy.get_param('~input_video_file','test.mp4') + # rospy.loginfo(f"Talker - params: use_camera={use_camera}, input_video_file={input_video_file}") + + # rospy.loginfo("Talker: Trying to open a video stream") + if use_camera == True: + cap = cv2.VideoCapture(0) + else: + cap = cv2.VideoCapture(input_video_file) + + pub = rospy.Publisher('image_topic', Image, queue_size=1) + rate = rospy.Rate(30) # 30hz + bridge = CvBridge() + + while not rospy.is_shutdown(): + ret, cv_image = cap.read() + if ret==False: + print("Talker: Video is over") + rospy.loginfo("Video is over") + return + + try: + image = bridge.cv2_to_imgmsg(cv_image, "bgr8") + except CvBridgeError as e: + rospy.logerr("Talker: cv2image conversion failed: ", e) + print(e) + continue + + rospy.loginfo("Talker: Publishing frame") + pub.publish(image) + rate.sleep() + +if __name__ == '__main__': + try: + talker() + except rospy.ROSInterruptException: + pass diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/src/main.cpp b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/src/main.cpp new file mode 100644 index 0000000000000000000000000000000000000000..5ecc0c8a3232b933337c7512dfd48b4b4d01b157 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/midas_cpp/src/main.cpp @@ -0,0 +1,285 @@ +#include +#include +#include +#include + +#include + +#include // One-stop header. + +#include +#include +#include +#include + +#include +#include + +// includes for OpenCV >= 3.x +#ifndef CV_VERSION_EPOCH +#include +#include +#include +#endif + +// OpenCV includes for OpenCV 2.x +#ifdef CV_VERSION_EPOCH +#include +#include +#include +#include +#endif + +static const std::string OPENCV_WINDOW = "Image window"; + +class Midas +{ + ros::NodeHandle nh_; + image_transport::ImageTransport it_; + image_transport::Subscriber image_sub_; + image_transport::Publisher image_pub_; + + torch::jit::script::Module module; + torch::Device device; + + auto ToTensor(cv::Mat img, bool show_output = false, bool unsqueeze = false, int unsqueeze_dim = 0) + { + //std::cout << "image shape: " << img.size() << std::endl; + at::Tensor tensor_image = torch::from_blob(img.data, { img.rows, img.cols, 3 }, at::kByte); + + if (unsqueeze) + { + tensor_image.unsqueeze_(unsqueeze_dim); + //std::cout << "tensors new shape: " << tensor_image.sizes() << std::endl; + } + + if (show_output) + { + std::cout << tensor_image.slice(2, 0, 1) << std::endl; + } + //std::cout << "tenor shape: " << tensor_image.sizes() << std::endl; + return tensor_image; + } + + auto ToInput(at::Tensor tensor_image) + { + // Create a vector of inputs. + return std::vector{tensor_image}; + } + + auto ToCvImage(at::Tensor tensor, int cv_type = CV_8UC3) + { + int width = tensor.sizes()[0]; + int height = tensor.sizes()[1]; + try + { + cv::Mat output_mat; + if (cv_type == CV_8UC4 || cv_type == CV_8UC3 || cv_type == CV_8UC2 || cv_type == CV_8UC1) { + cv::Mat cv_image(cv::Size{ height, width }, cv_type, tensor.data_ptr()); + output_mat = cv_image; + } + else if (cv_type == CV_32FC4 || cv_type == CV_32FC3 || cv_type == CV_32FC2 || cv_type == CV_32FC1) { + cv::Mat cv_image(cv::Size{ height, width }, cv_type, tensor.data_ptr()); + output_mat = cv_image; + } + else if (cv_type == CV_64FC4 || cv_type == CV_64FC3 || cv_type == CV_64FC2 || cv_type == CV_64FC1) { + cv::Mat cv_image(cv::Size{ height, width }, cv_type, tensor.data_ptr()); + output_mat = cv_image; + } + + //show_image(output_mat, "converted image from tensor"); + return output_mat.clone(); + } + catch (const c10::Error& e) + { + std::cout << "an error has occured : " << e.msg() << std::endl; + } + return cv::Mat(height, width, CV_8UC3); + } + + std::string input_topic, output_topic, model_name; + bool out_orig_size; + int net_width, net_height; + torch::NoGradGuard guard; + at::Tensor mean, std; + at::Tensor output, tensor; + +public: + Midas() + : nh_(), it_(nh_), device(torch::Device(torch::kCPU)) + { + ros::param::param("~input_topic", input_topic, "image_topic"); + ros::param::param("~output_topic", output_topic, "midas_topic"); + ros::param::param("~model_name", model_name, "model-small-traced.pt"); + ros::param::param("~out_orig_size", out_orig_size, true); + ros::param::param("~net_width", net_width, 256); + ros::param::param("~net_height", net_height, 256); + + std::cout << ", input_topic = " << input_topic << + ", output_topic = " << output_topic << + ", model_name = " << model_name << + ", out_orig_size = " << out_orig_size << + ", net_width = " << net_width << + ", net_height = " << net_height << + std::endl; + + // Subscrive to input video feed and publish output video feed + image_sub_ = it_.subscribe(input_topic, 1, &Midas::imageCb, this); + image_pub_ = it_.advertise(output_topic, 1); + + std::cout << "Try to load torchscript model \n"; + + try { + // Deserialize the ScriptModule from a file using torch::jit::load(). + module = torch::jit::load(model_name); + } + catch (const c10::Error& e) { + std::cerr << "error loading the model\n"; + exit(0); + } + + std::cout << "ok\n"; + + try { + module.eval(); + torch::jit::getProfilingMode() = false; + torch::jit::setGraphExecutorOptimize(true); + + mean = torch::tensor({ 0.485, 0.456, 0.406 }); + std = torch::tensor({ 0.229, 0.224, 0.225 }); + + if (torch::hasCUDA()) { + std::cout << "cuda is available" << std::endl; + at::globalContext().setBenchmarkCuDNN(true); + device = torch::Device(torch::kCUDA); + module.to(device); + mean = mean.to(device); + std = std.to(device); + } + } + catch (const c10::Error& e) + { + std::cerr << " module initialization: " << e.msg() << std::endl; + } + } + + ~Midas() + { + } + + void imageCb(const sensor_msgs::ImageConstPtr& msg) + { + cv_bridge::CvImagePtr cv_ptr; + try + { + // sensor_msgs::Image to cv::Mat + cv_ptr = cv_bridge::toCvCopy(msg, sensor_msgs::image_encodings::RGB8); + } + catch (cv_bridge::Exception& e) + { + ROS_ERROR("cv_bridge exception: %s", e.what()); + return; + } + + // pre-processing + auto tensor_cpu = ToTensor(cv_ptr->image); // OpenCV-image -> Libtorch-tensor + + try { + tensor = tensor_cpu.to(device); // move to device (CPU or GPU) + + tensor = tensor.toType(c10::kFloat); + tensor = tensor.permute({ 2, 0, 1 }); // HWC -> CHW + tensor = tensor.unsqueeze(0); + tensor = at::upsample_bilinear2d(tensor, { net_height, net_width }, true); // resize + tensor = tensor.squeeze(0); + tensor = tensor.permute({ 1, 2, 0 }); // CHW -> HWC + + tensor = tensor.div(255).sub(mean).div(std); // normalization + tensor = tensor.permute({ 2, 0, 1 }); // HWC -> CHW + tensor.unsqueeze_(0); // CHW -> NCHW + } + catch (const c10::Error& e) + { + std::cerr << " pre-processing exception: " << e.msg() << std::endl; + return; + } + + auto input_to_net = ToInput(tensor); // input to the network + + // inference + output; + try { + output = module.forward(input_to_net).toTensor(); // run inference + } + catch (const c10::Error& e) + { + std::cerr << " module.forward() exception: " << e.msg() << std::endl; + return; + } + + output = output.detach().to(torch::kF32); + + // move to CPU temporary + at::Tensor output_tmp = output; + output_tmp = output_tmp.to(torch::kCPU); + + // normalization + float min_val = std::numeric_limits::max(); + float max_val = std::numeric_limits::min(); + + for (int i = 0; i < net_width * net_height; ++i) { + float val = output_tmp.data_ptr()[i]; + if (min_val > val) min_val = val; + if (max_val < val) max_val = val; + } + float range_val = max_val - min_val; + + output = output.sub(min_val).div(range_val).mul(255.0F).clamp(0, 255).to(torch::kF32); // .to(torch::kU8); + + // resize to the original size if required + if (out_orig_size) { + try { + output = at::upsample_bilinear2d(output.unsqueeze(0), { cv_ptr->image.size().height, cv_ptr->image.size().width }, true); + output = output.squeeze(0); + } + catch (const c10::Error& e) + { + std::cout << " upsample_bilinear2d() exception: " << e.msg() << std::endl; + return; + } + } + output = output.permute({ 1, 2, 0 }).to(torch::kCPU); + + int cv_type = CV_32FC1; // CV_8UC1; + auto cv_img = ToCvImage(output, cv_type); + + sensor_msgs::Image img_msg; + + try { + // cv::Mat -> sensor_msgs::Image + std_msgs::Header header; // empty header + header.seq = 0; // user defined counter + header.stamp = ros::Time::now();// time + //cv_bridge::CvImage img_bridge = cv_bridge::CvImage(header, sensor_msgs::image_encodings::MONO8, cv_img); + cv_bridge::CvImage img_bridge = cv_bridge::CvImage(header, sensor_msgs::image_encodings::TYPE_32FC1, cv_img); + + img_bridge.toImageMsg(img_msg); // cv_bridge -> sensor_msgs::Image + } + catch (cv_bridge::Exception& e) + { + ROS_ERROR("cv_bridge exception: %s", e.what()); + return; + } + + // Output modified video stream + image_pub_.publish(img_msg); + } +}; + +int main(int argc, char** argv) +{ + ros::init(argc, argv, "midas", ros::init_options::AnonymousName); + Midas ic; + ros::spin(); + return 0; +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/run_talker_listener_test.sh b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/run_talker_listener_test.sh new file mode 100644 index 0000000000000000000000000000000000000000..181447a83afc9c273ab910cb7e71526b99069e9f --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/ros/run_talker_listener_test.sh @@ -0,0 +1,16 @@ +# place any test.mp4 file near with this file + +# roscore +# rosnode kill -a + +source ~/catkin_ws/devel/setup.bash + +roscore & +P1=$! +rosrun midas_cpp talker.py & +P2=$! +rosrun midas_cpp listener_original.py & +P3=$! +rosrun midas_cpp listener.py & +P4=$! +wait $P1 $P2 $P3 $P4 \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/run.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/run.py new file mode 100644 index 0000000000000000000000000000000000000000..55a3b65bb749954aa94602544ce2007bd5cb9145 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/run.py @@ -0,0 +1,279 @@ +"""Compute depth maps for images in the input folder. +""" +import os +import glob +import torch +import utils +import cv2 +import argparse +import time + +import numpy as np + +from imutils.video import VideoStream +from midas.model_loader import default_models, load_model + +from modules import devices + +first_execution = True +def process(device, model, model_type, image, input_size, target_size, optimize, use_camera): + """ + Run the inference and interpolate. + + Args: + device (torch.device): the torch device used + model: the model used for inference + model_type: the type of the model + image: the image fed into the neural network + input_size: the size (width, height) of the neural network input (for OpenVINO) + target_size: the size (width, height) the neural network output is interpolated to + optimize: optimize the model to half-floats on CUDA? + use_camera: is the camera used? + + Returns: + the prediction + """ + global first_execution + + if "openvino" in model_type: + if first_execution or not use_camera: + print(f" Input resized to {input_size[0]}x{input_size[1]} before entering the encoder") + first_execution = False + + sample = [np.reshape(image, (1, 3, *input_size))] + prediction = model(sample)[model.output(0)][0] + prediction = cv2.resize(prediction, dsize=target_size, + interpolation=cv2.INTER_CUBIC) + else: + sample = torch.from_numpy(image).to(device).unsqueeze(0) + + if optimize and device == torch.device("cuda"): + if first_execution: + print(" Optimization to half-floats activated. Use with caution, because models like Swin require\n" + " float precision to work properly and may yield non-finite depth values to some extent for\n" + " half-floats.") + sample = sample.to(memory_format=torch.channels_last) + sample = sample.half() + + if first_execution or not use_camera: + height, width = sample.shape[2:] + print(f" Input resized to {width}x{height} before entering the encoder") + first_execution = False + + prediction = model.forward(sample) + prediction = ( + torch.nn.functional.interpolate( + prediction.unsqueeze(1), + size=target_size[::-1], + mode="bicubic", + align_corners=False, + ) + .squeeze() + .cpu() + .numpy() + ) + + return prediction + + +def create_side_by_side(image, depth, grayscale): + """ + Take an RGB image and depth map and place them side by side. This includes a proper normalization of the depth map + for better visibility. + + Args: + image: the RGB image + depth: the depth map + grayscale: use a grayscale colormap? + + Returns: + the image and depth map place side by side + """ + depth_min = depth.min() + depth_max = depth.max() + normalized_depth = 255 * (depth - depth_min) / (depth_max - depth_min) + normalized_depth *= 3 + + right_side = np.repeat(np.expand_dims(normalized_depth, 2), 3, axis=2) / 3 + if not grayscale: + right_side = cv2.applyColorMap(np.uint8(right_side), cv2.COLORMAP_INFERNO) + + if image is None: + return right_side + else: + return np.concatenate((image, right_side), axis=1) + + +def run(input_path, output_path, model_path, model_type="dpt_beit_large_512", optimize=False, side=False, height=None, + square=False, grayscale=False): + """Run MonoDepthNN to compute depth maps. + + Args: + input_path (str): path to input folder + output_path (str): path to output folder + model_path (str): path to saved model + model_type (str): the model type + optimize (bool): optimize the model to half-floats on CUDA? + side (bool): RGB and depth side by side in output images? + height (int): inference encoder image height + square (bool): resize to a square resolution? + grayscale (bool): use a grayscale colormap? + """ + print("Initialize") + + # select device + device = devices.get_device_for("controlnet") if torch.cuda.is_available() else torch.device("cpu") + print("Device: %s" % device) + + model, transform, net_w, net_h = load_model(device, model_path, model_type, optimize, height, square) + + # get input + if input_path is not None: + image_names = glob.glob(os.path.join(input_path, "*")) + num_images = len(image_names) + else: + print("No input path specified. Grabbing images from camera.") + + # create output folder + if output_path is not None: + os.makedirs(output_path, exist_ok=True) + + print("Start processing") + + if input_path is not None: + if output_path is None: + print("Warning: No output path specified. Images will be processed but not shown or stored anywhere.") + for index, image_name in enumerate(image_names): + + print(" Processing {} ({}/{})".format(image_name, index + 1, num_images)) + + # input + original_image_rgb = utils.read_image(image_name) # in [0, 1] + image = transform({"image": original_image_rgb})["image"] + + # compute + with torch.no_grad(): + prediction = process(device, model, model_type, image, (net_w, net_h), original_image_rgb.shape[1::-1], + optimize, False) + + # output + if output_path is not None: + filename = os.path.join( + output_path, os.path.splitext(os.path.basename(image_name))[0] + '-' + model_type + ) + if not side: + utils.write_depth(filename, prediction, grayscale, bits=2) + else: + original_image_bgr = np.flip(original_image_rgb, 2) + content = create_side_by_side(original_image_bgr*255, prediction, grayscale) + cv2.imwrite(filename + ".png", content) + utils.write_pfm(filename + ".pfm", prediction.astype(np.float32)) + + else: + with torch.no_grad(): + fps = 1 + video = VideoStream(0).start() + time_start = time.time() + frame_index = 0 + while True: + frame = video.read() + if frame is not None: + original_image_rgb = np.flip(frame, 2) # in [0, 255] (flip required to get RGB) + image = transform({"image": original_image_rgb/255})["image"] + + prediction = process(device, model, model_type, image, (net_w, net_h), + original_image_rgb.shape[1::-1], optimize, True) + + original_image_bgr = np.flip(original_image_rgb, 2) if side else None + content = create_side_by_side(original_image_bgr, prediction, grayscale) + cv2.imshow('MiDaS Depth Estimation - Press Escape to close window ', content/255) + + if output_path is not None: + filename = os.path.join(output_path, 'Camera' + '-' + model_type + '_' + str(frame_index)) + cv2.imwrite(filename + ".png", content) + + alpha = 0.1 + if time.time()-time_start > 0: + fps = (1 - alpha) * fps + alpha * 1 / (time.time()-time_start) # exponential moving average + time_start = time.time() + print(f"\rFPS: {round(fps,2)}", end="") + + if cv2.waitKey(1) == 27: # Escape key + break + + frame_index += 1 + print() + + print("Finished") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-i', '--input_path', + default=None, + help='Folder with input images (if no input path is specified, images are tried to be grabbed ' + 'from camera)' + ) + + parser.add_argument('-o', '--output_path', + default=None, + help='Folder for output images' + ) + + parser.add_argument('-m', '--model_weights', + default=None, + help='Path to the trained weights of model' + ) + + parser.add_argument('-t', '--model_type', + default='dpt_beit_large_512', + help='Model type: ' + 'dpt_beit_large_512, dpt_beit_large_384, dpt_beit_base_384, dpt_swin2_large_384, ' + 'dpt_swin2_base_384, dpt_swin2_tiny_256, dpt_swin_large_384, dpt_next_vit_large_384, ' + 'dpt_levit_224, dpt_large_384, dpt_hybrid_384, midas_v21_384, midas_v21_small_256 or ' + 'openvino_midas_v21_small_256' + ) + + parser.add_argument('-s', '--side', + action='store_true', + help='Output images contain RGB and depth images side by side' + ) + + parser.add_argument('--optimize', dest='optimize', action='store_true', help='Use half-float optimization') + parser.set_defaults(optimize=False) + + parser.add_argument('--height', + type=int, default=None, + help='Preferred height of images feed into the encoder during inference. Note that the ' + 'preferred height may differ from the actual height, because an alignment to multiples of ' + '32 takes place. Many models support only the height chosen during training, which is ' + 'used automatically if this parameter is not set.' + ) + parser.add_argument('--square', + action='store_true', + help='Option to resize images to a square resolution by changing their widths when images are ' + 'fed into the encoder during inference. If this parameter is not set, the aspect ratio of ' + 'images is tried to be preserved if supported by the model.' + ) + parser.add_argument('--grayscale', + action='store_true', + help='Use a grayscale colormap instead of the inferno one. Although the inferno colormap, ' + 'which is used by default, is better for visibility, it does not allow storing 16-bit ' + 'depth values in PNGs but only 8-bit ones due to the precision limitation of this ' + 'colormap.' + ) + + args = parser.parse_args() + + + if args.model_weights is None: + args.model_weights = default_models[args.model_type] + + # set torch options + torch.backends.cudnn.enabled = True + torch.backends.cudnn.benchmark = True + + # compute depth maps + run(args.input_path, args.output_path, args.model_weights, args.model_type, args.optimize, args.side, args.height, + args.square, args.grayscale) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/README.md b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/README.md new file mode 100644 index 0000000000000000000000000000000000000000..dca3cf8efd6110c698e218759cc7ef0a83980c0d --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/README.md @@ -0,0 +1,147 @@ +## Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer + +### TensorFlow inference using `.pb` and `.onnx` models + +1. [Run inference on TensorFlow-model by using TensorFlow](#run-inference-on-tensorflow-model-by-using-tensorFlow) + +2. [Run inference on ONNX-model by using TensorFlow](#run-inference-on-onnx-model-by-using-tensorflow) + +3. [Make ONNX model from downloaded Pytorch model file](#make-onnx-model-from-downloaded-pytorch-model-file) + + +### Run inference on TensorFlow-model by using TensorFlow + +1) Download the model weights [model-f6b98070.pb](https://github.com/isl-org/MiDaS/releases/download/v2_1/model-f6b98070.pb) +and [model-small.pb](https://github.com/isl-org/MiDaS/releases/download/v2_1/model-small.pb) and place the +file in the `/tf/` folder. + +2) Set up dependencies: + +```shell +# install OpenCV +pip install --upgrade pip +pip install opencv-python + +# install TensorFlow +pip install -I grpcio tensorflow==2.3.0 tensorflow-addons==0.11.2 numpy==1.18.0 +``` + +#### Usage + +1) Place one or more input images in the folder `tf/input`. + +2) Run the model: + + ```shell + python tf/run_pb.py + ``` + + Or run the small model: + + ```shell + python tf/run_pb.py --model_weights model-small.pb --model_type small + ``` + +3) The resulting inverse depth maps are written to the `tf/output` folder. + + +### Run inference on ONNX-model by using ONNX-Runtime + +1) Download the model weights [model-f6b98070.onnx](https://github.com/isl-org/MiDaS/releases/download/v2_1/model-f6b98070.onnx) +and [model-small.onnx](https://github.com/isl-org/MiDaS/releases/download/v2_1/model-small.onnx) and place the +file in the `/tf/` folder. + +2) Set up dependencies: + +```shell +# install OpenCV +pip install --upgrade pip +pip install opencv-python + +# install ONNX +pip install onnx==1.7.0 + +# install ONNX Runtime +pip install onnxruntime==1.5.2 +``` + +#### Usage + +1) Place one or more input images in the folder `tf/input`. + +2) Run the model: + + ```shell + python tf/run_onnx.py + ``` + + Or run the small model: + + ```shell + python tf/run_onnx.py --model_weights model-small.onnx --model_type small + ``` + +3) The resulting inverse depth maps are written to the `tf/output` folder. + + + +### Make ONNX model from downloaded Pytorch model file + +1) Download the model weights [model-f6b98070.pt](https://github.com/isl-org/MiDaS/releases/download/v2_1/model-f6b98070.pt) and place the +file in the root folder. + +2) Set up dependencies: + +```shell +# install OpenCV +pip install --upgrade pip +pip install opencv-python + +# install PyTorch TorchVision +pip install -I torch==1.7.0 torchvision==0.8.0 + +# install TensorFlow +pip install -I grpcio tensorflow==2.3.0 tensorflow-addons==0.11.2 numpy==1.18.0 + +# install ONNX +pip install onnx==1.7.0 + +# install ONNX-TensorFlow +git clone https://github.com/onnx/onnx-tensorflow.git +cd onnx-tensorflow +git checkout 095b51b88e35c4001d70f15f80f31014b592b81e +pip install -e . +``` + +#### Usage + +1) Run the converter: + + ```shell + python tf/make_onnx_model.py + ``` + +2) The resulting `model-f6b98070.onnx` file is written to the `/tf/` folder. + + +### Requirements + + The code was tested with Python 3.6.9, PyTorch 1.5.1, TensorFlow 2.2.0, TensorFlow-addons 0.8.3, ONNX 1.7.0, ONNX-TensorFlow (GitHub-master-17.07.2020) and OpenCV 4.3.0. + +### Citation + +Please cite our paper if you use this code or any of the models: +``` +@article{Ranftl2019, + author = {Ren\'{e} Ranftl and Katrin Lasinger and David Hafner and Konrad Schindler and Vladlen Koltun}, + title = {Towards Robust Monocular Depth Estimation: Mixing Datasets for Zero-shot Cross-dataset Transfer}, + journal = {IEEE Transactions on Pattern Analysis and Machine Intelligence (TPAMI)}, + year = {2020}, +} +``` + +### License + +MIT License + + diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/input/.placeholder b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/input/.placeholder new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/make_onnx_model.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/make_onnx_model.py new file mode 100644 index 0000000000000000000000000000000000000000..49afaee6851e453a06f3b58ba9a9558375c90164 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/make_onnx_model.py @@ -0,0 +1,112 @@ +"""Compute depth maps for images in the input folder. +""" +import os +import ntpath +import glob +import torch +import utils +import cv2 +import numpy as np +from torchvision.transforms import Compose, Normalize +from torchvision import transforms + +from shutil import copyfile +import fileinput +import sys +sys.path.append(os.getcwd() + '/..') + +def modify_file(): + modify_filename = '../midas/blocks.py' + copyfile(modify_filename, modify_filename+'.bak') + + with open(modify_filename, 'r') as file : + filedata = file.read() + + filedata = filedata.replace('align_corners=True', 'align_corners=False') + filedata = filedata.replace('import torch.nn as nn', 'import torch.nn as nn\nimport torchvision.models as models') + filedata = filedata.replace('torch.hub.load("facebookresearch/WSL-Images", "resnext101_32x8d_wsl")', 'models.resnext101_32x8d()') + + with open(modify_filename, 'w') as file: + file.write(filedata) + +def restore_file(): + modify_filename = '../midas/blocks.py' + copyfile(modify_filename+'.bak', modify_filename) + +modify_file() + +from midas.midas_net import MidasNet +from midas.transforms import Resize, NormalizeImage, PrepareForNet + +restore_file() + + +class MidasNet_preprocessing(MidasNet): + """Network for monocular depth estimation. + """ + def forward(self, x): + """Forward pass. + + Args: + x (tensor): input data (image) + + Returns: + tensor: depth + """ + + mean = torch.tensor([0.485, 0.456, 0.406]) + std = torch.tensor([0.229, 0.224, 0.225]) + x.sub_(mean[None, :, None, None]).div_(std[None, :, None, None]) + + return MidasNet.forward(self, x) + + +def run(model_path): + """Run MonoDepthNN to compute depth maps. + + Args: + model_path (str): path to saved model + """ + print("initialize") + + # select device + + # load network + #model = MidasNet(model_path, non_negative=True) + model = MidasNet_preprocessing(model_path, non_negative=True) + + model.eval() + + print("start processing") + + # input + img_input = np.zeros((3, 384, 384), np.float32) + + # compute + with torch.no_grad(): + sample = torch.from_numpy(img_input).unsqueeze(0) + prediction = model.forward(sample) + prediction = ( + torch.nn.functional.interpolate( + prediction.unsqueeze(1), + size=img_input.shape[:2], + mode="bicubic", + align_corners=False, + ) + .squeeze() + .cpu() + .numpy() + ) + + torch.onnx.export(model, sample, ntpath.basename(model_path).rsplit('.', 1)[0]+'.onnx', opset_version=9) + + print("finished") + + +if __name__ == "__main__": + # set paths + # MODEL_PATH = "model.pt" + MODEL_PATH = "../model-f6b98070.pt" + + # compute depth maps + run(MODEL_PATH) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_onnx.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..48587111c3e6a7cc97674f3bb35ebbf222a7123d --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_onnx.py @@ -0,0 +1,119 @@ +"""Compute depth maps for images in the input folder. +""" +import os +import glob +import utils +import cv2 +import sys +import numpy as np +import argparse + +import onnx +import onnxruntime as rt + +from transforms import Resize, NormalizeImage, PrepareForNet + + +def run(input_path, output_path, model_path, model_type="large"): + """Run MonoDepthNN to compute depth maps. + + Args: + input_path (str): path to input folder + output_path (str): path to output folder + model_path (str): path to saved model + """ + print("initialize") + + # select device + device = "CUDA:0" + #device = "CPU" + print("device: %s" % device) + + # network resolution + if model_type == "large": + net_w, net_h = 384, 384 + elif model_type == "small": + net_w, net_h = 256, 256 + else: + print(f"model_type '{model_type}' not implemented, use: --model_type large") + assert False + + # load network + print("loading model...") + model = rt.InferenceSession(model_path) + input_name = model.get_inputs()[0].name + output_name = model.get_outputs()[0].name + + resize_image = Resize( + net_w, + net_h, + resize_target=None, + keep_aspect_ratio=False, + ensure_multiple_of=32, + resize_method="upper_bound", + image_interpolation_method=cv2.INTER_CUBIC, + ) + + def compose2(f1, f2): + return lambda x: f2(f1(x)) + + transform = compose2(resize_image, PrepareForNet()) + + # get input + img_names = glob.glob(os.path.join(input_path, "*")) + num_images = len(img_names) + + # create output folder + os.makedirs(output_path, exist_ok=True) + + print("start processing") + + for ind, img_name in enumerate(img_names): + + print(" processing {} ({}/{})".format(img_name, ind + 1, num_images)) + + # input + img = utils.read_image(img_name) + img_input = transform({"image": img})["image"] + + # compute + output = model.run([output_name], {input_name: img_input.reshape(1, 3, net_h, net_w).astype(np.float32)})[0] + prediction = np.array(output).reshape(net_h, net_w) + prediction = cv2.resize(prediction, (img.shape[1], img.shape[0]), interpolation=cv2.INTER_CUBIC) + + # output + filename = os.path.join( + output_path, os.path.splitext(os.path.basename(img_name))[0] + ) + utils.write_depth(filename, prediction, bits=2) + + print("finished") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-i', '--input_path', + default='input', + help='folder with input images' + ) + + parser.add_argument('-o', '--output_path', + default='output', + help='folder for output images' + ) + + parser.add_argument('-m', '--model_weights', + default='model-f6b98070.onnx', + help='path to the trained weights of model' + ) + + parser.add_argument('-t', '--model_type', + default='large', + help='model type: large or small' + ) + + args = parser.parse_args() + + # compute depth maps + run(args.input_path, args.output_path, args.model_weights, args.model_type) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_pb.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_pb.py new file mode 100644 index 0000000000000000000000000000000000000000..0f3e8cbbf433f4fe43878b221da0517f770cc81f --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/run_pb.py @@ -0,0 +1,135 @@ +"""Compute depth maps for images in the input folder. +""" +import os +import glob +import utils +import cv2 +import argparse + +import tensorflow as tf + +from transforms import Resize, NormalizeImage, PrepareForNet + +def run(input_path, output_path, model_path, model_type="large"): + """Run MonoDepthNN to compute depth maps. + + Args: + input_path (str): path to input folder + output_path (str): path to output folder + model_path (str): path to saved model + """ + print("initialize") + + # the runtime initialization will not allocate all memory on the device to avoid out of GPU memory + gpus = tf.config.experimental.list_physical_devices('GPU') + if gpus: + try: + for gpu in gpus: + #tf.config.experimental.set_memory_growth(gpu, True) + tf.config.experimental.set_virtual_device_configuration(gpu, + [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=4000)]) + except RuntimeError as e: + print(e) + + # network resolution + if model_type == "large": + net_w, net_h = 384, 384 + elif model_type == "small": + net_w, net_h = 256, 256 + else: + print(f"model_type '{model_type}' not implemented, use: --model_type large") + assert False + + # load network + graph_def = tf.compat.v1.GraphDef() + with tf.io.gfile.GFile(model_path, 'rb') as f: + graph_def.ParseFromString(f.read()) + tf.import_graph_def(graph_def, name='') + + + model_operations = tf.compat.v1.get_default_graph().get_operations() + input_node = '0:0' + output_layer = model_operations[len(model_operations) - 1].name + ':0' + print("Last layer name: ", output_layer) + + resize_image = Resize( + net_w, + net_h, + resize_target=None, + keep_aspect_ratio=False, + ensure_multiple_of=32, + resize_method="upper_bound", + image_interpolation_method=cv2.INTER_CUBIC, + ) + + def compose2(f1, f2): + return lambda x: f2(f1(x)) + + transform = compose2(resize_image, PrepareForNet()) + + # get input + img_names = glob.glob(os.path.join(input_path, "*")) + num_images = len(img_names) + + # create output folder + os.makedirs(output_path, exist_ok=True) + + print("start processing") + + with tf.compat.v1.Session() as sess: + try: + # load images + for ind, img_name in enumerate(img_names): + + print(" processing {} ({}/{})".format(img_name, ind + 1, num_images)) + + # input + img = utils.read_image(img_name) + img_input = transform({"image": img})["image"] + + # compute + prob_tensor = sess.graph.get_tensor_by_name(output_layer) + prediction, = sess.run(prob_tensor, {input_node: [img_input] }) + prediction = prediction.reshape(net_h, net_w) + prediction = cv2.resize(prediction, (img.shape[1], img.shape[0]), interpolation=cv2.INTER_CUBIC) + + # output + filename = os.path.join( + output_path, os.path.splitext(os.path.basename(img_name))[0] + ) + utils.write_depth(filename, prediction, bits=2) + + except KeyError: + print ("Couldn't find input node: ' + input_node + ' or output layer: " + output_layer + ".") + exit(-1) + + print("finished") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument('-i', '--input_path', + default='input', + help='folder with input images' + ) + + parser.add_argument('-o', '--output_path', + default='output', + help='folder for output images' + ) + + parser.add_argument('-m', '--model_weights', + default='model-f6b98070.pb', + help='path to the trained weights of model' + ) + + parser.add_argument('-t', '--model_type', + default='large', + help='model type: large or small' + ) + + args = parser.parse_args() + + # compute depth maps + run(args.input_path, args.output_path, args.model_weights, args.model_type) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/transforms.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/transforms.py new file mode 100644 index 0000000000000000000000000000000000000000..37d68afcebf67defadd7dfb2ff5494f257876575 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/transforms.py @@ -0,0 +1,234 @@ +import numpy as np +import cv2 +import math + + +def apply_min_size(sample, size, image_interpolation_method=cv2.INTER_AREA): + """Rezise the sample to ensure the given size. Keeps aspect ratio. + + Args: + sample (dict): sample + size (tuple): image size + + Returns: + tuple: new size + """ + shape = list(sample["disparity"].shape) + + if shape[0] >= size[0] and shape[1] >= size[1]: + return sample + + scale = [0, 0] + scale[0] = size[0] / shape[0] + scale[1] = size[1] / shape[1] + + scale = max(scale) + + shape[0] = math.ceil(scale * shape[0]) + shape[1] = math.ceil(scale * shape[1]) + + # resize + sample["image"] = cv2.resize( + sample["image"], tuple(shape[::-1]), interpolation=image_interpolation_method + ) + + sample["disparity"] = cv2.resize( + sample["disparity"], tuple(shape[::-1]), interpolation=cv2.INTER_NEAREST + ) + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + tuple(shape[::-1]), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return tuple(shape) + + +class Resize(object): + """Resize sample to given size (width, height). + """ + + def __init__( + self, + width, + height, + resize_target=True, + keep_aspect_ratio=False, + ensure_multiple_of=1, + resize_method="lower_bound", + image_interpolation_method=cv2.INTER_AREA, + ): + """Init. + + Args: + width (int): desired output width + height (int): desired output height + resize_target (bool, optional): + True: Resize the full sample (image, mask, target). + False: Resize image only. + Defaults to True. + keep_aspect_ratio (bool, optional): + True: Keep the aspect ratio of the input sample. + Output sample might not have the given width and height, and + resize behaviour depends on the parameter 'resize_method'. + Defaults to False. + ensure_multiple_of (int, optional): + Output width and height is constrained to be multiple of this parameter. + Defaults to 1. + resize_method (str, optional): + "lower_bound": Output will be at least as large as the given size. + "upper_bound": Output will be at max as large as the given size. (Output size might be smaller than given size.) + "minimal": Scale as least as possible. (Output size might be smaller than given size.) + Defaults to "lower_bound". + """ + self.__width = width + self.__height = height + + self.__resize_target = resize_target + self.__keep_aspect_ratio = keep_aspect_ratio + self.__multiple_of = ensure_multiple_of + self.__resize_method = resize_method + self.__image_interpolation_method = image_interpolation_method + + def constrain_to_multiple_of(self, x, min_val=0, max_val=None): + y = (np.round(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if max_val is not None and y > max_val: + y = (np.floor(x / self.__multiple_of) * self.__multiple_of).astype(int) + + if y < min_val: + y = (np.ceil(x / self.__multiple_of) * self.__multiple_of).astype(int) + + return y + + def get_size(self, width, height): + # determine new height and width + scale_height = self.__height / height + scale_width = self.__width / width + + if self.__keep_aspect_ratio: + if self.__resize_method == "lower_bound": + # scale such that output size is lower bound + if scale_width > scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "upper_bound": + # scale such that output size is upper bound + if scale_width < scale_height: + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + elif self.__resize_method == "minimal": + # scale as least as possbile + if abs(1 - scale_width) < abs(1 - scale_height): + # fit width + scale_height = scale_width + else: + # fit height + scale_width = scale_height + else: + raise ValueError( + f"resize_method {self.__resize_method} not implemented" + ) + + if self.__resize_method == "lower_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, min_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, min_val=self.__width + ) + elif self.__resize_method == "upper_bound": + new_height = self.constrain_to_multiple_of( + scale_height * height, max_val=self.__height + ) + new_width = self.constrain_to_multiple_of( + scale_width * width, max_val=self.__width + ) + elif self.__resize_method == "minimal": + new_height = self.constrain_to_multiple_of(scale_height * height) + new_width = self.constrain_to_multiple_of(scale_width * width) + else: + raise ValueError(f"resize_method {self.__resize_method} not implemented") + + return (new_width, new_height) + + def __call__(self, sample): + width, height = self.get_size( + sample["image"].shape[1], sample["image"].shape[0] + ) + + # resize sample + sample["image"] = cv2.resize( + sample["image"], + (width, height), + interpolation=self.__image_interpolation_method, + ) + + if self.__resize_target: + if "disparity" in sample: + sample["disparity"] = cv2.resize( + sample["disparity"], + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + + if "depth" in sample: + sample["depth"] = cv2.resize( + sample["depth"], (width, height), interpolation=cv2.INTER_NEAREST + ) + + sample["mask"] = cv2.resize( + sample["mask"].astype(np.float32), + (width, height), + interpolation=cv2.INTER_NEAREST, + ) + sample["mask"] = sample["mask"].astype(bool) + + return sample + + +class NormalizeImage(object): + """Normlize image by given mean and std. + """ + + def __init__(self, mean, std): + self.__mean = mean + self.__std = std + + def __call__(self, sample): + sample["image"] = (sample["image"] - self.__mean) / self.__std + + return sample + + +class PrepareForNet(object): + """Prepare sample for usage as network input. + """ + + def __init__(self): + pass + + def __call__(self, sample): + image = np.transpose(sample["image"], (2, 0, 1)) + sample["image"] = np.ascontiguousarray(image).astype(np.float32) + + if "mask" in sample: + sample["mask"] = sample["mask"].astype(np.float32) + sample["mask"] = np.ascontiguousarray(sample["mask"]) + + if "disparity" in sample: + disparity = sample["disparity"].astype(np.float32) + sample["disparity"] = np.ascontiguousarray(disparity) + + if "depth" in sample: + depth = sample["depth"].astype(np.float32) + sample["depth"] = np.ascontiguousarray(depth) + + return sample diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/utils.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..cdc1c60e3291ebc1ba57ab0cc4788dcb02fecd76 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/tf/utils.py @@ -0,0 +1,82 @@ +import numpy as np +import sys +import cv2 + + +def write_pfm(path, image, scale=1): + """Write pfm file. + Args: + path (str): pathto file + image (array): data + scale (int, optional): Scale. Defaults to 1. + """ + + with open(path, "wb") as file: + color = None + + if image.dtype.name != "float32": + raise Exception("Image dtype must be float32.") + + image = np.flipud(image) + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif ( + len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1 + ): # greyscale + color = False + else: + raise Exception("Image must have H x W x 3, H x W x 1 or H x W dimensions.") + + file.write("PF\n" if color else "Pf\n".encode()) + file.write("%d %d\n".encode() % (image.shape[1], image.shape[0])) + + endian = image.dtype.byteorder + + if endian == "<" or endian == "=" and sys.byteorder == "little": + scale = -scale + + file.write("%f\n".encode() % scale) + + image.tofile(file) + +def read_image(path): + """Read image and output RGB image (0-1). + Args: + path (str): path to file + Returns: + array: RGB image (0-1) + """ + img = cv2.imread(path) + + if img.ndim == 2: + img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) / 255.0 + + return img + +def write_depth(path, depth, bits=1): + """Write depth map to pfm and png file. + Args: + path (str): filepath without extension + depth (array): depth + """ + write_pfm(path + ".pfm", depth.astype(np.float32)) + + depth_min = depth.min() + depth_max = depth.max() + + max_val = (2**(8*bits))-1 + + if depth_max - depth_min > np.finfo("float").eps: + out = max_val * (depth - depth_min) / (depth_max - depth_min) + else: + out = 0 + + if bits == 1: + cv2.imwrite(path + ".png", out.astype("uint8")) + elif bits == 2: + cv2.imwrite(path + ".png", out.astype("uint16")) + + return \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/utils.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d836b7e196be6db8c0b145fe3b03357a10b4725d --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/utils.py @@ -0,0 +1,199 @@ +"""Utils for monoDepth. +""" +import sys +import re +import numpy as np +import cv2 +import torch + + +def read_pfm(path): + """Read pfm file. + + Args: + path (str): path to file + + Returns: + tuple: (data, scale) + """ + with open(path, "rb") as file: + + color = None + width = None + height = None + scale = None + endian = None + + header = file.readline().rstrip() + if header.decode("ascii") == "PF": + color = True + elif header.decode("ascii") == "Pf": + color = False + else: + raise Exception("Not a PFM file: " + path) + + dim_match = re.match(r"^(\d+)\s(\d+)\s$", file.readline().decode("ascii")) + if dim_match: + width, height = list(map(int, dim_match.groups())) + else: + raise Exception("Malformed PFM header.") + + scale = float(file.readline().decode("ascii").rstrip()) + if scale < 0: + # little-endian + endian = "<" + scale = -scale + else: + # big-endian + endian = ">" + + data = np.fromfile(file, endian + "f") + shape = (height, width, 3) if color else (height, width) + + data = np.reshape(data, shape) + data = np.flipud(data) + + return data, scale + + +def write_pfm(path, image, scale=1): + """Write pfm file. + + Args: + path (str): pathto file + image (array): data + scale (int, optional): Scale. Defaults to 1. + """ + + with open(path, "wb") as file: + color = None + + if image.dtype.name != "float32": + raise Exception("Image dtype must be float32.") + + image = np.flipud(image) + + if len(image.shape) == 3 and image.shape[2] == 3: # color image + color = True + elif ( + len(image.shape) == 2 or len(image.shape) == 3 and image.shape[2] == 1 + ): # greyscale + color = False + else: + raise Exception("Image must have H x W x 3, H x W x 1 or H x W dimensions.") + + file.write("PF\n" if color else "Pf\n".encode()) + file.write("%d %d\n".encode() % (image.shape[1], image.shape[0])) + + endian = image.dtype.byteorder + + if endian == "<" or endian == "=" and sys.byteorder == "little": + scale = -scale + + file.write("%f\n".encode() % scale) + + image.tofile(file) + + +def read_image(path): + """Read image and output RGB image (0-1). + + Args: + path (str): path to file + + Returns: + array: RGB image (0-1) + """ + img = cv2.imread(path) + + if img.ndim == 2: + img = cv2.cvtColor(img, cv2.COLOR_GRAY2BGR) + + img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) / 255.0 + + return img + + +def resize_image(img): + """Resize image and make it fit for network. + + Args: + img (array): image + + Returns: + tensor: data ready for network + """ + height_orig = img.shape[0] + width_orig = img.shape[1] + + if width_orig > height_orig: + scale = width_orig / 384 + else: + scale = height_orig / 384 + + height = (np.ceil(height_orig / scale / 32) * 32).astype(int) + width = (np.ceil(width_orig / scale / 32) * 32).astype(int) + + img_resized = cv2.resize(img, (width, height), interpolation=cv2.INTER_AREA) + + img_resized = ( + torch.from_numpy(np.transpose(img_resized, (2, 0, 1))).contiguous().float() + ) + img_resized = img_resized.unsqueeze(0) + + return img_resized + + +def resize_depth(depth, width, height): + """Resize depth map and bring to CPU (numpy). + + Args: + depth (tensor): depth + width (int): image width + height (int): image height + + Returns: + array: processed depth + """ + depth = torch.squeeze(depth[0, :, :, :]).to("cpu") + + depth_resized = cv2.resize( + depth.numpy(), (width, height), interpolation=cv2.INTER_CUBIC + ) + + return depth_resized + +def write_depth(path, depth, grayscale, bits=1): + """Write depth map to png file. + + Args: + path (str): filepath without extension + depth (array): depth + grayscale (bool): use a grayscale colormap? + """ + if not grayscale: + bits = 1 + + if not np.isfinite(depth).all(): + depth=np.nan_to_num(depth, nan=0.0, posinf=0.0, neginf=0.0) + print("WARNING: Non-finite depth values present") + + depth_min = depth.min() + depth_max = depth.max() + + max_val = (2**(8*bits))-1 + + if depth_max - depth_min > np.finfo("float").eps: + out = max_val * (depth - depth_min) / (depth_max - depth_min) + else: + out = np.zeros(depth.shape, dtype=depth.dtype) + + if not grayscale: + out = cv2.applyColorMap(np.uint8(out), cv2.COLORMAP_INFERNO) + + if bits == 1: + cv2.imwrite(path + ".png", out.astype("uint8")) + elif bits == 2: + cv2.imwrite(path + ".png", out.astype("uint16")) + + return diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/weights/.placeholder b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/base_models/midas_repo/weights/.placeholder new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/builder.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..a7c6f7bb6fa39af071b4d6ca267b53084b3ee5f6 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/builder.py @@ -0,0 +1,51 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +from importlib import import_module +from .depth_model import DepthModel + +def build_model(config) -> DepthModel: + """Builds a model from a config. The model is specified by the model name and version in the config. The model is then constructed using the build_from_config function of the model interface. + This function should be used to construct models for training and evaluation. + + Args: + config (dict): Config dict. Config is constructed in utils/config.py. Each model has its own config file(s) saved in its root model folder. + + Returns: + torch.nn.Module: Model corresponding to name and version as specified in config + """ + module_name = f"zoedepth.models.{config.model}" + try: + module = import_module(module_name) + except ModuleNotFoundError as e: + # print the original error message + print(e) + raise ValueError( + f"Model {config.model} not found. Refer above error for details.") from e + try: + get_version = getattr(module, "get_version") + except AttributeError as e: + raise ValueError( + f"Model {config.model} has no get_version function.") from e + return get_version(config.version_name).build_from_config(config) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/depth_model.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/depth_model.py new file mode 100644 index 0000000000000000000000000000000000000000..982750b0c4c2fc6f25aae75e0a7002f96dd8a87d --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/depth_model.py @@ -0,0 +1,148 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from torchvision import transforms +import PIL.Image +from PIL import Image +from typing import Union + + +class DepthModel(nn.Module): + def __init__(self): + super().__init__() + self.device = 'cpu' + + def forward(self, x, *args, **kwargs): + raise NotImplementedError + + def _infer(self, x: torch.Tensor): + """ + Inference interface for the model + Args: + x (torch.Tensor): input tensor of shape (b, c, h, w) + Returns: + torch.Tensor: output tensor of shape (b, 1, h, w) + """ + return self(x)['metric_depth'] + + def _infer_with_pad_aug(self, x: torch.Tensor, pad_input: bool=True, fh: float=3, fw: float=3, upsampling_mode: str='bicubic', padding_mode="reflect", **kwargs) -> torch.Tensor: + """ + Inference interface for the model with padding augmentation + Padding augmentation fixes the boundary artifacts in the output depth map. + Boundary artifacts are sometimes caused by the fact that the model is trained on NYU raw dataset which has a black or white border around the image. + This augmentation pads the input image and crops the prediction back to the original size / view. + + Note: This augmentation is not required for the models trained with 'avoid_boundary'=True. + Args: + x (torch.Tensor): input tensor of shape (b, c, h, w) + pad_input (bool, optional): whether to pad the input or not. Defaults to True. + fh (float, optional): height padding factor. The padding is calculated as sqrt(h/2) * fh. Defaults to 3. + fw (float, optional): width padding factor. The padding is calculated as sqrt(w/2) * fw. Defaults to 3. + upsampling_mode (str, optional): upsampling mode. Defaults to 'bicubic'. + padding_mode (str, optional): padding mode. Defaults to "reflect". + Returns: + torch.Tensor: output tensor of shape (b, 1, h, w) + """ + # assert x is nchw and c = 3 + assert x.dim() == 4, "x must be 4 dimensional, got {}".format(x.dim()) + assert x.shape[1] == 3, "x must have 3 channels, got {}".format(x.shape[1]) + + if pad_input: + assert fh > 0 or fw > 0, "atlease one of fh and fw must be greater than 0" + pad_h = int(np.sqrt(x.shape[2]/2) * fh) + pad_w = int(np.sqrt(x.shape[3]/2) * fw) + padding = [pad_w, pad_w] + if pad_h > 0: + padding += [pad_h, pad_h] + + x = F.pad(x, padding, mode=padding_mode, **kwargs) + out = self._infer(x) + if out.shape[-2:] != x.shape[-2:]: + out = F.interpolate(out, size=(x.shape[2], x.shape[3]), mode=upsampling_mode, align_corners=False) + if pad_input: + # crop to the original size, handling the case where pad_h and pad_w is 0 + if pad_h > 0: + out = out[:, :, pad_h:-pad_h,:] + if pad_w > 0: + out = out[:, :, :, pad_w:-pad_w] + return out + + def infer_with_flip_aug(self, x, pad_input: bool=True, **kwargs) -> torch.Tensor: + """ + Inference interface for the model with horizontal flip augmentation + Horizontal flip augmentation improves the accuracy of the model by averaging the output of the model with and without horizontal flip. + Args: + x (torch.Tensor): input tensor of shape (b, c, h, w) + pad_input (bool, optional): whether to use padding augmentation. Defaults to True. + Returns: + torch.Tensor: output tensor of shape (b, 1, h, w) + """ + # infer with horizontal flip and average + out = self._infer_with_pad_aug(x, pad_input=pad_input, **kwargs) + out_flip = self._infer_with_pad_aug(torch.flip(x, dims=[3]), pad_input=pad_input, **kwargs) + out = (out + torch.flip(out_flip, dims=[3])) / 2 + return out + + def infer(self, x, pad_input: bool=True, with_flip_aug: bool=True, **kwargs) -> torch.Tensor: + """ + Inference interface for the model + Args: + x (torch.Tensor): input tensor of shape (b, c, h, w) + pad_input (bool, optional): whether to use padding augmentation. Defaults to True. + with_flip_aug (bool, optional): whether to use horizontal flip augmentation. Defaults to True. + Returns: + torch.Tensor: output tensor of shape (b, 1, h, w) + """ + if with_flip_aug: + return self.infer_with_flip_aug(x, pad_input=pad_input, **kwargs) + else: + return self._infer_with_pad_aug(x, pad_input=pad_input, **kwargs) + + @torch.no_grad() + def infer_pil(self, pil_img, pad_input: bool=True, with_flip_aug: bool=True, output_type: str="numpy", **kwargs) -> Union[np.ndarray, PIL.Image.Image, torch.Tensor]: + """ + Inference interface for the model for PIL image + Args: + pil_img (PIL.Image.Image): input PIL image + pad_input (bool, optional): whether to use padding augmentation. Defaults to True. + with_flip_aug (bool, optional): whether to use horizontal flip augmentation. Defaults to True. + output_type (str, optional): output type. Supported values are 'numpy', 'pil' and 'tensor'. Defaults to "numpy". + """ + x = transforms.ToTensor()(pil_img).unsqueeze(0).to(self.device) + out_tensor = self.infer(x, pad_input=pad_input, with_flip_aug=with_flip_aug, **kwargs) + if output_type == "numpy": + return out_tensor.squeeze().cpu().numpy() + elif output_type == "pil": + # uint16 is required for depth pil image + out_16bit_numpy = (out_tensor.squeeze().cpu().numpy()*256).astype(np.uint16) + return Image.fromarray(out_16bit_numpy) + elif output_type == "tensor": + return out_tensor.squeeze().cpu() + else: + raise ValueError(f"output_type {output_type} not supported. Supported values are 'numpy', 'pil' and 'tensor'") + \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/attractor.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/attractor.py new file mode 100644 index 0000000000000000000000000000000000000000..b5e3473ca6e2271dc28666314cf8f92f52f7e3c6 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/attractor.py @@ -0,0 +1,208 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn + + +@torch.jit.script +def exp_attractor(dx, alpha: float = 300, gamma: int = 2): + """Exponential attractor: dc = exp(-alpha*|dx|^gamma) * dx , where dx = a - c, a = attractor point, c = bin center, dc = shift in bin centermmary for exp_attractor + + Args: + dx (torch.Tensor): The difference tensor dx = Ai - Cj, where Ai is the attractor point and Cj is the bin center. + alpha (float, optional): Proportional Attractor strength. Determines the absolute strength. Lower alpha = greater attraction. Defaults to 300. + gamma (int, optional): Exponential Attractor strength. Determines the "region of influence" and indirectly number of bin centers affected. Lower gamma = farther reach. Defaults to 2. + + Returns: + torch.Tensor : Delta shifts - dc; New bin centers = Old bin centers + dc + """ + return torch.exp(-alpha*(torch.abs(dx)**gamma)) * (dx) + + +@torch.jit.script +def inv_attractor(dx, alpha: float = 300, gamma: int = 2): + """Inverse attractor: dc = dx / (1 + alpha*dx^gamma), where dx = a - c, a = attractor point, c = bin center, dc = shift in bin center + This is the default one according to the accompanying paper. + + Args: + dx (torch.Tensor): The difference tensor dx = Ai - Cj, where Ai is the attractor point and Cj is the bin center. + alpha (float, optional): Proportional Attractor strength. Determines the absolute strength. Lower alpha = greater attraction. Defaults to 300. + gamma (int, optional): Exponential Attractor strength. Determines the "region of influence" and indirectly number of bin centers affected. Lower gamma = farther reach. Defaults to 2. + + Returns: + torch.Tensor: Delta shifts - dc; New bin centers = Old bin centers + dc + """ + return dx.div(1+alpha*dx.pow(gamma)) + + +class AttractorLayer(nn.Module): + def __init__(self, in_features, n_bins, n_attractors=16, mlp_dim=128, min_depth=1e-3, max_depth=10, + alpha=300, gamma=2, kind='sum', attractor_type='exp', memory_efficient=False): + """ + Attractor layer for bin centers. Bin centers are bounded on the interval (min_depth, max_depth) + """ + super().__init__() + + self.n_attractors = n_attractors + self.n_bins = n_bins + self.min_depth = min_depth + self.max_depth = max_depth + self.alpha = alpha + self.gamma = gamma + self.kind = kind + self.attractor_type = attractor_type + self.memory_efficient = memory_efficient + + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.ReLU(inplace=True), + nn.Conv2d(mlp_dim, n_attractors*2, 1, 1, 0), # x2 for linear norm + nn.ReLU(inplace=True) + ) + + def forward(self, x, b_prev, prev_b_embedding=None, interpolate=True, is_for_query=False): + """ + Args: + x (torch.Tensor) : feature block; shape - n, c, h, w + b_prev (torch.Tensor) : previous bin centers normed; shape - n, prev_nbins, h, w + + Returns: + tuple(torch.Tensor,torch.Tensor) : new bin centers normed and scaled; shape - n, nbins, h, w + """ + if prev_b_embedding is not None: + if interpolate: + prev_b_embedding = nn.functional.interpolate( + prev_b_embedding, x.shape[-2:], mode='bilinear', align_corners=True) + x = x + prev_b_embedding + + A = self._net(x) + eps = 1e-3 + A = A + eps + n, c, h, w = A.shape + A = A.view(n, self.n_attractors, 2, h, w) + A_normed = A / A.sum(dim=2, keepdim=True) # n, a, 2, h, w + A_normed = A[:, :, 0, ...] # n, na, h, w + + b_prev = nn.functional.interpolate( + b_prev, (h, w), mode='bilinear', align_corners=True) + b_centers = b_prev + + if self.attractor_type == 'exp': + dist = exp_attractor + else: + dist = inv_attractor + + if not self.memory_efficient: + func = {'mean': torch.mean, 'sum': torch.sum}[self.kind] + # .shape N, nbins, h, w + delta_c = func(dist(A_normed.unsqueeze( + 2) - b_centers.unsqueeze(1)), dim=1) + else: + delta_c = torch.zeros_like(b_centers, device=b_centers.device) + for i in range(self.n_attractors): + # .shape N, nbins, h, w + delta_c += dist(A_normed[:, i, ...].unsqueeze(1) - b_centers) + + if self.kind == 'mean': + delta_c = delta_c / self.n_attractors + + b_new_centers = b_centers + delta_c + B_centers = (self.max_depth - self.min_depth) * \ + b_new_centers + self.min_depth + B_centers, _ = torch.sort(B_centers, dim=1) + B_centers = torch.clip(B_centers, self.min_depth, self.max_depth) + return b_new_centers, B_centers + + +class AttractorLayerUnnormed(nn.Module): + def __init__(self, in_features, n_bins, n_attractors=16, mlp_dim=128, min_depth=1e-3, max_depth=10, + alpha=300, gamma=2, kind='sum', attractor_type='exp', memory_efficient=False): + """ + Attractor layer for bin centers. Bin centers are unbounded + """ + super().__init__() + + self.n_attractors = n_attractors + self.n_bins = n_bins + self.min_depth = min_depth + self.max_depth = max_depth + self.alpha = alpha + self.gamma = gamma + self.kind = kind + self.attractor_type = attractor_type + self.memory_efficient = memory_efficient + + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.ReLU(inplace=True), + nn.Conv2d(mlp_dim, n_attractors, 1, 1, 0), + nn.Softplus() + ) + + def forward(self, x, b_prev, prev_b_embedding=None, interpolate=True, is_for_query=False): + """ + Args: + x (torch.Tensor) : feature block; shape - n, c, h, w + b_prev (torch.Tensor) : previous bin centers normed; shape - n, prev_nbins, h, w + + Returns: + tuple(torch.Tensor,torch.Tensor) : new bin centers unbounded; shape - n, nbins, h, w. Two outputs just to keep the API consistent with the normed version + """ + if prev_b_embedding is not None: + if interpolate: + prev_b_embedding = nn.functional.interpolate( + prev_b_embedding, x.shape[-2:], mode='bilinear', align_corners=True) + x = x + prev_b_embedding + + A = self._net(x) + n, c, h, w = A.shape + + b_prev = nn.functional.interpolate( + b_prev, (h, w), mode='bilinear', align_corners=True) + b_centers = b_prev + + if self.attractor_type == 'exp': + dist = exp_attractor + else: + dist = inv_attractor + + if not self.memory_efficient: + func = {'mean': torch.mean, 'sum': torch.sum}[self.kind] + # .shape N, nbins, h, w + delta_c = func( + dist(A.unsqueeze(2) - b_centers.unsqueeze(1)), dim=1) + else: + delta_c = torch.zeros_like(b_centers, device=b_centers.device) + for i in range(self.n_attractors): + delta_c += dist(A[:, i, ...].unsqueeze(1) - + b_centers) # .shape N, nbins, h, w + + if self.kind == 'mean': + delta_c = delta_c / self.n_attractors + + b_new_centers = b_centers + delta_c + B_centers = b_new_centers + + return b_new_centers, B_centers diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/dist_layers.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/dist_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..d5ff793e94aeb43aa554ef8c2392080df5572e19 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/dist_layers.py @@ -0,0 +1,121 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn + + +def log_binom(n, k, eps=1e-7): + """ log(nCk) using stirling approximation """ + n = n + eps + k = k + eps + return n * torch.log(n) - k * torch.log(k) - (n-k) * torch.log(n-k+eps) + + +class LogBinomial(nn.Module): + def __init__(self, n_classes=256, act=torch.softmax): + """Compute log binomial distribution for n_classes + + Args: + n_classes (int, optional): number of output classes. Defaults to 256. + """ + super().__init__() + self.K = n_classes + self.act = act + self.register_buffer('k_idx', torch.arange( + 0, n_classes).view(1, -1, 1, 1)) + self.register_buffer('K_minus_1', torch.Tensor( + [self.K-1]).view(1, -1, 1, 1)) + + def forward(self, x, t=1., eps=1e-4): + """Compute log binomial distribution for x + + Args: + x (torch.Tensor - NCHW): probabilities + t (float, torch.Tensor - NCHW, optional): Temperature of distribution. Defaults to 1.. + eps (float, optional): Small number for numerical stability. Defaults to 1e-4. + + Returns: + torch.Tensor -NCHW: log binomial distribution logbinomial(p;t) + """ + if x.ndim == 3: + x = x.unsqueeze(1) # make it nchw + + one_minus_x = torch.clamp(1 - x, eps, 1) + x = torch.clamp(x, eps, 1) + y = log_binom(self.K_minus_1, self.k_idx) + self.k_idx * \ + torch.log(x) + (self.K - 1 - self.k_idx) * torch.log(one_minus_x) + return self.act(y/t, dim=1) + + +class ConditionalLogBinomial(nn.Module): + def __init__(self, in_features, condition_dim, n_classes=256, bottleneck_factor=2, p_eps=1e-4, max_temp=50, min_temp=1e-7, act=torch.softmax): + """Conditional Log Binomial distribution + + Args: + in_features (int): number of input channels in main feature + condition_dim (int): number of input channels in condition feature + n_classes (int, optional): Number of classes. Defaults to 256. + bottleneck_factor (int, optional): Hidden dim factor. Defaults to 2. + p_eps (float, optional): small eps value. Defaults to 1e-4. + max_temp (float, optional): Maximum temperature of output distribution. Defaults to 50. + min_temp (float, optional): Minimum temperature of output distribution. Defaults to 1e-7. + """ + super().__init__() + self.p_eps = p_eps + self.max_temp = max_temp + self.min_temp = min_temp + self.log_binomial_transform = LogBinomial(n_classes, act=act) + bottleneck = (in_features + condition_dim) // bottleneck_factor + self.mlp = nn.Sequential( + nn.Conv2d(in_features + condition_dim, bottleneck, + kernel_size=1, stride=1, padding=0), + nn.GELU(), + # 2 for p linear norm, 2 for t linear norm + nn.Conv2d(bottleneck, 2+2, kernel_size=1, stride=1, padding=0), + nn.Softplus() + ) + + def forward(self, x, cond): + """Forward pass + + Args: + x (torch.Tensor - NCHW): Main feature + cond (torch.Tensor - NCHW): condition feature + + Returns: + torch.Tensor: Output log binomial distribution + """ + pt = self.mlp(torch.concat((x, cond), dim=1)) + p, t = pt[:, :2, ...], pt[:, 2:, ...] + + p = p + self.p_eps + p = p[:, 0, ...] / (p[:, 0, ...] + p[:, 1, ...]) + + t = t + self.p_eps + t = t[:, 0, ...] / (t[:, 0, ...] + t[:, 1, ...]) + t = t.unsqueeze(1) + t = (self.max_temp - self.min_temp) * t + self.min_temp + + return self.log_binomial_transform(p, t) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/localbins_layers.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/localbins_layers.py new file mode 100644 index 0000000000000000000000000000000000000000..cda7b1f9c74ac0d54733ef6d1dcadc1c62ab4647 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/localbins_layers.py @@ -0,0 +1,169 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn + + +class SeedBinRegressor(nn.Module): + def __init__(self, in_features, n_bins=16, mlp_dim=256, min_depth=1e-3, max_depth=10): + """Bin center regressor network. Bin centers are bounded on (min_depth, max_depth) interval. + + Args: + in_features (int): input channels + n_bins (int, optional): Number of bin centers. Defaults to 16. + mlp_dim (int, optional): Hidden dimension. Defaults to 256. + min_depth (float, optional): Min depth value. Defaults to 1e-3. + max_depth (float, optional): Max depth value. Defaults to 10. + """ + super().__init__() + self.version = "1_1" + self.min_depth = min_depth + self.max_depth = max_depth + + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.ReLU(inplace=True), + nn.Conv2d(mlp_dim, n_bins, 1, 1, 0), + nn.ReLU(inplace=True) + ) + + def forward(self, x): + """ + Returns tensor of bin_width vectors (centers). One vector b for every pixel + """ + B = self._net(x) + eps = 1e-3 + B = B + eps + B_widths_normed = B / B.sum(dim=1, keepdim=True) + B_widths = (self.max_depth - self.min_depth) * \ + B_widths_normed # .shape NCHW + # pad has the form (left, right, top, bottom, front, back) + B_widths = nn.functional.pad( + B_widths, (0, 0, 0, 0, 1, 0), mode='constant', value=self.min_depth) + B_edges = torch.cumsum(B_widths, dim=1) # .shape NCHW + + B_centers = 0.5 * (B_edges[:, :-1, ...] + B_edges[:, 1:, ...]) + return B_widths_normed, B_centers + + +class SeedBinRegressorUnnormed(nn.Module): + def __init__(self, in_features, n_bins=16, mlp_dim=256, min_depth=1e-3, max_depth=10): + """Bin center regressor network. Bin centers are unbounded + + Args: + in_features (int): input channels + n_bins (int, optional): Number of bin centers. Defaults to 16. + mlp_dim (int, optional): Hidden dimension. Defaults to 256. + min_depth (float, optional): Not used. (for compatibility with SeedBinRegressor) + max_depth (float, optional): Not used. (for compatibility with SeedBinRegressor) + """ + super().__init__() + self.version = "1_1" + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.ReLU(inplace=True), + nn.Conv2d(mlp_dim, n_bins, 1, 1, 0), + nn.Softplus() + ) + + def forward(self, x): + """ + Returns tensor of bin_width vectors (centers). One vector b for every pixel + """ + B_centers = self._net(x) + return B_centers, B_centers + + +class Projector(nn.Module): + def __init__(self, in_features, out_features, mlp_dim=128): + """Projector MLP + + Args: + in_features (int): input channels + out_features (int): output channels + mlp_dim (int, optional): hidden dimension. Defaults to 128. + """ + super().__init__() + + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.ReLU(inplace=True), + nn.Conv2d(mlp_dim, out_features, 1, 1, 0), + ) + + def forward(self, x): + return self._net(x) + + + +class LinearSplitter(nn.Module): + def __init__(self, in_features, prev_nbins, split_factor=2, mlp_dim=128, min_depth=1e-3, max_depth=10): + super().__init__() + + self.prev_nbins = prev_nbins + self.split_factor = split_factor + self.min_depth = min_depth + self.max_depth = max_depth + + self._net = nn.Sequential( + nn.Conv2d(in_features, mlp_dim, 1, 1, 0), + nn.GELU(), + nn.Conv2d(mlp_dim, prev_nbins * split_factor, 1, 1, 0), + nn.ReLU() + ) + + def forward(self, x, b_prev, prev_b_embedding=None, interpolate=True, is_for_query=False): + """ + x : feature block; shape - n, c, h, w + b_prev : previous bin widths normed; shape - n, prev_nbins, h, w + """ + if prev_b_embedding is not None: + if interpolate: + prev_b_embedding = nn.functional.interpolate(prev_b_embedding, x.shape[-2:], mode='bilinear', align_corners=True) + x = x + prev_b_embedding + S = self._net(x) + eps = 1e-3 + S = S + eps + n, c, h, w = S.shape + S = S.view(n, self.prev_nbins, self.split_factor, h, w) + S_normed = S / S.sum(dim=2, keepdim=True) # fractional splits + + b_prev = nn.functional.interpolate(b_prev, (h,w), mode='bilinear', align_corners=True) + + + b_prev = b_prev / b_prev.sum(dim=1, keepdim=True) # renormalize for gurantees + # print(b_prev.shape, S_normed.shape) + # if is_for_query:(1).expand(-1, b_prev.size(0)//n, -1, -1, -1, -1).flatten(0,1) # TODO ? can replace all this with a single torch.repeat? + b = b_prev.unsqueeze(2) * S_normed + b = b.flatten(1,2) # .shape n, prev_nbins * split_factor, h, w + + # calculate bin centers for loss calculation + B_widths = (self.max_depth - self.min_depth) * b # .shape N, nprev * splitfactor, H, W + # pad has the form (left, right, top, bottom, front, back) + B_widths = nn.functional.pad(B_widths, (0,0,0,0,1,0), mode='constant', value=self.min_depth) + B_edges = torch.cumsum(B_widths, dim=1) # .shape NCHW + + B_centers = 0.5 * (B_edges[:, :-1, ...] + B_edges[:,1:,...]) + return b, B_centers \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/patch_transformer.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/patch_transformer.py new file mode 100644 index 0000000000000000000000000000000000000000..eacac4d38809db2b3c669d0134f91645225af531 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/layers/patch_transformer.py @@ -0,0 +1,91 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch +import torch.nn as nn + + +class PatchTransformerEncoder(nn.Module): + def __init__(self, in_channels, patch_size=10, embedding_dim=128, num_heads=4, use_class_token=False): + """ViT-like transformer block + + Args: + in_channels (int): Input channels + patch_size (int, optional): patch size. Defaults to 10. + embedding_dim (int, optional): Embedding dimension in transformer model. Defaults to 128. + num_heads (int, optional): number of attention heads. Defaults to 4. + use_class_token (bool, optional): Whether to use extra token at the start for global accumulation (called as "class token"). Defaults to False. + """ + super(PatchTransformerEncoder, self).__init__() + self.use_class_token = use_class_token + encoder_layers = nn.TransformerEncoderLayer( + embedding_dim, num_heads, dim_feedforward=1024) + self.transformer_encoder = nn.TransformerEncoder( + encoder_layers, num_layers=4) # takes shape S,N,E + + self.embedding_convPxP = nn.Conv2d(in_channels, embedding_dim, + kernel_size=patch_size, stride=patch_size, padding=0) + + def positional_encoding_1d(self, sequence_length, batch_size, embedding_dim, device='cpu'): + """Generate positional encodings + + Args: + sequence_length (int): Sequence length + embedding_dim (int): Embedding dimension + + Returns: + torch.Tensor SBE: Positional encodings + """ + position = torch.arange( + 0, sequence_length, dtype=torch.float32, device=device).unsqueeze(1) + index = torch.arange( + 0, embedding_dim, 2, dtype=torch.float32, device=device).unsqueeze(0) + div_term = torch.exp(index * (-torch.log(torch.tensor(10000.0, device=device)) / embedding_dim)) + pos_encoding = position * div_term + pos_encoding = torch.cat([torch.sin(pos_encoding), torch.cos(pos_encoding)], dim=1) + pos_encoding = pos_encoding.unsqueeze(1).repeat(1, batch_size, 1) + return pos_encoding + + + def forward(self, x): + """Forward pass + + Args: + x (torch.Tensor - NCHW): Input feature tensor + + Returns: + torch.Tensor - SNE: Transformer output embeddings. S - sequence length (=HW/patch_size^2), N - batch size, E - embedding dim + """ + embeddings = self.embedding_convPxP(x).flatten( + 2) # .shape = n,c,s = n, embedding_dim, s + if self.use_class_token: + # extra special token at start ? + embeddings = nn.functional.pad(embeddings, (1, 0)) + + # change to S,N,E format required by transformer + embeddings = embeddings.permute(2, 0, 1) + S, N, E = embeddings.shape + embeddings = embeddings + self.positional_encoding_1d(S, N, E, device=embeddings.device) + x = self.transformer_encoder(embeddings) # .shape = S, N, E + return x diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/model_io.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/model_io.py new file mode 100644 index 0000000000000000000000000000000000000000..f63a9a1ff09e98da78a9a3da63e58509471f728d --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/model_io.py @@ -0,0 +1,92 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import torch + +def load_state_dict(model, state_dict): + """Load state_dict into model, handling DataParallel and DistributedDataParallel. Also checks for "model" key in state_dict. + + DataParallel prefixes state_dict keys with 'module.' when saving. + If the model is not a DataParallel model but the state_dict is, then prefixes are removed. + If the model is a DataParallel model but the state_dict is not, then prefixes are added. + """ + state_dict = state_dict.get('model', state_dict) + # if model is a DataParallel model, then state_dict keys are prefixed with 'module.' + + do_prefix = isinstance( + model, (torch.nn.DataParallel, torch.nn.parallel.DistributedDataParallel)) + state = {} + for k, v in state_dict.items(): + if k.startswith('module.') and not do_prefix: + k = k[7:] + + if not k.startswith('module.') and do_prefix: + k = 'module.' + k + + state[k] = v + + model.load_state_dict(state) + print("Loaded successfully") + return model + + +def load_wts(model, checkpoint_path): + ckpt = torch.load(checkpoint_path, map_location='cpu') + return load_state_dict(model, ckpt) + + +def load_state_dict_from_url(model, url, **kwargs): + state_dict = torch.hub.load_state_dict_from_url(url, map_location='cpu', **kwargs) + return load_state_dict(model, state_dict) + + +def load_state_from_resource(model, resource: str): + """Loads weights to the model from a given resource. A resource can be of following types: + 1. URL. Prefixed with "url::" + e.g. url::http(s)://url.resource.com/ckpt.pt + + 2. Local path. Prefixed with "local::" + e.g. local::/path/to/ckpt.pt + + + Args: + model (torch.nn.Module): Model + resource (str): resource string + + Returns: + torch.nn.Module: Model with loaded weights + """ + print(f"Using pretrained resource {resource}") + + if resource.startswith('url::'): + url = resource.split('url::')[1] + return load_state_dict_from_url(model, url, progress=True) + + elif resource.startswith('local::'): + path = resource.split('local::')[1] + return load_wts(model, path) + + else: + raise ValueError("Invalid resource type, only url:: and local:: are supported") + \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth.json b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth.json new file mode 100644 index 0000000000000000000000000000000000000000..dfc9fa7b17615cf557b6ad01c8fedd6c0c32e88f --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth.json @@ -0,0 +1,58 @@ +{ + "model": { + "name": "ZoeDepth", + "version_name": "v1", + "n_bins": 64, + "bin_embedding_dim": 128, + "bin_centers_type": "softplus", + "n_attractors":[16, 8, 4, 1], + "attractor_alpha": 1000, + "attractor_gamma": 2, + "attractor_kind" : "mean", + "attractor_type" : "inv", + "midas_model_type" : "DPT_BEiT_L_384", + "min_temp": 0.0212, + "max_temp": 50.0, + "output_distribution": "logbinomial", + "memory_efficient": true, + "inverse_midas": false, + "img_size": [384, 512] + }, + + "train": { + "train_midas": true, + "use_pretrained_midas": true, + "trainer": "zoedepth", + "epochs": 5, + "bs": 16, + "optim_kwargs": {"lr": 0.000161, "wd": 0.01}, + "sched_kwargs": {"div_factor": 1, "final_div_factor": 10000, "pct_start": 0.7, "three_phase":false, "cycle_momentum": true}, + "same_lr": false, + "w_si": 1, + "w_domain": 0.2, + "w_reg": 0, + "w_grad": 0, + "avoid_boundary": false, + "random_crop": false, + "input_width": 640, + "input_height": 480, + "midas_lr_factor": 1, + "encoder_lr_factor":10, + "pos_enc_lr_factor":10, + "freeze_midas_bn": true + + }, + + "infer":{ + "train_midas": false, + "use_pretrained_midas": false, + "pretrained_resource" : null, + "force_keep_ar": true + }, + + "eval":{ + "train_midas": false, + "use_pretrained_midas": false, + "pretrained_resource" : null + } +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth_kitti.json b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth_kitti.json new file mode 100644 index 0000000000000000000000000000000000000000..3e7266ec2d7e918143f54ee728ea4d8d4e9adb11 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth/config_zoedepth_kitti.json @@ -0,0 +1,22 @@ +{ + "model": { + "bin_centers_type": "normed", + "img_size": [384, 768] + }, + + "train": { + }, + + "infer":{ + "train_midas": false, + "use_pretrained_midas": false, + "pretrained_resource" : "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_K.pt", + "force_keep_ar": true + }, + + "eval":{ + "train_midas": false, + "use_pretrained_midas": false, + "pretrained_resource" : "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_K.pt" + } +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth/zoedepth_v1.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth/zoedepth_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..4d5e8c2d272c3ae5f5ff1025aa96e6653b23d82f --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth/zoedepth_v1.py @@ -0,0 +1,250 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import itertools + +import torch +import torch.nn as nn +from ..depth_model import DepthModel +from ..base_models.midas import MidasCore +from ..layers.attractor import AttractorLayer, AttractorLayerUnnormed +from ..layers.dist_layers import ConditionalLogBinomial +from ..layers.localbins_layers import (Projector, SeedBinRegressor, + SeedBinRegressorUnnormed) +from ..model_io import load_state_from_resource + + +class ZoeDepth(DepthModel): + def __init__(self, core, n_bins=64, bin_centers_type="softplus", bin_embedding_dim=128, min_depth=1e-3, max_depth=10, + n_attractors=[16, 8, 4, 1], attractor_alpha=300, attractor_gamma=2, attractor_kind='sum', attractor_type='exp', min_temp=5, max_temp=50, train_midas=True, + midas_lr_factor=10, encoder_lr_factor=10, pos_enc_lr_factor=10, inverse_midas=False, **kwargs): + """ZoeDepth model. This is the version of ZoeDepth that has a single metric head + + Args: + core (models.base_models.midas.MidasCore): The base midas model that is used for extraction of "relative" features + n_bins (int, optional): Number of bin centers. Defaults to 64. + bin_centers_type (str, optional): "normed" or "softplus". Activation type used for bin centers. For "normed" bin centers, linear normalization trick is applied. This results in bounded bin centers. + For "softplus", softplus activation is used and thus are unbounded. Defaults to "softplus". + bin_embedding_dim (int, optional): bin embedding dimension. Defaults to 128. + min_depth (float, optional): Lower bound for normed bin centers. Defaults to 1e-3. + max_depth (float, optional): Upper bound for normed bin centers. Defaults to 10. + n_attractors (List[int], optional): Number of bin attractors at decoder layers. Defaults to [16, 8, 4, 1]. + attractor_alpha (int, optional): Proportional attractor strength. Refer to models.layers.attractor for more details. Defaults to 300. + attractor_gamma (int, optional): Exponential attractor strength. Refer to models.layers.attractor for more details. Defaults to 2. + attractor_kind (str, optional): Attraction aggregation "sum" or "mean". Defaults to 'sum'. + attractor_type (str, optional): Type of attractor to use; "inv" (Inverse attractor) or "exp" (Exponential attractor). Defaults to 'exp'. + min_temp (int, optional): Lower bound for temperature of output probability distribution. Defaults to 5. + max_temp (int, optional): Upper bound for temperature of output probability distribution. Defaults to 50. + train_midas (bool, optional): Whether to train "core", the base midas model. Defaults to True. + midas_lr_factor (int, optional): Learning rate reduction factor for base midas model except its encoder and positional encodings. Defaults to 10. + encoder_lr_factor (int, optional): Learning rate reduction factor for the encoder in midas model. Defaults to 10. + pos_enc_lr_factor (int, optional): Learning rate reduction factor for positional encodings in the base midas model. Defaults to 10. + """ + super().__init__() + + self.core = core + self.max_depth = max_depth + self.min_depth = min_depth + self.min_temp = min_temp + self.bin_centers_type = bin_centers_type + + self.midas_lr_factor = midas_lr_factor + self.encoder_lr_factor = encoder_lr_factor + self.pos_enc_lr_factor = pos_enc_lr_factor + self.train_midas = train_midas + self.inverse_midas = inverse_midas + + if self.encoder_lr_factor <= 0: + self.core.freeze_encoder( + freeze_rel_pos=self.pos_enc_lr_factor <= 0) + + N_MIDAS_OUT = 32 + btlnck_features = self.core.output_channels[0] + num_out_features = self.core.output_channels[1:] + + self.conv2 = nn.Conv2d(btlnck_features, btlnck_features, + kernel_size=1, stride=1, padding=0) # btlnck conv + + if bin_centers_type == "normed": + SeedBinRegressorLayer = SeedBinRegressor + Attractor = AttractorLayer + elif bin_centers_type == "softplus": + SeedBinRegressorLayer = SeedBinRegressorUnnormed + Attractor = AttractorLayerUnnormed + elif bin_centers_type == "hybrid1": + SeedBinRegressorLayer = SeedBinRegressor + Attractor = AttractorLayerUnnormed + elif bin_centers_type == "hybrid2": + SeedBinRegressorLayer = SeedBinRegressorUnnormed + Attractor = AttractorLayer + else: + raise ValueError( + "bin_centers_type should be one of 'normed', 'softplus', 'hybrid1', 'hybrid2'") + + self.seed_bin_regressor = SeedBinRegressorLayer( + btlnck_features, n_bins=n_bins, min_depth=min_depth, max_depth=max_depth) + self.seed_projector = Projector(btlnck_features, bin_embedding_dim) + self.projectors = nn.ModuleList([ + Projector(num_out, bin_embedding_dim) + for num_out in num_out_features + ]) + self.attractors = nn.ModuleList([ + Attractor(bin_embedding_dim, n_bins, n_attractors=n_attractors[i], min_depth=min_depth, max_depth=max_depth, + alpha=attractor_alpha, gamma=attractor_gamma, kind=attractor_kind, attractor_type=attractor_type) + for i in range(len(num_out_features)) + ]) + + last_in = N_MIDAS_OUT + 1 # +1 for relative depth + + # use log binomial instead of softmax + self.conditional_log_binomial = ConditionalLogBinomial( + last_in, bin_embedding_dim, n_classes=n_bins, min_temp=min_temp, max_temp=max_temp) + + def forward(self, x, return_final_centers=False, denorm=False, return_probs=False, **kwargs): + """ + Args: + x (torch.Tensor): Input image tensor of shape (B, C, H, W) + return_final_centers (bool, optional): Whether to return the final bin centers. Defaults to False. + denorm (bool, optional): Whether to denormalize the input image. This reverses ImageNet normalization as midas normalization is different. Defaults to False. + return_probs (bool, optional): Whether to return the output probability distribution. Defaults to False. + + Returns: + dict: Dictionary containing the following keys: + - rel_depth (torch.Tensor): Relative depth map of shape (B, H, W) + - metric_depth (torch.Tensor): Metric depth map of shape (B, 1, H, W) + - bin_centers (torch.Tensor): Bin centers of shape (B, n_bins). Present only if return_final_centers is True + - probs (torch.Tensor): Output probability distribution of shape (B, n_bins, H, W). Present only if return_probs is True + + """ + b, c, h, w = x.shape + # print("input shape ", x.shape) + self.orig_input_width = w + self.orig_input_height = h + rel_depth, out = self.core(x, denorm=denorm, return_rel_depth=True) + # print("output shapes", rel_depth.shape, out.shape) + + outconv_activation = out[0] + btlnck = out[1] + x_blocks = out[2:] + + x_d0 = self.conv2(btlnck) + x = x_d0 + _, seed_b_centers = self.seed_bin_regressor(x) + + if self.bin_centers_type == 'normed' or self.bin_centers_type == 'hybrid2': + b_prev = (seed_b_centers - self.min_depth) / \ + (self.max_depth - self.min_depth) + else: + b_prev = seed_b_centers + + prev_b_embedding = self.seed_projector(x) + + # unroll this loop for better performance + for projector, attractor, x in zip(self.projectors, self.attractors, x_blocks): + b_embedding = projector(x) + b, b_centers = attractor( + b_embedding, b_prev, prev_b_embedding, interpolate=True) + b_prev = b.clone() + prev_b_embedding = b_embedding.clone() + + last = outconv_activation + + if self.inverse_midas: + # invert depth followed by normalization + rel_depth = 1.0 / (rel_depth + 1e-6) + rel_depth = (rel_depth - rel_depth.min()) / \ + (rel_depth.max() - rel_depth.min()) + # concat rel depth with last. First interpolate rel depth to last size + rel_cond = rel_depth.unsqueeze(1) + rel_cond = nn.functional.interpolate( + rel_cond, size=last.shape[2:], mode='bilinear', align_corners=True) + last = torch.cat([last, rel_cond], dim=1) + + b_embedding = nn.functional.interpolate( + b_embedding, last.shape[-2:], mode='bilinear', align_corners=True) + x = self.conditional_log_binomial(last, b_embedding) + + # Now depth value is Sum px * cx , where cx are bin_centers from the last bin tensor + # print(x.shape, b_centers.shape) + b_centers = nn.functional.interpolate( + b_centers, x.shape[-2:], mode='bilinear', align_corners=True) + out = torch.sum(x * b_centers, dim=1, keepdim=True) + + # Structure output dict + output = dict(metric_depth=out) + if return_final_centers or return_probs: + output['bin_centers'] = b_centers + + if return_probs: + output['probs'] = x + + return output + + def get_lr_params(self, lr): + """ + Learning rate configuration for different layers of the model + Args: + lr (float) : Base learning rate + Returns: + list : list of parameters to optimize and their learning rates, in the format required by torch optimizers. + """ + param_conf = [] + if self.train_midas: + if self.encoder_lr_factor > 0: + param_conf.append({'params': self.core.get_enc_params_except_rel_pos( + ), 'lr': lr / self.encoder_lr_factor}) + + if self.pos_enc_lr_factor > 0: + param_conf.append( + {'params': self.core.get_rel_pos_params(), 'lr': lr / self.pos_enc_lr_factor}) + + midas_params = self.core.core.scratch.parameters() + midas_lr_factor = self.midas_lr_factor + param_conf.append( + {'params': midas_params, 'lr': lr / midas_lr_factor}) + + remaining_modules = [] + for name, child in self.named_children(): + if name != 'core': + remaining_modules.append(child) + remaining_params = itertools.chain( + *[child.parameters() for child in remaining_modules]) + + param_conf.append({'params': remaining_params, 'lr': lr}) + + return param_conf + + @staticmethod + def build(midas_model_type="DPT_BEiT_L_384", pretrained_resource=None, use_pretrained_midas=False, train_midas=False, freeze_midas_bn=True, **kwargs): + core = MidasCore.build(midas_model_type=midas_model_type, use_pretrained_midas=use_pretrained_midas, + train_midas=train_midas, fetch_features=True, freeze_bn=freeze_midas_bn, **kwargs) + model = ZoeDepth(core, **kwargs) + if pretrained_resource: + assert isinstance(pretrained_resource, str), "pretrained_resource must be a string" + model = load_state_from_resource(model, pretrained_resource) + return model + + @staticmethod + def build_from_config(config): + return ZoeDepth.build(**config) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth_nk/config_zoedepth_nk.json b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth_nk/config_zoedepth_nk.json new file mode 100644 index 0000000000000000000000000000000000000000..ae036e38243566e0bb79a4821e4897d9bc4aaae1 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth_nk/config_zoedepth_nk.json @@ -0,0 +1,67 @@ +{ + "model": { + "name": "ZoeDepthNK", + "version_name": "v1", + "bin_conf" : [ + { + "name": "nyu", + "n_bins": 64, + "min_depth": 1e-3, + "max_depth": 10.0 + }, + { + "name": "kitti", + "n_bins": 64, + "min_depth": 1e-3, + "max_depth": 80.0 + } + ], + "bin_embedding_dim": 128, + "bin_centers_type": "softplus", + "n_attractors":[16, 8, 4, 1], + "attractor_alpha": 1000, + "attractor_gamma": 2, + "attractor_kind" : "mean", + "attractor_type" : "inv", + "min_temp": 0.0212, + "max_temp": 50.0, + "memory_efficient": true, + "midas_model_type" : "DPT_BEiT_L_384", + "img_size": [384, 512] + }, + + "train": { + "train_midas": true, + "use_pretrained_midas": true, + "trainer": "zoedepth_nk", + "epochs": 5, + "bs": 16, + "optim_kwargs": {"lr": 0.0002512, "wd": 0.01}, + "sched_kwargs": {"div_factor": 1, "final_div_factor": 10000, "pct_start": 0.7, "three_phase":false, "cycle_momentum": true}, + "same_lr": false, + "w_si": 1, + "w_domain": 100, + "avoid_boundary": false, + "random_crop": false, + "input_width": 640, + "input_height": 480, + "w_grad": 0, + "w_reg": 0, + "midas_lr_factor": 10, + "encoder_lr_factor":10, + "pos_enc_lr_factor":10 + }, + + "infer": { + "train_midas": false, + "pretrained_resource": "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_NK.pt", + "use_pretrained_midas": false, + "force_keep_ar": true + }, + + "eval": { + "train_midas": false, + "pretrained_resource": "url::https://github.com/isl-org/ZoeDepth/releases/download/v1.0/ZoeD_M12_NK.pt", + "use_pretrained_midas": false + } +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth_nk/zoedepth_nk_v1.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth_nk/zoedepth_nk_v1.py new file mode 100644 index 0000000000000000000000000000000000000000..af0bfd914c005e06ab67638941d423ef2058e242 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/models/zoedepth_nk/zoedepth_nk_v1.py @@ -0,0 +1,333 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import itertools + +import torch +import torch.nn as nn + +from zoedepth.models.depth_model import DepthModel +from zoedepth.models.base_models.midas import MidasCore +from zoedepth.models.layers.attractor import AttractorLayer, AttractorLayerUnnormed +from zoedepth.models.layers.dist_layers import ConditionalLogBinomial +from zoedepth.models.layers.localbins_layers import (Projector, SeedBinRegressor, + SeedBinRegressorUnnormed) +from zoedepth.models.layers.patch_transformer import PatchTransformerEncoder +from zoedepth.models.model_io import load_state_from_resource + + +class ZoeDepthNK(DepthModel): + def __init__(self, core, bin_conf, bin_centers_type="softplus", bin_embedding_dim=128, + n_attractors=[16, 8, 4, 1], attractor_alpha=300, attractor_gamma=2, attractor_kind='sum', attractor_type='exp', + min_temp=5, max_temp=50, + memory_efficient=False, train_midas=True, + is_midas_pretrained=True, midas_lr_factor=1, encoder_lr_factor=10, pos_enc_lr_factor=10, inverse_midas=False, **kwargs): + """ZoeDepthNK model. This is the version of ZoeDepth that has two metric heads and uses a learned router to route to experts. + + Args: + core (models.base_models.midas.MidasCore): The base midas model that is used for extraction of "relative" features + + bin_conf (List[dict]): A list of dictionaries that contain the bin configuration for each metric head. Each dictionary should contain the following keys: + "name" (str, typically same as the dataset name), "n_bins" (int), "min_depth" (float), "max_depth" (float) + + The length of this list determines the number of metric heads. + bin_centers_type (str, optional): "normed" or "softplus". Activation type used for bin centers. For "normed" bin centers, linear normalization trick is applied. This results in bounded bin centers. + For "softplus", softplus activation is used and thus are unbounded. Defaults to "normed". + bin_embedding_dim (int, optional): bin embedding dimension. Defaults to 128. + + n_attractors (List[int], optional): Number of bin attractors at decoder layers. Defaults to [16, 8, 4, 1]. + attractor_alpha (int, optional): Proportional attractor strength. Refer to models.layers.attractor for more details. Defaults to 300. + attractor_gamma (int, optional): Exponential attractor strength. Refer to models.layers.attractor for more details. Defaults to 2. + attractor_kind (str, optional): Attraction aggregation "sum" or "mean". Defaults to 'sum'. + attractor_type (str, optional): Type of attractor to use; "inv" (Inverse attractor) or "exp" (Exponential attractor). Defaults to 'exp'. + + min_temp (int, optional): Lower bound for temperature of output probability distribution. Defaults to 5. + max_temp (int, optional): Upper bound for temperature of output probability distribution. Defaults to 50. + + memory_efficient (bool, optional): Whether to use memory efficient version of attractor layers. Memory efficient version is slower but is recommended incase of multiple metric heads in order save GPU memory. Defaults to False. + + train_midas (bool, optional): Whether to train "core", the base midas model. Defaults to True. + is_midas_pretrained (bool, optional): Is "core" pretrained? Defaults to True. + midas_lr_factor (int, optional): Learning rate reduction factor for base midas model except its encoder and positional encodings. Defaults to 10. + encoder_lr_factor (int, optional): Learning rate reduction factor for the encoder in midas model. Defaults to 10. + pos_enc_lr_factor (int, optional): Learning rate reduction factor for positional encodings in the base midas model. Defaults to 10. + + """ + + super().__init__() + + self.core = core + self.bin_conf = bin_conf + self.min_temp = min_temp + self.max_temp = max_temp + self.memory_efficient = memory_efficient + self.train_midas = train_midas + self.is_midas_pretrained = is_midas_pretrained + self.midas_lr_factor = midas_lr_factor + self.encoder_lr_factor = encoder_lr_factor + self.pos_enc_lr_factor = pos_enc_lr_factor + self.inverse_midas = inverse_midas + + N_MIDAS_OUT = 32 + btlnck_features = self.core.output_channels[0] + num_out_features = self.core.output_channels[1:] + # self.scales = [16, 8, 4, 2] # spatial scale factors + + self.conv2 = nn.Conv2d( + btlnck_features, btlnck_features, kernel_size=1, stride=1, padding=0) + + # Transformer classifier on the bottleneck + self.patch_transformer = PatchTransformerEncoder( + btlnck_features, 1, 128, use_class_token=True) + self.mlp_classifier = nn.Sequential( + nn.Linear(128, 128), + nn.ReLU(), + nn.Linear(128, 2) + ) + + if bin_centers_type == "normed": + SeedBinRegressorLayer = SeedBinRegressor + Attractor = AttractorLayer + elif bin_centers_type == "softplus": + SeedBinRegressorLayer = SeedBinRegressorUnnormed + Attractor = AttractorLayerUnnormed + elif bin_centers_type == "hybrid1": + SeedBinRegressorLayer = SeedBinRegressor + Attractor = AttractorLayerUnnormed + elif bin_centers_type == "hybrid2": + SeedBinRegressorLayer = SeedBinRegressorUnnormed + Attractor = AttractorLayer + else: + raise ValueError( + "bin_centers_type should be one of 'normed', 'softplus', 'hybrid1', 'hybrid2'") + self.bin_centers_type = bin_centers_type + # We have bins for each bin conf. + # Create a map (ModuleDict) of 'name' -> seed_bin_regressor + self.seed_bin_regressors = nn.ModuleDict( + {conf['name']: SeedBinRegressorLayer(btlnck_features, conf["n_bins"], mlp_dim=bin_embedding_dim//2, min_depth=conf["min_depth"], max_depth=conf["max_depth"]) + for conf in bin_conf} + ) + + self.seed_projector = Projector( + btlnck_features, bin_embedding_dim, mlp_dim=bin_embedding_dim//2) + self.projectors = nn.ModuleList([ + Projector(num_out, bin_embedding_dim, mlp_dim=bin_embedding_dim//2) + for num_out in num_out_features + ]) + + # Create a map (ModuleDict) of 'name' -> attractors (ModuleList) + self.attractors = nn.ModuleDict( + {conf['name']: nn.ModuleList([ + Attractor(bin_embedding_dim, n_attractors[i], + mlp_dim=bin_embedding_dim, alpha=attractor_alpha, + gamma=attractor_gamma, kind=attractor_kind, + attractor_type=attractor_type, memory_efficient=memory_efficient, + min_depth=conf["min_depth"], max_depth=conf["max_depth"]) + for i in range(len(n_attractors)) + ]) + for conf in bin_conf} + ) + + last_in = N_MIDAS_OUT + # conditional log binomial for each bin conf + self.conditional_log_binomial = nn.ModuleDict( + {conf['name']: ConditionalLogBinomial(last_in, bin_embedding_dim, conf['n_bins'], bottleneck_factor=4, min_temp=self.min_temp, max_temp=self.max_temp) + for conf in bin_conf} + ) + + def forward(self, x, return_final_centers=False, denorm=False, return_probs=False, **kwargs): + """ + Args: + x (torch.Tensor): Input image tensor of shape (B, C, H, W). Assumes all images are from the same domain. + return_final_centers (bool, optional): Whether to return the final centers of the attractors. Defaults to False. + denorm (bool, optional): Whether to denormalize the input image. Defaults to False. + return_probs (bool, optional): Whether to return the probabilities of the bins. Defaults to False. + + Returns: + dict: Dictionary of outputs with keys: + - "rel_depth": Relative depth map of shape (B, 1, H, W) + - "metric_depth": Metric depth map of shape (B, 1, H, W) + - "domain_logits": Domain logits of shape (B, 2) + - "bin_centers": Bin centers of shape (B, N, H, W). Present only if return_final_centers is True + - "probs": Bin probabilities of shape (B, N, H, W). Present only if return_probs is True + """ + b, c, h, w = x.shape + self.orig_input_width = w + self.orig_input_height = h + rel_depth, out = self.core(x, denorm=denorm, return_rel_depth=True) + + outconv_activation = out[0] + btlnck = out[1] + x_blocks = out[2:] + + x_d0 = self.conv2(btlnck) + x = x_d0 + + # Predict which path to take + embedding = self.patch_transformer(x)[0] # N, E + domain_logits = self.mlp_classifier(embedding) # N, 2 + domain_vote = torch.softmax(domain_logits.sum( + dim=0, keepdim=True), dim=-1) # 1, 2 + + # Get the path + bin_conf_name = ["nyu", "kitti"][torch.argmax( + domain_vote, dim=-1).squeeze().item()] + + try: + conf = [c for c in self.bin_conf if c.name == bin_conf_name][0] + except IndexError: + raise ValueError( + f"bin_conf_name {bin_conf_name} not found in bin_confs") + + min_depth = conf['min_depth'] + max_depth = conf['max_depth'] + + seed_bin_regressor = self.seed_bin_regressors[bin_conf_name] + _, seed_b_centers = seed_bin_regressor(x) + if self.bin_centers_type == 'normed' or self.bin_centers_type == 'hybrid2': + b_prev = (seed_b_centers - min_depth)/(max_depth - min_depth) + else: + b_prev = seed_b_centers + prev_b_embedding = self.seed_projector(x) + + attractors = self.attractors[bin_conf_name] + for projector, attractor, x in zip(self.projectors, attractors, x_blocks): + b_embedding = projector(x) + b, b_centers = attractor( + b_embedding, b_prev, prev_b_embedding, interpolate=True) + b_prev = b + prev_b_embedding = b_embedding + + last = outconv_activation + + b_centers = nn.functional.interpolate( + b_centers, last.shape[-2:], mode='bilinear', align_corners=True) + b_embedding = nn.functional.interpolate( + b_embedding, last.shape[-2:], mode='bilinear', align_corners=True) + + clb = self.conditional_log_binomial[bin_conf_name] + x = clb(last, b_embedding) + + # Now depth value is Sum px * cx , where cx are bin_centers from the last bin tensor + # print(x.shape, b_centers.shape) + # b_centers = nn.functional.interpolate(b_centers, x.shape[-2:], mode='bilinear', align_corners=True) + out = torch.sum(x * b_centers, dim=1, keepdim=True) + + output = dict(domain_logits=domain_logits, metric_depth=out) + if return_final_centers or return_probs: + output['bin_centers'] = b_centers + + if return_probs: + output['probs'] = x + return output + + def get_lr_params(self, lr): + """ + Learning rate configuration for different layers of the model + + Args: + lr (float) : Base learning rate + Returns: + list : list of parameters to optimize and their learning rates, in the format required by torch optimizers. + """ + param_conf = [] + if self.train_midas: + def get_rel_pos_params(): + for name, p in self.core.core.pretrained.named_parameters(): + if "relative_position" in name: + yield p + + def get_enc_params_except_rel_pos(): + for name, p in self.core.core.pretrained.named_parameters(): + if "relative_position" not in name: + yield p + + encoder_params = get_enc_params_except_rel_pos() + rel_pos_params = get_rel_pos_params() + midas_params = self.core.core.scratch.parameters() + midas_lr_factor = self.midas_lr_factor if self.is_midas_pretrained else 1.0 + param_conf.extend([ + {'params': encoder_params, 'lr': lr / self.encoder_lr_factor}, + {'params': rel_pos_params, 'lr': lr / self.pos_enc_lr_factor}, + {'params': midas_params, 'lr': lr / midas_lr_factor} + ]) + + remaining_modules = [] + for name, child in self.named_children(): + if name != 'core': + remaining_modules.append(child) + remaining_params = itertools.chain( + *[child.parameters() for child in remaining_modules]) + param_conf.append({'params': remaining_params, 'lr': lr}) + return param_conf + + def get_conf_parameters(self, conf_name): + """ + Returns parameters of all the ModuleDicts children that are exclusively used for the given bin configuration + """ + params = [] + for name, child in self.named_children(): + if isinstance(child, nn.ModuleDict): + for bin_conf_name, module in child.items(): + if bin_conf_name == conf_name: + params += list(module.parameters()) + return params + + def freeze_conf(self, conf_name): + """ + Freezes all the parameters of all the ModuleDicts children that are exclusively used for the given bin configuration + """ + for p in self.get_conf_parameters(conf_name): + p.requires_grad = False + + def unfreeze_conf(self, conf_name): + """ + Unfreezes all the parameters of all the ModuleDicts children that are exclusively used for the given bin configuration + """ + for p in self.get_conf_parameters(conf_name): + p.requires_grad = True + + def freeze_all_confs(self): + """ + Freezes all the parameters of all the ModuleDicts children + """ + for name, child in self.named_children(): + if isinstance(child, nn.ModuleDict): + for bin_conf_name, module in child.items(): + for p in module.parameters(): + p.requires_grad = False + + @staticmethod + def build(midas_model_type="DPT_BEiT_L_384", pretrained_resource=None, use_pretrained_midas=False, train_midas=False, freeze_midas_bn=True, **kwargs): + core = MidasCore.build(midas_model_type=midas_model_type, use_pretrained_midas=use_pretrained_midas, + train_midas=train_midas, fetch_features=True, freeze_bn=freeze_midas_bn, **kwargs) + model = ZoeDepthNK(core, **kwargs) + if pretrained_resource: + assert isinstance(pretrained_resource, str), "pretrained_resource must be a string" + model = load_state_from_resource(model, pretrained_resource) + return model + + @staticmethod + def build_from_config(config): + return ZoeDepthNK.build(**config) diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/arg_utils.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/arg_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..5124e8c617874b6457b7dbaebeec61c166577933 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/arg_utils.py @@ -0,0 +1,33 @@ + + +def infer_type(x): # hacky way to infer type from string args + if not isinstance(x, str): + return x + + try: + x = int(x) + return x + except ValueError: + pass + + try: + x = float(x) + return x + except ValueError: + pass + + return x + + +def parse_unknown(unknown_args): + clean = [] + for a in unknown_args: + if "=" in a: + k, v = a.split("=") + clean.extend([k, v]) + else: + clean.append(a) + + keys = clean[::2] + values = clean[1::2] + return {k.replace("--", ""): infer_type(v) for k, v in zip(keys, values)} diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/config.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/config.py new file mode 100644 index 0000000000000000000000000000000000000000..ff5cb7765994211de10946932a572e05901777d6 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/config.py @@ -0,0 +1,437 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import json +import os + +from .easydict import EasyDict as edict +from .arg_utils import infer_type + +import pathlib +import platform + +ROOT = pathlib.Path(__file__).parent.parent.resolve() + +HOME_DIR = os.path.expanduser("~") + +COMMON_CONFIG = { + "save_dir": os.path.expanduser("~/shortcuts/monodepth3_checkpoints"), + "project": "ZoeDepth", + "tags": '', + "notes": "", + "gpu": None, + "root": ".", + "uid": None, + "print_losses": False +} + +DATASETS_CONFIG = { + "kitti": { + "dataset": "kitti", + "min_depth": 0.001, + "max_depth": 80, + "data_path": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/raw"), + "gt_path": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/gts"), + "filenames_file": "./train_test_inputs/kitti_eigen_train_files_with_gt.txt", + "input_height": 352, + "input_width": 1216, # 704 + "data_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/raw"), + "gt_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/gts"), + "filenames_file_eval": "./train_test_inputs/kitti_eigen_test_files_with_gt.txt", + + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + + "do_random_rotate": True, + "degree": 1.0, + "do_kb_crop": True, + "garg_crop": True, + "eigen_crop": False, + "use_right": False + }, + "kitti_test": { + "dataset": "kitti", + "min_depth": 0.001, + "max_depth": 80, + "data_path": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/raw"), + "gt_path": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/gts"), + "filenames_file": "./train_test_inputs/kitti_eigen_train_files_with_gt.txt", + "input_height": 352, + "input_width": 1216, + "data_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/raw"), + "gt_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/kitti/gts"), + "filenames_file_eval": "./train_test_inputs/kitti_eigen_test_files_with_gt.txt", + + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + + "do_random_rotate": False, + "degree": 1.0, + "do_kb_crop": True, + "garg_crop": True, + "eigen_crop": False, + "use_right": False + }, + "nyu": { + "dataset": "nyu", + "avoid_boundary": False, + "min_depth": 1e-3, # originally 0.1 + "max_depth": 10, + "data_path": os.path.join(HOME_DIR, "shortcuts/datasets/nyu_depth_v2/sync/"), + "gt_path": os.path.join(HOME_DIR, "shortcuts/datasets/nyu_depth_v2/sync/"), + "filenames_file": "./train_test_inputs/nyudepthv2_train_files_with_gt.txt", + "input_height": 480, + "input_width": 640, + "data_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/nyu_depth_v2/official_splits/test/"), + "gt_path_eval": os.path.join(HOME_DIR, "shortcuts/datasets/nyu_depth_v2/official_splits/test/"), + "filenames_file_eval": "./train_test_inputs/nyudepthv2_test_files_with_gt.txt", + "min_depth_eval": 1e-3, + "max_depth_eval": 10, + "min_depth_diff": -10, + "max_depth_diff": 10, + + "do_random_rotate": True, + "degree": 1.0, + "do_kb_crop": False, + "garg_crop": False, + "eigen_crop": True + }, + "ibims": { + "dataset": "ibims", + "ibims_root": os.path.join(HOME_DIR, "shortcuts/datasets/ibims/ibims1_core_raw/"), + "eigen_crop": True, + "garg_crop": False, + "do_kb_crop": False, + "min_depth_eval": 0, + "max_depth_eval": 10, + "min_depth": 1e-3, + "max_depth": 10 + }, + "sunrgbd": { + "dataset": "sunrgbd", + "sunrgbd_root": os.path.join(HOME_DIR, "shortcuts/datasets/SUNRGBD/test/"), + "eigen_crop": True, + "garg_crop": False, + "do_kb_crop": False, + "min_depth_eval": 0, + "max_depth_eval": 8, + "min_depth": 1e-3, + "max_depth": 10 + }, + "diml_indoor": { + "dataset": "diml_indoor", + "diml_indoor_root": os.path.join(HOME_DIR, "shortcuts/datasets/diml_indoor_test/"), + "eigen_crop": True, + "garg_crop": False, + "do_kb_crop": False, + "min_depth_eval": 0, + "max_depth_eval": 10, + "min_depth": 1e-3, + "max_depth": 10 + }, + "diml_outdoor": { + "dataset": "diml_outdoor", + "diml_outdoor_root": os.path.join(HOME_DIR, "shortcuts/datasets/diml_outdoor_test/"), + "eigen_crop": False, + "garg_crop": True, + "do_kb_crop": False, + "min_depth_eval": 2, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 80 + }, + "diode_indoor": { + "dataset": "diode_indoor", + "diode_indoor_root": os.path.join(HOME_DIR, "shortcuts/datasets/diode_indoor/"), + "eigen_crop": True, + "garg_crop": False, + "do_kb_crop": False, + "min_depth_eval": 1e-3, + "max_depth_eval": 10, + "min_depth": 1e-3, + "max_depth": 10 + }, + "diode_outdoor": { + "dataset": "diode_outdoor", + "diode_outdoor_root": os.path.join(HOME_DIR, "shortcuts/datasets/diode_outdoor/"), + "eigen_crop": False, + "garg_crop": True, + "do_kb_crop": False, + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 80 + }, + "hypersim_test": { + "dataset": "hypersim_test", + "hypersim_test_root": os.path.join(HOME_DIR, "shortcuts/datasets/hypersim_test/"), + "eigen_crop": True, + "garg_crop": False, + "do_kb_crop": False, + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 10 + }, + "vkitti": { + "dataset": "vkitti", + "vkitti_root": os.path.join(HOME_DIR, "shortcuts/datasets/vkitti_test/"), + "eigen_crop": False, + "garg_crop": True, + "do_kb_crop": True, + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 80 + }, + "vkitti2": { + "dataset": "vkitti2", + "vkitti2_root": os.path.join(HOME_DIR, "shortcuts/datasets/vkitti2/"), + "eigen_crop": False, + "garg_crop": True, + "do_kb_crop": True, + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 80, + }, + "ddad": { + "dataset": "ddad", + "ddad_root": os.path.join(HOME_DIR, "shortcuts/datasets/ddad/ddad_val/"), + "eigen_crop": False, + "garg_crop": True, + "do_kb_crop": True, + "min_depth_eval": 1e-3, + "max_depth_eval": 80, + "min_depth": 1e-3, + "max_depth": 80, + }, +} + +ALL_INDOOR = ["nyu", "ibims", "sunrgbd", "diode_indoor", "hypersim_test"] +ALL_OUTDOOR = ["kitti", "diml_outdoor", "diode_outdoor", "vkitti2", "ddad"] +ALL_EVAL_DATASETS = ALL_INDOOR + ALL_OUTDOOR + +COMMON_TRAINING_CONFIG = { + "dataset": "nyu", + "distributed": True, + "workers": 16, + "clip_grad": 0.1, + "use_shared_dict": False, + "shared_dict": None, + "use_amp": False, + + "aug": True, + "random_crop": False, + "random_translate": False, + "translate_prob": 0.2, + "max_translation": 100, + + "validate_every": 0.25, + "log_images_every": 0.1, + "prefetch": False, +} + + +def flatten(config, except_keys=('bin_conf')): + def recurse(inp): + if isinstance(inp, dict): + for key, value in inp.items(): + if key in except_keys: + yield (key, value) + if isinstance(value, dict): + yield from recurse(value) + else: + yield (key, value) + + return dict(list(recurse(config))) + + +def split_combined_args(kwargs): + """Splits the arguments that are combined with '__' into multiple arguments. + Combined arguments should have equal number of keys and values. + Keys are separated by '__' and Values are separated with ';'. + For example, '__n_bins__lr=256;0.001' + + Args: + kwargs (dict): key-value pairs of arguments where key-value is optionally combined according to the above format. + + Returns: + dict: Parsed dict with the combined arguments split into individual key-value pairs. + """ + new_kwargs = dict(kwargs) + for key, value in kwargs.items(): + if key.startswith("__"): + keys = key.split("__")[1:] + values = value.split(";") + assert len(keys) == len( + values), f"Combined arguments should have equal number of keys and values. Keys are separated by '__' and Values are separated with ';'. For example, '__n_bins__lr=256;0.001. Given (keys,values) is ({keys}, {values})" + for k, v in zip(keys, values): + new_kwargs[k] = v + return new_kwargs + + +def parse_list(config, key, dtype=int): + """Parse a list of values for the key if the value is a string. The values are separated by a comma. + Modifies the config in place. + """ + if key in config: + if isinstance(config[key], str): + config[key] = list(map(dtype, config[key].split(','))) + assert isinstance(config[key], list) and all([isinstance(e, dtype) for e in config[key]] + ), f"{key} should be a list of values dtype {dtype}. Given {config[key]} of type {type(config[key])} with values of type {[type(e) for e in config[key]]}." + + +def get_model_config(model_name, model_version=None): + """Find and parse the .json config file for the model. + + Args: + model_name (str): name of the model. The config file should be named config_{model_name}[_{model_version}].json under the models/{model_name} directory. + model_version (str, optional): Specific config version. If specified config_{model_name}_{model_version}.json is searched for and used. Otherwise config_{model_name}.json is used. Defaults to None. + + Returns: + easydict: the config dictionary for the model. + """ + config_fname = f"config_{model_name}_{model_version}.json" if model_version is not None else f"config_{model_name}.json" + config_file = os.path.join(ROOT, "models", model_name, config_fname) + if not os.path.exists(config_file): + return None + + with open(config_file, "r") as f: + config = edict(json.load(f)) + + # handle dictionary inheritance + # only training config is supported for inheritance + if "inherit" in config.train and config.train.inherit is not None: + inherit_config = get_model_config(config.train["inherit"]).train + for key, value in inherit_config.items(): + if key not in config.train: + config.train[key] = value + return edict(config) + + +def update_model_config(config, mode, model_name, model_version=None, strict=False): + model_config = get_model_config(model_name, model_version) + if model_config is not None: + config = {**config, ** + flatten({**model_config.model, **model_config[mode]})} + elif strict: + raise ValueError(f"Config file for model {model_name} not found.") + return config + + +def check_choices(name, value, choices): + # return # No checks in dev branch + if value not in choices: + raise ValueError(f"{name} {value} not in supported choices {choices}") + + +KEYS_TYPE_BOOL = ["use_amp", "distributed", "use_shared_dict", "same_lr", "aug", "three_phase", + "prefetch", "cycle_momentum"] # Casting is not necessary as their int casted values in config are 0 or 1 + + +def get_config(model_name, mode='train', dataset=None, **overwrite_kwargs): + """Main entry point to get the config for the model. + + Args: + model_name (str): name of the desired model. + mode (str, optional): "train" or "infer". Defaults to 'train'. + dataset (str, optional): If specified, the corresponding dataset configuration is loaded as well. Defaults to None. + + Keyword Args: key-value pairs of arguments to overwrite the default config. + + The order of precedence for overwriting the config is (Higher precedence first): + # 1. overwrite_kwargs + # 2. "config_version": Config file version if specified in overwrite_kwargs. The corresponding config loaded is config_{model_name}_{config_version}.json + # 3. "version_name": Default Model version specific config specified in overwrite_kwargs. The corresponding config loaded is config_{model_name}_{version_name}.json + # 4. common_config: Default config for all models specified in COMMON_CONFIG + + Returns: + easydict: The config dictionary for the model. + """ + + + check_choices("Model", model_name, ["zoedepth", "zoedepth_nk"]) + check_choices("Mode", mode, ["train", "infer", "eval"]) + if mode == "train": + check_choices("Dataset", dataset, ["nyu", "kitti", "mix", None]) + + config = flatten({**COMMON_CONFIG, **COMMON_TRAINING_CONFIG}) + config = update_model_config(config, mode, model_name) + + # update with model version specific config + version_name = overwrite_kwargs.get("version_name", config["version_name"]) + config = update_model_config(config, mode, model_name, version_name) + + # update with config version if specified + config_version = overwrite_kwargs.get("config_version", None) + if config_version is not None: + print("Overwriting config with config_version", config_version) + config = update_model_config(config, mode, model_name, config_version) + + # update with overwrite_kwargs + # Combined args are useful for hyperparameter search + overwrite_kwargs = split_combined_args(overwrite_kwargs) + config = {**config, **overwrite_kwargs} + + # Casting to bool # TODO: Not necessary. Remove and test + for key in KEYS_TYPE_BOOL: + if key in config: + config[key] = bool(config[key]) + + # Model specific post processing of config + parse_list(config, "n_attractors") + + # adjust n_bins for each bin configuration if bin_conf is given and n_bins is passed in overwrite_kwargs + if 'bin_conf' in config and 'n_bins' in overwrite_kwargs: + bin_conf = config['bin_conf'] # list of dicts + n_bins = overwrite_kwargs['n_bins'] + new_bin_conf = [] + for conf in bin_conf: + conf['n_bins'] = n_bins + new_bin_conf.append(conf) + config['bin_conf'] = new_bin_conf + + if mode == "train": + orig_dataset = dataset + if dataset == "mix": + dataset = 'nyu' # Use nyu as default for mix. Dataset config is changed accordingly while loading the dataloader + if dataset is not None: + config['project'] = f"MonoDepth3-{orig_dataset}" # Set project for wandb + + if dataset is not None: + config['dataset'] = dataset + config = {**DATASETS_CONFIG[dataset], **config} + + + config['model'] = model_name + typed_config = {k: infer_type(v) for k, v in config.items()} + # add hostname to config + config['hostname'] = platform.node() + return edict(typed_config) + + +def change_dataset(config, new_dataset): + config.update(DATASETS_CONFIG[new_dataset]) + return config diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/geometry.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/geometry.py new file mode 100644 index 0000000000000000000000000000000000000000..5b405f2c90efb002c7382d159f35a8a58adacd24 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/geometry.py @@ -0,0 +1,98 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +import numpy as np + +def get_intrinsics(H,W): + """ + Intrinsics for a pinhole camera model. + Assume fov of 55 degrees and central principal point. + """ + f = 0.5 * W / np.tan(0.5 * 55 * np.pi / 180.0) + cx = 0.5 * W + cy = 0.5 * H + return np.array([[f, 0, cx], + [0, f, cy], + [0, 0, 1]]) + +def depth_to_points(depth, R=None, t=None): + + K = get_intrinsics(depth.shape[1], depth.shape[2]) + Kinv = np.linalg.inv(K) + if R is None: + R = np.eye(3) + if t is None: + t = np.zeros(3) + + # M converts from your coordinate to PyTorch3D's coordinate system + M = np.eye(3) + M[0, 0] = -1.0 + M[1, 1] = -1.0 + + height, width = depth.shape[1:3] + + x = np.arange(width) + y = np.arange(height) + coord = np.stack(np.meshgrid(x, y), -1) + coord = np.concatenate((coord, np.ones_like(coord)[:, :, [0]]), -1) # z=1 + coord = coord.astype(np.float32) + # coord = torch.as_tensor(coord, dtype=torch.float32, device=device) + coord = coord[None] # bs, h, w, 3 + + D = depth[:, :, :, None, None] + # print(D.shape, Kinv[None, None, None, ...].shape, coord[:, :, :, :, None].shape ) + pts3D_1 = D * Kinv[None, None, None, ...] @ coord[:, :, :, :, None] + # pts3D_1 live in your coordinate system. Convert them to Py3D's + pts3D_1 = M[None, None, None, ...] @ pts3D_1 + # from reference to targe tviewpoint + pts3D_2 = R[None, None, None, ...] @ pts3D_1 + t[None, None, None, :, None] + # pts3D_2 = pts3D_1 + # depth_2 = pts3D_2[:, :, :, 2, :] # b,1,h,w + return pts3D_2[:, :, :, :3, 0][0] + + +def create_triangles(h, w, mask=None): + """ + Reference: https://github.com/google-research/google-research/blob/e96197de06613f1b027d20328e06d69829fa5a89/infinite_nature/render_utils.py#L68 + Creates mesh triangle indices from a given pixel grid size. + This function is not and need not be differentiable as triangle indices are + fixed. + Args: + h: (int) denoting the height of the image. + w: (int) denoting the width of the image. + Returns: + triangles: 2D numpy array of indices (int) with shape (2(W-1)(H-1) x 3) + """ + x, y = np.meshgrid(range(w - 1), range(h - 1)) + tl = y * w + x + tr = y * w + x + 1 + bl = (y + 1) * w + x + br = (y + 1) * w + x + 1 + triangles = np.array([tl, bl, tr, br, tr, bl]) + triangles = np.transpose(triangles, (1, 2, 0)).reshape( + ((w - 1) * (h - 1) * 2, 3)) + if mask is not None: + mask = mask.reshape(-1) + triangles = triangles[mask[triangles].all(1)] + return triangles diff --git a/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/misc.py b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/misc.py new file mode 100644 index 0000000000000000000000000000000000000000..9fc1d469d1d7d831cd247f4c6634c957c2c9aba7 --- /dev/null +++ b/VideoX-Fun/comfyui/annotator/zoe/zoedepth/utils/misc.py @@ -0,0 +1,368 @@ +# MIT License + +# Copyright (c) 2022 Intelligent Systems Lab Org + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# File author: Shariq Farooq Bhat + +"""Miscellaneous utility functions.""" + +from scipy import ndimage + +import base64 +import math +import re +from io import BytesIO + +import matplotlib +import matplotlib.cm +import numpy as np +import requests +import torch +import torch.distributed as dist +import torch.nn +import torch.nn as nn +import torch.utils.data.distributed +from PIL import Image +from torchvision.transforms import ToTensor + + +class RunningAverage: + def __init__(self): + self.avg = 0 + self.count = 0 + + def append(self, value): + self.avg = (value + self.count * self.avg) / (self.count + 1) + self.count += 1 + + def get_value(self): + return self.avg + + +def denormalize(x): + """Reverses the imagenet normalization applied to the input. + + Args: + x (torch.Tensor - shape(N,3,H,W)): input tensor + + Returns: + torch.Tensor - shape(N,3,H,W): Denormalized input + """ + mean = torch.Tensor([0.485, 0.456, 0.406]).view(1, 3, 1, 1).to(x.device) + std = torch.Tensor([0.229, 0.224, 0.225]).view(1, 3, 1, 1).to(x.device) + return x * std + mean + + +class RunningAverageDict: + """A dictionary of running averages.""" + def __init__(self): + self._dict = None + + def update(self, new_dict): + if new_dict is None: + return + + if self._dict is None: + self._dict = dict() + for key, value in new_dict.items(): + self._dict[key] = RunningAverage() + + for key, value in new_dict.items(): + self._dict[key].append(value) + + def get_value(self): + if self._dict is None: + return None + return {key: value.get_value() for key, value in self._dict.items()} + + +def colorize(value, vmin=None, vmax=None, cmap='gray_r', invalid_val=-99, invalid_mask=None, background_color=(128, 128, 128, 255), gamma_corrected=False, value_transform=None): + """Converts a depth map to a color image. + + Args: + value (torch.Tensor, numpy.ndarry): Input depth map. Shape: (H, W) or (1, H, W) or (1, 1, H, W). All singular dimensions are squeezed + vmin (float, optional): vmin-valued entries are mapped to start color of cmap. If None, value.min() is used. Defaults to None. + vmax (float, optional): vmax-valued entries are mapped to end color of cmap. If None, value.max() is used. Defaults to None. + cmap (str, optional): matplotlib colormap to use. Defaults to 'magma_r'. + invalid_val (int, optional): Specifies value of invalid pixels that should be colored as 'background_color'. Defaults to -99. + invalid_mask (numpy.ndarray, optional): Boolean mask for invalid regions. Defaults to None. + background_color (tuple[int], optional): 4-tuple RGB color to give to invalid pixels. Defaults to (128, 128, 128, 255). + gamma_corrected (bool, optional): Apply gamma correction to colored image. Defaults to False. + value_transform (Callable, optional): Apply transform function to valid pixels before coloring. Defaults to None. + + Returns: + numpy.ndarray, dtype - uint8: Colored depth map. Shape: (H, W, 4) + """ + if isinstance(value, torch.Tensor): + value = value.detach().cpu().numpy() + + value = value.squeeze() + if invalid_mask is None: + invalid_mask = value == invalid_val + mask = np.logical_not(invalid_mask) + + # normalize + vmin = np.percentile(value[mask],2) if vmin is None else vmin + vmax = np.percentile(value[mask],85) if vmax is None else vmax + if vmin != vmax: + value = (value - vmin) / (vmax - vmin) # vmin..vmax + else: + # Avoid 0-division + value = value * 0. + + # squeeze last dim if it exists + # grey out the invalid values + + value[invalid_mask] = np.nan + cmapper = matplotlib.cm.get_cmap(cmap) + if value_transform: + value = value_transform(value) + # value = value / value.max() + value = cmapper(value, bytes=True) # (nxmx4) + + # img = value[:, :, :] + img = value[...] + img[invalid_mask] = background_color + + # return img.transpose((2, 0, 1)) + if gamma_corrected: + # gamma correction + img = img / 255 + img = np.power(img, 2.2) + img = img * 255 + img = img.astype(np.uint8) + return img + + +def count_parameters(model, include_all=False): + return sum(p.numel() for p in model.parameters() if p.requires_grad or include_all) + + +def compute_errors(gt, pred): + """Compute metrics for 'pred' compared to 'gt' + + Args: + gt (numpy.ndarray): Ground truth values + pred (numpy.ndarray): Predicted values + + gt.shape should be equal to pred.shape + + Returns: + dict: Dictionary containing the following metrics: + 'a1': Delta1 accuracy: Fraction of pixels that are within a scale factor of 1.25 + 'a2': Delta2 accuracy: Fraction of pixels that are within a scale factor of 1.25^2 + 'a3': Delta3 accuracy: Fraction of pixels that are within a scale factor of 1.25^3 + 'abs_rel': Absolute relative error + 'rmse': Root mean squared error + 'log_10': Absolute log10 error + 'sq_rel': Squared relative error + 'rmse_log': Root mean squared error on the log scale + 'silog': Scale invariant log error + """ + thresh = np.maximum((gt / pred), (pred / gt)) + a1 = (thresh < 1.25).mean() + a2 = (thresh < 1.25 ** 2).mean() + a3 = (thresh < 1.25 ** 3).mean() + + abs_rel = np.mean(np.abs(gt - pred) / gt) + sq_rel = np.mean(((gt - pred) ** 2) / gt) + + rmse = (gt - pred) ** 2 + rmse = np.sqrt(rmse.mean()) + + rmse_log = (np.log(gt) - np.log(pred)) ** 2 + rmse_log = np.sqrt(rmse_log.mean()) + + err = np.log(pred) - np.log(gt) + silog = np.sqrt(np.mean(err ** 2) - np.mean(err) ** 2) * 100 + + log_10 = (np.abs(np.log10(gt) - np.log10(pred))).mean() + return dict(a1=a1, a2=a2, a3=a3, abs_rel=abs_rel, rmse=rmse, log_10=log_10, rmse_log=rmse_log, + silog=silog, sq_rel=sq_rel) + + +def compute_metrics(gt, pred, interpolate=True, garg_crop=False, eigen_crop=True, dataset='nyu', min_depth_eval=0.1, max_depth_eval=10, **kwargs): + """Compute metrics of predicted depth maps. Applies cropping and masking as necessary or specified via arguments. Refer to compute_errors for more details on metrics. + """ + if 'config' in kwargs: + config = kwargs['config'] + garg_crop = config.garg_crop + eigen_crop = config.eigen_crop + min_depth_eval = config.min_depth_eval + max_depth_eval = config.max_depth_eval + + if gt.shape[-2:] != pred.shape[-2:] and interpolate: + pred = nn.functional.interpolate( + pred, gt.shape[-2:], mode='bilinear', align_corners=True) + + pred = pred.squeeze().cpu().numpy() + pred[pred < min_depth_eval] = min_depth_eval + pred[pred > max_depth_eval] = max_depth_eval + pred[np.isinf(pred)] = max_depth_eval + pred[np.isnan(pred)] = min_depth_eval + + gt_depth = gt.squeeze().cpu().numpy() + valid_mask = np.logical_and( + gt_depth > min_depth_eval, gt_depth < max_depth_eval) + + if garg_crop or eigen_crop: + gt_height, gt_width = gt_depth.shape + eval_mask = np.zeros(valid_mask.shape) + + if garg_crop: + eval_mask[int(0.40810811 * gt_height):int(0.99189189 * gt_height), + int(0.03594771 * gt_width):int(0.96405229 * gt_width)] = 1 + + elif eigen_crop: + # print("-"*10, " EIGEN CROP ", "-"*10) + if dataset == 'kitti': + eval_mask[int(0.3324324 * gt_height):int(0.91351351 * gt_height), + int(0.0359477 * gt_width):int(0.96405229 * gt_width)] = 1 + else: + # assert gt_depth.shape == (480, 640), "Error: Eigen crop is currently only valid for (480, 640) images" + eval_mask[45:471, 41:601] = 1 + else: + eval_mask = np.ones(valid_mask.shape) + valid_mask = np.logical_and(valid_mask, eval_mask) + return compute_errors(gt_depth[valid_mask], pred[valid_mask]) + + +#################################### Model uilts ################################################ + + +def parallelize(config, model, find_unused_parameters=True): + + if config.gpu is not None: + torch.cuda.set_device(config.gpu) + model = model.cuda(config.gpu) + + config.multigpu = False + if config.distributed: + # Use DDP + config.multigpu = True + config.rank = config.rank * config.ngpus_per_node + config.gpu + dist.init_process_group(backend=config.dist_backend, init_method=config.dist_url, + world_size=config.world_size, rank=config.rank) + config.batch_size = int(config.batch_size / config.ngpus_per_node) + # config.batch_size = 8 + config.workers = int( + (config.num_workers + config.ngpus_per_node - 1) / config.ngpus_per_node) + print("Device", config.gpu, "Rank", config.rank, "batch size", + config.batch_size, "Workers", config.workers) + torch.cuda.set_device(config.gpu) + model = nn.SyncBatchNorm.convert_sync_batchnorm(model) + model = model.cuda(config.gpu) + model = torch.nn.parallel.DistributedDataParallel(model, device_ids=[config.gpu], output_device=config.gpu, + find_unused_parameters=find_unused_parameters) + + elif config.gpu is None: + # Use DP + config.multigpu = True + model = model.cuda() + model = torch.nn.DataParallel(model) + + return model + + +################################################################################################# + + +##################################################################################################### + + +class colors: + '''Colors class: + Reset all colors with colors.reset + Two subclasses fg for foreground and bg for background. + Use as colors.subclass.colorname. + i.e. colors.fg.red or colors.bg.green + Also, the generic bold, disable, underline, reverse, strikethrough, + and invisible work with the main class + i.e. colors.bold + ''' + reset = '\033[0m' + bold = '\033[01m' + disable = '\033[02m' + underline = '\033[04m' + reverse = '\033[07m' + strikethrough = '\033[09m' + invisible = '\033[08m' + + class fg: + black = '\033[30m' + red = '\033[31m' + green = '\033[32m' + orange = '\033[33m' + blue = '\033[34m' + purple = '\033[35m' + cyan = '\033[36m' + lightgrey = '\033[37m' + darkgrey = '\033[90m' + lightred = '\033[91m' + lightgreen = '\033[92m' + yellow = '\033[93m' + lightblue = '\033[94m' + pink = '\033[95m' + lightcyan = '\033[96m' + + class bg: + black = '\033[40m' + red = '\033[41m' + green = '\033[42m' + orange = '\033[43m' + blue = '\033[44m' + purple = '\033[45m' + cyan = '\033[46m' + lightgrey = '\033[47m' + + +def printc(text, color): + print(f"{color}{text}{colors.reset}") + +############################################ + +def get_image_from_url(url): + response = requests.get(url) + img = Image.open(BytesIO(response.content)).convert("RGB") + return img + +def url_to_torch(url, size=(384, 384)): + img = get_image_from_url(url) + img = img.resize(size, Image.ANTIALIAS) + img = torch.from_numpy(np.asarray(img)).float() + img = img.permute(2, 0, 1) + img.div_(255) + return img + +def pil_to_batched_tensor(img): + return ToTensor()(img).unsqueeze(0) + +def save_raw_16bit(depth, fpath="raw.png"): + if isinstance(depth, torch.Tensor): + depth = depth.squeeze().cpu().numpy() + + assert isinstance(depth, np.ndarray), "Depth must be a torch tensor or numpy array" + assert depth.ndim == 2, "Depth must be 2D" + depth = depth * 256 # scale for 16-bit png + depth = depth.astype(np.uint16) + depth = Image.fromarray(depth) + depth.save(fpath) + print("Saved raw depth to", fpath) \ No newline at end of file diff --git a/VideoX-Fun/comfyui/camera_utils.py b/VideoX-Fun/comfyui/camera_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..efd05586b0c1c5fa24c9ae539da1ebbc72bdc575 --- /dev/null +++ b/VideoX-Fun/comfyui/camera_utils.py @@ -0,0 +1,80 @@ +"""Modified from https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/blob/main/camera_utils.py +""" +import copy +import numpy as np + +CAMERA = { + # T + "base_T_norm": 1.5, + "base_angle": np.pi/3, + + "Static": { "angle":[0., 0., 0.], "T":[0., 0., 0.]}, + "Pan Up": { "angle":[0., 0., 0.], "T":[0., 1., 0.]}, + "Pan Down": { "angle":[0., 0., 0.], "T":[0.,-1.,0.]}, + "Pan Left": { "angle":[0., 0., 0.], "T":[1.,0.,0.]}, + "Pan Right": { "angle":[0., 0., 0.], "T": [-1.,0.,0.]}, + "Zoom In": { "angle":[0., 0., 0.], "T": [0.,0.,-2.]}, + "Zoom Out": { "angle":[0., 0., 0.], "T": [0.,0.,2.]}, + "ACW": { "angle": [0., 0., 1.], "T":[0., 0., 0.]}, + "CW": { "angle": [0., 0., -1.], "T":[0., 0., 0.]}, +} + +def compute_R_form_rad_angle(angles): + theta_x, theta_y, theta_z = angles + Rx = np.array([[1, 0, 0], + [0, np.cos(theta_x), -np.sin(theta_x)], + [0, np.sin(theta_x), np.cos(theta_x)]]) + + Ry = np.array([[np.cos(theta_y), 0, np.sin(theta_y)], + [0, 1, 0], + [-np.sin(theta_y), 0, np.cos(theta_y)]]) + + Rz = np.array([[np.cos(theta_z), -np.sin(theta_z), 0], + [np.sin(theta_z), np.cos(theta_z), 0], + [0, 0, 1]]) + + # 计算相机外参的旋转矩阵 + R = np.dot(Rz, np.dot(Ry, Rx)) + return R + +def get_camera_motion(angle, T, speed, n=16): + RT = [] + for i in range(n): + _angle = (i/n)*speed*(CAMERA["base_angle"])*angle + R = compute_R_form_rad_angle(_angle) + # _T = (i/n)*speed*(T.reshape(3,1)) + _T=(i/n)*speed*(CAMERA["base_T_norm"])*(T.reshape(3,1)) + _RT = np.concatenate([R,_T], axis=1) + RT.append(_RT) + RT = np.stack(RT) + return RT + +def create_relative(RT_list, K_1=4.7, dataset="syn"): + RT = copy.deepcopy(RT_list[0]) + R_inv = RT[:,:3].T + T = RT[:,-1] + + temp = [] + for _RT in RT_list: + _RT[:,:3] = np.dot(_RT[:,:3], R_inv) + _RT[:,-1] = _RT[:,-1] - np.dot(_RT[:,:3], T) + temp.append(_RT) + RT_list = temp + + return RT_list + +def combine_camera_motion(RT_0, RT_1): + RT = copy.deepcopy(RT_0[-1]) + R = RT[:,:3] + R_inv = RT[:,:3].T + T = RT[:,-1] + + temp = [] + for _RT in RT_1: + _RT[:,:3] = np.dot(_RT[:,:3], R) + _RT[:,-1] = _RT[:,-1] + np.dot(np.dot(_RT[:,:3], R_inv), T) + temp.append(_RT) + + RT_1 = np.stack(temp) + + return np.concatenate([RT_0, RT_1], axis=0) \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/nodes.py b/VideoX-Fun/comfyui/cogvideox_fun/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..a6c97b0ed72e3bbe807ea278d434caae56b3d03f --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/nodes.py @@ -0,0 +1,671 @@ +"""Modified from https://github.com/kijai/ComfyUI-EasyAnimateWrapper/blob/main/nodes.py +""" +import gc +import json +import os + +import comfy.model_management as mm +import cv2 +import folder_paths +import copy +import numpy as np +import torch +from comfy.utils import ProgressBar, load_torch_file +from diffusers import EulerDiscreteScheduler +from einops import rearrange +from PIL import Image + +from ...videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + get_closest_ratio) +from ...videox_fun.models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) +from ...videox_fun.pipeline import (CogVideoXFunPipeline, + CogVideoXFunControlPipeline, + CogVideoXFunInpaintPipeline) +from ...videox_fun.ui.controller import all_cheduler_dict +from ...videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from ...videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) +from ...videox_fun.utils.fp8_optimization import convert_weight_dtype_wrapper, undo_convert_weight_dtype_wrapper +from ..comfyui_utils import (eas_cache_dir, script_directory, + search_model_in_possible_folders, to_pil) + +# Used in lora cache +transformer_cpu_cache = {} +# lora path before +lora_path_before = "" + +class LoadCogVideoXFunModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ( + [ + 'CogVideoX-Fun-2b-InP', + 'CogVideoX-Fun-5b-InP', + 'CogVideoX-Fun-V1.1-2b-InP', + 'CogVideoX-Fun-V1.1-5b-InP', + 'CogVideoX-Fun-V1.1-2b-Pose', + 'CogVideoX-Fun-V1.1-5b-Pose', + "CogVideoX-Fun-V1.1-2b-Control", + 'CogVideoX-Fun-V1.1-5b-Control', + 'CogVideoX-Fun-V1.5-5b-InP', + ], + { + "default": 'CogVideoX-Fun-V1.1-2b-InP', + } + ), + "model_type": ( + ["Inpaint", "Control"], + { + "default": "Inpaint", + } + ), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + "precision": ( + ['fp16', 'bf16'], + { + "default": 'fp16' + } + ), + }, + } + + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("cogvideoxfun_model",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, GPU_memory_mode, model, model_type, precision): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + + # Init processbar + pbar = ProgressBar(5) + + # Detect model is existing or not + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun", "Wan-AI"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + # Initialize model_name as None + model_name = search_model_in_possible_folders(possible_folders, model) + + vae = AutoencoderKLCogVideoX.from_pretrained( + model_name, + subfolder="vae", + ).to(weight_dtype) + # Update pbar + pbar.update(1) + + # Load Sampler + print("Load Sampler.") + scheduler = EulerDiscreteScheduler.from_pretrained(model_name, subfolder='scheduler') + # Update pbar + pbar.update(1) + + # Get Transformer + transformer = CogVideoXTransformer3DModel.from_pretrained( + model_name, + subfolder="transformer", + torch_dtype=torch.float8_e4m3fn if GPU_memory_mode == "model_cpu_offload_and_qfloat8" else weight_dtype, + ).to(weight_dtype) + # Update pbar + pbar.update(1) + + # Get tokenizer and text_encoder + tokenizer = T5Tokenizer.from_pretrained( + model_name, subfolder="tokenizer" + ) + pbar.update(1) + + text_encoder = T5EncoderModel.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype + ) + pbar.update(1) + + # Get pipeline + if model_type == "Inpaint": + if transformer.config.in_channels != vae.config.latent_channels: + pipeline = CogVideoXFunInpaintPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + else: + pipeline = CogVideoXFunPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + else: + pipeline = CogVideoXFunControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + + if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to("cuda") + + cogvideoxfun_model = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [], + } + return (cogvideoxfun_model,) + +class LoadCogVideoXFunLora: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "cogvideoxfun_model": ("FunModels",), + "lora_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), + "lora_cache":([False, True], {"default": False,}), + } + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("cogvideoxfun_model",) + FUNCTION = "load_lora" + CATEGORY = "CogVideoXFUNWrapper" + + def load_lora(self, cogvideoxfun_model, lora_name, strength_model, lora_cache): + new_funmodels = dict(cogvideoxfun_model) + + if lora_name is not None: + lora_path = folder_paths.get_full_path("loras", lora_name) + if lora_path is None: + raise FileNotFoundError(f"LoRA 文件未找到: {lora_name}") + + new_funmodels['lora_cache'] = lora_cache + new_funmodels['loras'] = cogvideoxfun_model.get("loras", []) + [lora_path] + new_funmodels['strength_model'] = cogvideoxfun_model.get("strength_model", []) + [strength_model] + + return (new_funmodels,) + +class CogVideoXFunT2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "cogvideoxfun_model": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 49, "min": 5, "max": 85, "step": 4} + ), + "width": ( + "INT", {"default": 1008, "min": 64, "max": 2048, "step": 16} + ), + "height": ( + "INT", {"default": 576, "min": 64, "max": 2048, "step": 16} + ), + "is_image":( + [ + False, + True + ], + { + "default": False, + } + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + [ + "Euler", + "Euler A", + "DPM++", + "PNDM", + "DDIM", + ], + { + "default": 'DDIM' + } + ), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, cogvideoxfun_model, prompt, negative_prompt, video_length, width, height, is_image, seed, steps, cfg, scheduler): + global transformer_cpu_cache + global lora_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = cogvideoxfun_model['pipeline'] + model_name = cogvideoxfun_model['model_name'] + weight_dtype = cogvideoxfun_model['dtype'] + + # Load Sampler + pipeline.scheduler = all_cheduler_dict[scheduler].from_pretrained(model_name, subfolder='scheduler') + + generator= torch.Generator(device).manual_seed(seed) + + video_length = 1 if is_image else video_length + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + input_video, input_video_mask, clip_image = get_image_to_video_latent(None, None, video_length=video_length, sample_size=(height, width)) + + # Apply lora + if cogvideoxfun_model.get("lora_cache", False): + if len(cogvideoxfun_model.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(cogvideoxfun_model.get("loras", []) + cogvideoxfun_model.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(cogvideoxfun_model.get("loras", []), cogvideoxfun_model.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + else: + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + print('Merge Lora') + for _lora_path, _lora_weight in zip(cogvideoxfun_model.get("loras", []), cogvideoxfun_model.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not cogvideoxfun_model.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(cogvideoxfun_model.get("loras", []), cogvideoxfun_model.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + return (videos,) + +class CogVideoXFunInpaintSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "cogvideoxfun_model": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 49, "min": 5, "max": 85, "step": 4} + ), + "base_resolution": ( + [ + 512, + 768, + 960, + 1024, + ], {"default": 768} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + [ + "Euler", + "Euler A", + "DPM++", + "PNDM", + "DDIM", + ], + { + "default": 'DDIM' + } + ) + }, + "optional":{ + "start_img": ("IMAGE",), + "end_img": ("IMAGE",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, cogvideoxfun_model, prompt, negative_prompt, video_length, base_resolution, seed, steps, cfg, scheduler, start_img=None, end_img=None): + global transformer_cpu_cache + global lora_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + start_img = [to_pil(_start_img) for _start_img in start_img] if start_img is not None else None + end_img = [to_pil(_end_img) for _end_img in end_img] if end_img is not None else None + # Count most suitable height and width + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + original_width, original_height = start_img[0].size if type(start_img) is list else Image.open(start_img).size + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height, width = [int(x / 16) * 16 for x in closest_size] + + # Get Pipeline + pipeline = cogvideoxfun_model['pipeline'] + model_name = cogvideoxfun_model['model_name'] + weight_dtype = cogvideoxfun_model['dtype'] + + # Load Sampler + pipeline.scheduler = all_cheduler_dict[scheduler].from_pretrained(model_name, subfolder='scheduler') + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_img, end_img, video_length=video_length, sample_size=(height, width)) + + # Apply lora + if cogvideoxfun_model.get("lora_cache", False): + if len(cogvideoxfun_model.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(cogvideoxfun_model.get("loras", []) + cogvideoxfun_model.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(cogvideoxfun_model.get("loras", []), cogvideoxfun_model.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + else: + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + print('Delete cpu state_dict') + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + print('Merge Lora') + for _lora_path, _lora_weight in zip(cogvideoxfun_model.get("loras", []), cogvideoxfun_model.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not cogvideoxfun_model.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(cogvideoxfun_model.get("loras", []), cogvideoxfun_model.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + return (videos,) + + +class CogVideoXFunV2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "cogvideoxfun_model": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 49, "min": 5, "max": 85, "step": 4} + ), + "base_resolution": ( + [ + 512, + 768, + 960, + 1024, + ], {"default": 768} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "denoise_strength": ( + "FLOAT", {"default": 1.00, "min": 0.05, "max": 1.00, "step": 0.01} + ), + "scheduler": ( + [ + "Euler", + "Euler A", + "DPM++", + "PNDM", + "DDIM", + ], + { + "default": 'DDIM' + } + ), + }, + "optional":{ + "validation_video": ("IMAGE",), + "control_video": ("IMAGE",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, cogvideoxfun_model, prompt, negative_prompt, video_length, base_resolution, seed, steps, cfg, denoise_strength, scheduler, validation_video=None, control_video=None): + global transformer_cpu_cache + global lora_path_before + + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = cogvideoxfun_model['pipeline'] + model_name = cogvideoxfun_model['model_name'] + weight_dtype = cogvideoxfun_model['dtype'] + model_type = cogvideoxfun_model['model_type'] + + # Count most suitable height and width + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if model_type == "Inpaint": + if type(validation_video) is str: + original_width, original_height = Image.fromarray(cv2.VideoCapture(validation_video).read()[1]).size + else: + validation_video = np.array(validation_video.cpu().numpy() * 255, np.uint8) + original_width, original_height = Image.fromarray(validation_video[0]).size + else: + if control_video is not None and type(control_video) is str: + original_width, original_height = Image.fromarray(cv2.VideoCapture(control_video).read()[1]).size + elif control_video is not None: + control_video = np.array(control_video.cpu().numpy() * 255, np.uint8) + original_width, original_height = Image.fromarray(control_video[0]).size + else: + original_width, original_height = 384 / 512 * base_resolution, 672 / 512 * base_resolution + + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height, width = [int(x / 16) * 16 for x in closest_size] + + # Load Sampler + pipeline.scheduler = all_cheduler_dict[scheduler].from_pretrained(model_name, subfolder='scheduler') + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + if model_type == "Inpaint": + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(validation_video, video_length=video_length, sample_size=(height, width), fps=8) + else: + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(control_video, video_length=video_length, sample_size=(height, width), fps=8) + + # Apply lora + if cogvideoxfun_model.get("lora_cache", False): + if len(cogvideoxfun_model.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(cogvideoxfun_model.get("loras", []) + cogvideoxfun_model.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(cogvideoxfun_model.get("loras", []), cogvideoxfun_model.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + else: + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + print('Merge Lora') + for _lora_path, _lora_weight in zip(cogvideoxfun_model.get("loras", []), cogvideoxfun_model.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + + if model_type == "Inpaint": + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + strength = float(denoise_strength), + comfyui_progressbar = True, + ).videos + else: + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + control_video = input_video, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not cogvideoxfun_model.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(cogvideoxfun_model.get("loras", []), cogvideoxfun_model.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device="cuda", dtype=weight_dtype) + return (videos,) diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_i2v.json b/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..132ff5d5a8e60ca23a226fe5682e692678a4e2d5 --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_i2v.json @@ -0,0 +1,451 @@ +{ + "last_node_id": 83, + "last_link_id": 46, + "nodes": [ + { + "id": 7, + "type": "LoadImage", + "pos": [ + 258.76883544921907, + 468.15773315429715 + ], + "size": [ + 378.07147216796875, + 314.0000114440918 + ], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 45 + ], + "shape": 3, + "label": "图像", + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3, + "label": "遮罩" + } + ], + "title": "Start Image（图片到视频的开始图片）", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "firework.png", + "image" + ] + }, + { + "id": 79, + "type": "Note", + "pos": [ + 16, + 460 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（在此上传开始图像）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 80, + "type": "Note", + "pos": [ + 20, + -300 + ], + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 43 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 82, + "type": "CogVideoXFunInpaintSampler", + "pos": [ + 758, + 93 + ], + "size": { + "0": 336, + "1": 282 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 42 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 43 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 44 + }, + { + "name": "start_img", + "type": "IMAGE", + "link": 45, + "slot_index": 3 + }, + { + "name": "end_img", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 46 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunInpaintSampler" + }, + "widgets_values": [ + 49, + 512, + 43, + "fixed", + 50, + 6, + "DDIM" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1134, + 93 + ], + "size": [ + 390.9534912109375, + 535.9734235491071 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 46, + "label": "图像", + "slot_index": 0 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null, + "label": "音频" + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理" + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3, + "label": "文件名", + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "CogVideoX-Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "CogVideoX-Fun_00003.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 83, + "type": "LoadCogVideoXFunModel", + "pos": [ + 300, + -294 + ], + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 42 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-V1.1-2b-InP", + "Inpaint", + "model_cpu_offload", + "bf16" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 44 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + } + ], + "links": [ + [ + 42, + 83, + 0, + 82, + 0, + "FunModels" + ], + [ + 43, + 75, + 0, + 82, + 1, + "STRING_PROMPT" + ], + [ + 44, + 73, + 0, + 82, + 2, + "STRING_PROMPT" + ], + [ + 45, + 7, + 0, + 82, + 3, + "IMAGE" + ], + [ + 46, + 82, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24 + }, + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24 + }, + { + "title": "Upload Your Start Image", + "bounding": [ + 218, + 382, + 452, + 418 + ], + "color": "#a1309b", + "font_size": 24 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015778, + "offset": [ + 268.77277812624413, + 436.3236112390962 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_t2v.json b/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..16bc2a77dd4e490862950471f1cd6b0a2ce20895 --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_t2v.json @@ -0,0 +1,359 @@ +{ + "last_node_id": 88, + "last_link_id": 52, + "nodes": [ + { + "id": 80, + "type": "Note", + "pos": [ + 20, + -300 + ], + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 0, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 50 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 88, + "type": "CogVideoXFunT2VSampler", + "pos": [ + 728, + -68 + ], + "size": { + "0": 327.6000061035156, + "1": 290 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 49 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 50 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 51, + "slot_index": 2 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 52 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunT2VSampler" + }, + "widgets_values": [ + 49, + 672, + 384, + false, + 43, + "fixed", + 50, + 6, + "DDIM" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1110, + -67 + ], + "size": [ + 390.9534912109375, + 535.9734235491071 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 52, + "label": "图像", + "slot_index": 0 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null, + "label": "音频" + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理" + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3, + "label": "文件名", + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "CogVideoX-Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "CogVideoX-Fun_00004.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 51 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + }, + { + "id": 87, + "type": "LoadCogVideoXFunModel", + "pos": [ + 302, + -285 + ], + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 49 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-V1.1-2b-InP", + "Inpaint", + "model_cpu_offload", + "bf16" + ] + } + ], + "links": [ + [ + 49, + 87, + 0, + 88, + 0, + "FunModels" + ], + [ + 50, + 75, + 0, + 88, + 1, + "STRING_PROMPT" + ], + [ + 51, + 73, + 0, + 88, + 2, + "STRING_PROMPT" + ], + [ + 52, + 88, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24 + }, + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 181.0702206286297, + 544.9672051634072 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_t2v_lora.json b/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_t2v_lora.json new file mode 100644 index 0000000000000000000000000000000000000000..b3e4f044687ec629267e7e3e0771b3d006ff08ca --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_t2v_lora.json @@ -0,0 +1,419 @@ +{ + "last_node_id": 92, + "last_link_id": 57, + "nodes": [ + { + "id": 80, + "type": "Note", + "pos": { + "0": 22.407291412353516, + "1": -302.0006103515625 + }, + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 50 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 51 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + }, + { + "id": 88, + "type": "CogVideoXFunT2VSampler", + "pos": { + "0": 728, + "1": -68 + }, + "size": { + "0": 327.6000061035156, + "1": 290 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 57 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 50 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 51, + "slot_index": 2 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 52 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunT2VSampler" + }, + "widgets_values": [ + 49, + 672, + 384, + false, + 43, + "fixed", + 50, + 6, + "DDIM" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1098, + "1": -69 + }, + "size": [ + 390.9534912109375, + 535.9734235491071 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 52, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "CogVideoX-Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "CogVideoX-Fun_00001.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 87, + "type": "LoadCogVideoXFunModel", + "pos": { + "0": 260, + "1": -298 + }, + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 56 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-V1.1-2b-InP", + "Inpaint", + "model_cpu_offload", + "bf16" + ] + }, + { + "id": 92, + "type": "LoadCogVideoXFunLora", + "pos": { + "0": 605, + "1": -297 + }, + "size": { + "0": 468.5999755859375, + "1": 82 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 56 + } + ], + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 57 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunLora" + }, + "widgets_values": [ + "cogvideox_fun/CogVideoX-Fun-V1.1-2b-InP-HPS2.1.safetensors", + 0.6 + ] + } + ], + "links": [ + [ + 50, + 75, + 0, + 88, + 1, + "STRING_PROMPT" + ], + [ + 51, + 73, + 0, + 88, + 2, + "STRING_PROMPT" + ], + [ + 52, + 88, + 0, + 17, + 0, + "IMAGE" + ], + [ + 56, + 87, + 0, + 92, + 0, + "FunModels" + ], + [ + 57, + 92, + 0, + 88, + 0, + "FunModels" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 222, + -382, + 873, + 235 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 143.5668017223784, + 513.2159371946544 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_v2v.json b/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_v2v.json new file mode 100644 index 0000000000000000000000000000000000000000..8e55eff73f75ec6073a3e9e24081202bd4d71d6e --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_v2v.json @@ -0,0 +1,492 @@ +{ + "last_node_id": 90, + "last_link_id": 57, + "nodes": [ + { + "id": 80, + "type": "Note", + "pos": [ + 20, + -300 + ], + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 0, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.739953613281248, + 462.38664912015946 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 336, + 470 + ], + "size": [ + 235.1999969482422, + 398.971426827567 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 56 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "00000125.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "00000125.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 88, + "type": "LoadCogVideoXFunModel", + "pos": [ + 309, + -286 + ], + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 53 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-V1.1-2b-InP", + "Inpaint", + "model_cpu_offload", + "bf16" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 54 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A cute cat is playing the guitar." + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 55 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + }, + { + "id": 90, + "type": "CogVideoXFunV2VSampler", + "pos": [ + 754, + 14 + ], + "size": { + "0": 317.4000244140625, + "1": 306 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 53 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 54 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 55 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": 56, + "slot_index": 3 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 57 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunV2VSampler" + }, + "widgets_values": [ + 49, + 768, + 43, + "randomize", + 50, + 6, + 0.7, + "DDIM" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1125, + 15 + ], + "size": [ + 390.9534912109375, + 535.9734235491071 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 57, + "label": "图像", + "slot_index": 0 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null, + "label": "音频" + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理" + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3, + "label": "文件名", + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "EasyAnimate", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "EasyAnimate_00045.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + } + ], + "links": [ + [ + 53, + 88, + 0, + 90, + 0, + "FunModels" + ], + [ + 54, + 75, + 0, + 90, + 1, + "STRING_PROMPT" + ], + [ + 55, + 73, + 0, + 90, + 2, + "STRING_PROMPT" + ], + [ + 56, + 85, + 0, + 90, + 3, + "IMAGE" + ], + [ + 57, + 90, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24 + }, + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24 + }, + { + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 456, + 498 + ], + "color": "#a1309b", + "font_size": 24 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.683013455365071, + "offset": [ + 314.4077746994681, + 444.69453403364594 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_v2v_control.json b/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_v2v_control.json new file mode 100644 index 0000000000000000000000000000000000000000..867a3df0ead8b5f0fd2a568af1b13fd96aa10b98 --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1.1/cogvideoxfunv1.1_workflow_v2v_control.json @@ -0,0 +1,492 @@ +{ + "last_node_id": 90, + "last_link_id": 59, + "nodes": [ + { + "id": 80, + "type": "Note", + "pos": [ + 20, + -300 + ], + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 0, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.739953613281248, + 462.38664912015946 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 55 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1125, + 15 + ], + "size": [ + 390.9534912109375, + 973.1686096191406 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 57, + "label": "图像", + "slot_index": 0 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null, + "label": "音频" + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理" + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3, + "label": "文件名", + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "CogVideoX-Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "CogVideoX-Fun_00007.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 88, + "type": "LoadCogVideoXFunModel", + "pos": [ + 309, + -286 + ], + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 53 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-V1.1-2b-Pose", + "Control", + "model_cpu_offload", + "bf16" + ] + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 336, + 470 + ], + "size": [ + 235.1999969482422, + 658.5777723524305 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 59 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 8, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 8, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 54 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A person wearing a knee-length white sleeveless dress and white high-heeled sandals performs a dance in a well-lit room with wooden flooring. The room's background features a closed door, a shelf displaying clear glass bottles of alcoholic beverages, and a partially visible dark-colored sofa. " + ] + }, + { + "id": 90, + "type": "CogVideoXFunV2VSampler", + "pos": [ + 754, + 14 + ], + "size": { + "0": 336, + "1": 306 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 53 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 54 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 55 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": null, + "slot_index": 3 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": 59 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 57 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunV2VSampler" + }, + "widgets_values": [ + 49, + 512, + 43, + "fixed", + 50, + 6, + 1, + "DDIM" + ] + } + ], + "links": [ + [ + 53, + 88, + 0, + 90, + 0, + "FunModels" + ], + [ + 54, + 75, + 0, + 90, + 1, + "STRING_PROMPT" + ], + [ + 55, + 73, + 0, + 90, + 2, + "STRING_PROMPT" + ], + [ + 57, + 90, + 0, + 17, + 0, + "IMAGE" + ], + [ + 59, + 85, + 0, + 90, + 4, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24 + }, + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24 + }, + { + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 457, + 776 + ], + "color": "#a1309b", + "font_size": 24 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650712, + "offset": [ + 250.2298948633902, + 399.72391778748613 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1.5/cogvideoxfunv1.5_workflow_i2v.json b/VideoX-Fun/comfyui/cogvideox_fun/v1.5/cogvideoxfunv1.5_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..a90947ad34f7483727c090d3c4fae92940076aaa --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1.5/cogvideoxfunv1.5_workflow_i2v.json @@ -0,0 +1,470 @@ +{ + "last_node_id": 83, + "last_link_id": 46, + "nodes": [ + { + "id": 79, + "type": "Note", + "pos": { + "0": 16, + "1": 460 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（在此上传开始图像）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 80, + "type": "Note", + "pos": { + "0": 20, + "1": -300 + }, + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 43 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1134, + "1": 93 + }, + "size": [ + 390.9534912109375, + 310 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 46, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "CogVideoX-Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "CogVideoX-Fun_00003.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 44 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + }, + { + "id": 83, + "type": "LoadCogVideoXFunModel", + "pos": { + "0": 300, + "1": -294 + }, + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 42 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-V1.5-5b-InP", + "Inpaint", + "model_cpu_offload", + "bf16" + ] + }, + { + "id": 7, + "type": "LoadImage", + "pos": { + "0": 259, + "1": 468 + }, + "size": { + "0": 378.07147216796875, + "1": 314 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 45 + ], + "slot_index": 0, + "shape": 3, + "label": "图像" + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3, + "label": "遮罩" + } + ], + "title": "Start Image（图片到视频的开始图片）", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "firework.png", + "image" + ] + }, + { + "id": 82, + "type": "CogVideoXFunInpaintSampler", + "pos": { + "0": 758, + "1": 93 + }, + "size": { + "0": 336, + "1": 282 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 42 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 43 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 44 + }, + { + "name": "start_img", + "type": "IMAGE", + "link": 45, + "slot_index": 3, + "shape": 7 + }, + { + "name": "end_img", + "type": "IMAGE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 46 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunInpaintSampler" + }, + "widgets_values": [ + 85, + 512, + 43, + "fixed", + 50, + 6, + "DDIM" + ] + } + ], + "links": [ + [ + 42, + 83, + 0, + 82, + 0, + "FunModels" + ], + [ + 43, + 75, + 0, + 82, + 1, + "STRING_PROMPT" + ], + [ + 44, + 73, + 0, + 82, + 2, + "STRING_PROMPT" + ], + [ + 45, + 7, + 0, + 82, + 3, + "IMAGE" + ], + [ + 46, + 82, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Upload Your Start Image", + "bounding": [ + 218, + 382, + 452, + 418 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015777, + "offset": [ + 253.7221767223781, + 421.7578981321543 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1.5/cogvideoxfunv1.5_workflow_t2v.json b/VideoX-Fun/comfyui/cogvideox_fun/v1.5/cogvideoxfunv1.5_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..3f444c19177c07c01de5ee25980a442b1eed83e3 --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1.5/cogvideoxfunv1.5_workflow_t2v.json @@ -0,0 +1,372 @@ +{ + "last_node_id": 88, + "last_link_id": 52, + "nodes": [ + { + "id": 80, + "type": "Note", + "pos": { + "0": 20, + "1": -300 + }, + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 50 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1110, + "1": -67 + }, + "size": [ + 390.9534912109375, + 310 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 52, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "CogVideoX-Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "CogVideoX-Fun_00004.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 51 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + }, + { + "id": 87, + "type": "LoadCogVideoXFunModel", + "pos": { + "0": 302, + "1": -285 + }, + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 49 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-V1.5-5b-InP", + "Inpaint", + "model_cpu_offload", + "bf16" + ] + }, + { + "id": 88, + "type": "CogVideoXFunT2VSampler", + "pos": { + "0": 728, + "1": -68 + }, + "size": { + "0": 327.6000061035156, + "1": 290 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 49 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 50 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 51, + "slot_index": 2 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 52 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunT2VSampler" + }, + "widgets_values": [ + 85, + 672, + 384, + "model_cpu_offload", + 43, + "fixed", + 50, + 6, + "DDIM" + ] + } + ], + "links": [ + [ + 49, + 87, + 0, + 88, + 0, + "FunModels" + ], + [ + 50, + 75, + 0, + 88, + 1, + "STRING_PROMPT" + ], + [ + 51, + 73, + 0, + 88, + 2, + "STRING_PROMPT" + ], + [ + 52, + 88, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015777, + "offset": [ + 302.3920634411272, + 585.6684684446539 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1.5/cogvideoxfunv1.5_workflow_v2v.json b/VideoX-Fun/comfyui/cogvideox_fun/v1.5/cogvideoxfunv1.5_workflow_v2v.json new file mode 100644 index 0000000000000000000000000000000000000000..c5e41f961121753722ee3468f27a75e26dddc4bc --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1.5/cogvideoxfunv1.5_workflow_v2v.json @@ -0,0 +1,512 @@ +{ + "last_node_id": 90, + "last_link_id": 57, + "nodes": [ + { + "id": 80, + "type": "Note", + "pos": { + "0": 20, + "1": -300 + }, + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": 15.739953994750977, + "1": 462.38665771484375 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": { + "0": 336, + "1": 470 + }, + "size": [ + 235.1999969482422, + 262 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 56 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "00000125.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "00000125.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 54 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A cute cat is playing the guitar." + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 55 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1125, + "1": 15 + }, + "size": [ + 390.9534912109375, + 310 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 57, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "EasyAnimate", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "EasyAnimate_00045.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 88, + "type": "LoadCogVideoXFunModel", + "pos": { + "0": 309, + "1": -286 + }, + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 53 + ], + "slot_index": 0, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-V1.5-5b-InP", + "Inpaint", + "model_cpu_offload", + "bf16" + ] + }, + { + "id": 90, + "type": "CogVideoXFunV2VSampler", + "pos": { + "0": 754, + "1": 14 + }, + "size": { + "0": 336, + "1": 306 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 53 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 54 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 55 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": 56, + "slot_index": 3, + "shape": 7 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 57 + ], + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunV2VSampler" + }, + "widgets_values": [ + 45, + 512, + 43, + "randomize", + 50, + 6, + 0.7, + "DDIM" + ] + } + ], + "links": [ + [ + 53, + 88, + 0, + 90, + 0, + "FunModels" + ], + [ + 54, + 75, + 0, + 90, + 1, + "STRING_PROMPT" + ], + [ + 55, + 73, + 0, + 90, + 2, + "STRING_PROMPT" + ], + [ + 56, + 85, + 0, + 90, + 3, + "IMAGE" + ], + [ + 57, + 90, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 456, + 498 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015777, + "offset": [ + 258.3078876598779, + 405.0892028196541 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1/cogvideoxfunv1_workflow_i2v.json b/VideoX-Fun/comfyui/cogvideox_fun/v1/cogvideoxfunv1_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..52b7e04cf444c086aa57f6d85f59431ef5ac2ff0 --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1/cogvideoxfunv1_workflow_i2v.json @@ -0,0 +1,451 @@ +{ + "last_node_id": 83, + "last_link_id": 46, + "nodes": [ + { + "id": 7, + "type": "LoadImage", + "pos": [ + 258.76883544921907, + 468.15773315429715 + ], + "size": [ + 378.07147216796875, + 314.0000114440918 + ], + "flags": {}, + "order": 0, + "mode": 0, + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 45 + ], + "shape": 3, + "label": "图像", + "slot_index": 0 + }, + { + "name": "MASK", + "type": "MASK", + "links": null, + "shape": 3, + "label": "遮罩" + } + ], + "title": "Start Image（图片到视频的开始图片）", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "firework.png", + "image" + ] + }, + { + "id": 79, + "type": "Note", + "pos": [ + 16, + 460 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（在此上传开始图像）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 80, + "type": "Note", + "pos": [ + 20, + -300 + ], + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 43 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 44 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + }, + { + "id": 82, + "type": "CogVideoXFunInpaintSampler", + "pos": [ + 758, + 93 + ], + "size": { + "0": 336, + "1": 282 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 42 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 43 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 44 + }, + { + "name": "start_img", + "type": "IMAGE", + "link": 45, + "slot_index": 3 + }, + { + "name": "end_img", + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 46 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunInpaintSampler" + }, + "widgets_values": [ + 49, + 512, + 43, + "fixed", + 50, + 6, + "DDIM" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1134, + 93 + ], + "size": [ + 390.9534912109375, + 535.9734235491071 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 46, + "label": "图像", + "slot_index": 0 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null, + "label": "音频" + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理" + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3, + "label": "文件名", + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "CogVideoX-Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "CogVideoX-Fun_00003.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 83, + "type": "LoadCogVideoXFunModel", + "pos": [ + 300, + -294 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 42 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-2b-InP", + "Inpaint", + "model_cpu_offload", + "bf16" + ] + } + ], + "links": [ + [ + 42, + 83, + 0, + 82, + 0, + "FunModels" + ], + [ + 43, + 75, + 0, + 82, + 1, + "STRING_PROMPT" + ], + [ + 44, + 73, + 0, + 82, + 2, + "STRING_PROMPT" + ], + [ + 45, + 7, + 0, + 82, + 3, + "IMAGE" + ], + [ + 46, + 82, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24 + }, + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24 + }, + { + "title": "Upload Your Start Image", + "bounding": [ + 218, + 382, + 452, + 418 + ], + "color": "#a1309b", + "font_size": 24 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015778, + "offset": [ + 265.8612156262443, + 436.6199667078462 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1/cogvideoxfunv1_workflow_t2v.json b/VideoX-Fun/comfyui/cogvideox_fun/v1/cogvideoxfunv1_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..a4d1b6c46dcfaae426d9988b1174a4a592afb318 --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1/cogvideoxfunv1_workflow_t2v.json @@ -0,0 +1,359 @@ +{ + "last_node_id": 88, + "last_link_id": 52, + "nodes": [ + { + "id": 80, + "type": "Note", + "pos": [ + 20, + -300 + ], + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 0, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 2, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 51 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 3, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 50 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 88, + "type": "CogVideoXFunT2VSampler", + "pos": [ + 728, + -68 + ], + "size": { + "0": 327.6000061035156, + "1": 290 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 49 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 50 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 51, + "slot_index": 2 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 52 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunT2VSampler" + }, + "widgets_values": [ + 49, + 672, + 384, + false, + 43, + "fixed", + 50, + 6, + "DDIM" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1110, + -67 + ], + "size": [ + 390.9534912109375, + 535.9734235491071 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 52, + "label": "图像", + "slot_index": 0 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null, + "label": "音频" + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理" + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3, + "label": "文件名", + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "CogVideoX-Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "CogVideoX-Fun_00004.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 87, + "type": "LoadCogVideoXFunModel", + "pos": [ + 302, + -285 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 49 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-2b-InP", + "Inpaint", + "model_cpu_offload", + "bf16" + ] + } + ], + "links": [ + [ + 49, + 87, + 0, + 88, + 0, + "FunModels" + ], + [ + 50, + 75, + 0, + 88, + 1, + "STRING_PROMPT" + ], + [ + 51, + 73, + 0, + 88, + 2, + "STRING_PROMPT" + ], + [ + 52, + 88, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24 + }, + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 171.40912687862968, + 545.5627520384072 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/cogvideox_fun/v1/cogvideoxfunv1_workflow_v2v.json b/VideoX-Fun/comfyui/cogvideox_fun/v1/cogvideoxfunv1_workflow_v2v.json new file mode 100644 index 0000000000000000000000000000000000000000..84c99e15d9b8a5821c447ef4f55ef8751b989a83 --- /dev/null +++ b/VideoX-Fun/comfyui/cogvideox_fun/v1/cogvideoxfunv1_workflow_v2v.json @@ -0,0 +1,489 @@ +{ + "last_node_id": 88, + "last_link_id": 52, + "nodes": [ + { + "id": 80, + "type": "Note", + "pos": [ + 20, + -300 + ], + "size": { + "0": 210, + "1": 66.98204040527344 + }, + "flags": {}, + "order": 0, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "Load model here\n（在此选择要使用的模型）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.739953613281248, + 462.38664912015946 + ], + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 336, + 470 + ], + "size": [ + 235.1999969482422, + 398.971426827567 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 49 + ], + "shape": 3, + "slot_index": 0 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "00000125.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "00000125.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 4, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 50 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "The video is not of a high quality, it has a low resolution. Watermark present in each frame. Strange motion trajectory. " + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 5, + "mode": 0, + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 51 + ], + "shape": 3, + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "A cute cat is playing the guitar." + ] + }, + { + "id": 87, + "type": "CogVideoXFunV2VSampler", + "pos": [ + 778, + 93 + ], + "size": { + "0": 336, + "1": 286 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "link": 52 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 51, + "slot_index": 1 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 50 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": 49, + "slot_index": 3 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 48 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "CogVideoXFunV2VSampler" + }, + "widgets_values": [ + 49, + 512, + 43, + "fixed", + 50, + 6, + 0.7, + "DDIM" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1134, + 93 + ], + "size": [ + 390.9534912109375, + 535.9734235491071 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 48, + "label": "图像", + "slot_index": 0 + }, + { + "name": "audio", + "type": "VHS_AUDIO", + "link": null, + "label": "音频" + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理" + }, + { + "name": "vae", + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "shape": 3, + "label": "文件名", + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 8, + "loop_count": 0, + "filename_prefix": "EasyAnimate", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "EasyAnimate_00045.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 88, + "type": "LoadCogVideoXFunModel", + "pos": [ + 309, + -286 + ], + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 6, + "mode": 0, + "outputs": [ + { + "name": "cogvideoxfun_model", + "type": "FunModels", + "links": [ + 52 + ], + "shape": 3, + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadCogVideoXFunModel" + }, + "widgets_values": [ + "CogVideoX-Fun-2b-InP", + "Inpaint", + "model_cpu_offload", + "bf16" + ] + } + ], + "links": [ + [ + 48, + 87, + 0, + 17, + 0, + "IMAGE" + ], + [ + 49, + 85, + 0, + 87, + 3, + "IMAGE" + ], + [ + 50, + 73, + 0, + 87, + 2, + "STRING_PROMPT" + ], + [ + 51, + 75, + 0, + 87, + 1, + "STRING_PROMPT" + ], + [ + 52, + 88, + 0, + 87, + 0, + "FunModels" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24 + }, + { + "title": "Load CogVideoX-Fun", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24 + }, + { + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 456, + 498 + ], + "color": "#a1309b", + "font_size": 24 + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.683013455365071, + "offset": [ + 322.5575500900931, + 444.05399028364593 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/comfyui_nodes.py b/VideoX-Fun/comfyui/comfyui_nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..7dde5c23e2641d27a70eef3605f6f7e9713d345e --- /dev/null +++ b/VideoX-Fun/comfyui/comfyui_nodes.py @@ -0,0 +1,579 @@ +import json +import os + +import cv2 +import numpy as np +import torch +import torch.nn.functional as F +from omegaconf import OmegaConf + +from .annotator.nodes import VideoToCanny, VideoToDepth, VideoToPose +from .camera_utils import CAMERA, combine_camera_motion, get_camera_motion +from .cogvideox_fun.nodes import (CogVideoXFunInpaintSampler, + CogVideoXFunT2VSampler, + CogVideoXFunV2VSampler, LoadCogVideoXFunLora, + LoadCogVideoXFunModel) +from .comfyui_utils import script_directory +from .qwenimage.nodes import (CombineQwenImagePipeline, LoadQwenImageLora, + LoadQwenImageModel, LoadQwenImageProcessor, QwenImageEditSampler, + LoadQwenImageTextEncoderModel, + LoadQwenImageTransformerModel, + LoadQwenImageVAEModel, QwenImageT2VSampler) +from .wan2_1.nodes import (CombineWanPipeline, LoadWanClipEncoderModel, + LoadWanLora, LoadWanModel, LoadWanTextEncoderModel, + LoadWanTransformerModel, LoadWanVAEModel, + WanI2VSampler, WanT2VSampler) +from .wan2_1_fun.nodes import (LoadWanFunLora, LoadWanFunModel, + WanFunInpaintSampler, WanFunT2VSampler, + WanFunV2VSampler) +from .wan2_2.nodes import (CombineWan2_2Pipeline, LoadWan2_2Lora, + LoadWan2_2Model, LoadWan2_2TransformerModel, + Wan2_2I2VSampler, Wan2_2T2VSampler) +from .wan2_2_fun.nodes import (LoadWan2_2FunLora, LoadWan2_2FunModel, + Wan2_2FunInpaintSampler, Wan2_2FunT2VSampler, + Wan2_2FunV2VSampler) +from .wan2_2_vace_fun.nodes import (CombineWan2_2VaceFunPipeline, + LoadVaceWanTransformer3DModel, + LoadWan2_2VaceFunModel, + Wan2_2VaceFunSampler) + + +class FunTextBox: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "prompt": ("STRING", {"multiline": True, "default": "",}), + }, + } + + RETURN_TYPES = ("STRING_PROMPT",) + RETURN_NAMES =("prompt",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, prompt): + return (prompt, ) + +class FunRiflex: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "riflex_k": ("INT", {"default": 6, "min": 0, "max": 10086}), + }, + } + + RETURN_TYPES = ("RIFLEXT_ARGS",) + RETURN_NAMES = ("riflex_k",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, riflex_k): + return (riflex_k, ) + +class FunCompile: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "cache_size_limit": ("INT", {"default": 64, "min": 0, "max": 10086}), + "funmodels": ("FunModels",) + } + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "compile" + CATEGORY = "CogVideoXFUNWrapper" + + def compile(self, cache_size_limit, funmodels): + torch._dynamo.config.cache_size_limit = cache_size_limit + + if funmodels["pipeline"].transformer.device == torch.device(type="meta"): + if hasattr(funmodels["pipeline"].transformer, "blocks"): + for i, block in enumerate(funmodels["pipeline"].transformer.blocks): + if hasattr(block, "_orig_mod"): + block = block._orig_mod + + if hasattr(funmodels["pipeline"], "transformer_2") and funmodels["pipeline"].transformer_2 is not None: + for i, block in enumerate(funmodels["pipeline"].transformer_2.blocks): + if hasattr(block, "_orig_mod"): + block = block._orig_mod + + elif hasattr(funmodels["pipeline"].transformer, "transformer_blocks"): + for i, block in enumerate(funmodels["pipeline"].transformer.transformer_blocks): + if hasattr(block, "_orig_mod"): + block = block._orig_mod + + if hasattr(funmodels["pipeline"], "transformer_2") and funmodels["pipeline"].transformer_2 is not None: + for i, block in enumerate(funmodels["pipeline"].transformer_2.transformer_blocks): + if hasattr(block, "_orig_mod"): + block = block._orig_mod + + print("Sequential cpu offload can not work with compile. Continue") + return (funmodels,) + + if hasattr(funmodels["pipeline"].transformer, "blocks"): + for i, block in enumerate(funmodels["pipeline"].transformer.blocks): + if hasattr(block, "_orig_mod"): + block = block._orig_mod + funmodels["pipeline"].transformer.blocks[i] = torch.compile(block) + + if hasattr(funmodels["pipeline"], "transformer_2") and funmodels["pipeline"].transformer_2 is not None: + for i, block in enumerate(funmodels["pipeline"].transformer_2.blocks): + if hasattr(block, "_orig_mod"): + block = block._orig_mod + funmodels["pipeline"].transformer_2.blocks[i] = torch.compile(block) + + elif hasattr(funmodels["pipeline"].transformer, "transformer_blocks"): + for i, block in enumerate(funmodels["pipeline"].transformer.transformer_blocks): + if hasattr(block, "_orig_mod"): + block = block._orig_mod + funmodels["pipeline"].transformer.transformer_blocks[i] = torch.compile(block) + + if hasattr(funmodels["pipeline"], "transformer_2") and funmodels["pipeline"].transformer_2 is not None: + for i, block in enumerate(funmodels["pipeline"].transformer_2.transformer_blocks): + if hasattr(block, "_orig_mod"): + block = block._orig_mod + funmodels["pipeline"].transformer_2.transformer_blocks[i] = torch.compile(block) + + else: + funmodels["pipeline"].transformer.forward = torch.compile(funmodels["pipeline"].transformer.forward) + + if hasattr(funmodels["pipeline"], "transformer_2") and funmodels["pipeline"].transformer_2 is not None: + funmodels["pipeline"].transformer_2.forward = torch.compile(funmodels["pipeline"].transformer_2.forward) + + print("Add Compile") + return (funmodels,) + +class FunAttention: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "attention_type": ( + ["flash", "sage", "torch"], + {"default": "flash"}, + ), + "funmodels": ("FunModels",) + } + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "funattention" + CATEGORY = "CogVideoXFUNWrapper" + + def funattention(self, attention_type, funmodels): + os.environ['VIDEOX_ATTENTION_TYPE'] = { + "flash": "FLASH_ATTENTION", + "sage": "SAGE_ATTENTION", + "torch": "TORCH_SCALED_DOT" + }[attention_type] + return (funmodels,) + +class LoadConfig: + @classmethod + def INPUT_TYPES(s): + return { + "required": ( + [ + "wan2.1/wan_civitai.yaml", + "wan2.2/wan_civitai_t2v.yaml", + "wan2.2/wan_civitai_i2v.yaml", + "wan2.2/wan_civitai_5b.yaml", + ], + { + "default": "wan2.2/wan_civitai_i2v.yaml", + } + ), + } + + RETURN_TYPES = ("FunConfig",) + RETURN_NAMES = ("config",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, config): + # Load config + config_path = f"{script_directory}/config/{config}" + config = OmegaConf.load(config_path) + return (config, ) + +def gen_gaussian_heatmap(imgSize=200): + circle_img = np.zeros((imgSize, imgSize,), np.float32) + circle_mask = cv2.circle(circle_img, (imgSize//2, imgSize//2), imgSize//2 - 1, 1, -1) + + isotropicGrayscaleImage = np.zeros((imgSize, imgSize), np.float32) + + # 生成高斯图 + for i in range(imgSize): + for j in range(imgSize): + isotropicGrayscaleImage[i, j] = 1 / (2 * np.pi * (40 ** 2)) * np.exp( + -1 / 2 * ((i - imgSize / 2) ** 2 / (40 ** 2) + (j - imgSize / 2) ** 2 / (40 ** 2))) + + isotropicGrayscaleImage = isotropicGrayscaleImage * circle_mask + isotropicGrayscaleImage = (isotropicGrayscaleImage / np.max(isotropicGrayscaleImage) * 255).astype(np.uint8) + return isotropicGrayscaleImage + +class CreateTrajectoryBasedOnKJNodes: + # Modified from https://github.com/kijai/ComfyUI-KJNodes/blob/main/nodes/curve_nodes.py + # Modify to meet the trajectory control requirements of EasyAnimate. + RETURN_TYPES = ("IMAGE", ) + RETURN_NAMES = ("image", ) + FUNCTION = "createtrajectory" + CATEGORY = "CogVideoXFUNWrapper" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "coordinates": ("STRING", {"forceInput": True}), + "masks": ("MASK", {"forceInput": True}), + }, + } + + def createtrajectory(self, coordinates, masks): + # Define the number of images in the batch + if len(coordinates) < 10: + coords_list = [] + for coords in coordinates: + coords = json.loads(coords.replace("'", '"')) + coords_list.append(coords) + else: + coords = json.loads(coordinates.replace("'", '"')) + coords_list = [coords] + + _, frame_height, frame_width = masks.size() + heatmap = gen_gaussian_heatmap() + + circle_size = int(50 * ((frame_height * frame_width) / (1280 * 720)) ** (1/2)) + + images_list = [] + for coords in coords_list: + _images_list = [] + for i in range(len(coords)): + _image = np.zeros((frame_height, frame_width, 3)) + center_coordinate = [coords[i][key] for key in coords[i]] + + y1 = max(center_coordinate[1] - circle_size, 0) + y2 = min(center_coordinate[1] + circle_size, np.shape(_image)[0] - 1) + x1 = max(center_coordinate[0] - circle_size, 0) + x2 = min(center_coordinate[0] + circle_size, np.shape(_image)[1] - 1) + + if x2 - x1 > 3 and y2 - y1 > 3: + need_map = cv2.resize(heatmap, (x2 - x1, y2 - y1))[:, :, None] + _image[y1:y2, x1:x2] = np.maximum(need_map.copy(), _image[y1:y2, x1:x2]) + + _image = np.expand_dims(_image, 0) / 255 + _images_list.append(_image) + images_list.append(np.concatenate(_images_list, axis=0)) + + out_images = torch.from_numpy(np.max(np.array(images_list), axis=0)) + return (out_images, ) + +class ImageMaximumNode: + RETURN_TYPES = ("IMAGE", ) + RETURN_NAMES = ("image", ) + FUNCTION = "imagemaximum" + CATEGORY = "CogVideoXFUNWrapper" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "video_1": ("IMAGE",), + "video_2": ("IMAGE",), + }, + } + + def imagemaximum(self, video_1, video_2): + length_1, h_1, w_1, c_1 = video_1.size() + length_2, h_2, w_2, c_2 = video_2.size() + + if h_1 != h_2 or w_1 != w_2: + video_1, video_2 = video_1.permute([0, 3, 1, 2]), video_2.permute([0, 3, 1, 2]) + video_2 = F.interpolate(video_2, video_1.size()[-2:]) + video_1, video_2 = video_1.permute([0, 2, 3, 1]), video_2.permute([0, 2, 3, 1]) + + if length_1 > length_2: + outputs = torch.maximum(video_1[:length_2], video_2) + else: + outputs = torch.maximum(video_1, video_2[:length_1]) + return (outputs, ) + +class ImageCollectNode: + RETURN_TYPES = ("IMAGE", ) + RETURN_NAMES = ("image", ) + FUNCTION = "imagecollect" + CATEGORY = "CogVideoXFUNWrapper" + + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "image_1": ("IMAGE",) + }, + "optional": { + "image_2": ("IMAGE",), + } + } + + def imagecollect(self, image_1, image_2): + image_out = [_image_1 for _image_1 in image_1] + [_image_2 for _image_2 in image_2] + return (image_out, ) + +class CameraBasicFromChaoJie: + # Copied from https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/blob/main/nodes.py + # Since ComfyUI-CameraCtrl-Wrapper requires a specific version of diffusers, which is not suitable for us. + # The code has been copied into the current repository. + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "camera_pose":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}), + "speed":("FLOAT",{"default":1.0}), + "video_length":("INT",{"default":16}), + }, + } + + RETURN_TYPES = ("CameraPose",) + FUNCTION = "run" + CATEGORY = "CameraCtrl" + + def run(self,camera_pose,speed,video_length): + camera_dict = { + "motion":[camera_pose], + "mode": "Basic Camera Poses", # "First A then B", "Both A and B", "Custom" + "speed": speed, + "complex": None + } + motion_list = camera_dict['motion'] + mode = camera_dict['mode'] + speed = camera_dict['speed'] + angle = np.array(CAMERA[motion_list[0]]["angle"]) + T = np.array(CAMERA[motion_list[0]]["T"]) + RT = get_camera_motion(angle, T, speed, video_length) + return (RT,) + +class CameraCombineFromChaoJie: + # Copied from https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/blob/main/nodes.py + # Since ComfyUI-CameraCtrl-Wrapper requires a specific version of diffusers, which is not suitable for us. + # The code has been copied into the current repository. + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "camera_pose1":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}), + "camera_pose2":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}), + "camera_pose3":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}), + "camera_pose4":(["Static","Pan Up","Pan Down","Pan Left","Pan Right","Zoom In","Zoom Out","ACW","CW"],{"default":"Static"}), + "speed":("FLOAT",{"default":1.0}), + "video_length":("INT",{"default":16}), + }, + } + + RETURN_TYPES = ("CameraPose",) + FUNCTION = "run" + CATEGORY = "CameraCtrl" + + def run(self,camera_pose1,camera_pose2,camera_pose3,camera_pose4,speed,video_length): + angle = np.array(CAMERA[camera_pose1]["angle"]) + np.array(CAMERA[camera_pose2]["angle"]) + np.array(CAMERA[camera_pose3]["angle"]) + np.array(CAMERA[camera_pose4]["angle"]) + T = np.array(CAMERA[camera_pose1]["T"]) + np.array(CAMERA[camera_pose2]["T"]) + np.array(CAMERA[camera_pose3]["T"]) + np.array(CAMERA[camera_pose4]["T"]) + RT = get_camera_motion(angle, T, speed, video_length) + return (RT,) + +class CameraJoinFromChaoJie: + # Copied from https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/blob/main/nodes.py + # Since ComfyUI-CameraCtrl-Wrapper requires a specific version of diffusers, which is not suitable for us. + # The code has been copied into the current repository. + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "camera_pose1":("CameraPose",), + "camera_pose2":("CameraPose",), + }, + } + + RETURN_TYPES = ("CameraPose",) + FUNCTION = "run" + CATEGORY = "CameraCtrl" + + def run(self,camera_pose1,camera_pose2): + RT = combine_camera_motion(camera_pose1, camera_pose2) + return (RT,) + +class CameraTrajectoryFromChaoJie: + # Copied from https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/blob/main/nodes.py + # Since ComfyUI-CameraCtrl-Wrapper requires a specific version of diffusers, which is not suitable for us. + # The code has been copied into the current repository. + @classmethod + def INPUT_TYPES(cls): + return { + "required": { + "camera_pose":("CameraPose",), + "fx":("FLOAT",{"default":0.474812461, "min": 0, "max": 1, "step": 0.000000001}), + "fy":("FLOAT",{"default":0.844111024, "min": 0, "max": 1, "step": 0.000000001}), + "cx":("FLOAT",{"default":0.5, "min": 0, "max": 1, "step": 0.01}), + "cy":("FLOAT",{"default":0.5, "min": 0, "max": 1, "step": 0.01}), + }, + } + + RETURN_TYPES = ("STRING","INT",) + RETURN_NAMES = ("camera_trajectory","video_length",) + FUNCTION = "run" + CATEGORY = "CameraCtrl" + + def run(self,camera_pose,fx,fy,cx,cy): + #print(camera_pose) + camera_pose_list=camera_pose.tolist() + trajs=[] + for cp in camera_pose_list: + traj=[fx,fy,cx,cy,0,0] + traj.extend(cp[0]) + traj.extend(cp[1]) + traj.extend(cp[2]) + trajs.append(traj) + return (json.dumps(trajs),len(trajs),) + +NODE_CLASS_MAPPINGS = { + "FunTextBox": FunTextBox, + "FunRiflex": FunRiflex, + "FunCompile": FunCompile, + "FunAttention": FunAttention, + + "LoadCogVideoXFunModel": LoadCogVideoXFunModel, + "LoadCogVideoXFunLora": LoadCogVideoXFunLora, + "CogVideoXFunT2VSampler": CogVideoXFunT2VSampler, + "CogVideoXFunInpaintSampler": CogVideoXFunInpaintSampler, + "CogVideoXFunV2VSampler": CogVideoXFunV2VSampler, + + "LoadQwenImageLora": LoadQwenImageLora, + "LoadQwenImageTextEncoderModel": LoadQwenImageTextEncoderModel, + "LoadQwenImageTransformerModel": LoadQwenImageTransformerModel, + "LoadQwenImageVAEModel": LoadQwenImageVAEModel, + "LoadQwenImageProcessor": LoadQwenImageProcessor, + "CombineQwenImagePipeline": CombineQwenImagePipeline, + + "LoadQwenImageModel": LoadQwenImageModel, + "QwenImageT2VSampler": QwenImageT2VSampler, + "QwenImageEditSampler": QwenImageEditSampler, + + "LoadWanClipEncoderModel": LoadWanClipEncoderModel, + "LoadWanTextEncoderModel": LoadWanTextEncoderModel, + "LoadWanTransformerModel": LoadWanTransformerModel, + "LoadWanVAEModel": LoadWanVAEModel, + "CombineWanPipeline": CombineWanPipeline, + "LoadWan2_2TransformerModel": LoadWan2_2TransformerModel, + "CombineWan2_2Pipeline": CombineWan2_2Pipeline, + + "LoadWanModel": LoadWanModel, + "LoadWanLora": LoadWanLora, + "WanT2VSampler": WanT2VSampler, + "WanI2VSampler": WanI2VSampler, + + "LoadWanFunModel": LoadWanFunModel, + "LoadWanFunLora": LoadWanFunLora, + "WanFunT2VSampler": WanFunT2VSampler, + "WanFunInpaintSampler": WanFunInpaintSampler, + "WanFunV2VSampler": WanFunV2VSampler, + + "LoadWan2_2Model": LoadWan2_2Model, + "LoadWan2_2Lora": LoadWan2_2Lora, + "Wan2_2T2VSampler": Wan2_2T2VSampler, + "Wan2_2I2VSampler": Wan2_2I2VSampler, + + "LoadWan2_2FunModel": LoadWan2_2FunModel, + "LoadWan2_2FunLora": LoadWan2_2FunLora, + "Wan2_2FunT2VSampler": Wan2_2FunT2VSampler, + "Wan2_2FunInpaintSampler": Wan2_2FunInpaintSampler, + "Wan2_2FunV2VSampler": Wan2_2FunV2VSampler, + + "LoadVaceWanTransformer3DModel": LoadVaceWanTransformer3DModel, + "CombineWan2_2VaceFunPipeline": CombineWan2_2VaceFunPipeline, + + "LoadWan2_2VaceFunModel": LoadWan2_2VaceFunModel, + "Wan2_2VaceFunSampler": Wan2_2VaceFunSampler, + + "VideoToCanny": VideoToCanny, + "VideoToDepth": VideoToDepth, + "VideoToOpenpose": VideoToPose, + + "CreateTrajectoryBasedOnKJNodes": CreateTrajectoryBasedOnKJNodes, + "CameraBasicFromChaoJie": CameraBasicFromChaoJie, + "CameraTrajectoryFromChaoJie": CameraTrajectoryFromChaoJie, + "CameraJoinFromChaoJie": CameraJoinFromChaoJie, + "CameraCombineFromChaoJie": CameraCombineFromChaoJie, + "ImageMaximumNode": ImageMaximumNode, + "ImageCollectNode": ImageCollectNode, +} +NODE_DISPLAY_NAME_MAPPINGS = { + "FunTextBox": "FunTextBox", + "FunRiflex": "FunRiflex", + "FunCompile": "FunCompile", + "FunAttention": "FunAttention", + + "LoadCogVideoXFunModel": "Load CogVideoX-Fun Model", + "LoadCogVideoXFunLora": "Load CogVideoX-Fun Lora", + "CogVideoXFunInpaintSampler": "CogVideoX-Fun Sampler for Image to Video", + "CogVideoXFunT2VSampler": "CogVideoX-Fun Sampler for Text to Video", + "CogVideoXFunV2VSampler": "CogVideoX-Fun Sampler for Video to Video", + + "LoadQwenImageLora": "Load QwenImage Lora", + "LoadQwenImageTextEncoderModel": "Load QwenImage TextEncoder Model", + "LoadQwenImageTransformerModel": "Load QwenImage Transformer Model", + "LoadQwenImageVAEModel": "Load QwenImage VAE Model", + "LoadQwenImageProcessor": "Load QwenImage Processor", + "CombineQwenImagePipeline": "Combine QwenImage Pipeline", + + "LoadQwenImageModel": "Load QwenImage Model", + "QwenImageT2VSampler": "QwenImage T2V Sampler", + "QwenImageEditSampler": "QwenImage Edit Sampler", + + "LoadWanClipEncoderModel": "Load Wan ClipEncoder Model", + "LoadWanTextEncoderModel": "Load Wan TextEncoder Model", + "LoadWanTransformerModel": "Load Wan Transformer Model", + "LoadWanVAEModel": "Load Wan VAE Model", + "CombineWanPipeline": "Combine Wan Pipeline", + "LoadWan2_2TransformerModel": "Load Wan2_2 Transformer Model", + "CombineWan2_2Pipeline": "Combine Wan2_2 Pipeline", + "LoadVaceWanTransformer3DModel": "Load Vace Wan Transformer 3DModel", + "CombineWan2_2VaceFunPipeline": "Combine Wan2_2 Vace Fun Pipeline", + + "LoadWanModel": "Load Wan Model", + "LoadWanLora": "Load Wan Lora", + "WanT2VSampler": "Wan Sampler for Text to Video", + "WanI2VSampler": "Wan Sampler for Image to Video", + + "LoadWanFunModel": "Load Wan Fun Model", + "LoadWanFunLora": "Load Wan Fun Lora", + "WanFunT2VSampler": "Wan Fun Sampler for Text to Video", + "WanFunInpaintSampler": "Wan Fun Sampler for Image to Video", + "WanFunV2VSampler": "Wan Fun Sampler for Video to Video", + + "LoadWan2_2Model": "Load Wan 2.2 Model", + "LoadWan2_2Lora": "Load Wan 2.2 Lora", + "Wan2_2T2VSampler": "Wan 2.2 Sampler for Text to Video", + "Wan2_2I2VSampler": "Wan 2.2 Sampler for Image to Video", + + "LoadWan2_2FunModel": "Load Wan 2.2 Fun Model", + "LoadWan2_2FunLora": "Load Wan 2.2 Fun Lora", + "Wan2_2FunT2VSampler": "Wan 2.2 Fun Sampler for Text to Video", + "Wan2_2FunInpaintSampler": "Wan 2.2 Fun Sampler for Image to Video", + "Wan2_2FunV2VSampler": "Wan 2.2 Fun Sampler for Video to Video", + + "LoadWan2_2VaceFunModel": "Load Wan2_2 Vace Fun Model", + "Wan2_2VaceFunSampler": "Wan2_2 Vace Fun Sampler", + + "VideoToCanny": "Video To Canny", + "VideoToDepth": "Video To Depth", + "VideoToOpenpose": "Video To Pose", + + "CreateTrajectoryBasedOnKJNodes": "Create Trajectory Based On KJNodes", + "CameraBasicFromChaoJie": "Camera Basic From ChaoJie", + "CameraTrajectoryFromChaoJie": "Camera Trajectory From ChaoJie", + "CameraJoinFromChaoJie": "Camera Join From ChaoJie", + "CameraCombineFromChaoJie": "Camera Combine From ChaoJie", + "ImageMaximumNode": "Image Maximum Node", + "ImageCollectNode": "Image Collect Node", +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/comfyui_utils.py b/VideoX-Fun/comfyui/comfyui_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0a51b10fa845f427c116e07ce34bf4ebcb885d4b --- /dev/null +++ b/VideoX-Fun/comfyui/comfyui_utils.py @@ -0,0 +1,88 @@ +import os + +import folder_paths +import numpy as np +import torch +from PIL import Image + +# Compatible with Alibaba EAS for quick launch +eas_cache_dir = '/stable-diffusion-cache/models' +# The directory of the cogvideoxfun +script_directory = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +def tensor2pil(image): + return Image.fromarray(np.clip(255. * image.cpu().numpy(), 0, 255).astype(np.uint8)) + +def numpy2pil(image): + return Image.fromarray(np.clip(255. * image, 0, 255).astype(np.uint8)) + +def to_pil(image): + if isinstance(image, Image.Image): + return image + if isinstance(image, torch.Tensor): + return tensor2pil(image) + if isinstance(image, np.ndarray): + return numpy2pil(image) + raise ValueError(f"Cannot convert {type(image)} to PIL.Image") + +def search_model_in_possible_folders(possible_folders, model): + model_name = None + # Check if the model exists in any of the possible folders within folder_paths.models_dir + for folder in possible_folders: + candidate_path = os.path.join(folder_paths.models_dir, folder, model) + if os.path.exists(candidate_path): + model_name = candidate_path + break + + # If model_name is still None, check eas_cache_dir for each possible folder + if model_name is None and os.path.exists(eas_cache_dir): + for folder in possible_folders: + candidate_path = os.path.join(eas_cache_dir, folder, model) + if os.path.exists(candidate_path): + model_name = candidate_path + break + + # If model_name is still None, prompt the user to download the model + if model_name is None: + print(f"Please download cogvideoxfun model to one of the following directories:") + for folder in possible_folders: + print(f"- {os.path.join(folder_paths.models_dir, folder)}") + if os.path.exists(eas_cache_dir): + print(f"- {os.path.join(eas_cache_dir, folder)}") + raise ValueError("Please download Fun model") + + return model_name + +def search_sub_dir_in_possible_folders(possible_folders, sub_dir_name="umt5-xxl"): + new_possible_folders = [] + # Check if the model exists in any of the possible folders within folder_paths.models_dir + for folder in possible_folders: + candidate_path = os.path.join(folder_paths.models_dir, folder) + if os.path.exists(candidate_path) and os.path.isdir(candidate_path): + new_possible_folders.append(candidate_path) + for sub_dir in os.listdir(candidate_path): + new_possible_folders.append(os.path.join(candidate_path, sub_dir)) + + # If model_name is still None, check eas_cache_dir for each possible folder + if os.path.exists(eas_cache_dir): + for folder in possible_folders: + candidate_path = os.path.join(eas_cache_dir, folder) + if os.path.exists(candidate_path) and os.path.isdir(candidate_path): + new_possible_folders.append(candidate_path) + for sub_dir in os.listdir(candidate_path): + new_possible_folders.append(os.path.join(candidate_path, sub_dir)) + + for folder in new_possible_folders + possible_folders: + final_possible_folder = os.path.join(folder, sub_dir_name) + final_possible_folder_basename = os.path.join(folder, os.path.basename(sub_dir_name)) + if os.path.exists(final_possible_folder) and os.path.isdir(final_possible_folder): + return final_possible_folder + if os.path.exists(final_possible_folder_basename) and os.path.isdir(final_possible_folder_basename): + return final_possible_folder_basename + + print(f"Please download {sub_dir_name} tokenizer model to one of the following directories:") + for folder in possible_folders: + print(f"- {os.path.join(folder_paths.models_dir, folder)}") + if os.path.exists(eas_cache_dir): + print(f"- {os.path.join(eas_cache_dir, folder)}") + raise ValueError("Please download Fun model") diff --git a/VideoX-Fun/comfyui/qwenimage/nodes.py b/VideoX-Fun/comfyui/qwenimage/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..1085f32a59ae7172987ac5ec936c1e48f0fbe7a7 --- /dev/null +++ b/VideoX-Fun/comfyui/qwenimage/nodes.py @@ -0,0 +1,1009 @@ +"""Modified from https://github.com/kijai/ComfyUI-EasyAnimateWrapper/blob/main/nodes.py +""" +import copy +import gc +import inspect +import json +import os + +import comfy.model_management as mm +import cv2 +import folder_paths +import numpy as np +import torch +from comfy.utils import ProgressBar, load_torch_file +from diffusers import FlowMatchEulerDiscreteScheduler +from einops import rearrange +from omegaconf import OmegaConf +from PIL import Image + +from ...videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + get_closest_ratio) +from ...videox_fun.models import (AutoencoderKLQwenImage, Qwen2_5_VLConfig, + Qwen2_5_VLForConditionalGeneration, Qwen2VLProcessor, + Qwen2Tokenizer, QwenImageTransformer2DModel) +from ...videox_fun.models.cache_utils import get_teacache_coefficients +from ...videox_fun.pipeline import QwenImagePipeline, QwenImageEditPipeline +from ...videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from ...videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from ...videox_fun.utils.fp8_optimization import ( + convert_model_weight_to_float8, convert_weight_dtype_wrapper, undo_convert_weight_dtype_wrapper, + replace_parameters_by_name) +from ...videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from ...videox_fun.utils.utils import filter_kwargs, get_image, get_autocast_dtype +from ..comfyui_utils import (eas_cache_dir, script_directory, to_pil, + search_model_in_possible_folders, + search_sub_dir_in_possible_folders) + +# Used in lora cache +transformer_cpu_cache = {} +# lora path before +lora_path_before = "" + +def get_qwen_scheduler(sampler_name, shift): + Chosen_Scheduler = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, + }[sampler_name] + scheduler_kwargs = { + "base_image_seq_len": 256, + "base_shift": 0.5, + "invert_sigmas": False, + "max_image_seq_len": 8192, + "max_shift": 0.9, + "num_train_timesteps": 1000, + "shift": 1.0, + "shift_terminal": 0.02, + "stochastic_sampling": False, + "time_shift_type": "exponential", + "use_beta_sigmas": False, + "use_dynamic_shifting": True, + "use_exponential_sigmas": False, + "use_karras_sigmas": False + } + scheduler_kwargs['shift'] = shift + scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, scheduler_kwargs) + ) + return scheduler + +class LoadQwenImageTransformerModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ( + folder_paths.get_filename_list("diffusion_models"), + {"default": "Wan2_1-T2V-1_3B_bf16.safetensors,"}, + ), + "precision": (["fp16", "bf16"], + {"default": "bf16"} + ), + }, + } + RETURN_TYPES = ("TransformerModel", "STRING") + RETURN_NAMES = ("transformer", "model_name") + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, precision): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + transformer = None + + model_path = folder_paths.get_full_path("diffusion_models", model_name) + transformer_state_dict = load_torch_file(model_path, safe_load=True) + + model_name_in_pipeline = "Qwen-Image" + kwargs = { + "attention_head_dim": 128, + "axes_dims_rope": [ + 16, + 56, + 56 + ], + "guidance_embeds": False, + "in_channels": 64, + "joint_attention_dim": 3584, + "num_attention_heads": 24, + "num_layers": 60, + "out_channels": 16, + "patch_size": 2, + "pooled_projection_dim": 768 + } + + sig = inspect.signature(QwenImageTransformer2DModel) + accepted = {k: v for k, v in kwargs.items() if k in sig.parameters} + transformer = QwenImageTransformer2DModel(**accepted) + transformer.load_state_dict(transformer_state_dict) + transformer = transformer.eval().to(device=offload_device, dtype=weight_dtype) + return (transformer, model_name_in_pipeline) + +class LoadQwenImageVAEModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ( + folder_paths.get_filename_list("vae"), + {"default": "QwenImage2.1_VAE.pth"} + ), + "precision": (["fp16", "bf16"], + {"default": "bf16"} + ), + }, + } + + RETURN_TYPES = ("VAEModel",) + RETURN_NAMES = ("vae", ) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, precision,): + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[precision] + model_path = folder_paths.get_full_path("vae", model_name) + vae_state_dict = load_torch_file(model_path, safe_load=True) + + if "conv1.weight" in vae_state_dict: + use_wan_compiled_vae = True + if not any(k.startswith("model.") for k in vae_state_dict.keys()): + vae_state_dict = {f"model.{k}": v for k, v in vae_state_dict.items()} + else: + use_wan_compiled_vae = False + + kwargs = { + "attn_scales": [], + "base_dim": 96, + "dim_mult": [ + 1, + 2, + 4, + 4 + ], + "dropout": 0.0, + "latents_mean": [ + -0.7571, + -0.7089, + -0.9113, + 0.1075, + -0.1745, + 0.9653, + -0.1517, + 1.5508, + 0.4134, + -0.0715, + 0.5517, + -0.3632, + -0.1922, + -0.9497, + 0.2503, + -0.2921 + ], + "latents_std": [ + 2.8184, + 1.4541, + 2.3275, + 2.6558, + 1.2196, + 1.7708, + 2.6052, + 2.0743, + 3.2687, + 2.1526, + 2.8652, + 1.5579, + 1.6382, + 1.1253, + 2.8251, + 1.916 + ], + "num_res_blocks": 2, + "temperal_downsample": [ + False, + True, + True + ], + "z_dim": 16 + } + + sig = inspect.signature(AutoencoderKLQwenImage) + accepted = {k: v for k, v in kwargs.items() if k in sig.parameters} + + if use_wan_compiled_vae: + from ...videox_fun.models.wan_vae import AutoencoderKLWanCompileQwenImage + vae = AutoencoderKLWanCompileQwenImage(**accepted) + else: + vae = AutoencoderKLQwenImage(**accepted) + vae.load_state_dict(vae_state_dict) + vae = vae.eval().to(device=offload_device, dtype=weight_dtype) + return (vae,) + +class LoadQwenImageTextEncoderModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ( + folder_paths.get_filename_list("text_encoders"), + {"default": "models_t5_umt5-xxl-enc-bf16.pth"} + ), + "precision": (["fp16", "bf16"], + {"default": "bf16"} + ), + }, + } + + RETURN_TYPES = ("TextEncoderModel", "Tokenizer") + RETURN_NAMES = ("text_encoder", "tokenizer") + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, precision,): + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[precision] + model_path = folder_paths.get_full_path("text_encoders", model_name) + text_state_dict = load_torch_file(model_path, safe_load=True) + + if not any(k.startswith("model.") for k in text_state_dict.keys()): + text_state_dict = {f"model.{k}": v for k, v in text_state_dict.items()} + + kwargs = { + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "image_token_id": 151655, + "initializer_range": 0.02, + "intermediate_size": 18944, + "max_position_embeddings": 128000, + "max_window_layers": 28, + "model_type": "qwen2_5_vl", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_section": [ + 16, + 24, + 24 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000.0, + "sliding_window": 32768, + "text_config": { + "architectures": [ + "Qwen2_5_VLForConditionalGeneration" + ], + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "hidden_act": "silu", + "hidden_size": 3584, + "image_token_id": None, + "initializer_range": 0.02, + "intermediate_size": 18944, + "layer_types": [ + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention", + "full_attention" + ], + "max_position_embeddings": 128000, + "max_window_layers": 28, + "model_type": "qwen2_5_vl_text", + "num_attention_heads": 28, + "num_hidden_layers": 28, + "num_key_value_heads": 4, + "rms_norm_eps": 1e-06, + "rope_scaling": { + "mrope_section": [ + 16, + 24, + 24 + ], + "rope_type": "default", + "type": "default" + }, + "rope_theta": 1000000.0, + "sliding_window": None, + "torch_dtype": "float32", + "use_cache": True, + "use_sliding_window": False, + "video_token_id": None, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652, + "vision_token_id": 151654, + "vocab_size": 152064 + }, + "tie_word_embeddings": False, + "torch_dtype": "bfloat16", + "transformers_version": "4.53.1", + "use_cache": True, + "use_sliding_window": False, + "video_token_id": 151656, + "vision_config": { + "depth": 32, + "fullatt_block_indexes": [ + 7, + 15, + 23, + 31 + ], + "hidden_act": "silu", + "hidden_size": 1280, + "in_channels": 3, + "in_chans": 3, + "initializer_range": 0.02, + "intermediate_size": 3420, + "model_type": "qwen2_5_vl", + "num_heads": 16, + "out_hidden_size": 3584, + "patch_size": 14, + "spatial_merge_size": 2, + "spatial_patch_size": 14, + "temporal_patch_size": 2, + "tokens_per_second": 2, + "torch_dtype": "float32", + "window_size": 112 + }, + "vision_end_token_id": 151653, + "vision_start_token_id": 151652, + "vision_token_id": 151654, + "vocab_size": 152064 + } + config = Qwen2_5_VLConfig(**kwargs) + text_encoder = Qwen2_5_VLForConditionalGeneration._from_config(config) + def transform_key(key): + key = key.replace("model.", "model.language_model.") + key = key.replace("visual.", "model.visual.") + return key + text_state_dict = {transform_key(k): v for k, v in text_state_dict.items()} + + text_encoder.load_state_dict(text_state_dict) + text_encoder = text_encoder.eval().to(device=offload_device, dtype=weight_dtype) + + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun", "Wan-AI", "Qwen"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + try: + tokenizer_path = search_sub_dir_in_possible_folders(possible_folders, sub_dir_name="qwen2_tokenizer") + except: + try: + tokenizer_path = os.path.join(search_sub_dir_in_possible_folders(possible_folders, sub_dir_name="Qwen-Image"), "tokenizer") + except: + tokenizer_path = search_sub_dir_in_possible_folders(possible_folders, sub_dir_name="Qwen2.5-VL-7B-Instruct") + + tokenizer = Qwen2Tokenizer.from_pretrained(tokenizer_path) + return (text_encoder, tokenizer) + + +class LoadQwenImageProcessor: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + }, + } + + RETURN_TYPES = ("Processor",) + RETURN_NAMES = ("processor",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, ): + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun", "Wan-AI", "Qwen"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + try: + processor_path = search_sub_dir_in_possible_folders(possible_folders, sub_dir_name="qwen2_processor") + except: + try: + processor_path = os.path.join(search_sub_dir_in_possible_folders(possible_folders, sub_dir_name="Qwen-Image-Edit"), "processor") + except: + processor_path = search_sub_dir_in_possible_folders(possible_folders, sub_dir_name="Qwen2.5-VL-7B-Instruct") + + # Get processor + processor = Qwen2VLProcessor.from_pretrained( + processor_path, + ) + return (processor, ) + + +class CombineQwenImagePipeline: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "transformer": ("TransformerModel",), + "vae": ("VAEModel",), + "text_encoder": ("TextEncoderModel",), + "tokenizer": ("Tokenizer",), + "model_name": ("STRING",), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + }, + "optional":{ + "processor": ("Processor",), + }, + } + + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, GPU_memory_mode, transformer, vae, text_encoder, tokenizer, processor=None, transformer_2=None): + # Get pipeline + weight_dtype = transformer.dtype if transformer.dtype not in [torch.float32, torch.float8_e4m3fn, torch.float8_e5m2] else get_autocast_dtype() + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + # Get pipeline + model_type = "Inpaint" + if model_type == "Inpaint": + if processor is not None: + pipeline = QwenImageEditPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=None, + processor=processor, + ) + else: + pipeline = QwenImagePipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=None, + ) + else: + raise ValueError("Not supported now.") + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + pipeline.to(device=offload_device) + transformer = transformer.to(weight_dtype) + + if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device=device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [] + } + return (funmodels,) + +class LoadQwenImageModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ( + [ + 'Qwen-Image', + 'Qwen-Image-Edit', + ], + { + "default": 'Qwen-Image', + } + ), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + "precision": ( + ['fp16', 'bf16'], + { + "default": 'fp16' + } + ), + }, + } + + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, GPU_memory_mode, model, precision): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + + # Init processbar + pbar = ProgressBar(5) + + # Detect model is existing or not + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun", "Wan-AI"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + # Initialize model_name as None + model_name = search_model_in_possible_folders(possible_folders, model) + + # Get Vae + vae = AutoencoderKLQwenImage.from_pretrained( + model_name, + subfolder="vae" + ).to(weight_dtype) + # Update pbar + pbar.update(1) + + # Load Sampler + print("Load Sampler.") + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + model_name, + subfolder="scheduler" + ) + # Update pbar + pbar.update(1) + + # Get Transformer + transformer = QwenImageTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + # Update pbar + pbar.update(1) + + # Get tokenizer and text_encoder + tokenizer = Qwen2Tokenizer.from_pretrained( + model_name, subfolder="tokenizer" + ) + pbar.update(1) + + text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype + ) + pbar.update(1) + + if os.path.exists(os.path.join(model_name, "processor")): + need_processor = True + # Get processor + processor = Qwen2VLProcessor.from_pretrained( + model_name, + subfolder="processor" + ) + else: + need_processor = False + + model_type = "Inpaint" + if model_type == "Inpaint": + if need_processor: + pipeline = QwenImageEditPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=None, + processor=processor, + ) + else: + pipeline = QwenImagePipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=None, + ) + else: + raise ValueError("Not supported now.") + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + + if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device=device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [] + } + return (funmodels,) + +class LoadQwenImageLora: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ("FunModels",), + "lora_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "lora_high_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), + "lora_cache":([False, True], {"default": False,}), + } + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "load_lora" + CATEGORY = "CogVideoXFUNWrapper" + + def load_lora(self, funmodels, lora_name, lora_high_name, strength_model, lora_cache): + new_funmodels = dict(funmodels) + if lora_name is not None: + loras = list(new_funmodels.get("loras", [])) + [folder_paths.get_full_path("loras", lora_name)] + loras_high = list(new_funmodels.get("loras_high", [])) + [folder_paths.get_full_path("loras", lora_high_name)] + strength_models = list(new_funmodels.get("strength_model", [])) + [strength_model] + new_funmodels['loras'] = loras + new_funmodels['loras_high'] = loras_high + new_funmodels['strength_model'] = strength_models + new_funmodels['lora_cache'] = lora_cache + return (new_funmodels,) + +class QwenImageT2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "width": ( + "INT", {"default": 1344, "min": 64, "max": 2048, "step": 16} + ), + "height": ( + "INT", {"default": 768, "min": 64, "max": 2048, "step": 16} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 4.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 1, "min": 1, "max": 100, "step": 1} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.250, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + } + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, width, height, seed, steps, cfg, scheduler, shift, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio): + global transformer_cpu_cache + global transformer_high_cpu_cache + global lora_path_before + global lora_high_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + + # Load Sampler + pipeline.scheduler = get_qwen_scheduler(scheduler, shift) + + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + else: + pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + else: + print('Merge Lora') + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + sample = pipeline( + prompt, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + true_cfg_scale = cfg, + num_inference_steps = steps, + comfyui_progressbar = True, + ).images + image = torch.Tensor(np.array(sample[0])).unsqueeze(0) / 255 + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + return (image,) + +class QwenImageEditSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "width": ( + "INT", {"default": 1344, "min": 64, "max": 2048, "step": 16} + ), + "height": ( + "INT", {"default": 768, "min": 64, "max": 2048, "step": 16} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 4.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 1, "min": 1, "max": 100, "step": 1} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional":{ + "image": ("IMAGE",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, width, height, seed, steps, cfg, scheduler, shift, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, image=None): + global transformer_cpu_cache + global transformer_high_cpu_cache + global lora_path_before + global lora_high_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + + # Load Sampler + pipeline.scheduler = get_qwen_scheduler(scheduler, shift) + + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + else: + pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + else: + print('Merge Lora') + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + image = [to_pil(image) for image in image] + image = get_image(image[0]) if image is not None else image + + sample = pipeline( + image = image, + prompt = prompt, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + true_cfg_scale = cfg, + num_inference_steps = steps, + comfyui_progressbar = True, + ).images + image = torch.Tensor(np.array(sample[0])).unsqueeze(0) / 255 + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + return (image,) diff --git a/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_chunked_loading_workflow_edit.json b/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_chunked_loading_workflow_edit.json new file mode 100644 index 0000000000000000000000000000000000000000..1243fee5120e9c5b78c64f0ab5e06b275f4e5baf --- /dev/null +++ b/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_chunked_loading_workflow_edit.json @@ -0,0 +1,596 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 98, + "last_link_id": 79, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 80, + "type": "Note", + "pos": [ + -92, + -294 + ], + "size": [ + 351.1499938964844, + 130.12660217285156 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 20B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用20B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 93, + "type": "LoadQwenImageVAEModel", + "pos": [ + 775.0554809570312, + -470.7688293457031 + ], + "size": [ + 377.8583984375, + 84.69844055175781 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 68 + ] + } + ], + "properties": { + "Node name for S&R": "LoadQwenImageVAEModel" + }, + "widgets_values": [ + "Qwen-Image-vae_bf16.safetensors", + "bf16" + ] + }, + { + "id": 91, + "type": "LoadQwenImageTextEncoderModel", + "pos": [ + 283.53765869140625, + -280.6837463378906 + ], + "size": [ + 407.4130859375, + 102 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 70 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 73 + ] + } + ], + "properties": { + "Node name for S&R": "LoadQwenImageTextEncoderModel" + }, + "widgets_values": [ + "Qwen-Image-text_encoder_bf16.safetensors", + "bf16" + ] + }, + { + "id": 96, + "type": "LoadQwenImageProcessor", + "pos": [ + 706.2975463867188, + -275.1779479980469 + ], + "size": [ + 217.32675170898438, + 26 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "processor", + "type": "Processor", + "links": [ + 74 + ] + } + ], + "properties": { + "Node name for S&R": "LoadQwenImageProcessor" + }, + "widgets_values": [] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 98, + "type": "LoadImage", + "pos": [ + 312.6856384277344, + 418.9110107421875 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 79 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "ref_1.png", + "image" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1070.207763671875, + -73.63389587402344 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 75 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 92, + "type": "LoadQwenImageTransformerModel", + "pos": [ + 275.9798278808594, + -465.2391052246094 + ], + "size": [ + 416.3677673339844, + 106.13789367675781 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 69 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 71 + ] + } + ], + "properties": { + "Node name for S&R": "LoadQwenImageTransformerModel" + }, + "widgets_values": [ + "Qwen-Image-Edit_bf16.safetensors", + "bf16" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "把相机转变成西瓜" + ] + }, + { + "id": 97, + "type": "QwenImageEditSampler", + "pos": [ + 747.4175415039062, + -76.30315399169922 + ], + "size": [ + 280.724609375, + 406 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 78 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "image", + "shape": 7, + "type": "IMAGE", + "link": 79 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "QwenImageEditSampler" + }, + "widgets_values": [ + 1344, + 768, + 373336117071181, + "randomize", + 40, + 4, + "Flow", + 1, + 0.25, + true, + 5, + true, + 0 + ] + }, + { + "id": 94, + "type": "CombineQwenImagePipeline", + "pos": [ + 945.2576293945312, + -330.913330078125 + ], + "size": [ + 321.2720642089844, + 162 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 69 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 68 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 70 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 73 + }, + { + "name": "processor", + "shape": 7, + "type": "Processor", + "link": 74 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 71 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "CombineQwenImagePipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8" + ] + } + ], + "links": [ + [ + 68, + 93, + 0, + 94, + 1, + "VAEModel" + ], + [ + 69, + 92, + 0, + 94, + 0, + "TransformerModel" + ], + [ + 70, + 91, + 0, + 94, + 2, + "TextEncoderModel" + ], + [ + 71, + 92, + 1, + 94, + 5, + "STRING" + ], + [ + 73, + 91, + 1, + 94, + 3, + "Tokenizer" + ], + [ + 74, + 96, + 0, + 94, + 4, + "Processor" + ], + [ + 75, + 97, + 0, + 88, + 0, + "IMAGE" + ], + [ + 76, + 75, + 0, + 97, + 1, + "STRING_PROMPT" + ], + [ + 77, + 73, + 0, + 97, + 2, + "STRING_PROMPT" + ], + [ + 78, + 94, + 0, + 97, + 0, + "FunModels" + ], + [ + 79, + 98, + 0, + 97, + 3, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 227.96267700195312, + -546.4359741210938, + 1053.5875244140625, + 397.3387756347656 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6905497838871149, + "offset": [ + 351.20689397709714, + 562.4271762478439 + ] + }, + "frontendVersion": "1.25.11", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "a97dd425909c3c3719fbbcb99e78061e2f0a237c", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_chunked_loading_workflow_t2v.json b/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_chunked_loading_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..14be765590d6fa513f849f7504a844cf670a4c3f --- /dev/null +++ b/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_chunked_loading_workflow_t2v.json @@ -0,0 +1,508 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 96, + "last_link_id": 83, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 75 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -92, + -294 + ], + "size": [ + 351.1499938964844, + 130.12660217285156 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 20B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用20B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 92, + "type": "LoadQwenImageTransformerModel", + "pos": [ + 275.9798278808594, + -465.2391052246094 + ], + "size": [ + 416.3677673339844, + 106.13789367675781 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 78 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "LoadQwenImageTransformerModel" + }, + "widgets_values": [ + "Qwen-Image_bf16.safetensors", + "bf16" + ] + }, + { + "id": 93, + "type": "LoadQwenImageVAEModel", + "pos": [ + 775.0554809570312, + -470.7688293457031 + ], + "size": [ + 377.8583984375, + 84.69844055175781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadQwenImageVAEModel" + }, + "widgets_values": [ + "Qwen-Image-vae_bf16.safetensors", + "bf16" + ] + }, + { + "id": 91, + "type": "LoadQwenImageTextEncoderModel", + "pos": [ + 283.53765869140625, + -280.6837463378906 + ], + "size": [ + 407.4130859375, + 102 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 80 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadQwenImageTextEncoderModel" + }, + "widgets_values": [ + "Qwen-Image-text_encoder_bf16.safetensors", + "bf16" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1070.207763671875, + -73.63389587402344 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 77 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 95, + "type": "QwenImageT2VSampler", + "pos": [ + 719.3201904296875, + -72.24609375 + ], + "size": [ + 280.724609375, + 386 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 83 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 75 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 76 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "QwenImageT2VSampler" + }, + "widgets_values": [ + 1344, + 768, + 867934328802019, + "randomize", + 40, + 4, + "Flow", + 1, + 0.25, + true, + 5, + true, + 0 + ] + }, + { + "id": 96, + "type": "CombineQwenImagePipeline", + "pos": [ + 790.3572998046875, + -328.7134094238281 + ], + "size": [ + 342.5804748535156, + 162 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 78 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 79 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 80 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 81 + }, + { + "name": "processor", + "shape": 7, + "type": "Processor", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 82 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 83 + ] + } + ], + "properties": { + "Node name for S&R": "CombineQwenImagePipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8" + ] + } + ], + "links": [ + [ + 75, + 75, + 0, + 95, + 1, + "STRING_PROMPT" + ], + [ + 76, + 73, + 0, + 95, + 2, + "STRING_PROMPT" + ], + [ + 77, + 95, + 0, + 88, + 0, + "IMAGE" + ], + [ + 78, + 92, + 0, + 96, + 0, + "TransformerModel" + ], + [ + 79, + 93, + 0, + 96, + 1, + "VAEModel" + ], + [ + 80, + 91, + 0, + 96, + 2, + "TextEncoderModel" + ], + [ + 81, + 91, + 1, + 96, + 3, + "Tokenizer" + ], + [ + 82, + 92, + 1, + 96, + 5, + "STRING" + ], + [ + 83, + 96, + 0, + 95, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 227.96267700195312, + -546.4359741210938, + 985.5581665039062, + 393.7902526855469 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015777, + "offset": [ + 338.4908515028006, + 640.3416144465855 + ] + }, + "frontendVersion": "1.25.11", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "a97dd425909c3c3719fbbcb99e78061e2f0a237c", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_workflow_edit.json b/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_workflow_edit.json new file mode 100644 index 0000000000000000000000000000000000000000..34b151bc564db619e4fc9cbfe3348b1ad7337c26 --- /dev/null +++ b/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_workflow_edit.json @@ -0,0 +1,375 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 99, + "last_link_id": 80, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 80, + "type": "Note", + "pos": [ + -92, + -294 + ], + "size": [ + 351.1499938964844, + 130.12660217285156 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 20B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用20B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 98, + "type": "LoadImage", + "pos": [ + 312.6856384277344, + 418.9110107421875 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 79 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "ref_1.png", + "image" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1070.207763671875, + -73.63389587402344 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 75 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 97, + "type": "QwenImageEditSampler", + "pos": [ + 747.4175415039062, + -76.30315399169922 + ], + "size": [ + 280.724609375, + 406 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 80 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "image", + "shape": 7, + "type": "IMAGE", + "link": 79 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "QwenImageEditSampler" + }, + "widgets_values": [ + 1344, + 768, + 686934831068040, + "randomize", + 40, + 4, + "Flow", + 1, + 0.25, + true, + 5, + true, + 0 + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "把相机转变成西瓜" + ] + }, + { + "id": 99, + "type": "LoadQwenImageModel", + "pos": [ + 294.5816650390625, + -309.4810485839844 + ], + "size": [ + 318.1009826660156, + 106 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "LoadQwenImageModel" + }, + "widgets_values": [ + "Qwen-Image-Edit", + "model_cpu_offload_and_qfloat8", + "bf16" + ] + } + ], + "links": [ + [ + 75, + 97, + 0, + 88, + 0, + "IMAGE" + ], + [ + 76, + 75, + 0, + 97, + 1, + "STRING_PROMPT" + ], + [ + 77, + 73, + 0, + 97, + 2, + "STRING_PROMPT" + ], + [ + 79, + 98, + 0, + 97, + 3, + "IMAGE" + ], + [ + 80, + 99, + 0, + 97, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 226.02244567871094, + -405.3177185058594, + 440.6474914550781, + 238.3169403076172 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6905497838871149, + "offset": [ + 542.3419639311205, + 433.4877940663054 + ] + }, + "frontendVersion": "1.25.11", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "36287cdcab8d5b6972bb6a2d208539c6e4bd81e2", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_workflow_t2v.json b/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..75cc7cef49aadb42017f8ca9642c457d49092da1 --- /dev/null +++ b/VideoX-Fun/comfyui/qwenimage/v1/qwenimage_workflow_t2v.json @@ -0,0 +1,324 @@ +{ + "id": "dcf2fcac-6293-4a86-b30b-f63e420177f2", + "revision": 0, + "last_node_id": 91, + "last_link_id": 70, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 68 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 69 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -92, + -294 + ], + "size": [ + 351.1499938964844, + 130.12660217285156 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 20B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用20B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 86, + "type": "LoadQwenImageModel", + "pos": [ + 314.6495666503906, + -281.51666259765625 + ], + "size": [ + 276.705078125, + 106 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "LoadQwenImageModel" + }, + "widgets_values": [ + "Qwen-Image", + "model_cpu_offload_and_qfloat8", + "bf16" + ] + }, + { + "id": 88, + "type": "PreviewImage", + "pos": [ + 1070.207763671875, + -73.63389587402344 + ], + "size": [ + 366.56134033203125, + 415.4429626464844 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 70 + } + ], + "outputs": [], + "properties": { + "Node name for S&R": "PreviewImage" + }, + "widgets_values": [] + }, + { + "id": 91, + "type": "QwenImageT2VSampler", + "pos": [ + 719.3201904296875, + -72.24609375 + ], + "size": [ + 280.724609375, + 386 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 67 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 68 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 69 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 70 + ] + } + ], + "properties": { + "Node name for S&R": "QwenImageT2VSampler" + }, + "widgets_values": [ + 1344, + 768, + 43, + "randomize", + 40, + 4, + "Flow", + 1, + 0.25, + true, + 5, + true, + 0 + ] + } + ], + "links": [ + [ + 67, + 86, + 0, + 91, + 0, + "FunModels" + ], + [ + 68, + 75, + 0, + 91, + 1, + "STRING_PROMPT" + ], + [ + 69, + 73, + 0, + 91, + 2, + "STRING_PROMPT" + ], + [ + 70, + 91, + 0, + 88, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7828871306993743, + "offset": [ + 512.3269100390406, + 507.26179044474577 + ] + }, + "frontendVersion": "1.25.11", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "a97dd425909c3c3719fbbcb99e78061e2f0a237c", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1/nodes.py b/VideoX-Fun/comfyui/wan2_1/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..a01d85045b919770de7008bc37e66969dcaf650c --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1/nodes.py @@ -0,0 +1,983 @@ +"""Modified from https://github.com/kijai/ComfyUI-EasyAnimateWrapper/blob/main/nodes.py +""" +import copy +import gc +import inspect +import json +import os + +import comfy.model_management as mm +import cv2 +import folder_paths +import numpy as np +import torch +from comfy.utils import ProgressBar, load_torch_file +from diffusers import FlowMatchEulerDiscreteScheduler +from einops import rearrange +from omegaconf import OmegaConf +from PIL import Image + +from ...videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + get_closest_ratio) +from ...videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from ...videox_fun.models.cache_utils import get_teacache_coefficients +from ...videox_fun.pipeline import (WanFunControlPipeline, WanI2VPipeline, + WanPipeline) +from ...videox_fun.ui.controller import all_cheduler_dict +from ...videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from ...videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from ...videox_fun.utils.fp8_optimization import ( + convert_model_weight_to_float8, convert_weight_dtype_wrapper, undo_convert_weight_dtype_wrapper, + replace_parameters_by_name) +from ...videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from ...videox_fun.utils.utils import (filter_kwargs, + get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid, get_autocast_dtype) +from ..comfyui_utils import (eas_cache_dir, script_directory, search_sub_dir_in_possible_folders, + search_model_in_possible_folders, to_pil) + +# Used in lora cache +transformer_cpu_cache = {} +# lora path before +lora_path_before = "" + + +def get_wan_scheduler(sampler_name, shift): + Chosen_Scheduler = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, + }[sampler_name] + scheduler_kwargs = { + "num_train_timesteps": 1000, + "shift": 5.0, + "use_dynamic_shifting": False, + "base_shift": 0.5, + "max_shift": 1.15, + "base_image_seq_len": 256, + "max_image_seq_len": 4096, + } + scheduler_kwargs['shift'] = shift + scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, scheduler_kwargs) + ) + return scheduler + +class LoadWanTransformerModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ( + folder_paths.get_filename_list("diffusion_models"), + {"default": "Wan2_1-T2V-1_3B_bf16.safetensors,"}, + ), + "precision": (["fp16", "bf16"], + {"default": "bf16"} + ), + }, + } + RETURN_TYPES = ("TransformerModel", "STRING") + RETURN_NAMES = ("transformer", "model_name") + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, precision): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + transformer = None + + model_path = folder_paths.get_full_path("diffusion_models", model_name) + transformer_state_dict = load_torch_file(model_path, safe_load=True) + + eps = 1e-6 + text_len = 512 + freq_dim = 256 + dim = transformer_state_dict["patch_embedding.weight"].shape[0] + hidden_size = dim + in_dim = transformer_state_dict["patch_embedding.weight"].shape[1] + in_channels = in_dim + ffn_dim = transformer_state_dict["blocks.0.ffn.0.bias"].shape[0] + + add_ref_conv = True if "ref_conv.weight" in transformer_state_dict else False + in_dim_ref_conv = transformer_state_dict["ref_conv.weight"].shape[1] if "ref_conv.weight" in transformer_state_dict else None + add_control_adapter = True if "control_adapter.conv.weight" in transformer_state_dict else False + in_dim_control_adapter = transformer_state_dict["control_adapter.conv.weight"].shape[1] if "control_adapter.conv.weight" in transformer_state_dict else None + + if dim == 5120: + num_heads = 40 + num_layers = 40 + out_dim = 16 + downscale_factor_control_adapter = 8 + if in_dim == out_dim * 2 + 4: + if "480" in model_name or "fun" in model_name.lower() \ + or "540" in model_name: + model_name_in_pipeline = "wan2.1-i2v-14b-480p" + else: + model_name_in_pipeline = "wan2.1-i2v-14b-720p" + elif in_dim == out_dim: + model_name_in_pipeline = "wan2.1-t2v-14b" + else: + model_name_in_pipeline = "wan2.1-fun-14b" + + elif dim == 3072: + num_heads = 24 + num_layers = 30 + out_dim = 48 + downscale_factor_control_adapter = 16 + model_name_in_pipeline = "wan2.1-t2v-1.3b" + else: + num_heads = 12 + num_layers = 30 + out_dim = 16 + downscale_factor_control_adapter = 8 + model_name_in_pipeline = "wan2.2-ti2v-5b" + + if in_dim != out_dim: + model_type = "i2v" + else: + model_type = "t2v" + + kwargs = dict( + dim = dim, + in_dim = in_dim, + eps = eps, + ffn_dim = ffn_dim, + freq_dim = freq_dim, + model_type = model_type, + num_heads = num_heads, + num_layers = num_layers, + out_dim = out_dim, + text_len = text_len, + in_channels = in_channels, + hidden_size = hidden_size, + add_control_adapter = add_control_adapter, + add_ref_conv = add_ref_conv, + in_dim_control_adapter = in_dim_control_adapter // downscale_factor_control_adapter // downscale_factor_control_adapter if in_dim_control_adapter is not None else in_dim_control_adapter, + in_dim_ref_conv = in_dim_ref_conv, + downscale_factor_control_adapter = downscale_factor_control_adapter, + ) + + sig = inspect.signature(WanTransformer3DModel) + accepted = {k: v for k, v in kwargs.items() if k in sig.parameters} + transformer = WanTransformer3DModel(**accepted) + transformer.load_state_dict(transformer_state_dict) + transformer = transformer.eval().to(device=offload_device, dtype=weight_dtype) + return (transformer, model_name_in_pipeline) + +class LoadWanVAEModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ( + folder_paths.get_filename_list("vae"), + {"default": "Wan2.1_VAE.pth"} + ), + "precision": (["fp16", "bf16"], + {"default": "bf16"} + ), + }, + } + + RETURN_TYPES = ("VAEModel",) + RETURN_NAMES = ("vae", ) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, precision,): + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[precision] + model_path = folder_paths.get_full_path("vae", model_name) + vae_state_dict = load_torch_file(model_path, safe_load=True) + + if not any(k.startswith("model.") for k in vae_state_dict.keys()): + vae_state_dict = {f"model.{k}": v for k, v in vae_state_dict.items()} + + Chosen_AutoencoderKL = { + 16: AutoencoderKLWan, + 48: AutoencoderKLWan3_8 + }[vae_state_dict["model.conv2.weight"].shape[0]] + + vae = Chosen_AutoencoderKL(latent_channels=vae_state_dict["model.conv2.weight"].shape[0]) + vae.load_state_dict(vae_state_dict) + vae = vae.eval().to(device=offload_device, dtype=weight_dtype) + return (vae,) + +class LoadWanTextEncoderModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ( + folder_paths.get_filename_list("text_encoders"), + {"default": "models_t5_umt5-xxl-enc-bf16.pth"} + ), + "precision": (["fp16", "bf16"], + {"default": "bf16"} + ), + }, + } + + RETURN_TYPES = ("TextEncoderModel", "Tokenizer") + RETURN_NAMES = ("text_encoder", "tokenizer") + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, precision,): + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[precision] + model_path = folder_paths.get_full_path("text_encoders", model_name) + text_state_dict = load_torch_file(model_path, safe_load=True) + + kwargs = { + "text_length": 512, + "vocab": 256384, + "dim": 4096, + "dim_attn": 4096, + "dim_ffn": 10240, + "num_heads": 64, + "num_layers": 24, + "num_buckets": 32, + "shared_pos": False, + "dropout": 0.0, + } + + sig = inspect.signature(WanT5EncoderModel) + accepted = {k: v for k, v in kwargs.items() if k in sig.parameters} + text_encoder = WanT5EncoderModel(**accepted) + text_encoder.load_state_dict(text_state_dict) + text_encoder = text_encoder.eval().to(device=offload_device, dtype=weight_dtype) + + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun", "Wan-AI"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + tokenizer = AutoTokenizer.from_pretrained(search_sub_dir_in_possible_folders(possible_folders, sub_dir_name="google/umt5-xxl")) + return (text_encoder, tokenizer) + +class LoadWanClipEncoderModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ( + folder_paths.get_filename_list("clip_vision") + folder_paths.get_filename_list("text_encoders"), + {"default": "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth"} + ), + "precision": (["fp16", "bf16"], + {"default": "bf16"} + ), + } + } + + RETURN_TYPES = ("ClipEncoderModel",) + RETURN_NAMES = ("clip_encoder", ) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, precision): + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[precision] + + model_path = folder_paths.get_full_path("clip_vision", model_name) + if model_path is None: + model_path = folder_paths.get_full_path("text_encoders", model_name) + clip_state_dict = load_torch_file(model_path, safe_load=True) + if not any(k.startswith("model.") for k in clip_state_dict.keys()): + clip_state_dict = {f"model.{k}": v for k, v in clip_state_dict.items()} + + clip_model = CLIPModel() + clip_model.load_state_dict(clip_state_dict) + clip_model = clip_model.eval().to(device=offload_device, dtype=weight_dtype) + + return (clip_model,) + +class LoadFlowScheduler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "sampler_name": (["Flow", "Flow_Unipc", "Flow_DPM++"], + {"default": "Flow"} + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + } + } + + RETURN_TYPES = ("FlowScheduler",) + RETURN_NAMES = ("scheduler", ) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, sampler_name, shift): + Chosen_Scheduler = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, + }[sampler_name] + scheduler_kwargs = { + "num_train_timesteps": 1000, + "shift": 5.0, + "use_dynamic_shifting": False, + "base_shift": 0.5, + "max_shift": 1.15, + "base_image_seq_len": 256, + "max_image_seq_len": 4096, + } + scheduler_kwargs['shift'] = shift + scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, scheduler_kwargs) + ) + return (scheduler,) + +class CombineWanPipeline: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "transformer": ("TransformerModel",), + "vae": ("VAEModel",), + "text_encoder": ("TextEncoderModel",), + "tokenizer": ("Tokenizer",), + "model_name": ("STRING",), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + "model_type": ( + ["Inpaint", "Control"], + { + "default": "Inpaint", + } + ), + }, + "optional":{ + "clip_encoder": ("ClipEncoderModel",), + }, + } + + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, GPU_memory_mode, model_type, transformer, vae, text_encoder, tokenizer, clip_encoder=None): + # Get pipeline + weight_dtype = transformer.dtype if transformer.dtype not in [torch.float32, torch.float8_e4m3fn, torch.float8_e5m2] else get_autocast_dtype() + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + if model_type == "Inpaint": + if transformer.config.in_channels != vae.config.latent_channels: + pipeline = WanI2VPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + clip_image_encoder=clip_encoder, + scheduler=None, + ) + else: + pipeline = WanPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=None, + ) + else: + pipeline = WanFunControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=None, + clip_image_encoder=clip_encoder + ) + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + pipeline.to(device=offload_device) + transformer = transformer.to(weight_dtype) + + if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",]) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",]) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [] + } + return (funmodels,) + +class LoadWanModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ( + [ + 'Wan2.1-T2V-1.3B', + 'Wan2.1-T2V-14B', + 'Wan2.1-I2V-14B-480P', + 'Wan2.1-I2V-14B-720P', + ], + { + "default": 'Wan2.1-T2V-1.3B', + } + ), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + "config": ( + [ + "wan2.1/wan_civitai.yaml", + ], + { + "default": "wan2.1/wan_civitai.yaml", + } + ), + "precision": ( + ['fp16', 'bf16'], + { + "default": 'bf16' + } + ), + }, + } + + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, GPU_memory_mode, model, precision, config): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + + # Init processbar + pbar = ProgressBar(5) + + # Load config + config_path = f"{script_directory}/config/{config}" + config = OmegaConf.load(config_path) + + # Detect model is existing or not + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun", "Wan-AI"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + # Initialize model_name as None + model_name = search_model_in_possible_folders(possible_folders, model) + + vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ).to(weight_dtype) + # Update pbar + pbar.update(1) + + # Load Sampler + print("Load Sampler.") + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + # Update pbar + pbar.update(1) + + # Get Transformer + transformer = WanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + # Update pbar + pbar.update(1) + + # Get tokenizer and text_encoder + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + pbar.update(1) + + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + pbar.update(1) + + if transformer.config.in_channels != vae.config.latent_channels: + # Get Clip Image Encoder + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(model_name, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ).to(weight_dtype) + clip_image_encoder = clip_image_encoder.eval() + + # Get pipeline + model_type = "Inpaint" + if model_type == "Inpaint": + if transformer.config.in_channels != vae.config.latent_channels: + pipeline = WanI2VPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder + ) + else: + pipeline = WanPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + else: + raise ValueError(f"Model type {model_type} not supported") + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + + if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",]) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",]) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [], + 'config': config, + } + return (funmodels,) + +class LoadWanLora: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ("FunModels",), + "lora_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), + "lora_cache":([False, True], {"default": False,}), + } + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "load_lora" + CATEGORY = "CogVideoXFUNWrapper" + + def load_lora(self, funmodels, lora_name, strength_model, lora_cache): + new_funmodels = dict(funmodels) + + if lora_name is not None: + lora_path = folder_paths.get_full_path("loras", lora_name) + + new_funmodels['lora_cache'] = lora_cache + new_funmodels['loras'] = funmodels.get("loras", []) + [lora_path] + new_funmodels['strength_model'] = funmodels.get("strength_model", []) + [strength_model] + + return (new_funmodels,) + + +class WanT2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 5, "max": 161, "step": 4} + ), + "width": ( + "INT", {"default": 832, "min": 64, "max": 2048, "step": 16} + ), + "height": ( + "INT", {"default": 480, "min": 64, "max": 2048, "step": 16} + ), + "is_image":( + [ + False, + True + ], + { + "default": False, + } + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional":{ + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, width, height, is_image, seed, steps, cfg, scheduler, shift, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, riflex_k=0): + global transformer_cpu_cache + global lora_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + else: + pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + + generator= torch.Generator(device).manual_seed(seed) + + video_length = 1 if is_image else video_length + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + else: + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + print('Merge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + shift = shift, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + return (videos,) + + +class WanI2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 5, "max": 161, "step": 4} + ), + "base_resolution": ( + [ + 512, + 640, + 768, + 896, + 960, + 1024, + ], {"default": 640} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional":{ + "start_img": ("IMAGE",), + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, base_resolution, seed, steps, cfg, scheduler, shift, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, start_img=None, end_img=None, riflex_k=0): + global transformer_cpu_cache + global lora_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + start_img = [to_pil(_start_img) for _start_img in start_img] if start_img is not None else None + end_img = [to_pil(_end_img) for _end_img in end_img] if end_img is not None else None + # Count most suitable height and width + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + original_width, original_height = start_img[0].size if type(start_img) is list else Image.open(start_img).size + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height, width = [int(x / 16) * 16 for x in closest_size] + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + else: + pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_img, end_img, video_length=video_length, sample_size=(height, width)) + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + else: + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + print('Delete cpu state_dict') + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + print('Merge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + comfyui_progressbar = True, + shift = shift, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + return (videos,) + diff --git a/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_chunked_loading_workflow_i2v.json b/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_chunked_loading_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..4bfa7c7e1709169af6e6a6e979f9907cb53754ee --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_chunked_loading_workflow_i2v.json @@ -0,0 +1,738 @@ +{ + "id": "7771a2ee-e50b-4606-8122-63139795c002", + "revision": 0, + "last_node_id": 119, + "last_link_id": 81, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 71 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 95, + "type": "Note", + "pos": [ + 35.71394348144531, + 536.9210205078125 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 72 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 87, + "type": "LoadImage", + "pos": [ + 280.5103759765625, + 477.9588623046875 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 74 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "2.png", + "image" + ] + }, + { + "id": 103, + "type": "LoadImage", + "pos": [ + 689.5659790039062, + 477.3172302246094 + ], + "size": [ + 318.56072998046875, + 326 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "End Image", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "070008-d3df70b6-1c49-43d5-a39b-ba357e48c17a-3149685904.png", + "image" + ] + }, + { + "id": 115, + "type": "LoadWanVAEModel", + "pos": [ + 250.1365203857422, + -450.6431884765625 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 118, + "type": "CombineWanPipeline", + "pos": [ + 714.62158203125, + -408.345703125 + ], + "size": [ + 362.322021484375, + 206 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 76 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 78 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 77 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 80 + }, + { + "name": "clip_encoder", + "type": "ClipEncoderModel", + "link": 79 + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 75 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWanPipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8", + "Inpaint" + ] + }, + { + "id": 114, + "type": "LoadWanClipEncoderModel", + "pos": [ + 708.2453002929688, + -615.9248046875 + ], + "size": [ + 365.69586181640625, + 92.28972625732422 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "clip_encoder", + "type": "ClipEncoderModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanClipEncoderModel" + }, + "widgets_values": [ + "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", + "bf16" + ] + }, + { + "id": 113, + "type": "LoadWanTransformerModel", + "pos": [ + 250.1365203857422, + -614.3566284179688 + ], + "size": [ + 401.8365478515625, + 102.11343383789062 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 76 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTransformerModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-InP_transformer.safetensors", + "bf16" + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 20.679954528808594, + -43.9061279296875 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 116, + "type": "WanI2VSampler", + "pos": [ + 807.667724609375, + -74.96307373046875 + ], + "size": [ + 302.2232360839844, + 426 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 81 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 71 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 72 + }, + { + "name": "start_img", + "shape": 7, + "type": "IMAGE", + "link": 74 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 73 + ] + } + ], + "properties": { + "Node name for S&R": "WanI2VSampler" + }, + "widgets_values": [ + 81, + 640, + 76970077943317, + "randomize", + 50, + 6, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 117, + "type": "LoadWanTextEncoderModel", + "pos": [ + 250.1365203857422, + -300.0733642578125 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 77 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "fp16" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1271, + -67 + ], + "size": [ + 390, + 546 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 73 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00121.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 119, + "type": "Note", + "pos": [ + 1109.983154296875, + -376.7947692871094 + ], + "size": [ + 257.3695983886719, + 117.2574234008789 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 71, + 75, + 0, + 116, + 1, + "STRING_PROMPT" + ], + [ + 72, + 73, + 0, + 116, + 2, + "STRING_PROMPT" + ], + [ + 73, + 116, + 0, + 17, + 0, + "IMAGE" + ], + [ + 74, + 87, + 0, + 116, + 3, + "IMAGE" + ], + [ + 75, + 113, + 1, + 118, + 6, + "STRING" + ], + [ + 76, + 113, + 0, + 118, + 0, + "TransformerModel" + ], + [ + 77, + 117, + 0, + 118, + 2, + "TextEncoderModel" + ], + [ + 78, + 115, + 0, + 118, + 1, + "VAEModel" + ], + [ + 79, + 114, + 0, + 118, + 4, + "ClipEncoderModel" + ], + [ + 80, + 117, + 1, + 118, + 3, + "Tokenizer" + ], + [ + 81, + 118, + 0, + 116, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Image", + "bounding": [ + 222, + 396, + 875.5350952148438, + 432.9040832519531 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model", + "bounding": [ + 212.720947265625, + -700.9775390625, + 924.4893798828125, + 529.3465576171875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.9090909090909092, + "offset": [ + 34.02887395686214, + 710.4654075123572 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "comfy-core": "0.3.44", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_chunked_loading_workflow_t2v.json b/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_chunked_loading_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..2bad143fdf6add885e8bf020b3204f94d457c8c3 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_chunked_loading_workflow_t2v.json @@ -0,0 +1,573 @@ +{ + "id": "7771a2ee-e50b-4606-8122-63139795c002", + "revision": 0, + "last_node_id": 121, + "last_link_id": 90, + "nodes": [ + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 88 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 115, + "type": "LoadWanVAEModel", + "pos": [ + 250.1365203857422, + -450.6431884765625 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 117, + "type": "LoadWanTextEncoderModel", + "pos": [ + 250.1365203857422, + -300.0733642578125 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 77 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 20.679954528808594, + -43.9061279296875 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 119, + "type": "Note", + "pos": [ + 1104.6458740234375, + -372.9275207519531 + ], + "size": [ + 257.3695983886719, + 117.2574234008789 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 113, + "type": "LoadWanTransformerModel", + "pos": [ + 250.1365203857422, + -614.3566284179688 + ], + "size": [ + 401.8365478515625, + 102.11343383789062 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 76 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTransformerModel" + }, + "widgets_values": [ + "Wan2.1-T2V-1.3B_transformer.safetensors", + "bf16" + ] + }, + { + "id": 118, + "type": "CombineWanPipeline", + "pos": [ + 714.62158203125, + -408.345703125 + ], + "size": [ + 362.322021484375, + 206 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 76 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 78 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 77 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 80 + }, + { + "name": "clip_encoder", + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 75 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 89 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWanPipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8", + "Inpaint" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 87 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1125.2930908203125, + -74.02973175048828 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 90 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00123.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 121, + "type": "WanT2VSampler", + "pos": [ + 762.89892578125, + -67.31344604492188 + ], + "size": [ + 288.9888610839844, + 454 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 89 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 87 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 88 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "WanT2VSampler" + }, + "widgets_values": [ + 81, + 832, + 480, + false, + 775022241121010, + "randomize", + 50, + 6, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 75, + 113, + 1, + 118, + 6, + "STRING" + ], + [ + 76, + 113, + 0, + 118, + 0, + "TransformerModel" + ], + [ + 77, + 117, + 0, + 118, + 2, + "TextEncoderModel" + ], + [ + 78, + 115, + 0, + 118, + 1, + "VAEModel" + ], + [ + 80, + 117, + 1, + 118, + 3, + "Tokenizer" + ], + [ + 87, + 75, + 0, + 121, + 1, + "STRING_PROMPT" + ], + [ + 88, + 73, + 0, + 121, + 2, + "STRING_PROMPT" + ], + [ + 89, + 118, + 0, + 121, + 0, + "FunModels" + ], + [ + 90, + 121, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model", + "bounding": [ + 212.720947265625, + -700.9775390625, + 924.4893798828125, + 529.3465576171875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917356, + "offset": [ + 67.75547551936205, + 679.2236887623583 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_workflow_i2v.json b/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..a361535268ae78e41648e6c07b85056ebe544bf9 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_workflow_i2v.json @@ -0,0 +1,491 @@ +{ + "last_node_id": 97, + "last_link_id": 63, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 60 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1277, + "1": -70 + }, + "size": [ + 390, + 310 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 63, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 87, + "type": "LoadImage", + "pos": { + "0": 306, + "1": 495 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 62 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "6.png", + "image" + ] + }, + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "Note", + "pos": { + "0": 17, + "1": -35 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": { + "0": 34, + "1": 550 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 97, + "type": "LoadWanModel", + "pos": { + "0": 304, + "1": -299 + }, + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 59 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanModel" + }, + "widgets_values": [ + "Wan2.1-I2V-14B-720P", + "sequential_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 96, + "type": "WanI2VSampler", + "pos": { + "0": 796, + "1": -70 + }, + "size": { + "0": 378, + "1": 358 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 59 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 60 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 61 + }, + { + "name": "start_img", + "type": "IMAGE", + "link": 62, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 63 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanI2VSampler" + }, + "widgets_values": [ + 81, + 640, + 507671483727948, + "randomize", + 50, + 6, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 80, + "type": "Note", + "pos": { + "0": -75, + "1": -297 + }, + "size": { + "0": 350.7127990722656, + "1": 125.54820251464844 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 61 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + } + ], + "links": [ + [ + 59, + 97, + 0, + 96, + 0, + "FunModels" + ], + [ + 60, + 75, + 0, + 96, + 1, + "STRING_PROMPT" + ], + [ + 61, + 73, + 0, + 96, + 2, + "STRING_PROMPT" + ], + [ + 62, + 87, + 0, + 96, + 3, + "IMAGE" + ], + [ + 63, + 96, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Load Model", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": "Group", + "bounding": [ + 220, + 409, + 458, + 436 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 356.367407042801, + 487.2362084107991 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "e054344e39c5030c23b0146ed4a1293bff2505ed", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "v0.2.7-3-g8afb97c" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_workflow_t2v.json b/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..5b515657fdbd37d693f5074b912eeaa509b58a78 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1/v1/wan2.1_workflow_t2v.json @@ -0,0 +1,379 @@ +{ + "last_node_id": 85, + "last_link_id": 53, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 49 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 85, + "type": "WanT2VSampler", + "pos": { + "0": 722, + "1": -73 + }, + "size": { + "0": 365.4000244140625, + "1": 386 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 48 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 49 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 52 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 53 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanT2VSampler" + }, + "widgets_values": [ + 81, + 832, + 480, + false, + 837135609139060, + "randomize", + 50, + 6, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1126, + "1": -73 + }, + "size": [ + 390.9534912109375, + 310 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 53, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 84, + "type": "LoadWanModel", + "pos": { + "0": 286, + "1": -297 + }, + "size": { + "0": 315, + "1": 130 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 48 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanModel" + }, + "widgets_values": [ + "Wan2.1-T2V-1.3B", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 80, + "type": "Note", + "pos": { + "0": -92, + "1": -294 + }, + "size": { + "0": 351.1499938964844, + "1": 130.12660217285156 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 52 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + } + ], + "links": [ + [ + 48, + 84, + 0, + 85, + 0, + "FunModels" + ], + [ + 49, + 75, + 0, + 85, + 1, + "STRING_PROMPT" + ], + [ + 52, + 73, + 0, + 85, + 2, + "STRING_PROMPT" + ], + [ + 53, + 85, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Load Model", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 356.367407042801, + 487.2362084107991 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "e054344e39c5030c23b0146ed4a1293bff2505ed", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/nodes.py b/VideoX-Fun/comfyui/wan2_1_fun/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..e5704d9cb1cb57802bbc8bb0f2bc9750a003ae04 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/nodes.py @@ -0,0 +1,858 @@ +"""Modified from https://github.com/kijai/ComfyUI-EasyAnimateWrapper/blob/main/nodes.py +""" +import copy +import gc +import json +import os + +import cv2 +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from einops import rearrange +from omegaconf import OmegaConf +from PIL import Image + +import comfy.model_management as mm +import folder_paths +from comfy.utils import ProgressBar, load_torch_file + +from ...videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + get_closest_ratio) +from ...videox_fun.data.dataset_image_video import process_pose_params +from ...videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, WanTransformer3DModel) +from ...videox_fun.models.cache_utils import get_teacache_coefficients +from ...videox_fun.pipeline import (WanFunControlPipeline, + WanFunInpaintPipeline, WanFunPipeline) +from ...videox_fun.ui.controller import all_cheduler_dict +from ...videox_fun.utils.fp8_optimization import ( + convert_model_weight_to_float8, convert_weight_dtype_wrapper, undo_convert_weight_dtype_wrapper, + replace_parameters_by_name) +from ...videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from ...videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) +from ..comfyui_utils import (eas_cache_dir, script_directory, + search_model_in_possible_folders, to_pil) +from ..wan2_1.nodes import get_wan_scheduler + +# Used in lora cache +transformer_cpu_cache = {} +# lora path before +lora_path_before = "" + +class LoadWanFunModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ( + [ + 'Wan2.1-Fun-1.3B-InP', + 'Wan2.1-Fun-14B-InP', + 'Wan2.1-Fun-1.3B-Control', + 'Wan2.1-Fun-14B-Control', + 'Wan2.1-Fun-V1.1-1.3B-InP', + 'Wan2.1-Fun-V1.1-14B-InP', + 'Wan2.1-Fun-V1.1-1.3B-Control', + 'Wan2.1-Fun-V1.1-14B-Control', + 'Wan2.1-Fun-V1.1-1.3B-Control-Camera', + 'Wan2.1-Fun-V1.1-14B-Control-Camera', + ], + { + "default": 'Wan2.1-Fun-1.3B-InP', + } + ), + "model_type": ( + ["Inpaint", "Control"], + { + "default": "Inpaint", + } + ), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + "config": ( + [ + "wan2.1/wan_civitai.yaml", + ], + { + "default": "wan2.1/wan_civitai.yaml", + } + ), + "precision": ( + ['fp16', 'bf16'], + { + "default": 'bf16' + } + ), + }, + } + + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, GPU_memory_mode, model_type, model, precision, config): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + + # Init processbar + pbar = ProgressBar(5) + + # Load config + config_path = f"{script_directory}/config/{config}" + config = OmegaConf.load(config_path) + + # Detect model is existing or not + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun", "Wan-AI"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + # Initialize model_name as None + model_name = search_model_in_possible_folders(possible_folders, model) + + vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ).to(weight_dtype) + # Update pbar + pbar.update(1) + + # Load Sampler + print("Load Sampler.") + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + # Update pbar + pbar.update(1) + + # Get Transformer + transformer = WanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + # Update pbar + pbar.update(1) + + # Get tokenizer and text_encoder + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + pbar.update(1) + + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + pbar.update(1) + + if transformer.config.in_channels != vae.config.latent_channels: + # Get Clip Image Encoder + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(model_name, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ).to(weight_dtype) + clip_image_encoder = clip_image_encoder.eval() + + # Get pipeline + if model_type == "Inpaint": + if transformer.config.in_channels != vae.config.latent_channels: + pipeline = WanFunInpaintPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder + ) + else: + pipeline = WanFunPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + else: + pipeline = WanFunControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder + ) + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + + if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",]) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",]) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [], + 'config': config, + } + return (funmodels,) + +class LoadWanFunLora: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ("FunModels",), + "lora_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), + "lora_cache":([False, True], {"default": False,}), + } + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "load_lora" + CATEGORY = "CogVideoXFUNWrapper" + + def load_lora(self, funmodels, lora_name, strength_model, lora_cache): + new_funmodels = dict(funmodels) + + if lora_name is not None: + lora_path = folder_paths.get_full_path("loras", lora_name) + + new_funmodels['lora_cache'] = lora_cache + new_funmodels['loras'] = funmodels.get("loras", []) + [lora_path] + new_funmodels['strength_model'] = funmodels.get("strength_model", []) + [strength_model] + + return (new_funmodels,) + +class WanFunT2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 5, "max": 161, "step": 4} + ), + "width": ( + "INT", {"default": 832, "min": 64, "max": 2048, "step": 16} + ), + "height": ( + "INT", {"default": 480, "min": 64, "max": 2048, "step": 16} + ), + "is_image":( + [ + False, + True + ], + { + "default": False, + } + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional": { + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, width, height, is_image, seed, steps, cfg, scheduler, shift, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, riflex_k=0): + global transformer_cpu_cache + global lora_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + else: + pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + + generator= torch.Generator(device).manual_seed(seed) + + video_length = 1 if is_image else video_length + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + else: + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + print('Merge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + if pipeline.transformer.config.in_channels != pipeline.vae.config.latent_channels: + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=(height, width)) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + comfyui_progressbar = True, + ).videos + else: + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + return (videos,) + + +class WanFunInpaintSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 5, "max": 161, "step": 4} + ), + "base_resolution": ( + [ + 512, + 640, + 768, + 896, + 960, + 1024, + ], {"default": 640} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional": { + "start_img": ("IMAGE",), + "end_img": ("IMAGE",), + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, base_resolution, seed, steps, cfg, scheduler, shift, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, start_img=None, end_img=None, riflex_k=0): + global transformer_cpu_cache + global lora_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + start_img = [to_pil(_start_img) for _start_img in start_img] if start_img is not None else None + end_img = [to_pil(_end_img) for _end_img in end_img] if end_img is not None else None + # Count most suitable height and width + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + original_width, original_height = start_img[0].size if type(start_img) is list else Image.open(start_img).size + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height, width = [int(x / 16) * 16 for x in closest_size] + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + else: + pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_img, end_img, video_length=video_length, sample_size=(height, width)) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + else: + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + print('Delete cpu state_dict') + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + print('Merge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + return (videos,) + + +class WanFunV2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 1, "max": 161, "step": 4} + ), + "base_resolution": ( + [ + 512, + 640, + 768, + 896, + 960, + 1024, + ], {"default": 640} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "denoise_strength": ( + "FLOAT", {"default": 1.00, "min": 0.05, "max": 1.00, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional": { + "validation_video": ("IMAGE",), + "control_video": ("IMAGE",), + "start_image": ("IMAGE",), + "ref_image": ("IMAGE",), + "camera_conditions": ("STRING", {"forceInput": True}), + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, base_resolution, seed, steps, cfg, denoise_strength, scheduler, shift, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, validation_video=None, control_video=None, start_image=None, ref_image=None, camera_conditions=None, riflex_k=0): + global transformer_cpu_cache + global lora_path_before + + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + model_type = funmodels['model_type'] + + # Count most suitable height and width + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if model_type == "Inpaint": + if type(validation_video) is str: + original_width, original_height = Image.fromarray(cv2.VideoCapture(validation_video).read()[1]).size + else: + validation_video = np.array(validation_video.cpu().numpy() * 255, np.uint8) + original_width, original_height = Image.fromarray(validation_video[0]).size + else: + if control_video is not None and type(control_video) is str: + original_width, original_height = Image.fromarray(cv2.VideoCapture(control_video).read()[1]).size + elif control_video is not None: + control_video = np.array(control_video.cpu().numpy() * 255, np.uint8) + original_width, original_height = Image.fromarray(control_video[0]).size + else: + original_width, original_height = 384 / 512 * base_resolution, 672 / 512 * base_resolution + + if ref_image is not None: + ref_image = [to_pil(_ref_image) for _ref_image in ref_image] + original_width, original_height = ref_image[0].size if type(ref_image) is list else Image.open(ref_image).size + + if start_image is not None: + start_image = [to_pil(_start_image) for _start_image in start_image] + original_width, original_height = start_image[0].size if type(start_image) is list else Image.open(start_image).size + + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height, width = [int(x / 16) * 16 for x in closest_size] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + else: + pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + if model_type == "Inpaint": + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(validation_video, video_length=video_length, sample_size=(height, width), fps=16, ref_image=ref_image[0] if ref_image is not None else ref_image) + else: + if ref_image is not None: + clip_image = ref_image[0].convert("RGB") + elif start_image is not None: + clip_image = start_image[0].convert("RGB") + else: + clip_image = None + + if ref_image is not None: + ref_image = get_image_latent(ref_image[0] if ref_image is not None else ref_image, sample_size=(height, width)) + + if start_image is not None: + start_image = get_image_latent(start_image[0] if start_image is not None else start_image, sample_size=(height, width)) + + if camera_conditions is not None and len(camera_conditions) > 0: + poses = json.loads(camera_conditions) + cam_params = np.array([[float(x) for x in pose] for pose in poses]) + cam_params = np.concatenate([np.zeros_like(cam_params[:, :1]), cam_params], 1) + control_camera_video = process_pose_params(cam_params, width=width, height=height) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + input_video, input_video_mask = None, None + else: + control_camera_video = None + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=(height, width), fps=16, ref_image=None) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + else: + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + print('Merge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + if model_type == "Inpaint": + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + strength = float(denoise_strength), + comfyui_progressbar = True, + ).videos + else: + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + ref_image = ref_image, + start_image = start_image, + clip_image = clip_image, + control_video = input_video, + control_camera_video = control_camera_video, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + return (videos,) + diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_control_camera.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_control_camera.json new file mode 100644 index 0000000000000000000000000000000000000000..4af05e6836cad9747f5705ad706ceed940fc59d3 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_control_camera.json @@ -0,0 +1,894 @@ +{ + "id": "addf6f13-291c-4b88-a80c-cd5785fa8f42", + "revision": 0, + "last_node_id": 138, + "last_link_id": 305, + "nodes": [ + { + "id": 107, + "type": "Note", + "pos": [ + 4, + 634 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 108, + "type": "Note", + "pos": [ + -110, + 842 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 122, + "type": "FunTextBox", + "pos": [ + 238, + 805 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 302 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 129, + "type": "CameraBasicFromChaoJie", + "pos": [ + 805.2059326171875, + 1012.381103515625 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CameraPose", + "type": "CameraPose", + "links": null + } + ], + "properties": { + "Node name for S&R": "CameraBasicFromChaoJie" + }, + "widgets_values": [ + "Static", + 1, + 16 + ] + }, + { + "id": 130, + "type": "CameraTrajectoryFromChaoJie", + "pos": [ + 1170.206298828125, + 763.3814697265625 + ], + "size": [ + 367.79998779296875, + 150 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "camera_pose", + "type": "CameraPose", + "link": 285 + } + ], + "outputs": [ + { + "name": "camera_trajectory", + "type": "STRING", + "slot_index": 0, + "links": [ + 304 + ] + }, + { + "name": "video_length", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "CameraTrajectoryFromChaoJie" + }, + "widgets_values": [ + 0.532139961, + 0.946026558, + 0.5, + 0.5 + ] + }, + { + "id": 106, + "type": "VHS_VideoCombine", + "pos": [ + 1408, + 68 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 305 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00127.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 121, + "type": "FunTextBox", + "pos": [ + 235, + 539 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 301 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "Fireworks light up the evening sky over a sprawling cityscape with gothic-style buildings featuring pointed towers and clock faces. The city is lit by both artificial lights from the buildings and the colorful bursts of the fireworks. The scene is viewed from an elevated angle, showcasing a vibrant urban environment set against a backdrop of a dramatic, partially cloudy sky at dusk." + ] + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 237.59738159179688, + 1164.597412109375 + ], + "size": [ + 378.07147216796875, + 314 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 303 + ] + }, + { + "label": "遮罩", + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image（图片到视频的开始图片）", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "5.png", + "image" + ] + }, + { + "id": 131, + "type": "CameraCombineFromChaoJie", + "pos": [ + 814.2059326171875, + 763.3814697265625 + ], + "size": [ + 315, + 178 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CameraPose", + "type": "CameraPose", + "links": [ + 285 + ] + } + ], + "properties": { + "Node name for S&R": "CameraCombineFromChaoJie" + }, + "widgets_values": [ + "Pan Right", + "Pan Up", + "Static", + "Static", + 1, + 81 + ] + }, + { + "id": 110, + "type": "Note", + "pos": [ + 1158.206298828125, + 970.381103515625 + ], + "size": [ + 608.1410522460938, + 188.2682342529297 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "CameraCombine is used to combine multiple camera movements, while CameraBasic produces a single camera movement. The nodes come from https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/. Since ComfyUI-CameraCtrl-Wrapper requires a specific version of diffusers, the code has been copied into the current repository.\n(CameraCombine用于组合多个镜头运动，CameraBasic产出单个镜头运动；节点来自于https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/，由于ComfyUI-CameraCtrl-Wrapper有具体diffusers版本要求，故复制代码到当前库中。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 133, + "type": "LoadWanClipEncoderModel", + "pos": [ + 292.5342102050781, + -5.799493789672852 + ], + "size": [ + 365.69586181640625, + 92.28972625732422 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "clip_encoder", + "type": "ClipEncoderModel", + "links": [ + 297 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanClipEncoderModel" + }, + "widgets_values": [ + "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", + "bf16" + ] + }, + { + "id": 136, + "type": "LoadWanVAEModel", + "pos": [ + -165.57484436035156, + 159.48216247558594 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 294 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 112, + "type": "Note", + "pos": [ + 245.56407165527344, + -216.91796875 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 135, + "type": "LoadWanTransformerModel", + "pos": [ + -165.57484436035156, + -4.231314182281494 + ], + "size": [ + 401.8365478515625, + 102.11343383789062 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 293 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 299 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTransformerModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control-Camera_transformer.safetensors", + "bf16" + ] + }, + { + "id": 134, + "type": "CombineWanPipeline", + "pos": [ + 298.9104919433594, + 201.77969360351562 + ], + "size": [ + 362.322021484375, + 206 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 293 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 294 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 295 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 296 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": 297 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 299 + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": null + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 300 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWanPipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8", + "Control" + ] + }, + { + "id": 137, + "type": "LoadWanTextEncoderModel", + "pos": [ + -165.57484436035156, + 310.0520324707031 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 295 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 296 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 138, + "type": "WanFunV2VSampler", + "pos": [ + 834.1106567382812, + 58.27174377441406 + ], + "size": [ + 445.0664978027344, + 530 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 300 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 301 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 302 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 303 + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": 304 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 305 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 173802627968141, + "randomize", + 50, + 6, + 1, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 285, + 131, + 0, + 130, + 0, + "CameraPose" + ], + [ + 293, + 135, + 0, + 134, + 0, + "TransformerModel" + ], + [ + 294, + 136, + 0, + 134, + 1, + "VAEModel" + ], + [ + 295, + 137, + 0, + 134, + 2, + "TextEncoderModel" + ], + [ + 296, + 137, + 1, + 134, + 3, + "Tokenizer" + ], + [ + 297, + 133, + 0, + 134, + 4, + "ClipEncoderModel" + ], + [ + 299, + 135, + 1, + 134, + 5, + "STRING" + ], + [ + 300, + 134, + 0, + 138, + 0, + "FunModels" + ], + [ + 301, + 121, + 0, + 138, + 1, + "STRING_PROMPT" + ], + [ + 302, + 122, + 0, + 138, + 2, + "STRING_PROMPT" + ], + [ + 303, + 100, + 0, + 138, + 5, + "IMAGE" + ], + [ + 304, + 130, + 0, + 138, + 7, + "STRING" + ], + [ + 305, + 138, + 0, + 106, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Generate Control Video", + "bounding": [ + 773, + 666, + 1025, + 531 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "First Image", + "bounding": [ + 191, + 1068, + 475, + 456 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 191, + 456, + 475, + 587 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + -175.57485961914062, + -79.39949798583984, + 846.8070678710938, + 501.4514465332031 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 312.885730326785, + 275.34866494065034 + ] + }, + "frontendVersion": "1.21.3", + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.44" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_i2v.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..0322fb396d99f3f7f3ef4f875617f81ba4394f0f --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_i2v.json @@ -0,0 +1,753 @@ +{ + "id": "7771a2ee-e50b-4606-8122-63139795c002", + "revision": 0, + "last_node_id": 120, + "last_link_id": 88, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 83 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 95, + "type": "Note", + "pos": [ + 35.71394348144531, + 536.9210205078125 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 84 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 87, + "type": "LoadImage", + "pos": [ + 280.5103759765625, + 477.9588623046875 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 86, + 88 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "2.png", + "image" + ] + }, + { + "id": 103, + "type": "LoadImage", + "pos": [ + 689.5659790039062, + 477.3172302246094 + ], + "size": [ + 318.56072998046875, + 326 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "End Image", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "070008-d3df70b6-1c49-43d5-a39b-ba357e48c17a-3149685904.png", + "image" + ] + }, + { + "id": 115, + "type": "LoadWanVAEModel", + "pos": [ + 250.1365203857422, + -450.6431884765625 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 114, + "type": "LoadWanClipEncoderModel", + "pos": [ + 708.2453002929688, + -615.9248046875 + ], + "size": [ + 365.69586181640625, + 92.28972625732422 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "clip_encoder", + "type": "ClipEncoderModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanClipEncoderModel" + }, + "widgets_values": [ + "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", + "bf16" + ] + }, + { + "id": 113, + "type": "LoadWanTransformerModel", + "pos": [ + 250.1365203857422, + -614.3566284179688 + ], + "size": [ + 401.8365478515625, + 102.11343383789062 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 76 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTransformerModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-InP_transformer.safetensors", + "bf16" + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 20.679954528808594, + -43.9061279296875 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 117, + "type": "LoadWanTextEncoderModel", + "pos": [ + 250.1365203857422, + -300.0733642578125 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 77 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "fp16" + ] + }, + { + "id": 119, + "type": "Note", + "pos": [ + 1109.983154296875, + -376.7947692871094 + ], + "size": [ + 257.3695983886719, + 117.2574234008789 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 118, + "type": "CombineWanPipeline", + "pos": [ + 714.62158203125, + -408.345703125 + ], + "size": [ + 362.322021484375, + 206 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 76 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 78 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 77 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 80 + }, + { + "name": "clip_encoder", + "type": "ClipEncoderModel", + "link": 79 + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 75 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWanPipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8", + "Inpaint" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1190.691162109375, + -90.75740051269531 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 87 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00124.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 120, + "type": "WanFunInpaintSampler", + "pos": [ + 776.4642333984375, + -92.8369140625 + ], + "size": [ + 330.2369079589844, + 446 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 82 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 83 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 84 + }, + { + "name": "start_img", + "shape": 7, + "type": "IMAGE", + "link": 88 + }, + { + "name": "end_img", + "shape": 7, + "type": "IMAGE", + "link": 86 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunInpaintSampler" + }, + "widgets_values": [ + 81, + 640, + 720614234271255, + "randomize", + 50, + 6, + "Flow", + 5, + 0.1, + true, + 5, + true, + 0 + ] + } + ], + "links": [ + [ + 75, + 113, + 1, + 118, + 6, + "STRING" + ], + [ + 76, + 113, + 0, + 118, + 0, + "TransformerModel" + ], + [ + 77, + 117, + 0, + 118, + 2, + "TextEncoderModel" + ], + [ + 78, + 115, + 0, + 118, + 1, + "VAEModel" + ], + [ + 79, + 114, + 0, + 118, + 4, + "ClipEncoderModel" + ], + [ + 80, + 117, + 1, + 118, + 3, + "Tokenizer" + ], + [ + 82, + 118, + 0, + 120, + 0, + "FunModels" + ], + [ + 83, + 75, + 0, + 120, + 1, + "STRING_PROMPT" + ], + [ + 84, + 73, + 0, + 120, + 2, + "STRING_PROMPT" + ], + [ + 86, + 87, + 0, + 120, + 4, + "IMAGE" + ], + [ + 87, + 120, + 0, + 17, + 0, + "IMAGE" + ], + [ + 88, + 87, + 0, + 120, + 3, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Image", + "bounding": [ + 222, + 396, + 875.5350952148438, + 432.9040832519531 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model", + "bounding": [ + 212.720947265625, + -700.9775390625, + 924.4893798828125, + 529.3465576171875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917356, + "offset": [ + 268.43715520686237, + 382.92364969985795 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "comfy-core": "0.3.44", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_t2v.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..0ed597273c3dd3a7a2bf721da7af71396595cbb1 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_t2v.json @@ -0,0 +1,613 @@ +{ + "id": "7771a2ee-e50b-4606-8122-63139795c002", + "revision": 0, + "last_node_id": 120, + "last_link_id": 85, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 83 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 84 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 115, + "type": "LoadWanVAEModel", + "pos": [ + 250.1365203857422, + -450.6431884765625 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 118, + "type": "CombineWanPipeline", + "pos": [ + 714.62158203125, + -408.345703125 + ], + "size": [ + 362.322021484375, + 206 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 76 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 78 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 77 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 80 + }, + { + "name": "clip_encoder", + "type": "ClipEncoderModel", + "link": 79 + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 75 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWanPipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8", + "Inpaint" + ] + }, + { + "id": 114, + "type": "LoadWanClipEncoderModel", + "pos": [ + 708.2453002929688, + -615.9248046875 + ], + "size": [ + 365.69586181640625, + 92.28972625732422 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "clip_encoder", + "type": "ClipEncoderModel", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanClipEncoderModel" + }, + "widgets_values": [ + "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", + "bf16" + ] + }, + { + "id": 113, + "type": "LoadWanTransformerModel", + "pos": [ + 250.1365203857422, + -614.3566284179688 + ], + "size": [ + 401.8365478515625, + 102.11343383789062 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 76 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTransformerModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-InP_transformer.safetensors", + "bf16" + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 20.679954528808594, + -43.9061279296875 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 117, + "type": "LoadWanTextEncoderModel", + "pos": [ + 250.1365203857422, + -300.0733642578125 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 77 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "fp16" + ] + }, + { + "id": 119, + "type": "Note", + "pos": [ + 1109.983154296875, + -376.7947692871094 + ], + "size": [ + 257.3695983886719, + 117.2574234008789 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 120, + "type": "WanFunT2VSampler", + "pos": [ + 833.6475219726562, + -69.29597473144531 + ], + "size": [ + 317.0025329589844, + 454 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 82 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 83 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 84 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 85 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunT2VSampler" + }, + "widgets_values": [ + 81, + 832, + 480, + false, + 45657127515004, + "randomize", + 50, + 6, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1264.4798583984375, + -72.03805541992188 + ], + "size": [ + 390, + 546 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 85 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00121.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + } + ], + "links": [ + [ + 75, + 113, + 1, + 118, + 6, + "STRING" + ], + [ + 76, + 113, + 0, + 118, + 0, + "TransformerModel" + ], + [ + 77, + 117, + 0, + 118, + 2, + "TextEncoderModel" + ], + [ + 78, + 115, + 0, + 118, + 1, + "VAEModel" + ], + [ + 79, + 114, + 0, + 118, + 4, + "ClipEncoderModel" + ], + [ + 80, + 117, + 1, + 118, + 3, + "Tokenizer" + ], + [ + 82, + 118, + 0, + 120, + 0, + "FunModels" + ], + [ + 83, + 75, + 0, + 120, + 1, + "STRING_PROMPT" + ], + [ + 84, + 73, + 0, + 120, + 2, + "STRING_PROMPT" + ], + [ + 85, + 120, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model", + "bounding": [ + 212.720947265625, + -700.9775390625, + 924.4893798828125, + 529.3465576171875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015778, + "offset": [ + 73.41944426936227, + 728.2656848561068 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_v2v_control.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_v2v_control.json new file mode 100644 index 0000000000000000000000000000000000000000..abcb2ded4e008aa0992f915caa7dad174af47cfd --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_v2v_control.json @@ -0,0 +1,794 @@ +{ + "id": "7fbc1be5-085f-4bd7-886a-31a294f07d9b", + "revision": 0, + "last_node_id": 104, + "last_link_id": 82, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.739953994750977, + 462.38665771484375 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 335, + 476 + ], + "size": [ + 252.056640625, + 688.545166015625 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 81 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 79 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + ] + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 98, + "type": "LoadWanClipEncoderModel", + "pos": [ + 548.1182250976562, + -611.0917358398438 + ], + "size": [ + 365.69586181640625, + 92.28972625732422 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "clip_encoder", + "type": "ClipEncoderModel", + "links": [ + 70 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanClipEncoderModel" + }, + "widgets_values": [ + "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", + "bf16" + ] + }, + { + "id": 101, + "type": "LoadWanVAEModel", + "pos": [ + 90.00912475585938, + -445.8097839355469 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 89, + "type": "Note", + "pos": [ + -378.2093811035156, + -515.0055541992188 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 100, + "type": "LoadWanTransformerModel", + "pos": [ + 90.00912475585938, + -609.5235595703125 + ], + "size": [ + 401.8365478515625, + 102.11343383789062 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 66 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 71 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTransformerModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control_transformer.safetensors", + "bf16" + ] + }, + { + "id": 99, + "type": "CombineWanPipeline", + "pos": [ + 554.4943237304688, + -403.5122985839844 + ], + "size": [ + 362.322021484375, + 206 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 66 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 67 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 68 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 69 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": 70 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 71 + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": null + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWanPipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8", + "Control" + ] + }, + { + "id": 102, + "type": "LoadWanTextEncoderModel", + "pos": [ + 90.00912475585938, + -295.2400207519531 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 68 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 69 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1274, + 10 + ], + "size": [ + 390.9534912109375, + 966.9860229492188 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 82 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00126.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 104, + "type": "WanFunV2VSampler", + "pos": [ + 792, + 11 + ], + "size": [ + 432.6514892578125, + 530 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 78 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 79 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 80 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 81 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "randomize", + 50, + 6, + 1, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 66, + 100, + 0, + 99, + 0, + "TransformerModel" + ], + [ + 67, + 101, + 0, + 99, + 1, + "VAEModel" + ], + [ + 68, + 102, + 0, + 99, + 2, + "TextEncoderModel" + ], + [ + 69, + 102, + 1, + 99, + 3, + "Tokenizer" + ], + [ + 70, + 98, + 0, + 99, + 4, + "ClipEncoderModel" + ], + [ + 71, + 100, + 1, + 99, + 5, + "STRING" + ], + [ + 78, + 99, + 0, + 104, + 0, + "FunModels" + ], + [ + 79, + 92, + 0, + 104, + 1, + "STRING_PROMPT" + ], + [ + 80, + 94, + 0, + 104, + 2, + "STRING_PROMPT" + ], + [ + 81, + 85, + 0, + 104, + 4, + "IMAGE" + ], + [ + 82, + 104, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 487, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 80.00911712646484, + -684.6915893554688, + 846.8070678710938, + 501.4514465332031 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917359, + "offset": [ + 235.4487553267823, + 718.4154520500232 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_v2v_control_ref.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_v2v_control_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..a3ef635c607f1f272cf211169e309a23842de01e --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_chunked_loading_workflow_v2v_control_ref.json @@ -0,0 +1,840 @@ +{ + "id": "7c39ed4d-e738-43a4-af77-13d5fc25f55b", + "revision": 0, + "last_node_id": 104, + "last_link_id": 80, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.145149230957031, + 527.0673217773438 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": [ + -192, + -293 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 321.473388671875, + 562.38037109375 + ], + "size": [ + 252.056640625, + 688.545166015625 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 78 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 266.0400085449219, + 188.32296752929688 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 262.0400085449219, + -35.67719650268555 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位动漫风格的女孩。她有着紫色的短发，头上戴着一个黑色和金色相间的蝴蝶结。她的表情显得有些严肃或沉思，眼睛大而有神。女孩穿着一件白色衬衫，外面搭配了一件深蓝色的背心，背心上有一个粉色的蝴蝶结装饰。她的裙子是白色的，裙摆蓬松，整体造型非常可爱且精致。背景是一个简单的圆形图案，颜色为粉红色和灰色相间，给人一种柔和的感觉。整个画面色调柔和，人物形象生动鲜明。" + ] + }, + { + "id": 97, + "type": "LoadImage", + "pos": [ + 616.0682983398438, + 574.69970703125 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 79 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 100, + "type": "LoadWanVAEModel", + "pos": [ + 256.8955078125, + -424.50177001953125 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 69 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 101, + "type": "LoadWanTextEncoderModel", + "pos": [ + 256.8955078125, + -273.9320373535156 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 70 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 71 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 103, + "type": "LoadWanClipEncoderModel", + "pos": [ + 715.0050659179688, + -589.7835083007812 + ], + "size": [ + 365.69586181640625, + 92.28972625732422 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "clip_encoder", + "type": "ClipEncoderModel", + "links": [ + 72 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanClipEncoderModel" + }, + "widgets_values": [ + "models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth", + "bf16" + ] + }, + { + "id": 102, + "type": "CombineWanPipeline", + "pos": [ + 721.3811645507812, + -382.20428466796875 + ], + "size": [ + 362.322021484375, + 206 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 68 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 69 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 70 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 71 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": 72 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 73 + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": null + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWanPipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8", + "Control" + ] + }, + { + "id": 99, + "type": "LoadWanTransformerModel", + "pos": [ + 256.8955078125, + -588.21533203125 + ], + "size": [ + 401.8365478515625, + 102.11343383789062 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 68 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 73 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTransformerModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control_transformer.safetensors", + "bf16" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1261.716064453125, + -85.7489242553711 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 80 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00128.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 104, + "type": "WanFunV2VSampler", + "pos": [ + 757.6497802734375, + -82.27965545654297 + ], + "size": [ + 408.6736145019531, + 530 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 75 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 78 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": 79 + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 655715659240565, + "randomize", + 50, + 6, + 1, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 68, + 99, + 0, + 102, + 0, + "TransformerModel" + ], + [ + 69, + 100, + 0, + 102, + 1, + "VAEModel" + ], + [ + 70, + 101, + 0, + 102, + 2, + "TextEncoderModel" + ], + [ + 71, + 101, + 1, + 102, + 3, + "Tokenizer" + ], + [ + 72, + 103, + 0, + 102, + 4, + "ClipEncoderModel" + ], + [ + 73, + 99, + 1, + 102, + 5, + "STRING" + ], + [ + 75, + 102, + 0, + 104, + 0, + "FunModels" + ], + [ + 76, + 92, + 0, + 104, + 1, + "STRING_PROMPT" + ], + [ + 77, + 94, + 0, + 104, + 2, + "STRING_PROMPT" + ], + [ + 78, + 85, + 0, + 104, + 4, + "IMAGE" + ], + [ + 79, + 97, + 0, + 104, + 6, + "IMAGE" + ], + [ + 80, + 104, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Video", + "bounding": [ + 204.06814575195312, + 471.69964599609375, + 761, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 226.03997802734375, + -116.67713165283203, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 246.89527893066406, + -663.3834228515625, + 846.8070678710938, + 501.4514465332031 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917358, + "offset": [ + 170.43200493615862, + 754.5590973625236 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "comfy-core": "0.3.44" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_control_camera.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_control_camera.json new file mode 100644 index 0000000000000000000000000000000000000000..0144791da43b698a7370e1f1b3f5420053621e78 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_control_camera.json @@ -0,0 +1,666 @@ +{ + "id": "addf6f13-291c-4b88-a80c-cd5785fa8f42", + "revision": 0, + "last_node_id": 132, + "last_link_id": 292, + "nodes": [ + { + "id": 107, + "type": "Note", + "pos": [ + 4, + 634 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 108, + "type": "Note", + "pos": [ + -110, + 842 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 122, + "type": "FunTextBox", + "pos": [ + 238, + 805 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 289 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 129, + "type": "CameraBasicFromChaoJie", + "pos": [ + 805.2059326171875, + 1012.381103515625 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CameraPose", + "type": "CameraPose", + "links": null + } + ], + "properties": { + "Node name for S&R": "CameraBasicFromChaoJie" + }, + "widgets_values": [ + "Static", + 1, + 16 + ] + }, + { + "id": 130, + "type": "CameraTrajectoryFromChaoJie", + "pos": [ + 1170.206298828125, + 763.3814697265625 + ], + "size": [ + 367.79998779296875, + 150 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "camera_pose", + "type": "CameraPose", + "link": 285 + } + ], + "outputs": [ + { + "name": "camera_trajectory", + "type": "STRING", + "slot_index": 0, + "links": [ + 292 + ] + }, + { + "name": "video_length", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "CameraTrajectoryFromChaoJie" + }, + "widgets_values": [ + 0.532139961, + 0.946026558, + 0.5, + 0.5 + ] + }, + { + "id": 106, + "type": "VHS_VideoCombine", + "pos": [ + 1408, + 68 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 291 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00061.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 121, + "type": "FunTextBox", + "pos": [ + 235, + 539 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 288 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "Fireworks light up the evening sky over a sprawling cityscape with gothic-style buildings featuring pointed towers and clock faces. The city is lit by both artificial lights from the buildings and the colorful bursts of the fireworks. The scene is viewed from an elevated angle, showcasing a vibrant urban environment set against a backdrop of a dramatic, partially cloudy sky at dusk." + ] + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 237.59738159179688, + 1164.597412109375 + ], + "size": [ + 378.07147216796875, + 314 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 290 + ] + }, + { + "label": "遮罩", + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image（图片到视频的开始图片）", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "5.png", + "image" + ] + }, + { + "id": 131, + "type": "CameraCombineFromChaoJie", + "pos": [ + 814.2059326171875, + 763.3814697265625 + ], + "size": [ + 315, + 178 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CameraPose", + "type": "CameraPose", + "links": [ + 285 + ] + } + ], + "properties": { + "Node name for S&R": "CameraCombineFromChaoJie" + }, + "widgets_values": [ + "Pan Right", + "Pan Up", + "Static", + "Static", + 1, + 81 + ] + }, + { + "id": 110, + "type": "Note", + "pos": [ + 1158.206298828125, + 970.381103515625 + ], + "size": [ + 608.1410522460938, + 188.2682342529297 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "CameraCombine is used to combine multiple camera movements, while CameraBasic produces a single camera movement. The nodes come from https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/. Since ComfyUI-CameraCtrl-Wrapper requires a specific version of diffusers, the code has been copied into the current repository.\n(CameraCombine用于组合多个镜头运动，CameraBasic产出单个镜头运动；节点来自于https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/，由于ComfyUI-CameraCtrl-Wrapper有具体diffusers版本要求，故复制代码到当前库中。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 132, + "type": "WanFunV2VSampler", + "pos": [ + 899, + 68 + ], + "size": [ + 428.4000244140625, + 506 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 287 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 288 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 289 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 290 + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": 292 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 291 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6, + 1, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 123, + "type": "LoadWanFunModel", + "pos": [ + 281, + 251 + ], + "size": [ + 315, + 154 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "slot_index": 0, + "links": [ + 287 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control-Camera", + "Control", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 112, + "type": "Note", + "pos": [ + -203, + 252 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 285, + 131, + 0, + 130, + 0, + "CameraPose" + ], + [ + 287, + 123, + 0, + 132, + 0, + "FunModels" + ], + [ + 288, + 121, + 0, + 132, + 1, + "STRING_PROMPT" + ], + [ + 289, + 122, + 0, + 132, + 2, + "STRING_PROMPT" + ], + [ + 290, + 100, + 0, + 132, + 5, + "IMAGE" + ], + [ + 291, + 132, + 0, + 106, + 0, + "IMAGE" + ], + [ + 292, + 130, + 0, + 132, + 7, + "STRING" + ] + ], + "groups": [ + { + "id": 1, + "title": "Generate Control Video", + "bounding": [ + 773, + 666, + 1025, + 531 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "First Image", + "bounding": [ + 191, + 1068, + 475, + 456 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 191, + 456, + 475, + 587 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 189, + 160, + 475, + 269 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 236.8040025924094, + -27.611371387475245 + ] + }, + "node_versions": { + "CogVideoX-Fun": "e935795d7684aaba72be65d2ad5ef580c0526f7a", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.44" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_control_trajectory.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_control_trajectory.json new file mode 100644 index 0000000000000000000000000000000000000000..9cc8650f752d084195d016ed51c9c586bad49454 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_control_trajectory.json @@ -0,0 +1,1041 @@ +{ + "id": "34fdaa39-a397-46b5-84d1-8846799c8c09", + "revision": 0, + "last_node_id": 126, + "last_link_id": 287, + "nodes": [ + { + "id": 107, + "type": "Note", + "pos": [ + 4, + 634 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 108, + "type": "Note", + "pos": [ + -110, + 842 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 238, + 1165 + ], + "size": [ + 378.07147216796875, + 314 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 285 + ] + }, + { + "label": "遮罩", + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image（图片到视频的开始图片）", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "1.png", + "image" + ] + }, + { + "id": 110, + "type": "Note", + "pos": [ + 847, + 613 + ], + "size": [ + 608.1410522460938, + 188.2682342529297 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Please set the mask height and mask width according to the height and width of the reference image. \nPlease set the video_length of the Spline Editor below to be the same as the video_length of the Sampler above. \nThe nodes are from KJNodes. For more details, please check https://github.com/kijai/ComfyUI-KJNodes/tree/main. \n\n请根据参考图片的高和宽设置mask height和mask width；\n请将下方Spline Editor的video_legnth置的与上方Sampler的video_legnth一样；\n部分节点来自于KJNodes，具体查看https://github.com/kijai/ComfyUI-KJNodes/tree/main；" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 118, + "type": "AppendStringsToList", + "pos": [ + 1140.1396484375, + 909.9193115234375 + ], + "size": [ + 315, + 82 + ], + "flags": { + "collapsed": false + }, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "string1", + "type": "STRING", + "link": 265 + }, + { + "name": "string2", + "type": "STRING", + "link": 266 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "slot_index": 0, + "links": [ + 267 + ] + } + ], + "properties": { + "Node name for S&R": "AppendStringsToList" + }, + "widgets_values": [] + }, + { + "id": 121, + "type": "FunTextBox", + "pos": [ + 235, + 539 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 282 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一只棕褐色的狗正摇晃着脑袋，坐在一个舒适的房间里的浅色沙发上。沙发看起来柔软而宽敞，为这只活泼的狗狗提供了一个完美的休息地点。在狗的后面，靠墙摆放着一个架子，架子上挂着一幅精美的镶框画，画中描绘着一些美丽的风景或场景。画框周围装饰着粉红色的花朵，这些花朵不仅增添了房间的色彩，还带来了一丝自然和生机。房间里的灯光柔和而温暖，从天花板上的吊灯和角落里的台灯散发出来，营造出一种温馨舒适的氛围。整个空间给人一种宁静和谐的感觉，仿佛时间在这里变得缓慢而美好。" + ] + }, + { + "id": 114, + "type": "ImageMaximumNode", + "pos": [ + 2074, + 905 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "video_1", + "type": "IMAGE", + "link": 287 + }, + { + "name": "video_2", + "type": "IMAGE", + "link": 262 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 264 + ] + } + ], + "properties": { + "Node name for S&R": "ImageMaximumNode" + }, + "widgets_values": [] + }, + { + "id": 95, + "type": "CreateTrajectoryBasedOnKJNodes", + "pos": [ + 1574.139404296875, + 929.9193115234375 + ], + "size": [ + 428.4000244140625, + 58 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "coordinates", + "type": "STRING", + "link": 267 + }, + { + "name": "masks", + "type": "MASK", + "link": 249 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 237, + 262, + 284 + ] + } + ], + "properties": { + "Node name for S&R": "CreateTrajectoryBasedOnKJNodes" + }, + "widgets_values": [] + }, + { + "id": 122, + "type": "FunTextBox", + "pos": [ + 238, + 805 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 283 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 97, + "type": "SplineEditor", + "pos": [ + 855.1397705078125, + 1058.91943359375 + ], + "size": [ + 645, + 832 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "bg_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "slot_index": 0, + "links": [ + 249 + ] + }, + { + "name": "coord_str", + "type": "STRING", + "slot_index": 1, + "links": [ + 265 + ] + }, + { + "name": "float", + "type": "FLOAT", + "links": null + }, + { + "name": "count", + "type": "INT", + "links": null + }, + { + "name": "normalized_str", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "SplineEditor", + "points": "SplineEditor", + "imgData": null + }, + "widgets_values": [ + "[{\"points\":[{\"x\":236.74497000000005,\"y\":169.10355000000004},{\"x\":263.53799999999995,\"y\":230.59575},{\"x\":321.61075881587004,\"y\":229.72197058276433},{\"x\":355.3934015486295,\"y\":168.9132136637973},{\"x\":343.23165016483614,\"y\":82.42964826793309},{\"x\":275.6663646993172,\"y\":55.40353408172552},{\"x\":197.29063355931524,\"y\":85.13225968655384},{\"x\":177.02104791965957,\"y\":148.64362802414163},{\"x\":205.39846781517753,\"y\":232.4245820013851},{\"x\":259.45069618759265,\"y\":167.56190795448694}],\"color\":\"#1f77b4\",\"name\":\"Spline 1\"}]", + "[[{\"x\":236.74496459960938,\"y\":169.10354614257812},{\"x\":238.94464111328125,\"y\":177.68576049804688},{\"x\":241.3327178955078,\"y\":186.21737670898438},{\"x\":243.95693969726562,\"y\":194.67919921875},{\"x\":246.88812255859375,\"y\":203.03933715820312},{\"x\":250.2393035888672,\"y\":211.23939514160156},{\"x\":254.20826721191406,\"y\":219.15655517578125},{\"x\":259.18768310546875,\"y\":226.46995544433594},{\"x\":265.87689208984375,\"y\":232.22171020507812},{\"x\":273.4639892578125,\"y\":236.787353515625},{\"x\":281.6380920410156,\"y\":240.17092895507812},{\"x\":290.3527526855469,\"y\":241.61793518066406},{\"x\":299.1367492675781,\"y\":240.67857360839844},{\"x\":307.49658203125,\"y\":237.7835235595703},{\"x\":315.3465576171875,\"y\":233.68600463867188},{\"x\":322.80859375,\"y\":228.9126739501953},{\"x\":330.00958251953125,\"y\":223.75323486328125},{\"x\":336.7490234375,\"y\":218.0097198486328},{\"x\":342.5494689941406,\"y\":211.32901000976562},{\"x\":346.96185302734375,\"y\":203.6618194580078},{\"x\":350.05450439453125,\"y\":195.3665771484375},{\"x\":352.2560729980469,\"y\":186.787109375},{\"x\":353.9389343261719,\"y\":178.08937072753906},{\"x\":355.33001708984375,\"y\":169.3397216796875},{\"x\":356.59344482421875,\"y\":160.57052612304688},{\"x\":357.7420959472656,\"y\":151.78565979003906},{\"x\":358.6518249511719,\"y\":142.9732208251953},{\"x\":359.1504211425781,\"y\":134.1287384033203},{\"x\":359.0207214355469,\"y\":125.2725601196289},{\"x\":358.035400390625,\"y\":116.4721908569336},{\"x\":356.0333251953125,\"y\":107.84730529785156},{\"x\":352.9960632324219,\"y\":99.53003692626953},{\"x\":349.046142578125,\"y\":91.60392761230469},{\"x\":344.37164306640625,\"y\":84.08070373535156},{\"x\":339.17694091796875,\"y\":76.9051742553711},{\"x\":333.4718322753906,\"y\":70.13098907470703},{\"x\":326.9710693359375,\"y\":64.12535858154297},{\"x\":319.4295654296875,\"y\":59.51286315917969},{\"x\":311.02655029296875,\"y\":56.7620735168457},{\"x\":302.25640869140625,\"y\":55.55453872680664},{\"x\":293.40484619140625,\"y\":55.21137619018555},{\"x\":284.5453796386719,\"y\":55.24800491333008},{\"x\":275.68701171875,\"y\":55.40314865112305},{\"x\":266.83221435546875,\"y\":55.690216064453125},{\"x\":257.9945373535156,\"y\":56.30451965332031},{\"x\":249.2041473388672,\"y\":57.398460388183594},{\"x\":240.5244140625,\"y\":59.16102981567383},{\"x\":232.06307983398438,\"y\":61.773162841796875},{\"x\":223.95306396484375,\"y\":65.32772827148438},{\"x\":216.29017639160156,\"y\":69.7664566040039},{\"x\":209.08375549316406,\"y\":74.91584777832031},{\"x\":202.27122497558594,\"y\":80.57798767089844},{\"x\":195.76087951660156,\"y\":86.58625030517578},{\"x\":189.468505859375,\"y\":92.8224105834961},{\"x\":183.6200714111328,\"y\":99.47235107421875},{\"x\":178.9163055419922,\"y\":106.95783233642578},{\"x\":176.3991241455078,\"y\":115.42044830322266},{\"x\":175.83934020996094,\"y\":124.25370788574219},{\"x\":176.12339782714844,\"y\":133.10797119140625},{\"x\":176.63381958007812,\"y\":141.95301818847656},{\"x\":177.14378356933594,\"y\":150.79808044433594},{\"x\":177.73876953125,\"y\":159.63778686523438},{\"x\":178.5083770751953,\"y\":168.46385192871094},{\"x\":179.4958038330078,\"y\":177.26815795898438},{\"x\":180.75762939453125,\"y\":186.03709411621094},{\"x\":182.37294006347656,\"y\":194.7476043701172},{\"x\":184.4583282470703,\"y\":203.35687255859375},{\"x\":187.1986083984375,\"y\":211.7789306640625},{\"x\":190.9047393798828,\"y\":219.81732177734375},{\"x\":196.108154296875,\"y\":226.9572296142578},{\"x\":203.41249084472656,\"y\":231.83749389648438},{\"x\":211.85409545898438,\"y\":230.89816284179688},{\"x\":218.7886199951172,\"y\":225.41867065429688},{\"x\":224.8160400390625,\"y\":218.92974853515625},{\"x\":230.3641357421875,\"y\":212.0236358642578},{\"x\":235.608642578125,\"y\":204.8833770751953},{\"x\":240.63941955566406,\"y\":197.59080505371094},{\"x\":245.50965881347656,\"y\":190.1898651123047},{\"x\":250.25364685058594,\"y\":182.7073516845703},{\"x\":254.8948974609375,\"y\":175.16050720214844},{\"x\":259.45068359375,\"y\":167.56190490722656}]]", + 600, + 382, + 81, + "path", + "cardinal", + 0.5, + 1, + "list", + 0, + 1, + "", + null + ] + }, + { + "id": 119, + "type": "SplineEditor", + "pos": [ + 1544.139404296875, + 1047.919189453125 + ], + "size": [ + 645, + 832 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "bg_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "slot_index": 0, + "links": [] + }, + { + "name": "coord_str", + "type": "STRING", + "slot_index": 1, + "links": [ + 266 + ] + }, + { + "name": "float", + "type": "FLOAT", + "links": null + }, + { + "name": "count", + "type": "INT", + "links": null + }, + { + "name": "normalized_str", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "SplineEditor", + "points": "SplineEditor", + "imgData": null + }, + "widgets_values": [ + "[{\"points\":[{\"x\":63.916760050380844,\"y\":114.45559357858895},{\"x\":71.34894145158792,\"y\":114.45559357858895}],\"color\":\"#1f77b4\",\"name\":\"Spline 1\"}]", + "[[{\"x\":63.9167594909668,\"y\":114.45559692382812},{\"x\":64.00965881347656,\"y\":114.45559692382812},{\"x\":64.1025619506836,\"y\":114.45559692382812},{\"x\":64.19546508789062,\"y\":114.45559692382812},{\"x\":64.28836822509766,\"y\":114.45559692382812},{\"x\":64.38127136230469,\"y\":114.45559692382812},{\"x\":64.47417449951172,\"y\":114.45559692382812},{\"x\":64.56707763671875,\"y\":114.45559692382812},{\"x\":64.65998077392578,\"y\":114.45559692382812},{\"x\":64.75287628173828,\"y\":114.45559692382812},{\"x\":64.84577941894531,\"y\":114.45559692382812},{\"x\":64.93868255615234,\"y\":114.45559692382812},{\"x\":65.03158569335938,\"y\":114.45559692382812},{\"x\":65.1244888305664,\"y\":114.45559692382812},{\"x\":65.21739196777344,\"y\":114.45559692382812},{\"x\":65.31029510498047,\"y\":114.45559692382812},{\"x\":65.4031982421875,\"y\":114.45559692382812},{\"x\":65.49609375,\"y\":114.45559692382812},{\"x\":65.58899688720703,\"y\":114.45559692382812},{\"x\":65.68190002441406,\"y\":114.45559692382812},{\"x\":65.7748031616211,\"y\":114.45559692382812},{\"x\":65.86770629882812,\"y\":114.45559692382812},{\"x\":65.96060943603516,\"y\":114.45559692382812},{\"x\":66.05351257324219,\"y\":114.45559692382812},{\"x\":66.14641571044922,\"y\":114.45559692382812},{\"x\":66.23931884765625,\"y\":114.45559692382812},{\"x\":66.33221435546875,\"y\":114.45559692382812},{\"x\":66.42511749267578,\"y\":114.45559692382812},{\"x\":66.51802062988281,\"y\":114.45559692382812},{\"x\":66.61092376708984,\"y\":114.45559692382812},{\"x\":66.70382690429688,\"y\":114.45559692382812},{\"x\":66.7967300415039,\"y\":114.45559692382812},{\"x\":66.88963317871094,\"y\":114.45559692382812},{\"x\":66.98253631591797,\"y\":114.45559692382812},{\"x\":67.075439453125,\"y\":114.45559692382812},{\"x\":67.1683349609375,\"y\":114.45559692382812},{\"x\":67.26123809814453,\"y\":114.45559692382812},{\"x\":67.35414123535156,\"y\":114.45559692382812},{\"x\":67.4470443725586,\"y\":114.45559692382812},{\"x\":67.53994750976562,\"y\":114.45559692382812},{\"x\":67.63285064697266,\"y\":114.45559692382812},{\"x\":67.72575378417969,\"y\":114.45559692382812},{\"x\":67.81864929199219,\"y\":114.45559692382812},{\"x\":67.91155242919922,\"y\":114.45559692382812},{\"x\":68.00445556640625,\"y\":114.45559692382812},{\"x\":68.09735870361328,\"y\":114.45559692382812},{\"x\":68.19026184082031,\"y\":114.45559692382812},{\"x\":68.28316497802734,\"y\":114.45559692382812},{\"x\":68.37606811523438,\"y\":114.45559692382812},{\"x\":68.4689712524414,\"y\":114.45559692382812},{\"x\":68.56187438964844,\"y\":114.45559692382812},{\"x\":68.65476989746094,\"y\":114.45559692382812},{\"x\":68.74767303466797,\"y\":114.45559692382812},{\"x\":68.840576171875,\"y\":114.45559692382812},{\"x\":68.93347930908203,\"y\":114.45559692382812},{\"x\":69.02638244628906,\"y\":114.45559692382812},{\"x\":69.1192855834961,\"y\":114.45559692382812},{\"x\":69.21218872070312,\"y\":114.45559692382812},{\"x\":69.30509185791016,\"y\":114.45559692382812},{\"x\":69.39799499511719,\"y\":114.45559692382812},{\"x\":69.49089050292969,\"y\":114.45559692382812},{\"x\":69.58379364013672,\"y\":114.45559692382812},{\"x\":69.67669677734375,\"y\":114.45559692382812},{\"x\":69.76959991455078,\"y\":114.45559692382812},{\"x\":69.86250305175781,\"y\":114.45559692382812},{\"x\":69.95540618896484,\"y\":114.45559692382812},{\"x\":70.04830932617188,\"y\":114.45559692382812},{\"x\":70.1412124633789,\"y\":114.45559692382812},{\"x\":70.2341079711914,\"y\":114.45559692382812},{\"x\":70.32701110839844,\"y\":114.45559692382812},{\"x\":70.41991424560547,\"y\":114.45559692382812},{\"x\":70.5128173828125,\"y\":114.45559692382812},{\"x\":70.60572052001953,\"y\":114.45559692382812},{\"x\":70.69862365722656,\"y\":114.45559692382812},{\"x\":70.7915267944336,\"y\":114.45559692382812},{\"x\":70.88442993164062,\"y\":114.45559692382812},{\"x\":70.97732543945312,\"y\":114.45559692382812},{\"x\":71.07022857666016,\"y\":114.45559692382812},{\"x\":71.16313171386719,\"y\":114.45559692382812},{\"x\":71.25603485107422,\"y\":114.45559692382812},{\"x\":71.34893798828125,\"y\":114.45559692382812}]]", + 600, + 382, + 81, + "path", + "cardinal", + 0.5, + 1, + "list", + 0, + 1, + "", + null + ] + }, + { + "id": 44, + "type": "VHS_VideoCombine", + "pos": [ + 2241.138427734375, + 1051.91943359375 + ], + "size": [ + 530, + 650.4000244140625 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 237 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "title": "Trajectory Outputs", + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Trajectory", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun-Trajectory_00002.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + }, + "muted": false + } + } + }, + { + "id": 115, + "type": "VHS_VideoCombine", + "pos": [ + 2819, + 1056 + ], + "size": [ + 530, + 310 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 264 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "title": "Video with Trajectory Outputs", + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Trajectory-Merge", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "EasyAnimate-Trajectory-Merge_00009.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + }, + "muted": false + } + } + }, + { + "id": 126, + "type": "WanFunV2VSampler", + "pos": [ + 902, + 60 + ], + "size": [ + 428.4000244140625, + 506 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 281 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 282 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 283 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 284 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 285 + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 286, + 287 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6, + 1, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 106, + "type": "VHS_VideoCombine", + "pos": [ + 1390, + 61 + ], + "size": [ + 390, + 310 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 286 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "EasyAnimate_00105.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 123, + "type": "LoadWanFunModel", + "pos": [ + 281, + 251 + ], + "size": [ + 315, + 154 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "slot_index": 0, + "links": [ + 281 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control", + "Control", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 112, + "type": "Note", + "pos": [ + -203, + 252 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 237, + 95, + 0, + 44, + 0, + "IMAGE" + ], + [ + 249, + 97, + 0, + 95, + 1, + "MASK" + ], + [ + 262, + 95, + 0, + 114, + 1, + "IMAGE" + ], + [ + 264, + 114, + 0, + 115, + 0, + "IMAGE" + ], + [ + 265, + 97, + 1, + 118, + 0, + "STRING" + ], + [ + 266, + 119, + 1, + 118, + 1, + "STRING" + ], + [ + 267, + 118, + 0, + 95, + 0, + "STRING" + ], + [ + 281, + 123, + 0, + 126, + 0, + "FunModels" + ], + [ + 282, + 121, + 0, + 126, + 1, + "STRING_PROMPT" + ], + [ + 283, + 122, + 0, + 126, + 2, + "STRING_PROMPT" + ], + [ + 284, + 95, + 0, + 126, + 4, + "IMAGE" + ], + [ + 285, + 100, + 0, + 126, + 5, + "IMAGE" + ], + [ + 286, + 126, + 0, + 106, + 0, + "IMAGE" + ], + [ + 287, + 126, + 0, + 114, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 189, + 160, + 475, + 269 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 191, + 456, + 475, + 587 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "First Image of Trajectory", + "bounding": [ + 191, + 1068, + 475, + 456 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Generate Control Video", + "bounding": [ + 786, + 841, + 2616, + 1056 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015782, + "offset": [ + 119.78785424088606, + -47.62140429778734 + ] + }, + "node_versions": { + "comfy-core": "0.3.44", + "ComfyUI-KJNodes": "ff49e1b01f10a14496b08e21bb89b64d2b15f333", + "CogVideoX-Fun": "e935795d7684aaba72be65d2ad5ef580c0526f7a", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_i2v.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..b4118b35410da4adff81e795855c76499a2444f0 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_i2v.json @@ -0,0 +1,497 @@ +{ + "last_node_id": 102, + "last_link_id": 68, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 66 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "Note", + "pos": { + "0": 17, + "1": -35 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": { + "0": 35.71394348144531, + "1": 536.9210205078125 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1271, + "1": -67 + }, + "size": [ + 390, + 537.4615384615385 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 64, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 80, + "type": "Note", + "pos": { + "0": -95.01139068603516, + "1": -334.30706787109375 + }, + "size": { + "0": 355.636474609375, + "1": 132.4238739013672 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 87, + "type": "LoadImage", + "pos": { + "0": 308, + "1": 482 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 68 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "2.png", + "image" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 67 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 99, + "type": "LoadWanFunModel", + "pos": { + "0": 290, + "1": -334 + }, + "size": { + "0": 315, + "1": 154 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 65 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-1.3B-InP", + "Inpaint", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 101, + "type": "WanFunInpaintSampler", + "pos": { + "0": 771, + "1": -67 + }, + "size": { + "0": 428.4000244140625, + "1": 378 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 65 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 66 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 67 + }, + { + "name": "start_img", + "type": "IMAGE", + "link": 68, + "shape": 7 + }, + { + "name": "end_img", + "type": "IMAGE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 64 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanFunInpaintSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "randomize", + 50, + 6, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 64, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 65, + 99, + 0, + 101, + 0, + "FunModels" + ], + [ + 66, + 75, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 67, + 73, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 68, + 87, + 0, + 101, + 3, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Load Image", + "bounding": [ + 222, + 396, + 458, + 436 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 227, + -416, + 469, + 256 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 73.82229219905072, + 565.0461564576733 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "10076a3aee6547541ca9442bf22389bfc2ea7526", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "v0.2.7-3-g8afb97c" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_i2v_lora.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_i2v_lora.json new file mode 100644 index 0000000000000000000000000000000000000000..667a6137fbf9e1e63566e058c78ee30692c7fd13 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_i2v_lora.json @@ -0,0 +1,545 @@ +{ + "last_node_id": 103, + "last_link_id": 70, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 66 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "Note", + "pos": { + "0": 17, + "1": -35 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": { + "0": 35.71394348144531, + "1": 536.9210205078125 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1271, + "1": -67 + }, + "size": [ + 390, + 546 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 64, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00021.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 80, + "type": "Note", + "pos": { + "0": -95.01139068603516, + "1": -334.30706787109375 + }, + "size": { + "0": 355.636474609375, + "1": 132.4238739013672 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 87, + "type": "LoadImage", + "pos": { + "0": 308, + "1": 482 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 68 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "2.png", + "image" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 67 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 99, + "type": "LoadWanFunModel", + "pos": { + "0": 290, + "1": -334 + }, + "size": { + "0": 315, + "1": 154 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 69 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-1.3B-InP", + "Inpaint", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 103, + "type": "LoadWanFunLora", + "pos": { + "0": 659, + "1": -335 + }, + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 69 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 70 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunLora" + }, + "widgets_values": [ + "cogvideox_fun/Wan2.1-Fun-1.3B-InP_MovieGen_tail_10_hpsv2.1_max_grad_norm0.3_lr1e-5_latent1_checkpoint-1000.safetensors", + 0.35, + false + ] + }, + { + "id": 101, + "type": "WanFunInpaintSampler", + "pos": { + "0": 771, + "1": -67 + }, + "size": { + "0": 428.4000244140625, + "1": 378 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 70 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 66 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 67 + }, + { + "name": "start_img", + "type": "IMAGE", + "link": 68, + "shape": 7 + }, + { + "name": "end_img", + "type": "IMAGE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 64 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "WanFunInpaintSampler" + }, + "widgets_values": [ + 81, + 640, + 844054136754572, + "randomize", + 50, + 6, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 64, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 66, + 75, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 67, + 73, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 68, + 87, + 0, + 101, + 3, + "IMAGE" + ], + [ + 69, + 99, + 0, + 103, + 0, + "FunModels" + ], + [ + 70, + 103, + 0, + 101, + 0, + "FunModels" + ] + ], + "groups": [ + { + "title": "Load Image", + "bounding": [ + 222, + 396, + 458, + 436 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 227, + -416, + 822, + 262 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 73.82229219905072, + 564.8949064576733 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "10076a3aee6547541ca9442bf22389bfc2ea7526", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "v0.2.7-3-g8afb97c" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_t2v.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..16539a585a5520ade3381191cf3e155c8dff22cf --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_t2v.json @@ -0,0 +1,379 @@ +{ + "last_node_id": 87, + "last_link_id": 58, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 57 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1185, + "1": -97 + }, + "size": [ + 390.9534912109375, + 538.0116295447716 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 55, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 87, + "type": "WanFunT2VSampler", + "pos": { + "0": 738, + "1": -97 + }, + "size": { + "0": 415.8000183105469, + "1": 386 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 56 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 57 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 58 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 55 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunT2VSampler" + }, + "widgets_values": [ + 81, + 832, + 480, + false, + 43, + "randomize", + 50, + 6, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 86, + "type": "LoadWanFunModel", + "pos": { + "0": 291, + "1": -317.7618408203125 + }, + "size": { + "0": 315, + "1": 154 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 56 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-1.3B-InP", + "Inpaint", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 80, + "type": "Note", + "pos": { + "0": -134, + "1": -324.7618408203125 + }, + "size": { + "0": 397.7547302246094, + "1": 125.90532684326172 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 58 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + } + ], + "links": [ + [ + 55, + 87, + 0, + 17, + 0, + "IMAGE" + ], + [ + 56, + 86, + 0, + 87, + 0, + "FunModels" + ], + [ + 57, + 75, + 0, + 87, + 1, + "STRING_PROMPT" + ], + [ + 58, + 73, + 0, + 87, + 2, + "STRING_PROMPT" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 220, + -401, + 469, + 251 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 137.06842501155074, + 523.7785392701735 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "10076a3aee6547541ca9442bf22389bfc2ea7526", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_t2v_lora.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_t2v_lora.json new file mode 100644 index 0000000000000000000000000000000000000000..7777955dc9fc4c0933b5ecc21c19a6d098b1ff59 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_t2v_lora.json @@ -0,0 +1,427 @@ +{ + "last_node_id": 89, + "last_link_id": 60, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 57 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1185, + "1": -97 + }, + "size": [ + 390.9534912109375, + 538.0116295447716 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 55, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00017.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 86, + "type": "LoadWanFunModel", + "pos": { + "0": 299.31610107421875, + "1": -334.70806884765625 + }, + "size": { + "0": 315, + "1": 154 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 59 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-1.3B-InP", + "Inpaint", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 58 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 80, + "type": "Note", + "pos": { + "0": -125.68385314941406, + "1": -341.70806884765625 + }, + "size": [ + 413.06488553532427, + 159.48434647201407 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "LoadWanFunLora", + "pos": { + "0": 646, + "1": -334 + }, + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 59 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 60 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunLora" + }, + "widgets_values": [ + "cogvideox_fun/Wan2.1-Fun-1.3B-InP_MovieGen_tail_10_hpsv2.1_max_grad_norm0.3_lr1e-5_latent1_checkpoint-1000.safetensors", + 0.55, + false + ] + }, + { + "id": 87, + "type": "WanFunT2VSampler", + "pos": { + "0": 738, + "1": -97 + }, + "size": { + "0": 415.8000183105469, + "1": 386 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 60 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 57 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 58 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 55 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunT2VSampler" + }, + "widgets_values": [ + 81, + 832, + 480, + false, + 637063819662010, + "randomize", + 50, + 6, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 55, + 87, + 0, + 17, + 0, + "IMAGE" + ], + [ + 57, + 75, + 0, + 87, + 1, + "STRING_PROMPT" + ], + [ + 58, + 73, + 0, + 87, + 2, + "STRING_PROMPT" + ], + [ + 59, + 86, + 0, + 89, + 0, + "FunModels" + ], + [ + 60, + 89, + 0, + 87, + 0, + "FunModels" + ] + ], + "groups": [ + { + "title": "Load Model", + "bounding": [ + 229, + -418, + 747, + 259 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 137.21967501155075, + 523.9297892701735 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "10076a3aee6547541ca9442bf22389bfc2ea7526", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control.json new file mode 100644 index 0000000000000000000000000000000000000000..8ecdc78c189337c33331bcc70a3ceace013173d5 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control.json @@ -0,0 +1,569 @@ +{ + "last_node_id": 97, + "last_link_id": 65, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": 15.739953994750977, + "1": 462.38665771484375 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": { + "0": -99, + "1": 197 + }, + "size": { + "0": 326.1556091308594, + "1": 145.20904541015625 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": { + "0": -192, + "1": -293 + }, + "size": { + "0": 427.074951171875, + "1": 143.9142608642578 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": { + "0": 335, + "1": 476 + }, + "size": [ + 252.056640625, + 688.5451388888889 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 64 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 92, + "type": "FunTextBox", + "pos": { + "0": 254, + "1": -46 + }, + "size": { + "0": 380.845703125, + "1": 157.68350219726562 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 62 + ], + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + ] + }, + { + "id": 94, + "type": "FunTextBox", + "pos": { + "0": 258, + "1": 178 + }, + "size": { + "0": 368.5529479980469, + "1": 159.4075927734375 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 63 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1274, + "1": 10 + }, + "size": [ + 390.9534912109375, + 966.9860514322917 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 65, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00004.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 97, + "type": "WanFunV2VSampler", + "pos": { + "0": 792, + "1": 11 + }, + "size": [ + 428.4000244140625, + 486 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 61 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 62 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 63 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": 64, + "shape": 7 + }, + { + "name": "start_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "riflex_k", + "type": "RIFLEXT_ARGS", + "link": null, + "shape": 7 + }, + { + "name": "camera_conditions", + "type": "STRING", + "link": null, + "widget": { + "name": "camera_conditions" + }, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 65 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6, + 1.0, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 95, + "type": "LoadWanFunModel", + "pos": { + "0": 275, + "1": -309 + }, + "size": { + "0": 435.2322082519531, + "1": 154 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 61 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control", + "Control", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + } + ], + "links": [ + [ + 61, + 95, + 0, + 97, + 0, + "FunModels" + ], + [ + 62, + 92, + 0, + 97, + 1, + "STRING_PROMPT" + ], + [ + 63, + 94, + 0, + 97, + 2, + "STRING_PROMPT" + ], + [ + 64, + 85, + 0, + 97, + 4, + "IMAGE" + ], + [ + 65, + 97, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 487, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 249.86519634240904, + 438.3739973625246 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "CogVideoX-Fun": "717f0629175ad192927dc51ec95c4376816a4212" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_canny.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_canny.json new file mode 100644 index 0000000000000000000000000000000000000000..14ceb34bc6e7d0f2e23e5031d995a41787085775 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_canny.json @@ -0,0 +1,699 @@ +{ + "last_node_id": 100, + "last_link_id": 71, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": 15.739953994750977, + "1": 462.38665771484375 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": { + "0": -99, + "1": 197 + }, + "size": { + "0": 326.1556091308594, + "1": 145.20904541015625 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": { + "0": -192, + "1": -293 + }, + "size": { + "0": 427.074951171875, + "1": 143.9142608642578 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 92, + "type": "FunTextBox", + "pos": { + "0": 254, + "1": -46 + }, + "size": { + "0": 380.845703125, + "1": 157.68350219726562 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 68 + ], + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + ] + }, + { + "id": 94, + "type": "FunTextBox", + "pos": { + "0": 258, + "1": 178 + }, + "size": { + "0": 368.5529479980469, + "1": 159.4075927734375 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 69 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": { + "0": 335, + "1": 476 + }, + "size": [ + 252.056640625, + 262 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 66 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "00339.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "00339.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 97, + "type": "VideoToCanny", + "pos": { + "0": 729, + "1": 566 + }, + "size": { + "0": 315, + "1": 106 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "input_video", + "type": "IMAGE", + "link": 66 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 65, + 70 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VideoToCanny" + }, + "widgets_values": [ + 100, + 200, + 81 + ] + }, + { + "id": 99, + "type": "VHS_VideoCombine", + "pos": { + "0": 1094, + "1": 559 + }, + "size": { + "0": 315, + "1": 310 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 65, + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Preprocess-Video", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00001.gif", + "subfolder": "", + "type": "output", + "format": "image/gif", + "frame_rate": 8 + } + } + } + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1467, + "1": -158 + }, + "size": [ + 390.9534912109375, + 966.9860514322917 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 71, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00004.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 100, + "type": "WanFunV2VSampler", + "pos": { + "0": 973, + "1": -160 + }, + "size": [ + 428.4000244140625, + 486 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 67 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 68 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 69 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": 70, + "shape": 7 + }, + { + "name": "start_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "riflex_k", + "type": "RIFLEXT_ARGS", + "link": null, + "shape": 7 + }, + { + "name": "camera_conditions", + "type": "STRING", + "link": null, + "widget": { + "name": "camera_conditions" + }, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 71 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6, + 1.0, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 95, + "type": "LoadWanFunModel", + "pos": { + "0": 275, + "1": -309 + }, + "size": { + "0": 435.2322082519531, + "1": 154 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 67 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control", + "Control", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + } + ], + "links": [ + [ + 65, + 97, + 0, + 99, + 0, + "IMAGE" + ], + [ + 66, + 85, + 0, + 97, + 0, + "IMAGE" + ], + [ + 67, + 95, + 0, + 100, + 0, + "FunModels" + ], + [ + 68, + 92, + 0, + 100, + 1, + "STRING_PROMPT" + ], + [ + 69, + 94, + 0, + 100, + 2, + "STRING_PROMPT" + ], + [ + 70, + 97, + 0, + 100, + 4, + "IMAGE" + ], + [ + 71, + 100, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 487, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 249.86519634240904, + 438.3739973625246 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "717f0629175ad192927dc51ec95c4376816a4212", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_depth.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_depth.json new file mode 100644 index 0000000000000000000000000000000000000000..c66bcdc7295727e8194a206bfc67ccaf9cd1f8cf --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_depth.json @@ -0,0 +1,697 @@ +{ + "last_node_id": 101, + "last_link_id": 75, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": 15.739953994750977, + "1": 462.38665771484375 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": { + "0": -99, + "1": 197 + }, + "size": { + "0": 326.1556091308594, + "1": 145.20904541015625 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": { + "0": -192, + "1": -293 + }, + "size": { + "0": 427.074951171875, + "1": 143.9142608642578 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 92, + "type": "FunTextBox", + "pos": { + "0": 254, + "1": -46 + }, + "size": { + "0": 380.845703125, + "1": 157.68350219726562 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 72 + ], + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + ] + }, + { + "id": 94, + "type": "FunTextBox", + "pos": { + "0": 258, + "1": 178 + }, + "size": { + "0": 368.5529479980469, + "1": 159.4075927734375 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 73 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": { + "0": 335, + "1": 476 + }, + "size": [ + 252.056640625, + 262 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 70 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "00339.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "00339.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 100, + "type": "VideoToDepth", + "pos": { + "0": 725, + "1": 477 + }, + "size": { + "0": 229.20001220703125, + "1": 58 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "input_video", + "type": "IMAGE", + "link": 70 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 68, + 74 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VideoToDepth" + }, + "widgets_values": [ + 81 + ] + }, + { + "id": 99, + "type": "VHS_VideoCombine", + "pos": { + "0": 996, + "1": 505 + }, + "size": { + "0": 315, + "1": 310 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 68, + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Preprocess-Video", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun-Preprocess-Video_00001.gif", + "subfolder": "", + "type": "output", + "format": "image/gif", + "frame_rate": 8 + } + } + } + }, + { + "id": 101, + "type": "WanFunV2VSampler", + "pos": { + "0": 989, + "1": -180 + }, + "size": [ + 428.4000244140625, + 486 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 71 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 72 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 73 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": 74, + "shape": 7 + }, + { + "name": "start_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "riflex_k", + "type": "RIFLEXT_ARGS", + "link": null, + "shape": 7 + }, + { + "name": "camera_conditions", + "type": "STRING", + "link": null, + "widget": { + "name": "camera_conditions" + }, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6, + 1.0, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 95, + "type": "LoadWanFunModel", + "pos": { + "0": 275, + "1": -309 + }, + "size": { + "0": 435.2322082519531, + "1": 154 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 71 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control", + "Control", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1459, + "1": -180 + }, + "size": [ + 390.9534912109375, + 966.9860514322917 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 75, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00039.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + } + ], + "links": [ + [ + 68, + 100, + 0, + 99, + 0, + "IMAGE" + ], + [ + 70, + 85, + 0, + 100, + 0, + "IMAGE" + ], + [ + 71, + 95, + 0, + 101, + 0, + "FunModels" + ], + [ + 72, + 92, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 73, + 94, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 74, + 100, + 0, + 101, + 4, + "IMAGE" + ], + [ + 75, + 101, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 487, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 249.86519634240904, + 438.3739973625246 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "717f0629175ad192927dc51ec95c4376816a4212", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_depth_ref.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_depth_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..0e850833cbafe61bf120e6e3ace547bceb853c6b --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_depth_ref.json @@ -0,0 +1,743 @@ +{ + "last_node_id": 104, + "last_link_id": 85, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": -111.46612548828125, + "1": 460.2178955078125 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": { + "0": -99, + "1": 197 + }, + "size": { + "0": 326.1556091308594, + "1": 145.20904541015625 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": { + "0": -192, + "1": -293 + }, + "size": { + "0": 427.074951171875, + "1": 143.9142608642578 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": { + "0": 258, + "1": 178 + }, + "size": { + "0": 368.5529479980469, + "1": 159.4075927734375 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 82 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1488, + "1": 8 + }, + "size": [ + 390.9534912109375, + 942.2558186848959 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 85, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00050.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 99, + "type": "VHS_VideoCombine", + "pos": { + "0": 1005, + "1": 554 + }, + "size": [ + 315, + 849.46875 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 79, + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Preprocess-Video", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun-Preprocess-Video_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 102, + "type": "LoadImage", + "pos": { + "0": 553, + "1": 598 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 84 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": { + "0": 254, + "1": -46 + }, + "size": { + "0": 380.845703125, + "1": 157.68350219726562 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 81 + ], + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位动漫风格的女孩。她有着紫色的短发，头上戴着一个黑色和金色相间的蝴蝶结。她的表情显得有些严肃或沉思，眼睛大而有神。女孩穿着一件白色衬衫，外面搭配了一件深蓝色的背心，背心上有一个粉色的蝴蝶结装饰。她的裙子是白色的，裙摆蓬松，整体造型非常可爱且精致。背景是一个简单的圆形图案，颜色为粉红色和灰色相间，给人一种柔和的感觉。整个画面色调柔和，人物形象生动鲜明。" + ] + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": { + "0": 207.79391479492188, + "1": 473.83123779296875 + }, + "size": [ + 252.056640625, + 688.5451388888889 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 76 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "000007.mp4", + "force_rate": 16, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 16, + "filename": "000007.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 103, + "type": "VideoToDepth", + "pos": { + "0": 553, + "1": 472 + }, + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "input_video", + "type": "IMAGE", + "link": 76 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 79, + 83 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VideoToDepth" + }, + "widgets_values": [ + 81 + ] + }, + { + "id": 104, + "type": "WanFunV2VSampler", + "pos": { + "0": 996, + "1": 9 + }, + "size": [ + 428.4000244140625, + 486 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 80 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 81 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 82 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": 83, + "shape": 7 + }, + { + "name": "start_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 84, + "shape": 7 + }, + { + "name": "riflex_k", + "type": "RIFLEXT_ARGS", + "link": null, + "shape": 7 + }, + { + "name": "camera_conditions", + "type": "STRING", + "link": null, + "widget": { + "name": "camera_conditions" + }, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 85 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 490544951141590, + "fixed", + 50, + 6, + 1.0, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 95, + "type": "LoadWanFunModel", + "pos": { + "0": 275, + "1": -309 + }, + "size": { + "0": 435.2322082519531, + "1": 154 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 80 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control", + "Control", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + } + ], + "links": [ + [ + 76, + 85, + 0, + 103, + 0, + "IMAGE" + ], + [ + 79, + 103, + 0, + 99, + 0, + "IMAGE" + ], + [ + 80, + 95, + 0, + 104, + 0, + "FunModels" + ], + [ + 81, + 92, + 0, + 104, + 1, + "STRING_PROMPT" + ], + [ + 82, + 94, + 0, + 104, + 2, + "STRING_PROMPT" + ], + [ + 83, + 103, + 0, + 104, + 4, + "IMAGE" + ], + [ + 84, + 102, + 0, + 104, + 6, + "IMAGE" + ], + [ + 85, + 104, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Upload Your Video And Reference Image", + "bounding": [ + 91, + 383, + 859, + 841 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 246.99418774865902, + 428.7658411125245 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "a7fa7028d52498f13e983eba012a81ebcae24977", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "v0.2.7-3-g8afb97c" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_pose.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_pose.json new file mode 100644 index 0000000000000000000000000000000000000000..5a922b9cb867d231d72161b89a8458872d3766f4 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_pose.json @@ -0,0 +1,697 @@ +{ + "last_node_id": 102, + "last_link_id": 79, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": 15.739953994750977, + "1": 462.38665771484375 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": { + "0": -99, + "1": 197 + }, + "size": { + "0": 326.1556091308594, + "1": 145.20904541015625 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": { + "0": -192, + "1": -293 + }, + "size": { + "0": 427.074951171875, + "1": 143.9142608642578 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 92, + "type": "FunTextBox", + "pos": { + "0": 254, + "1": -46 + }, + "size": { + "0": 380.845703125, + "1": 157.68350219726562 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 76 + ], + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + ] + }, + { + "id": 94, + "type": "FunTextBox", + "pos": { + "0": 258, + "1": 178 + }, + "size": { + "0": 368.5529479980469, + "1": 159.4075927734375 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 77 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": { + "0": 335, + "1": 476 + }, + "size": [ + 252.056640625, + 262 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 72 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "00339.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "00339.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 101, + "type": "VideoToOpenpose", + "pos": { + "0": 719, + "1": 542 + }, + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "input_video", + "type": "IMAGE", + "link": 72 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 73, + 78 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VideoToOpenpose" + }, + "widgets_values": [ + 81 + ] + }, + { + "id": 99, + "type": "VHS_VideoCombine", + "pos": { + "0": 1067, + "1": 540 + }, + "size": [ + 315, + 854.078125 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 73, + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Preprocess-Video", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun-Preprocess-Video_00003.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1432, + "1": -135 + }, + "size": [ + 390.9534912109375, + 966.9860514322917 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 79, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00041.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 102, + "type": "WanFunV2VSampler", + "pos": { + "0": 890, + "1": -136 + }, + "size": [ + 428.4000244140625, + 486 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 75 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": 78, + "shape": 7 + }, + { + "name": "start_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "riflex_k", + "type": "RIFLEXT_ARGS", + "link": null, + "shape": 7 + }, + { + "name": "camera_conditions", + "type": "STRING", + "link": null, + "widget": { + "name": "camera_conditions" + }, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 79 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6, + 1.0, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 95, + "type": "LoadWanFunModel", + "pos": { + "0": 275, + "1": -309 + }, + "size": { + "0": 435.2322082519531, + "1": 154 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 75 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control", + "Control", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + } + ], + "links": [ + [ + 72, + 85, + 0, + 101, + 0, + "IMAGE" + ], + [ + 73, + 101, + 0, + 99, + 0, + "IMAGE" + ], + [ + 75, + 95, + 0, + 102, + 0, + "FunModels" + ], + [ + 76, + 92, + 0, + 102, + 1, + "STRING_PROMPT" + ], + [ + 77, + 94, + 0, + 102, + 2, + "STRING_PROMPT" + ], + [ + 78, + 101, + 0, + 102, + 4, + "IMAGE" + ], + [ + 79, + 102, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 487, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 249.86519634240904, + 438.3739973625246 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "717f0629175ad192927dc51ec95c4376816a4212", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_pose_ref.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_pose_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..8d5e1c36e6044e5def7b5a0f3b8b0301e33b8f7c --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_pose_ref.json @@ -0,0 +1,743 @@ +{ + "last_node_id": 103, + "last_link_id": 81, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": -111.46612548828125, + "1": 460.2178955078125 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": { + "0": -99, + "1": 197 + }, + "size": { + "0": 326.1556091308594, + "1": 145.20904541015625 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": { + "0": -192, + "1": -293 + }, + "size": { + "0": 427.074951171875, + "1": 143.9142608642578 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": { + "0": 258, + "1": 178 + }, + "size": { + "0": 368.5529479980469, + "1": 159.4075927734375 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 78 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1488, + "1": 8 + }, + "size": [ + 390.9534912109375, + 942.2558186848959 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 81, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00049.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 99, + "type": "VHS_VideoCombine", + "pos": { + "0": 1005, + "1": 554 + }, + "size": [ + 315, + 849.46875 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 73, + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Preprocess-Video", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun-Preprocess-Video_00007.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 101, + "type": "VideoToOpenpose", + "pos": { + "0": 558, + "1": 474 + }, + "size": { + "0": 315, + "1": 58 + }, + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "input_video", + "type": "IMAGE", + "link": 72 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 73, + 79 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "VideoToOpenpose" + }, + "widgets_values": [ + 81 + ] + }, + { + "id": 102, + "type": "LoadImage", + "pos": { + "0": 553, + "1": 598 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 80 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 95, + "type": "LoadWanFunModel", + "pos": { + "0": 275, + "1": -309 + }, + "size": { + "0": 435.2322082519531, + "1": 154 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 76 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun1.1-1.3B-Control", + "Control", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": { + "0": 254, + "1": -46 + }, + "size": { + "0": 380.845703125, + "1": 157.68350219726562 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 77 + ], + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位动漫风格的女孩。她有着紫色的短发，头上戴着一个黑色和金色相间的蝴蝶结。她的表情显得有些严肃或沉思，眼睛大而有神。女孩穿着一件白色衬衫，外面搭配了一件深蓝色的背心，背心上有一个粉色的蝴蝶结装饰。她的裙子是白色的，裙摆蓬松，整体造型非常可爱且精致。背景是一个简单的圆形图案，颜色为粉红色和灰色相间，给人一种柔和的感觉。整个画面色调柔和，人物形象生动鲜明。" + ] + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": { + "0": 207.79391479492188, + "1": 473.83123779296875 + }, + "size": [ + 252.056640625, + 688.5451388888889 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 72 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "000007.mp4", + "force_rate": 16, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 16, + "filename": "000007.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 103, + "type": "WanFunV2VSampler", + "pos": { + "0": 996, + "1": 9 + }, + "size": [ + 428.4000244140625, + 486 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 76 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 78 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": 79, + "shape": 7 + }, + { + "name": "start_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 80, + "shape": 7 + }, + { + "name": "riflex_k", + "type": "RIFLEXT_ARGS", + "link": null, + "shape": 7 + }, + { + "name": "camera_conditions", + "type": "STRING", + "link": null, + "widget": { + "name": "camera_conditions" + }, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6, + 1.0, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 72, + 85, + 0, + 101, + 0, + "IMAGE" + ], + [ + 73, + 101, + 0, + 99, + 0, + "IMAGE" + ], + [ + 76, + 95, + 0, + 103, + 0, + "FunModels" + ], + [ + 77, + 92, + 0, + 103, + 1, + "STRING_PROMPT" + ], + [ + 78, + 94, + 0, + 103, + 2, + "STRING_PROMPT" + ], + [ + 79, + 101, + 0, + 103, + 4, + "IMAGE" + ], + [ + 80, + 102, + 0, + 103, + 6, + "IMAGE" + ], + [ + 81, + 103, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Upload Your Video And Reference Image", + "bounding": [ + 91, + 383, + 859, + 841 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 246.99418774865902, + 428.7658411125245 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "717f0629175ad192927dc51ec95c4376816a4212", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "v0.2.7-3-g8afb97c" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_ref.json b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..4b2015742c8531b87ceb73d0862890b4a77ed739 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_1_fun/v1/wan2.1_fun_workflow_v2v_control_ref.json @@ -0,0 +1,615 @@ +{ + "last_node_id": 98, + "last_link_id": 67, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": 15.145149230957031, + "1": 527.0673217773438 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": { + "0": -99, + "1": 197 + }, + "size": { + "0": 326.1556091308594, + "1": 145.20904541015625 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": { + "0": -192, + "1": -293 + }, + "size": { + "0": 427.074951171875, + "1": 143.9142608642578 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 1.3B model, you can set GPU_memory_mode to model_cpu_offload for faster generation. When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用1.3B模型时，可以设置GPU_memory_mode为model_cpu_offload进行更快速度的生成，在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": { + "0": 334.4051513671875, + "1": 540.6806640625 + }, + "size": [ + 252.056640625, + 688.5451388888889 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 65 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 94, + "type": "FunTextBox", + "pos": { + "0": 258, + "1": 178 + }, + "size": { + "0": 368.5529479980469, + "1": 159.4075927734375 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 64 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1274, + "1": -77 + }, + "size": [ + 390.9534912109375, + 942.2558186848959 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 67, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00051.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 92, + "type": "FunTextBox", + "pos": { + "0": 254, + "1": -46 + }, + "size": { + "0": 380.845703125, + "1": 157.68350219726562 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 63 + ], + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位动漫风格的女孩。她有着紫色的短发，头上戴着一个黑色和金色相间的蝴蝶结。她的表情显得有些严肃或沉思，眼睛大而有神。女孩穿着一件白色衬衫，外面搭配了一件深蓝色的背心，背心上有一个粉色的蝴蝶结装饰。她的裙子是白色的，裙摆蓬松，整体造型非常可爱且精致。背景是一个简单的圆形图案，颜色为粉红色和灰色相间，给人一种柔和的感觉。整个画面色调柔和，人物形象生动鲜明。" + ] + }, + { + "id": 97, + "type": "LoadImage", + "pos": { + "0": 629, + "1": 553 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 66 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 95, + "type": "LoadWanFunModel", + "pos": { + "0": 275, + "1": -309 + }, + "size": { + "0": 435.2322082519531, + "1": 154 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 62 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWanFunModel" + }, + "widgets_values": [ + "Wan2.1-Fun-V1.1-1.3B-Control", + "Control", + "model_cpu_offload", + "wan2.1/wan_civitai.yaml", + "bf16" + ] + }, + { + "id": 98, + "type": "WanFunV2VSampler", + "pos": { + "0": 798, + "1": -77 + }, + "size": [ + 428.4000244140625, + 486 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 62 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 63 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 64 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": 65, + "shape": 7 + }, + { + "name": "start_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 66, + "shape": 7 + }, + { + "name": "riflex_k", + "type": "RIFLEXT_ARGS", + "link": null, + "shape": 7 + }, + { + "name": "camera_conditions", + "type": "STRING", + "link": null, + "widget": { + "name": "camera_conditions" + }, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "WanFunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6, + 1.0, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 62, + 95, + 0, + 98, + 0, + "FunModels" + ], + [ + 63, + 92, + 0, + 98, + 1, + "STRING_PROMPT" + ], + [ + 64, + 94, + 0, + 98, + 2, + "STRING_PROMPT" + ], + [ + 65, + 85, + 0, + 98, + 4, + "IMAGE" + ], + [ + 66, + 97, + 0, + 98, + 6, + "IMAGE" + ], + [ + 67, + 98, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Upload Your Video", + "bounding": [ + 217, + 450, + 761, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917358, + "offset": [ + 23.894387748659057, + 254.96144111252457 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "CogVideoX-Fun": "a7fa7028d52498f13e983eba012a81ebcae24977", + "comfy-core": "v0.2.7-3-g8afb97c" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2/nodes.py b/VideoX-Fun/comfyui/wan2_2/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..b8153cca9851f51ace77e36091c2e7719fefc5c8 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2/nodes.py @@ -0,0 +1,916 @@ +"""Modified from https://github.com/kijai/ComfyUI-EasyAnimateWrapper/blob/main/nodes.py +""" +import copy +import gc +import inspect +import json +import os + +import cv2 +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from einops import rearrange +from omegaconf import OmegaConf +from PIL import Image + +import comfy.model_management as mm +import folder_paths +from comfy.utils import ProgressBar, load_torch_file + +from ...videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + get_closest_ratio) +from ...videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) +from ...videox_fun.models.cache_utils import get_teacache_coefficients +from ...videox_fun.pipeline import (Wan2_2FunControlPipeline, + Wan2_2FunInpaintPipeline, + Wan2_2FunPipeline, Wan2_2I2VPipeline, + Wan2_2Pipeline, Wan2_2TI2VPipeline) +from ...videox_fun.ui.controller import all_cheduler_dict +from ...videox_fun.utils.fp8_optimization import ( + convert_model_weight_to_float8, convert_weight_dtype_wrapper, undo_convert_weight_dtype_wrapper, + replace_parameters_by_name) +from ...videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from ...videox_fun.utils.utils import (filter_kwargs, + get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid, get_autocast_dtype) +from ..wan2_1.nodes import get_wan_scheduler +from ..comfyui_utils import (eas_cache_dir, script_directory, + search_model_in_possible_folders, to_pil) + +# Used in lora cache +transformer_cpu_cache = {} +transformer_high_cpu_cache = {} +# lora path before +lora_path_before = "" +lora_high_path_before = "" + +class LoadWan2_2TransformerModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ( + folder_paths.get_filename_list("diffusion_models"), + {"default": "Wan2_1-T2V-1_3B_bf16.safetensors,"}, + ), + "precision": (["fp16", "bf16"], + {"default": "bf16"} + ), + }, + } + RETURN_TYPES = ("TransformerModel", "STRING") + RETURN_NAMES = ("transformer", "model_name") + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, precision): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + transformer = None + + model_path = folder_paths.get_full_path("diffusion_models", model_name) + transformer_state_dict = load_torch_file(model_path, safe_load=True) + + eps = 1e-6 + text_len = 512 + freq_dim = 256 + dim = transformer_state_dict["patch_embedding.weight"].shape[0] + hidden_size = dim + in_dim = transformer_state_dict["patch_embedding.weight"].shape[1] + in_channels = in_dim + ffn_dim = transformer_state_dict["blocks.0.ffn.0.bias"].shape[0] + + add_ref_conv = True if "ref_conv.weight" in transformer_state_dict else False + in_dim_ref_conv = transformer_state_dict["ref_conv.weight"].shape[1] if "ref_conv.weight" in transformer_state_dict else None + add_control_adapter = True if "control_adapter.conv.weight" in transformer_state_dict else False + in_dim_control_adapter = transformer_state_dict["control_adapter.conv.weight"].shape[1] if "control_adapter.conv.weight" in transformer_state_dict else None + + if dim == 5120: + num_heads = 40 + num_layers = 40 + out_dim = 16 + downscale_factor_control_adapter = 8 + if in_dim == out_dim * 2 + 4: + model_name_in_pipeline = "wan2.2-i2v-a14b" + elif in_dim == out_dim: + model_name_in_pipeline = "wan2.2-t2v-a14b" + else: + model_name_in_pipeline = "wan2.2-fun-a14b" + + elif dim == 3072: + num_heads = 24 + num_layers = 30 + out_dim = 48 + downscale_factor_control_adapter = 16 + if in_dim == out_dim: + model_name_in_pipeline = "wan2.2-ti2v-5b" + else: + model_name_in_pipeline = "wan2.2-fun-5b" + else: + num_heads = 12 + num_layers = 30 + out_dim = 16 + downscale_factor_control_adapter = 8 + model_name_in_pipeline = "wan2.2-fun" + + if in_dim != out_dim: + model_type = "i2v" + else: + model_type = "t2v" + + kwargs = dict( + dim = dim, + in_dim = in_dim, + eps = eps, + ffn_dim = ffn_dim, + freq_dim = freq_dim, + model_type = model_type, + num_heads = num_heads, + num_layers = num_layers, + out_dim = out_dim, + text_len = text_len, + in_channels = in_channels, + hidden_size = hidden_size, + add_control_adapter = add_control_adapter, + add_ref_conv = add_ref_conv, + in_dim_control_adapter = in_dim_control_adapter // downscale_factor_control_adapter // downscale_factor_control_adapter if in_dim_control_adapter is not None else in_dim_control_adapter, + in_dim_ref_conv = in_dim_ref_conv, + downscale_factor_control_adapter = downscale_factor_control_adapter, + ) + + sig = inspect.signature(Wan2_2Transformer3DModel) + accepted = {k: v for k, v in kwargs.items() if k in sig.parameters} + transformer = Wan2_2Transformer3DModel(**accepted) + transformer.load_state_dict(transformer_state_dict) + transformer = transformer.eval().to(device=offload_device, dtype=weight_dtype) + return (transformer, model_name_in_pipeline) + +class CombineWan2_2Pipeline: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "transformer": ("TransformerModel",), + "vae": ("VAEModel",), + "text_encoder": ("TextEncoderModel",), + "tokenizer": ("Tokenizer",), + "model_name": ("STRING",), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + "model_type": ( + ["Inpaint", "Control"], + { + "default": "Inpaint", + } + ), + }, + "optional":{ + "clip_encoder": ("ClipEncoderModel",), + "transformer_2": ("TransformerModel",), + }, + } + + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, GPU_memory_mode, model_type, transformer, vae, text_encoder, tokenizer, clip_encoder=None, transformer_2=None): + # Get pipeline + weight_dtype = transformer.dtype if transformer.dtype not in [torch.float32, torch.float8_e4m3fn, torch.float8_e5m2] else get_autocast_dtype() + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + # Get pipeline + if model_type == "Inpaint": + if "5b" in model_name: + pipeline = Wan2_2TI2VPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + transformer_2=transformer_2, + scheduler=None, + ) + else: + if transformer.config.in_channels != vae.config.latent_channels: + pipeline = Wan2_2FunInpaintPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + transformer_2=transformer_2, + scheduler=None, + ) + else: + pipeline = Wan2_2FunPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + transformer_2=transformer_2, + scheduler=None, + ) + else: + pipeline = Wan2_2FunControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + transformer_2=transformer_2, + scheduler=None, + ) + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + pipeline.to(device=offload_device) + transformer = transformer.to(weight_dtype) + + if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",]) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",]) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [] + } + return (funmodels,) + +class LoadWan2_2Model: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ( + [ + 'Wan2.2-T2V-A14B', + 'Wan2.2-I2V-A14B', + 'Wan2.2-TI2V-5B', + ], + { + "default": 'Wan2.2-T2V-A14B', + } + ), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + "config": ( + [ + "wan2.2/wan_civitai_t2v.yaml", + "wan2.2/wan_civitai_i2v.yaml", + "wan2.2/wan_civitai_5b.yaml", + ], + { + "default": "wan2.2/wan_civitai_t2v.yaml", + } + ), + "precision": ( + ['fp16', 'bf16'], + { + "default": 'fp16' + } + ), + }, + } + + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, GPU_memory_mode, model, precision, config): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + + # Init processbar + pbar = ProgressBar(5) + + # Load config + config_path = f"{script_directory}/config/{config}" + config = OmegaConf.load(config_path) + + # Detect model is existing or not + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun", "Wan-AI"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + # Initialize model_name as None + model_name = search_model_in_possible_folders(possible_folders, model) + + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ).to(weight_dtype) + # Update pbar + pbar.update(1) + + # Load Sampler + print("Load Sampler.") + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + # Update pbar + pbar.update(1) + + # Get Transformer + transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + else: + transformer_2 = None + # Update pbar + pbar.update(1) + + # Get tokenizer and text_encoder + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + pbar.update(1) + + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + pbar.update(1) + + # Get pipeline + model_type = "Inpaint" + if model_type == "Inpaint": + if "wan_civitai_5b" in config_path: + pipeline = Wan2_2TI2VPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + transformer_2=transformer_2, + scheduler=scheduler, + ) + else: + if transformer.config.in_channels != vae.config.latent_channels: + pipeline = Wan2_2I2VPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + ) + else: + pipeline = Wan2_2Pipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + ) + else: + raise ValueError(f"Model type {model_type} not supported") + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + + if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device=device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [], + 'config': config, + } + return (funmodels,) + +class LoadWan2_2Lora: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ("FunModels",), + "lora_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "lora_high_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), + "lora_cache":([False, True], {"default": False,}), + } + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "load_lora" + CATEGORY = "CogVideoXFUNWrapper" + + def load_lora(self, funmodels, lora_name, lora_high_name, strength_model, lora_cache): + new_funmodels = dict(funmodels) + if lora_name is not None: + loras = list(new_funmodels.get("loras", [])) + [folder_paths.get_full_path("loras", lora_name)] + loras_high = list(new_funmodels.get("loras_high", [])) + [folder_paths.get_full_path("loras", lora_high_name)] + strength_models = list(new_funmodels.get("strength_model", [])) + [strength_model] + new_funmodels['loras'] = loras + new_funmodels['loras_high'] = loras_high + new_funmodels['strength_model'] = strength_models + new_funmodels['lora_cache'] = lora_cache + return (new_funmodels,) + +class Wan2_2T2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 5, "max": 161, "step": 4} + ), + "width": ( + "INT", {"default": 832, "min": 64, "max": 2048, "step": 16} + ), + "height": ( + "INT", {"default": 480, "min": 64, "max": 2048, "step": 16} + ), + "is_image":( + [ + False, + True + ], + { + "default": False, + } + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + "boundary": ( + "FLOAT", {"default": 0.875, "min": 0.00, "max": 1.00, "step": 0.001} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional":{ + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, width, height, is_image, seed, steps, cfg, scheduler, shift, boundary, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, riflex_k=0): + global transformer_cpu_cache + global transformer_high_cpu_cache + global lora_path_before + global lora_high_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + else: + pipeline.transformer.disable_teacache() + if pipeline.transformer_2 is not None: + pipeline.transformer_2.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + + generator= torch.Generator(device).manual_seed(seed) + + video_length = 1 if is_image else video_length + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + if pipeline.transformer_2 is not None: + # Save the original weights to cpu + if len(transformer_high_cpu_cache) == 0: + print('Save transformer high state_dict to cpu memory') + transformer_high_state_dict = pipeline.transformer_2.state_dict() + for key in transformer_high_state_dict: + transformer_high_cpu_cache[key] = transformer_high_state_dict[key].clone().cpu() + + lora_high_path_now = str(funmodels.get("loras_high", []) + funmodels.get("strength_model", [])) + if lora_high_path_now != lora_high_path_before: + print('Merge Lora High with Cache') + lora_high_path_before = copy.deepcopy(lora_high_path_now) + pipeline.transformer_2.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + else: + print('Merge Lora') + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + # Clear lora when switch from lora_cache=True to lora_cache=False. + if pipeline.transformer_2 is not None: + if len(transformer_high_cpu_cache) != 0: + pipeline.transformer_2.load_state_dict(transformer_high_cpu_cache) + transformer_high_cpu_cache = {} + lora_high_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + boundary = boundary, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + if pipeline.transformer_2 is not None: + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + return (videos,) + + +class Wan2_2I2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 5, "max": 161, "step": 4} + ), + "base_resolution": ( + [ + 512, + 640, + 768, + 896, + 960, + 1024, + ], {"default": 640} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + "boundary": ( + "FLOAT", {"default": 0.90, "min": 0.00, "max": 1.00, "step": 0.001} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional":{ + "start_img": ("IMAGE",), + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, base_resolution, seed, steps, cfg, scheduler, shift, boundary, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, start_img=None, end_img=None, riflex_k=0): + global transformer_cpu_cache + global transformer_high_cpu_cache + global lora_path_before + global lora_high_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + + start_img = [to_pil(_start_img) for _start_img in start_img] if start_img is not None else None + end_img = [to_pil(_end_img) for _end_img in end_img] if end_img is not None else None + # Count most suitable height and width + spatial_compression_ratio = pipeline.vae.config.spatial_compression_ratio if hasattr(pipeline.vae.config, "spatial_compression_ratio") else 8 + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + original_width, original_height = start_img[0].size if type(start_img) is list else Image.open(start_img).size + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height, width = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in closest_size] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + else: + pipeline.transformer.disable_teacache() + if pipeline.transformer_2 is not None: + pipeline.transformer_2.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_img, end_img, video_length=video_length, sample_size=(height, width)) + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + if pipeline.transformer_2 is not None: + # Save the original weights to cpu + if len(transformer_high_cpu_cache) == 0: + print('Save transformer high state_dict to cpu memory') + transformer_high_state_dict = pipeline.transformer_2.state_dict() + for key in transformer_high_state_dict: + transformer_high_cpu_cache[key] = transformer_high_state_dict[key].clone().cpu() + + lora_high_path_now = str(funmodels.get("loras_high", []) + funmodels.get("strength_model", [])) + if lora_high_path_now != lora_high_path_before: + print('Merge Lora High with Cache') + lora_high_path_before = copy.deepcopy(lora_high_path_now) + pipeline.transformer_2.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + else: + print('Merge Lora') + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + # Clear lora when switch from lora_cache=True to lora_cache=False. + if pipeline.transformer_2 is not None: + if len(transformer_high_cpu_cache) != 0: + pipeline.transformer_2.load_state_dict(transformer_high_cpu_cache) + transformer_high_cpu_cache = {} + lora_high_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + boundary = boundary, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + if pipeline.transformer_2 is not None: + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + return (videos,) \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_i2v.json b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..03a6ecd70a02357e49bb8f278645c231ae7ddb0d --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_i2v.json @@ -0,0 +1,708 @@ +{ + "id": "ae5280b3-6ce0-4651-b35f-7876c3aa5506", + "revision": 0, + "last_node_id": 107, + "last_link_id": 82, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 79 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 87, + "type": "LoadImage", + "pos": [ + 306, + 495 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 81 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "6.png", + "image" + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": [ + 34, + 550 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 80 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1277, + -70 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 82 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 104, + "type": "LoadWanVAEModel", + "pos": [ + 232.4375762939453, + -422.52117919921875 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 72 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 105, + "type": "LoadWanTextEncoderModel", + "pos": [ + 232.4375762939453, + -271.95159912109375 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 73 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 74 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -163.70957946777344, + -415.55194091796875 + ], + "size": [ + 350.7127990722656, + 125.54820251464844 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 103, + "type": "CombineWan2_2Pipeline", + "pos": [ + 689.4248046875, + -376.8700866699219 + ], + "size": [ + 360.3475341796875, + 206 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 71 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 72 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 73 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 74 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 75 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 76 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2Pipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload", + "Inpaint" + ] + }, + { + "id": 106, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 236.3606719970703, + -581.4786376953125 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 71 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 76 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2_2-I2V-A14B-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 102, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 668.324951171875, + -579.4934692382812 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 75 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2_2-I2V-A14B-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 107, + "type": "Wan2_2I2VSampler", + "pos": [ + 819.5313110351562, + -63.450443267822266 + ], + "size": [ + 325.5747985839844, + 450 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 78 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 79 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 80 + }, + { + "name": "start_img", + "shape": 7, + "type": "IMAGE", + "link": 81 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 82 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2I2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "randomize", + 50, + 6, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 71, + 106, + 0, + 103, + 0, + "TransformerModel" + ], + [ + 72, + 104, + 0, + 103, + 1, + "VAEModel" + ], + [ + 73, + 105, + 0, + 103, + 2, + "TextEncoderModel" + ], + [ + 74, + 105, + 1, + 103, + 3, + "Tokenizer" + ], + [ + 75, + 102, + 0, + 103, + 5, + "TransformerModel" + ], + [ + 76, + 106, + 1, + 103, + 6, + "STRING" + ], + [ + 78, + 103, + 0, + 107, + 0, + "FunModels" + ], + [ + 79, + 75, + 0, + 107, + 1, + "STRING_PROMPT" + ], + [ + 80, + 73, + 0, + 107, + 2, + "STRING_PROMPT" + ], + [ + 81, + 87, + 0, + 107, + 3, + "IMAGE" + ], + [ + 82, + 107, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Group", + "bounding": [ + 220, + 409, + 458, + 436 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 222.43731689453125, + -661.4031372070312, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 482.3495838700222, + 753.3669707232627 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "comfy-core": "0.3.44", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_i2v_5b.json b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_i2v_5b.json new file mode 100644 index 0000000000000000000000000000000000000000..c26f97d0820f44ddd2cd48c3f944cea4af03c854 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_i2v_5b.json @@ -0,0 +1,663 @@ +{ + "id": "ca87b2cd-bd4a-4f31-82ec-e5e028f8848c", + "revision": 0, + "last_node_id": 115, + "last_link_id": 112, + "nodes": [ + { + "id": 87, + "type": "LoadImage", + "pos": [ + 306, + 495 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 111 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "6.png", + "image" + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": [ + 34, + 550 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 110 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1277, + -70 + ], + "size": [ + 390, + 581.3912963867188 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 112 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00129.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 109 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 108, + "type": "LoadWanTextEncoderModel", + "pos": [ + 233.58401489257812, + -288.2000732421875 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 90 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 91 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 109, + "type": "Note", + "pos": [ + -211.7540740966797, + -428.57659912109375 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 111, + "type": "CombineWan2_2Pipeline", + "pos": [ + 690.572021484375, + -393.1186828613281 + ], + "size": [ + 360.3475341796875, + 206 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 88 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 89 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 90 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 91 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 87 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 108 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2Pipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8", + "Inpaint" + ] + }, + { + "id": 110, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 237.50721740722656, + -597.7264404296875 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 88 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-TI2V-5B_transformer.safetensors", + "bf16" + ] + }, + { + "id": 107, + "type": "LoadWanVAEModel", + "pos": [ + 233.58401489257812, + -438.769775390625 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 89 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.2_VAE.pth", + "bf16" + ] + }, + { + "id": 115, + "type": "Wan2_2I2VSampler", + "pos": [ + 749.0029296875, + -64.3254623413086 + ], + "size": [ + 386.6094665527344, + 450 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 108 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 109 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 110 + }, + { + "name": "start_img", + "shape": 7, + "type": "IMAGE", + "link": 111 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 112 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2I2VSampler" + }, + "widgets_values": [ + 81, + 960, + 1124619359413896, + "randomize", + 50, + 6, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 87, + 110, + 1, + 111, + 6, + "STRING" + ], + [ + 88, + 110, + 0, + 111, + 0, + "TransformerModel" + ], + [ + 89, + 107, + 0, + 111, + 1, + "VAEModel" + ], + [ + 90, + 108, + 0, + 111, + 2, + "TextEncoderModel" + ], + [ + 91, + 108, + 1, + 111, + 3, + "Tokenizer" + ], + [ + 108, + 111, + 0, + 115, + 0, + "FunModels" + ], + [ + 109, + 75, + 0, + 115, + 1, + "STRING_PROMPT" + ], + [ + 110, + 73, + 0, + 115, + 2, + "STRING_PROMPT" + ], + [ + 111, + 87, + 0, + 115, + 3, + "IMAGE" + ], + [ + 112, + 115, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Group", + "bounding": [ + 220, + 409, + 458, + 436 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 223.58383178710938, + -677.6509399414062, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 312.3881627762727, + 655.3866535357635 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "comfy-core": "0.3.44", + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_t2v.json b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..825c5a2fa76f2dcb1115994b8445a64d14997ebe --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_t2v.json @@ -0,0 +1,620 @@ +{ + "id": "98825915-fc5a-4287-b75e-c1a44df1155d", + "revision": 0, + "last_node_id": 93, + "last_link_id": 70, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 68 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1126, + -73 + ], + "size": [ + 390.9534912109375, + 538.0116577148438 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 70 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 69 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 90, + "type": "LoadWanTextEncoderModel", + "pos": [ + 228.3156280517578, + -279.8008728027344 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 60 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 61 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -142.44093322753906, + -428.1786804199219 + ], + "size": [ + 351.1499938964844, + 130.12660217285156 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 232.23873901367188, + -589.3279418945312 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 64 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 62 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2_2-T2V-A14B-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 92, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 670.7969970703125, + -593.7826538085938 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 65 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2_2-T2V-A14B-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 91, + "type": "CombineWan2_2Pipeline", + "pos": [ + 685.3029174804688, + -384.71942138671875 + ], + "size": [ + 360.3475341796875, + 206 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 64 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 59 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 60 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 61 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 65 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 62 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2Pipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload", + "Inpaint" + ] + }, + { + "id": 93, + "type": "Wan2_2T2VSampler", + "pos": [ + 746.95556640625, + -69.9979248046875 + ], + "size": [ + 312.3404235839844, + 478 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 67 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 68 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 69 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 70 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2T2VSampler" + }, + "widgets_values": [ + 81, + 832, + 480, + false, + 141471358015136, + "randomize", + 50, + 6, + "Flow", + 5, + 0.875, + 0.10000000000000002, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 89, + "type": "LoadWanVAEModel", + "pos": [ + 228.3156280517578, + -430.37054443359375 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 59 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + } + ], + "links": [ + [ + 59, + 89, + 0, + 91, + 1, + "VAEModel" + ], + [ + 60, + 90, + 0, + 91, + 2, + "TextEncoderModel" + ], + [ + 61, + 90, + 1, + 91, + 3, + "Tokenizer" + ], + [ + 62, + 88, + 1, + 91, + 6, + "STRING" + ], + [ + 64, + 88, + 0, + 91, + 0, + "TransformerModel" + ], + [ + 65, + 92, + 0, + 91, + 5, + "TransformerModel" + ], + [ + 67, + 91, + 0, + 93, + 0, + "FunModels" + ], + [ + 68, + 75, + 0, + 93, + 1, + "STRING_PROMPT" + ], + [ + 69, + 73, + 0, + 93, + 2, + "STRING_PROMPT" + ], + [ + 70, + 93, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model", + "bounding": [ + 218.3153839111328, + -669.25244140625, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 304.1988281365522, + 657.9109814576749 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_t2v_5b.json b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_t2v_5b.json new file mode 100644 index 0000000000000000000000000000000000000000..4d38553e086dea7dd9d8536a419b5a950d2427ff --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_chunked_loading_workflow_t2v_5b.json @@ -0,0 +1,575 @@ +{ + "id": "ca87b2cd-bd4a-4f31-82ec-e5e028f8848c", + "revision": 0, + "last_node_id": 111, + "last_link_id": 98, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 97 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 96 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 106, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 239.5579376220703, + -604.4466552734375 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 85 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 89 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-TI2V-5B_transformer.safetensors", + "bf16" + ] + }, + { + "id": 107, + "type": "LoadWanVAEModel", + "pos": [ + 235.63478088378906, + -445.4897766113281 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 86 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.2_VAE.pth", + "bf16" + ] + }, + { + "id": 109, + "type": "LoadWanTextEncoderModel", + "pos": [ + 235.63478088378906, + -294.9201354980469 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 87 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 88 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -135.5485382080078, + -450.6790466308594 + ], + "size": [ + 350.7127990722656, + 125.54820251464844 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 108, + "type": "CombineWan2_2Pipeline", + "pos": [ + 692.6224975585938, + -399.8387145996094 + ], + "size": [ + 360.3475341796875, + 206 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 85 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 86 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 87 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 88 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": null + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 89 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 95 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2Pipeline" + }, + "widgets_values": [ + "", + "model_cpu_offload_and_qfloat8", + "Inpaint" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1243.8077392578125, + -87.18865966796875 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 98 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00130.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 111, + "type": "Wan2_2T2VSampler", + "pos": [ + 748.5845947265625, + -88.83666229248047 + ], + "size": [ + 379.8106994628906, + 478 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 95 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 96 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 97 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 98 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2T2VSampler" + }, + "widgets_values": [ + 81, + 1280, + 704, + false, + 926814246464214, + "randomize", + 50, + 6, + "Flow", + 5, + 0.875, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 85, + 106, + 0, + 108, + 0, + "TransformerModel" + ], + [ + 86, + 107, + 0, + 108, + 1, + "VAEModel" + ], + [ + 87, + 109, + 0, + 108, + 2, + "TextEncoderModel" + ], + [ + 88, + 109, + 1, + 108, + 3, + "Tokenizer" + ], + [ + 89, + 106, + 1, + 108, + 6, + "STRING" + ], + [ + 95, + 108, + 0, + 111, + 0, + "FunModels" + ], + [ + 96, + 75, + 0, + 111, + 1, + "STRING_PROMPT" + ], + [ + 97, + 73, + 0, + 111, + 2, + "STRING_PROMPT" + ], + [ + 98, + 111, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model", + "bounding": [ + 225.63455200195312, + -684.3711547851562, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 578.0775994950219, + 731.5692461138882 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_i2v.json b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..fdc874b90ddbd16364435ebbb7c727d980e97c43 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_i2v.json @@ -0,0 +1,475 @@ +{ + "id": "ca87b2cd-bd4a-4f31-82ec-e5e028f8848c", + "revision": 0, + "last_node_id": 101, + "last_link_id": 70, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 68 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 87, + "type": "LoadImage", + "pos": [ + 306, + 495 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 66 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "6.png", + "image" + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": [ + 34, + 550 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 69 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -75, + -297 + ], + "size": [ + 350.7127990722656, + 125.54820251464844 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1277, + -70 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 70 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 101, + "type": "Wan2_2I2VSampler", + "pos": [ + 819.5313110351562, + -63.450443267822266 + ], + "size": [ + 325.5747985839844, + 402 + ], + "flags": { + "pinned": true + }, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 67 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 68 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 69 + }, + { + "name": "start_img", + "shape": 7, + "type": "IMAGE", + "link": 66 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 70 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2I2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 43, + 6, + "Flow", + 5, + 0.900, + 0.15, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 99, + "type": "LoadWan2_2Model", + "pos": [ + 347.27996826171875, + -299.0150146484375 + ], + "size": [ + 276.705078125, + 130 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2Model" + }, + "widgets_values": [ + "Wan2.2-I2V-A14B", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + } + ], + "links": [ + [ + 66, + 87, + 0, + 101, + 3, + "IMAGE" + ], + [ + 67, + 99, + 0, + 101, + 0, + "FunModels" + ], + [ + 68, + 75, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 69, + 73, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 70, + 101, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Group", + "bounding": [ + 220, + 409, + 458, + 436 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 272.66873113564736, + 429.81230900451357 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "ffe64f2b8c101da85e0707c59b355f21a83f5464", + "comfy-core": "0.3.44", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_i2v_5b.json b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_i2v_5b.json new file mode 100644 index 0000000000000000000000000000000000000000..b711363d587e5216b5ed6927988f0f2a62f8615f --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_i2v_5b.json @@ -0,0 +1,473 @@ +{ + "id": "ca87b2cd-bd4a-4f31-82ec-e5e028f8848c", + "revision": 0, + "last_node_id": 103, + "last_link_id": 80, + "nodes": [ + { + "id": 87, + "type": "LoadImage", + "pos": [ + 306, + 495 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 79 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "6.png", + "image" + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": [ + 34, + 550 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 78 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -75, + -297 + ], + "size": [ + 350.7127990722656, + 125.54820251464844 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1277, + -70 + ], + "size": [ + 390, + 577.7142944335938 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 80 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00107.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 99, + "type": "LoadWan2_2Model", + "pos": [ + 347.27996826171875, + -299.0150146484375 + ], + "size": [ + 276.705078125, + 130 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 76 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2Model" + }, + "widgets_values": [ + "Wan2.2-TI2V-5B", + "sequential_cpu_offload", + "wan2.2/wan_civitai_5b.yaml", + "bf16" + ] + }, + { + "id": 103, + "type": "Wan2_2I2VSampler", + "pos": [ + 819.5313110351562, + -63.450443267822266 + ], + "size": [ + 325.5747985839844, + 402 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 76 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 78 + }, + { + "name": "start_img", + "shape": 7, + "type": "IMAGE", + "link": 79 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2I2VSampler" + }, + "widgets_values": [ + 81, + 960, + 43, + "fixed", + 50, + 6, + "Flow", + 5, + 0.900, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + } + ], + "links": [ + [ + 76, + 99, + 0, + 103, + 0, + "FunModels" + ], + [ + 77, + 75, + 0, + 103, + 1, + "STRING_PROMPT" + ], + [ + 78, + 73, + 0, + 103, + 2, + "STRING_PROMPT" + ], + [ + 79, + 87, + 0, + 103, + 3, + "IMAGE" + ], + [ + 80, + 103, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Group", + "bounding": [ + 220, + 409, + 458, + 436 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 282.91743113564746, + 433.6498523638886 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "comfy-core": "0.3.44", + "CogVideoX-Fun": "5f2a55692d0834e00d477939b818b9bb63c06535", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_t2v.json b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..fac3b56be813faf4cea1385a44684edc2d03e6f3 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_t2v.json @@ -0,0 +1,385 @@ +{ + "id": "f52daf1e-5f66-4c8b-9f68-890324faf01e", + "revision": 0, + "last_node_id": 87, + "last_link_id": 57, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 55 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1126, + -73 + ], + "size": [ + 390.9534912109375, + 538.0116577148438 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 54 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 56 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 87, + "type": "LoadWan2_2Model", + "pos": [ + 305.3617858886719, + -300.4237060546875 + ], + "size": [ + 276.705078125, + 130 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 57 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2Model" + }, + "widgets_values": [ + "Wan2.2-T2V-A14B", + "sequential_cpu_offload", + "wan2.2/wan_civitai_t2v.yaml", + "fp16" + ] + }, + { + "id": 86, + "type": "Wan2_2T2VSampler", + "pos": [ + 746.95556640625, + -69.9979248046875 + ], + "size": [ + 312.3404235839844, + 430 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 57 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 55 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 56 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 54 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2T2VSampler" + }, + "widgets_values": [ + 81, + 832, + 480, + false, + 43, + "fixed", + 50, + 6, + "Flow", + 12, + 0.875, + 0.15, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -92, + -294 + ], + "size": [ + 351.1499938964844, + 130.12660217285156 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 54, + 86, + 0, + 17, + 0, + "IMAGE" + ], + [ + 55, + 75, + 0, + 86, + 1, + "STRING_PROMPT" + ], + [ + 56, + 73, + 0, + 86, + 2, + "STRING_PROMPT" + ], + [ + 57, + 87, + 0, + 86, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.9090909090909091, + "offset": [ + 149.60461329280074, + 495.60652716079886 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "ffe64f2b8c101da85e0707c59b355f21a83f5464", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_t2v_5b.json b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_t2v_5b.json new file mode 100644 index 0000000000000000000000000000000000000000..78f57d5dc52d36f4cafabf943f20b138644c4799 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2/v1/wan2.2_workflow_t2v_5b.json @@ -0,0 +1,385 @@ +{ + "id": "ca87b2cd-bd4a-4f31-82ec-e5e028f8848c", + "revision": 0, + "last_node_id": 105, + "last_link_id": 84, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 83 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -75, + -297 + ], + "size": [ + 350.7127990722656, + 125.54820251464844 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 82 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 99, + "type": "LoadWan2_2Model", + "pos": [ + 347.27996826171875, + -299.0150146484375 + ], + "size": [ + 276.705078125, + 130 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2Model" + }, + "widgets_values": [ + "Wan2.2-TI2V-5B", + "sequential_cpu_offload", + "wan2.2/wan_civitai_5b.yaml", + "bf16" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1277, + -70 + ], + "size": [ + 390, + 577.7142944335938 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 84 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00108.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 105, + "type": "Wan2_2FunT2VSampler", + "pos": [ + 827.6821899414062, + -69.17436981201172 + ], + "size": [ + 340.3540954589844, + 430 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 81 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 82 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 83 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 84 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunT2VSampler" + }, + "widgets_values": [ + 81, + 1280, + 704, + false, + 43, + "fixed", + 50, + 6, + "Flow", + 5, + 0.900, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 81, + 99, + 0, + 105, + 0, + "FunModels" + ], + [ + 82, + 75, + 0, + 105, + 1, + "STRING_PROMPT" + ], + [ + 83, + 73, + 0, + 105, + 2, + "STRING_PROMPT" + ], + [ + 84, + 105, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 220, + -380, + 472, + 232 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 313.88657762002225, + 431.4308258013887 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "5f2a55692d0834e00d477939b818b9bb63c06535", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/nodes.py b/VideoX-Fun/comfyui/wan2_2_fun/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..02ab6fd1485ba9c1f93b47c264c0e8802a9817df --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/nodes.py @@ -0,0 +1,1001 @@ +"""Modified from https://github.com/kijai/ComfyUI-EasyAnimateWrapper/blob/main/nodes.py +""" +import copy +import gc +import json +import os + +import cv2 +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from einops import rearrange +from omegaconf import OmegaConf +from PIL import Image + +import comfy.model_management as mm +import folder_paths +from comfy.utils import ProgressBar, load_torch_file + +from ...videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + get_closest_ratio) +from ...videox_fun.data.dataset_image_video import process_pose_params +from ...videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel, WanT5EncoderModel) +from ...videox_fun.models.cache_utils import get_teacache_coefficients +from ...videox_fun.pipeline import (Wan2_2FunControlPipeline, + Wan2_2FunInpaintPipeline, + Wan2_2FunPipeline) +from ...videox_fun.ui.controller import all_cheduler_dict +from ...videox_fun.utils.fp8_optimization import ( + convert_model_weight_to_float8, convert_weight_dtype_wrapper, undo_convert_weight_dtype_wrapper, + replace_parameters_by_name) +from ...videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from ...videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) +from ..wan2_1.nodes import get_wan_scheduler +from ..comfyui_utils import (eas_cache_dir, script_directory, + search_model_in_possible_folders, to_pil) + + +# Used in lora cache +transformer_cpu_cache = {} +transformer_high_cpu_cache = {} +# lora path before +lora_path_before = "" +lora_high_path_before = "" + + +class LoadWan2_2FunModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ( + [ + 'Wan2.2-Fun-A14B-InP', + 'Wan2.2-Fun-A14B-Control', + 'Wan2.2-Fun-A14B-Control-Camera', + 'Wan2.2-Fun-5B-InP', + 'Wan2.2-Fun-5B-Control', + 'Wan2.2-Fun-5B-Control-Camera', + ], + { + "default": 'Wan2.2-Fun-A14B-InP', + } + ), + "model_type": ( + ["Inpaint", "Control"], + { + "default": "Inpaint", + } + ), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + "config": ( + [ + "wan2.2/wan_civitai_i2v.yaml", + "wan2.2/wan_civitai_5b.yaml", + ], + { + "default": "wan2.2/wan_civitai_i2v.yaml", + } + ), + "precision": ( + ['fp16', 'bf16'], + { + "default": 'fp16' + } + ), + }, + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, GPU_memory_mode, model_type, model, precision, config): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + + # Init processbar + pbar = ProgressBar(5) + + # Load config + config_path = f"{script_directory}/config/{config}" + config = OmegaConf.load(config_path) + + # Detect model is existing or not + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun", "Wan-AI"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + # Initialize model_name as None + model_name = search_model_in_possible_folders(possible_folders, model) + + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ).to(weight_dtype) + # Update pbar + pbar.update(1) + + # Load Sampler + print("Load Sampler.") + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + # Update pbar + pbar.update(1) + + # Get Transformer + transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + + if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + else: + transformer_2 = None + # Update pbar + pbar.update(1) + + # Get tokenizer and text_encoder + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + pbar.update(1) + + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + pbar.update(1) + + # Get pipeline + if model_type == "Inpaint": + if transformer.config.in_channels != vae.config.latent_channels: + pipeline = Wan2_2FunInpaintPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + transformer_2=transformer_2, + scheduler=scheduler, + ) + else: + pipeline = Wan2_2FunPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + transformer_2=transformer_2, + scheduler=scheduler, + ) + else: + pipeline = Wan2_2FunControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + transformer_2=transformer_2, + scheduler=scheduler, + ) + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + + if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device=device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [], + 'config': config, + } + return (funmodels,) + +class LoadWan2_2FunLora: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ("FunModels",), + "lora_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "lora_high_name": (folder_paths.get_filename_list("loras"), {"default": None,}), + "strength_model": ("FLOAT", {"default": 1.0, "min": -100.0, "max": 100.0, "step": 0.01}), + "lora_cache":([False, True], {"default": False,}), + } + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "load_lora" + CATEGORY = "CogVideoXFUNWrapper" + + def load_lora(self, funmodels, lora_name, lora_high_name, strength_model, lora_cache): + new_funmodels = dict(funmodels) + if lora_name is not None: + loras = list(new_funmodels.get("loras", [])) + [folder_paths.get_full_path("loras", lora_name)] + loras_high = list(new_funmodels.get("loras_high", [])) + [folder_paths.get_full_path("loras", lora_high_name)] + strength_models = list(new_funmodels.get("strength_model", [])) + [strength_model] + new_funmodels['loras'] = loras + new_funmodels['loras_high'] = loras_high + new_funmodels['strength_model'] = strength_models + new_funmodels['lora_cache'] = lora_cache + return (new_funmodels,) + +class Wan2_2FunT2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 5, "max": 161, "step": 4} + ), + "width": ( + "INT", {"default": 832, "min": 64, "max": 2048, "step": 16} + ), + "height": ( + "INT", {"default": 480, "min": 64, "max": 2048, "step": 16} + ), + "is_image":( + [ + False, + True + ], + { + "default": False, + } + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + "boundary": ( + "FLOAT", {"default": 0.875, "min": 0.00, "max": 1.00, "step": 0.001} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional": { + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, width, height, is_image, seed, steps, cfg, scheduler, shift, boundary, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, riflex_k=0): + global transformer_cpu_cache + global transformer_high_cpu_cache + global lora_path_before + global lora_high_path_before + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + else: + pipeline.transformer.disable_teacache() + if pipeline.transformer_2 is not None: + pipeline.transformer_2.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + + generator= torch.Generator(device).manual_seed(seed) + + video_length = 1 if is_image else video_length + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, clip_image = get_image_to_video_latent(None, None, video_length=video_length, sample_size=(height, width)) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + if pipeline.transformer_2 is not None: + # Save the original weights to cpu + if len(transformer_high_cpu_cache) == 0: + print('Save transformer high state_dict to cpu memory') + transformer_high_state_dict = pipeline.transformer_2.state_dict() + for key in transformer_high_state_dict: + transformer_high_cpu_cache[key] = transformer_high_state_dict[key].clone().cpu() + + lora_high_path_now = str(funmodels.get("loras_high", []) + funmodels.get("strength_model", [])) + if lora_high_path_now != lora_high_path_before: + print('Merge Lora High with Cache') + lora_high_path_before = copy.deepcopy(lora_high_path_now) + pipeline.transformer_2.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + else: + print('Merge Lora') + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + # Clear lora when switch from lora_cache=True to lora_cache=False. + if pipeline.transformer_2 is not None: + if len(transformer_high_cpu_cache) != 0: + pipeline.transformer_2.load_state_dict(transformer_high_cpu_cache) + transformer_high_cpu_cache = {} + lora_high_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + boundary = boundary, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + if pipeline.transformer_2 is not None: + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + return (videos,) + + +class Wan2_2FunInpaintSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 5, "max": 161, "step": 4} + ), + "base_resolution": ( + [ + 512, + 640, + 768, + 896, + 960, + 1024, + ], {"default": 640} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + "boundary": ( + "FLOAT", {"default": 0.900, "min": 0.00, "max": 1.00, "step": 0.001} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional": { + "start_img": ("IMAGE",), + "end_img": ("IMAGE",), + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, base_resolution, seed, steps, cfg, scheduler, shift, boundary, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, start_img=None, end_img=None, riflex_k=0): + global transformer_cpu_cache + global transformer_high_cpu_cache + global lora_path_before + global lora_high_path_before + + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + + start_img = [to_pil(_start_img) for _start_img in start_img] if start_img is not None else None + end_img = [to_pil(_end_img) for _end_img in end_img] if end_img is not None else None + # Count most suitable height and width + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + original_width, original_height = start_img[0].size if type(start_img) is list else Image.open(start_img).size + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height, width = [int(x / 16) * 16 for x in closest_size] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + else: + pipeline.transformer.disable_teacache() + if pipeline.transformer_2 is not None: + pipeline.transformer_2.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_img, end_img, video_length=video_length, sample_size=(height, width)) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + if pipeline.transformer_2 is not None: + # Save the original weights to cpu + if len(transformer_high_cpu_cache) == 0: + print('Save transformer high state_dict to cpu memory') + transformer_high_state_dict = pipeline.transformer_2.state_dict() + for key in transformer_high_state_dict: + transformer_high_cpu_cache[key] = transformer_high_state_dict[key].clone().cpu() + + lora_high_path_now = str(funmodels.get("loras_high", []) + funmodels.get("strength_model", [])) + if lora_high_path_now != lora_high_path_before: + print('Merge Lora High with Cache') + lora_high_path_before = copy.deepcopy(lora_high_path_now) + pipeline.transformer_2.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + else: + print('Merge Lora') + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + # Clear lora when switch from lora_cache=True to lora_cache=False. + if pipeline.transformer_2 is not None: + if len(transformer_high_cpu_cache) != 0: + pipeline.transformer_2.load_state_dict(transformer_high_cpu_cache) + transformer_high_cpu_cache = {} + lora_high_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + boundary = boundary, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + if pipeline.transformer_2 is not None: + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + return (videos,) + + +class Wan2_2FunV2VSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 1, "max": 161, "step": 4} + ), + "base_resolution": ( + [ + 512, + 640, + 768, + 896, + 960, + 1024, + ], {"default": 640} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "denoise_strength": ( + "FLOAT", {"default": 1.00, "min": 0.05, "max": 1.00, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 5, "min": 1, "max": 100, "step": 1} + ), + "boundary": ( + "FLOAT", {"default": 0.900, "min": 0.00, "max": 1.00, "step": 0.001} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + }, + "optional": { + "validation_video": ("IMAGE",), + "control_video": ("IMAGE",), + "start_image": ("IMAGE",), + "end_image": ("IMAGE",), + "ref_image": ("IMAGE",), + "camera_conditions": ("STRING", {"forceInput": True}), + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, base_resolution, seed, steps, cfg, denoise_strength, scheduler, shift, boundary, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, validation_video=None, control_video=None, start_image=None, end_image=None, ref_image=None, camera_conditions=None, riflex_k=0): + global transformer_cpu_cache + global transformer_high_cpu_cache + global lora_path_before + global lora_high_path_before + + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + model_type = funmodels['model_type'] + + # Count most suitable height and width + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if model_type == "Inpaint": + if type(validation_video) is str: + original_width, original_height = Image.fromarray(cv2.VideoCapture(validation_video).read()[1]).size + else: + validation_video = np.array(validation_video.cpu().numpy() * 255, np.uint8) + original_width, original_height = Image.fromarray(validation_video[0]).size + else: + if control_video is not None and type(control_video) is str: + original_width, original_height = Image.fromarray(cv2.VideoCapture(control_video).read()[1]).size + elif control_video is not None: + control_video = np.array(control_video.cpu().numpy() * 255, np.uint8) + original_width, original_height = Image.fromarray(control_video[0]).size + else: + original_width, original_height = 384 / 512 * base_resolution, 672 / 512 * base_resolution + + if ref_image is not None: + ref_image = [to_pil(_ref_image) for _ref_image in ref_image] + original_width, original_height = ref_image[0].size if type(ref_image) is list else Image.open(ref_image).size + + if start_image is not None: + start_image = [to_pil(_start_image) for _start_image in start_image] + original_width, original_height = start_image[0].size if type(start_image) is list else Image.open(start_image).size + + if end_image is not None: + end_image = [to_pil(_end_image) for _end_image in end_image] + + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height, width = [int(x / 16) * 16 for x in closest_size] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + else: + pipeline.transformer.disable_teacache() + if pipeline.transformer_2 is not None: + pipeline.transformer_2.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + if model_type == "Inpaint": + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(validation_video, video_length=video_length, sample_size=(height, width), fps=16, ref_image=ref_image[0] if ref_image is not None else ref_image) + else: + if ref_image is not None: + clip_image = ref_image[0].convert("RGB") + elif start_image is not None: + clip_image = start_image[0].convert("RGB") + else: + clip_image = None + + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=(height, width)) + + if ref_image is not None: + ref_image = get_image_latent(ref_image[0] if ref_image is not None else ref_image, sample_size=(height, width)) + + if camera_conditions is not None and len(camera_conditions) > 0: + poses = json.loads(camera_conditions) + cam_params = np.array([[float(x) for x in pose] for pose in poses]) + cam_params = np.concatenate([np.zeros_like(cam_params[:, :1]), cam_params], 1) + control_camera_video = process_pose_params(cam_params, width=width, height=height) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + input_video, input_video_mask = None, None + else: + control_camera_video = None + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=(height, width), fps=16, ref_image=None) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + if pipeline.transformer_2 is not None: + # Save the original weights to cpu + if len(transformer_high_cpu_cache) == 0: + print('Save transformer high state_dict to cpu memory') + transformer_high_state_dict = pipeline.transformer_2.state_dict() + for key in transformer_high_state_dict: + transformer_high_cpu_cache[key] = transformer_high_state_dict[key].clone().cpu() + + lora_high_path_now = str(funmodels.get("loras_high", []) + funmodels.get("strength_model", [])) + if lora_high_path_now != lora_high_path_before: + print('Merge Lora High with Cache') + lora_high_path_before = copy.deepcopy(lora_high_path_now) + pipeline.transformer_2.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + else: + print('Merge Lora') + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + # Clear lora when switch from lora_cache=True to lora_cache=False. + if pipeline.transformer_2 is not None: + if len(transformer_high_cpu_cache) != 0: + pipeline.transformer_2.load_state_dict(transformer_high_cpu_cache) + transformer_high_cpu_cache = {} + lora_high_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + + if model_type == "Inpaint": + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + strength = float(denoise_strength), + comfyui_progressbar = True, + ).videos + else: + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = input_video, + control_camera_video = control_camera_video, + ref_image = ref_image, + boundary = boundary, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + if pipeline.transformer_2 is not None: + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + return (videos,) \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_control_camera.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_control_camera.json new file mode 100644 index 0000000000000000000000000000000000000000..85e4adcadabf10030b0ec3b47e878f2bf7b9a457 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_control_camera.json @@ -0,0 +1,906 @@ +{ + "id": "1464c536-caf0-472a-a5d5-6a0d06d5101a", + "revision": 0, + "last_node_id": 141, + "last_link_id": 313, + "nodes": [ + { + "id": 107, + "type": "Note", + "pos": [ + 4, + 634 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 108, + "type": "Note", + "pos": [ + -110, + 842 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 122, + "type": "FunTextBox", + "pos": [ + 238, + 805 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 310 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 129, + "type": "CameraBasicFromChaoJie", + "pos": [ + 805.2059326171875, + 1012.381103515625 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CameraPose", + "type": "CameraPose", + "links": null + } + ], + "properties": { + "Node name for S&R": "CameraBasicFromChaoJie" + }, + "widgets_values": [ + "Static", + 1, + 16 + ] + }, + { + "id": 130, + "type": "CameraTrajectoryFromChaoJie", + "pos": [ + 1170.206298828125, + 763.3814697265625 + ], + "size": [ + 367.79998779296875, + 150 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "camera_pose", + "type": "CameraPose", + "link": 285 + } + ], + "outputs": [ + { + "name": "camera_trajectory", + "type": "STRING", + "slot_index": 0, + "links": [ + 312 + ] + }, + { + "name": "video_length", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "CameraTrajectoryFromChaoJie" + }, + "widgets_values": [ + 0.532139961, + 0.946026558, + 0.5, + 0.5 + ] + }, + { + "id": 106, + "type": "VHS_VideoCombine", + "pos": [ + 1408, + 68 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 313 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00112.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 121, + "type": "FunTextBox", + "pos": [ + 235, + 539 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 309 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "Fireworks light up the evening sky over a sprawling cityscape with gothic-style buildings featuring pointed towers and clock faces. The city is lit by both artificial lights from the buildings and the colorful bursts of the fireworks. The scene is viewed from an elevated angle, showcasing a vibrant urban environment set against a backdrop of a dramatic, partially cloudy sky at dusk." + ] + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 237.59738159179688, + 1164.597412109375 + ], + "size": [ + 378.07147216796875, + 314 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 311 + ] + }, + { + "label": "遮罩", + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image（图片到视频的开始图片）", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "5.png", + "image" + ] + }, + { + "id": 131, + "type": "CameraCombineFromChaoJie", + "pos": [ + 814.2059326171875, + 763.3814697265625 + ], + "size": [ + 315, + 178 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CameraPose", + "type": "CameraPose", + "links": [ + 285 + ] + } + ], + "properties": { + "Node name for S&R": "CameraCombineFromChaoJie" + }, + "widgets_values": [ + "Pan Right", + "Pan Up", + "Static", + "Static", + 1, + 81 + ] + }, + { + "id": 110, + "type": "Note", + "pos": [ + 1158.206298828125, + 970.381103515625 + ], + "size": [ + 608.1410522460938, + 188.2682342529297 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "CameraCombine is used to combine multiple camera movements, while CameraBasic produces a single camera movement. The nodes come from https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/. Since ComfyUI-CameraCtrl-Wrapper requires a specific version of diffusers, the code has been copied into the current repository.\n(CameraCombine用于组合多个镜头运动，CameraBasic产出单个镜头运动；节点来自于https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/，由于ComfyUI-CameraCtrl-Wrapper有具体diffusers版本要求，故复制代码到当前库中。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 137, + "type": "CombineWan2_2Pipeline", + "pos": [ + 459.86553955078125, + 105.53478240966797 + ], + "size": [ + 360.3475341796875, + 206 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 301 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 302 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 303 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 304 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 305 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 306 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 308 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2Pipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload", + "Control" + ] + }, + { + "id": 138, + "type": "LoadWanTextEncoderModel", + "pos": [ + 2.877215623855591, + 210.45326232910156 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 303 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 304 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 139, + "type": "LoadWanVAEModel", + "pos": [ + 2.877215623855591, + 59.883544921875 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 302 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 112, + "type": "Note", + "pos": [ + -252.11000061035156, + 389.1165771484375 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 140, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 6.800307750701904, + -99.07373809814453 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 301 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 306 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control-Camera-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 136, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 438.7657470703125, + -97.08856964111328 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 305 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control-Camera-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 141, + "type": "Wan2_2FunV2VSampler", + "pos": [ + 923.9700317382812, + 64.22538757324219 + ], + "size": [ + 350.2320251464844, + 574 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 308 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 309 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 310 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 311 + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": 312 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 313 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 329092863634844, + "randomize", + 50, + 6, + 1, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 285, + 131, + 0, + 130, + 0, + "CameraPose" + ], + [ + 301, + 140, + 0, + 137, + 0, + "TransformerModel" + ], + [ + 302, + 139, + 0, + 137, + 1, + "VAEModel" + ], + [ + 303, + 138, + 0, + 137, + 2, + "TextEncoderModel" + ], + [ + 304, + 138, + 1, + 137, + 3, + "Tokenizer" + ], + [ + 305, + 136, + 0, + 137, + 5, + "TransformerModel" + ], + [ + 306, + 140, + 1, + 137, + 6, + "STRING" + ], + [ + 308, + 137, + 0, + 141, + 0, + "FunModels" + ], + [ + 309, + 121, + 0, + 141, + 1, + "STRING_PROMPT" + ], + [ + 310, + 122, + 0, + 141, + 2, + "STRING_PROMPT" + ], + [ + 311, + 100, + 0, + 141, + 5, + "IMAGE" + ], + [ + 312, + 130, + 0, + 141, + 8, + "STRING" + ], + [ + 313, + 141, + 0, + 106, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Generate Control Video", + "bounding": [ + 773, + 666, + 1025, + 531 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "First Image", + "bounding": [ + 191, + 1068, + 475, + 456 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 191, + 456, + 475, + 587 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 5, + "title": "Load Model", + "bounding": [ + -7.123280048370361, + -178.99830627441406, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 201.7628279830343, + 116.00196884690006 + ] + }, + "frontendVersion": "1.21.3", + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.44" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_flf.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_flf.json new file mode 100644 index 0000000000000000000000000000000000000000..02a9e805120951bc06b0879407e082e74fef75a6 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_flf.json @@ -0,0 +1,786 @@ +{ + "id": "3e85d93b-b3b0-4a45-8225-4a8c8bf27fb4", + "revision": 0, + "last_node_id": 111, + "last_link_id": 94, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 90 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "Note", + "pos": [ + 17, + -35 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": [ + 35.71394348144531, + 536.9210205078125 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1271, + -67 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 94 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 91 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 87, + "type": "LoadImage", + "pos": [ + 280.5103759765625, + 477.9588623046875 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 92 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "2.png", + "image" + ] + }, + { + "id": 103, + "type": "LoadImage", + "pos": [ + 689.5659790039062, + 477.3172302246094 + ], + "size": [ + 318.56072998046875, + 326 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 93 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "End Image", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "070008-d3df70b6-1c49-43d5-a39b-ba357e48c17a-3149685904.png", + "image" + ] + }, + { + "id": 105, + "type": "LoadWanVAEModel", + "pos": [ + 233.51658630371094, + -440.1548156738281 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 71 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 106, + "type": "LoadWanTextEncoderModel", + "pos": [ + 233.51658630371094, + -289.5851135253906 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 72 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 73 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -157.91722106933594, + -340.1963195800781 + ], + "size": [ + 355.636474609375, + 132.4238739013672 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 107, + "type": "CombineWan2_2Pipeline", + "pos": [ + 690.5042114257812, + -394.50360107421875 + ], + "size": [ + 360.3475341796875, + 206 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 70 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 71 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 72 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 73 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 74 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 75 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 89 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2Pipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload", + "Inpaint" + ] + }, + { + "id": 104, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 237.43971252441406, + -599.1122436523438 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 70 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-InP-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 108, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 669.4044189453125, + -597.1270751953125 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 74 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-InP-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 111, + "type": "Wan2_2FunInpaintSampler", + "pos": [ + 788.7388916015625, + -94.46605682373047 + ], + "size": [ + 353.5884704589844, + 470 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 89 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 90 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 91 + }, + { + "name": "start_img", + "shape": 7, + "type": "IMAGE", + "link": 92 + }, + { + "name": "end_img", + "shape": 7, + "type": "IMAGE", + "link": 93 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 94 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunInpaintSampler" + }, + "widgets_values": [ + 81, + 640, + 277302737621485, + "randomize", + 50, + 6, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 70, + 104, + 0, + 107, + 0, + "TransformerModel" + ], + [ + 71, + 105, + 0, + 107, + 1, + "VAEModel" + ], + [ + 72, + 106, + 0, + 107, + 2, + "TextEncoderModel" + ], + [ + 73, + 106, + 1, + 107, + 3, + "Tokenizer" + ], + [ + 74, + 108, + 0, + 107, + 5, + "TransformerModel" + ], + [ + 75, + 104, + 1, + 107, + 6, + "STRING" + ], + [ + 89, + 107, + 0, + 111, + 0, + "FunModels" + ], + [ + 90, + 75, + 0, + 111, + 1, + "STRING_PROMPT" + ], + [ + 91, + 73, + 0, + 111, + 2, + "STRING_PROMPT" + ], + [ + 92, + 87, + 0, + 111, + 3, + "IMAGE" + ], + [ + 93, + 103, + 0, + 111, + 4, + "IMAGE" + ], + [ + 94, + 111, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Image", + "bounding": [ + 222, + 396, + 875.5350952148438, + 432.9040832519531 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model", + "bounding": [ + 223.51626586914062, + -679.0366821289062, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650705, + "offset": [ + 333.3822625115507, + 670.6804120329575 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.44" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_i2v.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..3adcdeae6c9f5c1105314071b921d8cd0d574acc --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_i2v.json @@ -0,0 +1,740 @@ +{ + "id": "3e85d93b-b3b0-4a45-8225-4a8c8bf27fb4", + "revision": 0, + "last_node_id": 111, + "last_link_id": 94, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 90 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "Note", + "pos": [ + 17, + -35 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": [ + 35.71394348144531, + 536.9210205078125 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1271, + -67 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 94 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 91 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 87, + "type": "LoadImage", + "pos": [ + 280.5103759765625, + 477.9588623046875 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 92 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "2.png", + "image" + ] + }, + { + "id": 105, + "type": "LoadWanVAEModel", + "pos": [ + 233.51658630371094, + -440.1548156738281 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 71 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 106, + "type": "LoadWanTextEncoderModel", + "pos": [ + 233.51658630371094, + -289.5851135253906 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 72 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 73 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 80, + "type": "Note", + "pos": [ + -157.91722106933594, + -340.1963195800781 + ], + "size": [ + 355.636474609375, + 132.4238739013672 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 107, + "type": "CombineWan2_2Pipeline", + "pos": [ + 690.5042114257812, + -394.50360107421875 + ], + "size": [ + 360.3475341796875, + 206 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 70 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 71 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 72 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 73 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 74 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 75 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 89 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2Pipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload", + "Inpaint" + ] + }, + { + "id": 104, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 237.43971252441406, + -599.1122436523438 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 70 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-InP-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 108, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 669.4044189453125, + -597.1270751953125 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 74 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-InP-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 111, + "type": "Wan2_2FunInpaintSampler", + "pos": [ + 788.7388916015625, + -94.46605682373047 + ], + "size": [ + 353.5884704589844, + 470 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 89 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 90 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 91 + }, + { + "name": "start_img", + "shape": 7, + "type": "IMAGE", + "link": 92 + }, + { + "name": "end_img", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 94 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunInpaintSampler" + }, + "widgets_values": [ + 81, + 640, + 277302737621485, + "randomize", + 50, + 6, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 70, + 104, + 0, + 107, + 0, + "TransformerModel" + ], + [ + 71, + 105, + 0, + 107, + 1, + "VAEModel" + ], + [ + 72, + 106, + 0, + 107, + 2, + "TextEncoderModel" + ], + [ + 73, + 106, + 1, + 107, + 3, + "Tokenizer" + ], + [ + 74, + 108, + 0, + 107, + 5, + "TransformerModel" + ], + [ + 75, + 104, + 1, + 107, + 6, + "STRING" + ], + [ + 89, + 107, + 0, + 111, + 0, + "FunModels" + ], + [ + 90, + 75, + 0, + 111, + 1, + "STRING_PROMPT" + ], + [ + 91, + 73, + 0, + 111, + 2, + "STRING_PROMPT" + ], + [ + 92, + 87, + 0, + 111, + 3, + "IMAGE" + ], + [ + 94, + 111, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Image", + "bounding": [ + 222, + 396, + 422.8389892578125, + 427.9284973144531 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model", + "bounding": [ + 223.51626586914062, + -679.0366821289062, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015779, + "offset": [ + 126.29056352326933, + 918.635958298582 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.44" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_v2v_control.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_v2v_control.json new file mode 100644 index 0000000000000000000000000000000000000000..9c2642e78a8aaebb2be00fd8347b718215a5643a --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_v2v_control.json @@ -0,0 +1,806 @@ +{ + "id": "d9edc9c1-4b9b-467b-8063-98892514259c", + "revision": 0, + "last_node_id": 103, + "last_link_id": 77, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.739953994750977, + 462.38665771484375 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 335, + 476 + ], + "size": [ + 252.056640625, + 688.545166015625 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 76 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 74 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + ] + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 75 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1274, + 10 + ], + "size": [ + 390.9534912109375, + 966.9860229492188 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 77 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00004.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 100, + "type": "LoadWanVAEModel", + "pos": [ + 225.74212646484375, + -429.7904357910156 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 101, + "type": "LoadWanTextEncoderModel", + "pos": [ + 225.74212646484375, + -279.22076416015625 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 68 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 69 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 89, + "type": "Note", + "pos": [ + -243.11192321777344, + -404.67767333984375 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 103, + "type": "Wan2_2FunV2VSampler", + "pos": [ + 790.9247436523438, + 10.273669242858887 + ], + "size": [ + 350.2320251464844, + 574 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 73 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 74 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 75 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 76 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 1003658886389107, + "randomize", + 50, + 6, + 1, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 98, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 661.6300048828125, + -586.7626342773438 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 70 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 102, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 229.66526794433594, + -588.747802734375 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 66 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 71 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 99, + "type": "CombineWan2_2Pipeline", + "pos": [ + 682.7297973632812, + -384.1392517089844 + ], + "size": [ + 360.3475341796875, + 206 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 66 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 67 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 68 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 69 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 70 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 71 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 73 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2Pipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload", + "Control" + ] + } + ], + "links": [ + [ + 66, + 102, + 0, + 99, + 0, + "TransformerModel" + ], + [ + 67, + 100, + 0, + 99, + 1, + "VAEModel" + ], + [ + 68, + 101, + 0, + 99, + 2, + "TextEncoderModel" + ], + [ + 69, + 101, + 1, + 99, + 3, + "Tokenizer" + ], + [ + 70, + 98, + 0, + 99, + 5, + "TransformerModel" + ], + [ + 71, + 102, + 1, + 99, + 6, + "STRING" + ], + [ + 73, + 99, + 0, + 103, + 0, + "FunModels" + ], + [ + 74, + 92, + 0, + 103, + 1, + "STRING_PROMPT" + ], + [ + 75, + 94, + 0, + 103, + 2, + "STRING_PROMPT" + ], + [ + 76, + 85, + 0, + 103, + 4, + "IMAGE" + ], + [ + 77, + 103, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 487, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 215.74168395996094, + -668.6722412109375, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 534.0092600142825, + 618.3725102531495 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_v2v_control_pose_ref.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_v2v_control_pose_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..c83e195680d001be713e3b9cfe1c65072a6d7750 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_chunked_loading_workflow_v2v_control_pose_ref.json @@ -0,0 +1,989 @@ +{ + "id": "c5a53a91-88dc-41d4-b414-12e402c59e4c", + "revision": 0, + "last_node_id": 110, + "last_link_id": 98, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + -111.46612548828125, + 460.2178955078125 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 99, + "type": "VHS_VideoCombine", + "pos": [ + 1005, + 554 + ], + "size": [ + 315, + 849.46875 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 73 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Preprocess-Video", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun-Preprocess-Video_00007.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 207.79391479492188, + 473.83123779296875 + ], + "size": [ + 252.056640625, + 688.545166015625 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 72 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "000007.mp4", + "force_rate": 16, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 16, + "filename": "000007.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1488, + 8 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 84 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00049.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 102, + "type": "LoadImage", + "pos": [ + 553, + 598 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 86 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 101, + "type": "VideoToOpenpose", + "pos": [ + 558, + 474 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "input_video", + "type": "IMAGE", + "link": 72 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 73, + 87, + 91 + ] + } + ], + "properties": { + "Node name for S&R": "VideoToOpenpose" + }, + "widgets_values": [ + 81 + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 89 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位动漫风格的女孩。她有着紫色的短发，头上戴着一个黑色和金色相间的蝴蝶结。她的表情显得有些严肃或沉思，眼睛大而有神。女孩穿着一件白色衬衫，外面搭配了一件深蓝色的背心，背心上有一个粉色的蝴蝶结装饰。她的裙子是白色的，裙摆蓬松，整体造型非常可爱且精致。背景是一个简单的圆形图案，颜色为粉红色和灰色相间，给人一种柔和的感觉。整个画面色调柔和，人物形象生动鲜明。" + ] + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 105, + "type": "Wan2_2FunV2VSampler", + "pos": [ + 987.7456665039062, + -76.86135864257812 + ], + "size": [ + 434.938232421875, + 550 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 98 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 89 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 90 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": 91 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 87 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": 86 + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 84 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6.000000000000001, + 1, + "Flow", + 5, + 0.1, + true, + 3, + true, + 0.25000000000000006, + 0 + ] + }, + { + "id": 106, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 667.5018920898438, + -576.8204345703125 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 96 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 107, + "type": "LoadWan2_2TransformerModel", + "pos": [ + 235.53611755371094, + -578.8056030273438 + ], + "size": [ + 393.9992370605469, + 102 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 92 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 97 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2TransformerModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 108, + "type": "LoadWanVAEModel", + "pos": [ + 231.61300659179688, + -419.8486328125 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 93 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 109, + "type": "LoadWanTextEncoderModel", + "pos": [ + 231.61300659179688, + -269.279296875 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 94 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 95 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 89, + "type": "Note", + "pos": [ + -214.2266845703125, + -303.6688232421875 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 110, + "type": "CombineWan2_2Pipeline", + "pos": [ + 688.6017456054688, + -374.1976013183594 + ], + "size": [ + 360.3475341796875, + 206 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 92 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 93 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 94 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 95 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 96 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 97 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 98 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2Pipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload", + "Control" + ] + } + ], + "links": [ + [ + 72, + 85, + 0, + 101, + 0, + "IMAGE" + ], + [ + 73, + 101, + 0, + 99, + 0, + "IMAGE" + ], + [ + 84, + 105, + 0, + 17, + 0, + "IMAGE" + ], + [ + 86, + 102, + 0, + 105, + 7, + "IMAGE" + ], + [ + 87, + 101, + 0, + 105, + 4, + "IMAGE" + ], + [ + 89, + 92, + 0, + 105, + 1, + "STRING_PROMPT" + ], + [ + 90, + 94, + 0, + 105, + 2, + "STRING_PROMPT" + ], + [ + 91, + 101, + 0, + 105, + 3, + "IMAGE" + ], + [ + 92, + 107, + 0, + 110, + 0, + "TransformerModel" + ], + [ + 93, + 108, + 0, + 110, + 1, + "VAEModel" + ], + [ + 94, + 109, + 0, + 110, + 2, + "TextEncoderModel" + ], + [ + 95, + 109, + 1, + 110, + 3, + "Tokenizer" + ], + [ + 96, + 106, + 0, + 110, + 5, + "TransformerModel" + ], + [ + 97, + 107, + 1, + 110, + 6, + "STRING" + ], + [ + 98, + 110, + 0, + 105, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Video And Reference Image", + "bounding": [ + 91, + 383, + 859, + 841 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 221.61253356933594, + -658.7300415039062, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015781, + "offset": [ + 313.19959434428534, + 349.94392128752395 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.44", + "CogVideoX-Fun": "37bd73564702fa47ffe4576c87da6f82a50cb60e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_control_camera.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_control_camera.json new file mode 100644 index 0000000000000000000000000000000000000000..7e387f873a5eac6d562990282242a15e069f083f --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_control_camera.json @@ -0,0 +1,672 @@ +{ + "id": "f2736b24-5147-427c-a9dd-036fd3053b5d", + "revision": 0, + "last_node_id": 135, + "last_link_id": 300, + "nodes": [ + { + "id": 107, + "type": "Note", + "pos": [ + 4, + 634 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 108, + "type": "Note", + "pos": [ + -110, + 842 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 122, + "type": "FunTextBox", + "pos": [ + 238, + 805 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 297 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 129, + "type": "CameraBasicFromChaoJie", + "pos": [ + 805.2059326171875, + 1012.381103515625 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CameraPose", + "type": "CameraPose", + "links": null + } + ], + "properties": { + "Node name for S&R": "CameraBasicFromChaoJie" + }, + "widgets_values": [ + "Static", + 1, + 16 + ] + }, + { + "id": 130, + "type": "CameraTrajectoryFromChaoJie", + "pos": [ + 1170.206298828125, + 763.3814697265625 + ], + "size": [ + 367.79998779296875, + 150 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "camera_pose", + "type": "CameraPose", + "link": 285 + } + ], + "outputs": [ + { + "name": "camera_trajectory", + "type": "STRING", + "slot_index": 0, + "links": [ + 299 + ] + }, + { + "name": "video_length", + "type": "INT", + "links": null + } + ], + "properties": { + "Node name for S&R": "CameraTrajectoryFromChaoJie" + }, + "widgets_values": [ + 0.532139961, + 0.946026558, + 0.5, + 0.5 + ] + }, + { + "id": 106, + "type": "VHS_VideoCombine", + "pos": [ + 1408, + 68 + ], + "size": [ + 390, + 310 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 300 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00112.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 121, + "type": "FunTextBox", + "pos": [ + 235, + 539 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 296 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "Fireworks light up the evening sky over a sprawling cityscape with gothic-style buildings featuring pointed towers and clock faces. The city is lit by both artificial lights from the buildings and the colorful bursts of the fireworks. The scene is viewed from an elevated angle, showcasing a vibrant urban environment set against a backdrop of a dramatic, partially cloudy sky at dusk." + ] + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 237.59738159179688, + 1164.597412109375 + ], + "size": [ + 378.07147216796875, + 314 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 298 + ] + }, + { + "label": "遮罩", + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image（图片到视频的开始图片）", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "5.png", + "image" + ] + }, + { + "id": 131, + "type": "CameraCombineFromChaoJie", + "pos": [ + 814.2059326171875, + 763.3814697265625 + ], + "size": [ + 315, + 178 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "CameraPose", + "type": "CameraPose", + "links": [ + 285 + ] + } + ], + "properties": { + "Node name for S&R": "CameraCombineFromChaoJie" + }, + "widgets_values": [ + "Pan Right", + "Pan Up", + "Static", + "Static", + 1, + 81 + ] + }, + { + "id": 110, + "type": "Note", + "pos": [ + 1158.206298828125, + 970.381103515625 + ], + "size": [ + 608.1410522460938, + 188.2682342529297 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "CameraCombine is used to combine multiple camera movements, while CameraBasic produces a single camera movement. The nodes come from https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/. Since ComfyUI-CameraCtrl-Wrapper requires a specific version of diffusers, the code has been copied into the current repository.\n(CameraCombine用于组合多个镜头运动，CameraBasic产出单个镜头运动；节点来自于https://github.com/chaojie/ComfyUI-CameraCtrl-Wrapper/，由于ComfyUI-CameraCtrl-Wrapper有具体diffusers版本要求，故复制代码到当前库中。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 112, + "type": "Note", + "pos": [ + -203, + 252 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 134, + "type": "LoadWan2_2FunModel", + "pos": [ + 273.49224853515625, + 249.6970672607422 + ], + "size": [ + 350.6878662109375, + 155.20101928710938 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 295 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control-Camera", + "Control", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + }, + { + "id": 135, + "type": "Wan2_2FunV2VSampler", + "pos": [ + 923.0663452148438, + 107.64505767822266 + ], + "size": [ + 350.2320251464844, + 526 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 295 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 296 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 297 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 298 + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": 299 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 300 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6.000000000000001, + 1, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 285, + 131, + 0, + 130, + 0, + "CameraPose" + ], + [ + 295, + 134, + 0, + 135, + 0, + "FunModels" + ], + [ + 296, + 121, + 0, + 135, + 1, + "STRING_PROMPT" + ], + [ + 297, + 122, + 0, + 135, + 2, + "STRING_PROMPT" + ], + [ + 298, + 100, + 0, + 135, + 5, + "IMAGE" + ], + [ + 299, + 130, + 0, + 135, + 8, + "STRING" + ], + [ + 300, + 135, + 0, + 106, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Generate Control Video", + "bounding": [ + 773, + 666, + 1025, + 531 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "First Image", + "bounding": [ + 191, + 1068, + 475, + 456 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 191, + 456, + 475, + 587 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 189, + 160, + 475, + 269 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 246.58945220178464, + 51.36424150314978 + ] + }, + "node_versions": { + "CogVideoX-Fun": "e935795d7684aaba72be65d2ad5ef580c0526f7a", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.44" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_control_trajectory.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_control_trajectory.json new file mode 100644 index 0000000000000000000000000000000000000000..69d0185aa1e9aace9f03b17729735b50d54f8672 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_control_trajectory.json @@ -0,0 +1,1047 @@ +{ + "id": "a707e0b0-7fbd-4f91-9c23-1c5bdca19bad", + "revision": 0, + "last_node_id": 129, + "last_link_id": 294, + "nodes": [ + { + "id": 107, + "type": "Note", + "pos": [ + 4, + 634 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 108, + "type": "Note", + "pos": [ + -110, + 842 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 100, + "type": "LoadImage", + "pos": [ + 238, + 1165 + ], + "size": [ + 378.07147216796875, + 314 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "label": "图像", + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 292 + ] + }, + { + "label": "遮罩", + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image（图片到视频的开始图片）", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "1.png", + "image" + ] + }, + { + "id": 118, + "type": "AppendStringsToList", + "pos": [ + 1140.1396484375, + 909.9193115234375 + ], + "size": [ + 315, + 82 + ], + "flags": { + "collapsed": false + }, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "string1", + "type": "STRING", + "link": 265 + }, + { + "name": "string2", + "type": "STRING", + "link": 266 + } + ], + "outputs": [ + { + "name": "STRING", + "type": "STRING", + "slot_index": 0, + "links": [ + 267 + ] + } + ], + "properties": { + "Node name for S&R": "AppendStringsToList" + }, + "widgets_values": [] + }, + { + "id": 121, + "type": "FunTextBox", + "pos": [ + 235, + 539 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 289 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一只棕褐色的狗正摇晃着脑袋，坐在一个舒适的房间里的浅色沙发上。沙发看起来柔软而宽敞，为这只活泼的狗狗提供了一个完美的休息地点。在狗的后面，靠墙摆放着一个架子，架子上挂着一幅精美的镶框画，画中描绘着一些美丽的风景或场景。画框周围装饰着粉红色的花朵，这些花朵不仅增添了房间的色彩，还带来了一丝自然和生机。房间里的灯光柔和而温暖，从天花板上的吊灯和角落里的台灯散发出来，营造出一种温馨舒适的氛围。整个空间给人一种宁静和谐的感觉，仿佛时间在这里变得缓慢而美好。" + ] + }, + { + "id": 95, + "type": "CreateTrajectoryBasedOnKJNodes", + "pos": [ + 1574.139404296875, + 929.9193115234375 + ], + "size": [ + 428.4000244140625, + 58 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "coordinates", + "type": "STRING", + "link": 267 + }, + { + "name": "masks", + "type": "MASK", + "link": 249 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 237, + 262, + 291 + ] + } + ], + "properties": { + "Node name for S&R": "CreateTrajectoryBasedOnKJNodes" + }, + "widgets_values": [] + }, + { + "id": 122, + "type": "FunTextBox", + "pos": [ + 238, + 805 + ], + "size": [ + 400, + 200 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 290 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 97, + "type": "SplineEditor", + "pos": [ + 855.1397705078125, + 1058.91943359375 + ], + "size": [ + 645, + 832 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "bg_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "slot_index": 0, + "links": [ + 249 + ] + }, + { + "name": "coord_str", + "type": "STRING", + "slot_index": 1, + "links": [ + 265 + ] + }, + { + "name": "float", + "type": "FLOAT", + "links": null + }, + { + "name": "count", + "type": "INT", + "links": null + }, + { + "name": "normalized_str", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "SplineEditor", + "points": "SplineEditor", + "imgData": null + }, + "widgets_values": [ + "[{\"points\":[{\"x\":236.74497000000005,\"y\":169.10355000000004},{\"x\":263.53799999999995,\"y\":230.59575},{\"x\":321.61075881587004,\"y\":229.72197058276433},{\"x\":355.3934015486295,\"y\":168.9132136637973},{\"x\":343.23165016483614,\"y\":82.42964826793309},{\"x\":275.6663646993172,\"y\":55.40353408172552},{\"x\":197.29063355931524,\"y\":85.13225968655384},{\"x\":177.02104791965957,\"y\":148.64362802414163},{\"x\":205.39846781517753,\"y\":232.4245820013851},{\"x\":259.45069618759265,\"y\":167.56190795448694}],\"color\":\"#1f77b4\",\"name\":\"Spline 1\"}]", + "[[{\"x\":236.74496459960938,\"y\":169.10354614257812},{\"x\":238.94464111328125,\"y\":177.68576049804688},{\"x\":241.3327178955078,\"y\":186.21737670898438},{\"x\":243.95693969726562,\"y\":194.67919921875},{\"x\":246.88812255859375,\"y\":203.03933715820312},{\"x\":250.2393035888672,\"y\":211.23939514160156},{\"x\":254.20826721191406,\"y\":219.15655517578125},{\"x\":259.18768310546875,\"y\":226.46995544433594},{\"x\":265.87689208984375,\"y\":232.22171020507812},{\"x\":273.4639892578125,\"y\":236.787353515625},{\"x\":281.6380920410156,\"y\":240.17092895507812},{\"x\":290.3527526855469,\"y\":241.61793518066406},{\"x\":299.1367492675781,\"y\":240.67857360839844},{\"x\":307.49658203125,\"y\":237.7835235595703},{\"x\":315.3465576171875,\"y\":233.68600463867188},{\"x\":322.80859375,\"y\":228.9126739501953},{\"x\":330.00958251953125,\"y\":223.75323486328125},{\"x\":336.7490234375,\"y\":218.0097198486328},{\"x\":342.5494689941406,\"y\":211.32901000976562},{\"x\":346.96185302734375,\"y\":203.6618194580078},{\"x\":350.05450439453125,\"y\":195.3665771484375},{\"x\":352.2560729980469,\"y\":186.787109375},{\"x\":353.9389343261719,\"y\":178.08937072753906},{\"x\":355.33001708984375,\"y\":169.3397216796875},{\"x\":356.59344482421875,\"y\":160.57052612304688},{\"x\":357.7420959472656,\"y\":151.78565979003906},{\"x\":358.6518249511719,\"y\":142.9732208251953},{\"x\":359.1504211425781,\"y\":134.1287384033203},{\"x\":359.0207214355469,\"y\":125.2725601196289},{\"x\":358.035400390625,\"y\":116.4721908569336},{\"x\":356.0333251953125,\"y\":107.84730529785156},{\"x\":352.9960632324219,\"y\":99.53003692626953},{\"x\":349.046142578125,\"y\":91.60392761230469},{\"x\":344.37164306640625,\"y\":84.08070373535156},{\"x\":339.17694091796875,\"y\":76.9051742553711},{\"x\":333.4718322753906,\"y\":70.13098907470703},{\"x\":326.9710693359375,\"y\":64.12535858154297},{\"x\":319.4295654296875,\"y\":59.51286315917969},{\"x\":311.02655029296875,\"y\":56.7620735168457},{\"x\":302.25640869140625,\"y\":55.55453872680664},{\"x\":293.40484619140625,\"y\":55.21137619018555},{\"x\":284.5453796386719,\"y\":55.24800491333008},{\"x\":275.68701171875,\"y\":55.40314865112305},{\"x\":266.83221435546875,\"y\":55.690216064453125},{\"x\":257.9945373535156,\"y\":56.30451965332031},{\"x\":249.2041473388672,\"y\":57.398460388183594},{\"x\":240.5244140625,\"y\":59.16102981567383},{\"x\":232.06307983398438,\"y\":61.773162841796875},{\"x\":223.95306396484375,\"y\":65.32772827148438},{\"x\":216.29017639160156,\"y\":69.7664566040039},{\"x\":209.08375549316406,\"y\":74.91584777832031},{\"x\":202.27122497558594,\"y\":80.57798767089844},{\"x\":195.76087951660156,\"y\":86.58625030517578},{\"x\":189.468505859375,\"y\":92.8224105834961},{\"x\":183.6200714111328,\"y\":99.47235107421875},{\"x\":178.9163055419922,\"y\":106.95783233642578},{\"x\":176.3991241455078,\"y\":115.42044830322266},{\"x\":175.83934020996094,\"y\":124.25370788574219},{\"x\":176.12339782714844,\"y\":133.10797119140625},{\"x\":176.63381958007812,\"y\":141.95301818847656},{\"x\":177.14378356933594,\"y\":150.79808044433594},{\"x\":177.73876953125,\"y\":159.63778686523438},{\"x\":178.5083770751953,\"y\":168.46385192871094},{\"x\":179.4958038330078,\"y\":177.26815795898438},{\"x\":180.75762939453125,\"y\":186.03709411621094},{\"x\":182.37294006347656,\"y\":194.7476043701172},{\"x\":184.4583282470703,\"y\":203.35687255859375},{\"x\":187.1986083984375,\"y\":211.7789306640625},{\"x\":190.9047393798828,\"y\":219.81732177734375},{\"x\":196.108154296875,\"y\":226.9572296142578},{\"x\":203.41249084472656,\"y\":231.83749389648438},{\"x\":211.85409545898438,\"y\":230.89816284179688},{\"x\":218.7886199951172,\"y\":225.41867065429688},{\"x\":224.8160400390625,\"y\":218.92974853515625},{\"x\":230.3641357421875,\"y\":212.0236358642578},{\"x\":235.608642578125,\"y\":204.8833770751953},{\"x\":240.63941955566406,\"y\":197.59080505371094},{\"x\":245.50965881347656,\"y\":190.1898651123047},{\"x\":250.25364685058594,\"y\":182.7073516845703},{\"x\":254.8948974609375,\"y\":175.16050720214844},{\"x\":259.45068359375,\"y\":167.56190490722656}]]", + 600, + 382, + 81, + "path", + "cardinal", + 0.5, + 1, + "list", + 0, + 1, + "", + null + ] + }, + { + "id": 119, + "type": "SplineEditor", + "pos": [ + 1544.139404296875, + 1047.919189453125 + ], + "size": [ + 645, + 832 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "bg_image", + "shape": 7, + "type": "IMAGE", + "link": null + } + ], + "outputs": [ + { + "name": "mask", + "type": "MASK", + "slot_index": 0, + "links": [] + }, + { + "name": "coord_str", + "type": "STRING", + "slot_index": 1, + "links": [ + 266 + ] + }, + { + "name": "float", + "type": "FLOAT", + "links": null + }, + { + "name": "count", + "type": "INT", + "links": null + }, + { + "name": "normalized_str", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "SplineEditor", + "points": "SplineEditor", + "imgData": null + }, + "widgets_values": [ + "[{\"points\":[{\"x\":63.916760050380844,\"y\":114.45559357858895},{\"x\":71.34894145158792,\"y\":114.45559357858895}],\"color\":\"#1f77b4\",\"name\":\"Spline 1\"}]", + "[[{\"x\":63.9167594909668,\"y\":114.45559692382812},{\"x\":64.00965881347656,\"y\":114.45559692382812},{\"x\":64.1025619506836,\"y\":114.45559692382812},{\"x\":64.19546508789062,\"y\":114.45559692382812},{\"x\":64.28836822509766,\"y\":114.45559692382812},{\"x\":64.38127136230469,\"y\":114.45559692382812},{\"x\":64.47417449951172,\"y\":114.45559692382812},{\"x\":64.56707763671875,\"y\":114.45559692382812},{\"x\":64.65998077392578,\"y\":114.45559692382812},{\"x\":64.75287628173828,\"y\":114.45559692382812},{\"x\":64.84577941894531,\"y\":114.45559692382812},{\"x\":64.93868255615234,\"y\":114.45559692382812},{\"x\":65.03158569335938,\"y\":114.45559692382812},{\"x\":65.1244888305664,\"y\":114.45559692382812},{\"x\":65.21739196777344,\"y\":114.45559692382812},{\"x\":65.31029510498047,\"y\":114.45559692382812},{\"x\":65.4031982421875,\"y\":114.45559692382812},{\"x\":65.49609375,\"y\":114.45559692382812},{\"x\":65.58899688720703,\"y\":114.45559692382812},{\"x\":65.68190002441406,\"y\":114.45559692382812},{\"x\":65.7748031616211,\"y\":114.45559692382812},{\"x\":65.86770629882812,\"y\":114.45559692382812},{\"x\":65.96060943603516,\"y\":114.45559692382812},{\"x\":66.05351257324219,\"y\":114.45559692382812},{\"x\":66.14641571044922,\"y\":114.45559692382812},{\"x\":66.23931884765625,\"y\":114.45559692382812},{\"x\":66.33221435546875,\"y\":114.45559692382812},{\"x\":66.42511749267578,\"y\":114.45559692382812},{\"x\":66.51802062988281,\"y\":114.45559692382812},{\"x\":66.61092376708984,\"y\":114.45559692382812},{\"x\":66.70382690429688,\"y\":114.45559692382812},{\"x\":66.7967300415039,\"y\":114.45559692382812},{\"x\":66.88963317871094,\"y\":114.45559692382812},{\"x\":66.98253631591797,\"y\":114.45559692382812},{\"x\":67.075439453125,\"y\":114.45559692382812},{\"x\":67.1683349609375,\"y\":114.45559692382812},{\"x\":67.26123809814453,\"y\":114.45559692382812},{\"x\":67.35414123535156,\"y\":114.45559692382812},{\"x\":67.4470443725586,\"y\":114.45559692382812},{\"x\":67.53994750976562,\"y\":114.45559692382812},{\"x\":67.63285064697266,\"y\":114.45559692382812},{\"x\":67.72575378417969,\"y\":114.45559692382812},{\"x\":67.81864929199219,\"y\":114.45559692382812},{\"x\":67.91155242919922,\"y\":114.45559692382812},{\"x\":68.00445556640625,\"y\":114.45559692382812},{\"x\":68.09735870361328,\"y\":114.45559692382812},{\"x\":68.19026184082031,\"y\":114.45559692382812},{\"x\":68.28316497802734,\"y\":114.45559692382812},{\"x\":68.37606811523438,\"y\":114.45559692382812},{\"x\":68.4689712524414,\"y\":114.45559692382812},{\"x\":68.56187438964844,\"y\":114.45559692382812},{\"x\":68.65476989746094,\"y\":114.45559692382812},{\"x\":68.74767303466797,\"y\":114.45559692382812},{\"x\":68.840576171875,\"y\":114.45559692382812},{\"x\":68.93347930908203,\"y\":114.45559692382812},{\"x\":69.02638244628906,\"y\":114.45559692382812},{\"x\":69.1192855834961,\"y\":114.45559692382812},{\"x\":69.21218872070312,\"y\":114.45559692382812},{\"x\":69.30509185791016,\"y\":114.45559692382812},{\"x\":69.39799499511719,\"y\":114.45559692382812},{\"x\":69.49089050292969,\"y\":114.45559692382812},{\"x\":69.58379364013672,\"y\":114.45559692382812},{\"x\":69.67669677734375,\"y\":114.45559692382812},{\"x\":69.76959991455078,\"y\":114.45559692382812},{\"x\":69.86250305175781,\"y\":114.45559692382812},{\"x\":69.95540618896484,\"y\":114.45559692382812},{\"x\":70.04830932617188,\"y\":114.45559692382812},{\"x\":70.1412124633789,\"y\":114.45559692382812},{\"x\":70.2341079711914,\"y\":114.45559692382812},{\"x\":70.32701110839844,\"y\":114.45559692382812},{\"x\":70.41991424560547,\"y\":114.45559692382812},{\"x\":70.5128173828125,\"y\":114.45559692382812},{\"x\":70.60572052001953,\"y\":114.45559692382812},{\"x\":70.69862365722656,\"y\":114.45559692382812},{\"x\":70.7915267944336,\"y\":114.45559692382812},{\"x\":70.88442993164062,\"y\":114.45559692382812},{\"x\":70.97732543945312,\"y\":114.45559692382812},{\"x\":71.07022857666016,\"y\":114.45559692382812},{\"x\":71.16313171386719,\"y\":114.45559692382812},{\"x\":71.25603485107422,\"y\":114.45559692382812},{\"x\":71.34893798828125,\"y\":114.45559692382812}]]", + 600, + 382, + 81, + "path", + "cardinal", + 0.5, + 1, + "list", + 0, + 1, + "", + null + ] + }, + { + "id": 44, + "type": "VHS_VideoCombine", + "pos": [ + 2241.138427734375, + 1051.91943359375 + ], + "size": [ + 530, + 650.4000244140625 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 237 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "title": "Trajectory Outputs", + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Trajectory", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun-Trajectory_00002.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + }, + "muted": false + } + } + }, + { + "id": 115, + "type": "VHS_VideoCombine", + "pos": [ + 2819, + 1056 + ], + "size": [ + 530, + 310 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 264 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "title": "Video with Trajectory Outputs", + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Trajectory-Merge", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "EasyAnimate-Trajectory-Merge_00009.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + }, + "muted": false + } + } + }, + { + "id": 112, + "type": "Note", + "pos": [ + -203, + 252 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 128, + "type": "LoadWan2_2FunModel", + "pos": [ + 277.63623046875, + 254.2862548828125 + ], + "size": [ + 363.85333251953125, + 154 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 288 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control", + "Control", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + }, + { + "id": 110, + "type": "Note", + "pos": [ + 836.5111083984375, + 639.8340454101562 + ], + "size": [ + 608.1410522460938, + 188.2682342529297 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Please set the mask height and mask width according to the height and width of the reference image. \nPlease set the video_length of the Spline Editor below to be the same as the video_length of the Sampler above. \nThe nodes are from KJNodes. For more details, please check https://github.com/kijai/ComfyUI-KJNodes/tree/main. \n\n请根据参考图片的高和宽设置mask height和mask width；\n请将下方Spline Editor的video_legnth置的与上方Sampler的video_legnth一样；\n部分节点来自于KJNodes，具体查看https://github.com/kijai/ComfyUI-KJNodes/tree/main；" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 106, + "type": "VHS_VideoCombine", + "pos": [ + 1386.0306396484375, + 42.99613952636719 + ], + "size": [ + 390, + 310 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 293 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "EasyAnimate_00105.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 8 + } + } + } + }, + { + "id": 114, + "type": "ImageMaximumNode", + "pos": [ + 2114.7646484375, + 884.5827026367188 + ], + "size": [ + 210, + 46 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "video_1", + "type": "IMAGE", + "link": 294 + }, + { + "name": "video_2", + "type": "IMAGE", + "link": 262 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 264 + ] + } + ], + "properties": { + "Node name for S&R": "ImageMaximumNode" + }, + "widgets_values": [] + }, + { + "id": 129, + "type": "Wan2_2FunV2VSampler", + "pos": [ + 835.7046508789062, + 41.22541809082031 + ], + "size": [ + 470.3740234375, + 526 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 288 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 289 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 290 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 291 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 292 + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 293, + 294 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6.000000000000001, + 1, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 237, + 95, + 0, + 44, + 0, + "IMAGE" + ], + [ + 249, + 97, + 0, + 95, + 1, + "MASK" + ], + [ + 262, + 95, + 0, + 114, + 1, + "IMAGE" + ], + [ + 264, + 114, + 0, + 115, + 0, + "IMAGE" + ], + [ + 265, + 97, + 1, + 118, + 0, + "STRING" + ], + [ + 266, + 119, + 1, + 118, + 1, + "STRING" + ], + [ + 267, + 118, + 0, + 95, + 0, + "STRING" + ], + [ + 288, + 128, + 0, + 129, + 0, + "FunModels" + ], + [ + 289, + 121, + 0, + 129, + 1, + "STRING_PROMPT" + ], + [ + 290, + 122, + 0, + 129, + 2, + "STRING_PROMPT" + ], + [ + 291, + 95, + 0, + 129, + 4, + "IMAGE" + ], + [ + 292, + 100, + 0, + 129, + 5, + "IMAGE" + ], + [ + 293, + 129, + 0, + 106, + 0, + "IMAGE" + ], + [ + 294, + 129, + 0, + 114, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Model", + "bounding": [ + 189, + 160, + 475, + 269 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 191, + 456, + 475, + 587 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "First Image of Trajectory", + "bounding": [ + 191, + 1068, + 475, + 456 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Generate Control Video", + "bounding": [ + 786, + 841, + 2616, + 1056 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650711, + "offset": [ + 109.96141399088638, + -28.91205721966216 + ] + }, + "frontendVersion": "1.21.3", + "node_versions": { + "comfy-core": "0.3.44", + "ComfyUI-KJNodes": "ff49e1b01f10a14496b08e21bb89b64d2b15f333", + "CogVideoX-Fun": "e935795d7684aaba72be65d2ad5ef580c0526f7a", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_flf.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_flf.json new file mode 100644 index 0000000000000000000000000000000000000000..b690d5068c3a48c89f3a3f0015b2dc5d37456216 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_flf.json @@ -0,0 +1,553 @@ +{ + "id": "7771a2ee-e50b-4606-8122-63139795c002", + "revision": 0, + "last_node_id": 103, + "last_link_id": 69, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 66 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "Note", + "pos": [ + 17, + -35 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": [ + 35.71394348144531, + 536.9210205078125 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1271, + -67 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 64 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 80, + "type": "Note", + "pos": [ + -95.01139068603516, + -334.30706787109375 + ], + "size": [ + 355.636474609375, + 132.4238739013672 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 67 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 87, + "type": "LoadImage", + "pos": [ + 280.5103759765625, + 477.9588623046875 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 68 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "Start Image", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "2.png", + "image" + ] + }, + { + "id": 103, + "type": "LoadImage", + "pos": [ + 689.5659790039062, + 477.3172302246094 + ], + "size": [ + 318.56072998046875, + 326 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 69 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "title": "End Image", + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "070008-d3df70b6-1c49-43d5-a39b-ba357e48c17a-3149685904.png", + "image" + ] + }, + { + "id": 99, + "type": "LoadWan2_2FunModel", + "pos": [ + 290, + -334 + ], + "size": [ + 315, + 154 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 65 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-InP", + "Inpaint", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + }, + { + "id": 101, + "type": "Wan2_2FunInpaintSampler", + "pos": [ + 771, + -67 + ], + "size": [ + 428.4000244140625, + 422 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 65 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 66 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 67 + }, + { + "name": "start_img", + "shape": 7, + "type": "IMAGE", + "link": 68 + }, + { + "name": "end_img", + "shape": 7, + "type": "IMAGE", + "link": 69 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 64 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunInpaintSampler" + }, + "widgets_values": [ + 81, + 640, + 1035488842362745, + "randomize", + 50, + 6, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 64, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 65, + 99, + 0, + 101, + 0, + "FunModels" + ], + [ + 66, + 75, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 67, + 73, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 68, + 87, + 0, + 101, + 3, + "IMAGE" + ], + [ + 69, + 103, + 0, + 101, + 4, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Load Image", + "bounding": [ + 222, + 396, + 875.5350952148438, + 432.9040832519531 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model", + "bounding": [ + 227, + -416, + 469, + 256 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 37.99022188655044, + 404.3950236451732 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "a1b1e5e74589b3902bdf54d942862de0e6d30b08", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.44" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_i2v.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..23a13455f20eecb5999c19e972dccf58161c0aa7 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_i2v.json @@ -0,0 +1,498 @@ +{ + "last_node_id": 102, + "last_link_id": 68, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": -50 + }, + "size": { + "0": 383.54010009765625, + "1": 156.71620178222656 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 66 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "Note", + "pos": { + "0": 17, + "1": -35 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 95, + "type": "Note", + "pos": { + "0": 35.71394348144531, + "1": 536.9210205078125 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload image here\n（你可以在此上传图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1271, + "1": -67 + }, + "size": [ + 390, + 537.4615384615385 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 64, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 80, + "type": "Note", + "pos": { + "0": -95.01139068603516, + "1": -334.30706787109375 + }, + "size": { + "0": 355.636474609375, + "1": 132.4238739013672 + }, + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 87, + "type": "LoadImage", + "pos": { + "0": 308, + "1": 482 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 68 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "2.png", + "image" + ] + }, + { + "id": 73, + "type": "FunTextBox", + "pos": { + "0": 250, + "1": 160 + }, + "size": { + "0": 383.7149963378906, + "1": 183.83506774902344 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 67 + ], + "slot_index": 0, + "shape": 3 + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 99, + "type": "LoadWan2_2FunModel", + "pos": { + "0": 290, + "1": -334 + }, + "size": { + "0": 315, + "1": 154 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 65 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-InP", + "Inpaint", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + }, + { + "id": 101, + "type": "Wan2_2FunInpaintSampler", + "pos": { + "0": 771, + "1": -67 + }, + "size": { + "0": 428.4000244140625, + "1": 378 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 65 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 66 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 67 + }, + { + "name": "start_img", + "type": "IMAGE", + "link": 68, + "shape": 7 + }, + { + "name": "end_img", + "type": "IMAGE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 64 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunInpaintSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "randomize", + 50, + 6, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 64, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 65, + 99, + 0, + 101, + 0, + "FunModels" + ], + [ + 66, + 75, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 67, + 73, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 68, + 87, + 0, + 101, + 3, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Load Image", + "bounding": [ + 222, + 396, + 458, + 436 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 227, + -416, + 469, + 256 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 73.82229219905072, + 565.0461564576733 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "10076a3aee6547541ca9442bf22389bfc2ea7526", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "v0.2.7-3-g8afb97c" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_t2v.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_t2v.json new file mode 100644 index 0000000000000000000000000000000000000000..7e4702a04ef8f443f2c383add218c58ad9bc6e11 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_t2v.json @@ -0,0 +1,411 @@ +{ + "id": "8d9a378f-1cac-4610-8858-351d5982a6ab", + "revision": 0, + "last_node_id": 103, + "last_link_id": 73, + "nodes": [ + { + "id": 75, + "type": "FunTextBox", + "pos": [ + 250, + -50 + ], + "size": [ + 383.54010009765625, + 156.71620178222656 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 72 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "fireworks display over night city. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + ] + }, + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "Note", + "pos": [ + 17, + -35 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 80, + "type": "Note", + "pos": [ + -95.01139068603516, + -334.30706787109375 + ], + "size": [ + 355.636474609375, + 132.4238739013672 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 73, + "type": "FunTextBox", + "pos": [ + 250, + 160 + ], + "size": [ + 383.7149963378906, + 183.83506774902344 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 73 + ] + } + ], + "title": "Negtive Prompt（反向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1257.373291015625, + -146.444091796875 + ], + "size": [ + 390, + 537.4615478515625 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 70 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00008.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 99, + "type": "LoadWan2_2FunModel", + "pos": [ + 290, + -334 + ], + "size": [ + 315, + 154 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 71 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-InP", + "Inpaint", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + }, + { + "id": 103, + "type": "Wan2_2FunT2VSampler", + "pos": [ + 819.128173828125, + -145.90399169921875 + ], + "size": [ + 340.3540954589844, + 430 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 71 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 72 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 73 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 70 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunT2VSampler" + }, + "widgets_values": [ + 81, + 832, + 480, + false, + 43, + "fixed", + 50, + 6, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 70, + 103, + 0, + 17, + 0, + "IMAGE" + ], + [ + 71, + 99, + 0, + 103, + 0, + "FunModels" + ], + [ + 72, + 75, + 0, + 103, + 1, + "STRING_PROMPT" + ], + [ + 73, + 73, + 0, + 103, + 2, + "STRING_PROMPT" + ] + ], + "groups": [ + { + "id": 2, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Load Model", + "bounding": [ + 227, + -416, + 469, + 256 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917354, + "offset": [ + 175.2685031365507, + 463.74646895767313 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "5f2a55692d0834e00d477939b818b9bb63c06535", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control.json new file mode 100644 index 0000000000000000000000000000000000000000..ce3e6547ca4112129e2e4465e63a696e7136a198 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control.json @@ -0,0 +1,576 @@ +{ + "last_node_id": 97, + "last_link_id": 65, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": 15.739953994750977, + "1": 462.38665771484375 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": { + "0": -99, + "1": 197 + }, + "size": { + "0": 326.1556091308594, + "1": 145.20904541015625 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": { + "0": -192, + "1": -293 + }, + "size": { + "0": 427.074951171875, + "1": 143.9142608642578 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": { + "0": 335, + "1": 476 + }, + "size": [ + 252.056640625, + 688.5451388888889 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 64 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 92, + "type": "FunTextBox", + "pos": { + "0": 254, + "1": -46 + }, + "size": { + "0": 380.845703125, + "1": 157.68350219726562 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 62 + ], + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + ] + }, + { + "id": 94, + "type": "FunTextBox", + "pos": { + "0": 258, + "1": 178 + }, + "size": { + "0": 368.5529479980469, + "1": 159.4075927734375 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 63 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1274, + "1": 10 + }, + "size": [ + 390.9534912109375, + 966.9860514322917 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 65, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00004.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 97, + "type": "Wan2_2FunV2VSampler", + "pos": { + "0": 792, + "1": 11 + }, + "size": [ + 428.4000244140625, + 486 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 61 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 62 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 63 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": 64, + "shape": 7 + }, + { + "name": "start_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "end_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "riflex_k", + "type": "RIFLEXT_ARGS", + "link": null, + "shape": 7 + }, + { + "name": "camera_conditions", + "type": "STRING", + "link": null, + "widget": { + "name": "camera_conditions" + }, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 65 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6, + 1.0, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 95, + "type": "LoadWan2_2FunModel", + "pos": { + "0": 275, + "1": -309 + }, + "size": { + "0": 435.2322082519531, + "1": 154 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 61 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control", + "Control", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + } + ], + "links": [ + [ + 61, + 95, + 0, + 97, + 0, + "FunModels" + ], + [ + 62, + 92, + 0, + 97, + 1, + "STRING_PROMPT" + ], + [ + 63, + 94, + 0, + 97, + 2, + "STRING_PROMPT" + ], + [ + 64, + 85, + 0, + 97, + 4, + "IMAGE" + ], + [ + 65, + 97, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 487, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 249.86519634240904, + 438.3739973625246 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "CogVideoX-Fun": "717f0629175ad192927dc51ec95c4376816a4212" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_canny.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_canny.json new file mode 100644 index 0000000000000000000000000000000000000000..07fd76ed553f9c3ccfe8025748edb556d8a97b75 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_canny.json @@ -0,0 +1,702 @@ +{ + "id": "90329caf-a94f-48a6-80d7-d6e167a8b1e3", + "revision": 0, + "last_node_id": 102, + "last_link_id": 78, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.739953994750977, + 462.38665771484375 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": [ + -192, + -293 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 73 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + ] + }, + { + "id": 99, + "type": "VHS_VideoCombine", + "pos": [ + 1094, + 559 + ], + "size": [ + 315, + 851.4834594726562 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 65 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Preprocess-Video", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00001.gif", + "subfolder": "", + "type": "output", + "format": "image/gif", + "frame_rate": 8 + } + } + } + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 74 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 97, + "type": "VideoToCanny", + "pos": [ + 729, + 566 + ], + "size": [ + 315, + 106 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "input_video", + "type": "IMAGE", + "link": 66 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 65, + 75 + ] + } + ], + "properties": { + "Node name for S&R": "VideoToCanny" + }, + "widgets_values": [ + 100, + 200, + 81 + ] + }, + { + "id": 101, + "type": "LoadWan2_2FunModel", + "pos": [ + 306.2298583984375, + -313.5478515625 + ], + "size": [ + 397.9565734863281, + 154 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control", + "Control", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1465.6671142578125, + -174.95152282714844 + ], + "size": [ + 390.9534912109375, + 966.9860229492188 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 77 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00004.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 335, + 476 + ], + "size": [ + 252.056640625, + 409.87884521484375 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 66 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "00000005.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "00000005.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 102, + "type": "Wan2_2FunV2VSampler", + "pos": [ + 848.4100341796875, + -180.23484802246094 + ], + "size": [ + 458.36383056640625, + 526 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 78 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 73 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 74 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 75 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6.000000000000001, + 1, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 65, + 97, + 0, + 99, + 0, + "IMAGE" + ], + [ + 66, + 85, + 0, + 97, + 0, + "IMAGE" + ], + [ + 73, + 92, + 0, + 102, + 1, + "STRING_PROMPT" + ], + [ + 74, + 94, + 0, + 102, + 2, + "STRING_PROMPT" + ], + [ + 75, + 97, + 0, + 102, + 4, + "IMAGE" + ], + [ + 77, + 102, + 0, + 17, + 0, + "IMAGE" + ], + [ + 78, + 101, + 0, + 102, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 487, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 192.62803696741028, + 413.1926211906501 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "e935795d7684aaba72be65d2ad5ef580c0526f7a", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_depth.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_depth.json new file mode 100644 index 0000000000000000000000000000000000000000..2df29d718392a953a0d66646305fc876c1f5d875 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_depth.json @@ -0,0 +1,699 @@ +{ + "id": "90329caf-a94f-48a6-80d7-d6e167a8b1e3", + "revision": 0, + "last_node_id": 103, + "last_link_id": 83, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.739953994750977, + 462.38665771484375 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": [ + -192, + -293 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 73 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + ] + }, + { + "id": 99, + "type": "VHS_VideoCombine", + "pos": [ + 1094, + 559 + ], + "size": [ + 315, + 851.4834594726562 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 81 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Preprocess-Video", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "AnimateDiff_00001.gif", + "subfolder": "", + "type": "output", + "format": "image/gif", + "frame_rate": 8 + } + } + } + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 74 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 101, + "type": "LoadWan2_2FunModel", + "pos": [ + 306.2298583984375, + -313.5478515625 + ], + "size": [ + 397.9565734863281, + 154 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 78 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control", + "Control", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1465.6671142578125, + -174.95152282714844 + ], + "size": [ + 390.9534912109375, + 966.9860229492188 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 77 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00004.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 335, + 476 + ], + "size": [ + 252.056640625, + 409.87884521484375 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 82 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "00000005.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "00000005.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 102, + "type": "Wan2_2FunV2VSampler", + "pos": [ + 848.4100341796875, + -180.23484802246094 + ], + "size": [ + 458.36383056640625, + 526 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 78 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 73 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 74 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 83 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6.000000000000001, + 1, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + }, + { + "id": 103, + "type": "VideoToDepth", + "pos": [ + 734.0958862304688, + 474.7727355957031 + ], + "size": [ + 270, + 58 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "input_video", + "type": "IMAGE", + "link": 82 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 81, + 83 + ] + } + ], + "properties": { + "Node name for S&R": "VideoToDepth" + }, + "widgets_values": [ + 81 + ] + } + ], + "links": [ + [ + 73, + 92, + 0, + 102, + 1, + "STRING_PROMPT" + ], + [ + 74, + 94, + 0, + 102, + 2, + "STRING_PROMPT" + ], + [ + 77, + 102, + 0, + 17, + 0, + "IMAGE" + ], + [ + 78, + 101, + 0, + 102, + 0, + "FunModels" + ], + [ + 81, + 103, + 0, + 99, + 0, + "IMAGE" + ], + [ + 82, + 85, + 0, + 103, + 0, + "IMAGE" + ], + [ + 83, + 103, + 0, + 102, + 4, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Video", + "bounding": [ + 218, + 385, + 487, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6830134553650709, + "offset": [ + 192.62803696741028, + 413.1926211906501 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "e935795d7684aaba72be65d2ad5ef580c0526f7a", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_pose_ref.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_pose_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..c05790cb802e91495b2daa88355bbfc52fe68c9e --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_pose_ref.json @@ -0,0 +1,755 @@ +{ + "id": "c5a53a91-88dc-41d4-b414-12e402c59e4c", + "revision": 0, + "last_node_id": 105, + "last_link_id": 91, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + -111.46612548828125, + 460.2178955078125 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": [ + -192, + -293 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 99, + "type": "VHS_VideoCombine", + "pos": [ + 1005, + 554 + ], + "size": [ + 315, + 849.46875 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 73 + }, + { + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun-Preprocess-Video", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 19, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun-Preprocess-Video_00007.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 207.79391479492188, + 473.83123779296875 + ], + "size": [ + 252.056640625, + 688.545166015625 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 72 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "000007.mp4", + "force_rate": 16, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 16, + "filename": "000007.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1488, + 8 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 84 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00049.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 102, + "type": "LoadImage", + "pos": [ + 553, + 598 + ], + "size": [ + 315, + 314 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 86 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 101, + "type": "VideoToOpenpose", + "pos": [ + 558, + 474 + ], + "size": [ + 315, + 58 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "input_video", + "type": "IMAGE", + "link": 72 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 73, + 87, + 91 + ] + } + ], + "properties": { + "Node name for S&R": "VideoToOpenpose" + }, + "widgets_values": [ + 81 + ] + }, + { + "id": 104, + "type": "LoadWan2_2FunModel", + "pos": [ + 303.9479064941406, + -309.996337890625 + ], + "size": [ + 390.1385192871094, + 154 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 88 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control", + "Control", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 89 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位动漫风格的女孩。她有着紫色的短发，头上戴着一个黑色和金色相间的蝴蝶结。她的表情显得有些严肃或沉思，眼睛大而有神。女孩穿着一件白色衬衫，外面搭配了一件深蓝色的背心，背心上有一个粉色的蝴蝶结装饰。她的裙子是白色的，裙摆蓬松，整体造型非常可爱且精致。背景是一个简单的圆形图案，颜色为粉红色和灰色相间，给人一种柔和的感觉。整个画面色调柔和，人物形象生动鲜明。" + ] + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 105, + "type": "Wan2_2FunV2VSampler", + "pos": [ + 987.7456665039062, + -76.86135864257812 + ], + "size": [ + 434.938232421875, + 526 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 88 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 89 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 90 + }, + { + "name": "validation_video", + "shape": 7, + "type": "IMAGE", + "link": 91 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 87 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "ref_image", + "shape": 7, + "type": "IMAGE", + "link": 86 + }, + { + "name": "camera_conditions", + "shape": 7, + "type": "STRING", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 84 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6.000000000000001, + 1, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 72, + 85, + 0, + 101, + 0, + "IMAGE" + ], + [ + 73, + 101, + 0, + 99, + 0, + "IMAGE" + ], + [ + 84, + 105, + 0, + 17, + 0, + "IMAGE" + ], + [ + 86, + 102, + 0, + 105, + 7, + "IMAGE" + ], + [ + 87, + 101, + 0, + 105, + 4, + "IMAGE" + ], + [ + 88, + 104, + 0, + 105, + 0, + "FunModels" + ], + [ + 89, + 92, + 0, + 105, + 1, + "STRING_PROMPT" + ], + [ + 90, + 94, + 0, + 105, + 2, + "STRING_PROMPT" + ], + [ + 91, + 101, + 0, + 105, + 3, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Video And Reference Image", + "bounding": [ + 91, + 383, + 859, + 841 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.7513148009015781, + "offset": [ + 235.41928184428536, + 445.7655228500244 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.44", + "CogVideoX-Fun": "e935795d7684aaba72be65d2ad5ef580c0526f7a" + }, + "frontendVersion": "1.21.3" + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_ref.json b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..600fadb8c4c0b5d58f3fe7fd660fe0329308634c --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_fun/v1/wan2.2_fun_workflow_v2v_control_ref.json @@ -0,0 +1,622 @@ +{ + "last_node_id": 98, + "last_link_id": 67, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": { + "0": 18, + "1": -46 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": { + "0": 15.145149230957031, + "1": 527.0673217773438 + }, + "size": { + "0": 210, + "1": 58 + }, + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": { + "0": -99, + "1": 197 + }, + "size": { + "0": 326.1556091308594, + "1": 145.20904541015625 + }, + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": { + "0": -192, + "1": -293 + }, + "size": { + "0": 427.074951171875, + "1": 143.9142608642578 + }, + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": { + "0": 334.4051513671875, + "1": 540.6806640625 + }, + "size": [ + 252.056640625, + 688.5451388888889 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 65 + ], + "slot_index": 0, + "shape": 3 + }, + { + "name": "frame_count", + "type": "INT", + "links": null, + "shape": 3 + }, + { + "name": "audio", + "type": "AUDIO", + "links": null, + "shape": 3 + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null, + "shape": 3 + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 94, + "type": "FunTextBox", + "pos": { + "0": 258, + "1": 178 + }, + "size": { + "0": 368.5529479980469, + "1": 159.4075927734375 + }, + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 64 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": { + "0": 1274, + "1": -77 + }, + "size": [ + 390.9534912109375, + 942.2558186848959 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "images", + "type": "IMAGE", + "link": 67, + "slot_index": 0, + "label": "图像", + "shape": 7 + }, + { + "name": "audio", + "type": "AUDIO", + "link": null, + "label": "音频", + "shape": 7 + }, + { + "name": "meta_batch", + "type": "VHS_BatchManager", + "link": null, + "label": "批次管理", + "shape": 7 + }, + { + "name": "vae", + "type": "VAE", + "link": null, + "shape": 7 + } + ], + "outputs": [ + { + "name": "Filenames", + "type": "VHS_FILENAMES", + "links": null, + "slot_index": 0, + "shape": 3, + "label": "文件名" + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00051.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 92, + "type": "FunTextBox", + "pos": { + "0": 254, + "1": -46 + }, + "size": { + "0": 380.845703125, + "1": 157.68350219726562 + }, + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "links": [ + 63 + ], + "slot_index": 0 + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位动漫风格的女孩。她有着紫色的短发，头上戴着一个黑色和金色相间的蝴蝶结。她的表情显得有些严肃或沉思，眼睛大而有神。女孩穿着一件白色衬衫，外面搭配了一件深蓝色的背心，背心上有一个粉色的蝴蝶结装饰。她的裙子是白色的，裙摆蓬松，整体造型非常可爱且精致。背景是一个简单的圆形图案，颜色为粉红色和灰色相间，给人一种柔和的感觉。整个画面色调柔和，人物形象生动鲜明。" + ] + }, + { + "id": 97, + "type": "LoadImage", + "pos": { + "0": 629, + "1": 553 + }, + "size": { + "0": 315, + "1": 314 + }, + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 66 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 95, + "type": "LoadWan2_2FunModel", + "pos": { + "0": 275, + "1": -309 + }, + "size": { + "0": 435.2322082519531, + "1": 154 + }, + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 62 + ], + "slot_index": 0 + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunModel" + }, + "widgets_values": [ + "Wan2.2-Fun-A14B-Control", + "Control", + "sequential_cpu_offload", + "wan2.2/wan_civitai_i2v.yaml", + "bf16" + ] + }, + { + "id": 98, + "type": "Wan2_2FunV2VSampler", + "pos": { + "0": 798, + "1": -77 + }, + "size": [ + 428.4000244140625, + 486 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 62 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 63 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 64 + }, + { + "name": "validation_video", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "control_video", + "type": "IMAGE", + "link": 65, + "shape": 7 + }, + { + "name": "start_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "end_image", + "type": "IMAGE", + "link": null, + "shape": 7 + }, + { + "name": "ref_image", + "type": "IMAGE", + "link": 66, + "shape": 7 + }, + { + "name": "riflex_k", + "type": "RIFLEXT_ARGS", + "link": null, + "shape": 7 + }, + { + "name": "camera_conditions", + "type": "STRING", + "link": null, + "widget": { + "name": "camera_conditions" + }, + "shape": 7 + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 67 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2FunV2VSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "fixed", + 50, + 6, + 1.0, + "Flow", + 5, + 0.9, + 0.1, + true, + 3, + true, + 0.25000000000000006 + ] + } + ], + "links": [ + [ + 62, + 95, + 0, + 98, + 0, + "FunModels" + ], + [ + 63, + 92, + 0, + 98, + 1, + "STRING_PROMPT" + ], + [ + 64, + 94, + 0, + 98, + 2, + "STRING_PROMPT" + ], + [ + 65, + 85, + 0, + 98, + 4, + "IMAGE" + ], + [ + 66, + 97, + 0, + 98, + 6, + "IMAGE" + ], + [ + 67, + 98, + 0, + 17, + 0, + "IMAGE" + ] + ], + "groups": [ + { + "title": "Upload Your Video", + "bounding": [ + 217, + 450, + 761, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "title": "Load Model", + "bounding": [ + 218, + -387, + 542, + 248 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.8264462809917358, + "offset": [ + 23.894387748659057, + 254.96144111252457 + ] + }, + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "CogVideoX-Fun": "a7fa7028d52498f13e983eba012a81ebcae24977", + "comfy-core": "v0.2.7-3-g8afb97c" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/nodes.py b/VideoX-Fun/comfyui/wan2_2_vace_fun/nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..6e7af9304b69bc372c29323951bb72da372bab0a --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/nodes.py @@ -0,0 +1,635 @@ +"""Modified from https://github.com/kijai/ComfyUI-EasyAnimateWrapper/blob/main/nodes.py +""" +import copy +import gc +import inspect +import json +import os + +import comfy.model_management as mm +import cv2 +import folder_paths +import numpy as np +import torch +from comfy.utils import ProgressBar, load_torch_file +from diffusers import FlowMatchEulerDiscreteScheduler +from einops import rearrange +from omegaconf import OmegaConf +from PIL import Image + +from ...videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + get_closest_ratio) +from ...videox_fun.data.dataset_image_video import process_pose_params +from ...videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + VaceWanTransformer3DModel, WanT5EncoderModel) +from ...videox_fun.models.cache_utils import get_teacache_coefficients +from ...videox_fun.pipeline import Wan2_2VaceFunPipeline +from ...videox_fun.utils.fp8_optimization import ( + convert_model_weight_to_float8, convert_weight_dtype_wrapper, undo_convert_weight_dtype_wrapper, + replace_parameters_by_name) +from ...videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from ...videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, + get_video_to_video_latent, get_autocast_dtype) +from ..comfyui_utils import (script_directory, + search_model_in_possible_folders, to_pil) +from ..wan2_1.nodes import get_wan_scheduler + +# Used in lora cache +transformer_cpu_cache = {} +transformer_high_cpu_cache = {} +# lora path before +lora_path_before = "" +lora_high_path_before = "" + +class LoadVaceWanTransformer3DModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model_name": ( + folder_paths.get_filename_list("diffusion_models"), + {"default": "Wan2_1-T2V-1_3B_bf16.safetensors,"}, + ), + "precision": (["fp16", "bf16"], + {"default": "bf16"} + ), + }, + } + RETURN_TYPES = ("TransformerModel", "STRING") + RETURN_NAMES = ("transformer", "model_name") + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, precision): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + transformer = None + + model_path = folder_paths.get_full_path("diffusion_models", model_name) + transformer_state_dict = load_torch_file(model_path, safe_load=True) + + eps = 1e-6 + text_len = 512 + freq_dim = 256 + dim = transformer_state_dict["patch_embedding.weight"].shape[0] + hidden_size = dim + in_dim = transformer_state_dict["patch_embedding.weight"].shape[1] + in_channels = in_dim + ffn_dim = transformer_state_dict["blocks.0.ffn.0.bias"].shape[0] + vace_layers = [0, 5, 10, 15, 20, 25, 30, 35] + vace_in_dim = 96 + + add_ref_conv = True if "ref_conv.weight" in transformer_state_dict else False + in_dim_ref_conv = transformer_state_dict["ref_conv.weight"].shape[1] if "ref_conv.weight" in transformer_state_dict else None + add_control_adapter = True if "control_adapter.conv.weight" in transformer_state_dict else False + in_dim_control_adapter = transformer_state_dict["control_adapter.conv.weight"].shape[1] if "control_adapter.conv.weight" in transformer_state_dict else None + + if dim == 5120: + num_heads = 40 + num_layers = 40 + out_dim = 16 + downscale_factor_control_adapter = 8 + model_name_in_pipeline = "wan2.2-vace-fun-a14b" + + elif dim == 3072: + num_heads = 24 + num_layers = 30 + out_dim = 48 + downscale_factor_control_adapter = 16 + model_name_in_pipeline = "wan2.2-vace-fun-5b" + else: + num_heads = 12 + num_layers = 30 + out_dim = 16 + downscale_factor_control_adapter = 8 + model_name_in_pipeline = "wan2.1-vace-1.3b" + + model_type = "t2v" + + kwargs = dict( + dim = dim, + in_dim = in_dim, + eps = eps, + ffn_dim = ffn_dim, + freq_dim = freq_dim, + model_type = model_type, + num_heads = num_heads, + num_layers = num_layers, + out_dim = out_dim, + text_len = text_len, + in_channels = in_channels, + hidden_size = hidden_size, + add_control_adapter = add_control_adapter, + add_ref_conv = add_ref_conv, + in_dim_control_adapter = in_dim_control_adapter // downscale_factor_control_adapter // downscale_factor_control_adapter if in_dim_control_adapter is not None else in_dim_control_adapter, + in_dim_ref_conv = in_dim_ref_conv, + downscale_factor_control_adapter = downscale_factor_control_adapter, + vace_layers = vace_layers, + vace_in_dim = vace_in_dim + ) + + sig = inspect.signature(VaceWanTransformer3DModel) + accepted = {k: v for k, v in kwargs.items() if k in sig.parameters} + transformer = VaceWanTransformer3DModel(**accepted) + transformer.load_state_dict(transformer_state_dict) + transformer = transformer.eval().to(device=offload_device, dtype=weight_dtype) + return (transformer, model_name_in_pipeline) + +class CombineWan2_2VaceFunPipeline: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "transformer": ("TransformerModel",), + "vae": ("VAEModel",), + "text_encoder": ("TextEncoderModel",), + "tokenizer": ("Tokenizer",), + "model_name": ("STRING",), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8","model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + }, + "optional":{ + "clip_encoder": ("ClipEncoderModel",), + "transformer_2": ("TransformerModel",), + }, + } + + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, model_name, GPU_memory_mode, transformer, vae, text_encoder, tokenizer, clip_encoder=None, transformer_2=None, model_type="Control"): + weight_dtype = transformer.dtype if transformer.dtype not in [torch.float32, torch.float8_e4m3fn, torch.float8_e5m2] else get_autocast_dtype() + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + # Get pipeline + pipeline = Wan2_2VaceFunPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + transformer_2=transformer_2, + scheduler=None, + ) + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + pipeline.to(device=offload_device) + transformer = transformer.to(weight_dtype) + + if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",]) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload() + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",]) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [] + } + return (funmodels,) + + +class LoadWan2_2VaceFunModel: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "model": ( + [ + 'Wan2.2-VACE-Fun-A14B', + ], + { + "default": 'Wan2.2-VACE-Fun-A14B', + } + ), + "GPU_memory_mode":( + ["model_full_load", "model_full_load_and_qfloat8", "model_cpu_offload", "model_cpu_offload_and_qfloat8", "sequential_cpu_offload"], + { + "default": "model_cpu_offload", + } + ), + "config": ( + [ + "wan2.2/wan_civitai_t2v.yaml", + ], + { + "default": "wan2.2/wan_civitai_t2v.yaml", + } + ), + "precision": ( + ['fp16', 'bf16'], + { + "default": 'bf16' + } + ), + }, + } + RETURN_TYPES = ("FunModels",) + RETURN_NAMES = ("funmodels",) + FUNCTION = "loadmodel" + CATEGORY = "CogVideoXFUNWrapper" + + def loadmodel(self, GPU_memory_mode, model, precision, config, model_type="Control"): + # Init weight_dtype and device + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + weight_dtype = {"bf16": torch.bfloat16, "fp16": torch.float16, "fp32": torch.float32}[precision] + + mm.unload_all_models() + mm.cleanup_models() + mm.soft_empty_cache() + + # Init processbar + pbar = ProgressBar(5) + + # Load config + config_path = f"{script_directory}/config/{config}" + config = OmegaConf.load(config_path) + + # Detect model is existing or not + possible_folders = ["CogVideoX_Fun", "Fun_Models", "VideoX_Fun"] + \ + [os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "models/Diffusion_Transformer")] # Possible folder names to check + # Initialize model_name as None + model_name = search_model_in_possible_folders(possible_folders, model) + + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ).to(weight_dtype) + # Update pbar + pbar.update(1) + + # Load Sampler + print("Load Sampler.") + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + # Update pbar + pbar.update(1) + + # Get Transformer + transformer = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + + if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + else: + transformer_2 = None + # Update pbar + pbar.update(1) + + # Get tokenizer and text_encoder + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + pbar.update(1) + + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + pbar.update(1) + + # Get pipeline + pipeline = Wan2_2VaceFunPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + transformer_2=transformer_2, + scheduler=scheduler, + ) + + pipeline.remove_all_hooks() + undo_convert_weight_dtype_wrapper(transformer) + + if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) + elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) + else: + pipeline.to(device=device) + + funmodels = { + 'pipeline': pipeline, + 'dtype': weight_dtype, + 'model_name': model_name, + 'model_type': model_type, + 'loras': [], + 'strength_model': [], + 'config': config, + } + return (funmodels,) + + +class Wan2_2VaceFunSampler: + @classmethod + def INPUT_TYPES(s): + return { + "required": { + "funmodels": ( + "FunModels", + ), + "prompt": ( + "STRING_PROMPT", + ), + "negative_prompt": ( + "STRING_PROMPT", + ), + "video_length": ( + "INT", {"default": 81, "min": 1, "max": 161, "step": 4} + ), + "base_resolution": ( + [ + 512, + 640, + 768, + 896, + 960, + 1024, + ], {"default": 640} + ), + "seed": ( + "INT", {"default": 43, "min": 0, "max": 0xffffffffffffffff} + ), + "steps": ( + "INT", {"default": 50, "min": 1, "max": 200, "step": 1} + ), + "cfg": ( + "FLOAT", {"default": 6.0, "min": 1.0, "max": 20.0, "step": 0.01} + ), + "scheduler": ( + ["Flow", "Flow_Unipc", "Flow_DPM++"], + { + "default": 'Flow' + } + ), + "shift": ( + "INT", {"default": 12, "min": 1, "max": 100, "step": 1} + ), + "boundary": ( + "FLOAT", {"default": 0.875, "min": 0.00, "max": 1.00, "step": 0.001} + ), + "teacache_threshold": ( + "FLOAT", {"default": 0.10, "min": 0.00, "max": 1.00, "step": 0.005} + ), + "enable_teacache":( + [False, True], {"default": True,} + ), + "num_skip_start_steps": ( + "INT", {"default": 5, "min": 0, "max": 50, "step": 1} + ), + "teacache_offload":( + [False, True], {"default": True,} + ), + "cfg_skip_ratio":( + "FLOAT", {"default": 0, "min": 0, "max": 1, "step": 0.01} + ), + "padding_in_subject_ref_images":( + [False, True], {"default": True,} + ), + }, + "optional": { + "control_video": ("IMAGE",), + "start_image": ("IMAGE",), + "end_image": ("IMAGE",), + "subject_ref_images": ("IMAGE",), + "riflex_k": ("RIFLEXT_ARGS",), + }, + } + + RETURN_TYPES = ("IMAGE",) + RETURN_NAMES =("images",) + FUNCTION = "process" + CATEGORY = "CogVideoXFUNWrapper" + + def process(self, funmodels, prompt, negative_prompt, video_length, base_resolution, seed, steps, cfg, scheduler, shift, boundary, teacache_threshold, enable_teacache, num_skip_start_steps, teacache_offload, cfg_skip_ratio, padding_in_subject_ref_images, control_video=None, start_image=None, end_image=None, subject_ref_images=None, riflex_k=0): + global transformer_cpu_cache + global transformer_high_cpu_cache + global lora_path_before + global lora_high_path_before + + device = mm.get_torch_device() + offload_device = mm.unet_offload_device() + + mm.soft_empty_cache() + gc.collect() + + # Get Pipeline + pipeline = funmodels['pipeline'] + model_name = funmodels['model_name'] + weight_dtype = funmodels['dtype'] + model_type = funmodels['model_type'] + + # Count most suitable height and width + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + + if control_video is not None and type(control_video) is str: + original_width, original_height = Image.fromarray(cv2.VideoCapture(control_video).read()[1]).size + elif control_video is not None: + control_video = np.array(control_video.cpu().numpy() * 255, np.uint8) + original_width, original_height = Image.fromarray(control_video[0]).size + else: + original_width, original_height = 384 / 512 * base_resolution, 672 / 512 * base_resolution + + if subject_ref_images is not None: + subject_ref_images = [to_pil(_subject_ref_images) for _subject_ref_images in subject_ref_images] + original_width, original_height = subject_ref_images[0].size if type(subject_ref_images) is list else Image.open(subject_ref_images).size + + if start_image is not None: + start_image = [to_pil(_start_image) for _start_image in start_image] + original_width, original_height = start_image[0].size if type(start_image) is list else Image.open(start_image).size + + if end_image is not None: + end_image = [to_pil(_end_image) for _end_image in end_image] + + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height, width = [int(x / 16) * 16 for x in closest_size] + + # Load Sampler + pipeline.scheduler = get_wan_scheduler(scheduler, shift) + coefficients = get_teacache_coefficients(model_name) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + else: + pipeline.transformer.disable_teacache() + if pipeline.transformer_2 is not None: + pipeline.transformer_2.disable_teacache() + + if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, steps) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + + generator= torch.Generator(device).manual_seed(seed) + + with torch.no_grad(): + video_length = int((video_length - 1) // pipeline.vae.config.temporal_compression_ratio * pipeline.vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + + if riflex_k > 0: + latent_frames = (video_length - 1) // pipeline.vae.config.temporal_compression_ratio + 1 + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if pipeline.transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + inpaint_video, inpaint_video_mask, _ = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=(height, width)) + + if subject_ref_images is not None: + subject_ref_images = [get_image_latent(_subject_ref_image, sample_size=(height, width), padding=padding_in_subject_ref_images) for _subject_ref_image in subject_ref_images] + subject_ref_images = torch.cat(subject_ref_images, dim=2) + + control_video, control_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=(height, width), fps=16, ref_image=None) + + # Apply lora + if funmodels.get("lora_cache", False): + if len(funmodels.get("loras", [])) != 0: + # Save the original weights to cpu + if len(transformer_cpu_cache) == 0: + print('Save transformer state_dict to cpu memory') + transformer_state_dict = pipeline.transformer.state_dict() + for key in transformer_state_dict: + transformer_cpu_cache[key] = transformer_state_dict[key].clone().cpu() + + lora_path_now = str(funmodels.get("loras", []) + funmodels.get("strength_model", [])) + if lora_path_now != lora_path_before: + print('Merge Lora with Cache') + lora_path_before = copy.deepcopy(lora_path_now) + pipeline.transformer.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + if pipeline.transformer_2 is not None: + # Save the original weights to cpu + if len(transformer_high_cpu_cache) == 0: + print('Save transformer high state_dict to cpu memory') + transformer_high_state_dict = pipeline.transformer_2.state_dict() + for key in transformer_high_state_dict: + transformer_high_cpu_cache[key] = transformer_high_state_dict[key].clone().cpu() + + lora_high_path_now = str(funmodels.get("loras_high", []) + funmodels.get("strength_model", [])) + if lora_high_path_now != lora_high_path_before: + print('Merge Lora High with Cache') + lora_high_path_before = copy.deepcopy(lora_high_path_now) + pipeline.transformer_2.load_state_dict(transformer_cpu_cache) + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + else: + print('Merge Lora') + # Clear lora when switch from lora_cache=True to lora_cache=False. + if len(transformer_cpu_cache) != 0: + pipeline.transformer.load_state_dict(transformer_cpu_cache) + transformer_cpu_cache = {} + lora_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + + # Clear lora when switch from lora_cache=True to lora_cache=False. + if pipeline.transformer_2 is not None: + if len(transformer_high_cpu_cache) != 0: + pipeline.transformer_2.load_state_dict(transformer_high_cpu_cache) + transformer_high_cpu_cache = {} + lora_high_path_before = "" + gc.collect() + + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = merge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = height, + width = width, + generator = generator, + guidance_scale = cfg, + num_inference_steps = steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = subject_ref_images, + boundary = boundary, + comfyui_progressbar = True, + ).videos + videos = rearrange(sample, "b c t h w -> (b t) h w c") + + if not funmodels.get("lora_cache", False): + print('Unmerge Lora') + for _lora_path, _lora_weight in zip(funmodels.get("loras", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype) + if pipeline.transformer_2 is not None: + for _lora_path, _lora_weight in zip(funmodels.get("loras_high", []), funmodels.get("strength_model", [])): + pipeline = unmerge_lora(pipeline, _lora_path, _lora_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + return (videos,) \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_i2v.json b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..2f6e2f0a80ff618d0e9f4f471990d16b7be70910 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_i2v.json @@ -0,0 +1,750 @@ +{ + "id": "71b53971-2ab9-41eb-a405-78c174d97672", + "revision": 0, + "last_node_id": 109, + "last_link_id": 91, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 80 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一只棕色的狗舔了一下它的舌头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1179.8734130859375, + -215.58523559570312 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 83 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00143.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.145149230957031, + 527.0673217773438 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload start image here\n（在此开始图像）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 97, + "type": "LoadImage", + "pos": [ + 290.19805908203125, + 480.74139404296875 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 102, + "type": "Wan2_2VaceFunSampler", + "pos": [ + 842.1512451171875, + -215.00350952148438 + ], + "size": [ + 280.724609375, + 534 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 91 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 80 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 81 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 82 + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "subject_ref_images", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 83 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2VaceFunSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "randomize", + 50, + 6, + "Flow", + 12, + 0.875, + 0.1, + true, + 3, + true, + 0.25000000000000006, + true + ] + }, + { + "id": 104, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 699.5308837890625, + -714.7778930664062 + ], + "size": [ + 354.0464782714844, + 102 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 89 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 106, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 245.1029510498047, + -717.7963256835938 + ], + "size": [ + 408.1667785644531, + 102 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 85 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 107, + "type": "LoadWanVAEModel", + "pos": [ + 247.4856414794922, + -555.1431884765625 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 86 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 108, + "type": "LoadWanTextEncoderModel", + "pos": [ + 248.16152954101562, + -406.9620361328125 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 87 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 88 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 105, + "type": "CombineWan2_2VaceFunPipeline", + "pos": [ + 700.357421875, + -538.5968627929688 + ], + "size": [ + 361.8251953125, + 182 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 85 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 86 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 87 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 88 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 89 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 90 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 91 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2VaceFunPipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload" + ] + }, + { + "id": 109, + "type": "Note", + "pos": [ + -222.97146606445312, + -434.9617004394531 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 80, + 92, + 0, + 102, + 1, + "STRING_PROMPT" + ], + [ + 81, + 94, + 0, + 102, + 2, + "STRING_PROMPT" + ], + [ + 82, + 97, + 0, + 102, + 4, + "IMAGE" + ], + [ + 83, + 102, + 0, + 17, + 0, + "IMAGE" + ], + [ + 85, + 106, + 0, + 105, + 0, + "TransformerModel" + ], + [ + 86, + 107, + 0, + 105, + 1, + "VAEModel" + ], + [ + 87, + 108, + 0, + 105, + 2, + "TextEncoderModel" + ], + [ + 88, + 108, + 1, + 105, + 3, + "Tokenizer" + ], + [ + 89, + 104, + 0, + 105, + 5, + "TransformerModel" + ], + [ + 90, + 106, + 1, + 105, + 6, + "STRING" + ], + [ + 91, + 105, + 0, + 102, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Start Image", + "bounding": [ + 218.27899169921875, + 391.4151916503906, + 449.0052795410156, + 420.1466369628906 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 227.10231018066406, + -793.4209594726562, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.751314800901578, + "offset": [ + 421.5189072799089, + 1243.2841129875237 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "a4d5cca5c80bb811e1ee35915ad1e096ca62a3bf", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_subjects.json b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_subjects.json new file mode 100644 index 0000000000000000000000000000000000000000..d47b5e9bdf7552b7ce41751145d54edb76fd5b7c --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_subjects.json @@ -0,0 +1,844 @@ +{ + "id": "71b53971-2ab9-41eb-a405-78c174d97672", + "revision": 0, + "last_node_id": 110, + "last_link_id": 91, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1179.8734130859375, + -215.58523559570312 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 80 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00143.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 79, + "type": "Note", + "pos": [ + -215.824951171875, + 490.2828369140625 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload ref images here\n（在此上传参考图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 104, + "type": "ImageCollectNode", + "pos": [ + 835.30908203125, + 491.93438720703125 + ], + "size": [ + 162.85116577148438, + 46 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 83 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": 82 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 84 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCollectNode" + }, + "widgets_values": [] + }, + { + "id": 103, + "type": "LoadImage", + "pos": [ + 409.56060791015625, + 457.4063415527344 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "ref_1.png", + "image" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "海风作曲，浪花打拍。她握着一台亮黄色相机，双臂如翼般舒展流转。双手托举相机，在胸前划出轻柔的圆弧，时而高举过肩，迎向天际的光晕，时而缓缓贴近心口，倾听快门低语般的节奏。紫色长发在风中扬起，拂过肩头，仿佛与镜头共舞。她的目光随取景框游走，每一次微倾与回旋，都让镜头掠过海天相接的边际、摇曳的花丛。最后，呼吸一凝，稳稳对准水中倒影，轻轻按下快门——将风、光与心动，悄然收藏。" + ] + }, + { + "id": 97, + "type": "LoadImage", + "pos": [ + 51.6423454284668, + 459.73138427734375 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 83 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "8.png", + "image" + ] + }, + { + "id": 101, + "type": "Wan2_2VaceFunSampler", + "pos": [ + 842.1512451171875, + -215.00350952148438 + ], + "size": [ + 280.724609375, + 534 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 91 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "subject_ref_images", + "shape": 7, + "type": "IMAGE", + "link": 84 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2VaceFunSampler" + }, + "widgets_values": [ + 81, + 640, + 360428861626678, + "randomize", + 50, + 6, + "Flow", + 12, + 0.875, + 0.1, + true, + 3, + true, + 0.25000000000000006, + true + ] + }, + { + "id": 105, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 238.0269317626953, + -745.763671875 + ], + "size": [ + 408.1667785644531, + 102 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 85 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 106, + "type": "LoadWanVAEModel", + "pos": [ + 240.40968322753906, + -583.1104736328125 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 86 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 107, + "type": "LoadWanTextEncoderModel", + "pos": [ + 241.08558654785156, + -434.9287109375 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 87 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 88 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 109, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 692.4532470703125, + -742.7451782226562 + ], + "size": [ + 354.0464782714844, + 102 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 89 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 108, + "type": "CombineWan2_2VaceFunPipeline", + "pos": [ + 693.27978515625, + -566.5641479492188 + ], + "size": [ + 361.8251953125, + 182 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 85 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 86 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 87 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 88 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 89 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 90 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 91 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2VaceFunPipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload" + ] + }, + { + "id": 110, + "type": "Note", + "pos": [ + -220.1790008544922, + -468.36676025390625 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + } + ], + "links": [ + [ + 76, + 92, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 77, + 94, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 80, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 82, + 103, + 0, + 104, + 1, + "IMAGE" + ], + [ + 83, + 97, + 0, + 104, + 0, + "IMAGE" + ], + [ + 84, + 104, + 0, + 101, + 6, + "IMAGE" + ], + [ + 85, + 105, + 0, + 108, + 0, + "TransformerModel" + ], + [ + 86, + 106, + 0, + 108, + 1, + "VAEModel" + ], + [ + 87, + 107, + 0, + 108, + 2, + "TextEncoderModel" + ], + [ + 88, + 107, + 1, + 108, + 3, + "Tokenizer" + ], + [ + 89, + 109, + 0, + 108, + 5, + "TransformerModel" + ], + [ + 90, + 105, + 1, + 108, + 6, + "STRING" + ], + [ + 91, + 108, + 0, + 101, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Ref Images", + "bounding": [ + -0.4573218524456024, + 378.5783386230469, + 1071.616943359375, + 451.41473388671875 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 220.02639770507812, + -821.3883056640625, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.751314800901578, + "offset": [ + 342.74554399865866, + 838.8420856437743 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "a4d5cca5c80bb811e1ee35915ad1e096ca62a3bf", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_subjects_4steps_lora.json b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_subjects_4steps_lora.json new file mode 100644 index 0000000000000000000000000000000000000000..fdfa5a4bdecb31a6bda9d56df78d8665e885b91b --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_subjects_4steps_lora.json @@ -0,0 +1,892 @@ +{ + "id": "71b53971-2ab9-41eb-a405-78c174d97672", + "revision": 0, + "last_node_id": 111, + "last_link_id": 93, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1179.8734130859375, + -215.58523559570312 + ], + "size": [ + 390.9534912109375, + 966.9860229492188 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 80 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00154.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 79, + "type": "Note", + "pos": [ + -215.824951171875, + 490.2828369140625 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload ref images here\n（在此上传参考图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 104, + "type": "ImageCollectNode", + "pos": [ + 835.30908203125, + 491.93438720703125 + ], + "size": [ + 162.85116577148438, + 46 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 83 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": 82 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 84 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCollectNode" + }, + "widgets_values": [] + }, + { + "id": 103, + "type": "LoadImage", + "pos": [ + 409.56060791015625, + 457.4063415527344 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "ref_1.png", + "image" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "海风作曲，浪花打拍。她握着一台亮黄色相机，双臂如翼般舒展流转。双手托举相机，在胸前划出轻柔的圆弧，时而高举过肩，迎向天际的光晕，时而缓缓贴近心口，倾听快门低语般的节奏。紫色长发在风中扬起，拂过肩头，仿佛与镜头共舞。她的目光随取景框游走，每一次微倾与回旋，都让镜头掠过海天相接的边际、摇曳的花丛。最后，呼吸一凝，稳稳对准水中倒影，轻轻按下快门——将风、光与心动，悄然收藏。" + ] + }, + { + "id": 97, + "type": "LoadImage", + "pos": [ + 51.6423454284668, + 459.73138427734375 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 83 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "8.png", + "image" + ] + }, + { + "id": 105, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 238.0269317626953, + -745.763671875 + ], + "size": [ + 408.1667785644531, + 102 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 85 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 106, + "type": "LoadWanVAEModel", + "pos": [ + 240.40968322753906, + -583.1104736328125 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 86 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 107, + "type": "LoadWanTextEncoderModel", + "pos": [ + 241.08558654785156, + -434.9287109375 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 87 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 88 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 109, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 692.4532470703125, + -742.7451782226562 + ], + "size": [ + 354.0464782714844, + 102 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 89 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 110, + "type": "Note", + "pos": [ + -220.1790008544922, + -468.36676025390625 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 108, + "type": "CombineWan2_2VaceFunPipeline", + "pos": [ + 693.27978515625, + -566.5641479492188 + ], + "size": [ + 361.8251953125, + 182 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 85 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 86 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 87 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 88 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 89 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 90 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 92 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2VaceFunPipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload" + ] + }, + { + "id": 111, + "type": "LoadWan2_2FunLora", + "pos": [ + 1122.6082763671875, + -506.19036865234375 + ], + "size": [ + 601.3900756835938, + 130 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 92 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 93 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunLora" + }, + "widgets_values": [ + "Wan2.2-T2V-A14B-4steps-lora-rank64-Seko-V1.1_low_noise_model.safetensors", + "Wan2.2-T2V-A14B-4steps-lora-rank64-Seko-V1.1_high_noise_model.safetensors", + 1, + false + ] + }, + { + "id": 101, + "type": "Wan2_2VaceFunSampler", + "pos": [ + 842.1512451171875, + -215.00350952148438 + ], + "size": [ + 280.724609375, + 534 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 93 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "subject_ref_images", + "shape": 7, + "type": "IMAGE", + "link": 84 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2VaceFunSampler" + }, + "widgets_values": [ + 81, + 640, + 580941324226511, + "randomize", + 4, + 1.0000000000000002, + "Flow_Unipc", + 12, + 0.875, + 0.1, + false, + 3, + true, + 0, + true + ] + } + ], + "links": [ + [ + 76, + 92, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 77, + 94, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 80, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 82, + 103, + 0, + 104, + 1, + "IMAGE" + ], + [ + 83, + 97, + 0, + 104, + 0, + "IMAGE" + ], + [ + 84, + 104, + 0, + 101, + 6, + "IMAGE" + ], + [ + 85, + 105, + 0, + 108, + 0, + "TransformerModel" + ], + [ + 86, + 106, + 0, + 108, + 1, + "VAEModel" + ], + [ + 87, + 107, + 0, + 108, + 2, + "TextEncoderModel" + ], + [ + 88, + 107, + 1, + 108, + 3, + "Tokenizer" + ], + [ + 89, + 109, + 0, + 108, + 5, + "TransformerModel" + ], + [ + 90, + 105, + 1, + 108, + 6, + "STRING" + ], + [ + 92, + 108, + 0, + 111, + 0, + "FunModels" + ], + [ + 93, + 111, + 0, + 101, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Ref Images", + "bounding": [ + -0.4573218524456024, + 378.5783386230469, + 1071.616943359375, + 451.41473388671875 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 220.02639770507812, + -821.3883056640625, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 1.1, + "offset": [ + -67.03623665930887, + 397.7621525676092 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "abeac38889da94b1f215b456a8180198a24d5032", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_subjects_lora.json b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_subjects_lora.json new file mode 100644 index 0000000000000000000000000000000000000000..0bb0453c638d501406666cefb0988ee89fb5d42c --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_subjects_lora.json @@ -0,0 +1,892 @@ +{ + "id": "71b53971-2ab9-41eb-a405-78c174d97672", + "revision": 0, + "last_node_id": 111, + "last_link_id": 93, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1179.8734130859375, + -215.58523559570312 + ], + "size": [ + 390.9534912109375, + 966.9860229492188 + ], + "flags": {}, + "order": 16, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 80 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00154.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 79, + "type": "Note", + "pos": [ + -215.824951171875, + 490.2828369140625 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload ref images here\n（在此上传参考图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 104, + "type": "ImageCollectNode", + "pos": [ + 835.30908203125, + 491.93438720703125 + ], + "size": [ + 162.85116577148438, + 46 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 83 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": 82 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 84 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCollectNode" + }, + "widgets_values": [] + }, + { + "id": 103, + "type": "LoadImage", + "pos": [ + 409.56060791015625, + 457.4063415527344 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "ref_1.png", + "image" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "海风作曲，浪花打拍。她握着一台亮黄色相机，双臂如翼般舒展流转。双手托举相机，在胸前划出轻柔的圆弧，时而高举过肩，迎向天际的光晕，时而缓缓贴近心口，倾听快门低语般的节奏。紫色长发在风中扬起，拂过肩头，仿佛与镜头共舞。她的目光随取景框游走，每一次微倾与回旋，都让镜头掠过海天相接的边际、摇曳的花丛。最后，呼吸一凝，稳稳对准水中倒影，轻轻按下快门——将风、光与心动，悄然收藏。" + ] + }, + { + "id": 97, + "type": "LoadImage", + "pos": [ + 51.6423454284668, + 459.73138427734375 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 83 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "8.png", + "image" + ] + }, + { + "id": 105, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 238.0269317626953, + -745.763671875 + ], + "size": [ + 408.1667785644531, + 102 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 85 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 90 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 106, + "type": "LoadWanVAEModel", + "pos": [ + 240.40968322753906, + -583.1104736328125 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 86 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 107, + "type": "LoadWanTextEncoderModel", + "pos": [ + 241.08558654785156, + -434.9287109375 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 87 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 88 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 109, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 692.4532470703125, + -742.7451782226562 + ], + "size": [ + 354.0464782714844, + 102 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 89 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 110, + "type": "Note", + "pos": [ + -220.1790008544922, + -468.36676025390625 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 108, + "type": "CombineWan2_2VaceFunPipeline", + "pos": [ + 693.27978515625, + -566.5641479492188 + ], + "size": [ + 361.8251953125, + 182 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 85 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 86 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 87 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 88 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 89 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 90 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 92 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2VaceFunPipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload" + ] + }, + { + "id": 111, + "type": "LoadWan2_2FunLora", + "pos": [ + 1122.6082763671875, + -506.19036865234375 + ], + "size": [ + 601.3900756835938, + 130 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 92 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 93 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2FunLora" + }, + "widgets_values": [ + "Wan2.2-T2V-A14B-4steps-lora-rank64-Seko-V1.1_low_noise_model.safetensors", + "Wan2.2-T2V-A14B-4steps-lora-rank64-Seko-V1.1_high_noise_model.safetensors", + 1, + false + ] + }, + { + "id": 101, + "type": "Wan2_2VaceFunSampler", + "pos": [ + 842.1512451171875, + -215.00350952148438 + ], + "size": [ + 280.724609375, + 534 + ], + "flags": {}, + "order": 15, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 93 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "subject_ref_images", + "shape": 7, + "type": "IMAGE", + "link": 84 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2VaceFunSampler" + }, + "widgets_values": [ + 81, + 640, + 580941324226511, + "randomize", + 40, + 1.0000000000000002, + "Flow", + 12, + 0.875, + 0.1, + true, + 3, + true, + 0.25000000000000006, + true + ] + } + ], + "links": [ + [ + 76, + 92, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 77, + 94, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 80, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 82, + 103, + 0, + 104, + 1, + "IMAGE" + ], + [ + 83, + 97, + 0, + 104, + 0, + "IMAGE" + ], + [ + 84, + 104, + 0, + 101, + 6, + "IMAGE" + ], + [ + 85, + 105, + 0, + 108, + 0, + "TransformerModel" + ], + [ + 86, + 106, + 0, + 108, + 1, + "VAEModel" + ], + [ + 87, + 107, + 0, + 108, + 2, + "TextEncoderModel" + ], + [ + 88, + 107, + 1, + 108, + 3, + "Tokenizer" + ], + [ + 89, + 109, + 0, + 108, + 5, + "TransformerModel" + ], + [ + 90, + 105, + 1, + 108, + 6, + "STRING" + ], + [ + 92, + 108, + 0, + 111, + 0, + "FunModels" + ], + [ + 93, + 111, + 0, + 101, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Ref Images", + "bounding": [ + -0.4573218524456024, + 378.5783386230469, + 1071.616943359375, + 451.41473388671875 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 220.02639770507812, + -821.3883056640625, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.9090909090909095, + "offset": [ + -50.780612464991464, + 485.5421122795407 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "abeac38889da94b1f215b456a8180198a24d5032", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_v2v_control.json b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_v2v_control.json new file mode 100644 index 0000000000000000000000000000000000000000..2d70c054d44a3ba75bec0905ba13e74c5964a897 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_v2v_control.json @@ -0,0 +1,793 @@ +{ + "id": "71b53971-2ab9-41eb-a405-78c174d97672", + "revision": 0, + "last_node_id": 115, + "last_link_id": 108, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.145149230957031, + 527.0673217773438 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 96 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 95 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位年轻女子站在阳光明媚的海岸线上，身穿清爽的白色衬衫与裙子，在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" + ] + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 312.67236328125, + 479.96942138671875 + ], + "size": [ + 252.056640625, + 688.545166015625 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 97 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1179.8734130859375, + -215.58523559570312 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 98 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00143.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 108, + "type": "Note", + "pos": [ + -217.46144104003906, + -468.01708984375 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 106, + "type": "LoadWanTextEncoderModel", + "pos": [ + 228.37794494628906, + -433.6276550292969 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 104 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 108 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 105, + "type": "LoadWanVAEModel", + "pos": [ + 228.37794494628906, + -584.197021484375 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 103 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 114, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 678.4397583007812, + -747.5654296875 + ], + "size": [ + 354.0464782714844, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 105 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 113, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 236.3780059814453, + -747.4537353515625 + ], + "size": [ + 391.59686279296875, + 102 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 102 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 106 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 115, + "type": "CombineWan2_2VaceFunPipeline", + "pos": [ + 672.4246826171875, + -564.6559448242188 + ], + "size": [ + 361.8251953125, + 182 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 102 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 103 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 104 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 108 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 105 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 106 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 107 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2VaceFunPipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload" + ] + }, + { + "id": 112, + "type": "Wan2_2VaceFunSampler", + "pos": [ + 789.3981323242188, + -218.46241760253906 + ], + "size": [ + 333.453125, + 534 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 107 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 95 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 96 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 97 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "subject_ref_images", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 98 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2VaceFunSampler" + }, + "widgets_values": [ + 81, + 640, + 591039721331570, + "randomize", + 50, + 6, + "Flow", + 12, + 0.875, + 0.1, + true, + 3, + true, + 0.25000000000000006, + true + ] + } + ], + "links": [ + [ + 95, + 92, + 0, + 112, + 1, + "STRING_PROMPT" + ], + [ + 96, + 94, + 0, + 112, + 2, + "STRING_PROMPT" + ], + [ + 97, + 85, + 0, + 112, + 3, + "IMAGE" + ], + [ + 98, + 112, + 0, + 17, + 0, + "IMAGE" + ], + [ + 102, + 113, + 0, + 115, + 0, + "TransformerModel" + ], + [ + 103, + 105, + 0, + 115, + 1, + "VAEModel" + ], + [ + 104, + 106, + 0, + 115, + 2, + "TextEncoderModel" + ], + [ + 105, + 114, + 0, + 115, + 5, + "TransformerModel" + ], + [ + 106, + 113, + 1, + 115, + 6, + "STRING" + ], + [ + 107, + 115, + 0, + 112, + 0, + "FunModels" + ], + [ + 108, + 106, + 1, + 115, + 3, + "Tokenizer" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Video", + "bounding": [ + 218.27899169921875, + 391.4151916503906, + 455.1039733886719, + 797.0015869140625 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 218.37744140625, + -823.078369140625, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6209213230591556, + "offset": [ + 827.0994132162837, + 864.1815898774307 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "a4d5cca5c80bb811e1ee35915ad1e096ca62a3bf", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_v2v_control_ref.json b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_v2v_control_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..138779e721a5c8315e96a57960e681241dda2154 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_chunked_loading_workflow_v2v_control_ref.json @@ -0,0 +1,839 @@ +{ + "id": "f615e8ca-31ec-49be-859c-42d80fd8a4ff", + "revision": 0, + "last_node_id": 108, + "last_link_id": 88, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.145149230957031, + 527.0673217773438 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位年轻女子站在阳光明媚的海岸线上，身穿清爽的白色衬衫与裙子，在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" + ] + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 312.67236328125, + 479.96942138671875 + ], + "size": [ + 252.056640625, + 688.545166015625 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 78 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 97, + "type": "LoadImage", + "pos": [ + 621.7833862304688, + 490.2975769042969 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 79 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1179.8734130859375, + -215.58523559570312 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 14, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 80 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00143.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 101, + "type": "Wan2_2VaceFunSampler", + "pos": [ + 842.1512451171875, + -215.00350952148438 + ], + "size": [ + 280.724609375, + 534 + ], + "flags": {}, + "order": 13, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 88 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 78 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "subject_ref_images", + "shape": 7, + "type": "IMAGE", + "link": 79 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2VaceFunSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "randomize", + 50, + 6, + "Flow", + 12, + 0.875, + 0.1, + true, + 3, + true, + 0.25000000000000006, + true + ] + }, + { + "id": 108, + "type": "Note", + "pos": [ + -215.6398162841797, + -449.5701599121094 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 106, + "type": "LoadWanVAEModel", + "pos": [ + 240.581787109375, + -566.3518676757812 + ], + "size": [ + 406.1996154785156, + 82 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "vae", + "type": "VAEModel", + "links": [ + 83 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanVAEModel" + }, + "widgets_values": [ + "Wan2.1_VAE.pth", + "bf16" + ] + }, + { + "id": 107, + "type": "LoadWanTextEncoderModel", + "pos": [ + 241.25767517089844, + -418.1702880859375 + ], + "size": [ + 404.7171936035156, + 102 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "text_encoder", + "type": "TextEncoderModel", + "links": [ + 84 + ] + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "links": [ + 85 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWanTextEncoderModel" + }, + "widgets_values": [ + "models_t5_umt5-xxl-enc-bf16.pth", + "bf16" + ] + }, + { + "id": 103, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 238.1990203857422, + -729.0050659179688 + ], + "size": [ + 408.1667785644531, + 102 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 82 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": [ + 87 + ] + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-LOW_bf16.safetensors", + "bf16" + ] + }, + { + "id": 104, + "type": "LoadVaceWanTransformer3DModel", + "pos": [ + 692.6251831054688, + -725.986572265625 + ], + "size": [ + 354.0464782714844, + 102 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "links": [ + 86 + ] + }, + { + "name": "model_name", + "type": "STRING", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadVaceWanTransformer3DModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B-HIGH_bf16.safetensors", + "bf16" + ] + }, + { + "id": 105, + "type": "CombineWan2_2VaceFunPipeline", + "pos": [ + 693.45166015625, + -549.8055419921875 + ], + "size": [ + 361.8251953125, + 182 + ], + "flags": {}, + "order": 12, + "mode": 0, + "inputs": [ + { + "name": "transformer", + "type": "TransformerModel", + "link": 82 + }, + { + "name": "vae", + "type": "VAEModel", + "link": 83 + }, + { + "name": "text_encoder", + "type": "TextEncoderModel", + "link": 84 + }, + { + "name": "tokenizer", + "type": "Tokenizer", + "link": 85 + }, + { + "name": "clip_encoder", + "shape": 7, + "type": "ClipEncoderModel", + "link": null + }, + { + "name": "transformer_2", + "shape": 7, + "type": "TransformerModel", + "link": 86 + }, + { + "name": "model_name", + "type": "STRING", + "widget": { + "name": "model_name" + }, + "link": 87 + } + ], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 88 + ] + } + ], + "properties": { + "Node name for S&R": "CombineWan2_2VaceFunPipeline" + }, + "widgets_values": [ + "", + "sequential_cpu_offload" + ] + } + ], + "links": [ + [ + 76, + 92, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 77, + 94, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 78, + 85, + 0, + 101, + 3, + "IMAGE" + ], + [ + 79, + 97, + 0, + 101, + 6, + "IMAGE" + ], + [ + 80, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 82, + 103, + 0, + 105, + 0, + "TransformerModel" + ], + [ + 83, + 106, + 0, + 105, + 1, + "VAEModel" + ], + [ + 84, + 107, + 0, + 105, + 2, + "TextEncoderModel" + ], + [ + 85, + 107, + 1, + 105, + 3, + "Tokenizer" + ], + [ + 86, + 104, + 0, + 105, + 5, + "TransformerModel" + ], + [ + 87, + 103, + 1, + 105, + 6, + "STRING" + ], + [ + 88, + 105, + 0, + 101, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Video", + "bounding": [ + 218.27899169921875, + 391.4151916503906, + 761, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + }, + { + "id": 4, + "title": "Load Model", + "bounding": [ + 220.19847106933594, + -804.6296997070312, + 863.243896484375, + 511.2655029296875 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.751314800901578, + "offset": [ + 462.8838916549091, + 1171.5400934562742 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "a4d5cca5c80bb811e1ee35915ad1e096ca62a3bf", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_i2v.json b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_i2v.json new file mode 100644 index 0000000000000000000000000000000000000000..806ef2bb66653a2b3c3b0868937cd8d8f273a896 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_i2v.json @@ -0,0 +1,516 @@ +{ + "id": "71b53971-2ab9-41eb-a405-78c174d97672", + "revision": 0, + "last_node_id": 103, + "last_link_id": 84, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": [ + -192, + -293 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 80 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一只棕色的狗舔了一下它的舌头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1179.8734130859375, + -215.58523559570312 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 83 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00143.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.145149230957031, + 527.0673217773438 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload start image here\n（在此开始图像）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 97, + "type": "LoadImage", + "pos": [ + 290.19805908203125, + 480.74139404296875 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 103, + "type": "LoadWan2_2VaceFunModel", + "pos": [ + 302.4931640625, + -302.9169616699219 + ], + "size": [ + 291.3540954589844, + 130 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 84 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2VaceFunModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B", + "sequential_cpu_offload", + "wan2.2/wan_civitai_t2v.yaml", + "bf16" + ] + }, + { + "id": 102, + "type": "Wan2_2VaceFunSampler", + "pos": [ + 842.1512451171875, + -215.00350952148438 + ], + "size": [ + 280.724609375, + 510 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 84 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 80 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 81 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": 82 + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "subject_ref_images", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 83 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2VaceFunSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "randomize", + 50, + 6, + "Flow", + 12, + 0.875, + 0.1, + true, + 3, + true, + 0.25000000000000006, + true + ] + } + ], + "links": [ + [ + 80, + 92, + 0, + 102, + 1, + "STRING_PROMPT" + ], + [ + 81, + 94, + 0, + 102, + 2, + "STRING_PROMPT" + ], + [ + 82, + 97, + 0, + 102, + 4, + "IMAGE" + ], + [ + 83, + 102, + 0, + 17, + 0, + "IMAGE" + ], + [ + 84, + 103, + 0, + 102, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Start Image", + "bounding": [ + 218.27899169921875, + 391.4151916503906, + 449.0052795410156, + 420.1466369628906 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Load Model", + "bounding": [ + 218, + -387, + 448.53363037109375, + 238.44383239746094 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.751314800901578, + "offset": [ + 118.37845806115907, + 460.2817692375243 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "b82d3946141f6daba603fb941418cebdc51f5f62", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_subjects.json b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_subjects.json new file mode 100644 index 0000000000000000000000000000000000000000..51d2b5332a41bacbf5115c1b1b2b0dbf8bbeb48a --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_subjects.json @@ -0,0 +1,609 @@ +{ + "id": "71b53971-2ab9-41eb-a405-78c174d97672", + "revision": 0, + "last_node_id": 104, + "last_link_id": 84, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": [ + -192, + -293 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1179.8734130859375, + -215.58523559570312 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 11, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 80 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00143.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 102, + "type": "LoadWan2_2VaceFunModel", + "pos": [ + 299.8675537109375, + -299.02276611328125 + ], + "size": [ + 291.3540954589844, + 130 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2VaceFunModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B", + "sequential_cpu_offload", + "wan2.2/wan_civitai_t2v.yaml", + "bf16" + ] + }, + { + "id": 79, + "type": "Note", + "pos": [ + -215.824951171875, + 490.2828369140625 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload ref images here\n（在此上传参考图片）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 104, + "type": "ImageCollectNode", + "pos": [ + 835.30908203125, + 491.93438720703125 + ], + "size": [ + 162.85116577148438, + 46 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "image_1", + "type": "IMAGE", + "link": 83 + }, + { + "name": "image_2", + "shape": 7, + "type": "IMAGE", + "link": 82 + } + ], + "outputs": [ + { + "name": "image", + "type": "IMAGE", + "links": [ + 84 + ] + } + ], + "properties": { + "Node name for S&R": "ImageCollectNode" + } + }, + { + "id": 103, + "type": "LoadImage", + "pos": [ + 409.56060791015625, + 457.4063415527344 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 82 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "ref_1.png", + "image" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "海风作曲，浪花打拍。她握着一台亮黄色相机，双臂如翼般舒展流转。双手托举相机，在胸前划出轻柔的圆弧，时而高举过肩，迎向天际的光晕，时而缓缓贴近心口，倾听快门低语般的节奏。紫色长发在风中扬起，拂过肩头，仿佛与镜头共舞。她的目光随取景框游走，每一次微倾与回旋，都让镜头掠过海天相接的边际、摇曳的花丛。最后，呼吸一凝，稳稳对准水中倒影，轻轻按下快门——将风、光与心动，悄然收藏。" + ] + }, + { + "id": 97, + "type": "LoadImage", + "pos": [ + 51.6423454284668, + 459.73138427734375 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 83 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "8.png", + "image" + ] + }, + { + "id": 101, + "type": "Wan2_2VaceFunSampler", + "pos": [ + 842.1512451171875, + -215.00350952148438 + ], + "size": [ + 280.724609375, + 510 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 81 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "subject_ref_images", + "shape": 7, + "type": "IMAGE", + "link": 84 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2VaceFunSampler" + }, + "widgets_values": [ + 81, + 640, + 360428861626678, + "randomize", + 50, + 6, + "Flow", + 12, + 0.875, + 0.1, + true, + 3, + true, + 0.25000000000000006, + true + ] + } + ], + "links": [ + [ + 76, + 92, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 77, + 94, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 80, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 81, + 102, + 0, + 101, + 0, + "FunModels" + ], + [ + 82, + 103, + 0, + 104, + 1, + "IMAGE" + ], + [ + 83, + 97, + 0, + 104, + 0, + "IMAGE" + ], + [ + 84, + 104, + 0, + 101, + 6, + "IMAGE" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Ref Images", + "bounding": [ + -0.4573218524456024, + 378.5783386230469, + 1071.616943359375, + 451.41473388671875 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Load Model", + "bounding": [ + 218, + -387, + 449.3915100097656, + 247.22012329101562 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.751314800901578, + "offset": [ + 178.80897759240935, + 429.71036298752415 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "b82d3946141f6daba603fb941418cebdc51f5f62", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_v2v_control.json b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_v2v_control.json new file mode 100644 index 0000000000000000000000000000000000000000..72e2cf38c9e0d71877ae35f80781ae343ca499b7 --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_v2v_control.json @@ -0,0 +1,559 @@ +{ + "id": "71b53971-2ab9-41eb-a405-78c174d97672", + "revision": 0, + "last_node_id": 102, + "last_link_id": 81, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.145149230957031, + 527.0673217773438 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": [ + -192, + -293 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位年轻女子站在阳光明媚的海岸线上，身穿清爽的白色衬衫与裙子，在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" + ] + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 312.67236328125, + 479.96942138671875 + ], + "size": [ + 252.056640625, + 688.545166015625 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 78 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1179.8734130859375, + -215.58523559570312 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 80 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00143.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 102, + "type": "LoadWan2_2VaceFunModel", + "pos": [ + 299.8675537109375, + -299.02276611328125 + ], + "size": [ + 291.3540954589844, + 130 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2VaceFunModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B", + "sequential_cpu_offload", + "wan2.2/wan_civitai_t2v.yaml", + "bf16" + ] + }, + { + "id": 101, + "type": "Wan2_2VaceFunSampler", + "pos": [ + 842.1512451171875, + -215.00350952148438 + ], + "size": [ + 280.724609375, + 510 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 81 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 78 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "subject_ref_images", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2VaceFunSampler" + }, + "widgets_values": [ + 81, + 640, + 58957163437492, + "randomize", + 50, + 6, + "Flow", + 12, + 0.875, + 0.1, + true, + 3, + true, + 0.25000000000000006, + true + ] + } + ], + "links": [ + [ + 76, + 92, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 77, + 94, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 78, + 85, + 0, + 101, + 3, + "IMAGE" + ], + [ + 80, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 81, + 102, + 0, + 101, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Video", + "bounding": [ + 218.27899169921875, + 391.4151916503906, + 455.1039733886719, + 797.0015869140625 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Load Model", + "bounding": [ + 218, + -387, + 449.3915100097656, + 247.22012329101562 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.6209213230591555, + "offset": [ + 441.47341292722155, + 382.1698606313368 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "b82d3946141f6daba603fb941418cebdc51f5f62", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_v2v_control_ref.json b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_v2v_control_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..c03d99aa9b130f3591bc3ceb6696c1814357284f --- /dev/null +++ b/VideoX-Fun/comfyui/wan2_2_vace_fun/v1/wan2.2_vace_fun_workflow_v2v_control_ref.json @@ -0,0 +1,605 @@ +{ + "id": "71b53971-2ab9-41eb-a405-78c174d97672", + "revision": 0, + "last_node_id": 102, + "last_link_id": 81, + "nodes": [ + { + "id": 78, + "type": "Note", + "pos": [ + 18, + -46 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 0, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can write prompt here\n（你可以在此填写提示词）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 79, + "type": "Note", + "pos": [ + 15.145149230957031, + 527.0673217773438 + ], + "size": [ + 210, + 88 + ], + "flags": {}, + "order": 1, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "You can upload video here\n（在此上传视频）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 88, + "type": "Note", + "pos": [ + -99, + 197 + ], + "size": [ + 326.1556091308594, + 145.20904541015625 + ], + "flags": {}, + "order": 2, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "Using longer neg prompt such as \"Blurring, mutation, deformation, distortion, dark and solid, comics.\" can increase stability. Adding words such as \"quiet, solid\" to the neg prompt can increase dynamism.\n(使用更长的neg prompt如\"模糊，突变，变形，失真，画面暗，画面固定，连环画，漫画，线稿，没有主体。\"，可以增加稳定性。在neg prompt中添加\"安静，固定\"等词语可以增加动态性。)" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 89, + "type": "Note", + "pos": [ + -192, + -293 + ], + "size": [ + 427.074951171875, + 143.9142608642578 + ], + "flags": {}, + "order": 3, + "mode": 0, + "inputs": [], + "outputs": [], + "properties": { + "text": "" + }, + "widgets_values": [ + "When using the 14B model, you can use sequential_cpu_offload to save GPU memory during generation.\n（在使用14B模型时，可以使用sequential_cpu_offload节省显存，进行生成。）" + ], + "color": "#432", + "bgcolor": "#653" + }, + { + "id": 94, + "type": "FunTextBox", + "pos": [ + 258, + 178 + ], + "size": [ + 368.5529479980469, + 159.4075927734375 + ], + "flags": {}, + "order": 4, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 77 + ] + } + ], + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + ] + }, + { + "id": 92, + "type": "FunTextBox", + "pos": [ + 254, + -46 + ], + "size": [ + 380.845703125, + 157.68350219726562 + ], + "flags": {}, + "order": 5, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "prompt", + "type": "STRING_PROMPT", + "slot_index": 0, + "links": [ + 76 + ] + } + ], + "title": "Positive Prompt（正向提示词）", + "properties": { + "Node name for S&R": "FunTextBox" + }, + "widgets_values": [ + "一位年轻女子站在阳光明媚的海岸线上，身穿清爽的白色衬衫与裙子，在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" + ] + }, + { + "id": 85, + "type": "VHS_LoadVideo", + "pos": [ + 312.67236328125, + 479.96942138671875 + ], + "size": [ + 252.056640625, + 688.545166015625 + ], + "flags": {}, + "order": 6, + "mode": 0, + "inputs": [ + { + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "slot_index": 0, + "links": [ + 78 + ] + }, + { + "name": "frame_count", + "type": "INT", + "links": null + }, + { + "name": "audio", + "type": "AUDIO", + "links": null + }, + { + "name": "video_info", + "type": "VHS_VIDEOINFO", + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_LoadVideo" + }, + "widgets_values": { + "video": "pose.mp4", + "force_rate": 0, + "force_size": "Disabled", + "custom_width": 512, + "custom_height": 512, + "frame_load_cap": 0, + "skip_first_frames": 0, + "select_every_nth": 1, + "choose video to upload": "image", + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "frame_load_cap": 0, + "skip_first_frames": 0, + "force_rate": 0, + "filename": "pose.mp4", + "type": "input", + "format": "video/mp4", + "select_every_nth": 1 + } + } + } + }, + { + "id": 97, + "type": "LoadImage", + "pos": [ + 621.7833862304688, + 490.2975769042969 + ], + "size": [ + 315, + 314.0000305175781 + ], + "flags": {}, + "order": 7, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "IMAGE", + "type": "IMAGE", + "links": [ + 79 + ] + }, + { + "name": "MASK", + "type": "MASK", + "links": null + } + ], + "properties": { + "Node name for S&R": "LoadImage" + }, + "widgets_values": [ + "9.png", + "image" + ] + }, + { + "id": 17, + "type": "VHS_VideoCombine", + "pos": [ + 1179.8734130859375, + -215.58523559570312 + ], + "size": [ + 390.9534912109375, + 942.2557983398438 + ], + "flags": {}, + "order": 10, + "mode": 0, + "inputs": [ + { + "label": "图像", + "name": "images", + "shape": 7, + "type": "IMAGE", + "link": 80 + }, + { + "label": "音频", + "name": "audio", + "shape": 7, + "type": "AUDIO", + "link": null + }, + { + "label": "批次管理", + "name": "meta_batch", + "shape": 7, + "type": "VHS_BatchManager", + "link": null + }, + { + "name": "vae", + "shape": 7, + "type": "VAE", + "link": null + } + ], + "outputs": [ + { + "label": "文件名", + "name": "Filenames", + "type": "VHS_FILENAMES", + "slot_index": 0, + "links": null + } + ], + "properties": { + "Node name for S&R": "VHS_VideoCombine" + }, + "widgets_values": { + "frame_rate": 16, + "loop_count": 0, + "filename_prefix": "Fun", + "format": "video/h264-mp4", + "pix_fmt": "yuv420p", + "crf": 22, + "save_metadata": true, + "pingpong": false, + "save_output": true, + "videopreview": { + "hidden": false, + "paused": false, + "params": { + "filename": "Fun_00143.mp4", + "subfolder": "", + "type": "output", + "format": "video/h264-mp4", + "frame_rate": 16 + } + } + } + }, + { + "id": 102, + "type": "LoadWan2_2VaceFunModel", + "pos": [ + 299.8675537109375, + -299.02276611328125 + ], + "size": [ + 291.3540954589844, + 130 + ], + "flags": {}, + "order": 8, + "mode": 0, + "inputs": [], + "outputs": [ + { + "name": "funmodels", + "type": "FunModels", + "links": [ + 81 + ] + } + ], + "properties": { + "Node name for S&R": "LoadWan2_2VaceFunModel" + }, + "widgets_values": [ + "Wan2.2-VACE-Fun-A14B", + "sequential_cpu_offload", + "wan2.2/wan_civitai_t2v.yaml", + "bf16" + ] + }, + { + "id": 101, + "type": "Wan2_2VaceFunSampler", + "pos": [ + 842.1512451171875, + -215.00350952148438 + ], + "size": [ + 280.724609375, + 510 + ], + "flags": {}, + "order": 9, + "mode": 0, + "inputs": [ + { + "name": "funmodels", + "type": "FunModels", + "link": 81 + }, + { + "name": "prompt", + "type": "STRING_PROMPT", + "link": 76 + }, + { + "name": "negative_prompt", + "type": "STRING_PROMPT", + "link": 77 + }, + { + "name": "control_video", + "shape": 7, + "type": "IMAGE", + "link": 78 + }, + { + "name": "start_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "end_image", + "shape": 7, + "type": "IMAGE", + "link": null + }, + { + "name": "subject_ref_images", + "shape": 7, + "type": "IMAGE", + "link": 79 + }, + { + "name": "riflex_k", + "shape": 7, + "type": "RIFLEXT_ARGS", + "link": null + } + ], + "outputs": [ + { + "name": "images", + "type": "IMAGE", + "links": [ + 80 + ] + } + ], + "properties": { + "Node name for S&R": "Wan2_2VaceFunSampler" + }, + "widgets_values": [ + 81, + 640, + 43, + "randomize", + 50, + 6, + "Flow", + 12, + 0.875, + 0.1, + true, + 3, + true, + 0.25000000000000006, + true + ] + } + ], + "links": [ + [ + 76, + 92, + 0, + 101, + 1, + "STRING_PROMPT" + ], + [ + 77, + 94, + 0, + 101, + 2, + "STRING_PROMPT" + ], + [ + 78, + 85, + 0, + 101, + 3, + "IMAGE" + ], + [ + 79, + 97, + 0, + 101, + 6, + "IMAGE" + ], + [ + 80, + 101, + 0, + 17, + 0, + "IMAGE" + ], + [ + 81, + 102, + 0, + 101, + 0, + "FunModels" + ] + ], + "groups": [ + { + "id": 1, + "title": "Upload Your Video", + "bounding": [ + 218.27899169921875, + 391.4151916503906, + 761, + 789 + ], + "color": "#a1309b", + "font_size": 24, + "flags": {} + }, + { + "id": 2, + "title": "Load Model", + "bounding": [ + 218, + -387, + 449.3915100097656, + 247.22012329101562 + ], + "color": "#b06634", + "font_size": 24, + "flags": {} + }, + { + "id": 3, + "title": "Prompts", + "bounding": [ + 218, + -127, + 450, + 483 + ], + "color": "#3f789e", + "font_size": 24, + "flags": {} + } + ], + "config": {}, + "extra": { + "ds": { + "scale": 0.751314800901578, + "offset": [ + 195.69084087365928, + 421.60998017502436 + ] + }, + "frontendVersion": "1.21.3", + "workspace_info": { + "id": "776b62b4-bd17-4ed3-9923-b7aad000b1ea" + }, + "node_versions": { + "CogVideoX-Fun": "b82d3946141f6daba603fb941418cebdc51f5f62", + "ComfyUI-VideoHelperSuite": "70faa9bcef65932ab72e7404d6373fb300013a2e", + "comfy-core": "0.3.57" + } + }, + "version": 0.4 +} \ No newline at end of file diff --git a/VideoX-Fun/compile_all_subgraphs.sh b/VideoX-Fun/compile_all_subgraphs.sh new file mode 100644 index 0000000000000000000000000000000000000000..78c16612319585c56393e1939d566879906bff16 --- /dev/null +++ b/VideoX-Fun/compile_all_subgraphs.sh @@ -0,0 +1,237 @@ +#!/bin/bash +# +# Script to compile all ONNX subgraph models using pulsar2 +# ./compile_all_subgraphs.sh --onnx-dir ./transformers_body_only_1728_992_split_onnx --config-dir pulsar2_configs/subgraphs_1728x992 --output-base-dir ./compiled_transformers_body_only_1728x992/out_all --final-output-dir ./compiled_transformers_body_only_1728x992/out_final +# + +# Don't exit on error - we want to continue even if some models fail +set +e + +# Default directories (can be overridden via CLI) +ONNX_DIR="transformers_body_only_split_onnx" +CONFIG_DIR="pulsar2_configs/subgraphs" +OUTPUT_BASE_DIR="comliled_subgraph_from_all_onnx" +FINAL_OUTPUT_DIR="comliled_subgraph_from_all_onnx" + +usage() { + cat <<'EOF' +Usage: ./compile_all_subgraphs.sh [options] + +Options: + --onnx-dir PATH ONNX 子图目录，默认为 transformers_body_only_split_onnx + --config-dir PATH 配置文件目录，默认为 pulsar2_configs/subgraphs + --output-base-dir PATH 每个子图编译输出的基目录，默认为 comliled_subgraph_from_all_onnx + --final-output-dir PATH 收集 .axmodel 的最终目录，默认为 comliled_subgraph_from_all_onnx + -h, --help 显示本帮助 +EOF +} + +# Parse CLI args +while [[ $# -gt 0 ]]; do + case "$1" in + --onnx-dir) + ONNX_DIR="$2"; shift 2;; + --onnx-dir=*) + ONNX_DIR="${1#*=}"; shift;; + --config-dir) + CONFIG_DIR="$2"; shift 2;; + --config-dir=*) + CONFIG_DIR="${1#*=}"; shift;; + --output-base-dir) + OUTPUT_BASE_DIR="$2"; shift 2;; + --output-base-dir=*) + OUTPUT_BASE_DIR="${1#*=}"; shift;; + --final-output-dir) + FINAL_OUTPUT_DIR="$2"; shift 2;; + --final-output-dir=*) + FINAL_OUTPUT_DIR="${1#*=}"; shift;; + -h|--help) + usage; exit 0;; + *) + echo "Unknown option: $1" >&2 + usage + exit 1;; + esac +done + +# Create output directories if they don't exist +mkdir -p "${OUTPUT_BASE_DIR}" "${FINAL_OUTPUT_DIR}" + +# Log files +LOG_FILE="${OUTPUT_BASE_DIR}/compile_log_$(date +%Y%m%d_%H%M%S).txt" +ERROR_LOG="${OUTPUT_BASE_DIR}/compile_errors_$(date +%Y%m%d_%H%M%S).txt" + +echo "===============================================" | tee -a "${LOG_FILE}" +echo "ONNX Subgraph Compilation Script" | tee -a "${LOG_FILE}" +echo "Start time: $(date)" | tee -a "${LOG_FILE}" +echo "===============================================" | tee -a "${LOG_FILE}" +echo "" | tee -a "${LOG_FILE}" + +# Counter variables +total_files=0 +success_count=0 +failed_count=0 +skip_count=0 + +# Array to store failed models +declare -a failed_models + +# Get all ONNX files sorted +onnx_files=($(ls "${ONNX_DIR}"/*.onnx 2>/dev/null | sort)) + +if [ ${#onnx_files[@]} -eq 0 ]; then + echo "Error: No ONNX files found in ${ONNX_DIR}" | tee -a "${LOG_FILE}" "${ERROR_LOG}" + exit 1 +fi + +total_files=${#onnx_files[@]} +echo "Found ${total_files} ONNX files to compile" | tee -a "${LOG_FILE}" +echo "" | tee -a "${LOG_FILE}" + +# Loop through each ONNX file +for onnx_file in "${onnx_files[@]}"; do + # Extract filename without path and extension + filename=$(basename "${onnx_file}") + basename_no_ext="${filename%.onnx}" + + # Extract prefix (e.g., "auto_00" or "cfg_00") + # Pattern: auto_XX or cfg_XX + if [[ $basename_no_ext =~ ^(auto_[0-9]{2}|cfg_[0-9]{2}) ]]; then + prefix="${BASH_REMATCH[1]}" + else + echo "Warning: Cannot extract prefix from ${filename}, skipping..." | tee -a "${LOG_FILE}" "${ERROR_LOG}" + continue + fi + + # Config file path + config_file="${CONFIG_DIR}/${prefix}.json" + + # Check if config file exists + if [ ! -f "${config_file}" ]; then + echo "Error: Config file not found: ${config_file}" | tee -a "${LOG_FILE}" "${ERROR_LOG}" + failed_models+=("${basename_no_ext}") + ((failed_count++)) + continue + fi + + # Output directory for this model + output_dir="${OUTPUT_BASE_DIR}/compiled_${prefix}" + + # Output model/log names + output_name="${basename_no_ext}.axmodel" + final_model_path="${FINAL_OUTPUT_DIR}/${output_name}" + model_log="${FINAL_OUTPUT_DIR}/${basename_no_ext}_build.txt" + + # Skip compilation if final output already exists + if [ -f "${final_model_path}" ]; then + echo "-----------------------------------------------" | tee -a "${LOG_FILE}" + echo "[$(date +%H:%M:%S)] [$((success_count + failed_count + skip_count + 1))/${total_files}] Skip: ${filename} 已存在，跳过 pulsar2 编译" | tee -a "${LOG_FILE}" + echo " Final file already present at ${final_model_path}" | tee -a "${LOG_FILE}" + { + echo "[$(date)] Skip pulsar2 build for ${output_name}."; + echo "Reason: ${final_model_path} already exists."; + } > "${model_log}" + ((skip_count++)) + echo "" | tee -a "${LOG_FILE}" + continue + fi + + mkdir -p "${output_dir}" + + echo "-----------------------------------------------" | tee -a "${LOG_FILE}" + echo "[$(date +%H:%M:%S)] [$((success_count + failed_count + 1))/${total_files}] Compiling: ${filename}" | tee -a "${LOG_FILE}" + echo " Config: ${config_file}" | tee -a "${LOG_FILE}" + echo " Output: ${output_dir}/${output_name}" | tee -a "${LOG_FILE}" + + # Build the pulsar2 command + compile_cmd="pulsar2 build \ + --input ${onnx_file} \ + --model_type ONNX \ + --output_dir ${output_dir} \ + --output_name ${output_name} \ + --config ${config_file} \ + --target_hardware AX650 \ + --npu_mode NPU3 \ + --debug.dump_frontend_graph true" + + # Prepare per-model log + { + echo "[$(date)] pulsar2 build log for ${basename_no_ext}"; + echo "Command: ${compile_cmd}"; + echo "---- output ----"; + } > "${model_log}" + + # Execute compilation + if eval "${compile_cmd}" 2>&1 | tee -a "${model_log}" >> "${LOG_FILE}"; then + echo " ✓ Success" | tee -a "${LOG_FILE}" + ((success_count++)) + + # Copy the generated .axmodel file to the final output directory + if [ -f "${output_dir}/${output_name}" ]; then + if cp "${output_dir}/${output_name}" "${FINAL_OUTPUT_DIR}/"; then + echo " ✓ Copied ${output_name} to ${FINAL_OUTPUT_DIR}/" | tee -a "${LOG_FILE}" + if rm -rf "${output_dir}"; then + echo " ✓ Removed build directory ${output_dir}" | tee -a "${LOG_FILE}" + else + echo " ⚠ Warning: failed to remove ${output_dir}" | tee -a "${LOG_FILE}" "${ERROR_LOG}" + fi + else + echo " ⚠ Warning: copy ${output_name} to ${FINAL_OUTPUT_DIR} failed" | tee -a "${LOG_FILE}" "${ERROR_LOG}" + fi + else + echo " ⚠ Warning: ${output_name} not found in ${output_dir}" | tee -a "${LOG_FILE}" + fi + else + echo " ✗ Failed" | tee -a "${LOG_FILE}" "${ERROR_LOG}" + echo " Model: ${filename}" >> "${ERROR_LOG}" + echo " Command: ${compile_cmd}" >> "${ERROR_LOG}" + echo " See ${LOG_FILE} for details" >> "${ERROR_LOG}" + echo "" >> "${ERROR_LOG}" + failed_models+=("${basename_no_ext}") + ((failed_count++)) + fi + + echo "" | tee -a "${LOG_FILE}" +done + +# Copy all remaining .axmodel files to final output directory (in case some were missed) +echo "-----------------------------------------------" | tee -a "${LOG_FILE}" +echo "Copying all .axmodel files to ${FINAL_OUTPUT_DIR}/" | tee -a "${LOG_FILE}" +find "${OUTPUT_BASE_DIR}/compiled_"*/ -name "*.axmodel" -type f 2>/dev/null | while read -r axmodel_file; do + filename=$(basename "${axmodel_file}") + if [ ! -f "${FINAL_OUTPUT_DIR}/${filename}" ]; then + cp "${axmodel_file}" "${FINAL_OUTPUT_DIR}/" + echo " Copied: ${filename}" | tee -a "${LOG_FILE}" + fi +done + +# Print summary +echo "" | tee -a "${LOG_FILE}" +echo "===============================================" | tee -a "${LOG_FILE}" +echo "COMPILATION SUMMARY" | tee -a "${LOG_FILE}" +echo "===============================================" | tee -a "${LOG_FILE}" +echo "Total files: ${total_files}" | tee -a "${LOG_FILE}" +echo "Successful: ${success_count}" | tee -a "${LOG_FILE}" +echo "Skipped: ${skip_count}" | tee -a "${LOG_FILE}" +echo "Failed: ${failed_count}" | tee -a "${LOG_FILE}" +echo "End time: $(date)" | tee -a "${LOG_FILE}" +echo "" | tee -a "${LOG_FILE}" + +if [ ${failed_count} -gt 0 ]; then + echo "Failed models:" | tee -a "${LOG_FILE}" + for model in "${failed_models[@]}"; do + echo " - ${model}" | tee -a "${LOG_FILE}" + done + echo "" | tee -a "${LOG_FILE}" + echo "Check error log: ${ERROR_LOG}" | tee -a "${LOG_FILE}" +fi + +echo "Full log saved to: ${LOG_FILE}" | tee -a "${LOG_FILE}" +echo "===============================================" | tee -a "${LOG_FILE}" + +# List generated .axmodel files +echo "" | tee -a "${LOG_FILE}" +echo "Generated .axmodel files in ${FINAL_OUTPUT_DIR}:" | tee -a "${LOG_FILE}" +ls -lh "${FINAL_OUTPUT_DIR}"/*.axmodel 2>/dev/null | tee -a "${LOG_FILE}" + +exit 0 diff --git a/VideoX-Fun/config/flux2/flux2_control.yaml b/VideoX-Fun/config/flux2/flux2_control.yaml new file mode 100644 index 0000000000000000000000000000000000000000..56b2f39f5a49bde0faeaf32526117d0d504d7935 --- /dev/null +++ b/VideoX-Fun/config/flux2/flux2_control.yaml @@ -0,0 +1,5 @@ +format: diffusers +pipeline: flux2 +transformer_additional_kwargs: + control_layers: [0, 2, 4, 6] + control_in_dim: 260 \ No newline at end of file diff --git a/VideoX-Fun/config/wan2.1/wan_civitai.yaml b/VideoX-Fun/config/wan2.1/wan_civitai.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3a2747bdc9e86a6812c3ca09d2a8653aaea6cb54 --- /dev/null +++ b/VideoX-Fun/config/wan2.1/wan_civitai.yaml @@ -0,0 +1,39 @@ +format: civitai +pipeline: Wan +transformer_additional_kwargs: + transformer_subpath: ./ + dict_mapping: + in_dim: in_channels + dim: hidden_size + +vae_kwargs: + vae_subpath: Wan2.1_VAE.pth + temporal_compression_ratio: 4 + spatial_compression_ratio: 8 + +text_encoder_kwargs: + text_encoder_subpath: models_t5_umt5-xxl-enc-bf16.pth + tokenizer_subpath: google/umt5-xxl + text_length: 512 + vocab: 256384 + dim: 4096 + dim_attn: 4096 + dim_ffn: 10240 + num_heads: 64 + num_layers: 24 + num_buckets: 32 + shared_pos: False + dropout: 0.0 + +scheduler_kwargs: + scheduler_subpath: null + num_train_timesteps: 1000 + shift: 5.0 + use_dynamic_shifting: false + base_shift: 0.5 + max_shift: 1.15 + base_image_seq_len: 256 + max_image_seq_len: 4096 + +image_encoder_kwargs: + image_encoder_subpath: models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth \ No newline at end of file diff --git a/VideoX-Fun/config/wan2.2/wan_civitai_5b.yaml b/VideoX-Fun/config/wan2.2/wan_civitai_5b.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f465e28e00ba377596a093ec3c2a23b4bc07e8b6 --- /dev/null +++ b/VideoX-Fun/config/wan2.2/wan_civitai_5b.yaml @@ -0,0 +1,41 @@ +format: civitai +pipeline: Wan +transformer_additional_kwargs: + transformer_low_noise_model_subpath: ./ + transformer_combination_type: "single" + dict_mapping: + in_dim: in_channels + dim: hidden_size + +vae_kwargs: + vae_type: "AutoencoderKLWan3_8" + vae_subpath: Wan2.2_VAE.pth + temporal_compression_ratio: 4 + spatial_compression_ratio: 16 + +text_encoder_kwargs: + text_encoder_subpath: models_t5_umt5-xxl-enc-bf16.pth + tokenizer_subpath: google/umt5-xxl + text_length: 512 + vocab: 256384 + dim: 4096 + dim_attn: 4096 + dim_ffn: 10240 + num_heads: 64 + num_layers: 24 + num_buckets: 32 + shared_pos: False + dropout: 0.0 + +scheduler_kwargs: + scheduler_subpath: null + num_train_timesteps: 1000 + shift: 5.0 + use_dynamic_shifting: false + base_shift: 0.5 + max_shift: 1.15 + base_image_seq_len: 256 + max_image_seq_len: 4096 + +image_encoder_kwargs: + image_encoder_subpath: models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth \ No newline at end of file diff --git a/VideoX-Fun/config/wan2.2/wan_civitai_animate.yaml b/VideoX-Fun/config/wan2.2/wan_civitai_animate.yaml new file mode 100644 index 0000000000000000000000000000000000000000..9ed6d1a65c114513c1bca38f57501bc94a24c82d --- /dev/null +++ b/VideoX-Fun/config/wan2.2/wan_civitai_animate.yaml @@ -0,0 +1,41 @@ +format: civitai +pipeline: Wan +transformer_additional_kwargs: + transformer_low_noise_model_subpath: ./ + transformer_combination_type: "single" + dict_mapping: + in_dim: in_channels + dim: hidden_size + +vae_kwargs: + vae_type: "AutoencoderKLWan" + vae_subpath: Wan2.1_VAE.pth + temporal_compression_ratio: 4 + spatial_compression_ratio: 8 + +text_encoder_kwargs: + text_encoder_subpath: models_t5_umt5-xxl-enc-bf16.pth + tokenizer_subpath: google/umt5-xxl + text_length: 512 + vocab: 256384 + dim: 4096 + dim_attn: 4096 + dim_ffn: 10240 + num_heads: 64 + num_layers: 24 + num_buckets: 32 + shared_pos: False + dropout: 0.0 + +scheduler_kwargs: + scheduler_subpath: null + num_train_timesteps: 1000 + shift: 5.0 + use_dynamic_shifting: false + base_shift: 0.5 + max_shift: 1.15 + base_image_seq_len: 256 + max_image_seq_len: 4096 + +image_encoder_kwargs: + image_encoder_subpath: models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth diff --git a/VideoX-Fun/config/wan2.2/wan_civitai_i2v.yaml b/VideoX-Fun/config/wan2.2/wan_civitai_i2v.yaml new file mode 100644 index 0000000000000000000000000000000000000000..030b62e22db7e6e3c2e2fc7ed9935f0a9fffe835 --- /dev/null +++ b/VideoX-Fun/config/wan2.2/wan_civitai_i2v.yaml @@ -0,0 +1,43 @@ +format: civitai +pipeline: Wan +transformer_additional_kwargs: + transformer_low_noise_model_subpath: ./low_noise_model + transformer_high_noise_model_subpath: ./high_noise_model + transformer_combination_type: "moe" + boundary: 0.900 + dict_mapping: + in_dim: in_channels + dim: hidden_size + +vae_kwargs: + vae_type: "AutoencoderKLWan" + vae_subpath: Wan2.1_VAE.pth + temporal_compression_ratio: 4 + spatial_compression_ratio: 8 + +text_encoder_kwargs: + text_encoder_subpath: models_t5_umt5-xxl-enc-bf16.pth + tokenizer_subpath: google/umt5-xxl + text_length: 512 + vocab: 256384 + dim: 4096 + dim_attn: 4096 + dim_ffn: 10240 + num_heads: 64 + num_layers: 24 + num_buckets: 32 + shared_pos: False + dropout: 0.0 + +scheduler_kwargs: + scheduler_subpath: null + num_train_timesteps: 1000 + shift: 5.0 + use_dynamic_shifting: false + base_shift: 0.5 + max_shift: 1.15 + base_image_seq_len: 256 + max_image_seq_len: 4096 + +image_encoder_kwargs: + image_encoder_subpath: models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth \ No newline at end of file diff --git a/VideoX-Fun/config/wan2.2/wan_civitai_s2v.yaml b/VideoX-Fun/config/wan2.2/wan_civitai_s2v.yaml new file mode 100644 index 0000000000000000000000000000000000000000..81d99215914c21070efd55741d943144e312bb90 --- /dev/null +++ b/VideoX-Fun/config/wan2.2/wan_civitai_s2v.yaml @@ -0,0 +1,44 @@ +format: civitai +pipeline: Wan +transformer_additional_kwargs: + transformer_low_noise_model_subpath: ./ + transformer_combination_type: "single" + dict_mapping: + in_dim: in_channels + dim: hidden_size + +vae_kwargs: + vae_type: "AutoencoderKLWan" + vae_subpath: Wan2.1_VAE.pth + temporal_compression_ratio: 4 + spatial_compression_ratio: 8 + +text_encoder_kwargs: + text_encoder_subpath: models_t5_umt5-xxl-enc-bf16.pth + tokenizer_subpath: google/umt5-xxl + text_length: 512 + vocab: 256384 + dim: 4096 + dim_attn: 4096 + dim_ffn: 10240 + num_heads: 64 + num_layers: 24 + num_buckets: 32 + shared_pos: False + dropout: 0.0 + +audio_encoder_kwargs: + audio_encoder_subpath: wav2vec2-large-xlsr-53-english + +scheduler_kwargs: + scheduler_subpath: null + num_train_timesteps: 1000 + shift: 3.0 + use_dynamic_shifting: false + base_shift: 0.5 + max_shift: 1.15 + base_image_seq_len: 256 + max_image_seq_len: 4096 + +image_encoder_kwargs: + image_encoder_subpath: models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth \ No newline at end of file diff --git a/VideoX-Fun/config/wan2.2/wan_civitai_t2v.yaml b/VideoX-Fun/config/wan2.2/wan_civitai_t2v.yaml new file mode 100644 index 0000000000000000000000000000000000000000..407f28562f2d392d79087097a677e2a961779284 --- /dev/null +++ b/VideoX-Fun/config/wan2.2/wan_civitai_t2v.yaml @@ -0,0 +1,43 @@ +format: civitai +pipeline: Wan +transformer_additional_kwargs: + transformer_low_noise_model_subpath: ./low_noise_model + transformer_high_noise_model_subpath: ./high_noise_model + transformer_combination_type: "moe" + boundary: 0.875 + dict_mapping: + in_dim: in_channels + dim: hidden_size + +vae_kwargs: + vae_type: "AutoencoderKLWan" + vae_subpath: Wan2.1_VAE.pth + temporal_compression_ratio: 4 + spatial_compression_ratio: 8 + +text_encoder_kwargs: + text_encoder_subpath: models_t5_umt5-xxl-enc-bf16.pth + tokenizer_subpath: google/umt5-xxl + text_length: 512 + vocab: 256384 + dim: 4096 + dim_attn: 4096 + dim_ffn: 10240 + num_heads: 64 + num_layers: 24 + num_buckets: 32 + shared_pos: False + dropout: 0.0 + +scheduler_kwargs: + scheduler_subpath: null + num_train_timesteps: 1000 + shift: 12.0 + use_dynamic_shifting: false + base_shift: 0.5 + max_shift: 1.15 + base_image_seq_len: 256 + max_image_seq_len: 4096 + +image_encoder_kwargs: + image_encoder_subpath: models_clip_open-clip-xlm-roberta-large-vit-huge-14.pth \ No newline at end of file diff --git a/VideoX-Fun/config/z_image/z_image_control.yaml b/VideoX-Fun/config/z_image/z_image_control.yaml new file mode 100644 index 0000000000000000000000000000000000000000..869e3ba162255e33351499529d98f2f62488c2b0 --- /dev/null +++ b/VideoX-Fun/config/z_image/z_image_control.yaml @@ -0,0 +1,5 @@ +format: diffusers +pipeline: flux2 +transformer_additional_kwargs: + control_layers_places: [0, 5, 10, 15, 20, 25] + control_in_dim: 16 \ No newline at end of file diff --git a/VideoX-Fun/config/zero_stage2_config.json b/VideoX-Fun/config/zero_stage2_config.json new file mode 100644 index 0000000000000000000000000000000000000000..e60ea05a563827d4286955c1421e82d3b1bbe5cc --- /dev/null +++ b/VideoX-Fun/config/zero_stage2_config.json @@ -0,0 +1,16 @@ +{ + "bf16": { + "enabled": true + }, + "train_micro_batch_size_per_gpu": 1, + "train_batch_size": "auto", + "gradient_accumulation_steps": "auto", + "dump_state": true, + "zero_optimization": { + "stage": 2, + "overlap_comm": true, + "contiguous_gradients": true, + "sub_group_size": 1e9, + "reduce_bucket_size": 5e8 + } +} \ No newline at end of file diff --git a/VideoX-Fun/config/zero_stage3_config.json b/VideoX-Fun/config/zero_stage3_config.json new file mode 100644 index 0000000000000000000000000000000000000000..60e59347b74f0d09df96586720977dd25fa1aded --- /dev/null +++ b/VideoX-Fun/config/zero_stage3_config.json @@ -0,0 +1,28 @@ +{ + "bf16": { + "enabled": true + }, + "train_micro_batch_size_per_gpu": 1, + "train_batch_size": "auto", + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "wall_clock_breakdown": false, + "zero_optimization": { + "stage": 3, + "overlap_comm": true, + "contiguous_gradients": true, + "reduce_bucket_size": 5e8, + "sub_group_size": 1e9, + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": "auto", + "offload_optimizer": { + "device": "none" + }, + "offload_param": { + "device": "none" + } + } +} + diff --git a/VideoX-Fun/config/zero_stage3_config_cpu_offload.json b/VideoX-Fun/config/zero_stage3_config_cpu_offload.json new file mode 100644 index 0000000000000000000000000000000000000000..19f779dea3cce78936b7848ce277f7875abb3526 --- /dev/null +++ b/VideoX-Fun/config/zero_stage3_config_cpu_offload.json @@ -0,0 +1,28 @@ +{ + "bf16": { + "enabled": true + }, + "train_micro_batch_size_per_gpu": 1, + "train_batch_size": "auto", + "gradient_accumulation_steps": "auto", + "gradient_clipping": "auto", + "steps_per_print": 2000, + "wall_clock_breakdown": false, + "zero_optimization": { + "stage": 3, + "overlap_comm": true, + "contiguous_gradients": true, + "reduce_bucket_size": 5e8, + "sub_group_size": 1e9, + "stage3_max_live_parameters": 1e9, + "stage3_max_reuse_distance": 1e9, + "stage3_gather_16bit_weights_on_model_save": "auto", + "offload_optimizer": { + "device": "cpu" + }, + "offload_param": { + "device": "cpu" + } + } +} + diff --git a/VideoX-Fun/examples/cogvideox_fun/app.py b/VideoX-Fun/examples/cogvideox_fun/app.py new file mode 100644 index 0000000000000000000000000000000000000000..63463fb5e7e6155b4f24ad41172231540e76026e --- /dev/null +++ b/VideoX-Fun/examples/cogvideox_fun/app.py @@ -0,0 +1,73 @@ +import os +import sys +import time + +import torch + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.api.api import (infer_forward_api, + update_diffusion_transformer_api) +from videox_fun.ui.controller import ddpm_scheduler_dict +from videox_fun.ui.cogvideox_fun_ui import ui, ui_client, ui_host + +if __name__ == "__main__": + # Choose the ui mode + # "normal" refers to the standard UI, which allows users to click to switch models, change model types, and more. + # "host" represents the hosting mode, where the model is loaded directly at startup and can be accessed via + # the API to return generation results. + # "client" represents the client mode, offering a simple UI that sends requests to a remote API for generation. + ui_mode = "normal" + + # GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. + # model_full_load means that the entire model will be moved to the GPU. + # + # model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. + # + # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, + # and the transformer model has been quantized to float8, which can save more GPU memory. + # + # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, + # resulting in slower speeds but saving a large amount of GPU memory. + GPU_memory_mode = "model_cpu_offload_and_qfloat8" + # Compile will give a speedup in fixed resolution and need a little GPU memory. + # The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. + compile_dit = False + + # Use torch.float16 if GPU does not support torch.bfloat16 + # ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 + weight_dtype = torch.bfloat16 + + # Server ip + server_name = "0.0.0.0" + server_port = 7860 + + # Params below is used when ui_mode = "host" + model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" + # "Inpaint" or "Control" + model_type = "Inpaint" + + if ui_mode == "host": + demo, controller = ui_host(GPU_memory_mode, ddpm_scheduler_dict, model_name, model_type, compile_dit, weight_dtype) + elif ui_mode == "client": + demo, controller = ui_client(ddpm_scheduler_dict, model_name) + else: + demo, controller = ui(GPU_memory_mode, ddpm_scheduler_dict, compile_dit, weight_dtype) + + # launch gradio + app, _, _ = demo.queue(status_update_rate=1).launch( + server_name=server_name, + server_port=server_port, + prevent_thread_lock=True + ) + + # launch api + infer_forward_api(None, app, controller) + update_diffusion_transformer_api(None, app, controller) + + # not close the python + while True: + time.sleep(5) \ No newline at end of file diff --git a/VideoX-Fun/examples/cogvideox_fun/launch_api.py b/VideoX-Fun/examples/cogvideox_fun/launch_api.py new file mode 100644 index 0000000000000000000000000000000000000000..067330c224d3151475fe8f6ee994f2f5c8f81bf3 --- /dev/null +++ b/VideoX-Fun/examples/cogvideox_fun/launch_api.py @@ -0,0 +1,90 @@ +import argparse +import os +import sys +import time + +import gradio as gr +import ray +import torch + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.api.api_multi_nodes import (MultiNodesEngine, + multi_nodes_infer_forward_api) +from videox_fun.ui.controller import flow_scheduler_dict +from videox_fun.ui.cogvideox_fun_ui import CogVideoXFunController + +def main(): + parser = argparse.ArgumentParser(description='xDiT HTTP Service') + parser.add_argument('--world_size', type=int, default=8, help='Number of parallel workers') + parser.add_argument( + '--gpu_memory_mode', type=str, default="model_cpu_offload", help=''' +GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8]. +model_full_load means that the entire model will be moved to the GPU. + +model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +and the transformer model has been quantized to float8, which can save more GPU memory. + +model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. + +model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +and the transformer model has been quantized to float8, which can save more GPU memory. + ''' + ) + parser.add_argument( + '--compile_dit', action='store_true', help=''' +Enable compile dit. +Compile will give a speedup in fixed resolution and need a little GPU memory. +The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. + ''' + ) + parser.add_argument('--fsdp_dit', action='store_true', help="Use DIT FSDP to save more GPU memory in multi gpus.") + parser.add_argument('--fsdp_text_encoder', action='store_true', help="Use Text Encoder FSDP to save more GPU memory in multi gpus.") + parser.add_argument('--ulysses_degree', type=int, default=4, help='Degree of Ulysses configuration') + parser.add_argument('--ring_degree', type=int, default=2, help='Degree of Ring configuration') + parser.add_argument('--weight_dtype', type=str, default='bf16', help='Weight data type') + parser.add_argument('--server_name', type=str, default="0.0.0.0", help='Server IP address') + parser.add_argument('--server_port', type=int, default=7860, help='Server Port') + parser.add_argument('--model_name', type=str, default="models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP", help='Model path') + parser.add_argument('--model_type', type=str, default="Inpaint", help='Model type (Inpaint/Control)') + parser.add_argument('--savedir_sample', type=str, default=None, help='The save directory for samples') + args = parser.parse_args() + + weight_dtype = torch.float32 + if args.weight_dtype == "bf16": + weight_dtype = torch.bfloat16 + elif args.weight_dtype == "fp16": + weight_dtype = torch.float16 + + engine = MultiNodesEngine( + world_size=args.world_size, Controller=CogVideoXFunController, + GPU_memory_mode=args.gpu_memory_mode, scheduler_dict=flow_scheduler_dict, model_name=args.model_name, model_type=args.model_type, config_path=None, + ulysses_degree=args.ulysses_degree, ring_degree=args.ring_degree, + fsdp_dit=args.fsdp_dit, fsdp_text_encoder=args.fsdp_text_encoder, compile_dit=args.compile_dit, + weight_dtype=weight_dtype, savedir_sample=args.savedir_sample, + ) + + def gr_launch(): + # launch gradio + with gr.Blocks() as demo: + gr.Markdown("") + app, _, _ = demo.queue(status_update_rate=1).launch( + server_name=args.server_name, + server_port=args.server_port, + prevent_thread_lock=True + ) + + # launch api + multi_nodes_infer_forward_api(None, app, engine) + + gr_launch() + + # not close the python + while True: + time.sleep(5) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/examples/cogvideox_fun/post_infer.py b/VideoX-Fun/examples/cogvideox_fun/post_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..20cd09e0a6fc5406a34b0b6055fe1360e13c966e --- /dev/null +++ b/VideoX-Fun/examples/cogvideox_fun/post_infer.py @@ -0,0 +1,150 @@ +import base64 +import json +import time +from datetime import datetime + +import requests +import base64 + + +def post_diffusion_transformer(diffusion_transformer_path, url='http://127.0.0.1:7860'): + datas = json.dumps({ + "diffusion_transformer_path": diffusion_transformer_path + }) + r = requests.post(f'{url}/cogvideox_fun/update_diffusion_transformer', data=datas, timeout=1500) + data = r.content.decode('utf-8') + return data + +def post_update_edition(edition, url='http://0.0.0.0:7860'): + datas = json.dumps({ + "edition": edition + }) + r = requests.post(f'{url}/cogvideox_fun/update_edition', data=datas, timeout=1500) + data = r.content.decode('utf-8') + return data + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5000, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion.", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43 +): + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + data = post_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # The Url you want to post + POST_URL = 'http://0.0.0.0:7860' + # Used in EAS. If you don't need Authorization, please set it to empty string. + TOKEN = '' + + # -------------------------- # + # Step 1: update edition + # -------------------------- # + # diffusion_transformer_path = "models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" + # outputs = post_diffusion_transformer(diffusion_transformer_path) + # print('Output update edition: ', outputs) + + # -------------------------- # + # Step 2: infer + # -------------------------- # + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 49 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + negative_prompt_textbox = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion." + # Sampler name + sampler_dropdown = "Euler" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 672 + height_slider = 384 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + url=POST_URL, + POST_TOKEN=TOKEN + ) + + # Get decoded data + outputs = json.loads(outputs) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') \ No newline at end of file diff --git a/VideoX-Fun/examples/cogvideox_fun/post_infer_queue.py b/VideoX-Fun/examples/cogvideox_fun/post_infer_queue.py new file mode 100644 index 0000000000000000000000000000000000000000..079175099d7d7d5b067980e413f6c5570f57e35c --- /dev/null +++ b/VideoX-Fun/examples/cogvideox_fun/post_infer_queue.py @@ -0,0 +1,145 @@ +import base64 +import json +import time +import urllib.parse +import requests + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion.", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43 +): + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 49 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + negative_prompt_textbox = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion." + # Sampler name + sampler_dropdown = "Euler" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 672 + height_slider = 384 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + url=EAS_URL, + POST_TOKEN=TOKEN + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') \ No newline at end of file diff --git a/VideoX-Fun/examples/cogvideox_fun/predict_i2v.py b/VideoX-Fun/examples/cogvideox_fun/predict_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..0475af23a8a3a93a7b313b79d3db53b8eb587af1 --- /dev/null +++ b/VideoX-Fun/examples/cogvideox_fun/predict_i2v.py @@ -0,0 +1,320 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import (CogVideoXDDIMScheduler, DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, + PNDMScheduler) +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) +from videox_fun.pipeline import (CogVideoXFunInpaintPipeline, + CogVideoXFunPipeline) +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Config and model path +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" + +# Choose the sampler in "Euler" "Euler A" "DPM++" "PNDM" "DDIM_Cog" and "DDIM_Origin" +sampler_name = "DDIM_Origin" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [384, 672] +# V1.0 and V1.1 support up to 49 frames of video generation, +# while V1.5 supports up to 85 frames. +video_length = 49 +fps = 8 + +# If you want to generate ultra long videos, please set partial_video_length as the length of each sub video segment +partial_video_length = None +overlap_video_length = 4 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# If you want to generate from text, please set the validation_image_start = None and validation_image_end = None +validation_image_start = "asset/1.png" +validation_image_end = None + +# prompts +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +negative_prompt = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion. " +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/cogvideox-fun-videos_i2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = CogVideoXTransformer3DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLCogVideoX.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = T5Tokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = T5EncoderModel.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Euler": EulerDiscreteScheduler, + "Euler A": EulerAncestralDiscreteScheduler, + "DPM++": DPMSolverMultistepScheduler, + "PNDM": PNDMScheduler, + "DDIM_Cog": CogVideoXDDIMScheduler, + "DDIM_Origin": DDIMScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +if transformer.config.in_channels != vae.config.latent_channels: + pipeline = CogVideoXFunInpaintPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) +else: + pipeline = CogVideoXFunPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=[], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=[], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +if partial_video_length is not None: + partial_video_length = int((partial_video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (partial_video_length - 1) // vae.config.temporal_compression_ratio + 1 + if partial_video_length != 1 and transformer.config.patch_size_t is not None and latent_frames % transformer.config.patch_size_t != 0: + additional_frames = transformer.config.patch_size_t - latent_frames % transformer.config.patch_size_t + partial_video_length += additional_frames * vae.config.temporal_compression_ratio + + init_frames = 0 + last_frames = init_frames + partial_video_length + while init_frames < video_length: + if last_frames >= video_length: + _partial_video_length = video_length - init_frames + _partial_video_length = int((_partial_video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 + latent_frames = (_partial_video_length - 1) // vae.config.temporal_compression_ratio + 1 + if _partial_video_length != 1 and transformer.config.patch_size_t is not None and latent_frames % transformer.config.patch_size_t != 0: + additional_frames = transformer.config.patch_size_t - latent_frames % transformer.config.patch_size_t + _partial_video_length += additional_frames * vae.config.temporal_compression_ratio + + if _partial_video_length <= 0: + break + else: + _partial_video_length = partial_video_length + + input_video, input_video_mask, clip_image = get_image_to_video_latent(validation_image, None, video_length=_partial_video_length, sample_size=sample_size) + + with torch.no_grad(): + sample = pipeline( + prompt, + num_frames = _partial_video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = input_video, + mask_video = input_video_mask + ).videos + + if init_frames != 0: + mix_ratio = torch.from_numpy( + np.array([float(_index) / float(overlap_video_length) for _index in range(overlap_video_length)], np.float32) + ).unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + + new_sample[:, :, -overlap_video_length:] = new_sample[:, :, -overlap_video_length:] * (1 - mix_ratio) + \ + sample[:, :, :overlap_video_length] * mix_ratio + new_sample = torch.cat([new_sample, sample[:, :, overlap_video_length:]], dim = 2) + + sample = new_sample + else: + new_sample = sample + + if last_frames >= video_length: + break + + validation_image = [ + Image.fromarray( + (sample[0, :, _index].transpose(0, 1).transpose(1, 2) * 255).numpy().astype(np.uint8) + ) for _index in range(-overlap_video_length, 0) + ] + + init_frames = init_frames + _partial_video_length - overlap_video_length + last_frames = init_frames + _partial_video_length +else: + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + if video_length != 1 and transformer.config.patch_size_t is not None and latent_frames % transformer.config.patch_size_t != 0: + additional_frames = transformer.config.patch_size_t - latent_frames % transformer.config.patch_size_t + video_length += additional_frames * vae.config.temporal_compression_ratio + input_video, input_video_mask, clip_image = get_image_to_video_latent(validation_image_start, validation_image_end, video_length=video_length, sample_size=sample_size) + + with torch.no_grad(): + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = input_video, + mask_video = input_video_mask + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/cogvideox_fun/predict_t2v.py b/VideoX-Fun/examples/cogvideox_fun/predict_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..376615b01c87a52bde7db94221056f9549f2852c --- /dev/null +++ b/VideoX-Fun/examples/cogvideox_fun/predict_t2v.py @@ -0,0 +1,260 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import (CogVideoXDDIMScheduler, DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, + PNDMScheduler) +from PIL import Image +from transformers import T5EncoderModel + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) +from videox_fun.pipeline import (CogVideoXFunPipeline, + CogVideoXFunInpaintPipeline) +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid +from videox_fun.dist import set_multi_gpus_devices, shard_model + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# model path +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" + +# Choose the sampler in "Euler" "Euler A" "DPM++" "PNDM" "DDIM_Cog" and "DDIM_Origin" +sampler_name = "DDIM_Origin" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [384, 672] +# V1.0 and V1.1 support up to 49 frames of video generation, +# while V1.5 supports up to 85 frames. +video_length = 49 +fps = 8 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +prompt = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +negative_prompt = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion. " +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/cogvideox-fun-videos-t2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = CogVideoXTransformer3DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLCogVideoX.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = T5Tokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = T5EncoderModel.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Euler": EulerDiscreteScheduler, + "Euler A": EulerAncestralDiscreteScheduler, + "DPM++": DPMSolverMultistepScheduler, + "PNDM": PNDMScheduler, + "DDIM_Cog": CogVideoXDDIMScheduler, + "DDIM_Origin": DDIMScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +if transformer.config.in_channels != vae.config.latent_channels: + pipeline = CogVideoXFunInpaintPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) +else: + pipeline = CogVideoXFunPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=[], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=[], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + if video_length != 1 and transformer.config.patch_size_t is not None and latent_frames % transformer.config.patch_size_t != 0: + additional_frames = transformer.config.patch_size_t - latent_frames % transformer.config.patch_size_t + video_length += additional_frames * vae.config.temporal_compression_ratio + + if transformer.config.in_channels != vae.config.latent_channels: + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=sample_size) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = input_video, + mask_video = input_video_mask, + ).videos + else: + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/cogvideox_fun/predict_v2v.py b/VideoX-Fun/examples/cogvideox_fun/predict_v2v.py new file mode 100644 index 0000000000000000000000000000000000000000..84d10192dcf33f0e1157606f2a52a7f66c91082f --- /dev/null +++ b/VideoX-Fun/examples/cogvideox_fun/predict_v2v.py @@ -0,0 +1,255 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import (CogVideoXDDIMScheduler, DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, + PNDMScheduler) +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) +from videox_fun.pipeline import (CogVideoXFunPipeline, + CogVideoXFunInpaintPipeline) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.utils import get_video_to_video_latent, save_videos_grid +from videox_fun.dist import set_multi_gpus_devices, shard_model + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# model path +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" + +# Choose the sampler in "Euler" "Euler A" "DPM++" "PNDM" "DDIM_Cog" and "DDIM_Origin" +sampler_name = "DDIM_Origin" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None +# Other params +sample_size = [384, 672] +# V1.0 and V1.1 support up to 49 frames of video generation, +# while V1.5 supports up to 85 frames. +video_length = 49 +fps = 8 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# If you are preparing to redraw the reference video, set validation_video and validation_video_mask. +# If you do not use validation_video_mask, the entire video will be redrawn; +# if you use validation_video_mask, only a portion of the video will be redrawn. +# Please set a larger denoise_strength when using validation_video_mask, such as 1.00 instead of 0.70 +validation_video = "asset/1.mp4" +validation_video_mask = None +denoise_strength = 0.70 + +# prompts +prompt = "A cute cat is playing the guitar. " +negative_prompt = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion. " +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/cogvideox-fun-videos_v2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = CogVideoXTransformer3DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLCogVideoX.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = T5Tokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = T5EncoderModel.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Euler": EulerDiscreteScheduler, + "Euler A": EulerAncestralDiscreteScheduler, + "DPM++": DPMSolverMultistepScheduler, + "PNDM": PNDMScheduler, + "DDIM_Cog": CogVideoXDDIMScheduler, + "DDIM_Origin": DDIMScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +if transformer.config.in_channels != vae.config.latent_channels: + pipeline = CogVideoXFunInpaintPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) +else: + pipeline = CogVideoXFunPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=[], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=[], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 +latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 +if video_length != 1 and transformer.config.patch_size_t is not None and latent_frames % transformer.config.patch_size_t != 0: + additional_frames = transformer.config.patch_size_t - latent_frames % transformer.config.patch_size_t + video_length += additional_frames * vae.config.temporal_compression_ratio +input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(validation_video, video_length=video_length, sample_size=sample_size, validation_video_mask=validation_video_mask, fps=fps) + +with torch.no_grad(): + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = input_video, + mask_video = input_video_mask, + strength = denoise_strength, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/cogvideox_fun/predict_v2v_control.py b/VideoX-Fun/examples/cogvideox_fun/predict_v2v_control.py new file mode 100644 index 0000000000000000000000000000000000000000..1193d33c5f506f0ef90d69d1fc1a299ac7af49f6 --- /dev/null +++ b/VideoX-Fun/examples/cogvideox_fun/predict_v2v_control.py @@ -0,0 +1,240 @@ +import os +import sys + +import cv2 +import numpy as np +import torch +from diffusers import (CogVideoXDDIMScheduler, DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, + PNDMScheduler) +from PIL import Image +from transformers import T5EncoderModel + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) +from videox_fun.pipeline import (CogVideoXFunControlPipeline, + CogVideoXFunInpaintPipeline) +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import get_video_to_video_latent, save_videos_grid +from videox_fun.dist import set_multi_gpus_devices, shard_model + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# model path +model_name = "models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-Pose" + +# Choose the sampler in "Euler" "Euler A" "DPM++" "PNDM" "DDIM_Cog" and "DDIM_Origin" +sampler_name = "DDIM_Origin" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None +# Other params +sample_size = [672, 384] +# V1.0 and V1.1 support up to 49 frames of video generation, +# while V1.5 supports up to 85 frames. +video_length = 49 +fps = 8 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = "asset/pose.mp4" + +# prompts +prompt = "A young woman with beautiful face, dressed in white, is moving her body. " +negative_prompt = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion. " +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/cogvideox-fun-videos_control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = CogVideoXTransformer3DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLCogVideoX.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = T5Tokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = T5EncoderModel.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Euler": EulerDiscreteScheduler, + "Euler A": EulerAncestralDiscreteScheduler, + "DPM++": DPMSolverMultistepScheduler, + "PNDM": PNDMScheduler, + "DDIM_Cog": CogVideoXDDIMScheduler, + "DDIM_Origin": DDIMScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = CogVideoXFunControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=[], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=[], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 +latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 +if video_length != 1 and transformer.config.patch_size_t is not None and latent_frames % transformer.config.patch_size_t != 0: + additional_frames = transformer.config.patch_size_t - latent_frames % transformer.config.patch_size_t + video_length += additional_frames * vae.config.temporal_compression_ratio +input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps) + +with torch.no_grad(): + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + control_video = input_video, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/fantasytalking/predict_s2v.py b/VideoX-Fun/examples/fantasytalking/predict_s2v.py new file mode 100644 index 0000000000000000000000000000000000000000..ec8e5edda8539dcf22f7d0964b0ad69cbc3f6579 --- /dev/null +++ b/VideoX-Fun/examples/fantasytalking/predict_s2v.py @@ -0,0 +1,332 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + FantasyTalkingTransformer3DModel, FantasyTalkingAudioEncoder, + WanT5EncoderModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import FantasyTalkingPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, + get_video_to_video_latent, + merge_video_audio, save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +# Please Download https://modelscope.cn/models/AI-ModelScope/wav2vec2-base-960h/summary +# to models/Diffusion_Transformer/Wan2.1-I2V-14B-720P/audio_encoder for encoding audio. +model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +# The fantasytalking_model.ckpt can be downloaded in https://www.modelscope.cn/models/amap_cvlab/FantasyTalking/ +transformer_path = "models/Personalized_Model/fantasytalking_model.ckpt" +vae_path = None +# Load lora model if need +lora_path = None + +# Other params +sample_size = [832, 480] +video_length = 81 +fps = 23 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# If you want to generate from text, please set the validation_image_start = None +validation_image_start = "asset/8.png" +audio_path = "asset/talk.wav" + +# prompts +prompt = "一个女孩在海边说话。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 4.5 +seed = 43 +num_inference_steps = 40 +lora_weight = 0.55 +save_path = "samples/fantasy-talking-videos-speech2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = FantasyTalkingTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + + if "audio_processor" in state_dict: + audio_processor_dict = state_dict["audio_processor"] if "audio_processor" in state_dict else state_dict + m, u = transformer.load_state_dict(audio_processor_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + proj_model_dict = state_dict["proj_model"] if "proj_model" in state_dict else state_dict + proj_model_dict = {"proj_model." + k : v for k, v in proj_model_dict.items()} + m, u = transformer.load_state_dict(proj_model_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + else: + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Clip Image Encoder +clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(model_name, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), +).to(weight_dtype) +clip_image_encoder = clip_image_encoder.eval() + +audio_encoder = FantasyTalkingAudioEncoder( + os.path.join(model_name, "audio_encoder") +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = FantasyTalkingPipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + audio_encoder=audio_encoder, + clip_image_encoder=clip_image_encoder, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, clip_image = get_image_to_video_latent(validation_image_start, None, video_length=video_length, sample_size=sample_size) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + audio_path = audio_path, + shift = shift, + fps = fps + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + + merge_video_audio(video_path=video_path, audio_path=audio_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/flux/predict_t2i.py b/VideoX-Fun/examples/flux/predict_t2i.py new file mode 100644 index 0000000000000000000000000000000000000000..5fb55e6426c6db958108220ca46f754e7de28e87 --- /dev/null +++ b/VideoX-Fun/examples/flux/predict_t2i.py @@ -0,0 +1,216 @@ +import os +import sys + +import torch +from diffusers import FlowMatchEulerDiscreteScheduler + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, CLIPTextModel, CLIPTokenizer, + FluxTransformer2DModel, T5EncoderModel, + T5TokenizerFast) +from videox_fun.pipeline import FluxPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# model path +model_name = "models/Diffusion_Transformer/FLUX.1-dev" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [1344, 768] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +negative_prompt = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion. " +guidance_scale = 1.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/flux-t2i" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = FluxTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = CLIPTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = CLIPTextModel.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype +) + +tokenizer_2 = T5TokenizerFast.from_pretrained( + model_name, subfolder="tokenizer_2" +) +text_encoder_2 = T5EncoderModel.from_pretrained( + model_name, subfolder="text_encoder_2", torch_dtype=weight_dtype +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = FluxPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + tokenizer_2=tokenizer_2, + text_encoder_2=text_encoder_2, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.text_model.encoder.layers) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + sample = pipeline( + prompt, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + true_cfg_scale = guidance_scale, + num_inference_steps = num_inference_steps, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/flux2/predict_t2i.py b/VideoX-Fun/examples/flux2/predict_t2i.py new file mode 100644 index 0000000000000000000000000000000000000000..dce3c3e885f9e5e17b2cce75f5299a3e6e2c4ec8 --- /dev/null +++ b/VideoX-Fun/examples/flux2/predict_t2i.py @@ -0,0 +1,211 @@ +import os +import sys + +import torch + +from diffusers import (FlowMatchEulerDiscreteScheduler) + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLFlux2, + Mistral3ForConditionalGeneration, + PixtralProcessor, Flux2Transformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Flux2Pipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# model path +model_name = "models/Diffusion_Transformer/FLUX.2-dev" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [1344, 768] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +negative_prompt = " " +guidance_scale = 4.00 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/flux2-t2i" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = Flux2Transformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLFlux2.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = PixtralProcessor.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Mistral3ForConditionalGeneration.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = Flux2Pipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + sample = pipeline( + prompt = prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/flux2_fun/predict_i2i_inpaint.py b/VideoX-Fun/examples/flux2_fun/predict_i2i_inpaint.py new file mode 100644 index 0000000000000000000000000000000000000000..1c4aca6ad610d1b34c902a1c66fbc960bc5f437c --- /dev/null +++ b/VideoX-Fun/examples/flux2_fun/predict_i2i_inpaint.py @@ -0,0 +1,251 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLFlux2, + Mistral3ForConditionalGeneration, + PixtralProcessor, Flux2ControlTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Flux2ControlPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image, get_image_latent, + get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Config and model path +config_path = "config/flux2/flux2_control.yaml" +# model path +model_name = "models/Diffusion_Transformer/FLUX.2-dev" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +image = None +control_image = None +inpaint_image = "asset/8.png" +mask_image = "asset/mask.png" +control_context_scale = 0.75 + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "This is a panoramic portrait photo of a young man. He has medium-length flowing hair with a soft lavender-like color. He is wearing a white sleeveless shirt with a blue ribbon bow tied around the collar. He has a confident posture, with his left hand naturally hanging down and his right hand in his pocket, and his legs slightly apart. He looks straight at the camera. The sea breeze gently brushes his hair as he stands on the sunny seaside path, surrounded by blooming purple seaside flowers and smooth pebbles, with the sparkling sea and blue sky behind him. The scene presents a bright summer atmosphere, with soft and natural lighting, realistic details, and 8K ultra high definition image quality, clearly presenting fine textures such as clothing and hair." +negative_prompt = " " +guidance_scale = 4.00 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/flux2-t2i-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = Flux2ControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLFlux2.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = PixtralProcessor.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Mistral3ForConditionalGeneration.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = Flux2ControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers, ignored_modules=[text_encoder.language_model.embed_tokens], transformer_layer_cls_to_wrap=["MistralDecoderLayer", "PixtralTransformer"]) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + if image is not None: + if not isinstance(image, list): + image = get_image(image) + else: + image = [get_image(_image) for _image in image] + + if inpaint_image is not None: + inpaint_image = get_image_latent(inpaint_image, sample_size=sample_size)[:, :, 0] + else: + inpaint_image = torch.zeros([1, 3, sample_size[0], sample_size[1]]) + + if mask_image is not None: + mask_image = get_image_latent(mask_image, sample_size=sample_size)[:, :1, 0] + else: + mask_image = torch.ones([1, 1, sample_size[0], sample_size[1]]) * 255 + + if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] + + sample = pipeline( + prompt = prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + image = image, + inpaint_image = inpaint_image, + mask_image = mask_image, + control_image = control_image, + num_inference_steps = num_inference_steps, + control_context_scale = control_context_scale, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/flux2_fun/predict_t2i_control.py b/VideoX-Fun/examples/flux2_fun/predict_t2i_control.py new file mode 100644 index 0000000000000000000000000000000000000000..722aa1b51938e19c1d619150454eaaed60ffab93 --- /dev/null +++ b/VideoX-Fun/examples/flux2_fun/predict_t2i_control.py @@ -0,0 +1,251 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLFlux2, + Mistral3ForConditionalGeneration, + PixtralProcessor, Flux2ControlTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Flux2ControlPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image, get_image_latent, + get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Config and model path +config_path = "config/flux2/flux2_control.yaml" +# model path +model_name = "models/Diffusion_Transformer/FLUX.2-dev" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +image = None +control_image = "asset/pose.jpg" +inpaint_image = None +mask_image = None +control_context_scale = 0.75 + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "This is a panoramic portrait photo of a young woman. She has flowing long hair and a soft lavender like color. She is wearing a white sleeveless dress with a blue ribbon bow tied around the collar. She has a confident posture, with her left hand naturally hanging down and her right hand in her pocket, and her legs slightly apart. Look straight at the camera. The sea breeze gently brushed her long hair, and they stood on the sunny seaside path, surrounded by blooming purple seaside flowers and smooth pebbles, with the sparkling sea and blue sky behind them. The screen presents a bright summer atmosphere, with soft and natural lighting, realistic details, and 8K ultra high definition image quality, clearly presenting fine textures such as clothing and hair. " +negative_prompt = " " +guidance_scale = 4.00 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/flux2-t2i-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = Flux2ControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLFlux2.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = PixtralProcessor.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Mistral3ForConditionalGeneration.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = Flux2ControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers, ignored_modules=[text_encoder.language_model.embed_tokens], transformer_layer_cls_to_wrap=["MistralDecoderLayer", "PixtralTransformer"]) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + if image is not None: + if not isinstance(image, list): + image = get_image(image) + else: + image = [get_image(_image) for _image in image] + + if inpaint_image is not None: + inpaint_image = get_image_latent(inpaint_image, sample_size=sample_size)[:, :, 0] + else: + inpaint_image = torch.zeros([1, 3, sample_size[0], sample_size[1]]) + + if mask_image is not None: + mask_image = get_image_latent(mask_image, sample_size=sample_size)[:, :1, 0] + else: + mask_image = torch.ones([1, 1, sample_size[0], sample_size[1]]) * 255 + + if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] + + sample = pipeline( + prompt = prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + image = image, + inpaint_image = inpaint_image, + mask_image = mask_image, + control_image = control_image, + num_inference_steps = num_inference_steps, + control_context_scale = control_context_scale, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/flux2_fun/predict_t2i_control_ref.py b/VideoX-Fun/examples/flux2_fun/predict_t2i_control_ref.py new file mode 100644 index 0000000000000000000000000000000000000000..2f621d3b8993c6fc4a2ee6561ec4f1f897dce1d2 --- /dev/null +++ b/VideoX-Fun/examples/flux2_fun/predict_t2i_control_ref.py @@ -0,0 +1,251 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLFlux2, + Mistral3ForConditionalGeneration, + PixtralProcessor, Flux2ControlTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Flux2ControlPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image, get_image_latent, + get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Config and model path +config_path = "config/flux2/flux2_control.yaml" +# model path +model_name = "models/Diffusion_Transformer/FLUX.2-dev" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/FLUX.2-dev-Fun-Controlnet-Union.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +image = "asset/8.png" +control_image = "asset/pose.jpg" +inpaint_image = None +mask_image = None +control_context_scale = 0.75 + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "This is a panoramic portrait photo of a young woman. She has flowing long hair and a soft lavender like color. She is wearing a white sleeveless dress with a blue ribbon bow tied around the collar. She has a confident posture, with her left hand naturally hanging down and her right hand in her pocket, and her legs slightly apart. Look straight at the camera. The sea breeze gently brushed her long hair, and they stood on the sunny seaside path, surrounded by blooming purple seaside flowers and smooth pebbles, with the sparkling sea and blue sky behind them. The screen presents a bright summer atmosphere, with soft and natural lighting, realistic details, and 8K ultra high definition image quality, clearly presenting fine textures such as clothing and hair. " +negative_prompt = " " +guidance_scale = 4.00 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/flux2-t2i-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = Flux2ControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLFlux2.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = PixtralProcessor.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Mistral3ForConditionalGeneration.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = Flux2ControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers, ignored_modules=[text_encoder.language_model.embed_tokens], transformer_layer_cls_to_wrap=["MistralDecoderLayer", "PixtralTransformer"]) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + if image is not None: + if not isinstance(image, list): + image = get_image(image) + else: + image = [get_image(_image) for _image in image] + + if inpaint_image is not None: + inpaint_image = get_image_latent(inpaint_image, sample_size=sample_size)[:, :, 0] + else: + inpaint_image = torch.zeros([1, 3, sample_size[0], sample_size[1]]) + + if mask_image is not None: + mask_image = get_image_latent(mask_image, sample_size=sample_size)[:, :1, 0] + else: + mask_image = torch.ones([1, 1, sample_size[0], sample_size[1]]) * 255 + + if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] + + sample = pipeline( + prompt = prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + image = image, + inpaint_image = inpaint_image, + mask_image = mask_image, + control_image = control_image, + num_inference_steps = num_inference_steps, + control_context_scale = control_context_scale, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/hunyuanvideo/predict_i2v.py b/VideoX-Fun/examples/hunyuanvideo/predict_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..22e8cf8cf1d8dc123b1a9b331703624f8a92fcfd --- /dev/null +++ b/VideoX-Fun/examples/hunyuanvideo/predict_i2v.py @@ -0,0 +1,262 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import export_to_video +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from diffusers.schedulers.scheduling_unipc_multistep import \ + UniPCMultistepScheduler + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLHunyuanVideo, CLIPTextModel, CLIPImageProcessor, + CLIPTokenizer, HunyuanVideoTransformer3DModel, + LlavaForConditionalGeneration, LlamaTokenizerFast) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import HunyuanVideoPipeline, HunyuanVideoI2VPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.utils import get_image + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# model path +model_name = "models/Diffusion_Transformer/HunyuanVideo-I2V" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# If you want to generate from text, please set the validation_image_start = None and validation_image_end = None +validation_image_start = "asset/1.png" + +# prompts +prompt = "The dog is shaking head. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." +negative_prompt = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion. " +guidance_scale = 1.0 +seed = 43 +num_inference_steps = 40 +lora_weight = 0.55 +save_path = "samples/hunyuanvideo-videos-i2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = HunyuanVideoTransformer3DModel.from_pretrained( + os.path.join(model_name, 'transformer'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLHunyuanVideo.from_pretrained( + os.path.join(model_name, 'vae') +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = LlamaTokenizerFast.from_pretrained( + os.path.join(model_name, 'tokenizer'), +) + +# Get Text encoder +text_encoder = LlavaForConditionalGeneration.from_pretrained( + os.path.join(model_name, 'text_encoder'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +# Get Tokenizer 2 +tokenizer_2 = CLIPTokenizer.from_pretrained( + os.path.join(model_name, 'tokenizer_2'), +) + +# Get Text encoder 2 +text_encoder_2 = CLIPTextModel.from_pretrained( + os.path.join(model_name, 'text_encoder_2'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +# Get Image Processor +image_processor = CLIPImageProcessor.from_pretrained( + os.path.join(model_name, 'image_processor'), +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + os.path.join(model_name, 'scheduler'), +) + +# Get Pipeline +pipeline = HunyuanVideoI2VPipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + tokenizer_2=tokenizer_2, + text_encoder_2=text_encoder_2, + scheduler=scheduler, + image_processor=image_processor, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_embedder", "context_embedder", "time_text_embed", "rope", "proj_out"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_embedder", "context_embedder", "time_text_embed", "rope", "proj_out"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + # open + image = get_image(validation_image_start) + + sample = pipeline( + prompt, + image = image, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + true_cfg_scale = guidance_scale, + num_inference_steps = num_inference_steps, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/hunyuanvideo/predict_t2v.py b/VideoX-Fun/examples/hunyuanvideo/predict_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..966186b2645f2e472e0fe3b202f258267f404d42 --- /dev/null +++ b/VideoX-Fun/examples/hunyuanvideo/predict_t2v.py @@ -0,0 +1,247 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import export_to_video +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from diffusers.schedulers.scheduling_unipc_multistep import \ + UniPCMultistepScheduler + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLHunyuanVideo, CLIPTextModel, + CLIPTokenizer, HunyuanVideoTransformer3DModel, + LlamaModel, LlamaTokenizerFast) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import HunyuanVideoPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# model path +model_name = "models/Diffusion_Transformer/HunyuanVideo" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [832, 480] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +negative_prompt = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion. " +guidance_scale = 1.0 +seed = 43 +num_inference_steps = 40 +lora_weight = 0.55 +save_path = "samples/hunyuanvideo-videos-t2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = HunyuanVideoTransformer3DModel.from_pretrained( + os.path.join(model_name, 'transformer'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLHunyuanVideo.from_pretrained( + os.path.join(model_name, 'vae') +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = LlamaTokenizerFast.from_pretrained( + os.path.join(model_name, 'tokenizer'), +) + +# Get Text encoder +text_encoder = LlamaModel.from_pretrained( + os.path.join(model_name, 'text_encoder'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +# Get Tokenizer 2 +tokenizer_2 = CLIPTokenizer.from_pretrained( + os.path.join(model_name, 'tokenizer_2'), +) + +# Get Text encoder 2 +text_encoder_2 = CLIPTextModel.from_pretrained( + os.path.join(model_name, 'text_encoder_2'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + os.path.join(model_name, 'scheduler'), +) + +# Get Pipeline +pipeline = HunyuanVideoPipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + tokenizer_2=tokenizer_2, + text_encoder_2=text_encoder_2, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.layers) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_embedder", "context_embedder", "time_text_embed", "rope", "proj_out"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["x_embedder", "context_embedder", "time_text_embed", "rope", "proj_out"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + true_cfg_scale = guidance_scale, + num_inference_steps = num_inference_steps, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/phantom/predict_s2v.py b/VideoX-Fun/examples/phantom/predict_s2v.py new file mode 100644 index 0000000000000000000000000000000000000000..0d7e8f4e3e76db2e987d48f9873b9091acc01f64 --- /dev/null +++ b/VideoX-Fun/examples/phantom/predict_s2v.py @@ -0,0 +1,300 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, + WanT5EncoderModel, WanTransformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanFunPhantomPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | Model Name | threshold | +# | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | +# | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow_Unipc" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 3 + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/Phantom-Wan-1.3B.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +subject_ref_images = ["asset/ref_1.png", "asset/ref_2.png"] + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "暖阳漫过草地，扎着双马尾、头戴绿色蝴蝶结、身穿浅绿色连衣裙的小女孩蹲在盛开的雏菊旁。她身旁一只棕白相间的狗狗吐着舌头，毛茸茸尾巴欢快摇晃。小女孩笑着举起黄红配色、带有蓝色按钮的玩具相机，将和狗狗的欢乐瞬间定格。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/wan-videos-phantom" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = WanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = WanFunPhantomPipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + if subject_ref_images is not None: + subject_ref_images = [get_image_latent(_subject_ref_image, sample_size=sample_size, padding=True) for _subject_ref_image in subject_ref_images] + subject_ref_images = torch.cat(subject_ref_images, dim=2) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + subject_ref_images = subject_ref_images, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/qwenimage/predict_t2i.py b/VideoX-Fun/examples/qwenimage/predict_t2i.py new file mode 100644 index 0000000000000000000000000000000000000000..644afc1e4ef510eecba304ed960b0c8de9f6bc7b --- /dev/null +++ b/VideoX-Fun/examples/qwenimage/predict_t2i.py @@ -0,0 +1,237 @@ +import os +import sys + +import torch + +from diffusers import (FlowMatchEulerDiscreteScheduler) + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLQwenImage, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, QwenImageTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import QwenImagePipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +teacache_threshold = 0.250 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# model path +model_name = "models/Diffusion_Transformer/Qwen-Image" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [1344, 768] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +prompt = "1girl, black_hair, brown_eyes, earrings, freckles, grey_background, jewelry, lips, long_hair, looking_at_viewer, nose, piercing, realistic, red_lips, solo, upper_body" +negative_prompt = " " +guidance_scale = 4.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/qwenimage-t2i" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = QwenImageTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLQwenImage.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = Qwen2Tokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = QwenImagePipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + sample = pipeline( + prompt, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + true_cfg_scale = guidance_scale, + num_inference_steps = num_inference_steps, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/qwenimage/predict_t2i_edit.py b/VideoX-Fun/examples/qwenimage/predict_t2i_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..bc58f7872df7acc7d16e75b10e9796b9cb07ba9c --- /dev/null +++ b/VideoX-Fun/examples/qwenimage/predict_t2i_edit.py @@ -0,0 +1,255 @@ +import os +import sys + +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLQwenImage, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, Qwen2VLProcessor, + QwenImageTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import QwenImageEditPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import get_image + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +teacache_threshold = 0.250 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# model path +model_name = "models/Diffusion_Transformer/Qwen-Image-Edit" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [1344, 768] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +image = "asset/8.png" +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女子站在阳光明媚的海岸线上，身穿清爽的白色衬衫与裙子，在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" +negative_prompt = " " +guidance_scale = 4.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/qwenimage-t2i-edit" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = QwenImageTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLQwenImage.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = Qwen2Tokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype +) + +# Get processor +processor = Qwen2VLProcessor.from_pretrained( + model_name, + subfolder="processor" +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = QwenImageEditPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + processor=processor, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +# for prompt in prompts: +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +# open +image = get_image(image) + +with torch.no_grad(): + sample = pipeline( + prompt = prompt, + image = image, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + true_cfg_scale = guidance_scale, + num_inference_steps = num_inference_steps, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/qwenimage/predict_t2i_edit_plus.py b/VideoX-Fun/examples/qwenimage/predict_t2i_edit_plus.py new file mode 100644 index 0000000000000000000000000000000000000000..9a9e9e1b5373ad144150e49889da5d22287ce9e3 --- /dev/null +++ b/VideoX-Fun/examples/qwenimage/predict_t2i_edit_plus.py @@ -0,0 +1,255 @@ +import os +import sys + +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLQwenImage, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, Qwen2VLProcessor, + QwenImageTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import QwenImageEditPlusPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import get_image + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload_and_qfloat8" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +teacache_threshold = 0.30 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# model path +model_name = "models/Diffusion_Transformer/Qwen-Image-Edit-2509" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [1344, 768] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +image = ["asset/8.png", "asset/ref_1.png"] +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "女生拿着相机" +negative_prompt = " " +guidance_scale = 4.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/qwenimage-t2i-edit-plus" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = QwenImageTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLQwenImage.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = Qwen2Tokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype +) + +# Get processor +processor = Qwen2VLProcessor.from_pretrained( + model_name, + subfolder="processor" +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = QwenImageEditPlusPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + processor=processor, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +# for prompt in prompts: +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +# open +image = [get_image(_image) for _image in image] + +with torch.no_grad(): + sample = pipeline( + prompt = prompt, + image = image, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + true_cfg_scale = guidance_scale, + num_inference_steps = num_inference_steps, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1/app.py b/VideoX-Fun/examples/wan2.1/app.py new file mode 100644 index 0000000000000000000000000000000000000000..aee89fb9d189765ec9963c082699f8355e3aaf40 --- /dev/null +++ b/VideoX-Fun/examples/wan2.1/app.py @@ -0,0 +1,77 @@ +import os +import sys +import time + +import torch + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.api.api import (infer_forward_api, + update_diffusion_transformer_api) +from videox_fun.ui.controller import flow_scheduler_dict +from videox_fun.ui.wan_ui import ui, ui_client, ui_host + +if __name__ == "__main__": + # Choose the ui mode + # "normal" refers to the standard UI, which allows users to click to switch models, change model types, and more. + # "host" represents the hosting mode, where the model is loaded directly at startup and can be accessed via + # the API to return generation results. + # "client" represents the client mode, offering a simple UI that sends requests to a remote API for generation. + ui_mode = "normal" + + # GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. + # model_full_load means that the entire model will be moved to the GPU. + # + # model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. + # + # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, + # and the transformer model has been quantized to float8, which can save more GPU memory. + # + # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, + # resulting in slower speeds but saving a large amount of GPU memory. + GPU_memory_mode = "sequential_cpu_offload" + # Compile will give a speedup in fixed resolution and need a little GPU memory. + # The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. + compile_dit = False + + # Use torch.float16 if GPU does not support torch.bfloat16 + # ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 + weight_dtype = torch.bfloat16 + + # Server ip + server_name = "0.0.0.0" + server_port = 7860 + + # Config path + config_path = "config/wan2.1/wan_civitai.yaml" + # Params below is used when ui_mode = "host" + # Model path of the pretrained model + model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" + + if ui_mode == "host": + demo, controller = ui_host(GPU_memory_mode, flow_scheduler_dict, model_name, "Inpaint", config_path, compile_dit, weight_dtype) + elif ui_mode == "client": + demo, controller = ui_client(flow_scheduler_dict, model_name) + else: + demo, controller = ui(GPU_memory_mode, flow_scheduler_dict, config_path, compile_dit, weight_dtype) + + def gr_launch(): + # launch gradio + app, _, _ = demo.queue(status_update_rate=1).launch( + server_name=server_name, + server_port=server_port, + prevent_thread_lock=True + ) + + # launch api + infer_forward_api(None, app, controller) + update_diffusion_transformer_api(None, app, controller) + + gr_launch() + + # not close the python + while True: + time.sleep(5) \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1/launch_api.py b/VideoX-Fun/examples/wan2.1/launch_api.py new file mode 100644 index 0000000000000000000000000000000000000000..a9d8c436a75cadac372bea0fef7adbfc5386422f --- /dev/null +++ b/VideoX-Fun/examples/wan2.1/launch_api.py @@ -0,0 +1,91 @@ +import argparse +import os +import sys +import time + +import gradio as gr +import ray +import torch + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.api.api_multi_nodes import (MultiNodesEngine, + multi_nodes_infer_forward_api) +from videox_fun.ui.controller import flow_scheduler_dict +from videox_fun.ui.wan_ui import Wan_Controller + +def main(): + parser = argparse.ArgumentParser(description='xDiT HTTP Service') + parser.add_argument('--world_size', type=int, default=8, help='Number of parallel workers') + parser.add_argument( + '--gpu_memory_mode', type=str, default="model_cpu_offload", help=''' +GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8]. +model_full_load means that the entire model will be moved to the GPU. + +model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +and the transformer model has been quantized to float8, which can save more GPU memory. + +model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. + +model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +and the transformer model has been quantized to float8, which can save more GPU memory. + ''' + ) + parser.add_argument('--ulysses_degree', type=int, default=4, help='Degree of Ulysses configuration') + parser.add_argument('--ring_degree', type=int, default=2, help='Degree of Ring configuration') + parser.add_argument( + '--compile_dit', action='store_true', help=''' +Enable compile dit. +Compile will give a speedup in fixed resolution and need a little GPU memory. +The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. + ''' + ) + parser.add_argument('--fsdp_dit', action='store_true', help="Use DIT FSDP to save more GPU memory in multi gpus.") + parser.add_argument('--fsdp_text_encoder', action='store_true', help="Use Text Encoder FSDP to save more GPU memory in multi gpus.") + parser.add_argument('--weight_dtype', type=str, default='bf16', help='Weight data type') + parser.add_argument('--server_name', type=str, default="0.0.0.0", help='Server IP address') + parser.add_argument('--server_port', type=int, default=7860, help='Server Port') + parser.add_argument('--config_path', type=str, default="config/wan2.1/wan_civitai.yaml", help='Path to config file') + parser.add_argument('--model_name', type=str, default="models/Diffusion_Transformer/Wan2.1-T2V-1.3B", help='Model path') + parser.add_argument('--model_type', type=str, default="Inpaint", help='Model type (Inpaint/Control)') + parser.add_argument('--savedir_sample', type=str, default=None, help='The save directory for samples') + args = parser.parse_args() + + weight_dtype = torch.float32 + if args.weight_dtype == "bf16": + weight_dtype = torch.bfloat16 + elif args.weight_dtype == "fp16": + weight_dtype = torch.float16 + + engine = MultiNodesEngine( + world_size=args.world_size, Controller=Wan_Controller, + GPU_memory_mode=args.gpu_memory_mode, scheduler_dict=flow_scheduler_dict, model_name=args.model_name, model_type=args.model_type, config_path=args.config_path, + ulysses_degree=args.ulysses_degree, ring_degree=args.ring_degree, + fsdp_dit=args.fsdp_dit, fsdp_text_encoder=args.fsdp_text_encoder, compile_dit=args.compile_dit, + weight_dtype=weight_dtype, savedir_sample=args.savedir_sample, + ) + + def gr_launch(): + # launch gradio + with gr.Blocks() as demo: + gr.Markdown("") + app, _, _ = demo.queue(status_update_rate=1).launch( + server_name=args.server_name, + server_port=args.server_port, + prevent_thread_lock=True + ) + + # launch api + multi_nodes_infer_forward_api(None, app, engine) + + gr_launch() + + # not close the python + while True: + time.sleep(5) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1/post_infer.py b/VideoX-Fun/examples/wan2.1/post_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..a00245740ed05d5220640221ff0aca8b1bea080d --- /dev/null +++ b/VideoX-Fun/examples/wan2.1/post_infer.py @@ -0,0 +1,150 @@ +import base64 +import json +import time +from datetime import datetime + +import requests +import base64 + + +def post_diffusion_transformer(diffusion_transformer_path, url='http://127.0.0.1:7860'): + datas = json.dumps({ + "diffusion_transformer_path": diffusion_transformer_path + }) + r = requests.post(f'{url}/cogvideox_fun/update_diffusion_transformer', data=datas, timeout=1500) + data = r.content.decode('utf-8') + return data + +def post_update_edition(edition, url='http://0.0.0.0:7860'): + datas = json.dumps({ + "edition": edition + }) + r = requests.post(f'{url}/cogvideox_fun/update_edition', data=datas, timeout=1500) + data = r.content.decode('utf-8') + return data + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5000, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion.", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43 +): + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + }) + + # Initialize session and set headers + session = requests.session() + if url[-1] == "/": + url = url[:-1] + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + data = post_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # The Url you want to post + POST_URL = 'http://0.0.0.0:7860' + # Used in EAS. If you don't need Authorization, please set it to empty string. + TOKEN = '' + + # -------------------------- # + # Step 1: update edition + # -------------------------- # + # diffusion_transformer_path = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" + # outputs = post_diffusion_transformer(diffusion_transformer_path) + # print('Output update edition: ', outputs) + + # -------------------------- # + # Step 2: infer + # -------------------------- # + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 49 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + negative_prompt_textbox = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion." + # Sampler name + sampler_dropdown = "Flow_Unipc" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + url=POST_URL, + POST_TOKEN=TOKEN + ) + + # Get decoded data + outputs = json.loads(outputs) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1/post_infer_queue.py b/VideoX-Fun/examples/wan2.1/post_infer_queue.py new file mode 100644 index 0000000000000000000000000000000000000000..872d4de7a4a1187ab58de900d9ad50a449883034 --- /dev/null +++ b/VideoX-Fun/examples/wan2.1/post_infer_queue.py @@ -0,0 +1,192 @@ +import base64 +import json +import time +import urllib.parse +import requests + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, +): + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # Support TeaCache. + enable_teacache = True + # Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, + # but it may cause slight differences between the generated content and the original content. + # # --------------------------------------------------------------------------------------------------- # + # | Model Name | threshold | Model Name | threshold | Model Name | threshold | + # | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | + # | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | + # # --------------------------------------------------------------------------------------------------- # + teacache_threshold = 0.10 + # The number of steps to skip TeaCache at the beginning of the inference process, which can + # reduce the impact of TeaCache on generated video quality. + num_skip_start_steps = 5 + # Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. + teacache_offload = False + + # Skip some cfg steps in inference + # Recommended to be set between 0.00 and 0.25 + cfg_skip_ratio = 0 + + # Riflex config + enable_riflex = False + # Index of intrinsic frequency + riflex_k = 6 + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 81 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + negative_prompt_textbox = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + # Sampler name + sampler_dropdown = "Flow_Unipc" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + url=EAS_URL, + POST_TOKEN=TOKEN + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1/post_infer_queue_i2v.py b/VideoX-Fun/examples/wan2.1/post_infer_queue_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..f453cc35ddb515d56ab429a5d8af02f9f21a5a69 --- /dev/null +++ b/VideoX-Fun/examples/wan2.1/post_infer_queue_i2v.py @@ -0,0 +1,212 @@ +import base64 +import json +import time +import urllib.parse +import requests +from PIL import Image +from io import BytesIO + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + start_image=None +): + if start_image: + try: + if not start_image.startswith("http"): + image = Image.open(start_image).convert("RGB") + # 将图片转换为 Base64 编码 + buffered = BytesIO() + image.save(buffered, format="JPEG") + start_image = base64.b64encode(buffered.getvalue()).decode('utf-8') + except Exception as e: + print(f"Error processing start_image: {e}") + raise + + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + "start_image": start_image + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # Support TeaCache. + enable_teacache = True + # Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, + # but it may cause slight differences between the generated content and the original content. + # # --------------------------------------------------------------------------------------------------- # + # | Model Name | threshold | Model Name | threshold | Model Name | threshold | + # | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | + # | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | + # # --------------------------------------------------------------------------------------------------- # + teacache_threshold = 0.10 + # The number of steps to skip TeaCache at the beginning of the inference process, which can + # reduce the impact of TeaCache on generated video quality. + num_skip_start_steps = 5 + # Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. + teacache_offload = False + + # Skip some cfg steps in inference + # Recommended to be set between 0.00 and 0.25 + cfg_skip_ratio = 0 + + # Riflex config + enable_riflex = False + # Index of intrinsic frequency + riflex_k = 6 + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 81 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" + negative_prompt_textbox = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + # Sampler name + sampler_dropdown = "Flow_Unipc" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + # 起始图片路径 + start_image_path = "asset/1.png" # 替换为实际的图片路径 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + url=EAS_URL, + POST_TOKEN=TOKEN, + start_image=start_image_path # 传递起始图片路径 + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') diff --git a/VideoX-Fun/examples/wan2.1/predict_i2v.py b/VideoX-Fun/examples/wan2.1/predict_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..c2de0184e047f463d50acdbd403389d9a7d1381d --- /dev/null +++ b/VideoX-Fun/examples/wan2.1/predict_i2v.py @@ -0,0 +1,301 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, WanTransformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanI2VPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | Model Name | threshold | +# | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | +# | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-I2V-14B-480P" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow_Unipc" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 3 + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# If you want to generate from text, please set the validation_image_start = None and validation_image_end = None +validation_image_start = "asset/1.png" + +# prompts +prompt = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/wan-videos-i2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = WanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Clip Image Encoder +clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(model_name, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), +).to(weight_dtype) +clip_image_encoder = clip_image_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = WanI2VPipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, clip_image = get_image_to_video_latent(validation_image_start, None, video_length=video_length, sample_size=sample_size) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1/predict_t2v.py b/VideoX-Fun/examples/wan2.1/predict_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..2e96d13b8085080cfefb3d4eeec69de05b4660cb --- /dev/null +++ b/VideoX-Fun/examples/wan2.1/predict_t2v.py @@ -0,0 +1,282 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, WanT5EncoderModel, AutoTokenizer, + WanTransformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | Model Name | threshold | +# | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | +# | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-T2V-1.3B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow_Unipc" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 3 + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +prompt = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/wan-videos-t2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = WanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = WanPipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_fun/app.py b/VideoX-Fun/examples/wan2.1_fun/app.py new file mode 100644 index 0000000000000000000000000000000000000000..80b459d78f5941bc06671c925574a7a8d9b8b600 --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/app.py @@ -0,0 +1,79 @@ +import os +import sys +import time + +import torch + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.api.api import (infer_forward_api, + update_diffusion_transformer_api) +from videox_fun.ui.controller import flow_scheduler_dict +from videox_fun.ui.wan_fun_ui import ui, ui_client, ui_host + +if __name__ == "__main__": + # Choose the ui mode + # "normal" refers to the standard UI, which allows users to click to switch models, change model types, and more. + # "host" represents the hosting mode, where the model is loaded directly at startup and can be accessed via + # the API to return generation results. + # "client" represents the client mode, offering a simple UI that sends requests to a remote API for generation. + ui_mode = "normal" + + # GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. + # model_full_load means that the entire model will be moved to the GPU. + # + # model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. + # + # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, + # and the transformer model has been quantized to float8, which can save more GPU memory. + # + # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, + # resulting in slower speeds but saving a large amount of GPU memory. + GPU_memory_mode = "sequential_cpu_offload" + # Compile will give a speedup in fixed resolution and need a little GPU memory. + # The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. + compile_dit = False + + # Use torch.float16 if GPU does not support torch.bfloat16 + # ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 + weight_dtype = torch.bfloat16 + + # Server ip + server_name = "0.0.0.0" + server_port = 7860 + + # Config path + config_path = "config/wan2.1/wan_civitai.yaml" + # Params below is used when ui_mode = "host" + # Model path of the pretrained model + model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" + # "Inpaint" or "Control" + model_type = "Inpaint" + + if ui_mode == "host": + demo, controller = ui_host(GPU_memory_mode, flow_scheduler_dict, model_name, model_type, config_path, compile_dit, weight_dtype) + elif ui_mode == "client": + demo, controller = ui_client(flow_scheduler_dict, model_name) + else: + demo, controller = ui(GPU_memory_mode, flow_scheduler_dict, config_path, compile_dit, weight_dtype) + + def gr_launch(): + # launch gradio + app, _, _ = demo.queue(status_update_rate=1).launch( + server_name=server_name, + server_port=server_port, + prevent_thread_lock=True + ) + + # launch api + infer_forward_api(None, app, controller) + update_diffusion_transformer_api(None, app, controller) + + gr_launch() + + # not close the python + while True: + time.sleep(5) \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_fun/launch_api.py b/VideoX-Fun/examples/wan2.1_fun/launch_api.py new file mode 100644 index 0000000000000000000000000000000000000000..2f900944e7ad53baa7065e334f9298ccfcc51a6b --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/launch_api.py @@ -0,0 +1,91 @@ +import argparse +import os +import sys +import time + +import gradio as gr +import ray +import torch + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.api.api_multi_nodes import (MultiNodesEngine, + multi_nodes_infer_forward_api) +from videox_fun.ui.controller import flow_scheduler_dict +from videox_fun.ui.wan_fun_ui import Wan_Fun_Controller + +def main(): + parser = argparse.ArgumentParser(description='xDiT HTTP Service') + parser.add_argument('--world_size', type=int, default=8, help='Number of parallel workers') + parser.add_argument( + '--gpu_memory_mode', type=str, default="model_full_load", help=''' +GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8]. +model_full_load means that the entire model will be moved to the GPU. + +model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +and the transformer model has been quantized to float8, which can save more GPU memory. + +model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. + +model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +and the transformer model has been quantized to float8, which can save more GPU memory. + ''' + ) + parser.add_argument('--ulysses_degree', type=int, default=4, help='Degree of Ulysses configuration') + parser.add_argument('--ring_degree', type=int, default=2, help='Degree of Ring configuration') + parser.add_argument( + '--compile_dit', action='store_true', help=''' +Enable compile dit. +Compile will give a speedup in fixed resolution and need a little GPU memory. +The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. + ''' + ) + parser.add_argument('--fsdp_dit', action='store_true', help="Use DIT FSDP to save more GPU memory in multi gpus.") + parser.add_argument('--fsdp_text_encoder', action='store_true', help="Use Text Encoder FSDP to save more GPU memory in multi gpus.") + parser.add_argument('--weight_dtype', type=str, default='bf16', help='Weight data type') + parser.add_argument('--server_name', type=str, default="0.0.0.0", help='Server IP address') + parser.add_argument('--server_port', type=int, default=7860, help='Server Port') + parser.add_argument('--config_path', type=str, default="config/wan2.1/wan_civitai.yaml", help='Path to config file') + parser.add_argument('--model_name', type=str, default="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP", help='Model path') + parser.add_argument('--model_type', type=str, default="Inpaint", help='Model type (Inpaint/Control)') + parser.add_argument('--savedir_sample', type=str, default=None, help='The save directory for samples') + args = parser.parse_args() + + weight_dtype = torch.float32 + if args.weight_dtype == "bf16": + weight_dtype = torch.bfloat16 + elif args.weight_dtype == "fp16": + weight_dtype = torch.float16 + + engine = MultiNodesEngine( + world_size=args.world_size, Controller=Wan_Fun_Controller, + GPU_memory_mode=args.gpu_memory_mode, scheduler_dict=flow_scheduler_dict, model_name=args.model_name, model_type=args.model_type, config_path=args.config_path, + ulysses_degree=args.ulysses_degree, ring_degree=args.ring_degree, + fsdp_dit=args.fsdp_dit, fsdp_text_encoder=args.fsdp_text_encoder, compile_dit=args.compile_dit, + weight_dtype=weight_dtype, savedir_sample=args.savedir_sample, + ) + + def gr_launch(): + # launch gradio + with gr.Blocks() as demo: + gr.Markdown("") + app, _, _ = demo.queue(status_update_rate=1).launch( + server_name=args.server_name, + server_port=args.server_port, + prevent_thread_lock=True + ) + + # launch api + multi_nodes_infer_forward_api(None, app, engine) + + gr_launch() + + # not close the python + while True: + time.sleep(5) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_fun/post_infer.py b/VideoX-Fun/examples/wan2.1_fun/post_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..0493c9baec8f6a2b568ddba1c8a14210fb878f01 --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/post_infer.py @@ -0,0 +1,150 @@ +import base64 +import json +import time +from datetime import datetime + +import requests +import base64 + + +def post_diffusion_transformer(diffusion_transformer_path, url='http://127.0.0.1:7860'): + datas = json.dumps({ + "diffusion_transformer_path": diffusion_transformer_path + }) + r = requests.post(f'{url}/videox_fun/update_diffusion_transformer', data=datas, timeout=1500) + data = r.content.decode('utf-8') + return data + +def post_update_edition(edition, url='http://0.0.0.0:7860'): + datas = json.dumps({ + "edition": edition + }) + r = requests.post(f'{url}/videox_fun/update_edition', data=datas, timeout=1500) + data = r.content.decode('utf-8') + return data + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5000, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion.", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43 +): + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + data = post_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # The Url you want to post + POST_URL = 'http://0.0.0.0:7860' + # Used in EAS. If you don't need Authorization, please set it to empty string. + TOKEN = '' + + # -------------------------- # + # Step 1: update edition + # -------------------------- # + # diffusion_transformer_path = "models/Diffusion_Transformer/Wan2.1-Fun-1.3B-InP" + # outputs = post_diffusion_transformer(diffusion_transformer_path) + # print('Output update edition: ', outputs) + + # -------------------------- # + # Step 2: infer + # -------------------------- # + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 49 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + negative_prompt_textbox = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion." + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + url=POST_URL, + POST_TOKEN=TOKEN + ) + + # Get decoded data + outputs = json.loads(outputs) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_fun/post_infer_queue.py b/VideoX-Fun/examples/wan2.1_fun/post_infer_queue.py new file mode 100644 index 0000000000000000000000000000000000000000..bc6e8682922f54298ab8b52707aa16d9bdf298a7 --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/post_infer_queue.py @@ -0,0 +1,192 @@ +import base64 +import json +import time +import urllib.parse +import requests + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, +): + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # Support TeaCache. + enable_teacache = True + # Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, + # but it may cause slight differences between the generated content and the original content. + # # --------------------------------------------------------------------------------------------------- # + # | Model Name | threshold | Model Name | threshold | Model Name | threshold | + # | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | + # | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | + # # --------------------------------------------------------------------------------------------------- # + teacache_threshold = 0.10 + # The number of steps to skip TeaCache at the beginning of the inference process, which can + # reduce the impact of TeaCache on generated video quality. + num_skip_start_steps = 5 + # Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. + teacache_offload = False + + # Skip some cfg steps in inference + # Recommended to be set between 0.00 and 0.25 + cfg_skip_ratio = 0 + + # Riflex config + enable_riflex = False + # Index of intrinsic frequency + riflex_k = 6 + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 81 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + negative_prompt_textbox = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + url=EAS_URL, + POST_TOKEN=TOKEN + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_fun/post_infer_queue_i2v.py b/VideoX-Fun/examples/wan2.1_fun/post_infer_queue_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..d24d20bfa173c88df2f96944cd42f8b34ed6b9d5 --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/post_infer_queue_i2v.py @@ -0,0 +1,213 @@ +import base64 +import json +import time +import urllib.parse +import requests +from PIL import Image +from io import BytesIO + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + start_image = None +): + if start_image: + try: + if not start_image.startswith("http"): + image = Image.open(start_image).convert("RGB") + # 将图片转换为 Base64 编码 + buffered = BytesIO() + image.save(buffered, format="JPEG") + start_image = base64.b64encode(buffered.getvalue()).decode('utf-8') + except Exception as e: + print(f"Error processing start_image: {e}") + raise + + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + "start_image": start_image + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # Support TeaCache. + enable_teacache = True + # Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, + # but it may cause slight differences between the generated content and the original content. + # # --------------------------------------------------------------------------------------------------- # + # | Model Name | threshold | Model Name | threshold | Model Name | threshold | + # | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | + # | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | + # # --------------------------------------------------------------------------------------------------- # + teacache_threshold = 0.10 + # The number of steps to skip TeaCache at the beginning of the inference process, which can + # reduce the impact of TeaCache on generated video quality. + num_skip_start_steps = 5 + # Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. + teacache_offload = False + + # Skip some cfg steps in inference + # Recommended to be set between 0.00 and 0.25 + cfg_skip_ratio = 0 + + # Riflex config + enable_riflex = False + # Index of intrinsic frequency + riflex_k = 6 + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 81 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" + negative_prompt_textbox = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + # 起始图片路径 + start_image_path = "asset/1.png" # 替换为实际的图片路径 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + url=EAS_URL, + POST_TOKEN=TOKEN, + start_image=start_image_path # 传递起始图片路径 + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') diff --git a/VideoX-Fun/examples/wan2.1_fun/post_infer_queue_v2v_control.py b/VideoX-Fun/examples/wan2.1_fun/post_infer_queue_v2v_control.py new file mode 100644 index 0000000000000000000000000000000000000000..23a6c58b297b2473ebd3e7555f3501d858b2a01b --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/post_infer_queue_v2v_control.py @@ -0,0 +1,230 @@ +import base64 +import json +import time +import urllib.parse +from io import BytesIO + +import requests +from PIL import Image + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + control_video = None, + ref_image = None +): + if control_video: + try: + if not control_video.startswith("http"): + with open(control_video, "rb") as file: + video_data = file.read() + + control_video = base64.b64encode(video_data).decode('utf-8') + except Exception as e: + print(f"Error processing control_video: {e}") + raise + + if ref_image: + try: + if not ref_image.startswith("http"): + image = Image.open(ref_image).convert("RGB") + # 将图片转换为 Base64 编码 + buffered = BytesIO() + image.save(buffered, format="JPEG") + ref_image = base64.b64encode(buffered.getvalue()).decode('utf-8') + except Exception as e: + print(f"Error processing ref_image: {e}") + raise + + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "ref_image": ref_image, + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + "control_video": control_video + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # Support TeaCache. + enable_teacache = True + # Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, + # but it may cause slight differences between the generated content and the original content. + # # --------------------------------------------------------------------------------------------------- # + # | Model Name | threshold | Model Name | threshold | Model Name | threshold | + # | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | + # | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | + # # --------------------------------------------------------------------------------------------------- # + teacache_threshold = 0.10 + # The number of steps to skip TeaCache at the beginning of the inference process, which can + # reduce the impact of TeaCache on generated video quality. + num_skip_start_steps = 5 + # Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. + teacache_offload = False + + # Skip some cfg steps in inference + # Recommended to be set between 0.00 and 0.25 + cfg_skip_ratio = 0 + + # Riflex config + enable_riflex = False + # Index of intrinsic frequency + riflex_k = 6 + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 81 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + negative_prompt_textbox = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 480 + height_slider = 832 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + # 控制视频路径（可以是本地路径或 URL） + control_video_path = "asset/000000.mp4" # 替换为实际的视频路径 + # 参考图片路径 + ref_image_path = None # 替换为实际的图片路径 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + url=EAS_URL, + POST_TOKEN=TOKEN, + control_video=control_video_path, # 传递控制视频路径 + ref_image=ref_image_path # 传递参考图片路径 + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') diff --git a/VideoX-Fun/examples/wan2.1_fun/predict_i2v.py b/VideoX-Fun/examples/wan2.1_fun/predict_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..dc86a68b109299623bdb104607f8f251c5accffe --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/predict_i2v.py @@ -0,0 +1,302 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanFunInpaintPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | Model Name | threshold | +# | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | +# | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 3 + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# If you want to generate from text, please set the validation_image_start = None and validation_image_end = None +validation_image_start = "asset/1.png" +validation_image_end = None + +# prompts +prompt = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/wan-videos-fun-i2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = WanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Clip Image Encoder +clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(model_name, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), +).to(weight_dtype) +clip_image_encoder = clip_image_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = WanFunInpaintPipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, clip_image = get_image_to_video_latent(validation_image_start, validation_image_end, video_length=video_length, sample_size=sample_size) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_fun/predict_t2v.py b/VideoX-Fun/examples/wan2.1_fun/predict_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..bfb27598f068f763867be07ea978b52ade566e61 --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/predict_t2v.py @@ -0,0 +1,321 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanFunInpaintPipeline, WanFunPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | Model Name | threshold | +# | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | +# | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 3 + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +prompt = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/wan-videos-fun-t2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = WanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +if transformer.config.in_channels != vae.config.latent_channels: + # Get Clip Image Encoder + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(model_name, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ).to(weight_dtype) + clip_image_encoder = clip_image_encoder.eval() +else: + clip_image_encoder = None + clip_image_processor = None + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +if transformer.config.in_channels != vae.config.latent_channels: + pipeline = WanFunInpaintPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) +else: + pipeline = WanFunPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + if transformer.config.in_channels != vae.config.latent_channels: + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=sample_size) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = input_video, + mask_video = input_video_mask, + shift = shift, + ).videos + else: + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_fun/predict_v2v_control.py b/VideoX-Fun/examples/wan2.1_fun/predict_v2v_control.py new file mode 100644 index 0000000000000000000000000000000000000000..85a74340ebeedf1f91aa8be3002891ee31534532 --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/predict_v2v_control.py @@ -0,0 +1,333 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, WanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanFunControlPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | Model Name | threshold | +# | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | +# | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-Control" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 3 + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [832, 480] +video_length = 49 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = "asset/pose.mp4" +control_camera_txt = None +start_image = None +ref_image = None + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/wan-videos-fun-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = WanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Clip Image Encoder +clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(model_name, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), +).to(weight_dtype) +clip_image_encoder = clip_image_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = WanFunControlPipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + if ref_image is not None: + clip_image = Image.open(ref_image).convert("RGB") + elif start_image is not None: + clip_image = Image.open(start_image).convert("RGB") + else: + clip_image = None + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=sample_size) + + if start_image is not None: + start_image = get_image_latent(start_image, sample_size=sample_size) + + if control_camera_txt is not None: + input_video, input_video_mask = None, None + control_camera_video = process_pose_file(control_camera_txt, sample_size[1], sample_size[0]) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + else: + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + control_camera_video = None + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + control_video = input_video, + control_camera_video = control_camera_video, + ref_image = ref_image, + start_image = start_image, + clip_image = clip_image, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_fun/predict_v2v_control_camera.py b/VideoX-Fun/examples/wan2.1_fun/predict_v2v_control_camera.py new file mode 100644 index 0000000000000000000000000000000000000000..40adc1308c712fb04d7064917287b7531976b5db --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/predict_v2v_control_camera.py @@ -0,0 +1,333 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, WanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanFunControlPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | Model Name | threshold | +# | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | +# | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-Control-Camera" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 3 + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = None +control_camera_txt = "asset/Pan_Left.txt" +start_image = "asset/7.png" +ref_image = None + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一个小女孩正在户外玩耍。她穿着一件蓝色的短袖上衣和粉色的短裤，头发扎成一个可爱的辫子。她的脚上没有穿鞋，显得非常自然和随意。她正用一把红色的小铲子在泥土里挖土，似乎在进行某种有趣的活动，可能是种花或是挖掘宝藏。地上有一根长长的水管，可能是用来浇水的。背景是一片草地和一些绿色植物，阳光明媚，整个场景充满了童趣和生机。小女孩专注的表情和认真的动作让人感受到她的快乐和好奇心。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/wan-videos-fun-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = WanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Clip Image Encoder +clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(model_name, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), +).to(weight_dtype) +clip_image_encoder = clip_image_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = WanFunControlPipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + if ref_image is not None: + clip_image = Image.open(ref_image).convert("RGB") + elif start_image is not None: + clip_image = Image.open(start_image).convert("RGB") + else: + clip_image = None + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=sample_size) + + if start_image is not None: + start_image = get_image_latent(start_image, sample_size=sample_size) + + if control_camera_txt is not None: + input_video, input_video_mask = None, None + control_camera_video = process_pose_file(control_camera_txt, sample_size[1], sample_size[0]) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + else: + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + control_camera_video = None + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + control_video = input_video, + control_camera_video = control_camera_video, + ref_image = ref_image, + start_image = start_image, + clip_image = clip_image, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_fun/predict_v2v_control_ref.py b/VideoX-Fun/examples/wan2.1_fun/predict_v2v_control_ref.py new file mode 100644 index 0000000000000000000000000000000000000000..b71154434333a8e9c1103e18e0f53fc93943d094 --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_fun/predict_v2v_control_ref.py @@ -0,0 +1,333 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, WanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanFunControlPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | Model Name | threshold | +# | Wan2.1-T2V-1.3B | 0.05~0.10 | Wan2.1-T2V-14B | 0.10~0.15 | Wan2.1-I2V-14B-720P | 0.20~0.30 | +# | Wan2.1-I2V-14B-480P | 0.20~0.25 | Wan2.1-Fun-*-1.3B-* | 0.05~0.10 | Wan2.1-Fun-*-14B-* | 0.20~0.30 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-Control" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 3 + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [832, 480] +video_length = 49 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = "asset/pose.mp4" +control_camera_txt = None +start_image = None +ref_image = "asset/6.png" + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女性穿着一件粉色的连衣裙，裙子上有白色的装饰和粉色的纽扣。她的头发是紫色的，头上戴着一个红色的大蝴蝶结，显得非常可爱和精致。她还戴着一个红色的领结，整体造型充满了少女感和活力。她的表情温柔，双手轻轻交叉放在身前，姿态优雅。背景是简单的灰色，没有任何多余的装饰，使得人物更加突出。她的妆容清淡自然，突显了她的清新气质。整体画面给人一种甜美、梦幻的感觉，仿佛置身于童话世界中。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +lora_weight = 0.55 +save_path = "samples/wan-videos-fun-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = WanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Clip Image Encoder +clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(model_name, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), +).to(weight_dtype) +clip_image_encoder = clip_image_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = WanFunControlPipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + if ref_image is not None: + clip_image = Image.open(ref_image).convert("RGB") + elif start_image is not None: + clip_image = Image.open(start_image).convert("RGB") + else: + clip_image = None + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=sample_size) + + if start_image is not None: + start_image = get_image_latent(start_image, sample_size=sample_size) + + if control_camera_txt is not None: + input_video, input_video_mask = None, None + control_camera_video = process_pose_file(control_camera_txt, sample_size[1], sample_size[0]) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + else: + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + control_camera_video = None + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + control_video = input_video, + control_camera_video = control_camera_video, + ref_image = ref_image, + start_image = start_image, + clip_image = clip_image, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_vace/predict_i2v.py b/VideoX-Fun/examples/wan2.1_vace/predict_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..050d4010ac47e39042d6786437fc794ce6ac9c5c --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_vace/predict_i2v.py @@ -0,0 +1,311 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, + WanT5EncoderModel, VaceWanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanVacePipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.1-VACE-1.3B | 0.05~0.10 | Wan2.1-VACE-14B | 0.10~0.15 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.05 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-VACE-1.3B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow_Unipc" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 16 + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = None +start_image = "asset/1.png" +end_image = None +subject_ref_images = None +vace_context_scale = 1.00 + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一只棕色的狗舔了一下它的舌头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 5.0 +seed = 43 +num_inference_steps = 40 +lora_weight = 0.55 +save_path = "samples/vace-videos" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = WanVacePipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + if subject_ref_images is not None: + subject_ref_images = [get_image_latent(_subject_ref_image, sample_size=sample_size, padding=True) for _subject_ref_image in subject_ref_images] + subject_ref_images = torch.cat(subject_ref_images, dim=2) + + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + control_video, _, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = subject_ref_images, + shift = shift, + vace_context_scale = vace_context_scale, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_vace/predict_s2v.py b/VideoX-Fun/examples/wan2.1_vace/predict_s2v.py new file mode 100644 index 0000000000000000000000000000000000000000..c44c937af0a4cee8e848c3a03f95eaa9817dcfba --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_vace/predict_s2v.py @@ -0,0 +1,311 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, + WanT5EncoderModel, VaceWanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanVacePipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.1-VACE-1.3B | 0.05~0.10 | Wan2.1-VACE-14B | 0.10~0.15 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.05 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-VACE-1.3B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow_Unipc" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 16 + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [832, 480] +video_length = 81 +fps = 16 +vace_context_scale = 1.00 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = None +start_image = None +end_image = None +subject_ref_images = ["asset/ref_1.png", "asset/ref_2.png"] + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "暖阳漫过草地，扎着双马尾、头戴绿色蝴蝶结、身穿浅绿色连衣裙的小女孩蹲在盛开的雏菊旁。她身旁一只棕白相间的狗狗吐着舌头，毛茸茸尾巴欢快摇晃。小女孩笑着举起黄红配色、带有蓝色按钮的玩具相机，将和狗狗的欢乐瞬间定格。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 5.0 +seed = 43 +num_inference_steps = 40 +lora_weight = 0.55 +save_path = "samples/vace-videos" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = WanVacePipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + if subject_ref_images is not None: + subject_ref_images = [get_image_latent(_subject_ref_image, sample_size=sample_size, padding=True) for _subject_ref_image in subject_ref_images] + subject_ref_images = torch.cat(subject_ref_images, dim=2) + + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + control_video, _, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = subject_ref_images, + shift = shift, + vace_context_scale = vace_context_scale, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.1_vace/predict_v2v_control.py b/VideoX-Fun/examples/wan2.1_vace/predict_v2v_control.py new file mode 100644 index 0000000000000000000000000000000000000000..2c09c1892c6cfe796864afda4a1fa2539650850e --- /dev/null +++ b/VideoX-Fun/examples/wan2.1_vace/predict_v2v_control.py @@ -0,0 +1,311 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, + WanT5EncoderModel, VaceWanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import WanVacePipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.1-VACE-1.3B | 0.05~0.10 | Wan2.1-VACE-14B | 0.10~0.15 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.05 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.1/wan_civitai.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.1-VACE-1.3B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow_Unipc" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 16 + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [832, 480] +video_length = 81 +fps = 16 +vace_context_scale = 1.00 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = "asset/pose.mp4" +start_image = None +end_image = None +subject_ref_images = None + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女子站在阳光明媚的海岸线上，身穿深蓝色背心与清爽的白色衬衫，外搭一条简洁的白色围裙，围裙在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 5.0 +seed = 43 +num_inference_steps = 40 +lora_weight = 0.55 +save_path = "samples/vace-videos" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKLWan.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = WanVacePipeline( + transformer=transformer, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + + if subject_ref_images is not None: + subject_ref_images = [get_image_latent(_subject_ref_image, sample_size=sample_size, padding=True) for _subject_ref_image in subject_ref_images] + subject_ref_images = torch.cat(subject_ref_images, dim=2) + + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + control_video, _, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = subject_ref_images, + shift = shift, + vace_context_scale = vace_context_scale, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2/app.py b/VideoX-Fun/examples/wan2.2/app.py new file mode 100644 index 0000000000000000000000000000000000000000..07f33bcb75815636b6bbcbbd08c73ac5bf7276de --- /dev/null +++ b/VideoX-Fun/examples/wan2.2/app.py @@ -0,0 +1,79 @@ +import os +import sys +import time + +import torch + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.api.api import (infer_forward_api, + update_diffusion_transformer_api) +from videox_fun.ui.controller import flow_scheduler_dict +from videox_fun.ui.wan2_2_ui import ui, ui_client, ui_host + +if __name__ == "__main__": + # Choose the ui mode + # "normal" refers to the standard UI, which allows users to click to switch models, change model types, and more. + # "host" represents the hosting mode, where the model is loaded directly at startup and can be accessed via + # the API to return generation results. + # "client" represents the client mode, offering a simple UI that sends requests to a remote API for generation. + ui_mode = "normal" + + # GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. + # model_full_load means that the entire model will be moved to the GPU. + # + # model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. + # + # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, + # and the transformer model has been quantized to float8, which can save more GPU memory. + # + # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, + # resulting in slower speeds but saving a large amount of GPU memory. + GPU_memory_mode = "sequential_cpu_offload" + # Compile will give a speedup in fixed resolution and need a little GPU memory. + # The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. + compile_dit = False + + # Use torch.float16 if GPU does not support torch.bfloat16 + # ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 + weight_dtype = torch.bfloat16 + + # Server ip + server_name = "0.0.0.0" + server_port = 7860 + + # Config path + config_path = "config/wan2.2/wan_civitai_i2v.yaml" + # Params below is used when ui_mode = "host" + # Model path of the pretrained model + model_name = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" + # "Inpaint" or "Control" + model_type = "Inpaint" + + if ui_mode == "host": + demo, controller = ui_host(GPU_memory_mode, flow_scheduler_dict, model_name, model_type, config_path, compile_dit, weight_dtype) + elif ui_mode == "client": + demo, controller = ui_client(flow_scheduler_dict, model_name) + else: + demo, controller = ui(GPU_memory_mode, flow_scheduler_dict, config_path, compile_dit, weight_dtype) + + def gr_launch(): + # launch gradio + app, _, _ = demo.queue(status_update_rate=1).launch( + server_name=server_name, + server_port=server_port, + prevent_thread_lock=True + ) + + # launch api + infer_forward_api(None, app, controller) + update_diffusion_transformer_api(None, app, controller) + + gr_launch() + + # not close the python + while True: + time.sleep(5) \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2/launch_api.py b/VideoX-Fun/examples/wan2.2/launch_api.py new file mode 100644 index 0000000000000000000000000000000000000000..c5d30053d9292fb09600f750bf2c4dd52423d64b --- /dev/null +++ b/VideoX-Fun/examples/wan2.2/launch_api.py @@ -0,0 +1,91 @@ +import argparse +import os +import sys +import time + +import gradio as gr +import ray +import torch + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.api.api_multi_nodes import (MultiNodesEngine, + multi_nodes_infer_forward_api) +from videox_fun.ui.controller import flow_scheduler_dict +from videox_fun.ui.wan2_2_ui import Wan2_2_Controller + +def main(): + parser = argparse.ArgumentParser(description='xDiT HTTP Service') + parser.add_argument('--world_size', type=int, default=8, help='Number of parallel workers') + parser.add_argument( + '--gpu_memory_mode', type=str, default="model_full_load", help=''' +GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8]. +model_full_load means that the entire model will be moved to the GPU. + +model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +and the transformer model has been quantized to float8, which can save more GPU memory. + +model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. + +model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +and the transformer model has been quantized to float8, which can save more GPU memory. + ''' + ) + parser.add_argument('--ulysses_degree', type=int, default=4, help='Degree of Ulysses configuration') + parser.add_argument('--ring_degree', type=int, default=2, help='Degree of Ring configuration') + parser.add_argument( + '--compile_dit', action='store_true', help=''' +Enable compile dit. +Compile will give a speedup in fixed resolution and need a little GPU memory. +The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. + ''' + ) + parser.add_argument('--fsdp_dit', action='store_true', help="Use DIT FSDP to save more GPU memory in multi gpus.") + parser.add_argument('--fsdp_text_encoder', action='store_true', help="Use Text Encoder FSDP to save more GPU memory in multi gpus.") + parser.add_argument('--weight_dtype', type=str, default='bf16', help='Weight data type') + parser.add_argument('--server_name', type=str, default="0.0.0.0", help='Server IP address') + parser.add_argument('--server_port', type=int, default=7860, help='Server Port') + parser.add_argument('--config_path', type=str, default="config/wan2.2/wan_civitai_i2v.yaml", help='Path to config file') + parser.add_argument('--model_name', type=str, default="models/Diffusion_Transformer/Wan2.2-I2V-A14B", help='Model path') + parser.add_argument('--model_type', type=str, default="Inpaint", help='Model type (Inpaint/Control)') + parser.add_argument('--savedir_sample', type=str, default=None, help='The save directory for samples') + args = parser.parse_args() + + weight_dtype = torch.float32 + if args.weight_dtype == "bf16": + weight_dtype = torch.bfloat16 + elif args.weight_dtype == "fp16": + weight_dtype = torch.float16 + + engine = MultiNodesEngine( + world_size=args.world_size, Controller=Wan2_2_Controller, + GPU_memory_mode=args.gpu_memory_mode, scheduler_dict=flow_scheduler_dict, model_name=args.model_name, model_type=args.model_type, config_path=args.config_path, + ulysses_degree=args.ulysses_degree, ring_degree=args.ring_degree, + fsdp_dit=args.fsdp_dit, fsdp_text_encoder=args.fsdp_text_encoder, compile_dit=args.compile_dit, + weight_dtype=weight_dtype, savedir_sample=args.savedir_sample, + ) + + def gr_launch(): + # launch gradio + with gr.Blocks() as demo: + gr.Markdown("") + app, _, _ = demo.queue(status_update_rate=1).launch( + server_name=args.server_name, + server_port=args.server_port, + prevent_thread_lock=True + ) + + # launch api + multi_nodes_infer_forward_api(None, app, engine) + + gr_launch() + + # not close the python + while True: + time.sleep(5) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2/post_infer.py b/VideoX-Fun/examples/wan2.2/post_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..13787ca6b32febd3495fa40a131712f11eb8b39c --- /dev/null +++ b/VideoX-Fun/examples/wan2.2/post_infer.py @@ -0,0 +1,169 @@ +import base64 +import json +import time +from datetime import datetime +from io import BytesIO + +import requests +from PIL import Image + + +def post_diffusion_transformer(diffusion_transformer_path, url='http://127.0.0.1:7860'): + datas = json.dumps({ + "diffusion_transformer_path": diffusion_transformer_path + }) + r = requests.post(f'{url}/videox_fun/update_diffusion_transformer', data=datas, timeout=1500) + data = r.content.decode('utf-8') + return data + +def post_update_edition(edition, url='http://0.0.0.0:7860'): + datas = json.dumps({ + "edition": edition + }) + r = requests.post(f'{url}/videox_fun/update_edition', data=datas, timeout=1500) + data = r.content.decode('utf-8') + return data + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5000, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion.", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + start_image = None +): + if start_image: + try: + if not start_image.startswith("http"): + image = Image.open(start_image).convert("RGB") + # 将图片转换为 Base64 编码 + buffered = BytesIO() + image.save(buffered, format="JPEG") + start_image = base64.b64encode(buffered.getvalue()).decode('utf-8') + except Exception as e: + print(f"Error processing start_image: {e}") + raise + + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "start_image": start_image + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + data = post_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # The Url you want to post + POST_URL = 'http://0.0.0.0:7860' + # Used in EAS. If you don't need Authorization, please set it to empty string. + TOKEN = '' + + # -------------------------- # + # Step 1: update edition + # -------------------------- # + # diffusion_transformer_path = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" + # outputs = post_diffusion_transformer(diffusion_transformer_path) + # print('Output update edition: ', outputs) + + # -------------------------- # + # Step 2: infer + # -------------------------- # + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 49 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + negative_prompt_textbox = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion." + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + # 起始图片路径 + start_image_path = "asset/3.png" # 替换为实际的图片路径 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + url=POST_URL, + POST_TOKEN=TOKEN, + start_image=start_image_path + ) + + # Get decoded data + outputs = json.loads(outputs) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2/post_infer_queue.py b/VideoX-Fun/examples/wan2.2/post_infer_queue.py new file mode 100644 index 0000000000000000000000000000000000000000..2d4f9915789fa599bb857404515ae408795b1728 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2/post_infer_queue.py @@ -0,0 +1,191 @@ +import base64 +import json +import time +import urllib.parse +import requests + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, +): + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # Support TeaCache. + enable_teacache = True + # Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, + # but it may cause slight differences between the generated content and the original content. + # # --------------------------------------------------------------------------------------------------- # + # | Model Name | threshold | Model Name | threshold | + # | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | + # # --------------------------------------------------------------------------------------------------- # + teacache_threshold = 0.10 + # The number of steps to skip TeaCache at the beginning of the inference process, which can + # reduce the impact of TeaCache on generated video quality. + num_skip_start_steps = 5 + # Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. + teacache_offload = False + + # Skip some cfg steps in inference + # Recommended to be set between 0.00 and 0.25 + cfg_skip_ratio = 0 + + # Riflex config + enable_riflex = False + # Index of intrinsic frequency + riflex_k = 6 + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 81 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + negative_prompt_textbox = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + url=EAS_URL, + POST_TOKEN=TOKEN + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2/post_infer_queue_i2v.py b/VideoX-Fun/examples/wan2.2/post_infer_queue_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..9dd1f4357a19790af4656065a1fb228ce5328b1e --- /dev/null +++ b/VideoX-Fun/examples/wan2.2/post_infer_queue_i2v.py @@ -0,0 +1,212 @@ +import base64 +import json +import time +import urllib.parse +import requests +from PIL import Image +from io import BytesIO + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + start_image = None +): + if start_image: + try: + if not start_image.startswith("http"): + image = Image.open(start_image).convert("RGB") + # 将图片转换为 Base64 编码 + buffered = BytesIO() + image.save(buffered, format="JPEG") + start_image = base64.b64encode(buffered.getvalue()).decode('utf-8') + except Exception as e: + print(f"Error processing start_image: {e}") + raise + + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + "start_image": start_image + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # Support TeaCache. + enable_teacache = True + # Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, + # but it may cause slight differences between the generated content and the original content. + # # --------------------------------------------------------------------------------------------------- # + # | Model Name | threshold | Model Name | threshold | + # | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | + # # --------------------------------------------------------------------------------------------------- # + teacache_threshold = 0.10 + # The number of steps to skip TeaCache at the beginning of the inference process, which can + # reduce the impact of TeaCache on generated video quality. + num_skip_start_steps = 5 + # Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. + teacache_offload = False + + # Skip some cfg steps in inference + # Recommended to be set between 0.00 and 0.25 + cfg_skip_ratio = 0 + + # Riflex config + enable_riflex = False + # Index of intrinsic frequency + riflex_k = 6 + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 81 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" + negative_prompt_textbox = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + # 起始图片路径 + start_image_path = "asset/1.png" # 替换为实际的图片路径 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + url=EAS_URL, + POST_TOKEN=TOKEN, + start_image=start_image_path # 传递起始图片路径 + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') diff --git a/VideoX-Fun/examples/wan2.2/predict_animate.py b/VideoX-Fun/examples/wan2.2/predict_animate.py new file mode 100644 index 0000000000000000000000000000000000000000..c209c887e9c38b7afad8cfff4c969d3f629aa038 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2/predict_animate.py @@ -0,0 +1,379 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel_Animate, + WanT5EncoderModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2AnimatePipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image, + get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_animate.yaml" +# model path +model_name = "./models/Diffusion_Transformer/Wan2.2-Animate-14B/" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow_Unipc" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +src_root_path = "asset/wan_animate/replace/process_results/" +src_pose_path = os.path.join(src_root_path, "src_pose.mp4") +src_face_path = os.path.join(src_root_path, "src_face.mp4") +src_ref_path = os.path.join(src_root_path, "src_ref.png") +src_bg_path = os.path.join(src_root_path, "src_bg.mp4") +src_mask_path = os.path.join(src_root_path, "src_mask.mp4") + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +prompt = "视频中的人在做动作" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 4.0 +seed = 43 +num_inference_steps = 20 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-animate" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Clip Image Encoder +clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(model_name, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), +).to(weight_dtype) +clip_image_encoder = clip_image_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2AnimatePipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + clip_image_encoder=clip_image_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + pose_video, _, _, _ = get_video_to_video_latent(src_pose_path, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + + face_video, _, _, _ = get_video_to_video_latent(src_face_path, video_length=video_length, sample_size=[512, 512], fps=fps, ref_image=None) + + ref_image = get_image(src_ref_path) + + if os.path.exists(src_bg_path): + bg_video, _, _, _ = get_video_to_video_latent(src_bg_path, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + mask_video, _, _, _ = get_video_to_video_latent(src_mask_path, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + mask_video = mask_video[:, :1] + replace_flag = True + else: + bg_video = None + mask_video = None + replace_flag = False + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + boundary = boundary, + pose_video = pose_video, + face_video = face_video, + ref_image = ref_image, + bg_video = bg_video, + mask_video = mask_video, + replace_flag = replace_flag, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() diff --git a/VideoX-Fun/examples/wan2.2/predict_i2v.py b/VideoX-Fun/examples/wan2.2/predict_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..52c93c2df57da0e8fb2cbc5b5013e9cc7134cd5b --- /dev/null +++ b/VideoX-Fun/examples/wan2.2/predict_i2v.py @@ -0,0 +1,337 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2I2VPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_i2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-I2V-A14B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow_Unipc" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# If you want to generate from text, please set the validation_image_start = None and validation_image_end = None +validation_image_start = "asset/1.png" + +# prompts +prompt = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-i2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2I2VPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, clip_image = get_image_to_video_latent(validation_image_start, None, video_length=video_length, sample_size=sample_size) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + boundary = boundary, + + video = input_video, + mask_video = input_video_mask, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2/predict_s2v.py b/VideoX-Fun/examples/wan2.2/predict_s2v.py new file mode 100644 index 0000000000000000000000000000000000000000..2c526f2cc01434e0707a58a9173361358fa37b6f --- /dev/null +++ b/VideoX-Fun/examples/wan2.2/predict_s2v.py @@ -0,0 +1,376 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel_S2V, WanAudioEncoder, + WanT5EncoderModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2S2VPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, + get_video_to_video_latent, + merge_video_audio, save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_s2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-S2V-14B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 3 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [832, 480] +video_length = 80 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# The path of the pose control video +control_video = "asset/pose.mp4" +# The path of the reference image +ref_image = "asset/8.png" +# Use ref_image as the first frame +init_first_frame = False +# The path of the audio +audio_path = "asset/talk.wav" + +# prompts +prompt = "一个女孩在海边说话。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 4.5 +seed = 43 +num_inference_steps = 40 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-speech2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + +transformer = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +audio_encoder = WanAudioEncoder( + os.path.join(model_name, config['audio_encoder_kwargs'].get('audio_encoder_subpath', 'audio_encoder')), + "cpu" +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2S2VPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + audio_encoder=audio_encoder, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = video_length // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio if video_length != 1 else 1 + latent_frames = video_length // vae.config.temporal_compression_ratio + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=sample_size) + + pose_video, _, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + boundary = boundary, + + ref_image = ref_image, + pose_video = pose_video, + audio_path = audio_path, + shift = shift, + fps = fps, + init_first_frame = init_first_frame + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + + merge_video_audio(video_path=video_path, audio_path=audio_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2/predict_t2v.py b/VideoX-Fun/examples/wan2.2/predict_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..28d56cef16ed8bf4c066d92bda63fd3424de44b6 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2/predict_t2v.py @@ -0,0 +1,327 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, WanT5EncoderModel, AutoTokenizer, + Wan2_2Transformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2Pipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_t2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-T2V-A14B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow_Unipc" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 12 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +prompt = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-t2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2Pipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + boundary = boundary, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2/predict_ti2v.py b/VideoX-Fun/examples/wan2.2/predict_ti2v.py new file mode 100644 index 0000000000000000000000000000000000000000..d039210cfd407d6d0e3bce3891eaafea21ea1712 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2/predict_ti2v.py @@ -0,0 +1,355 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan3_8, AutoencoderKLWan, WanT5EncoderModel, AutoTokenizer, + Wan2_2Transformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2TI2VPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_5b.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-TI2V-5B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow_Unipc" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +# Since Wan2.2-5b consists of only one model, only transformer_path is used. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +# Since Wan2.2-5b consists of only one model, only lora_path is used. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [704, 1280] +video_length = 121 +fps = 24 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# If you want to generate from text, please set the validation_image_start = None and validation_image_end = None +validation_image_start = "asset/1.png" + +# prompts +prompt = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-t2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2TI2VPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + if validation_image_start is not None: + input_video, input_video_mask, clip_image = get_image_to_video_latent(validation_image_start, None, video_length=video_length, sample_size=sample_size) + else: + input_video, input_video_mask, clip_image = None, None, None + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + boundary = boundary, + + video = input_video, + mask_video = input_video_mask, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/app.py b/VideoX-Fun/examples/wan2.2_fun/app.py new file mode 100644 index 0000000000000000000000000000000000000000..4091a88777f1d74426cbb27c55a2242802a288fe --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/app.py @@ -0,0 +1,79 @@ +import os +import sys +import time + +import torch + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.api.api import (infer_forward_api, + update_diffusion_transformer_api) +from videox_fun.ui.controller import flow_scheduler_dict +from videox_fun.ui.wan2_2_fun_ui import ui, ui_client, ui_host + +if __name__ == "__main__": + # Choose the ui mode + # "normal" refers to the standard UI, which allows users to click to switch models, change model types, and more. + # "host" represents the hosting mode, where the model is loaded directly at startup and can be accessed via + # the API to return generation results. + # "client" represents the client mode, offering a simple UI that sends requests to a remote API for generation. + ui_mode = "normal" + + # GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. + # model_full_load means that the entire model will be moved to the GPU. + # + # model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. + # + # model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, + # and the transformer model has been quantized to float8, which can save more GPU memory. + # + # sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, + # resulting in slower speeds but saving a large amount of GPU memory. + GPU_memory_mode = "sequential_cpu_offload" + # Compile will give a speedup in fixed resolution and need a little GPU memory. + # The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. + compile_dit = False + + # Use torch.float16 if GPU does not support torch.bfloat16 + # ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 + weight_dtype = torch.bfloat16 + + # Server ip + server_name = "0.0.0.0" + server_port = 7860 + + # Config path + config_path = "config/wan2.2/wan_civitai_i2v.yaml" + # Params below is used when ui_mode = "host" + # Model path of the pretrained model + model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" + # "Inpaint" or "Control" + model_type = "Inpaint" + + if ui_mode == "host": + demo, controller = ui_host(GPU_memory_mode, flow_scheduler_dict, model_name, model_type, config_path, compile_dit, weight_dtype) + elif ui_mode == "client": + demo, controller = ui_client(flow_scheduler_dict, model_name) + else: + demo, controller = ui(GPU_memory_mode, flow_scheduler_dict, config_path, compile_dit, weight_dtype) + + def gr_launch(): + # launch gradio + app, _, _ = demo.queue(status_update_rate=1).launch( + server_name=server_name, + server_port=server_port, + prevent_thread_lock=True + ) + + # launch api + infer_forward_api(None, app, controller) + update_diffusion_transformer_api(None, app, controller) + + gr_launch() + + # not close the python + while True: + time.sleep(5) \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/launch_api.py b/VideoX-Fun/examples/wan2.2_fun/launch_api.py new file mode 100644 index 0000000000000000000000000000000000000000..851be75dd77c917aa87d2150b53f29a7ec00597c --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/launch_api.py @@ -0,0 +1,91 @@ +import argparse +import os +import sys +import time + +import gradio as gr +import ray +import torch + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.api.api_multi_nodes import (MultiNodesEngine, + multi_nodes_infer_forward_api) +from videox_fun.ui.controller import flow_scheduler_dict +from videox_fun.ui.wan2_2_fun_ui import Wan2_2_Fun_Controller + +def main(): + parser = argparse.ArgumentParser(description='xDiT HTTP Service') + parser.add_argument('--world_size', type=int, default=8, help='Number of parallel workers') + parser.add_argument( + '--gpu_memory_mode', type=str, default="model_full_load", help=''' +GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8]. +model_full_load means that the entire model will be moved to the GPU. + +model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +and the transformer model has been quantized to float8, which can save more GPU memory. + +model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. + +model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +and the transformer model has been quantized to float8, which can save more GPU memory. + ''' + ) + parser.add_argument('--ulysses_degree', type=int, default=4, help='Degree of Ulysses configuration') + parser.add_argument('--ring_degree', type=int, default=2, help='Degree of Ring configuration') + parser.add_argument( + '--compile_dit', action='store_true', help=''' +Enable compile dit. +Compile will give a speedup in fixed resolution and need a little GPU memory. +The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. + ''' + ) + parser.add_argument('--fsdp_dit', action='store_true', help="Use DIT FSDP to save more GPU memory in multi gpus.") + parser.add_argument('--fsdp_text_encoder', action='store_true', help="Use Text Encoder FSDP to save more GPU memory in multi gpus.") + parser.add_argument('--weight_dtype', type=str, default='bf16', help='Weight data type') + parser.add_argument('--server_name', type=str, default="0.0.0.0", help='Server IP address') + parser.add_argument('--server_port', type=int, default=7860, help='Server Port') + parser.add_argument('--config_path', type=str, default="config/wan2.1/wan_civitai.yaml", help='Path to config file') + parser.add_argument('--model_name', type=str, default="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP", help='Model path') + parser.add_argument('--model_type', type=str, default="Inpaint", help='Model type (Inpaint/Control)') + parser.add_argument('--savedir_sample', type=str, default=None, help='The save directory for samples') + args = parser.parse_args() + + weight_dtype = torch.float32 + if args.weight_dtype == "bf16": + weight_dtype = torch.bfloat16 + elif args.weight_dtype == "fp16": + weight_dtype = torch.float16 + + engine = MultiNodesEngine( + world_size=args.world_size, Controller=Wan2_2_Fun_Controller, + GPU_memory_mode=args.gpu_memory_mode, scheduler_dict=flow_scheduler_dict, model_name=args.model_name, model_type=args.model_type, config_path=args.config_path, + ulysses_degree=args.ulysses_degree, ring_degree=args.ring_degree, + fsdp_dit=args.fsdp_dit, fsdp_text_encoder=args.fsdp_text_encoder, compile_dit=args.compile_dit, + weight_dtype=weight_dtype, savedir_sample=args.savedir_sample, + ) + + def gr_launch(): + # launch gradio + with gr.Blocks() as demo: + gr.Markdown("") + app, _, _ = demo.queue(status_update_rate=1).launch( + server_name=args.server_name, + server_port=args.server_port, + prevent_thread_lock=True + ) + + # launch api + multi_nodes_infer_forward_api(None, app, engine) + + gr_launch() + + # not close the python + while True: + time.sleep(5) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/post_infer.py b/VideoX-Fun/examples/wan2.2_fun/post_infer.py new file mode 100644 index 0000000000000000000000000000000000000000..0493c9baec8f6a2b568ddba1c8a14210fb878f01 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/post_infer.py @@ -0,0 +1,150 @@ +import base64 +import json +import time +from datetime import datetime + +import requests +import base64 + + +def post_diffusion_transformer(diffusion_transformer_path, url='http://127.0.0.1:7860'): + datas = json.dumps({ + "diffusion_transformer_path": diffusion_transformer_path + }) + r = requests.post(f'{url}/videox_fun/update_diffusion_transformer', data=datas, timeout=1500) + data = r.content.decode('utf-8') + return data + +def post_update_edition(edition, url='http://0.0.0.0:7860'): + datas = json.dumps({ + "edition": edition + }) + r = requests.post(f'{url}/videox_fun/update_edition', data=datas, timeout=1500) + data = r.content.decode('utf-8') + return data + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5000, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion.", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43 +): + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + data = post_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # The Url you want to post + POST_URL = 'http://0.0.0.0:7860' + # Used in EAS. If you don't need Authorization, please set it to empty string. + TOKEN = '' + + # -------------------------- # + # Step 1: update edition + # -------------------------- # + # diffusion_transformer_path = "models/Diffusion_Transformer/Wan2.1-Fun-1.3B-InP" + # outputs = post_diffusion_transformer(diffusion_transformer_path) + # print('Output update edition: ', outputs) + + # -------------------------- # + # Step 2: infer + # -------------------------- # + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 49 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + negative_prompt_textbox = "The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion." + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + url=POST_URL, + POST_TOKEN=TOKEN + ) + + # Get decoded data + outputs = json.loads(outputs) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/post_infer_queue.py b/VideoX-Fun/examples/wan2.2_fun/post_infer_queue.py new file mode 100644 index 0000000000000000000000000000000000000000..1ece0628c43dad57c802582a067d49accd06f317 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/post_infer_queue.py @@ -0,0 +1,192 @@ +import base64 +import json +import time +import urllib.parse +import requests + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, +): + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # Support TeaCache. + enable_teacache = True + # Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, + # but it may cause slight differences between the generated content and the original content. + # # --------------------------------------------------------------------------------------------------- # + # | Model Name | threshold | Model Name | threshold | + # | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | + # | Wan2.2-Fun-A14B-* | 0.15~0.20 | + # # --------------------------------------------------------------------------------------------------- # + teacache_threshold = 0.10 + # The number of steps to skip TeaCache at the beginning of the inference process, which can + # reduce the impact of TeaCache on generated video quality. + num_skip_start_steps = 5 + # Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. + teacache_offload = False + + # Skip some cfg steps in inference + # Recommended to be set between 0.00 and 0.25 + cfg_skip_ratio = 0 + + # Riflex config + enable_riflex = False + # Index of intrinsic frequency + riflex_k = 6 + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 81 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic." + negative_prompt_textbox = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + url=EAS_URL, + POST_TOKEN=TOKEN + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/post_infer_queue_i2v.py b/VideoX-Fun/examples/wan2.2_fun/post_infer_queue_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..0e54e9079ffe193c8fb697c5e8af99378fc9727c --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/post_infer_queue_i2v.py @@ -0,0 +1,213 @@ +import base64 +import json +import time +import urllib.parse +import requests +from PIL import Image +from io import BytesIO + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + start_image = None +): + if start_image: + try: + if not start_image.startswith("http"): + image = Image.open(start_image).convert("RGB") + # 将图片转换为 Base64 编码 + buffered = BytesIO() + image.save(buffered, format="JPEG") + start_image = base64.b64encode(buffered.getvalue()).decode('utf-8') + except Exception as e: + print(f"Error processing start_image: {e}") + raise + + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + "start_image": start_image + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # Support TeaCache. + enable_teacache = True + # Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, + # but it may cause slight differences between the generated content and the original content. + # # --------------------------------------------------------------------------------------------------- # + # | Model Name | threshold | Model Name | threshold | + # | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | + # | Wan2.2-Fun-A14B-* | 0.15~0.20 | + # # --------------------------------------------------------------------------------------------------- # + teacache_threshold = 0.10 + # The number of steps to skip TeaCache at the beginning of the inference process, which can + # reduce the impact of TeaCache on generated video quality. + num_skip_start_steps = 5 + # Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. + teacache_offload = False + + # Skip some cfg steps in inference + # Recommended to be set between 0.00 and 0.25 + cfg_skip_ratio = 0 + + # Riflex config + enable_riflex = False + # Index of intrinsic frequency + riflex_k = 6 + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 81 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" + negative_prompt_textbox = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 832 + height_slider = 480 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + # 起始图片路径 + start_image_path = "asset/1.png" # 替换为实际的图片路径 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + url=EAS_URL, + POST_TOKEN=TOKEN, + start_image=start_image_path # 传递起始图片路径 + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') diff --git a/VideoX-Fun/examples/wan2.2_fun/post_infer_queue_v2v_control.py b/VideoX-Fun/examples/wan2.2_fun/post_infer_queue_v2v_control.py new file mode 100644 index 0000000000000000000000000000000000000000..a97a6e6ee96b55b030f2e7c6d3fbbf54b782cee6 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/post_infer_queue_v2v_control.py @@ -0,0 +1,230 @@ +import base64 +import json +import time +import urllib.parse +from io import BytesIO + +import requests +from PIL import Image + + +def post_infer( + generation_method, + length_slider, + url='http://127.0.0.1:7860', + POST_TOKEN="", + timeout=5, + base_model_path="none", + lora_model_path="none", + lora_alpha_slider=0.55, + prompt_textbox="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt_textbox="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + sampler_dropdown="Flow", + sample_step_slider=50, + width_slider=672, + height_slider=384, + cfg_scale_slider=6, + seed_textbox=43, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + control_video = None, + ref_image = None +): + if control_video: + try: + if not control_video.startswith("http"): + with open(control_video, "rb") as file: + video_data = file.read() + + control_video = base64.b64encode(video_data).decode('utf-8') + except Exception as e: + print(f"Error processing control_video: {e}") + raise + + if ref_image: + try: + if not ref_image.startswith("http"): + image = Image.open(ref_image).convert("RGB") + # 将图片转换为 Base64 编码 + buffered = BytesIO() + image.save(buffered, format="JPEG") + ref_image = base64.b64encode(buffered.getvalue()).decode('utf-8') + except Exception as e: + print(f"Error processing ref_image: {e}") + raise + + # Prepare the data payload + datas = json.dumps({ + "base_model_path": base_model_path, + "lora_model_path": lora_model_path, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "width_slider": width_slider, + "height_slider": height_slider, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "seed_textbox": seed_textbox, + + "ref_image": ref_image, + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + "control_video": control_video + }) + + # Initialize session and set headers + session = requests.session() + session.headers.update({"Authorization": POST_TOKEN}) + + # Send POST request + if url[-1] == "/": + url = url[:-1] + post_r = session.post(f'{url}/videox_fun/infer_forward', data=datas, timeout=timeout) + + # Extract request ID from POST response headers + request_id = post_r.headers.get("X-Eas-Queueservice-Request-Id") + + # Prepare query parameters for GET request + query = { + '_index_': '0', + '_length_': '1', + '_timeout_': str(timeout), + '_raw_': 'false', + '_auto_delete_': 'true', + } + if request_id: + query['requestId'] = request_id + + query_str = urllib.parse.urlencode(query) + + # Polling GET request until status code is not 204 + status_code = 204 + while status_code == 204: + if query_str: + get_r = session.get(f'{url}/sink?{query_str}', timeout=timeout) + else: + get_r = session.get(f'{url}/sink', timeout=timeout) + status_code = get_r.status_code + # Decode and return the response content + data = get_r.content.decode('utf-8') + return data + + +if __name__ == '__main__': + # initiate time + time_start = time.time() + + # EAS队列配置 + EAS_URL = 'http://17xxxxxxxxx.pai-eas.aliyuncs.com/api/predict/xxxxxxxx' + # Use in EAS Queue + TOKEN = 'xxxxxxxx' + + # Support TeaCache. + enable_teacache = True + # Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, + # but it may cause slight differences between the generated content and the original content. + # # --------------------------------------------------------------------------------------------------- # + # | Model Name | threshold | Model Name | threshold | + # | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | + # | Wan2.2-Fun-A14B-* | 0.15~0.20 | + # # --------------------------------------------------------------------------------------------------- # + teacache_threshold = 0.10 + # The number of steps to skip TeaCache at the beginning of the inference process, which can + # reduce the impact of TeaCache on generated video quality. + num_skip_start_steps = 5 + # Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. + teacache_offload = False + + # Skip some cfg steps in inference + # Recommended to be set between 0.00 and 0.25 + cfg_skip_ratio = 0 + + # Riflex config + enable_riflex = False + # Index of intrinsic frequency + riflex_k = 6 + + # "Video Generation" and "Image Generation" + generation_method = "Video Generation" + # Video length + length_slider = 81 + # Used in Lora models + lora_model_path = "none" + lora_alpha_slider = 0.55 + # Prompts + prompt_textbox = "在这个阳光明媚的户外花园里，美女身穿一袭及膝的白色无袖连衣裙，裙摆在她轻盈的舞姿中轻柔地摆动，宛如一只翩翩起舞的蝴蝶。阳光透过树叶间洒下斑驳的光影，映衬出她柔和的脸庞和清澈的眼眸，显得格外优雅。仿佛每一个动作都在诉说着青春与活力，她在草地上旋转，裙摆随之飞扬，仿佛整个花园都因她的舞动而欢愉。周围五彩缤纷的花朵在微风中摇曳，玫瑰、菊花、百合，各自释放出阵阵香气，营造出一种轻松而愉快的氛围。" + negative_prompt_textbox = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + # Sampler name + sampler_dropdown = "Flow" + # Sampler steps + sample_step_slider = 50 + # height and width + width_slider = 480 + height_slider = 832 + # cfg scale + cfg_scale_slider = 6 + seed_textbox = 43 + + # 控制视频路径（可以是本地路径或 URL） + control_video_path = "asset/000000.mp4" # 替换为实际的视频路径 + # 参考图片路径 + ref_image_path = None # 替换为实际的图片路径 + + outputs = post_infer( + generation_method, + length_slider, + lora_model_path=lora_model_path, + lora_alpha_slider=lora_alpha_slider, + prompt_textbox=prompt_textbox, + negative_prompt_textbox=negative_prompt_textbox, + sampler_dropdown=sampler_dropdown, + sample_step_slider=sample_step_slider, + width_slider=width_slider, + height_slider=height_slider, + cfg_scale_slider=cfg_scale_slider, + seed_textbox=seed_textbox, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + url=EAS_URL, + POST_TOKEN=TOKEN, + control_video=control_video_path, # 传递控制视频路径 + ref_image=ref_image_path # 传递参考图片路径 + ) + # Get decoded data + outputs = json.loads(base64.b64decode(json.loads(outputs)[0]['data'])) + base64_encoding = outputs["base64_encoding"] + decoded_data = base64.b64decode(base64_encoding) + + is_image = True if generation_method == "Image Generation" else False + if is_image or length_slider == 1: + file_path = "1.png" + else: + file_path = "1.mp4" + with open(file_path, "wb") as file: + file.write(decoded_data) + + # End of record time + # The calculated time difference is the execution time of the program, expressed in seconds / s + time_end = time.time() + time_sum = (time_end - time_start) + print('# --------------------------------------------------------- #') + print(f'# Total expenditure: {time_sum}s') + print('# --------------------------------------------------------- #') diff --git a/VideoX-Fun/examples/wan2.2_fun/predict_i2v.py b/VideoX-Fun/examples/wan2.2_fun/predict_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..79181b4d33f4f6fada0db43e0cea2e0abd7e5bc1 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/predict_i2v.py @@ -0,0 +1,354 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2FunInpaintPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# | Wan2.2-Fun-A14B-* | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_i2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# If you want to generate from text, please set the validation_image_start = None and validation_image_end = None +validation_image_start = "asset/1.png" +validation_image_end = None + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-fun-i2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2FunInpaintPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, clip_image = get_image_to_video_latent(validation_image_start, validation_image_end, video_length=video_length, sample_size=sample_size) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + boundary = boundary, + + video = input_video, + mask_video = input_video_mask, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/predict_i2v_5b.py b/VideoX-Fun/examples/wan2.2_fun/predict_i2v_5b.py new file mode 100644 index 0000000000000000000000000000000000000000..1bbccaa089daceefb8cde08fbfe3327b6ef5210b --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/predict_i2v_5b.py @@ -0,0 +1,356 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, AutoencoderKLWan3_8, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2FunInpaintPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# | Wan2.2-Fun-A14B-* | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_5b.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-InP/" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +# Since Wan2.2-5b consists of only one model, only transformer_path is used. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +# Since Wan2.2-5b consists of only one model, only lora_path is used. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [704, 1280] +video_length = 121 +fps = 24 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# If you want to generate from text, please set the validation_image_start = None and validation_image_end = None +validation_image_start = "asset/1.png" +validation_image_end = None + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一只棕色的狗舔了一下它的舌头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-fun-i2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2FunInpaintPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, clip_image = get_image_to_video_latent(validation_image_start, validation_image_end, video_length=video_length, sample_size=sample_size) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + boundary = boundary, + + video = input_video, + mask_video = input_video_mask, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/predict_t2v.py b/VideoX-Fun/examples/wan2.2_fun/predict_t2v.py new file mode 100644 index 0000000000000000000000000000000000000000..347128ac0ef7a9c8bfca72a09d40dffd13ce3d1b --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/predict_t2v.py @@ -0,0 +1,336 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, AutoencoderKLWan3_8, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2FunInpaintPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# | Wan2.2-Fun-A14B-* | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_i2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-fun-t2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2FunInpaintPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=sample_size) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = input_video, + mask_video = input_video_mask, + boundary = boundary, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/predict_t2v_5b.py b/VideoX-Fun/examples/wan2.2_fun/predict_t2v_5b.py new file mode 100644 index 0000000000000000000000000000000000000000..49c5bb236912b55633071b90a35658c5525c6fd8 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/predict_t2v_5b.py @@ -0,0 +1,350 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2FunInpaintPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# TeaCache config +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# | Wan2.2-Fun-A14B-* | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_5b.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-InP/" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [704, 1280] +video_length = 121 +fps = 21 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一只棕色的狗摇着头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-fun-t2v" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2FunInpaintPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=sample_size) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = input_video, + mask_video = input_video_mask, + boundary = boundary, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control.py b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control.py new file mode 100644 index 0000000000000000000000000000000000000000..b707b93679861c522943290b8f7aa94ff2e39751 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control.py @@ -0,0 +1,381 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# | Wan2.2-Fun-A14B-* | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_i2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [832, 480] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = "asset/pose.mp4" +control_camera_txt = None +start_image = None +end_image = None +ref_image = None + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女子站在阳光明媚的海岸线上，身穿深蓝色背心与清爽的白色衬衫，外搭一条简洁的白色围裙，围裙在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-fun-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2FunControlPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=sample_size) + + if control_camera_txt is not None: + input_video, input_video_mask = None, None + control_camera_video = process_pose_file(control_camera_txt, sample_size[1], sample_size[0]) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + else: + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + control_camera_video = None + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = input_video, + control_camera_video = control_camera_video, + ref_image = ref_image, + boundary = boundary, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_5b.py b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_5b.py new file mode 100644 index 0000000000000000000000000000000000000000..16dab5d6ad2abb92a8aba95c0eaa6d5d09f59b8f --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_5b.py @@ -0,0 +1,381 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# | Wan2.2-Fun-A14B-* | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_5b.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-Control/" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +# Since Wan2.2-5b consists of only one model, only transformer_path is used. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +# Since Wan2.2-5b consists of only one model, only lora_path is used. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [1280, 704] +video_length = 121 +fps = 24 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = "asset/pose.mp4" +control_camera_txt = None +start_image = None +end_image = None +ref_image = None + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女子站在阳光明媚的海岸线上，身穿深蓝色背心与清爽的白色衬衫，外搭一条简洁的白色围裙，围裙在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 40 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-fun-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2FunControlPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=sample_size) + + if control_camera_txt is not None: + input_video, input_video_mask = None, None + control_camera_video = process_pose_file(control_camera_txt, sample_size[1], sample_size[0]) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + else: + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + control_camera_video = None + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = input_video, + control_camera_video = control_camera_video, + ref_image = ref_image, + boundary = boundary, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_camera.py b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_camera.py new file mode 100644 index 0000000000000000000000000000000000000000..f112394aaf35b4118f98c82205803e9fbaae729d --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_camera.py @@ -0,0 +1,381 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# | Wan2.2-Fun-A14B-* | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_i2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control-Camera" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = None +control_camera_txt = "asset/Zoom_In.txt" +start_image = "asset/7.png" +end_image = None +ref_image = None + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一个小女孩正在户外玩耍。她穿着一件蓝色的短袖上衣和粉色的短裤，头发扎成一个可爱的辫子。她的脚上没有穿鞋，显得非常自然和随意。她正用一把红色的小铲子在泥土里挖土，似乎在进行某种有趣的活动，可能是种花或是挖掘宝藏。地上有一根长长的水管，可能是用来浇水的。背景是一片草地和一些绿色植物，阳光明媚，整个场景充满了童趣和生机。小女孩专注的表情和认真的动作让人感受到她的快乐和好奇心。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 6.0 +seed = 42 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-fun-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2FunControlPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=sample_size) + + if control_camera_txt is not None: + input_video, input_video_mask = None, None + control_camera_video = process_pose_file(control_camera_txt, sample_size[1], sample_size[0]) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + else: + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + control_camera_video = None + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = input_video, + control_camera_video = control_camera_video, + ref_image = ref_image, + boundary = boundary, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_camera_5b.py b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_camera_5b.py new file mode 100644 index 0000000000000000000000000000000000000000..bc7e494414298358203c8720dc9231028b1431b4 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_camera_5b.py @@ -0,0 +1,381 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# | Wan2.2-Fun-A14B-* | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_5b.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-Control-Camera/" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +# Since Wan2.2-5b consists of only one model, only transformer_path is used. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +# Since Wan2.2-5b consists of only one model, only lora_path is used. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [704, 1280] +video_length = 81 +fps = 24 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = None +control_camera_txt = "asset/Zoom_In.txt" +start_image = "asset/7.png" +end_image = None +ref_image = None + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一个小女孩正在户外玩耍。她穿着一件蓝色的短袖上衣和粉色的短裤，头发扎成一个可爱的辫子。她的脚上没有穿鞋，显得非常自然和随意。她正用一把红色的小铲子在泥土里挖土，似乎在进行某种有趣的活动，可能是种花或是挖掘宝藏。地上有一根长长的水管，可能是用来浇水的。背景是一片草地和一些绿色植物，阳光明媚，整个场景充满了童趣和生机。小女孩专注的表情和认真的动作让人感受到她的快乐和好奇心。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 6.0 +seed = 42 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-fun-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + # low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + # low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2FunControlPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=sample_size) + + if control_camera_txt is not None: + input_video, input_video_mask = None, None + control_camera_video = process_pose_file(control_camera_txt, sample_size[1], sample_size[0]) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + else: + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + control_camera_video = None + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = input_video, + control_camera_video = control_camera_video, + ref_image = ref_image, + boundary = boundary, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_ref.py b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_ref.py new file mode 100644 index 0000000000000000000000000000000000000000..e842870ee1759d75bad44fb3308e4554d1b50cd4 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_ref.py @@ -0,0 +1,381 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# | Wan2.2-Fun-A14B-* | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_i2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +# If you want to generate a 480p video, it is recommended to set the shift value to 3.0. +# If you want to generate a 720p video, it is recommended to set the shift value to 5.0. +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [832, 480] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = "asset/pose.mp4" +control_camera_txt = None +start_image = None +end_image = None +ref_image = "asset/8.png" + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女子站在阳光明媚的海岸线上，身穿深蓝色背心与清爽的白色衬衫，外搭一条简洁的白色围裙，围裙在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-fun-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2FunControlPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=sample_size) + + if control_camera_txt is not None: + input_video, input_video_mask = None, None + control_camera_video = process_pose_file(control_camera_txt, sample_size[1], sample_size[0]) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + else: + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + control_camera_video = None + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = input_video, + control_camera_video = control_camera_video, + ref_image = ref_image, + boundary = boundary, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_ref_5b.py b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_ref_5b.py new file mode 100644 index 0000000000000000000000000000000000000000..bf9bd04ebba071563a72590605626c14a2108da1 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_fun/predict_v2v_control_ref_5b.py @@ -0,0 +1,381 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2FunControlPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_latent, get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.2-T2V-A14B | 0.10~0.15 | Wan2.2-I2V-A14B | 0.15~0.20 | +# | Wan2.2-Fun-A14B-* | 0.15~0.20 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_5b.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-Fun-5B-Control/" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 5 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +# Since Wan2.2-5b consists of only one model, only transformer_path is used. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +# Since Wan2.2-5b consists of only one model, only lora_path is used. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [1280, 704] +video_length = 121 +fps = 24 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = "asset/pose.mp4" +control_camera_txt = None +start_image = None +end_image = None +ref_image = "asset/8.png" + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女子站在阳光明媚的海岸线上，身穿深蓝色背心与清爽的白色衬衫，外搭一条简洁的白色围裙，围裙在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 6.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/wan-videos-fun-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + # low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + # low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2FunControlPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=sample_size) + + if control_camera_txt is not None: + input_video, input_video_mask = None, None + control_camera_video = process_pose_file(control_camera_txt, sample_size[1], sample_size[0]) + control_camera_video = control_camera_video[:video_length].permute([3, 0, 1, 2]).unsqueeze(0) + else: + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + control_camera_video = None + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = input_video, + control_camera_video = control_camera_video, + ref_image = ref_image, + boundary = boundary, + shift = shift, + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_vace_fun/predict_i2v.py b/VideoX-Fun/examples/wan2.2_vace_fun/predict_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..45f6065c5b702244c4b98ea11df06f2fb432a24d --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_vace_fun/predict_i2v.py @@ -0,0 +1,387 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, VaceWanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2VaceFunPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.1-VACE-1.3B | 0.05~0.10 | Wan2.1-VACE-14B | 0.10~0.15 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_t2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-VACE-Fun-A14B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 12.0 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = None +start_image = "asset/1.png" +end_image = None +# Use inpaint video instead of start image and end image. +inpaint_video = None +inpaint_video_mask = None +subject_ref_images = None +vace_context_scale = 1.00 +# Sometimes, when generating a video from a reference image, white borders appear. +# Because the padding is mistakenly treated as part of the image. +# If the aspect ratio of the reference image is close to the final output, you can omit the white padding. +padding_in_subject_ref_images = True + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一只棕色的狗舔了一下它的舌头，坐在舒适房间里的浅色沙发上。在狗的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 5.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/vace-videos-fun" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2VaceFunPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + if subject_ref_images is not None: + subject_ref_images = [get_image_latent(_subject_ref_image, sample_size=sample_size, padding=padding_in_subject_ref_images) for _subject_ref_image in subject_ref_images] + subject_ref_images = torch.cat(subject_ref_images, dim=2) + + if inpaint_video is not None: + if inpaint_video_mask is None: + raise ValueError("inpaint_video_mask is required when inpaint_video is provided") + inpaint_video, _, _, _ = get_video_to_video_latent(inpaint_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + inpaint_video_mask, _, _, _ = get_video_to_video_latent(inpaint_video_mask, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + inpaint_video_mask = inpaint_video_mask[:, :1] + else: + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + control_video, _, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = subject_ref_images, + boundary = boundary, + shift = shift, + vace_context_scale = vace_context_scale + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_vace_fun/predict_s2v.py b/VideoX-Fun/examples/wan2.2_vace_fun/predict_s2v.py new file mode 100644 index 0000000000000000000000000000000000000000..7d53f2cbf5107c38eb27802ed8405f16b02168b3 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_vace_fun/predict_s2v.py @@ -0,0 +1,387 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, VaceWanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2VaceFunPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.1-VACE-1.3B | 0.05~0.10 | Wan2.1-VACE-14B | 0.10~0.15 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_t2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-VACE-Fun-A14B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 12.0 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = None +start_image = None +end_image = None +# Use inpaint video instead of start image and end image. +inpaint_video = None +inpaint_video_mask = None +subject_ref_images = ["asset/8.png", "asset/ref_1.png"] +vace_context_scale = 1.00 +# Sometimes, when generating a video from a reference image, white borders appear. +# Because the padding is mistakenly treated as part of the image. +# If the aspect ratio of the reference image is close to the final output, you can omit the white padding. +padding_in_subject_ref_images = True + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "海风作曲，浪花打拍。她握着一台亮黄色相机，双臂如翼般舒展流转。双手托举相机，在胸前划出轻柔的圆弧，时而高举过肩，迎向天际的光晕，时而缓缓贴近心口，倾听快门低语般的节奏。紫色长发在风中扬起，拂过肩头，仿佛与镜头共舞。她的目光随取景框游走，每一次微倾与回旋，都让镜头掠过海天相接的边际、摇曳的花丛。最后，呼吸一凝，稳稳对准水中倒影，轻轻按下快门——将风、光与心动，悄然收藏。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 5.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/vace-videos-fun" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2VaceFunPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + if subject_ref_images is not None: + subject_ref_images = [get_image_latent(_subject_ref_image, sample_size=sample_size, padding=padding_in_subject_ref_images) for _subject_ref_image in subject_ref_images] + subject_ref_images = torch.cat(subject_ref_images, dim=2) + + if inpaint_video is not None: + if inpaint_video_mask is None: + raise ValueError("inpaint_video_mask is required when inpaint_video is provided") + inpaint_video, _, _, _ = get_video_to_video_latent(inpaint_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + inpaint_video_mask, _, _, _ = get_video_to_video_latent(inpaint_video_mask, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + inpaint_video_mask = inpaint_video_mask[:, :1] + else: + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + control_video, _, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = subject_ref_images, + boundary = boundary, + shift = shift, + vace_context_scale = vace_context_scale + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_vace_fun/predict_v2v_control.py b/VideoX-Fun/examples/wan2.2_vace_fun/predict_v2v_control.py new file mode 100644 index 0000000000000000000000000000000000000000..a23f35c5819a7d86c7ffd3667a881c4bb291c2cd --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_vace_fun/predict_v2v_control.py @@ -0,0 +1,387 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, VaceWanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2VaceFunPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.1-VACE-1.3B | 0.05~0.10 | Wan2.1-VACE-14B | 0.10~0.15 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_t2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-VACE-Fun-A14B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 12.0 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [832, 480] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = "asset/pose.mp4" +start_image = None +end_image = None +# Use inpaint video instead of start image and end image. +inpaint_video = None +inpaint_video_mask = None +subject_ref_images = None +vace_context_scale = 1.00 +# Sometimes, when generating a video from a reference image, white borders appear. +# Because the padding is mistakenly treated as part of the image. +# If the aspect ratio of the reference image is close to the final output, you can omit the white padding. +padding_in_subject_ref_images = True + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女子站在阳光明媚的海岸线上，身穿清爽的白色衬衫与裙子，在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 5.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/vace-videos-fun" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2VaceFunPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + if subject_ref_images is not None: + subject_ref_images = [get_image_latent(_subject_ref_image, sample_size=sample_size, padding=padding_in_subject_ref_images) for _subject_ref_image in subject_ref_images] + subject_ref_images = torch.cat(subject_ref_images, dim=2) + + if inpaint_video is not None: + if inpaint_video_mask is None: + raise ValueError("inpaint_video_mask is required when inpaint_video is provided") + inpaint_video, _, _, _ = get_video_to_video_latent(inpaint_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + inpaint_video_mask, _, _, _ = get_video_to_video_latent(inpaint_video_mask, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + inpaint_video_mask = inpaint_video_mask[:, :1] + else: + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + control_video, _, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = subject_ref_images, + boundary = boundary, + shift = shift, + vace_context_scale = vace_context_scale + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_vace_fun/predict_v2v_control_ref.py b/VideoX-Fun/examples/wan2.2_vace_fun/predict_v2v_control_ref.py new file mode 100644 index 0000000000000000000000000000000000000000..e0538361f41b48acc30b95f6406faa4f656eef3c --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_vace_fun/predict_v2v_control_ref.py @@ -0,0 +1,387 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, VaceWanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2VaceFunPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.1-VACE-1.3B | 0.05~0.10 | Wan2.1-VACE-14B | 0.10~0.15 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_t2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-VACE-Fun-A14B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 12.0 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [832, 480] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = "asset/pose.mp4" +start_image = None +end_image = None +# Use inpaint video instead of start image and end image. +inpaint_video = None +inpaint_video_mask = None +subject_ref_images = ["asset/8.png"] +vace_context_scale = 1.00 +# Sometimes, when generating a video from a reference image, white borders appear. +# Because the padding is mistakenly treated as part of the image. +# If the aspect ratio of the reference image is close to the final output, you can omit the white padding. +padding_in_subject_ref_images = True + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女子站在阳光明媚的海岸线上，身穿清爽的白色衬衫与裙子，在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 5.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/vace-videos-fun" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2VaceFunPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + if subject_ref_images is not None: + subject_ref_images = [get_image_latent(_subject_ref_image, sample_size=sample_size, padding=padding_in_subject_ref_images) for _subject_ref_image in subject_ref_images] + subject_ref_images = torch.cat(subject_ref_images, dim=2) + + if inpaint_video is not None: + if inpaint_video_mask is None: + raise ValueError("inpaint_video_mask is required when inpaint_video is provided") + inpaint_video, _, _, _ = get_video_to_video_latent(inpaint_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + inpaint_video_mask, _, _, _ = get_video_to_video_latent(inpaint_video_mask, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + inpaint_video_mask = inpaint_video_mask[:, :1] + else: + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + control_video, _, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = subject_ref_images, + boundary = boundary, + shift = shift, + vace_context_scale = vace_context_scale + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/wan2.2_vace_fun/predict_v2v_mask.py b/VideoX-Fun/examples/wan2.2_vace_fun/predict_v2v_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..31ddc896400dbac2dde8076c3d698aab94484af1 --- /dev/null +++ b/VideoX-Fun/examples/wan2.2_vace_fun/predict_v2v_mask.py @@ -0,0 +1,387 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image +from transformers import AutoTokenizer + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, VaceWanTransformer3DModel) +from videox_fun.data.dataset_image_video import process_pose_file +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import Wan2_2VaceFunPipeline, WanPipeline +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, + get_video_to_video_latent, + save_videos_grid) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +# GPU memory mode, which can be chosen in [model_full_load, model_cpu_offload_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "sequential_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = True +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Support TeaCache. +enable_teacache = True +# Recommended to be set between 0.05 and 0.30. A larger threshold can cache more steps, speeding up the inference process, +# but it may cause slight differences between the generated content and the original content. +# # --------------------------------------------------------------------------------------------------- # +# | Model Name | threshold | Model Name | threshold | +# | Wan2.1-VACE-1.3B | 0.05~0.10 | Wan2.1-VACE-14B | 0.10~0.15 | +# # --------------------------------------------------------------------------------------------------- # +teacache_threshold = 0.10 +# The number of steps to skip TeaCache at the beginning of the inference process, which can +# reduce the impact of TeaCache on generated video quality. +num_skip_start_steps = 5 +# Whether to offload TeaCache tensors to cpu to save a little bit of GPU memory. +teacache_offload = False + +# Skip some cfg steps in inference for acceleration +# Recommended to be set between 0.00 and 0.25 +cfg_skip_ratio = 0 + +# Riflex config +enable_riflex = False +# Index of intrinsic frequency +riflex_k = 6 + +# Config and model path +config_path = "config/wan2.2/wan_civitai_t2v.yaml" +# model path +model_name = "models/Diffusion_Transformer/Wan2.2-VACE-Fun-A14B" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" +# [NOTE]: Noise schedule shift parameter. Affects temporal dynamics. +# Used when the sampler is in "Flow_Unipc", "Flow_DPM++". +shift = 12.0 + +# Load pretrained model if need +# The transformer_path is used for low noise model, the transformer_high_path is used for high noise model. +transformer_path = None +transformer_high_path = None +vae_path = None +# Load lora model if need +# The lora_path is used for low noise model, the lora_high_path is used for high noise model. +lora_path = None +lora_high_path = None + +# Other params +sample_size = [480, 832] +video_length = 81 +fps = 16 + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_video = None +start_image = None +end_image = None +# Use inpaint video instead of start image and end image. +inpaint_video = "asset/inpaint_video.mp4" +inpaint_video_mask = "asset/inpaint_video_mask.mp4" +subject_ref_images = None +vace_context_scale = 1.00 +# Sometimes, when generating a video from a reference image, white borders appear. +# Because the padding is mistakenly treated as part of the image. +# If the aspect ratio of the reference image is close to the final output, you can omit the white padding. +padding_in_subject_ref_images = True + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一只棕色的兔子舔了一下它的舌头，坐在舒适房间里的浅色沙发上。在兔子的后面，架子上有一幅镶框的画，周围是粉红色的花朵。房间里柔和温暖的灯光营造出舒适的氛围。" +negative_prompt = "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" + +# Using longer neg prompt such as "Blurring, mutation, deformation, distortion, dark and solid, comics, text subtitles, line art." can increase stability +# Adding words such as "quiet, solid" to the neg prompt can increase dynamism. +# prompt = "A young woman with beautiful, clear eyes and blonde hair stands in the forest, wearing a white dress and a crown. Her expression is serene, reminiscent of a movie star, with fair and youthful skin. Her brown long hair flows in the wind. The video quality is very high, with a clear view. High quality, masterpiece, best quality, high resolution, ultra-fine, fantastical." +# negative_prompt = "Twisted body, limb deformities, text captions, comic, static, ugly, error, messy code." +guidance_scale = 5.0 +seed = 43 +num_inference_steps = 50 +# The lora_weight is used for low noise model, the lora_high_weight is used for high noise model. +lora_weight = 0.55 +lora_high_weight = 0.55 +save_path = "samples/vace-videos-fun" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) +boundary = config['transformer_additional_kwargs'].get('boundary', 0.875) + +transformer = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +if config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + transformer_2 = VaceWanTransformer3DModel.from_pretrained( + os.path.join(model_name, config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) +else: + transformer_2 = None + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +if transformer_2 is not None: + if transformer_high_path is not None: + print(f"From checkpoint: {transformer_high_path}") + if transformer_high_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_high_path) + else: + state_dict = torch.load(transformer_high_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer_2.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 +}[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] +vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(model_name, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Tokenizer +tokenizer = AutoTokenizer.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), +) + +# Get Text encoder +text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(model_name, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +) +text_encoder = text_encoder.eval() + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +if sampler_name == "Flow_Unipc" or sampler_name == "Flow_DPM++": + config['scheduler_kwargs']['shift'] = 1 +scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(config['scheduler_kwargs'])) +) + +# Get Pipeline +pipeline = Wan2_2VaceFunPipeline( + transformer=transformer, + transformer_2=transformer_2, + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, +) +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if transformer_2 is not None: + transformer_2.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.transformer = shard_fn(pipeline.transformer) + if transformer_2 is not None: + pipeline.transformer_2 = shard_fn(pipeline.transformer_2) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype) + pipeline.text_encoder = shard_fn(pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.blocks)): + pipeline.transformer.blocks[i] = torch.compile(pipeline.transformer.blocks[i]) + if transformer_2 is not None: + for i in range(len(pipeline.transformer_2.blocks)): + pipeline.transformer_2.blocks[i] = torch.compile(pipeline.transformer_2.blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(transformer, ["modulation",], device=device) + transformer.freqs = transformer.freqs.to(device=device) + if transformer_2 is not None: + replace_parameters_by_name(transformer_2, ["modulation",], device=device) + transformer_2.freqs = transformer_2.freqs.to(device=device) + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + if transformer_2 is not None: + convert_model_weight_to_float8(transformer_2, exclude_module_name=["modulation",], device=device) + convert_weight_dtype_wrapper(transformer_2, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +coefficients = get_teacache_coefficients(model_name) if enable_teacache else None +if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + pipeline.transformer.enable_teacache( + coefficients, num_inference_steps, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if transformer_2 is not None: + pipeline.transformer_2.share_teacache(transformer=pipeline.transformer) + +if cfg_skip_ratio is not None: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, num_inference_steps) + if transformer_2 is not None: + pipeline.transformer_2.share_cfg_skip(transformer=pipeline.transformer) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = merge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +with torch.no_grad(): + video_length = int((video_length - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if video_length != 1 else 1 + latent_frames = (video_length - 1) // vae.config.temporal_compression_ratio + 1 + + if enable_riflex: + pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames) + if transformer_2 is not None: + pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames) + + if subject_ref_images is not None: + subject_ref_images = [get_image_latent(_subject_ref_image, sample_size=sample_size, padding=padding_in_subject_ref_images) for _subject_ref_image in subject_ref_images] + subject_ref_images = torch.cat(subject_ref_images, dim=2) + + if inpaint_video is not None: + if inpaint_video_mask is None: + raise ValueError("inpaint_video_mask is required when inpaint_video is provided") + inpaint_video, _, _, _ = get_video_to_video_latent(inpaint_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + inpaint_video_mask, _, _, _ = get_video_to_video_latent(inpaint_video_mask, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + inpaint_video_mask = inpaint_video_mask[:, :1] + else: + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=video_length, sample_size=sample_size) + + control_video, _, _, _ = get_video_to_video_latent(control_video, video_length=video_length, sample_size=sample_size, fps=fps, ref_image=None) + + sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = subject_ref_images, + boundary = boundary, + shift = shift, + vace_context_scale = vace_context_scale + ).videos + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + if transformer_2 is not None: + pipeline = unmerge_lora(pipeline, lora_high_path, lora_high_weight, device=device, dtype=weight_dtype, sub_transformer_name="transformer_2") + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + if video_length == 1: + video_path = os.path.join(save_path, prefix + ".png") + + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(video_path) + else: + video_path = os.path.join(save_path, prefix + ".mp4") + save_videos_grid(sample, video_path, fps=fps) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/z_image/predict_t2i.py b/VideoX-Fun/examples/z_image/predict_t2i.py new file mode 100644 index 0000000000000000000000000000000000000000..e07dd08f202beb65f83f0376b0c09a7010e19e3b --- /dev/null +++ b/VideoX-Fun/examples/z_image/predict_t2i.py @@ -0,0 +1,209 @@ +import os +import sys + +import torch +from diffusers import FlowMatchEulerDiscreteScheduler + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoTokenizer, Qwen3ForCausalLM, + ZImageTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import ZImagePipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# model path +model_name = "models/Diffusion_Transformer/Z-Image-Turbo" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = None +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女子站在阳光明媚的海岸线上，白裙在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" +negative_prompt = " " +guidance_scale = 0.00 +seed = 43 +num_inference_steps = 9 +lora_weight = 0.55 +save_path = "samples/z-image-t2i" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) + +transformer = ZImageTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = AutoTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = ZImagePipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers, ignored_modules=[text_encoder.language_model.embed_tokens], transformer_layer_cls_to_wrap=["MistralDecoderLayer", "PixtralTransformer"]) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + sample = pipeline( + prompt = prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + num_inference_steps = num_inference_steps, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/z_image_fun/collect_onnx_inputs.py b/VideoX-Fun/examples/z_image_fun/collect_onnx_inputs.py new file mode 100644 index 0000000000000000000000000000000000000000..4ee07d13a0002fd88133630b0755d90de5b502c6 --- /dev/null +++ b/VideoX-Fun/examples/z_image_fun/collect_onnx_inputs.py @@ -0,0 +1,567 @@ +import argparse +import os +import sys +from typing import List, Optional, Union + +import numpy as np +import torch +from omegaconf import OmegaConf + +# Determine repo root and model locations +current_file_path = os.path.abspath(__file__) +project_roots = [ + os.path.dirname(current_file_path), + os.path.dirname(os.path.dirname(current_file_path)), + os.path.dirname(os.path.dirname(os.path.dirname(current_file_path))), +] +for project_root in project_roots: + if project_root not in sys.path: + sys.path.insert(0, project_root) +repo_root = project_roots[-1] + +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.image_processor import VaeImageProcessor +from diffusers.utils.torch_utils import randn_tensor +from videox_fun.models import ( + AutoTokenizer, + AutoencoderKL, + Qwen3ForCausalLM, + ZImageTransformer2DModel, +) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + + +def parse_args(): + parser = argparse.ArgumentParser(description="Collect ONNX calibration inputs for Z-Image model") + + # Model and config paths + parser.add_argument( + "--config_path", + type=str, + default="config/z_image/z_image.yaml", + help="Path to config file", + ) + parser.add_argument( + "--model_name", + type=str, + default="models/Diffusion_Transformer/Z-Image-Turbo/", + help="Path to pretrained model directory (must contain tokenizer/, text_encoder/, transformer/, vae/, scheduler/ subfolders)", + ) + + # Output directory + parser.add_argument( + "--output_dir", + type=str, + default="onnx-calibration-no-controlnet", + help="Output directory for calibration data", + ) + + # Inference hyperparameters + parser.add_argument( + "--height", + type=int, + default=512, + help="Image height", + ) + parser.add_argument( + "--width", + type=int, + default=512, + help="Image width", + ) + parser.add_argument( + "--num_inference_steps", + type=int, + default=9, + help="Number of inference steps", + ) + parser.add_argument( + "--seed", + type=int, + default=None, + help="Random seed (if None, use random seed per prompt)", + ) + parser.add_argument( + "--sampler_name", + type=str, + default="Flow", + choices=["Flow", "Flow_Unipc", "Flow_DPM++"], + help="Sampler name", + ) + parser.add_argument( + "--max_sequence_length", + type=int, + default=128, + help="Maximum sequence length for text encoder", + ) + parser.add_argument( + "--prompts_file", + type=str, + default=None, + help="Path to a text file containing prompts (one per line). If not provided, use default prompts.", + ) + parser.add_argument( + "--device", + type=str, + default="cuda:0", + help="Device to use (e.g., 'cuda:0', 'cpu')", + ) + + return parser.parse_args() + + +# Model and config paths (will be overridden by args) +config_path_default = "config/z_image/z_image.yaml" +model_name = "models/Diffusion_Transformer/Z-Image-Turbo/" + +# Prompts to sweep; adjust as needed +DEFAULT_PROMPTS = [ + "(masterpiece, best quality) solo female on a tropical beach, golden hour rim light, cinematic grading", + "nighttime cyberpunk boulevard, neon reflections on wet asphalt, volumetric fog, wide shot", + "sunrise over alpine mountains, low clouds in valleys, god rays, ultra-detailed landscape", + "modern minimal living room, soft natural light, Scandinavian design, high-resolution interior render", + "classical oil painting of a renaissance noblewoman, chiaroscuro lighting, rich textures", + "macro photography of a dewdrop on a leaf, extreme detail, shallow depth of field", + "futuristic sports car parked under neon lights, glossy paint, cinematic 35mm look", + "ancient library with towering bookshelves, warm candlelight, dust motes in air", + "portrait of an astronaut in full suit, visor reflection showing earth, studio lighting", + "stormy sea with a lone lighthouse, crashing waves, dramatic clouds, long exposure feel", + "cybernetic samurai standing in rain, backlit silhouette, moody blue-orange palette", + "lush rainforest waterfall, soft mist, saturated greens, wide-angle composition", + "product shot of a smartwatch on marble, softbox lighting, crisp shadows, advertisement style", + "architectural exterior of a glass skyscraper at dusk, warm interior lights, reflections", + "vintage film photograph of a 1950s diner at night, grain and halation, neon signage", + "hyperrealistic bowl of ramen, steam rising, glossy broth, detailed toppings", + "fantasy castle on a floating island, waterfalls falling into clouds, sunset lighting", + "high-fashion editorial portrait, dramatic chiaroscuro, sharp focus on eyes", + "aerial view of winding river through autumn forest, golden and crimson leaves", + "studio shot of running shoes mid-air, motion blur trails, vibrant background gradient", + "noir city alley in the 1940s, hard shadows, rain-slick pavement, moody atmosphere", + "desert caravan at twilight, silhouettes of camels, soft purple sky, cinematic scope", + "close-up of a mechanical watch movement, intricate gears, metallic reflections", + "bioluminescent underwater reef, glowing corals, schools of fish, deep blue tones", + "portrait of an elderly man with weathered face, soft window light, fine skin detail", + "snowy village at night, warm cabin lights, smoke from chimneys, peaceful mood", + "futuristic data center aisle, cool cyan lighting, depth and symmetry", + "oil painting of a bowl of fruit in Dutch masters style, rich textures, dramatic lighting", + "sunlit meadow with wildflowers, shallow depth of field, pastel color palette", + "sci-fi corridor with volumetric light shafts, pristine white surfaces, wide lens", + "luxury wristwatch on black velvet, high contrast, advertisement macro shot", + "medieval marketplace at dawn, merchants setting up, soft warm light, lively details", +] + +# Output directory for saved npy dictionaries and sample images (will be overridden by args) +OUTPUT_DIR = os.path.join(repo_root, "onnx-calibration-no-controlnet") +OUTPUT_IMG_DIR = os.path.join(OUTPUT_DIR, "samples") + +# Inference hyperparameters (will be overridden by args) +sample_size = [512, 512] # H, W +num_inference_steps = 9 # align with launcher_no_controlnet.py default +# If None, use a fresh random seed per prompt; otherwise use base_seed + prompt_idx +seed = None +sampler_name = "Flow" +vae_scale_factor = 8 +vae_scale = vae_scale_factor * 2 +max_sequence_length = 128 # Will be overridden by args + +def _select_weight_dtype(device: torch.device) -> torch.dtype: + if device.type == "cuda": + # Prefer bfloat16 when supported; otherwise float16 + if torch.cuda.is_bf16_supported(): + return torch.bfloat16 + return torch.float16 + return torch.float32 + + +weight_dtype = None # set in main based on device +control_context_scale = 0.75 + +# Scheduler selection (will be set based on args) +def get_scheduler(sampler_name): + Chosen_Scheduler = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, + }[sampler_name] + return Chosen_Scheduler + + +def _resolve_config_path(path: str) -> Optional[str]: + candidate = path if os.path.isabs(path) else os.path.join(repo_root, path) + return candidate if os.path.exists(candidate) else None + + +def _infer_module_device(module: torch.nn.Module) -> torch.device: + param = next(module.parameters(), None) + if param is not None: + return param.device + buffer = next(module.buffers(), None) + if buffer is not None: + return buffer.device + return torch.device("cpu") + + +def _encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device], + tokenizer: AutoTokenizer, + text_encoder: Qwen3ForCausalLM, + max_sequence_length: int = 512, +) -> List[torch.FloatTensor]: + device = device or _infer_module_device(text_encoder) + if isinstance(prompt, str): + prompt = [prompt] + + for i, prompt_item in enumerate(prompt): + messages = [{"role": "user", "content": prompt_item}] + prompt_item = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + prompt[i] = prompt_item + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + + with torch.no_grad(): + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] + + embeddings_list = [] + for i in range(len(prompt_embeds)): + embeddings_list.append(prompt_embeds[i]) + return embeddings_list + + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device], + tokenizer: AutoTokenizer, + text_encoder: Qwen3ForCausalLM, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + max_sequence_length: int = 512, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + prompt_embeds = _encode_prompt( + prompt=prompt, + device=device, + tokenizer=tokenizer, + text_encoder=text_encoder, + max_sequence_length=max_sequence_length, + ) + + if do_classifier_free_guidance: + if negative_prompt is None: + negative_prompt = ["" for _ in prompt] + else: + negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + assert len(prompt) == len(negative_prompt) + negative_prompt_embeds = _encode_prompt( + prompt=negative_prompt, + device=device, + tokenizer=tokenizer, + text_encoder=text_encoder, + max_sequence_length=max_sequence_length, + ) + else: + negative_prompt_embeds = [] + return prompt_embeds, negative_prompt_embeds + + +def _stack_prompt_embeddings(prompt_embeds_input): + if isinstance(prompt_embeds_input, list): + return torch.stack(prompt_embeds_input, dim=0) + return prompt_embeds_input + + +def prepare_latents( + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, +): + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + shape = (batch_size, num_channels_latents, height, width) + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + return latents + + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + **kwargs, +): + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, len(timesteps) + + +def _to_float32_np(tensor: torch.Tensor) -> np.ndarray: + return tensor.detach().to(dtype=torch.float32, device="cpu").numpy() + + +def save_npy_dict(path: str, data: dict): + os.makedirs(os.path.dirname(path), exist_ok=True) + np.save(path, data) + + +def _slugify(text: str, max_len: int = 64) -> str: + safe = [] + for ch in text.lower(): + if ch.isalnum(): + safe.append(ch) + elif ch in [" ", "-", ",", ".", "_"]: + safe.append("_") + slug = "".join(safe).strip("_") + if not slug: + slug = "prompt" + return slug[:max_len] + + +def _build_generator(device: torch.device, base_seed: Optional[int], prompt_idx: int) -> torch.Generator: + gen = torch.Generator(device=device) + if base_seed is not None: + gen = gen.manual_seed(base_seed + prompt_idx) + return gen + + +def main(): + args = parse_args() + + # Determine device + if args.device == "cuda:0" and not torch.cuda.is_available(): + device = torch.device("cpu") + print("CUDA not available, using CPU instead") + else: + device = torch.device(args.device) + + global weight_dtype + weight_dtype = _select_weight_dtype(device) + + # Resolve paths + config_path = _resolve_config_path(args.config_path) + config = OmegaConf.load(config_path) if config_path else None + + model_name = args.model_name if os.path.isabs(args.model_name) else os.path.join(repo_root, args.model_name) + + # Validate model directory structure + required_subfolders = ["tokenizer", "text_encoder", "transformer", "vae", "scheduler"] + missing_subfolders = [sf for sf in required_subfolders if not os.path.exists(os.path.join(model_name, sf))] + if missing_subfolders: + raise ValueError( + f"Model directory '{model_name}' is missing required subfolders: {missing_subfolders}\n" + f"Expected structure: {model_name}/{{tokenizer,text_encoder,transformer,vae,scheduler}}/\n" + f"Note: --model_name should point to a HuggingFace model directory, not an ONNX file." + ) + + output_dir = args.output_dir if os.path.isabs(args.output_dir) else os.path.join(repo_root, args.output_dir) + output_img_dir = os.path.join(output_dir, "samples") + + # Load prompts + if args.prompts_file: + prompts_path = args.prompts_file if os.path.isabs(args.prompts_file) else os.path.join(repo_root, args.prompts_file) + if os.path.exists(prompts_path): + with open(prompts_path, 'r', encoding='utf-8') as f: + prompts = [line.strip() for line in f if line.strip()] + print(f"Loaded {len(prompts)} prompts from {prompts_path}") + else: + print(f"Prompts file not found: {prompts_path}, using default prompts") + prompts = DEFAULT_PROMPTS + else: + prompts = DEFAULT_PROMPTS + print(f"Using {len(prompts)} default prompts") + + extra_kwargs = {} + if config is not None and hasattr(config, "transformer_additional_kwargs"): + extra_kwargs = OmegaConf.to_container(config.transformer_additional_kwargs, resolve=True) + + tokenizer = AutoTokenizer.from_pretrained(model_name, subfolder="tokenizer") + text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, + subfolder="text_encoder", + torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + ).to(device=device, dtype=weight_dtype) + text_encoder.eval() + + transformer = ZImageTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + **({"transformer_additional_kwargs": extra_kwargs} if extra_kwargs else {}), + ).to(device=device, dtype=weight_dtype) + + vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae", + ).to(device=device, dtype=weight_dtype) + vae.eval() + + image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + vae_config_shift_factor = getattr(vae.config, "shift_factor", 0.0) + vae_config_scaling_factor = getattr(vae.config, "scaling_factor", 1.0) + + Chosen_Scheduler = get_scheduler(args.sampler_name) + scheduler = Chosen_Scheduler.from_pretrained(model_name, subfolder="scheduler") + + height, width = args.height, args.width + if height % vae_scale != 0 or width % vae_scale != 0: + raise ValueError(f"Height/Width must be divisible by {vae_scale}") + + num_channels_latents = 16 + do_classifier_free_guidance = False + + for prompt_idx, prompt in enumerate(prompts): + prompt_embeds, negative_prompt_embeds = encode_prompt( + prompt=prompt, + negative_prompt="", + do_classifier_free_guidance=do_classifier_free_guidance, + prompt_embeds=None, + negative_prompt_embeds=None, + device=device, + tokenizer=tokenizer, + text_encoder=text_encoder, + max_sequence_length=args.max_sequence_length, + ) + + generator = _build_generator(device, args.seed, prompt_idx) + latents = prepare_latents( + batch_size=1, + num_channels_latents=num_channels_latents, + height=height, + width=width, + dtype=torch.float32, + device=device, + generator=generator, + ) + + prompt_embeds_tensor = _stack_prompt_embeddings(prompt_embeds) + image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2) + mu = calculate_shift( + image_seq_len, + scheduler.config.get("base_image_seq_len", 256), + scheduler.config.get("max_image_seq_len", 4096), + scheduler.config.get("base_shift", 0.5), + scheduler.config.get("max_shift", 1.15), + ) + scheduler.sigma_min = 0.0 + scheduler_kwargs = {"mu": mu} + timesteps, _ = retrieve_timesteps( + scheduler, + num_inference_steps=args.num_inference_steps, + device=device, + **scheduler_kwargs, + ) + + for step_idx, t in enumerate(timesteps): + timestep = t.expand(latents.shape[0]) + timestep_model_input = (1000 - timestep) / 1000 + + latent_model_input = latents.to(transformer.dtype) + latent_for_onnx = latent_model_input.unsqueeze(2) + prompt_for_onnx = prompt_embeds_tensor + + transformer_inputs = { + "latent_model_input": _to_float32_np(latent_for_onnx), + "timestep": _to_float32_np(timestep_model_input), + "prompt_embeds": _to_float32_np(prompt_for_onnx), + } + transformer_save_path = os.path.join( + output_dir, + f"transformer_inputs_prompt{prompt_idx:03d}_step{step_idx:02d}.npy", + ) + save_npy_dict(transformer_save_path, transformer_inputs) + + timestep_model_input = timestep_model_input.to(device=device, dtype=transformer.dtype) + latent_model_input_list = list(latent_for_onnx.unbind(dim=0)) + + if device.type == "cuda": + with torch.autocast(device_type="cuda", dtype=transformer.dtype): + model_out_list = transformer( + latent_model_input_list, + timestep_model_input, + prompt_embeds, + patch_size=2, + f_patch_size=1, + )[0] + else: + model_out_list = transformer( + latent_model_input_list, + timestep_model_input, + prompt_embeds, + patch_size=2, + f_patch_size=1, + )[0] + + noise_pred = torch.stack([t.float() for t in model_out_list], dim=0) + noise_pred = noise_pred.squeeze(2) + noise_pred = -noise_pred + + latents = scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0] + latents = latents.to(device=device, dtype=torch.float32) + + latents_dec = latents.to(vae.dtype) + latents_dec = (latents_dec / vae_config_scaling_factor) + vae_config_shift_factor + vae_decoder_inputs = {"latents": _to_float32_np(latents_dec)} + vae_save_path = os.path.join( + output_dir, + f"vae_decoder_inputs_prompt{prompt_idx:03d}.npy", + ) + save_npy_dict(vae_save_path, vae_decoder_inputs) + + # Decode once to ensure latents are valid and save sample image for inspection + with torch.no_grad(): + if device.type == "cuda": + with torch.autocast(device_type="cuda", dtype=vae.dtype): + image = vae.decode(latents_dec, return_dict=False)[0] + else: + image = vae.decode(latents_dec, return_dict=False)[0] + images = image_processor.postprocess(image, output_type="pil") + if images: + os.makedirs(output_img_dir, exist_ok=True) + slug = _slugify(prompt) + img_path = os.path.join(output_img_dir, f"prompt{prompt_idx:03d}_{slug}.png") + images[0].save(img_path) + + print(f"Saved ONNX calibration inputs to {output_dir}") + + +if __name__ == "__main__": + torch.set_grad_enabled(False) + main() diff --git a/VideoX-Fun/examples/z_image_fun/collect_subgraph_inputs.py b/VideoX-Fun/examples/z_image_fun/collect_subgraph_inputs.py new file mode 100644 index 0000000000000000000000000000000000000000..a56d2f7847d61753500190fe263298d8bfcad9be --- /dev/null +++ b/VideoX-Fun/examples/z_image_fun/collect_subgraph_inputs.py @@ -0,0 +1,884 @@ +#!/usr/bin/env python3 +"""收集所有子ONNX模型的输入数据，用于量化校准。 + +此脚本通过推理完整的原始ONNX模型来收集各个子图的输入数据。 +会为每个子图创建独立的文件夹，保存输入数据，并打包成tar文件。 +""" +import argparse +import json +import os +import sys +import tarfile +from pathlib import Path +from typing import Dict, List, Optional, Union + +import numpy as np +import onnxruntime as ort +import torch +from omegaconf import OmegaConf + +# Determine repo root and model locations +current_file_path = os.path.abspath(__file__) +project_roots = [ + os.path.dirname(current_file_path), + os.path.dirname(os.path.dirname(current_file_path)), + os.path.dirname(os.path.dirname(os.path.dirname(current_file_path))), +] +for project_root in project_roots: + if project_root not in sys.path: + sys.path.insert(0, project_root) +repo_root = project_roots[-1] + +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.image_processor import VaeImageProcessor +from diffusers.utils.torch_utils import randn_tensor +from videox_fun.models import ( + AutoTokenizer, + AutoencoderKL, + Qwen3ForCausalLM, + ZImageTransformer2DModel, +) +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + + +# Model and config paths +config_path_default = "config/z_image/z_image.yaml" +model_name = "models/Diffusion_Transformer/Z-Image-Turbo/" + +# 原始ONNX模型路径和子图配置 +ORIGINAL_ONNX = os.path.join(repo_root, "onnx-models/z_image_transformer_body_only_simp_slim.onnx") +SUBGRAPH_CONFIG = os.path.join(repo_root, "pulsar2_configs/transformers_subgraph.json") + +# 校准数据prompts - 只使用部分prompts来加快处理 +PROMPTS = [ + "(masterpiece, best quality) solo female on a tropical beach, golden hour rim light, cinematic grading", + "nighttime cyberpunk boulevard, neon reflections on wet asphalt, volumetric fog, wide shot", + "sunrise over alpine mountains, low clouds in valleys, god rays, ultra-detailed landscape", + "modern minimal living room, soft natural light, Scandinavian design, high-resolution interior render", +] + +# 输出目录 +OUTPUT_BASE_DIR = os.path.join(repo_root, "onnx-calibration-subgraphs") +TAR_LIST_FILE = os.path.join(OUTPUT_BASE_DIR, "subgraph_calibration_paths.txt") + +# 推理参数默认值 +sample_size = [512, 512] # H, W +num_inference_steps = 9 +seed = 42 # 固定种子保证可复现 +sampler_name = "Flow" +vae_scale_factor = 8 +vae_scale = vae_scale_factor * 2 +max_sequence_length = 128 + + +def _select_weight_dtype(device: torch.device) -> torch.dtype: + if device.type == "cuda": + if torch.cuda.is_bf16_supported(): + return torch.bfloat16 + return torch.float16 + return torch.float32 + + +SCHEDULER_MAP = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +} + + +def _resolve_config_path(path: str) -> Optional[str]: + candidate = path if os.path.isabs(path) else os.path.join(repo_root, path) + return candidate if os.path.exists(candidate) else None + + +def _infer_module_device(module: torch.nn.Module) -> torch.device: + param = next(module.parameters(), None) + if param is not None: + return param.device + buffer = next(module.buffers(), None) + if buffer is not None: + return buffer.device + return torch.device("cpu") + + +def _encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device], + tokenizer: AutoTokenizer, + text_encoder: Qwen3ForCausalLM, + max_sequence_length: int = 512, +) -> List[torch.FloatTensor]: + device = device or _infer_module_device(text_encoder) + if isinstance(prompt, str): + prompt = [prompt] + + for i, prompt_item in enumerate(prompt): + messages = [{"role": "user", "content": prompt_item}] + prompt_item = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + prompt[i] = prompt_item + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + + with torch.no_grad(): + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] + + embeddings_list = [] + for i in range(len(prompt_embeds)): + embeddings_list.append(prompt_embeds[i]) + return embeddings_list + + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device], + tokenizer: AutoTokenizer, + text_encoder: Qwen3ForCausalLM, + do_classifier_free_guidance: bool = False, + negative_prompt: Optional[Union[str, List[str]]] = None, + max_sequence_length: int = 512, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + prompt_embeds = _encode_prompt( + prompt=prompt, + device=device, + tokenizer=tokenizer, + text_encoder=text_encoder, + max_sequence_length=max_sequence_length, + ) + + if do_classifier_free_guidance: + if negative_prompt is None: + negative_prompt = ["" for _ in prompt] + else: + negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_embeds = _encode_prompt( + prompt=negative_prompt, + device=device, + tokenizer=tokenizer, + text_encoder=text_encoder, + max_sequence_length=max_sequence_length, + ) + else: + negative_prompt_embeds = [] + return prompt_embeds, negative_prompt_embeds + + +def _stack_prompt_embeddings(prompt_embeds_input): + if isinstance(prompt_embeds_input, list): + return torch.stack(prompt_embeds_input, dim=0) + return prompt_embeds_input + + +def prepare_latents(batch_size, num_channels_latents, height, width, dtype, device, generator): + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + shape = (batch_size, num_channels_latents, height, width) + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + return latents + + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +def retrieve_timesteps(scheduler, num_inference_steps: Optional[int] = None, device=None, **kwargs): + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, len(timesteps) + + +def _to_float32_np(tensor: torch.Tensor) -> np.ndarray: + return tensor.detach().to(dtype=torch.float32, device="cpu").numpy() + + +def load_subgraph_config(config_path: str) -> List[dict]: + """加载子图配置""" + with open(config_path, 'r') as f: + config = json.load(f) + sub_configs = config.get("compiler", {}).get("sub_configs", []) + return sub_configs + + +def add_intermediate_outputs_to_onnx(onnx_path: str, output_tensor_names: List[str], output_path: str): + """为ONNX模型添加中间输出，以便提取中间张量值""" + import onnx + + print(f"加载ONNX模型: {onnx_path}") + model = onnx.load(onnx_path, load_external_data=False) # 只加载结构 + + # 收集现有的输出名称 + existing_outputs = {out.name for out in model.graph.output} + + # 为每个需要的张量添加为输出（如果还不是输出） + added_count = 0 + for tensor_name in output_tensor_names: + if tensor_name not in existing_outputs: + # 查找这个张量的value_info + value_info = None + for vi in model.graph.value_info: + if vi.name == tensor_name: + value_info = vi + break + + if value_info is None: + # 如果没有找到，创建一个未定义类型的 + from onnx import TensorProto + value_info = onnx.helper.make_tensor_value_info( + tensor_name, TensorProto.UNDEFINED, None + ) + + # 添加为输出 + output_vi = onnx.ValueInfoProto() + output_vi.CopyFrom(value_info) + model.graph.output.append(output_vi) + added_count += 1 + + if added_count > 0: + print(f"添加了 {added_count} 个中间输出") + onnx.save(model, output_path, save_as_external_data=False) + print(f"修改后的模型已保存到: {output_path}") + return output_path + else: + print("所有需要的输出已存在，无需修改模型") + return onnx_path + + +def create_onnx_session(onnx_path: str, providers: List[str]): + """创建ONNX推理会话""" + sess_options = ort.SessionOptions() + sess_options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL + + # 加载模型 + session = ort.InferenceSession(onnx_path, sess_options, providers=providers) + + return session + + +def run_inference_and_collect( + tokenizer, + text_encoder, + transformer, + scheduler, + device, + weight_dtype, + sub_configs: List[dict], + onnx_session, + output_base_dir: str, + skip_existing: bool = False, +): + """运行推理并收集所有子图的输入数据""" + + # 检查已存在的数据 + existing_data = check_existing_data(output_base_dir) if skip_existing else {} + + # 为每个子图创建数据存储 (包括 cfg_00 到 cfg_32) + subgraph_data = {} + need_inference = False # 标记是否需要推理 + + for idx, config in enumerate(sub_configs): + subgraph_label = f"cfg_{idx:02d}" + + # 如果跳过已存在且数据存在，则加载已有数据 + if skip_existing and existing_data.get(subgraph_label, False): + loaded_data = load_existing_subgraph_data(subgraph_label, output_base_dir) + if loaded_data: + subgraph_data[subgraph_label] = loaded_data + continue + + # 否则创建新的数据结构 + subgraph_data[subgraph_label] = { + "start_tensors": config["start_tensor_names"], + "end_tensors": config["end_tensor_names"], + "samples": [] + } + need_inference = True + + # 添加 auto_00 子图（自动生成的最后一个子图） + if skip_existing and existing_data.get("auto_00", False): + loaded_data = load_existing_subgraph_data("auto_00", output_base_dir) + if loaded_data: + subgraph_data["auto_00"] = loaded_data + else: + subgraph_data["auto_00"] = { + "start_tensors": ["/model/layers.29/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensors": ["sample"], + "samples": [] + } + need_inference = True + else: + subgraph_data["auto_00"] = { + "start_tensors": ["/model/layers.29/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensors": ["sample"], + "samples": [] + } + need_inference = True + + # 如果所有数据都已存在，直接返回 + if not need_inference: + print("\n所有子图数据都已存在，跳过推理过程") + return subgraph_data + + height, width = sample_size + num_channels_latents = 16 + + # 获取ONNX session的输入输出名称 + onnx_inputs = {inp.name: inp for inp in onnx_session.get_inputs()} + onnx_outputs = {out.name: out for out in onnx_session.get_outputs()} + + print(f"ONNX模型输入: {list(onnx_inputs.keys())}") + print(f"ONNX模型输出数量: {len(onnx_outputs)}") + print(f"收集子图数量: {len(subgraph_data)} (包括 auto_00)") + + for prompt_idx, prompt in enumerate(PROMPTS): + print(f"\n处理 Prompt {prompt_idx + 1}/{len(PROMPTS)}: {prompt[:60]}...") + + # 编码prompt + prompt_embeds, _ = encode_prompt( + prompt=prompt, + negative_prompt="", + do_classifier_free_guidance=False, + device=device, + tokenizer=tokenizer, + text_encoder=text_encoder, + max_sequence_length=max_sequence_length, + ) + + # 准备latents + generator = torch.Generator(device=device).manual_seed(seed + prompt_idx) + latents = prepare_latents( + batch_size=1, + num_channels_latents=num_channels_latents, + height=height, + width=width, + dtype=torch.float32, + device=device, + generator=generator, + ) + + prompt_embeds_tensor = _stack_prompt_embeddings(prompt_embeds) + image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2) + mu = calculate_shift( + image_seq_len, + scheduler.config.get("base_image_seq_len", 256), + scheduler.config.get("max_image_seq_len", 4096), + scheduler.config.get("base_shift", 0.5), + scheduler.config.get("max_shift", 1.15), + ) + scheduler.sigma_min = 0.0 + timesteps, _ = retrieve_timesteps( + scheduler, + num_inference_steps=num_inference_steps, + device=device, + mu=mu, + ) + + # 对每个timestep进行推理 + for step_idx, t in enumerate(timesteps): + print(f" Step {step_idx + 1}/{len(timesteps)}") + + # 确保latents是4维 + if latents.dim() == 5: + print(f" 警告: latents是5维，squeeze到4维") + latents = latents.squeeze(2) + + timestep = t.expand(latents.shape[0]) + timestep_model_input = (1000 - timestep) / 1000 + + latent_model_input = latents + # ONNX模型期望5维输入: (B, C, 1, H, W) + latent_for_onnx = latent_model_input.unsqueeze(2) # (B, C, H, W) -> (B, C, 1, H, W) + prompt_for_onnx = prompt_embeds_tensor + + # 准备ONNX输入 + onnx_feed = { + "latent_model_input": _to_float32_np(latent_for_onnx), + "timestep": _to_float32_np(timestep_model_input), + "prompt_embeds": _to_float32_np(prompt_for_onnx), + } + + if step_idx == 0: + print(f" 调试: latents原始 shape = {latents.shape}") + print(f" 调试: latent_for_onnx shape = {latent_for_onnx.shape}") + print(f" 调试: timestep shape = {timestep_model_input.shape}") + print(f" 调试: prompt shape = {prompt_for_onnx.shape}") + + # 运行ONNX推理 + try: + onnx_results = onnx_session.run(None, onnx_feed) + onnx_output_dict = {out.name: onnx_results[i] for i, out in enumerate(onnx_session.get_outputs())} + + # 保存各个子图的输入 + sample_id = f"prompt{prompt_idx:03d}_step{step_idx:02d}" + + # 为每个子图收集输入 (cfg_00 到 cfg_32) + for idx, config in enumerate(sub_configs): + subgraph_label = f"cfg_{idx:02d}" + start_tensors = config["start_tensor_names"] + + # 构建输入字典 + inputs_dict = {} + for tensor_name in start_tensors: + if tensor_name in onnx_feed: + # 这是原始输入 + inputs_dict[tensor_name] = onnx_feed[tensor_name] + elif tensor_name in onnx_output_dict: + # 这是中间层输出 + inputs_dict[tensor_name] = onnx_output_dict[tensor_name] + else: + print(f" 警告: 子图 {subgraph_label} 缺少输入张量: {tensor_name}") + + if inputs_dict: + subgraph_data[subgraph_label]["samples"].append({ + "id": sample_id, + "inputs": inputs_dict + }) + + # 收集 auto_00 子图的输入 + auto_start_tensors = subgraph_data["auto_00"]["start_tensors"] + auto_inputs_dict = {} + for tensor_name in auto_start_tensors: + if tensor_name in onnx_feed: + auto_inputs_dict[tensor_name] = onnx_feed[tensor_name] + elif tensor_name in onnx_output_dict: + auto_inputs_dict[tensor_name] = onnx_output_dict[tensor_name] + else: + print(f" 警告: 子图 auto_00 缺少输入张量: {tensor_name}") + + if auto_inputs_dict: + subgraph_data["auto_00"]["samples"].append({ + "id": sample_id, + "inputs": auto_inputs_dict + }) + + # 使用ONNX的输出更新latents + if "sample" in onnx_output_dict: + noise_pred = torch.from_numpy(onnx_output_dict["sample"]).to(device=device, dtype=torch.float32) + # noise_pred可能是4维或5维，确保是4维 + if noise_pred.dim() == 5: + noise_pred = noise_pred.squeeze(2) # (B, C, 1, H, W) -> (B, C, H, W) + + latents = scheduler.step(noise_pred, t, latents, return_dict=False)[0] + latents = latents.to(device=device, dtype=torch.float32) + + # 确保latents是4维 + if latents.dim() == 5: + latents = latents.squeeze(2) + else: + print(" 警告: ONNX输出中没有'sample'，跳过latents更新") + break + + except Exception as e: + print(f" 错误: ONNX推理失败: {e}") + print(" 回退到PyTorch推理...") + + # 使用PyTorch模型推理 + timestep_model_input_device = timestep_model_input.to(device=device, dtype=transformer.dtype) + # PyTorch transformer需要一个list，每个元素是4维张量 (C, F, H, W) + # latent_for_onnx是5维 (B, C, F, H, W)，squeeze掉batch维度后unbind + latent_5d = latent_for_onnx.to(transformer.dtype) # (B, C, 1, H, W) + # 先squeeze掉F维度变成4维，再unbind成list + # 注意：transformer期望的是 list of (C, F, H, W)，而F=1 + latent_model_input_list = list(latent_5d.unbind(dim=0)) # list of (C, 1, H, W) + + if device.type == "cuda": + with torch.autocast(device_type="cuda", dtype=transformer.dtype): + model_out_list = transformer( + latent_model_input_list, + timestep_model_input_device, + prompt_embeds, + patch_size=2, + f_patch_size=1, + )[0] + else: + model_out_list = transformer( + latent_model_input_list, + timestep_model_input_device, + prompt_embeds, + patch_size=2, + f_patch_size=1, + )[0] + + noise_pred = torch.stack([t.float() for t in model_out_list], dim=0) + # noise_pred可能是5维或4维，确保是4维 + if noise_pred.dim() == 5: + noise_pred = noise_pred.squeeze(2) + noise_pred = -noise_pred + + # 使用PyTorch收集基本输入（无法获取中间层） + sample_id = f"prompt{prompt_idx:03d}_step{step_idx:02d}" + for idx in range(min(3, len(sub_configs))): # 只保存前几个子图 + subgraph_label = f"cfg_{idx:02d}" + config = sub_configs[idx] + + inputs_dict = {} + for tensor_name in config["start_tensor_names"]: + if tensor_name == "timestep": + inputs_dict[tensor_name] = _to_float32_np(timestep_model_input) + elif tensor_name == "prompt_embeds": + inputs_dict[tensor_name] = _to_float32_np(prompt_for_onnx) + elif tensor_name == "latent_model_input": + inputs_dict[tensor_name] = _to_float32_np(latent_for_onnx) + + if inputs_dict: + subgraph_data[subgraph_label]["samples"].append({ + "id": sample_id, + "inputs": inputs_dict + }) + + latents = scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0] + latents = latents.to(device=device, dtype=torch.float32) + + # 确保latents是4维 + if latents.dim() == 5: + latents = latents.squeeze(2) + + return subgraph_data + + +def check_existing_data(output_base_dir: str) -> Dict[str, bool]: + """检查哪些子图的tar文件已经存在 + + Returns: + Dict[str, bool]: 子图标签到是否存在的映射 + """ + existing = {} + output_dir = Path(output_base_dir) + if not output_dir.exists(): + return existing + + # 检查所有可能的tar文件 + for i in range(33): # cfg_00 到 cfg_32 + subgraph_label = f"cfg_{i:02d}" + tar_path = output_dir / f"{subgraph_label}.tar" + existing[subgraph_label] = tar_path.exists() + + # 检查 auto_00 + auto_tar = output_dir / "auto_00.tar" + existing["auto_00"] = auto_tar.exists() + + return existing + + +def load_existing_subgraph_data(subgraph_label: str, output_base_dir: str) -> Optional[Dict]: + """从已存在的tar文件加载子图数据 + + Returns: + Dict or None: 子图数据结构，如果加载失败返回None + """ + tar_path = Path(output_base_dir) / f"{subgraph_label}.tar" + if not tar_path.exists(): + return None + + try: + # 从tar中提取并统计样本数 + sample_count = 0 + with tarfile.open(tar_path, "r") as tar: + members = tar.getmembers() + # 只计算.npy文件 + sample_count = sum(1 for m in members if m.name.endswith('.npy')) + + print(f" 从已存在的tar加载: {subgraph_label} ({sample_count} 样本)") + return { + "loaded_from_existing": True, + "sample_count": sample_count, + "tar_path": str(tar_path.absolute()), + "samples": [] # 添加空的samples列表，避免后续访问时KeyError + } + except Exception as e: + print(f" 警告: 加载已存在的数据失败 ({subgraph_label}): {e}") + return None + + +def save_subgraph_data(subgraph_data: Dict, output_base_dir: str, skip_existing: bool = False) -> List[str]: + """保存子图数据并创建tar文件""" + os.makedirs(output_base_dir, exist_ok=True) + tar_paths = [] + + for subgraph_label, data in subgraph_data.items(): + # 如果是从已存在的tar加载的，直接添加到列表并跳过 + if data.get("loaded_from_existing", False): + tar_path = data.get("tar_path") + if tar_path and os.path.exists(tar_path): + tar_paths.append(tar_path) + print(f" 跳过已存在: {subgraph_label} ({data.get('sample_count', 0)} 样本)") + continue + + print(f"\n保存子图数据: {subgraph_label}") + + # 创建子图目录 + subgraph_dir = os.path.join(output_base_dir, subgraph_label) + os.makedirs(subgraph_dir, exist_ok=True) + + # 保存每个样本 + for sample in data["samples"]: + sample_file = os.path.join(subgraph_dir, f"{sample['id']}.npy") + np.save(sample_file, sample["inputs"]) + + # 创建tar文件 + tar_path = os.path.join(output_base_dir, f"{subgraph_label}.tar") + with tarfile.open(tar_path, "w") as tar: + tar.add(subgraph_dir, arcname=subgraph_label) + + tar_paths.append(os.path.abspath(tar_path)) + print(f" 已创建: {tar_path} (包含 {len(data['samples'])} 个样本)") + + return tar_paths + + +def write_tar_list(tar_paths: List[str], output_file: str): + """将所有tar文件路径写入文本文件""" + with open(output_file, 'w') as f: + for path in tar_paths: + f.write(path + '\n') + print(f"\n所有tar文件路径已写入: {output_file}") + + +def main(): + global sample_size, num_inference_steps, seed, sampler_name, vae_scale_factor, vae_scale, max_sequence_length + + parser = argparse.ArgumentParser(description="收集子图ONNX模型的量化校准数据") + parser.add_argument( + "--skip-existing", + action="store_true", + help="跳过已经存在的tar文件，不重新生成数据" + ) + parser.add_argument( + "--onnx", + default=ORIGINAL_ONNX, + help="原始ONNX模型路径，默认使用项目内置模型" + ) + parser.add_argument( + "--subgraph-config", + dest="subgraph_config", + default=SUBGRAPH_CONFIG, + help="子图配置json路径，默认使用项目内置配置" + ) + parser.add_argument( + "--output-dir", + dest="output_base_dir", + default=OUTPUT_BASE_DIR, + help="输出目录，存放子图输入数据及tar文件" + ) + parser.add_argument( + "--tar-list-file", + default=None, + help="tar列表文件路径，未提供时默认写入输出目录下的subgraph_calibration_paths.txt" + ) + parser.add_argument( + "--sample-size", + nargs=2, + type=int, + metavar=("H", "W"), + default=sample_size, + help="推理分辨率，格式: H W，默认 512 512", + ) + parser.add_argument( + "--num-inference-steps", + type=int, + default=num_inference_steps, + help="推理步数，默认 9", + ) + parser.add_argument( + "--seed", + type=int, + default=seed, + help="随机种子，默认 42", + ) + parser.add_argument( + "--sampler", + dest="sampler_name", + choices=sorted(SCHEDULER_MAP.keys()), + default=sampler_name, + help="采样器名称，默认 Flow", + ) + parser.add_argument( + "--vae-scale-factor", + type=int, + default=vae_scale_factor, + help="VAE 下采样因子，默认 8", + ) + parser.add_argument( + "--max-seq-len", + type=int, + default=max_sequence_length, + help="最大序列长度，默认 128", + ) + args = parser.parse_args() + + print("=" * 80) + print("收集子图ONNX模型的量化校准数据") + print("=" * 80) + + # 应用命令行参数到全局推理参数 + sample_size = args.sample_size + num_inference_steps = args.num_inference_steps + seed = args.seed + sampler_name = args.sampler_name + vae_scale_factor = args.vae_scale_factor + vae_scale = vae_scale_factor * 2 + max_sequence_length = args.max_seq_len + + if args.skip_existing: + print("模式: 跳过已存在的数据") + else: + print("模式: 重新生成所有数据") + + original_onnx = os.path.expanduser(args.onnx) + subgraph_config_path = os.path.expanduser(args.subgraph_config) + output_base_dir = os.path.expanduser(args.output_base_dir) + tar_list_file = os.path.expanduser(args.tar_list_file) if args.tar_list_file else os.path.join( + output_base_dir, + os.path.basename(TAR_LIST_FILE), + ) + + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + weight_dtype = _select_weight_dtype(device) + + print(f"\n设备: {device}") + print(f"数据类型: {weight_dtype}") + + # 加载子图配置 + print(f"\n加载子图配置: {subgraph_config_path}") + sub_configs = load_subgraph_config(subgraph_config_path) + print(f"找到 {len(sub_configs)} 个子图配置") + + # 收集所有需要作为输出的中间张量 + all_tensor_names = set() + for config in sub_configs: + all_tensor_names.update(config["start_tensor_names"]) + all_tensor_names.update(config["end_tensor_names"]) + print(f"需要提取 {len(all_tensor_names)} 个张量") + + # 准备修改后的ONNX模型（添加中间输出） + modified_onnx_path = original_onnx.replace(".onnx", "_with_outputs.onnx") + + # 加载模型 + print(f"\n加载PyTorch模型: {model_name}") + config_path = _resolve_config_path(config_path_default) + config = OmegaConf.load(config_path) if config_path else None + extra_kwargs = {} + if config is not None and hasattr(config, "transformer_additional_kwargs"): + extra_kwargs = OmegaConf.to_container(config.transformer_additional_kwargs, resolve=True) + + tokenizer = AutoTokenizer.from_pretrained(model_name, subfolder="tokenizer") + text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, + subfolder="text_encoder", + torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + ).to(device=device, dtype=weight_dtype) + text_encoder.eval() + + transformer = ZImageTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + **({"transformer_additional_kwargs": extra_kwargs} if extra_kwargs else {}), + ).to(device=device, dtype=weight_dtype) + transformer.eval() + + scheduler_cls = SCHEDULER_MAP.get(sampler_name) + if scheduler_cls is None: + raise ValueError(f"不支持的采样器: {sampler_name}") + scheduler = scheduler_cls.from_pretrained(model_name, subfolder="scheduler") + + # 准备ONNX会话 + onnx_session = None + if os.path.exists(original_onnx): + print(f"\n找到原始ONNX模型: {original_onnx}") + try: + # 尝试修改ONNX添加中间输出 + print("准备添加中间输出到ONNX模型...") + modified_path = add_intermediate_outputs_to_onnx( + original_onnx, + list(all_tensor_names), + modified_onnx_path + ) + + # 创建ONNX会话 + print("创建ONNX推理会话...") + providers = ['CUDAExecutionProvider', 'CPUExecutionProvider'] if device.type == 'cuda' else ['CPUExecutionProvider'] + onnx_session = create_onnx_session(modified_path, providers) + print(f"ONNX会话创建成功，使用provider: {onnx_session.get_providers()}") + except Exception as e: + print(f"无法加载ONNX模型: {e}") + print("将使用PyTorch模型进行推理") + + if onnx_session is None: + print("\n警告: 未能创建ONNX会话，将只收集基本输入数据") + print("这意味着无法获取中间层的数据，只能收集前几个子图的输入") + print("如需完整数据，请确保ONNX模型可用") + + # 运行推理并收集数据 + print(f"\n开始推理并收集数据...") + print(f"Prompts数量: {len(PROMPTS)}") + print(f"推理步数: {num_inference_steps}") + + subgraph_data = run_inference_and_collect( + tokenizer, + text_encoder, + transformer, + scheduler, + device, + weight_dtype, + sub_configs, + onnx_session, + output_base_dir, + skip_existing=args.skip_existing, + ) + + # 保存数据 + print(f"\n保存数据到: {output_base_dir}") + tar_paths = save_subgraph_data(subgraph_data, output_base_dir, skip_existing=args.skip_existing) + + # 写入tar列表 + write_tar_list(tar_paths, tar_list_file) + + print("\n" + "=" * 80) + print("完成!") + print(f"总共创建了 {len(tar_paths)} 个tar文件") + print(f"tar文件列表: {tar_list_file}") + print("=" * 80) + + +if __name__ == "__main__": + """ + python examples/z_image_fun/collect_subgraph_inputs.py \ + --onnx /path/model.onnx \ + --subgraph-config /path/subgraph.json \ + --output-dir /data/out \ + --tar-list-file /data/out/subgraph_calibration_paths.txt \ + --sample-size 640 640 \ + --max-seq-len 256 + """ + torch.set_grad_enabled(False) + main() diff --git a/VideoX-Fun/examples/z_image_fun/export_onnx.py b/VideoX-Fun/examples/z_image_fun/export_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..f0bac8e689a0823ea79e7a84d8ec93cb547febe7 --- /dev/null +++ b/VideoX-Fun/examples/z_image_fun/export_onnx.py @@ -0,0 +1,494 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + Qwen3ForCausalLM, ZImageControlTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import ZImageControlPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, get_image, + get_video_to_video_latent, + save_videos_grid) + +from loguru import logger +import onnx +import subprocess + + +def run_onnxslim(input_file="vae.onnx", output_file="vae_slim.onnx"): + """ + 执行 onnxslim 命令压缩 ONNX 模型 + """ + try: + # 使用完整的命令路径（如果知道的话） + cmd = ["onnxslim", input_file, output_file] + + print(f"执行命令: {' '.join(cmd)}") + + # 执行命令, 实时输出 + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + universal_newlines=True + ) + + # 实时打印输出 + for line in process.stdout: + print(line, end='') + + # 等待命令完成 + stdout, stderr = process.communicate() + + if process.returncode != 0: + print(f"命令执行失败, 错误信息:\n{stderr}") + return False + else: + print("ONNX模型压缩完成!") + return True + + except FileNotFoundError: + print("错误: 未找到 onnxslim 命令, 请确保已安装 onnxslim") + print("安装方法: pip install onnx-simplifier") + return False + except Exception as e: + print(f"执行命令时发生错误: {e}") + return False + + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Config and model path +config_path = "config/z_image/z_image_control.yaml" +# model path +model_name = "models/Diffusion_Transformer/Z-Image-Turbo/" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] # H, W + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_image = "asset/pose.jpg" +control_context_scale = 0.75 + +# 使用更长的neg prompt如"模糊, 突变, 变形, 失真, 画面暗, 文本字幕, 画面固定, 连环画, 漫画, 线稿, 没有主体.", 可以增加稳定性 +# 在neg prompt中添加"安静, 固定"等词语可以增加动态性. +prompt = "一位年轻女子站在阳光明媚的海岸线上, 白裙在轻拂的海风中微微飘动.她拥有一头鲜艳的紫色长发, 在风中轻盈舞动, 发间系着一个精致的黑色蝴蝶结, 与身后柔和的蔚蓝天空形成鲜明对比.她面容清秀, 眉目精致, 透着一股甜美的青春气息；神情柔和, 略带羞涩, 目光静静地凝望着远方的地平线, 双手自然交叠于身前, 仿佛沉浸在思绪之中.在她身后, 是辽阔无垠、波光粼粼的大海, 阳光洒在海面上, 映出温暖的金色光晕." +# prompt = "一位身穿白色仙袍的仙人女子手持青色仙剑, 她抬头望着天空疾驰而来的雷霆, 面容严肃, 一袭紫色长发在风中飘扬, 仿佛与天地间的风雷共舞.她站立在一片古老的山巅, 背后是连绵起伏的群山和翻滚的乌云, 整个场景充满了神秘而壮丽的气息.天空中闪电划过, 照亮了她坚定的眼神和手中的仙剑, 彷佛预示着一场即将到来的大战.她的姿态优雅而坚定, 彷佛是天地间的守护者, 准备迎接任何挑战." +negative_prompt = " " +guidance_scale = 0.00 +seed = 43 +num_inference_steps = 9 +lora_weight = 0.55 +save_path = "samples/z-image-t2i-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = ZImageControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + +if False: + + class DummyControlTransformerWrapper(torch.nn.Module): + def __init__(self, model): + super().__init__() + self.model = model + self.control_context_scale = 0.75 + + def forward( + self, + latent_model_input, + timestep, + prompt_embeds, + control_context, + ): + model_out = self.model( + latent_model_input, + timestep, + prompt_embeds, + control_context=control_context, + control_context_scale=self.control_context_scale, + ) + return model_out + + # 使用 Torch 导出 transformer onnx 模型 + dummy_input = { + "latent_model_input": [torch.randn(16, 1, sample_size[0] // 8, sample_size[1] // 8, device="cpu", dtype=torch.float32)], + "timestep": torch.tensor([0.], device="cpu", dtype=torch.float32), + "prompt_embeds": [torch.randn(512, 2560, device="cpu", dtype=torch.float32)], # TODO: 这里需要支持最大长度 + "control_context": torch.randn(1, 16, 1, sample_size[0] // 8, sample_size[1] // 8, device="cpu", dtype=torch.float32), + # "control_context_scale": 0.75, + } + + # import pdb; pdb.set_trace() + transformer_warpper = DummyControlTransformerWrapper(transformer) + transformer_warpper.eval() + + transformer_path = "onnx-models/trans/" + transformer_onnx_path = os.path.join(transformer_path, "z_image_control_transformer.onnx") + if not os.path.exists(transformer_path): + os.makedirs(transformer_path, exist_ok=True) + + torch.onnx.export( + transformer_warpper.to(device="cpu", dtype=torch.float32), + tuple(dummy_input.values()), + transformer_onnx_path, + opset_version=17, + input_names=list(dummy_input.keys()), + output_names=["model_out"], + do_constant_folding=True, + export_params=True, + verbose=False + ) + trans_onnx = onnx.load(transformer_onnx_path) + + simp_onnx_data = "onnx-models/z_image_control_transformer.onnx" + onnx.save( + trans_onnx, + simp_onnx_data, + save_as_external_data=True, + all_tensors_to_one_file=True + ) + + logger.info("Transformer ONNX model exported, start to simplify.") + # 在 python 中执行终端指令: onnxslim vae.onnx vae_slim.onnx 实现模型简化 + success = run_onnxslim(simp_onnx_data, simp_onnx_data.replace(".onnx", "_slim.onnx")) + if success: + logger.info("Transformer ONNX model exported successfully.") + else: + sys.exit(1) + + exit() + """ + (Pdb) latent_model_input_list[0].shape + torch.Size([16, 1, 216, 124]) + (Pdb) timestep_model_input + tensor([0.], device='cuda:0') + (Pdb) prompt_embeds_model_input[0].shape + torch.Size([165, 2560]) + (Pdb) control_context.shape + torch.Size([1, 16, 1, 216, 124]) + (Pdb) control_context_scale + 0.75 + (Pdb) model_out_list[0].shape + torch.Size([16, 1, 216, 124]) + """ + + +# Get Vae +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + + +if False: + + class DummyVAEEncoderWrapper(torch.nn.Module): + def __init__(self, model): + super().__init__() + self.model = model + + def forward(self, x): + latent_dist = self.model.encode(x)[0].mode() + return latent_dist + + class DummyVAEDecoderWrapper(torch.nn.Module): + def __init__(self, model): + super().__init__() + self.model = model + + def forward(self, latents): + image = self.model.decode(latents, return_dict=False)[0] + return image + + def export_vae_onnx(vae, sample_size): + # 使用 Torch 导出 vae onnx 模型 + vae.eval() + if not os.path.exists("./onnx-models"): + os.makedirs("./onnx-models", exist_ok=True) + + ## 导出 VAE Decoder + vae_encoder_onnx_path = "./onnx-models/vae_encoder.onnx" + dummy_input = torch.randn(1, 3, sample_size[0], sample_size[1], device="cpu", dtype=torch.float32) + vae_encode_wrapper = DummyVAEEncoderWrapper(vae) + vae_encode_wrapper.eval() + torch.onnx.export(vae_encode_wrapper.to(torch.float32), dummy_input, vae_encoder_onnx_path, opset_version=17) + # import pdb; pdb.set_trace() + onnx.checker.check_model(vae_encoder_onnx_path) + logger.info("VAE-Encoder ONNX model exported, start to simplify.") + # 在 python 中执行终端指令: onnxslim vae.onnx vae_slim.onnx 实现模型简化 + success = run_onnxslim(vae_encoder_onnx_path, vae_encoder_onnx_path.replace(".onnx", "_slim.onnx")) + + ## 导出 VAE Decoder + vae_decoder_onnx_path = "./onnx-models/vae_decoder.onnx" + dummy_latent = torch.randn(1, vae.config.latent_channels, sample_size[0] // 8, sample_size[1] // 8, device="cpu", dtype=torch.float32) + vae_decode_wrapper = DummyVAEDecoderWrapper(vae) + vae_decode_wrapper.eval() + torch.onnx.export(vae_decode_wrapper.to(torch.float32), dummy_latent, input_names=["latent"], output_names=["image"], f=vae_decoder_onnx_path, opset_version=17) + onnx.checker.check_model(vae_decoder_onnx_path) + logger.info("VAE-Decoder ONNX model exported, start to simplify.") + success = run_onnxslim(vae_decoder_onnx_path, vae_decoder_onnx_path.replace(".onnx", "_slim.onnx")) + + if success: + logger.info("VAE ONNX model exported successfully.") + else: + sys.exit(1) + export_vae_onnx(vae, sample_size) + exit() + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = AutoTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +if False: + # 目前使用 llm_build 方法进行编译, Qwen3 架构 + text_encoder.eval() + text_encoder.config.use_cache = False + text_encoder.config.output_attentions = False + text_encoder.config.output_hidden_states = False + + # 创建包装器 + import torch.nn as nn + class Qwen3TextEncoderExporter(nn.Module): + def __init__(self, model): + super().__init__() + self.model = model + + def forward(self, input_ids, attention_mask=None): + # 返回 hidden_states[-2] + outputs = self.model( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + return_dict=True + ) + return outputs.hidden_states[-2] + + wrapped_text_encoder = Qwen3TextEncoderExporter(text_encoder) + wrapped_text_encoder.eval() + + # 导出 text_encoder 的 onnx 模型 + max_sequence_length = 512 + text_encoder_onnx_path = "./onnx-models/text_encoder.onnx" + """ + NOTE: 注意输入 onnx 的 mask 的 size 与 input_ids 的 size 一致. 前 N 个有效输入为 True, 后面的 padding 为 False. + 例如 input_ids 的 size 为 (1, 512), 实际有效输入长度为 20, 则 attention_mask 应该为: + attention_mask = [True, True, ..., True, False, False, ..., False] # 共 512 个元素, 前 20 个为 True, 后 492 个为 False + 这样可以确保 ONNX 模型在推理时正确处理 padding 部分, 避免无效计算. + """ + input_ids = torch.randint(0, tokenizer.vocab_size, (1, max_sequence_length), device="cpu", dtype=torch.long) + attention_mask = torch.ones((1, max_sequence_length), device="cpu", dtype=torch.long) + + # 测试前向传播 + with torch.no_grad(): + test_output = wrapped_text_encoder(input_ids, attention_mask) + print(f"测试输出形状: {test_output.shape}") + import pdb; pdb.set_trace() + + torch.onnx.export( + wrapped_text_encoder, + (input_ids, attention_mask), + text_encoder_onnx_path, + opset_version=17, + input_names=["input_ids", "attention_mask"], + output_names=["last_hidden_state"], + do_constant_folding=True, + export_params=True, + verbose=False + ) + onnx.checker.check_model(text_encoder_onnx_path) + logger.info("Text Encoder ONNX model exported successfully.") + + if not os.path.exists("./onnx-models"): + os.makedirs("./onnx-models", exist_ok=True) + + logger.info("Text Encoder ONNX model exported, start to simplify.") + # 在 python 中执行终端指令: onnxslim text_encoder.onnx text_encoder_slim.onnx 实现模型简化 + success = run_onnxslim(text_encoder_onnx_path, text_encoder_onnx_path.replace(".onnx", "_slim.onnx")) + if success: + logger.info("Text Encoder ONNX model exported successfully.") + else: + sys.exit(1) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = ZImageControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +# if ulysses_degree > 1 or ring_degree > 1: +# from functools import partial +# transformer.enable_multi_gpus_inference() +# if fsdp_dit: +# shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) +# pipeline.transformer = shard_fn(pipeline.transformer) +# print("Add FSDP DIT") +# if fsdp_text_encoder: +# shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers, ignored_modules=[text_encoder.language_model.embed_tokens], transformer_layer_cls_to_wrap=["MistralDecoderLayer", "PixtralTransformer"]) +# text_encoder = shard_fn(text_encoder) +# print("Add FSDP TEXT ENCODER") + +# if compile_dit: +# for i in range(len(pipeline.transformer.transformer_blocks)): +# pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) +# print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +# if lora_path is not None: +# pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] # torch.Size([1, 3, sample_size[0], sample_size[1]]) + + sample = pipeline( + prompt = prompt, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + control_image = control_image, + num_inference_steps = num_inference_steps, + control_context_scale = control_context_scale, + ).images + +# if lora_path is not None: +# pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +# if ulysses_degree * ring_degree > 1: +# import torch.distributed as dist +# if dist.get_rank() == 0: +# save_results() +# else: +# save_results() + +save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/z_image_fun/gradio_app.py b/VideoX-Fun/examples/z_image_fun/gradio_app.py new file mode 100644 index 0000000000000000000000000000000000000000..4cef692f6945e520e392d5bf1f339d963daf81e5 --- /dev/null +++ b/VideoX-Fun/examples/z_image_fun/gradio_app.py @@ -0,0 +1,243 @@ +import os +import sys +import random +from pathlib import Path +from typing import Optional, Tuple + +import gradio as gr +import torch +from PIL import Image +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from safetensors.torch import load_file + +# 将仓库根目录加入 sys.path，方便直接运行本脚本 +CURRENT_FILE = Path(__file__).resolve() +PROJECT_ROOTS = [CURRENT_FILE.parent, CURRENT_FILE.parent.parent, CURRENT_FILE.parent.parent.parent] +for root in PROJECT_ROOTS: + root_str = str(root) + if root_str not in sys.path: + sys.path.insert(0, root_str) +REPO_ROOT = PROJECT_ROOTS[-1] + +from videox_fun.models import ( # noqa: E402 + AutoencoderKL, + AutoTokenizer, + Qwen3ForCausalLM, + ZImageControlTransformer2DModel, +) +from videox_fun.pipeline import ZImageControlPipeline # noqa: E402 +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler # noqa: E402 +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler # noqa: E402 + +# 基础配置，可按需修改 +CONFIG_PATH = REPO_ROOT / "config" / "z_image" / "z_image_control.yaml" +MODEL_NAME = REPO_ROOT / "models" / "Diffusion_Transformer" / "Z-Image-Turbo" +TRANSFORMER_CKPT = REPO_ROOT / "models" / "Personalized_Model" / "Z-Image-Turbo-Fun-Controlnet-Union.safetensors" +DEFAULT_POSE_PATH = REPO_ROOT / "asset" / "pose_1024x1024.png" +SAMPLERS = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +} +DEFAULT_SAMPLER = "Flow" + +PIPELINE: Optional[ZImageControlPipeline] = None +PIPELINE_DEVICE: Optional[torch.device] = None +PIPELINE_DTYPE: Optional[torch.dtype] = None + + +def _pick_dtype() -> torch.dtype: + if torch.cuda.is_available(): + if torch.cuda.is_bf16_supported(): + return torch.bfloat16 + return torch.float16 + return torch.float32 + + +def _load_pipeline() -> Tuple[ZImageControlPipeline, torch.device, torch.dtype]: + global PIPELINE, PIPELINE_DEVICE, PIPELINE_DTYPE + if PIPELINE is not None: + return PIPELINE, PIPELINE_DEVICE, PIPELINE_DTYPE + + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + dtype = _pick_dtype() + + config = OmegaConf.load(CONFIG_PATH) + + transformer = ZImageControlTransformer2DModel.from_pretrained( + MODEL_NAME, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=dtype, + transformer_additional_kwargs=OmegaConf.to_container(config["transformer_additional_kwargs"]), + ).to(dtype) + + if TRANSFORMER_CKPT.exists(): + if TRANSFORMER_CKPT.suffix == ".safetensors": + state_dict = load_file(TRANSFORMER_CKPT) + else: + state_dict = torch.load(TRANSFORMER_CKPT, map_location="cpu") + if "state_dict" in state_dict: + state_dict = state_dict["state_dict"] + missing, unexpected = transformer.load_state_dict(state_dict, strict=False) + print(f"[load] transformer ckpt loaded, missing={len(missing)}, unexpected={len(unexpected)}") + else: + print(f"[warn] transformer checkpoint not found at {TRANSFORMER_CKPT}, using base weights") + + vae = AutoencoderKL.from_pretrained(MODEL_NAME, subfolder="vae").to(dtype) + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, subfolder="tokenizer") + text_encoder = Qwen3ForCausalLM.from_pretrained( + MODEL_NAME, + subfolder="text_encoder", + torch_dtype=dtype, + low_cpu_mem_usage=True, + ) + + scheduler_cls = SAMPLERS.get(DEFAULT_SAMPLER, FlowMatchEulerDiscreteScheduler) + scheduler = scheduler_cls.from_pretrained(MODEL_NAME, subfolder="scheduler") + + pipe = ZImageControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + + if torch.cuda.is_available(): + pipe.to(device=device) + else: + # CPU 推理较慢，这里保留在 CPU 上避免超出显存 + pipe.to(device) + + PIPELINE = pipe + PIPELINE_DEVICE = device + PIPELINE_DTYPE = dtype + return pipe, device, dtype + + +def _ensure_pose_image(pose_image: Optional[Image.Image]) -> Image.Image: + if pose_image is None: + return Image.open(DEFAULT_POSE_PATH).convert("RGB") + if pose_image.mode != "RGB": + pose_image = pose_image.convert("RGB") + return pose_image + + +def _align_size(value: int) -> int: + # pipeline 要求可被 16 整除 + return max(256, (value // 16) * 16) + + +def infer( + prompt: str, + negative_prompt: str, + pose_image: Optional[Image.Image], + height: int, + width: int, + steps: int, + guidance_scale: float, + control_strength: float, + seed: int, +): + pipe, device, _ = _load_pipeline() + + if not prompt.strip(): + raise gr.Error("提示词不能为空") + + pose_image = _ensure_pose_image(pose_image) + height = _align_size(height) + width = _align_size(width) + + if seed is None or seed < 0: + seed = random.randint(1, 2**31 - 1) + generator = torch.Generator(device=device).manual_seed(seed) + + with torch.inference_mode(): + result = pipe( + prompt=prompt, + negative_prompt=negative_prompt, + height=height, + width=width, + num_inference_steps=int(steps), + guidance_scale=float(guidance_scale), + generator=generator, + control_image=pose_image, + control_context_scale=float(control_strength), + max_sequence_length=128, + ).images[0] + + return result, seed + + +def build_ui(): + css = """ + .compact-slider {padding-top: 4px; padding-bottom: 4px;} + """ + + with gr.Blocks(title="Z-Image Turbo 文生图 (Pose)", css=css) as demo: + gr.Markdown("## Z-Image Turbo 文生图 (含 Pose 控制)") + gr.Markdown( + "上传姿态图，输入提示词即可生成图像。右侧为缩略图预览，可放大/下载原分辨率。", + ) + + with gr.Row(): + with gr.Column(scale=1, min_width=320): + prompt = gr.Textbox( + label="提示词", + placeholder="描述你想生成的画面", + lines=4, + value="1 girl, on the beach, summer, full body, highly detailed", + ) + negative_prompt = gr.Textbox( + label="反向提示词", + placeholder="不希望出现的元素，例如 '低质量, 模糊'", + lines=3, + value="lowres, blurry, text, watermark", + ) + steps = gr.Slider(minimum=4, maximum=30, step=1, value=9, label="采样步数", elem_classes=["compact-slider"]) + guidance_scale = gr.Slider(minimum=0.0, maximum=6.0, step=0.1, value=0.0, label="CFG 指数 (>=1 生效)", elem_classes=["compact-slider"]) + control_strength = gr.Slider(minimum=0.0, maximum=2.0, step=0.05, value=0.75, label="Pose 强度", elem_classes=["compact-slider"]) + height = gr.Slider(minimum=512, maximum=1792, step=16, value=1024, label="高度 (16 的倍数)", elem_classes=["compact-slider"]) + width = gr.Slider(minimum=512, maximum=1792, step=16, value=1024, label="宽度 (16 的倍数)", elem_classes=["compact-slider"]) + seed = gr.Number(value=-1, label="随机种子 (-1 表示随机)", precision=0) + run_btn = gr.Button("生成", variant="primary") + + with gr.Column(scale=2.4): + with gr.Row(): + with gr.Column(scale=0.8, min_width=200): + pose_image = gr.Image( + label="姿态图上传 (RGB)", + type="pil", + height=320, + width=240, + show_download_button=True, + ) + with gr.Column(scale=2.6): + result_img = gr.Image( + label="生成结果 (缩略图)", + type="pil", + height=520, + show_download_button=True, + show_fullscreen_button=True, + ) + used_seed = gr.Number(label="实际种子", precision=0) + + run_btn.click( + infer, + inputs=[prompt, negative_prompt, pose_image, height, width, steps, guidance_scale, control_strength, seed], + outputs=[result_img, used_seed], + ) + + return demo + + +def main(): + _load_pipeline() + demo = build_ui() + demo.queue().launch(server_name="0.0.0.0", server_port=7860, inbrowser=False, share=False) + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/examples/z_image_fun/launcher.py b/VideoX-Fun/examples/z_image_fun/launcher.py new file mode 100644 index 0000000000000000000000000000000000000000..8caefec2e5351695a20b88326b6b73100d641b12 --- /dev/null +++ b/VideoX-Fun/examples/z_image_fun/launcher.py @@ -0,0 +1,977 @@ +import os +import sys +import numpy as np +import torch +import onnxruntime as ort +from contextlib import contextmanager + +from omegaconf import OmegaConf +from PIL import Image +from loguru import logger + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None +repo_root = project_roots[-1] +onnx_models_dir = os.path.join(repo_root, "onnx-models") +vae_encoder_onnx_path = os.path.join(onnx_models_dir, "vae_encoder_simp_slim.onnx") +vae_decoder_onnx_path = os.path.join(onnx_models_dir, "vae_decoder_simp_slim.onnx") +# vae_encoder_onnx_path = "" +# vae_decoder_onnx_path = "" +use_transformer_onnx = True +use_controlnet = False # 控制是否启用 ControlNet，默认启用 +controlnet_onnx_path = os.path.join(onnx_models_dir, "z_image_controlnet_simp_slim.onnx") +transformer_body_onnx_path = os.path.join(onnx_models_dir, "z_image_transformer_body_simp_slim.onnx") +onnx_export_dtype = torch.float16 +_printed_onnx_debug = False +_body_input_cache = {} + + +def _select_onnx_providers(): + avail = ort.get_available_providers() + if "CUDAExecutionProvider" in avail: + return ["CUDAExecutionProvider", "CPUExecutionProvider"] + if "AzureExecutionProvider" in avail: + return ["AzureExecutionProvider", "CPUExecutionProvider"] + return ["CPUExecutionProvider"] + + +onnx_providers = _select_onnx_providers() + +from diffusers import FlowMatchEulerDiscreteScheduler +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + Qwen3ForCausalLM, ZImageControlTransformer2DModel) +from typing import List, Optional, Union +from diffusers.utils.torch_utils import randn_tensor +from videox_fun.utils.utils import get_image_latent + +torch.set_grad_enabled(False) + +_ort_sessions = {} + + +def _ort_type_to_torch(type_str: str): + if type_str is None: + return None + if "float16" in type_str: + return torch.float16 + if "bfloat16" in type_str: + return torch.bfloat16 + if "float" in type_str: + return torch.float32 + return None + + +def ort_inference(onnx_model_path: str, inputs: dict, providers=None): + providers = providers or ["CPUExecutionProvider"] + cache_key = (onnx_model_path, tuple(providers)) + if cache_key not in _ort_sessions: + if not os.path.exists(onnx_model_path): + raise FileNotFoundError(f"ONNX model not found: {onnx_model_path}") + _ort_sessions[cache_key] = ort.InferenceSession(onnx_model_path, providers=providers) + session = _ort_sessions[cache_key] + + # 按 ONNX 输入类型做 dtype 对齐, bfloat16 等不被 numpy 支持的类型会被安全转换 + input_type_map = {i.name: _ort_type_to_torch(getattr(i, "type", None)) for i in session.get_inputs()} + inputs_onnx = {} + for k, v in inputs.items(): + target_torch_dtype = input_type_map.get(k) + if isinstance(v, np.ndarray): + if target_torch_dtype is not None: + np_dtype = np.float16 if target_torch_dtype == torch.float16 else np.float32 + if v.dtype != np_dtype: + v = v.astype(np_dtype) + inputs_onnx[k] = v + elif torch.is_tensor(v): + if target_torch_dtype is not None and v.dtype != target_torch_dtype: + v = v.to(dtype=target_torch_dtype) + elif v.dtype == torch.bfloat16: + v = v.to(dtype=torch.float32) + inputs_onnx[k] = v.detach().to("cpu").numpy() + else: + raise TypeError(f"Unsupported input type for key {k}: {type(v)}") + return session.run(None, inputs_onnx) + + +def _infer_module_device(module: torch.nn.Module) -> torch.device: + param = next(module.parameters(), None) + if param is not None: + return param.device + buffer = next(module.buffers(), None) + if buffer is not None: + return buffer.device + return torch.device("cpu") + + +@contextmanager +def module_to_device(module: torch.nn.Module, target_device: torch.device): + if module is None: + yield module + return + original_device = _infer_module_device(module) + target_device = target_device or original_device + needs_move = original_device != target_device + moved_to_cuda = needs_move and target_device.type == "cuda" + if needs_move: + module.to(target_device) + try: + yield module + finally: + if needs_move: + module.to(original_device) + if moved_to_cuda and torch.cuda.is_available(): + cache_device = target_device.index + if cache_device is None: + cache_device = torch.cuda.current_device() + with torch.cuda.device(cache_device): + torch.cuda.empty_cache() + + +# Config and model path +config_path = "config/z_image/z_image_control.yaml" +model_name = "models/Diffusion_Transformer/Z-Image-Turbo/" + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +# control_image = "asset/pose.jpg" +control_image = "asset/pose_1024x1024.png" +control_context_scale = 0.75 + +device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + +# Get tokenizer and text_encoder +tokenizer = AutoTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) +text_encoder.eval() + +def _encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + max_sequence_length: int = 512, +) -> List[torch.FloatTensor]: + device = device or torch.device("cpu") + + if prompt_embeds is not None: + return prompt_embeds + + if isinstance(prompt, str): + prompt = [prompt] + + for i, prompt_item in enumerate(prompt): + messages = [ + {"role": "user", "content": prompt_item}, + ] + prompt_item = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + prompt[i] = prompt_item + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + + with module_to_device(text_encoder, device): + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] # 固定为 torch.Size([1, 512, 2560] + + embeddings_list = [] + + for i in range(len(prompt_embeds)): + # embeddings_list.append(prompt_embeds[i][prompt_masks[i]]) + embeddings_list.append(prompt_embeds[i]) + + return embeddings_list + + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + max_sequence_length: int = 512, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + prompt_embeds = _encode_prompt( + prompt=prompt, + device=device, + prompt_embeds=prompt_embeds, + max_sequence_length=max_sequence_length, + ) + + if do_classifier_free_guidance: + if negative_prompt is None: + negative_prompt = ["" for _ in prompt] + else: + negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + assert len(prompt) == len(negative_prompt) + negative_prompt_embeds = _encode_prompt( + prompt=negative_prompt, + device=device, + prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + ) + else: + negative_prompt_embeds = [] + return prompt_embeds, negative_prompt_embeds + + +def _stack_prompt_embeddings(prompt_embeds_input): + if isinstance(prompt_embeds_input, list): + return torch.stack(prompt_embeds_input, dim=0) + return prompt_embeds_input + + +def _onnx_shape_compatible(model_path: str, providers, latent_shape, prompt_shape, verbose=False): + try: + sess = ort.InferenceSession(model_path, providers=providers) + inputs = {i.name: i for i in sess.get_inputs()} + ok = True + mismatch_msgs = [] + if "latent_model_input" in inputs: + shape = inputs["latent_model_input"].shape + exp_h, exp_w = shape[3], shape[4] + if isinstance(exp_h, int) and exp_h != latent_shape[3]: + ok = False + mismatch_msgs.append(f"latent_h expected {exp_h}, got {latent_shape[3]}") + if isinstance(exp_w, int) and exp_w != latent_shape[4]: + ok = False + mismatch_msgs.append(f"latent_w expected {exp_w}, got {latent_shape[4]}") + if "prompt_embeds" in inputs: + pshape = inputs["prompt_embeds"].shape + exp_seq = pshape[1] + if isinstance(exp_seq, int) and exp_seq != prompt_shape[1]: + ok = False + mismatch_msgs.append(f"seq_len expected {exp_seq}, got {prompt_shape[1]}") + if verbose and (not ok or mismatch_msgs): + print(f"[DEBUG] ONNX shape check for {model_path}") + print(f" providers={providers}") + print(f" model latent shape={inputs.get('latent_model_input').shape if 'latent_model_input' in inputs else 'n/a'}") + print(f" model prompt shape={inputs.get('prompt_embeds').shape if 'prompt_embeds' in inputs else 'n/a'}") + print(f" runtime latent shape={latent_shape}, prompt shape={prompt_shape}") + if mismatch_msgs: + print(f" mismatch: {', '.join(mismatch_msgs)}") + return ok + except Exception as exc: + print(f"ONNX shape check failed for {model_path}: {exc}") + return True + + +def prepare_latents( + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, +): + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + + shape = (batch_size, num_channels_latents, height, width) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + return latents + + +def _make_zero_control_hints(model_path: str, providers, device, dtype): + cache_key = (model_path, tuple(providers)) + if cache_key not in _body_input_cache: + sess = ort.InferenceSession(model_path, providers=providers) + inputs = {i.name: i for i in sess.get_inputs()} + if "control_hints" not in inputs: + raise RuntimeError("Body ONNX 未找到 control_hints 输入，无法构造零填充。") + _body_input_cache[cache_key] = inputs["control_hints"].shape + shape = _body_input_cache[cache_key] + # 将 None 或字符串符号维度用 1 填充（静态模型应全是 int） + resolved = [] + for d in shape: + if isinstance(d, int): + resolved.append(d) + else: + resolved.append(1) + return torch.zeros(resolved, device=device, dtype=dtype) + + + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + + +# model path + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [512, 512] # H, W + + +# 使用更长的neg prompt如"模糊, 突变, 变形, 失真, 画面暗, 文本字幕, 画面固定, 连环画, 漫画, 线稿, 没有主体.", 可以增加稳定性 +# 在neg prompt中添加"安静, 固定"等词语可以增加动态性. +# prompt = "一位年轻女子站在阳光明媚的海岸线上, 白裙在轻拂的海风中微微飘动.她拥有一头鲜艳的紫色长发, 在风中轻盈舞动, 发间系着一个精致的黑色蝴蝶结, 与身后柔和的蔚蓝天空形成鲜明对比.她面容清秀, 眉目精致, 透着一股甜美的青春气息；神情柔和, 略带羞涩, 目光静静地凝望着远方的地平线, 双手自然交叠于身前, 仿佛沉浸在思绪之中.在她身后, 是辽阔无垠、波光粼粼的大海, 阳光洒在海面上, 映出温暖的金色光晕." +# prompt = "一位身穿白色仙袍的仙人女子手持青色仙剑, 她抬头望着天空疾驰而来的雷霆, 面容严肃, 一袭紫色长发在风中飘扬, 仿佛与天地间的风雷共舞.她站立在一片古老的山巅, 背后是连绵起伏的群山和翻滚的乌云, 整个场景充满了神秘而壮丽的气息.天空中闪电划过, 照亮了她坚定的眼神和手中的仙剑, 彷佛预示着一场即将到来的大战.她的姿态优雅而坚定, 彷佛是天地间的守护者, 准备迎接任何挑战." +prompt = "(masterpiece, best quality, ultra detailed, 8k, CG unity wallpaper),1 young beautiful girl, full body, official art, extremely detailed, highly detailed, 1 girl, aqua eyes, light smile, grey hair, hair flower,bracelet, choker, ribbon, JK, looking at viewer, on the beach, in summer," +negative_prompt = " " +guidance_scale = 0.00 +seed = 43 +num_inference_steps = 9 +lora_weight = 0.55 +save_path = "samples/z-image-t2i-control" + +config = OmegaConf.load(config_path) + +transformer = ZImageControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype).to(device) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + +# Get Scheduler +Chosen_Scheduler = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] + +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +# pipeline = ZImageControlPipeline( +# vae=vae, +# tokenizer=tokenizer, +# text_encoder=text_encoder, +# transformer=transformer, +# scheduler=scheduler, +# ) + +# if ulysses_degree > 1 or ring_degree > 1: +# from functools import partial +# transformer.enable_multi_gpus_inference() +# if fsdp_dit: +# shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) +# pipeline.transformer = shard_fn(pipeline.transformer) +# print("Add FSDP DIT") +# if fsdp_text_encoder: +# shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers, ignored_modules=[text_encoder.language_model.embed_tokens], transformer_layer_cls_to_wrap=["MistralDecoderLayer", "PixtralTransformer"]) +# text_encoder = shard_fn(text_encoder) +# print("Add FSDP TEXT ENCODER") + +# if compile_dit: +# for i in range(len(pipeline.transformer.transformer_blocks)): +# pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) +# print("Add Compile") + +# if GPU_memory_mode == "sequential_cpu_offload": +# pipeline.enable_sequential_cpu_offload(device=device) +# elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": +# convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) +# convert_weight_dtype_wrapper(transformer, weight_dtype) +# pipeline.enable_model_cpu_offload(device=device) +# elif GPU_memory_mode == "model_cpu_offload": +# pipeline.enable_model_cpu_offload(device=device) +# elif GPU_memory_mode == "model_full_load_and_qfloat8": +# convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) +# convert_weight_dtype_wrapper(transformer, weight_dtype) +# pipeline.to(device=device) +# else: +# pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +# if lora_path is not None: +# pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +# import pdb; pdb.set_trace() +if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] # torch.Size([1, 3, sample_size[0], sample_size[1]]) + +print(control_image.shape) + + +height, width = sample_size + +vae_scale_factor = 8 +vae_scale = vae_scale_factor * 2 + +if height % vae_scale != 0: + raise ValueError( + f"Height must be divisible by {vae_scale} (got {height}). " + f"Please adjust the height to a multiple of {vae_scale}." + ) +if width % vae_scale != 0: + raise ValueError( + f"Width must be divisible by {vae_scale} (got {width}). " + f"Please adjust the width to a multiple of {vae_scale}." + ) + +_guidance_scale = guidance_scale = 0.0 +_joint_attention_kwargs = joint_attention_kwargs = None +_interrupt = False +_cfg_normalization = cfg_normalization = False +_cfg_truncation = cfg_truncation = 1.0 + + +# 2. Define call parameters +prompt_embeds = None +if prompt is not None and isinstance(prompt, str): + batch_size = 1 +elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) +else: + batch_size = len(prompt_embeds) + +batch_size = 1 + +weight_dtype = text_encoder.dtype # text_encoder.dtype +num_channels_latents = 16 # transformer.in_channels +inpaint_latent = None # TODO: 原始代码中未定义该变量 + +from diffusers.image_processor import VaeImageProcessor +image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + +# Get Vae +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) +vae.eval() +vae_config_shift_factor = getattr(vae.config, "shift_factor", 0.0) +vae_config_scaling_factor = getattr(vae.config, "scaling_factor", 1.0) + +if control_image is not None: + control_image = image_processor.preprocess(control_image, height=height, width=width) + control_image = control_image.to(dtype=weight_dtype, device=device) + if os.path.exists(vae_encoder_onnx_path): + control_latents = ort_inference( + vae_encoder_onnx_path, + {"pixel_values": control_image}, + )[0] + control_latents = torch.from_numpy(control_latents).to(device=device, dtype=weight_dtype) + else: + with module_to_device(vae, device): + control_latents = vae.encode(control_image)[0].mode() + control_latents = (control_latents - vae_config_shift_factor) * vae_config_scaling_factor +else: + control_latents = torch.zeros_like(inpaint_latent) + +control_context = control_latents.unsqueeze(2) +control_context_scale_tensor = torch.tensor([control_context_scale], device=device, dtype=torch.float32) + +if not use_controlnet: + control_context = torch.zeros_like(control_context) + control_context_scale_tensor = torch.zeros_like(control_context_scale_tensor) + +onnx_transformer_enabled = use_transformer_onnx and os.path.exists(controlnet_onnx_path) and os.path.exists(transformer_body_onnx_path) +if use_transformer_onnx and not onnx_transformer_enabled: + print(f"ONNX transformer requested but missing files, fallback to torch. controlnet={controlnet_onnx_path}, body={transformer_body_onnx_path}") +else: + print(f"[DEBUG] ONNX providers: {onnx_providers}") + print(f"[DEBUG] controlnet onnx path: {controlnet_onnx_path}") + print(f"[DEBUG] transformer body onnx path: {transformer_body_onnx_path}") + + +do_classifier_free_guidance = False +negative_prompt_embeds = None +max_sequence_length = 128 # 512 + +prompt_embeds, negative_prompt_embeds = encode_prompt( + prompt=prompt, + negative_prompt=negative_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + device=device, + max_sequence_length=max_sequence_length, +) + +num_images_per_prompt = 1 +latents = None + +""" +(Pdb) latents[0, 0, 100:105, 100] +tensor([-0.9203, 1.3958, 0.8130, -0.5280, -1.9788], device='cuda:0') +""" + +# 4. Prepare latent variables +latents = prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + torch.float32, + device, + generator, + latents, +) # torch.Size([1, 16, 216, 124]) +print(latents.shape) +# 以上代码验证正确 + +# Repeat prompt_embeds for num_images_per_prompt +if num_images_per_prompt > 1: + prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)] + if do_classifier_free_guidance and negative_prompt_embeds: + negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)] + +actual_batch_size = batch_size * num_images_per_prompt +image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2) + + +# Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +# 5. Prepare timesteps +mu = calculate_shift( + image_seq_len, + scheduler.config.get("base_image_seq_len", 256), + scheduler.config.get("max_image_seq_len", 4096), + scheduler.config.get("base_shift", 0.5), + scheduler.config.get("max_shift", 1.15), +) +scheduler.sigma_min = 0.0 +scheduler_kwargs = {"mu": mu} + + +import inspect +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +sigmas = None +timesteps, num_inference_steps = retrieve_timesteps( + scheduler, + num_inference_steps, + device, + sigmas=sigmas, + **scheduler_kwargs, +) +num_warmup_steps = max(len(timesteps) - num_inference_steps * scheduler.order, 0) +""" +(Pdb) timesteps +tensor([1000.0000, 954.5454, 900.0000, 833.3333, 750.0000, 642.8571, + 500.0000, 300.0000, 0.0000], device='cuda:0') +""" +_num_timesteps = len(timesteps) # 9 +# 以上代码验证正确 +# import pdb; pdb.set_trace() + +callback_on_step_end = None +callback_on_step_end_tensor_inputs = ['latents'] + +# 6. Denoising loop +# with progress_bar(total=num_inference_steps) as progress_bar: +for i, t in enumerate(timesteps): + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latents.shape[0]) + timestep = (1000 - timestep) / 1000 + # Normalized time for time-aware config (0 at start, 1 at end) + t_norm = timestep[0].item() + + # Handle cfg truncation + current_guidance_scale = guidance_scale + if ( + do_classifier_free_guidance + and _cfg_truncation is not None + and float(_cfg_truncation) <= 1 + ): + if t_norm > _cfg_truncation: + current_guidance_scale = 0.0 + + # Run CFG only if configured AND scale is non-zero + apply_cfg = do_classifier_free_guidance and current_guidance_scale > 0 + + if apply_cfg: + latents_typed = latents.to(transformer.dtype) + latent_model_input = latents_typed.repeat(2, 1, 1, 1) + prompt_embeds_model_input = [p.to(transformer.dtype) for p in (prompt_embeds + negative_prompt_embeds)] + timestep_model_input = timestep.repeat(2) + else: + latent_model_input = latents.to(transformer.dtype) + prompt_embeds_model_input = [p.to(transformer.dtype) for p in prompt_embeds] + timestep_model_input = timestep + + latent_model_input = latent_model_input.unsqueeze(2) + prompt_embeds_tensor = _stack_prompt_embeddings(prompt_embeds_model_input) + + if not _printed_onnx_debug: + print(f"[DEBUG] runtime latent shape: {latent_model_input.shape}, dtype={latent_model_input.dtype}") + print(f"[DEBUG] runtime prompt shape: {prompt_embeds_tensor.shape}, dtype={prompt_embeds_tensor.dtype}") + print(f"[DEBUG] runtime control_context shape: {control_context.shape}, dtype={control_context.dtype}") + + if onnx_transformer_enabled: + onnx_latent = latent_model_input.to(dtype=onnx_export_dtype) + onnx_prompt = prompt_embeds_tensor.to(dtype=onnx_export_dtype) + onnx_control = control_context.to(dtype=onnx_export_dtype) + if apply_cfg: + onnx_control = onnx_control.repeat(2, 1, 1, 1, 1) + + if not _onnx_shape_compatible(controlnet_onnx_path, onnx_providers, onnx_latent.shape, onnx_prompt.shape, verbose=True): + print(f"ONNX controlnet 输入尺寸与当前推理不匹配，回退到 torch。模型期望与当前 latent/prompt 尺寸不同。") + onnx_transformer_enabled = False + if onnx_transformer_enabled and not _onnx_shape_compatible(transformer_body_onnx_path, onnx_providers, onnx_latent.shape, onnx_prompt.shape, verbose=True): + print(f"ONNX transformer body 输入尺寸与当前推理不匹配，回退到 torch。模型期望与当前 latent/prompt 尺寸不同。") + + if onnx_transformer_enabled: + + control_inputs = { + "latent_model_input": onnx_latent, + "timestep": timestep_model_input.to(dtype=torch.float32), + "prompt_embeds": onnx_prompt, + "control_context": onnx_control, + } + if use_controlnet: + control_hints = ort_inference(controlnet_onnx_path, control_inputs, providers=onnx_providers)[0] + control_hints_tensor = torch.from_numpy(control_hints).to(device=device, dtype=onnx_export_dtype) + else: + control_hints_tensor = _make_zero_control_hints(transformer_body_onnx_path, onnx_providers, device, onnx_export_dtype) + + body_inputs = { + "latent_model_input": onnx_latent, + "timestep": timestep_model_input.to(dtype=torch.float32), + "prompt_embeds": onnx_prompt, + "control_hints": control_hints_tensor, + "control_context_scale": torch.zeros_like(control_context_scale_tensor).to(dtype=torch.float32) if not use_controlnet else control_context_scale_tensor.to(dtype=torch.float32), + } + model_out = ort_inference(transformer_body_onnx_path, body_inputs, providers=onnx_providers)[0] + model_out_tensor = torch.from_numpy(model_out).to(device=device, dtype=onnx_export_dtype) + model_out_list = list(model_out_tensor) + else: + latent_model_input_list = [t.to(transformer.dtype) for t in latent_model_input.unbind(dim=0)] + control_context_input = control_context.repeat(2, 1, 1, 1, 1) if apply_cfg else control_context + if not use_controlnet: + control_context_input = torch.zeros_like(control_context_input) + control_context_scale_zero = torch.zeros_like(control_context_scale_tensor) + else: + control_context_scale_zero = control_context_scale_tensor + + control_context_input = control_context_input.to(dtype=transformer.dtype) + control_context_scale_zero = control_context_scale_zero.to(dtype=transformer.dtype) + + # timestep 也保持与权重一致的 dtype，避免 mat1/mat2 dtype 冲突 + timestep_model_input = timestep_model_input.to(dtype=transformer.dtype) + + # import pdb; pdb.set_trace() + if device.type == "cuda": + with torch.autocast(device_type="cuda", dtype=transformer.dtype): + model_out_list = transformer( + latent_model_input_list, + timestep_model_input, + prompt_embeds_model_input, + control_context=control_context_input, + control_context_scale=control_context_scale_zero, + )[0] + else: + model_out_list = transformer( + latent_model_input_list, + timestep_model_input, + prompt_embeds_model_input, + control_context=control_context_input, + control_context_scale=control_context_scale_zero, + )[0] + + if not _printed_onnx_debug: + _printed_onnx_debug = True + + if apply_cfg: + # Perform CFG + pos_out = model_out_list[:actual_batch_size] + neg_out = model_out_list[actual_batch_size:] + + noise_pred = [] + for j in range(actual_batch_size): + pos = pos_out[j].float() + neg = neg_out[j].float() + + pred = pos + current_guidance_scale * (pos - neg) + + # Renormalization + if _cfg_normalization and float(_cfg_normalization) > 0.0: + ori_pos_norm = torch.linalg.vector_norm(pos) + new_pos_norm = torch.linalg.vector_norm(pred) + max_new_norm = ori_pos_norm * float(_cfg_normalization) + if new_pos_norm > max_new_norm: + pred = pred * (max_new_norm / new_pos_norm) + + noise_pred.append(pred) + + noise_pred = torch.stack(noise_pred, dim=0) + else: + noise_pred = torch.stack([t.float() for t in model_out_list], dim=0) + + noise_pred = noise_pred.squeeze(2) + noise_pred = -noise_pred + + # compute the previous noisy sample x_t -> x_t-1 + latents = scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0] + assert latents.dtype == torch.float32 + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + # if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % scheduler.order == 0): + # progress_bar.update() + +# 显存优化, 80G -> 20G + +""" +# 原始数据 (vae onnx 版本) +(Pdb) latents.shape +torch.Size([1, 16, 216, 124]) +(Pdb) latents[0, 0, 100, 100:105] +tensor([ 0.8545, 1.0117, 0.7908, -0.7002, 0.3965], device='cuda:0') +(Pdb) + +# 当前数据 launcher.py +(Pdb) latents.shape +torch.Size([1, 16, 216, 124]) +(Pdb) latents[0, 0, 100, 100:105] +tensor([ 0.2899, 0.7049, 0.4407, -1.2531, -0.0161], device='cuda:2') +(Pdb) + +# 对齐 embedding 后的结果(torch版本) +(Pdb) latents.shape +torch.Size([1, 16, 216, 124]) +(Pdb) latents[0, 0, 100, 100:105] +tensor([ 0.8763, 1.0138, 0.8031, -0.6851, 0.3751], device='cuda:2') # cuda:0 结果相同 +(Pdb) + +# 修改为 onnx 版本 +(Pdb) latents[0, 0, 100, 100:105] +tensor([ 0.8545, 1.0117, 0.7908, -0.7002, 0.3965], device='cuda:0') + +# 以上结果对齐 +""" + +output_type = "pil" +if output_type == "latent": + image = latents + +else: + latents = latents.to(vae.dtype) + latents = (latents / vae.config.scaling_factor) + vae.config.shift_factor # torch.Size([1, 16, 216, 124]) + + if os.path.exists(vae_decoder_onnx_path): + image = ort_inference( + vae_decoder_onnx_path, + {"latents": latents}, + )[0] + image = torch.from_numpy(image).to(device=device, dtype=vae.dtype) + else: + with module_to_device(vae, device): + image = vae.decode(latents, return_dict=False)[0] # torch.Size([1, 3, 1728, 992]) + """ + (Pdb) latents[0, 0, 100, 100:105] + tensor([ 2.4844, 2.9062, 2.2969, -1.8203, 1.2188], device='cuda:0', + dtype=torch.bfloat16) + (Pdb) image[0, 0, 100, 100:105] + tensor([0.3906, 0.3848, 0.3809, 0.3809, 0.3848], device='cuda:0', + dtype=torch.bfloat16) + """ + # image = ort_inference( + # "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-models/vae_decoder_slim.onnx", + # {"latent": latents}, + # )[0] + # image = torch.from_numpy(image).to(device=device, dtype=vae.dtype) + image = image_processor.postprocess(image, output_type=output_type) + +# Offload all models +# maybe_free_model_hooks() +# return_dict = True +# if not return_dict: +# return (image,) + +# return ZImagePipelineOutput(images=image) + +# with torch.no_grad(): +# if control_image is not None: +# control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] # torch.Size([1, 3, sample_size[0], sample_size[1]]) + +# sample = pipeline( +# prompt = prompt, +# negative_prompt = negative_prompt, +# height = sample_size[0], +# width = sample_size[1], +# generator = generator, +# guidance_scale = guidance_scale, +# control_image = control_image, +# num_inference_steps = num_inference_steps, +# control_context_scale = control_context_scale, +# ).images + +# if lora_path is not None: +# pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +sample = image + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +# if ulysses_degree * ring_degree > 1: +# import torch.distributed as dist +# if dist.get_rank() == 0: +# save_results() +# else: +# save_results() + +save_results() +logger.info(f"Saved image to {save_path}") \ No newline at end of file diff --git a/VideoX-Fun/examples/z_image_fun/launcher.py.save.v1 b/VideoX-Fun/examples/z_image_fun/launcher.py.save.v1 new file mode 100644 index 0000000000000000000000000000000000000000..5569297b1aa5fcd85bac94cab43daecc18694334 --- /dev/null +++ b/VideoX-Fun/examples/z_image_fun/launcher.py.save.v1 @@ -0,0 +1,671 @@ +import os +import sys +import numpy as np +import torch + +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from diffusers import FlowMatchEulerDiscreteScheduler +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + Qwen3ForCausalLM, ZImageControlTransformer2DModel) +from typing import List, Optional, Union +from diffusers.utils.torch_utils import randn_tensor +from videox_fun.utils.utils import get_image_latent + + +# Config and model path +config_path = "config/z_image/z_image_control.yaml" +model_name = "models/Diffusion_Transformer/Z-Image-Turbo/" + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_image = "asset/pose.jpg" +control_context_scale = 0.75 + +device = torch.device('cuda:2') if torch.cuda.is_available() else torch.device('cpu') + +# Get tokenizer and text_encoder +tokenizer = AutoTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +).to(device) + +def _encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + max_sequence_length: int = 512, +) -> List[torch.FloatTensor]: + # device = device or _execution_device + + if prompt_embeds is not None: + return prompt_embeds + + if isinstance(prompt, str): + prompt = [prompt] + + for i, prompt_item in enumerate(prompt): + messages = [ + {"role": "user", "content": prompt_item}, + ] + prompt_item = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + prompt[i] = prompt_item + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] # 固定为 torch.Size([1, 512, 2560] + + embeddings_list = [] + + for i in range(len(prompt_embeds)): # TODO: 对于 ONNX 模型不支持动态 shape + embeddings_list.append(prompt_embeds[i][prompt_masks[i]]) + # embeddings_list.append(prompt_embeds[i]) + + return embeddings_list + + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + max_sequence_length: int = 512, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + prompt_embeds = _encode_prompt( + prompt=prompt, + device=device, + prompt_embeds=prompt_embeds, + max_sequence_length=max_sequence_length, + ) + + if do_classifier_free_guidance: + if negative_prompt is None: + negative_prompt = ["" for _ in prompt] + else: + negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + assert len(prompt) == len(negative_prompt) + negative_prompt_embeds = _encode_prompt( + prompt=negative_prompt, + device=device, + prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + ) + else: + negative_prompt_embeds = [] + return prompt_embeds, negative_prompt_embeds + + +def prepare_latents( + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, +): + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + + shape = (batch_size, num_channels_latents, height, width) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + return latents + + + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + + +# model path + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] # H, W + + +# 使用更长的neg prompt如"模糊, 突变, 变形, 失真, 画面暗, 文本字幕, 画面固定, 连环画, 漫画, 线稿, 没有主体.", 可以增加稳定性 +# 在neg prompt中添加"安静, 固定"等词语可以增加动态性. +prompt = "一位年轻女子站在阳光明媚的海岸线上, 白裙在轻拂的海风中微微飘动.她拥有一头鲜艳的紫色长发, 在风中轻盈舞动, 发间系着一个精致的黑色蝴蝶结, 与身后柔和的蔚蓝天空形成鲜明对比.她面容清秀, 眉目精致, 透着一股甜美的青春气息；神情柔和, 略带羞涩, 目光静静地凝望着远方的地平线, 双手自然交叠于身前, 仿佛沉浸在思绪之中.在她身后, 是辽阔无垠、波光粼粼的大海, 阳光洒在海面上, 映出温暖的金色光晕." +# prompt = "一位身穿白色仙袍的仙人女子手持青色仙剑, 她抬头望着天空疾驰而来的雷霆, 面容严肃, 一袭紫色长发在风中飘扬, 仿佛与天地间的风雷共舞.她站立在一片古老的山巅, 背后是连绵起伏的群山和翻滚的乌云, 整个场景充满了神秘而壮丽的气息.天空中闪电划过, 照亮了她坚定的眼神和手中的仙剑, 彷佛预示着一场即将到来的大战.她的姿态优雅而坚定, 彷佛是天地间的守护者, 准备迎接任何挑战." +negative_prompt = " " +guidance_scale = 0.00 +seed = 43 +num_inference_steps = 9 +lora_weight = 0.55 +save_path = "samples/z-image-t2i-control" + +config = OmegaConf.load(config_path) + +transformer = ZImageControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype).to(device) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + +# Get Scheduler +Chosen_Scheduler = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] + +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +# pipeline = ZImageControlPipeline( +# vae=vae, +# tokenizer=tokenizer, +# text_encoder=text_encoder, +# transformer=transformer, +# scheduler=scheduler, +# ) + +# if ulysses_degree > 1 or ring_degree > 1: +# from functools import partial +# transformer.enable_multi_gpus_inference() +# if fsdp_dit: +# shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) +# pipeline.transformer = shard_fn(pipeline.transformer) +# print("Add FSDP DIT") +# if fsdp_text_encoder: +# shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers, ignored_modules=[text_encoder.language_model.embed_tokens], transformer_layer_cls_to_wrap=["MistralDecoderLayer", "PixtralTransformer"]) +# text_encoder = shard_fn(text_encoder) +# print("Add FSDP TEXT ENCODER") + +# if compile_dit: +# for i in range(len(pipeline.transformer.transformer_blocks)): +# pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) +# print("Add Compile") + +# if GPU_memory_mode == "sequential_cpu_offload": +# pipeline.enable_sequential_cpu_offload(device=device) +# elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": +# convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) +# convert_weight_dtype_wrapper(transformer, weight_dtype) +# pipeline.enable_model_cpu_offload(device=device) +# elif GPU_memory_mode == "model_cpu_offload": +# pipeline.enable_model_cpu_offload(device=device) +# elif GPU_memory_mode == "model_full_load_and_qfloat8": +# convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) +# convert_weight_dtype_wrapper(transformer, weight_dtype) +# pipeline.to(device=device) +# else: +# pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +# if lora_path is not None: +# pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +# import pdb; pdb.set_trace() +if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] # torch.Size([1, 3, sample_size[0], sample_size[1]]) + +print(control_image.shape) + + +height, width = sample_size + +vae_scale_factor = 8 +vae_scale = vae_scale_factor * 2 + +if height % vae_scale != 0: + raise ValueError( + f"Height must be divisible by {vae_scale} (got {height}). " + f"Please adjust the height to a multiple of {vae_scale}." + ) +if width % vae_scale != 0: + raise ValueError( + f"Width must be divisible by {vae_scale} (got {width}). " + f"Please adjust the width to a multiple of {vae_scale}." + ) + +_guidance_scale = guidance_scale = 0.0 +_joint_attention_kwargs = joint_attention_kwargs = None +_interrupt = False +_cfg_normalization = cfg_normalization = False +_cfg_truncation = cfg_truncation = 1.0 + + +# 2. Define call parameters +prompt_embeds = None +if prompt is not None and isinstance(prompt, str): + batch_size = 1 +elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) +else: + batch_size = len(prompt_embeds) + +batch_size = 1 + +weight_dtype = text_encoder.dtype # text_encoder.dtype +num_channels_latents = 16 # transformer.in_channels +vae_config_shift_factor = 0.1159 +vae_config_scaling_factor = 0.3611 +inpaint_latent = None # TODO: 原始代码中未定义该变量 + +from diffusers.image_processor import VaeImageProcessor +image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + +# Get Vae +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype).to(device) + +if control_image is not None: + control_image = image_processor.preprocess(control_image, height=height, width=width) + control_image = control_image.to(dtype=weight_dtype, device=device) + control_latents = vae.encode(control_image)[0].mode() + # control_latents = ort_inference( + # "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-models/vae_encoder_slim.onnx", + # {"x": control_image}, + # )[0] + # control_latents = torch.from_numpy(control_latents).to(device=device, dtype=weight_dtype) + control_latents = (control_latents - vae_config_shift_factor) * vae_config_scaling_factor +else: + control_latents = torch.zeros_like(inpaint_latent) + +control_context = control_latents.unsqueeze(2) + + +do_classifier_free_guidance = False +negative_prompt_embeds = None +max_sequence_length = 512 + +prompt_embeds, negative_prompt_embeds = encode_prompt( + prompt=prompt, + negative_prompt=negative_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + device=device, + max_sequence_length=max_sequence_length, +) + +num_images_per_prompt = 1 +latents = None + +""" +(Pdb) latents[0, 0, 100:105, 100] +tensor([-0.9203, 1.3958, 0.8130, -0.5280, -1.9788], device='cuda:0') +""" + +# 4. Prepare latent variables +latents = prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + torch.float32, + device, + generator, + latents, +) # torch.Size([1, 16, 216, 124]) +print(latents.shape) +# 以上代码验证正确 + +# Repeat prompt_embeds for num_images_per_prompt +if num_images_per_prompt > 1: + prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)] + if do_classifier_free_guidance and negative_prompt_embeds: + negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)] + +actual_batch_size = batch_size * num_images_per_prompt +image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2) + + +# Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +# 5. Prepare timesteps +mu = calculate_shift( + image_seq_len, + scheduler.config.get("base_image_seq_len", 256), + scheduler.config.get("max_image_seq_len", 4096), + scheduler.config.get("base_shift", 0.5), + scheduler.config.get("max_shift", 1.15), +) +scheduler.sigma_min = 0.0 +scheduler_kwargs = {"mu": mu} + + +import inspect +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +sigmas = None +timesteps, num_inference_steps = retrieve_timesteps( + scheduler, + num_inference_steps, + device, + sigmas=sigmas, + **scheduler_kwargs, +) +num_warmup_steps = max(len(timesteps) - num_inference_steps * scheduler.order, 0) +""" +(Pdb) timesteps +tensor([1000.0000, 954.5454, 900.0000, 833.3333, 750.0000, 642.8571, + 500.0000, 300.0000, 0.0000], device='cuda:0') +""" +_num_timesteps = len(timesteps) # 9 +# 以上代码验证正确 +# import pdb; pdb.set_trace() + +callback_on_step_end = None +callback_on_step_end_tensor_inputs = ['latents'] + +# 6. Denoising loop +# with progress_bar(total=num_inference_steps) as progress_bar: +for i, t in enumerate(timesteps): + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latents.shape[0]) + timestep = (1000 - timestep) / 1000 + # Normalized time for time-aware config (0 at start, 1 at end) + t_norm = timestep[0].item() + + # Handle cfg truncation + current_guidance_scale = guidance_scale + if ( + do_classifier_free_guidance + and _cfg_truncation is not None + and float(_cfg_truncation) <= 1 + ): + if t_norm > _cfg_truncation: + current_guidance_scale = 0.0 + + # Run CFG only if configured AND scale is non-zero + apply_cfg = do_classifier_free_guidance and current_guidance_scale > 0 + + if apply_cfg: + latents_typed = latents.to(transformer.dtype) + latent_model_input = latents_typed.repeat(2, 1, 1, 1) + prompt_embeds_model_input = prompt_embeds + negative_prompt_embeds + timestep_model_input = timestep.repeat(2) + else: + latent_model_input = latents.to(transformer.dtype) + prompt_embeds_model_input = prompt_embeds + timestep_model_input = timestep + + latent_model_input = latent_model_input.unsqueeze(2) + latent_model_input_list = list(latent_model_input.unbind(dim=0)) + + import pdb; pdb.set_trace() + model_out_list = transformer( + latent_model_input_list, + timestep_model_input, + prompt_embeds_model_input, + control_context=control_context, + control_context_scale=control_context_scale, + )[0] + + if apply_cfg: + # Perform CFG + pos_out = model_out_list[:actual_batch_size] + neg_out = model_out_list[actual_batch_size:] + + noise_pred = [] + for j in range(actual_batch_size): + pos = pos_out[j].float() + neg = neg_out[j].float() + + pred = pos + current_guidance_scale * (pos - neg) + + # Renormalization + if _cfg_normalization and float(_cfg_normalization) > 0.0: + ori_pos_norm = torch.linalg.vector_norm(pos) + new_pos_norm = torch.linalg.vector_norm(pred) + max_new_norm = ori_pos_norm * float(_cfg_normalization) + if new_pos_norm > max_new_norm: + pred = pred * (max_new_norm / new_pos_norm) + + noise_pred.append(pred) + + noise_pred = torch.stack(noise_pred, dim=0) + else: + noise_pred = torch.stack([t.float() for t in model_out_list], dim=0) + + noise_pred = noise_pred.squeeze(2) + noise_pred = -noise_pred + + # compute the previous noisy sample x_t -> x_t-1 + latents = scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0] + assert latents.dtype == torch.float32 + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + # if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % scheduler.order == 0): + # progress_bar.update() + +output_type = "pil" +if output_type == "latent": + image = latents + +else: + latents = latents.to(vae.dtype) + latents = (latents / vae.config.scaling_factor) + vae.config.shift_factor + + image = vae.decode(latents, return_dict=False)[0] # torch.Size([1, 3, 1728, 992]) + """ + (Pdb) image[0, 0, 100, 100:105] + tensor([0.3906, 0.3848, 0.3809, 0.3809, 0.3848], device='cuda:0', + dtype=torch.bfloat16) + """ + import pdb; pdb.set_trace() + # image = ort_inference( + # "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-models/vae_decoder_slim.onnx", + # {"latent": latents}, + # )[0] + # image = torch.from_numpy(image).to(device=device, dtype=vae.dtype) + image = image_processor.postprocess(image, output_type=output_type) + +# # Offload all models +# maybe_free_model_hooks() + +# if not return_dict: +# return (image,) + +# return ZImagePipelineOutput(images=image) + +# # with torch.no_grad(): + +# # sample = pipeline( +# # prompt = prompt, +# # negative_prompt = negative_prompt, +# # height = sample_size[0], +# # width = sample_size[1], +# # generator = generator, +# # guidance_scale = guidance_scale, +# # control_image = control_image, +# # num_inference_steps = num_inference_steps, +# # control_context_scale = control_context_scale, +# # ).images + +# # if lora_path is not None: +# # pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +# # def save_results(): +# # if not os.path.exists(save_path): +# # os.makedirs(save_path, exist_ok=True) + +# # index = len([path for path in os.listdir(save_path)]) + 1 +# # prefix = str(index).zfill(8) +# # video_path = os.path.join(save_path, prefix + ".png") +# # image = sample[0] +# # image.save(video_path) + +# # if ulysses_degree * ring_degree > 1: +# # import torch.distributed as dist +# # if dist.get_rank() == 0: +# # save_results() +# # else: +# # save_results() + +# # save_results() \ No newline at end of file diff --git a/VideoX-Fun/examples/z_image_fun/launcher_axmodel.py b/VideoX-Fun/examples/z_image_fun/launcher_axmodel.py new file mode 100644 index 0000000000000000000000000000000000000000..47d4a047977ee1abbe85da1825848a8520af1c0f --- /dev/null +++ b/VideoX-Fun/examples/z_image_fun/launcher_axmodel.py @@ -0,0 +1,575 @@ +#!/usr/bin/env python3 +"""使用 AXModel 推理链路 (transformer + VAE decoder)。""" +from __future__ import annotations + +import argparse +import json +import os +import random +import sys +from contextlib import contextmanager +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Tuple, Union + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if REPO_ROOT.as_posix() not in sys.path: + sys.path.insert(0, REPO_ROOT.as_posix()) + +import numpy as np +import torch +from axengine import InferenceSession as AxInferenceSession +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.image_processor import VaeImageProcessor +from diffusers.utils.torch_utils import randn_tensor +from omegaconf import OmegaConf +from PIL import Image +from loguru import logger +from tqdm import tqdm + +from videox_fun.models import AutoTokenizer, Qwen3ForCausalLM +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.utils import get_image_latent + + +# ----------------------------------------------------------------------------- +# 模型与资源路径 +# ----------------------------------------------------------------------------- +MODEL_NAME = "models/Diffusion_Transformer/Z-Image-Turbo/" +CONFIG_PATH = REPO_ROOT / "VideoX-Fun" / "config" / "z_image" / "z_image.yaml" +TRANSFORMER_CONFIG_PATH = REPO_ROOT / "VideoX-Fun" / "pulsar2_configs" / "transformers_subgraph.json" +TRANSFORMER_ONNX_PATH = REPO_ROOT / "VideoX-Fun" / "compiled_subgraph_from_onnx" / "frontend" / "optimized_quant_axmodel.onnx" +TRANSFORMER_AXMODEL_DIR = REPO_ROOT / "VideoX-Fun" / "comliled_subgraph_from_all_onnx" # compiled_slice_quant_onnx +VAE_DECODER_AXMODEL = REPO_ROOT / "VideoX-Fun" / "vae_decoder.axmodel" +SAVE_DIR = REPO_ROOT / "VideoX-Fun" / "samples" / "z-image-t2i-axmodel" + +# ----------------------------------------------------------------------------- +# 运行配置 +# ----------------------------------------------------------------------------- +DEFAULT_PROMPTS = [ + "(masterpiece, best quality) solo female on a tropical beach, golden hour rim light, cinematic grading", + "nighttime cyberpunk boulevard, neon reflections on wet asphalt, volumetric fog, wide shot", + "sunrise over alpine mountains, low clouds in valleys, god rays, ultra-detailed landscape", + "modern minimal living room, soft natural light, Scandinavian design, high-resolution interior render", + "classical oil painting of a renaissance noblewoman, chiaroscuro lighting, rich textures", + "macro photography of a dewdrop on a leaf, extreme detail, shallow depth of field", + "futuristic sports car parked under neon lights, glossy paint, cinematic 35mm look", + "ancient library with towering bookshelves, warm candlelight, dust motes in air", + "portrait of an astronaut in full suit, visor reflection showing earth, studio lighting", + "stormy sea with a lone lighthouse, crashing waves, dramatic clouds, long exposure feel", + "cybernetic samurai standing in rain, backlit silhouette, moody blue-orange palette", + "lush rainforest waterfall, soft mist, saturated greens, wide-angle composition", + "product shot of a smartwatch on marble, softbox lighting, crisp shadows, advertisement style", + "architectural exterior of a glass skyscraper at dusk, warm interior lights, reflections", + "vintage film photograph of a 1950s diner at night, grain and halation, neon signage", + "hyperrealistic bowl of ramen, steam rising, glossy broth, detailed toppings", + "fantasy castle on a floating island, waterfalls falling into clouds, sunset lighting", + "high-fashion editorial portrait, dramatic chiaroscuro, sharp focus on eyes", + "aerial view of winding river through autumn forest, golden and crimson leaves", + "studio shot of running shoes mid-air, motion blur trails, vibrant background gradient", + "noir city alley in the 1940s, hard shadows, rain-slick pavement, moody atmosphere", + "desert caravan at twilight, silhouettes of camels, soft purple sky, cinematic scope", + "close-up of a mechanical watch movement, intricate gears, metallic reflections", + "bioluminescent underwater reef, glowing corals, schools of fish, deep blue tones", + "portrait of an elderly man with weathered face, soft window light, fine skin detail", + "snowy village at night, warm cabin lights, smoke from chimneys, peaceful mood", + "futuristic data center aisle, cool cyan lighting, depth and symmetry", + "oil painting of a bowl of fruit in Dutch masters style, rich textures, dramatic lighting", + "sunlit meadow with wildflowers, shallow depth of field, pastel color palette", + "sci-fi corridor with volumetric light shafts, pristine white surfaces, wide lens", + "luxury wristwatch on black velvet, high contrast, advertisement macro shot", + "medieval marketplace at dawn, merchants setting up, soft warm light, lively details", + "((masterpiece,best quality))1 young beautiful girl,ultra detailed,official art,unity 8k wallpaper,masterpiece, best quality, official art, extremely detailed CG unity 8k wallpaper, highly detailed, 1 girl, aqua eyes, light smile, ((grey hair)), hair flower, bracelet, choker, ribbon, JK, look at viewer, on the beach, in summer," +] +prompt_idx = random.randint(0, len(DEFAULT_PROMPTS) - 1) +PROMPT = DEFAULT_PROMPTS[prompt_idx] +NEG_PROMPT = " " +GUIDANCE_SCALE = 0.0 +SEED = 42 +HEIGHT, WIDTH = 512, 512 +NUM_INFERENCE_STEPS = 9 +NUM_CHANNELS_LATENTS = 16 +VAE_SCALE_FACTOR = 8 +PATCH_SIZE = 2 +FPATCH_SIZE = 1 +MAX_SEQ_LEN = 128 +VAE_SCALING_FACTOR = 0.3611 +VAE_SHIFT_FACTOR = 0.1159 + +SAMPLER_MAP = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +} +SAMPLER_NAME = "Flow" + +# 默认最终输出，如果不存在 auto 子图则回退到最后一个 cfg 输出 +DEFAULT_FINAL_OUTPUT = None + +# ----------------------------------------------------------------------------- +# 工具函数 (复制自原 launcher 并微调) +# ----------------------------------------------------------------------------- + +def _infer_module_device(module: torch.nn.Module) -> torch.device: + param = next(module.parameters(), None) + if param is not None: + return param.device + buffer = next(module.buffers(), None) + if buffer is not None: + return buffer.device + return torch.device("cpu") + + +@contextmanager +def module_to_device(module: torch.nn.Module, target_device: torch.device): + if module is None: + yield module + return + original_device = _infer_module_device(module) + target_device = target_device or original_device + needs_move = original_device != target_device + moved_to_cuda = needs_move and target_device.type == "cuda" + if needs_move: + module.to(target_device) + try: + yield module + finally: + if needs_move: + module.to(original_device) + if moved_to_cuda and torch.cuda.is_available(): + cache_device = target_device.index or torch.cuda.current_device() + with torch.cuda.device(cache_device): + torch.cuda.empty_cache() + + +def _encode_prompt( + tokenizer: AutoTokenizer, + text_encoder: Qwen3ForCausalLM, + prompt: Union[str, List[str]], + device: torch.device, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + max_sequence_length: int = 512, +) -> List[torch.FloatTensor]: + if prompt_embeds is not None: + return prompt_embeds + prompts = [prompt] if isinstance(prompt, str) else list(prompt) + for idx, item in enumerate(prompts): + messages = [{"role": "user", "content": item}] + prompts[idx] = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, enable_thinking=True + ) + text_inputs = tokenizer( + prompts, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + with module_to_device(text_encoder, device): + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] + return [prompt_embeds[i] for i in range(len(prompt_embeds))] + + +def encode_prompt( + tokenizer: AutoTokenizer, + text_encoder: Qwen3ForCausalLM, + prompt: Union[str, List[str]], + device: torch.device, + do_classifier_free_guidance: bool, + negative_prompt: Optional[Union[str, List[str]]], + max_sequence_length: int, +) -> Tuple[List[torch.FloatTensor], List[torch.FloatTensor]]: + prompt_embeds = _encode_prompt( + tokenizer, text_encoder, prompt, device, None, max_sequence_length + ) + negative_embeds: List[torch.FloatTensor] = [] + if do_classifier_free_guidance: + neg = negative_prompt or "" + negative_list = [neg] if isinstance(neg, str) else list(neg) + negative_embeds = _encode_prompt( + tokenizer, text_encoder, negative_list, device, None, max_sequence_length + ) + return prompt_embeds, negative_embeds + + +def _stack_prompt_embeddings(prompt_embeds_input): + if isinstance(prompt_embeds_input, list): + return torch.stack(prompt_embeds_input, dim=0) + return prompt_embeds_input + + +def prepare_latents( + batch_size: int, + num_channels_latents: int, + height: int, + width: int, + dtype: torch.dtype, + device: torch.device, + generator: torch.Generator, +) -> torch.FloatTensor: + height = 2 * (int(height) // (VAE_SCALE_FACTOR * 2)) + width = 2 * (int(width) // (VAE_SCALE_FACTOR * 2)) + shape = (batch_size, num_channels_latents, height, width) + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + return latents + + +def calculate_shift( + image_seq_len: int, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +) -> float: + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + return image_seq_len * m + b + + +def retrieve_timesteps( + scheduler, + num_inference_steps: int, + device: torch.device, + **kwargs, +): + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + return scheduler.timesteps + + +# ----------------------------------------------------------------------------- +# 参数 +# ----------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="AXModel 推理 (transformer + VAE)") + parser.add_argument("--prompt", type=str, default=None, help="正向提示词，不填则使用预置随机样本") + parser.add_argument("--negative-prompt", type=str, default=NEG_PROMPT, help="反向提示词") + parser.add_argument("--steps", type=int, default=NUM_INFERENCE_STEPS, help="迭代步数") + parser.add_argument("--height", type=int, default=HEIGHT, help="生成高度，需被 16 整除") + parser.add_argument("--width", type=int, default=WIDTH, help="生成宽度，需被 16 整除") + parser.add_argument("--seed", type=int, default=SEED, help="随机种子") + parser.add_argument("--sampler", type=str, choices=list(SAMPLER_MAP.keys()), default=SAMPLER_NAME, help="采样器") + parser.add_argument("--max-seq-len", type=int, default=MAX_SEQ_LEN, help="最大文本长度") + parser.add_argument("--save-dir", type=str, default=str(SAVE_DIR), help="结果输出目录") + parser.add_argument("--transformer-config", type=str, required=True, help="子图配置 json") + parser.add_argument("--transformer-onnx", type=str, default=None, help="原始 transformer onnx（可选，sub_configs 已覆盖可不填）") + parser.add_argument("--transformer-subgraph-dir", type=str, required=True, help="子图 axmodel 目录") + parser.add_argument("--vae-axmodel", type=str, required=True, help="VAE decoder axmodel 路径") + parser.add_argument("--final-output-name", type=str, default=None, help="指定最终输出 tensor 名称，默认自动推断") + parser.add_argument("--save-decoder-input", action="store_true", help="是否保存 decoder 输入 npy") + parser.add_argument("--no-progress", action="store_true", help="关闭进度条输出") + return parser.parse_args() + + +# ----------------------------------------------------------------------------- +# AX transformer 子图执行器 +# ----------------------------------------------------------------------------- + +from scripts.split_quant_onnx_by_subconfigs import SubGraphSpec, sanitize + + +class AxSplitTransformer: + def __init__(self, config_path: Path, onnx_path: Optional[Path], model_dir: Path): + self.config_path = config_path + self.onnx_path = onnx_path + self.model_dir = model_dir + self._session_cache: Dict[str, AxInferenceSession] = {} + config_specs = self._load_specs() + auto_specs = self._load_auto_specs() + self.specs = config_specs + auto_specs + last_group = auto_specs if auto_specs else config_specs + self.final_outputs = list(last_group[-1].end) + self.default_final_output = DEFAULT_FINAL_OUTPUT or self.final_outputs[0] + + def _get_session(self, spec: SubGraphSpec) -> AxInferenceSession: + if spec.label not in self._session_cache: + path = self._expected_path(spec) + self._session_cache[spec.label] = AxInferenceSession(path.as_posix()) + return self._session_cache[spec.label] + + def close(self) -> None: + # 显式释放缓存的 session + for key, sess in list(self._session_cache.items()): + try: + del sess + finally: + self._session_cache.pop(key, None) + + def _load_specs(self) -> List[SubGraphSpec]: + with self.config_path.open("r", encoding="utf-8") as f: + config = json.load(f) + sub_configs = config.get("compiler", {}).get("sub_configs", []) + if not sub_configs: + raise ValueError("配置文件缺少 compiler.sub_configs") + specs: List[SubGraphSpec] = [] + for idx, entry in enumerate(sub_configs): + start = [name for name in entry.get("start_tensor_names", []) if name] + end = [name for name in entry.get("end_tensor_names", []) if name] + if not start or not end: + raise ValueError(f"sub_config[{idx}] 定义不完整") + spec = SubGraphSpec( + label=f"cfg_{idx:02d}", + start=start, + end=end, + node_names=set(), + source="config", + ) + specs.append(spec) + return specs + + def _load_auto_specs(self) -> List[SubGraphSpec]: + specs: List[SubGraphSpec] = [] + for path in sorted(self.model_dir.glob("auto_*.axmodel")): + # Ax compiler有两种输出目录命名：compiled_{stem} 或 compiled_auto_XX + candidates = [path.parent / f"compiled_{path.stem}"] + # 兼容实际产物 compiled_auto_00 这类名称 + parts = path.stem.split("_", 3) + if len(parts) >= 2: + candidates.append(path.parent / f"compiled_{parts[0]}_{parts[1]}") + + ctx_path = None + for cand in candidates: + candidate_ctx = cand / "build_context.json" + if candidate_ctx.exists(): + ctx_path = candidate_ctx + break + + if ctx_path is None: + logger.warning(f"跳过 {path.name}，缺少 build_context.json (尝试: {[c.as_posix() for c in candidates]})") + continue + if not ctx_path.exists(): + logger.warning(f"跳过 {path.name}，缺少 build_context.json") + continue + try: + with ctx_path.open("r", encoding="utf-8") as f: + ctx = json.load(f) + inputs = [p.get("tensor_name") for p in ctx.get("cfg", {}).get("input_processors", []) if p.get("tensor_name")] + outputs = [p.get("tensor_name") for p in ctx.get("cfg", {}).get("output_processors", []) if p.get("tensor_name")] + except Exception as exc: # pragma: no cover - defensive + logger.warning(f"解析 {ctx_path} 失败: {exc}") + continue + + if not inputs or not outputs: + logger.warning(f"跳过 {path.name}，未找到有效的输入/输出定义") + continue + + specs.append( + SubGraphSpec( + label=path.stem, + start=inputs, + end=outputs, + node_names=set(), + source="auto", + output_path=path, + ) + ) + return specs + + def _expected_path(self, spec: SubGraphSpec) -> Path: + if spec.output_path is not None: + path = spec.output_path + else: + head = sanitize(spec.start[0]) if spec.start else "const" + tail = sanitize(spec.end[0]) if spec.end else "out" + filename = f"{spec.label}_{head}_to_{tail}_{spec.source}.axmodel" + path = self.model_dir / filename + if not path.exists(): + raise FileNotFoundError(f"缺少 AXModel: {path}") + return path + + def __call__( + self, + latent_np: np.ndarray, + prompt_np: np.ndarray, + timestep_np: np.ndarray, + final_output_name: Optional[str] = None, + ) -> np.ndarray: + tensor_store: Dict[str, np.ndarray] = { + "latent_model_input": latent_np, + "prompt_embeds": prompt_np, + "timestep": timestep_np, + } + executed = set() + target = final_output_name or self.default_final_output + + # 就绪驱动执行，单个子图跑完立刻释放 session + while target not in tensor_store: + progressed = False + for spec in self.specs: + if spec.label in executed: + continue + if not all(name in tensor_store for name in spec.start): + continue + session = self._get_session(spec) + inputs = {name: tensor_store[name] for name in spec.start} + outputs = session.run(spec.end, inputs) + for out_name, value in zip(spec.end, outputs): + tensor_store[out_name] = value + executed.add(spec.label) + progressed = True + if not progressed: + missing = [ + (spec.label, [name for name in spec.start if name not in tensor_store]) + for spec in self.specs + if spec.label not in executed + ] + raise RuntimeError( + f"子图调度中断，缺少输入: {missing}; 当前可用: {list(tensor_store.keys())}" + ) + + return tensor_store[target] + + +# ----------------------------------------------------------------------------- +# 主流程 +# ----------------------------------------------------------------------------- + +def main() -> None: + args = parse_args() + prompt_text = args.prompt if args.prompt is not None else PROMPT + logger.info(f"使用的 prompt: {prompt_text}") + + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + torch.set_grad_enabled(False) + + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, subfolder="tokenizer") + text_encoder = Qwen3ForCausalLM.from_pretrained( + MODEL_NAME, + subfolder="text_encoder", + torch_dtype=torch.bfloat16, + low_cpu_mem_usage=True, + ) + text_encoder.eval() + + scheduler_cls = SAMPLER_MAP[args.sampler] + scheduler = scheduler_cls.from_pretrained(MODEL_NAME, subfolder="scheduler") + + image_processor = VaeImageProcessor(vae_scale_factor=VAE_SCALE_FACTOR * 2) + + prompt_embeds, _ = encode_prompt( + tokenizer, + text_encoder, + prompt_text, + device, + do_classifier_free_guidance=False, + negative_prompt=args.negative_prompt, + max_sequence_length=args.max_seq_len, + ) + + latents = prepare_latents( + batch_size=1, + num_channels_latents=NUM_CHANNELS_LATENTS, + height=args.height, + width=args.width, + dtype=torch.float32, + device=device, + generator=torch.Generator(device=device).manual_seed(args.seed), + ) + + image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2) + mu = calculate_shift( + image_seq_len, + scheduler.config.get("base_image_seq_len", 256), + scheduler.config.get("max_image_seq_len", 4096), + scheduler.config.get("base_shift", 0.5), + scheduler.config.get("max_shift", 1.15), + ) + timesteps = retrieve_timesteps(scheduler, args.steps, device=device, mu=mu) + + onnx_path = Path(args.transformer_onnx) if args.transformer_onnx else None + transformer_runner = AxSplitTransformer( + Path(args.transformer_config), + onnx_path, + Path(args.transformer_subgraph_dir), + ) + # 优先使用 auto_* 子图里的 sample 输出，避免误用中间特征导致 shape 对不上 + available_outputs = [name for spec in transformer_runner.specs for name in getattr(spec, "end", [])] + preferred_output = "sample" if "sample" in available_outputs else transformer_runner.default_final_output + final_output_name = args.final_output_name or preferred_output + if final_output_name not in available_outputs: + raise ValueError(f"指定的输出 {final_output_name} 不存在，可选: {available_outputs}") + + prompt_embeds_tensor = _stack_prompt_embeddings(prompt_embeds) + + iterator = timesteps if args.no_progress else tqdm(timesteps, desc="AX Denoising", dynamic_ncols=True) + for t in iterator: + timestep = t.expand(latents.shape[0]) + timestep_model_input = (1000 - timestep) / 1000 + + latent_model_input = latents.to(torch.float32) + latent_np = latent_model_input.unsqueeze(2).to(dtype=torch.float32).cpu().numpy() + prompt_np = prompt_embeds_tensor.to(dtype=torch.float32).cpu().numpy() + timestep_np = timestep_model_input.to(dtype=torch.float32).cpu().numpy() + + model_out = transformer_runner(latent_np, prompt_np, timestep_np, final_output_name) + if model_out.ndim == 5 and model_out.shape[2] == 1: + model_out = np.squeeze(model_out, axis=2) + model_out_tensor = torch.from_numpy(model_out).to(device=device, dtype=torch.float32) + if model_out_tensor.dim() == 5 and model_out_tensor.size(2) == 1: + model_out_tensor = model_out_tensor.squeeze(2) + noise_pred = -model_out_tensor + + latents = scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + # 释放 transformer 缓存的 session + transformer_runner.close() + + latents = (latents / VAE_SCALING_FACTOR) + VAE_SHIFT_FACTOR + decoder_input = latents.to(dtype=torch.float32).cpu().numpy() + + if args.save_decoder_input: + save_dir_path = Path(args.save_dir) + save_dir_path.mkdir(parents=True, exist_ok=True) + np.save(save_dir_path / "decoder_input.npy", decoder_input) + logger.info("已保存 decoder 输入为 npy") + + del transformer_runner + + vae_decoder_session = AxInferenceSession(Path(args.vae_axmodel).as_posix()) + + if decoder_input.ndim == 5 and decoder_input.shape[2] == 1: + decoder_input = np.squeeze(decoder_input, axis=2) + image = vae_decoder_session.run(None, {"latents": decoder_input})[0] + + del vae_decoder_session + image = torch.from_numpy(image).to(device=device, dtype=torch.float32) + image = image_processor.postprocess(image, output_type="pil") + + save_dir = Path(args.save_dir) + save_dir.mkdir(parents=True, exist_ok=True) + target_path = save_dir / f"z_image_axmodel_{prompt_idx}.png" + image[0].save(target_path) + logger.info(f"AXModel 推理完成，结果保存到 {target_path}") + + +if __name__ == "__main__": + """ + # 512x512 生成示例命令： + python3 examples/z_image_fun/launcher_axmodel.py \ + --transformer-config pulsar2_configs/transformers_subgraph.json \ + --transformer-subgraph-dir comliled_subgraph_from_all_onnx \ + --vae-axmodel vae_decoder.axmodel + + # 1728x992 生成示例命令: + python3 examples/z_image_fun/launcher_axmodel.py \ + --transformer-config pulsar2_configs/transformers_subgraph_1728x992.json \ + --transformer-subgraph-dir transformers_body_only_1728_992_split_onnx \ + --vae-axmodel onnx-models-1728x992/vae_decoder_simp_slim.axmodel \ + --max-seq-len 256 \ + --height 1728 --width 992 + """ + + main() \ No newline at end of file diff --git a/VideoX-Fun/examples/z_image_fun/launcher_no_controlnet.py b/VideoX-Fun/examples/z_image_fun/launcher_no_controlnet.py new file mode 100644 index 0000000000000000000000000000000000000000..6b61aa40863da4a4b11ddd9145dbeed7ca6c352a --- /dev/null +++ b/VideoX-Fun/examples/z_image_fun/launcher_no_controlnet.py @@ -0,0 +1,612 @@ +import os +import sys +import numpy as np +import torch +import onnxruntime as ort +from contextlib import contextmanager +from typing import List, Optional, Union + +from omegaconf import OmegaConf +from PIL import Image +from loguru import logger + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None +repo_root = project_roots[-1] +onnx_models_dir = os.path.join(repo_root, "onnx-models") +# vae_encoder_onnx_path = "" +# vae_decoder_onnx_path = "" +vae_encoder_onnx_path = os.path.join(onnx_models_dir, "vae_encoder_simp_slim.onnx") +vae_decoder_onnx_path = os.path.join(onnx_models_dir, "vae_decoder_simp_slim.onnx") +use_transformer_onnx = True +transformer_body_onnx_path = os.path.join(onnx_models_dir, "z_image_transformer_body_only_simp_slim.onnx") +onnx_export_dtype = torch.float16 +_printed_onnx_debug = False + + +def _select_onnx_providers(): + avail = ort.get_available_providers() + if "CUDAExecutionProvider" in avail: + return ["CUDAExecutionProvider", "CPUExecutionProvider"] + if "AzureExecutionProvider" in avail: + return ["AzureExecutionProvider", "CPUExecutionProvider"] + return ["CPUExecutionProvider"] + + +onnx_providers = _select_onnx_providers() + +from diffusers import FlowMatchEulerDiscreteScheduler +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + Qwen3ForCausalLM, ZImageTransformer2DModel) +from diffusers.utils.torch_utils import randn_tensor +from videox_fun.utils.utils import get_image_latent + + +torch.set_grad_enabled(False) +_ort_sessions = {} +def _ort_type_to_torch(type_str: str): + if type_str is None: + return None + if "float16" in type_str: + return torch.float16 + if "bfloat16" in type_str: + return torch.bfloat16 + if "float" in type_str: + return torch.float32 + return None + + +def ort_inference(onnx_model_path: str, inputs: dict, providers=None): + providers = providers or ["CPUExecutionProvider"] + cache_key = (onnx_model_path, tuple(providers)) + if cache_key not in _ort_sessions: + if not os.path.exists(onnx_model_path): + raise FileNotFoundError(f"ONNX model not found: {onnx_model_path}") + _ort_sessions[cache_key] = ort.InferenceSession(onnx_model_path, providers=providers) + session = _ort_sessions[cache_key] + + # 按 ONNX 输入类型对齐 dtype，避免 float/bfloat16 转换导致的类型冲突 + input_type_map = {i.name: _ort_type_to_torch(getattr(i, "type", None)) for i in session.get_inputs()} + inputs_onnx = {} + for k, v in inputs.items(): + target_torch_dtype = input_type_map.get(k) + if isinstance(v, np.ndarray): + if target_torch_dtype is not None: + np_dtype = np.float16 if target_torch_dtype == torch.float16 else np.float32 + if v.dtype != np_dtype: + v = v.astype(np_dtype) + inputs_onnx[k] = v + elif torch.is_tensor(v): + if target_torch_dtype is not None and v.dtype != target_torch_dtype: + v = v.to(dtype=target_torch_dtype) + elif v.dtype == torch.bfloat16: + # numpy 不支持 bfloat16，主动转换为 float32 + v = v.to(dtype=torch.float32) + inputs_onnx[k] = v.detach().to("cpu").numpy() + else: + raise TypeError(f"Unsupported input type for key {k}: {type(v)}") + return session.run(None, inputs_onnx) + + +def _infer_module_device(module: torch.nn.Module) -> torch.device: + param = next(module.parameters(), None) + if param is not None: + return param.device + buffer = next(module.buffers(), None) + if buffer is not None: + return buffer.device + return torch.device("cpu") + + +@contextmanager +def module_to_device(module: torch.nn.Module, target_device: torch.device): + if module is None: + yield module + return + original_device = _infer_module_device(module) + target_device = target_device or original_device + needs_move = original_device != target_device + moved_to_cuda = needs_move and target_device.type == "cuda" + if needs_move: + module.to(target_device) + try: + yield module + finally: + if needs_move: + module.to(original_device) + if moved_to_cuda and torch.cuda.is_available(): + cache_device = target_device.index + if cache_device is None: + cache_device = torch.cuda.current_device() + with torch.cuda.device(cache_device): + torch.cuda.empty_cache() + + +# Config and model path（纯 body，无 control） +config_path_default = "config/z_image/z_image.yaml" +model_name = "models/Diffusion_Transformer/Z-Image-Turbo/" + +# Use torch.float16 if GPU does not support torch.bfloat16 +weight_dtype = torch.bfloat16 + +control_context_scale = 0.75 + +device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + +# Get tokenizer and text_encoder +tokenizer = AutoTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) +text_encoder.eval() + + +def _encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + max_sequence_length: int = 512, +) -> List[torch.FloatTensor]: + device = device or _infer_module_device(text_encoder) + + if prompt_embeds is not None: + return prompt_embeds + + if isinstance(prompt, str): + prompt = [prompt] + + for i, prompt_item in enumerate(prompt): + messages = [ + {"role": "user", "content": prompt_item}, + ] + prompt_item = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + prompt[i] = prompt_item + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + + with module_to_device(text_encoder, device): + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] + + embeddings_list = [] + for i in range(len(prompt_embeds)): + # embeddings_list.append(prompt_embeds[i][prompt_masks[i]]) + embeddings_list.append(prompt_embeds[i]) + return embeddings_list + + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + max_sequence_length: int = 512, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + prompt_embeds = _encode_prompt( + prompt=prompt, + device=device, + prompt_embeds=prompt_embeds, + max_sequence_length=max_sequence_length, + ) + + if do_classifier_free_guidance: + if negative_prompt is None: + negative_prompt = ["" for _ in prompt] + else: + negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + assert len(prompt) == len(negative_prompt) + negative_prompt_embeds = _encode_prompt( + prompt=negative_prompt, + device=device, + prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + ) + else: + negative_prompt_embeds = [] + return prompt_embeds, negative_prompt_embeds + + +def _stack_prompt_embeddings(prompt_embeds_input): + if isinstance(prompt_embeds_input, list): + return torch.stack(prompt_embeds_input, dim=0) + return prompt_embeds_input + + +def _onnx_shape_compatible(model_path: str, providers, latent_shape, prompt_shape, verbose=False): + try: + sess = ort.InferenceSession(model_path, providers=providers) + inputs = {i.name: i for i in sess.get_inputs()} + ok = True + mismatch_msgs = [] + if "latent_model_input" in inputs: + shape = inputs["latent_model_input"].shape + exp_h, exp_w = shape[3], shape[4] + if isinstance(exp_h, int) and exp_h != latent_shape[3]: + ok = False + mismatch_msgs.append(f"latent_h expected {exp_h}, got {latent_shape[3]}") + if isinstance(exp_w, int) and exp_w != latent_shape[4]: + ok = False + mismatch_msgs.append(f"latent_w expected {latent_w} got {latent_shape[4]}") + if "prompt_embeds" in inputs: + pshape = inputs["prompt_embeds"].shape + exp_seq = pshape[1] + if isinstance(exp_seq, int) and exp_seq != prompt_shape[1]: + ok = False + mismatch_msgs.append(f"seq_len expected {exp_seq}, got {prompt_shape[1]}") + if verbose and (not ok or mismatch_msgs): + print(f"[DEBUG] ONNX shape check for {model_path}") + print(f" providers={providers}") + print(f" model latent shape={inputs.get('latent_model_input').shape if 'latent_model_input' in inputs else 'n/a'}") + print(f" model prompt shape={inputs.get('prompt_embeds').shape if 'prompt_embeds' in inputs else 'n/a'}") + print(f" runtime latent shape={latent_shape}, prompt shape={prompt_shape}") + if mismatch_msgs: + print(f" mismatch: {', '.join(mismatch_msgs)}") + return ok + except Exception as exc: + print(f"ONNX shape check failed for {model_path}: {exc}") + return True + + +def prepare_latents( + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, +): + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + + shape = (batch_size, num_channels_latents, height, width) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + return latents + + +# GPU memory mode (保留占位，当前脚本默认全量加载) +GPU_memory_mode = "model_full_load" +ulysses_degree = 1 +ring_degree = 1 +fsdp_dit = False +fsdp_text_encoder = False +compile_dit = False + +sampler_name = "Flow" +transformer_path = None +vae_path = None +lora_path = None + +sample_size = [512, 512] # H, W +prompt = "(masterpiece, best quality), 1 girl on the beach" +negative_prompt = " " +guidance_scale = 0.0 +seed = 42 +num_inference_steps = 9 +lora_weight = 0.55 +save_path = "samples/z-image-t2i-nocontrol" + +def _resolve_config_path(path: str) -> Optional[str]: + candidate = path if os.path.isabs(path) else os.path.join(repo_root, path) + return candidate if os.path.exists(candidate) else None + + +config_path = _resolve_config_path(config_path_default) +config = OmegaConf.load(config_path) if config_path else None + +extra_kwargs = {} +if config is not None and hasattr(config, "transformer_additional_kwargs"): + extra_kwargs = OmegaConf.to_container(config.transformer_additional_kwargs, resolve=True) + +transformer = ZImageTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + **({"transformer_additional_kwargs": extra_kwargs} if extra_kwargs else {}), +).to(weight_dtype).to(device) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +Chosen_Scheduler = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] + +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +height, width = sample_size +vae_scale_factor = 8 +vae_scale = vae_scale_factor * 2 + +if height % vae_scale != 0 or width % vae_scale != 0: + raise ValueError(f"Height/Width must be divisible by {vae_scale}") + +_guidance_scale = guidance_scale +_joint_attention_kwargs = None +_interrupt = False +_cfg_normalization = False +_cfg_truncation = 1.0 + +prompt_embeds = None +if isinstance(prompt, str): + batch_size = 1 +elif isinstance(prompt, list): + batch_size = len(prompt) +else: + batch_size = len(prompt_embeds) + +batch_size = 1 + +weight_dtype = text_encoder.dtype +num_channels_latents = 16 +inpaint_latent = None + +from diffusers.image_processor import VaeImageProcessor +image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor * 2) + +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) +vae.eval() +vae_config_shift_factor = getattr(vae.config, "shift_factor", 0.0) +vae_config_scaling_factor = getattr(vae.config, "scaling_factor", 1.0) + +# No control image path in this launcher +control_context_scale_tensor = torch.tensor([control_context_scale], device=device, dtype=torch.float32) + +onnx_transformer_enabled = use_transformer_onnx and os.path.exists(transformer_body_onnx_path) +if use_transformer_onnx and not onnx_transformer_enabled: + print(f"ONNX transformer requested but missing file, fallback to torch. body={transformer_body_onnx_path}") +else: + print(f"[DEBUG] ONNX providers: {onnx_providers}") + print(f"[DEBUG] transformer body onnx path: {transformer_body_onnx_path}") + +do_classifier_free_guidance = False +negative_prompt_embeds = None +max_sequence_length = 128 + +prompt_embeds, negative_prompt_embeds = encode_prompt( + prompt=prompt, + negative_prompt=negative_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + device=device, + max_sequence_length=max_sequence_length, +) + +num_images_per_prompt = 1 +latents = None + +latents = prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + torch.float32, + device, + torch.Generator(device=device).manual_seed(seed), + latents, +) + +if num_images_per_prompt > 1: + prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)] + if do_classifier_free_guidance and negative_prompt_embeds: + negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)] + +actual_batch_size = batch_size * num_images_per_prompt +image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2) + + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +mu = calculate_shift( + image_seq_len, + scheduler.config.get("base_image_seq_len", 256), + scheduler.config.get("max_image_seq_len", 4096), + scheduler.config.get("base_shift", 0.5), + scheduler.config.get("max_shift", 1.15), +) +scheduler.sigma_min = 0.0 +scheduler_kwargs = {"mu": mu} + + +import inspect + +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +timesteps, num_inference_steps = retrieve_timesteps( + scheduler, + num_inference_steps=num_inference_steps, + device=device, + **scheduler_kwargs, +) + +num_warmup_steps = max(len(timesteps) - num_inference_steps * scheduler.order, 0) + +for i, t in enumerate(timesteps): + timestep = t.expand(latents.shape[0]) + timestep_model_input = (1000 - timestep) / 1000 + + latent_model_input = latents.to(transformer.dtype) + prompt_embeds_tensor = _stack_prompt_embeddings(prompt_embeds) + + if not _printed_onnx_debug: + print(f"[DEBUG] runtime latent shape: {latent_model_input.unsqueeze(2).shape}, dtype={latent_model_input.dtype}") + print(f"[DEBUG] runtime prompt shape: {prompt_embeds_tensor.shape}, dtype={prompt_embeds_tensor.dtype}") + + if onnx_transformer_enabled: + onnx_latent = latent_model_input.unsqueeze(2).to(dtype=onnx_export_dtype) + onnx_prompt = prompt_embeds_tensor.to(dtype=onnx_export_dtype) + + if not _onnx_shape_compatible(transformer_body_onnx_path, onnx_providers, onnx_latent.shape, onnx_prompt.shape, verbose=True): + print("ONNX transformer 输入尺寸与当前推理不匹配，回退到 torch。") + onnx_transformer_enabled = False + else: + body_inputs = { + "latent_model_input": onnx_latent, + "timestep": timestep_model_input.to(dtype=torch.float32), + "prompt_embeds": onnx_prompt, + } + model_out = ort_inference(transformer_body_onnx_path, body_inputs, providers=onnx_providers)[0] + model_out_tensor = torch.from_numpy(model_out).to(device=device, dtype=onnx_export_dtype) + model_out_list = list(model_out_tensor) + if not onnx_transformer_enabled: + latent_model_input_list = list(latent_model_input.unsqueeze(2).unbind(dim=0)) + timestep_model_input = timestep_model_input.to(dtype=transformer.dtype) + + if device.type == "cuda": + with torch.autocast(device_type="cuda", dtype=transformer.dtype): + model_out_list = transformer( + latent_model_input_list, + timestep_model_input, + prompt_embeds, + patch_size=2, + f_patch_size=1, + )[0] + else: + model_out_list = transformer( + latent_model_input_list, + timestep_model_input, + prompt_embeds, + patch_size=2, + f_patch_size=1, + )[0] + + noise_pred = torch.stack([t.float() for t in model_out_list], dim=0) + noise_pred = noise_pred.squeeze(2) + noise_pred = -noise_pred + + latents = scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0] + assert latents.dtype == torch.float32 + + if not _printed_onnx_debug: + _printed_onnx_debug = True + +# Decode +output_type = "pil" +if output_type == "latent": + image = latents +else: + latents = latents.to(vae.dtype) + latents = (latents / vae.config.scaling_factor) + vae.config.shift_factor + + if os.path.exists(vae_decoder_onnx_path): + image = ort_inference( + vae_decoder_onnx_path, + {"latents": latents}, + )[0] + image = torch.from_numpy(image).to(device=device, dtype=vae.dtype) + else: + with module_to_device(vae, device): + image = vae.decode(latents, return_dict=False)[0] + image = image_processor.postprocess(image, output_type=output_type) + +sample = image + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +save_results() +logger.info(f"Saved image to {save_path}") diff --git a/VideoX-Fun/examples/z_image_fun/launcher_onnx.py b/VideoX-Fun/examples/z_image_fun/launcher_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..532670d367157f759adfe4761fa41594e153b1d2 --- /dev/null +++ b/VideoX-Fun/examples/z_image_fun/launcher_onnx.py @@ -0,0 +1,530 @@ +#!/usr/bin/env python3 +"""使用 onnxruntime 推理链路 (transformer + VAE decoder)。""" +from __future__ import annotations + +import argparse +import json +import os +import random +import sys +from contextlib import contextmanager +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Tuple, Union + +import numpy as np +import onnxruntime as ort +import torch +from tqdm import tqdm +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.image_processor import VaeImageProcessor +from diffusers.utils.torch_utils import randn_tensor +from loguru import logger +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [ + os.path.dirname(current_file_path), + os.path.dirname(os.path.dirname(current_file_path)), + os.path.dirname(os.path.dirname(os.path.dirname(current_file_path))), +] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +SCRIPT_DIR = Path(__file__).resolve().parent +REPO_ROOT = SCRIPT_DIR.parents[2] +if REPO_ROOT.as_posix() not in sys.path: + sys.path.insert(0, REPO_ROOT.as_posix()) + +from videox_fun.models import AutoTokenizer, Qwen3ForCausalLM +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.utils import get_image_latent +from scripts.split_quant_onnx_by_subconfigs import ( + SubGraphSpec, + sanitize, +) + +# ----------------------------------------------------------------------------- +# 模型与资源路径 +# ----------------------------------------------------------------------------- +MODEL_NAME = "models/Diffusion_Transformer/Z-Image-Turbo/" +CONFIG_PATH = REPO_ROOT / "VideoX-Fun" / "config" / "z_image" / "z_image.yaml" +TRANSFORMER_CONFIG_PATH = REPO_ROOT / "VideoX-Fun" / "pulsar2_configs" / "transformers_subgraph.json" +TRANSFORMER_ONNX_DIR = REPO_ROOT / "VideoX-Fun" / "compiled_subgraph_from_onnx" / "frontend" +VAE_DECODER_ONNX = REPO_ROOT / "VideoX-Fun" / "compiled_output_vae_decoder" / "frontend" / "optimized.onnx" +SAVE_DIR = REPO_ROOT / "VideoX-Fun" / "samples" / "z-image-t2i-onnx" + +# ----------------------------------------------------------------------------- +# 运行配置 +# ----------------------------------------------------------------------------- +DEFAULT_PROMPTS = [ + "(masterpiece, best quality) solo female on a tropical beach, golden hour rim light, cinematic grading", + "nighttime cyberpunk boulevard, neon reflections on wet asphalt, volumetric fog, wide shot", + "sunrise over alpine mountains, low clouds in valleys, god rays, ultra-detailed landscape", + "modern minimal living room, soft natural light, Scandinavian design, high-resolution interior render", + "classical oil painting of a renaissance noblewoman, chiaroscuro lighting, rich textures", + "macro photography of a dewdrop on a leaf, extreme detail, shallow depth of field", + "futuristic sports car parked under neon lights, glossy paint, cinematic 35mm look", + "ancient library with towering bookshelves, warm candlelight, dust motes in air", + "portrait of an astronaut in full suit, visor reflection showing earth, studio lighting", + "stormy sea with a lone lighthouse, crashing waves, dramatic clouds, long exposure feel", + "cybernetic samurai standing in rain, backlit silhouette, moody blue-orange palette", + "lush rainforest waterfall, soft mist, saturated greens, wide-angle composition", + "product shot of a smartwatch on marble, softbox lighting, crisp shadows, advertisement style", + "architectural exterior of a glass skyscraper at dusk, warm interior lights, reflections", + "vintage film photograph of a 1950s diner at night, grain and halation, neon signage", + "hyperrealistic bowl of ramen, steam rising, glossy broth, detailed toppings", + "fantasy castle on a floating island, waterfalls falling into clouds, sunset lighting", + "high-fashion editorial portrait, dramatic chiaroscuro, sharp focus on eyes", + "aerial view of winding river through autumn forest, golden and crimson leaves", + "studio shot of running shoes mid-air, motion blur trails, vibrant background gradient", + "noir city alley in the 1940s, hard shadows, rain-slick pavement, moody atmosphere", + "desert caravan at twilight, silhouettes of camels, soft purple sky, cinematic scope", + "close-up of a mechanical watch movement, intricate gears, metallic reflections", + "bioluminescent underwater reef, glowing corals, schools of fish, deep blue tones", + "portrait of an elderly man with weathered face, soft window light, fine skin detail", + "snowy village at night, warm cabin lights, smoke from chimneys, peaceful mood", + "futuristic data center aisle, cool cyan lighting, depth and symmetry", + "oil painting of a bowl of fruit in Dutch masters style, rich textures, dramatic lighting", + "sunlit meadow with wildflowers, shallow depth of field, pastel color palette", + "sci-fi corridor with volumetric light shafts, pristine white surfaces, wide lens", + "luxury wristwatch on black velvet, high contrast, advertisement macro shot", + "medieval marketplace at dawn, merchants setting up, soft warm light, lively details", + "((masterpiece,best quality))1 young beautiful girl,ultra detailed,official art,unity 8k wallpaper,masterpiece, best quality, official art, extremely detailed CG unity 8k wallpaper, highly detailed, 1 girl, aqua eyes, light smile, ((grey hair)), hair flower, bracelet, choker, ribbon, JK, look at viewer, on the beach, in summer,", +] +idx = random.randint(0, len(DEFAULT_PROMPTS) - 1) +PROMPT = DEFAULT_PROMPTS[idx] +NEG_PROMPT = " " +GUIDANCE_SCALE = 0.0 +SEED = 42 +HEIGHT, WIDTH = 512, 512 +NUM_INFERENCE_STEPS = 9 +NUM_CHANNELS_LATENTS = 16 +VAE_SCALE_FACTOR = 8 +PATCH_SIZE = 2 +FPATCH_SIZE = 1 +MAX_SEQ_LEN = 128 +VAE_SCALING_FACTOR = 0.3611 +VAE_SHIFT_FACTOR = 0.1159 + +SAMPLER_MAP = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +} +SAMPLER_NAME = "Flow" + +# ----------------------------------------------------------------------------- +# 工具函数 +# ----------------------------------------------------------------------------- + +def _infer_module_device(module: torch.nn.Module) -> torch.device: + param = next(module.parameters(), None) + if param is not None: + return param.device + buffer = next(module.buffers(), None) + if buffer is not None: + return buffer.device + return torch.device("cpu") + + +@contextmanager +def module_to_device(module: torch.nn.Module, target_device: torch.device): + if module is None: + yield module + return + original_device = _infer_module_device(module) + target_device = target_device or original_device + needs_move = original_device != target_device + moved_to_cuda = needs_move and target_device.type == "cuda" + if needs_move: + module.to(target_device) + try: + yield module + finally: + if needs_move: + module.to(original_device) + if moved_to_cuda and torch.cuda.is_available(): + cache_device = target_device.index or torch.cuda.current_device() + with torch.cuda.device(cache_device): + torch.cuda.empty_cache() + + +def _encode_prompt( + tokenizer: AutoTokenizer, + text_encoder: Qwen3ForCausalLM, + prompt: Union[str, List[str]], + device: torch.device, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + max_sequence_length: int = 512, +) -> List[torch.FloatTensor]: + if prompt_embeds is not None: + return prompt_embeds + prompts = [prompt] if isinstance(prompt, str) else list(prompt) + for idx, item in enumerate(prompts): + messages = [{"role": "user", "content": item}] + prompts[idx] = tokenizer.apply_chat_template( + messages, tokenize=False, add_generation_prompt=True, enable_thinking=True + ) + text_inputs = tokenizer( + prompts, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + with module_to_device(text_encoder, device): + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] + return [prompt_embeds[i] for i in range(len(prompt_embeds))] + + +def encode_prompt( + tokenizer: AutoTokenizer, + text_encoder: Qwen3ForCausalLM, + prompt: Union[str, List[str]], + device: torch.device, + do_classifier_free_guidance: bool, + negative_prompt: Optional[Union[str, List[str]]], + max_sequence_length: int, +) -> Tuple[List[torch.FloatTensor], List[torch.FloatTensor]]: + prompt_embeds = _encode_prompt( + tokenizer, text_encoder, prompt, device, None, max_sequence_length + ) + negative_embeds: List[torch.FloatTensor] = [] + if do_classifier_free_guidance: + neg = negative_prompt or "" + negative_list = [neg] if isinstance(neg, str) else list(neg) + negative_embeds = _encode_prompt( + tokenizer, text_encoder, negative_list, device, None, max_sequence_length + ) + return prompt_embeds, negative_embeds + + +def _stack_prompt_embeddings(prompt_embeds_input): + if isinstance(prompt_embeds_input, list): + return torch.stack(prompt_embeds_input, dim=0) + return prompt_embeds_input + + +def prepare_latents( + batch_size: int, + num_channels_latents: int, + height: int, + width: int, + dtype: torch.dtype, + device: torch.device, + generator: torch.Generator, +) -> torch.FloatTensor: + height = 2 * (int(height) // (VAE_SCALE_FACTOR * 2)) + width = 2 * (int(width) // (VAE_SCALE_FACTOR * 2)) + shape = (batch_size, num_channels_latents, height, width) + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + return latents + + +def calculate_shift( + image_seq_len: int, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +) -> float: + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + return image_seq_len * m + b + + +def retrieve_timesteps( + scheduler, + num_inference_steps: int, + device: torch.device, + **kwargs, +): + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + return scheduler.timesteps + + +def resolve_providers(use_cuda: bool) -> List[str]: + available = ort.get_available_providers() + if use_cuda and "CUDAExecutionProvider" in available: + return ["CUDAExecutionProvider", "CPUExecutionProvider"] + return ["CPUExecutionProvider"] + + +# ----------------------------------------------------------------------------- +# ONNX transformer 子图执行器 +# ----------------------------------------------------------------------------- + + +class OnnxSplitTransformer: + def __init__(self, config_path: Path, model_dir: Path, providers: List[str]): + self.config_path = config_path + self.model_dir = model_dir + self.providers = providers + self.specs = self._load_specs() + self.sessions, self.default_final_output = self._load_sessions_and_outputs() + + def _load_specs(self) -> List[SubGraphSpec]: + with self.config_path.open("r", encoding="utf-8") as f: + config = json.load(f) + sub_configs = config.get("compiler", {}).get("sub_configs", []) + if not sub_configs: + raise ValueError("配置文件缺少 compiler.sub_configs") + + specs: List[SubGraphSpec] = [] + for idx, entry in enumerate(sub_configs): + start = [name for name in entry.get("start_tensor_names", []) if name] + end = [name for name in entry.get("end_tensor_names", []) if name] + if not start or not end: + raise ValueError(f"sub_config[{idx}] 定义不完整") + specs.append( + SubGraphSpec( + label=f"cfg_{idx:02d}", + start=start, + end=end, + node_names=set(), + source="config", + ) + ) + return specs + + def _expected_path(self, spec: SubGraphSpec) -> Path: + head = sanitize(spec.start[0]) if spec.start else "const" + tail = sanitize(spec.end[0]) if spec.end else "out" + filename = f"{spec.label}_{head}_to_{tail}_{spec.source}.onnx" + path = self.model_dir / filename + if not path.exists(): + raise FileNotFoundError(f"缺少 ONNX 模型: {path}") + return path + + def _load_sessions_and_outputs(self) -> Tuple[List[Tuple[SubGraphSpec, ort.InferenceSession]], str]: + sessions: List[Tuple[SubGraphSpec, ort.InferenceSession]] = [] + # 先加载 config 中的子图 + for spec in self.specs: + path = self._expected_path(spec) + sess = ort.InferenceSession(path.as_posix(), providers=self.providers) + sessions.append((spec, sess)) + + # 再尝试加载额外 auto_* 子图，输入/输出直接从 onnx 读取 + auto_specs: List[SubGraphSpec] = [] + for path in sorted(self.model_dir.glob("auto_*.onnx")): + sess = ort.InferenceSession(path.as_posix(), providers=self.providers) + inputs = [i.name for i in sess.get_inputs()] + outputs = [o.name for o in sess.get_outputs()] + auto_spec = SubGraphSpec( + label=path.stem, + start=inputs, + end=outputs, + node_names=set(), + source="auto", + ) + auto_specs.append(auto_spec) + sessions.append((auto_spec, sess)) + + # 默认输出：优先使用最后一个 auto 子图的首个输出，否则用最后一个 config 子图的首个输出 + if auto_specs: + default_output = auto_specs[-1].end[0] + else: + default_output = self.specs[-1].end[0] + + # 将 auto_specs 也存进 self.specs，方便调度错误提示 + self.specs = self.specs + auto_specs + return sessions, default_output + + def __call__( + self, + latent_np: np.ndarray, + prompt_np: np.ndarray, + timestep_np: np.ndarray, + final_output_name: Optional[str] = None, + ) -> np.ndarray: + tensor_store: Dict[str, np.ndarray] = { + "latent_model_input": latent_np, + "prompt_embeds": prompt_np, + "timestep": timestep_np, + } + executed = set() + target = final_output_name or self.default_final_output + + # 动态调度：当所有输入就绪才运行对应子图 + while target not in tensor_store: + progressed = False + for spec, session in self.sessions: + if spec.label in executed: + continue + if not all(name in tensor_store for name in spec.start): + continue + inputs = {name: tensor_store[name] for name in spec.start} + outputs = session.run(spec.end, inputs) + for out_name, value in zip(spec.end, outputs): + tensor_store[out_name] = value + executed.add(spec.label) + progressed = True + if not progressed: + missing = [ + (spec.label, [name for name in spec.start if name not in tensor_store]) + for spec, _ in self.sessions + if spec.label not in executed + ] + raise RuntimeError( + f"子图调度中断，缺少输入: {missing}; 当前可用: {list(tensor_store.keys())}" + ) + + return tensor_store[target] + + +# ----------------------------------------------------------------------------- +# 主流程 +# ----------------------------------------------------------------------------- + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="ONNXRuntime 推理 (transformer + VAE)") + parser.add_argument("--prompt", type=str, default=None, help="正向提示词，不填则使用预置随机样本") + parser.add_argument("--negative-prompt", type=str, default=NEG_PROMPT, help="反向提示词") + parser.add_argument("--steps", type=int, default=NUM_INFERENCE_STEPS, help="迭代步数") + parser.add_argument("--height", type=int, default=HEIGHT, help="生成高度，需被 16 整除") + parser.add_argument("--width", type=int, default=WIDTH, help="生成宽度，需被 16 整除") + parser.add_argument("--seed", type=int, default=SEED, help="随机种子") + parser.add_argument("--sampler", type=str, choices=list(SAMPLER_MAP.keys()), default=SAMPLER_NAME, help="采样器") + parser.add_argument("--max-seq-len", type=int, default=MAX_SEQ_LEN, help="最大文本长度") + parser.add_argument("--save-dir", type=str, default=str(SAVE_DIR), help="结果输出目录") + parser.add_argument("--transformer-config", type=str, default=str(TRANSFORMER_CONFIG_PATH), help="子图配置 json") + parser.add_argument("--transformer-subgraph-dir", type=str, default=str(TRANSFORMER_ONNX_DIR), help="子图 onnx 目录") + parser.add_argument("--vae-onnx", type=str, default=str(VAE_DECODER_ONNX), help="VAE decoder onnx 路径") + parser.add_argument("--use-cuda-provider", action="store_true", help="优先使用 CUDAExecutionProvider") + parser.add_argument("--save-decoder-input", action="store_true", help="是否保存 decoder 输入 npy") + parser.add_argument("--final-output-name", type=str, default=None, help="指定最终输出 tensor 名称，默认为最后一个子图的第一个输出") + parser.add_argument("--no-progress", action="store_true", help="关闭进度条输出") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + prompt_text = args.prompt if args.prompt is not None else PROMPT + logger.info(f"使用的 prompt: {prompt_text}") + + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + torch.set_grad_enabled(False) + + tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, subfolder="tokenizer") + text_encoder = Qwen3ForCausalLM.from_pretrained( + MODEL_NAME, + subfolder="text_encoder", + torch_dtype=torch.bfloat16, + low_cpu_mem_usage=True, + ) + text_encoder.eval() + + scheduler_cls = SAMPLER_MAP[args.sampler] + scheduler = scheduler_cls.from_pretrained(MODEL_NAME, subfolder="scheduler") + + image_processor = VaeImageProcessor(vae_scale_factor=VAE_SCALE_FACTOR * 2) + + prompt_embeds, _ = encode_prompt( + tokenizer, + text_encoder, + prompt_text, + device, + do_classifier_free_guidance=False, + negative_prompt=args.negative_prompt, + max_sequence_length=args.max_seq_len, + ) + + latents = prepare_latents( + batch_size=1, + num_channels_latents=NUM_CHANNELS_LATENTS, + height=args.height, + width=args.width, + dtype=torch.float32, + device=device, + generator=torch.Generator(device=device).manual_seed(args.seed), + ) + + image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2) + mu = calculate_shift( + image_seq_len, + scheduler.config.get("base_image_seq_len", 256), + scheduler.config.get("max_image_seq_len", 4096), + scheduler.config.get("base_shift", 0.5), + scheduler.config.get("max_shift", 1.15), + ) + timesteps = retrieve_timesteps(scheduler, args.steps, device=device, mu=mu) + + providers = resolve_providers(args.use_cuda_provider) + transformer_runner = OnnxSplitTransformer( + Path(args.transformer_config), Path(args.transformer_subgraph_dir), providers + ) + final_output_name = args.final_output_name or transformer_runner.default_final_output + + prompt_embeds_tensor = _stack_prompt_embeddings(prompt_embeds) + + iterator = timesteps if args.no_progress else tqdm(timesteps, desc="Denoising", dynamic_ncols=True) + + for t in iterator: + timestep = t.expand(latents.shape[0]) + timestep_model_input = (1000 - timestep) / 1000 + + latent_model_input = latents.to(torch.float32) + latent_np = latent_model_input.unsqueeze(2).to(dtype=torch.float32).cpu().numpy() + prompt_np = prompt_embeds_tensor.to(dtype=torch.float32).cpu().numpy() + timestep_np = timestep_model_input.to(dtype=torch.float32).cpu().numpy() + + model_out = transformer_runner(latent_np, prompt_np, timestep_np, final_output_name) + if model_out.ndim == 5 and model_out.shape[2] == 1: + model_out = np.squeeze(model_out, axis=2) + model_out_tensor = torch.from_numpy(model_out).to(device=device, dtype=torch.float32) + if model_out_tensor.dim() == 5 and model_out_tensor.size(2) == 1: + model_out_tensor = model_out_tensor.squeeze(2) + noise_pred = -model_out_tensor + + latents = scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + latents = (latents / VAE_SCALING_FACTOR) + VAE_SHIFT_FACTOR + decoder_input = latents.to(dtype=torch.float32).cpu().numpy() + + if args.save_decoder_input: + SAVE_DIR_PATH = Path(args.save_dir) + SAVE_DIR_PATH.mkdir(parents=True, exist_ok=True) + np.save(SAVE_DIR_PATH / "decoder_input.npy", decoder_input) + logger.info("已保存 decoder 输入为 npy") + + if decoder_input.ndim == 5 and decoder_input.shape[2] == 1: + decoder_input = np.squeeze(decoder_input, axis=2) + + vae_session = ort.InferenceSession(Path(args.vae_onnx).as_posix(), providers=providers) + image = vae_session.run(None, {"latents": decoder_input})[0] + + image = torch.from_numpy(image).to(device=device, dtype=torch.float32) + image = image_processor.postprocess(image, output_type="pil") + + save_dir = Path(args.save_dir) + save_dir.mkdir(parents=True, exist_ok=True) + target_path = save_dir / f"z_image_onnx_{idx}.png" + image[0].save(target_path) + logger.info(f"ONNXRuntime 推理完成，结果保存到 {target_path}") + + +if __name__ == "__main__": + """ + # 512x512 生成示例命令： + python examples/z_image_fun/launcher_onnx.py \ + --transformer-config pulsar2_configs/transformers_subgraph.json \ + --transformer-subgraph-dir transformers_body_only_split_onnx --vae-onnx onnx-models/vae_decoder_simp_slim.onnx + + # 1728x992 生成示例命令: + python examples/z_image_fun/launcher_onnx.py \ + --transformer-config pulsar2_configs/transformers_subgraph_1728x992.json \ + --transformer-subgraph-dir transformers_body_only_1728_992_split_onnx \ + --vae-onnx onnx-models-1728x992/vae_decoder_simp_slim.onnx \ + --max-seq-len 256 \ + --height 1728 --width 992 + """ + main() diff --git a/VideoX-Fun/examples/z_image_fun/predict_t2i_control.py b/VideoX-Fun/examples/z_image_fun/predict_t2i_control.py new file mode 100644 index 0000000000000000000000000000000000000000..788bd2e45cc89405237de91cbde0ab7e4e43a6b1 --- /dev/null +++ b/VideoX-Fun/examples/z_image_fun/predict_t2i_control.py @@ -0,0 +1,228 @@ +import os +import sys + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from omegaconf import OmegaConf +from PIL import Image + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + Qwen3ForCausalLM, ZImageControlTransformer2DModel) +from videox_fun.models.cache_utils import get_teacache_coefficients +from videox_fun.pipeline import ZImageControlPipeline +from videox_fun.utils.fm_solvers import FlowDPMSolverMultistepScheduler +from videox_fun.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from videox_fun.utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper) +from videox_fun.utils.lora_utils import merge_lora, unmerge_lora +from videox_fun.utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, get_image, + get_video_to_video_latent, + save_videos_grid) + +# GPU memory mode, which can be chosen in [model_full_load, model_full_load_and_qfloat8, model_cpu_offload, model_cpu_offload_and_qfloat8, sequential_cpu_offload]. +# model_full_load means that the entire model will be moved to the GPU. +# +# model_full_load_and_qfloat8 means that the entire model will be moved to the GPU, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# model_cpu_offload means that the entire model will be moved to the CPU after use, which can save some GPU memory. +# +# model_cpu_offload_and_qfloat8 indicates that the entire model will be moved to the CPU after use, +# and the transformer model has been quantized to float8, which can save more GPU memory. +# +# sequential_cpu_offload means that each layer of the model will be moved to the CPU after use, +# resulting in slower speeds but saving a large amount of GPU memory. +GPU_memory_mode = "model_cpu_offload" +# Multi GPUs config +# Please ensure that the product of ulysses_degree and ring_degree equals the number of GPUs used. +# For example, if you are using 8 GPUs, you can set ulysses_degree = 2 and ring_degree = 4. +# If you are using 1 GPU, you can set ulysses_degree = 1 and ring_degree = 1. +ulysses_degree = 1 +ring_degree = 1 +# Use FSDP to save more GPU memory in multi gpus. +fsdp_dit = False +fsdp_text_encoder = False +# Compile will give a speedup in fixed resolution and need a little GPU memory. +# The compile_dit is not compatible with the fsdp_dit and sequential_cpu_offload. +compile_dit = False + +# Config and model path +config_path = "config/z_image/z_image_control.yaml" +# model path +model_name = "models/Diffusion_Transformer/Z-Image-Turbo/" + +# Choose the sampler in "Flow", "Flow_Unipc", "Flow_DPM++" +sampler_name = "Flow" + +# Load pretrained model if need +transformer_path = "models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union.safetensors" +vae_path = None +lora_path = None + +# Other params +sample_size = [1728, 992] + +# Use torch.float16 if GPU does not support torch.bfloat16 +# ome graphics cards, such as v100, 2080ti, do not support torch.bfloat16 +weight_dtype = torch.bfloat16 +control_image = "asset/pose.jpg" +control_context_scale = 0.75 + +# 使用更长的neg prompt如"模糊，突变，变形，失真，画面暗，文本字幕，画面固定，连环画，漫画，线稿，没有主体。"，可以增加稳定性 +# 在neg prompt中添加"安静，固定"等词语可以增加动态性。 +prompt = "一位年轻女子站在阳光明媚的海岸线上，白裙在轻拂的海风中微微飘动。她拥有一头鲜艳的紫色长发，在风中轻盈舞动，发间系着一个精致的黑色蝴蝶结，与身后柔和的蔚蓝天空形成鲜明对比。她面容清秀，眉目精致，透着一股甜美的青春气息；神情柔和，略带羞涩，目光静静地凝望着远方的地平线，双手自然交叠于身前，仿佛沉浸在思绪之中。在她身后，是辽阔无垠、波光粼粼的大海，阳光洒在海面上，映出温暖的金色光晕。" +negative_prompt = " " +guidance_scale = 0.00 +seed = 43 +num_inference_steps = 9 +lora_weight = 0.55 +save_path = "samples/z-image-t2i-control" + +device = set_multi_gpus_devices(ulysses_degree, ring_degree) +config = OmegaConf.load(config_path) + +transformer = ZImageControlTransformer2DModel.from_pretrained( + model_name, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), +).to(weight_dtype) + +if transformer_path is not None: + print(f"From checkpoint: {transformer_path}") + if transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(transformer_path) + else: + state_dict = torch.load(transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get Vae +vae = AutoencoderKL.from_pretrained( + model_name, + subfolder="vae" +).to(weight_dtype) + +if vae_path is not None: + print(f"From checkpoint: {vae_path}") + if vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(vae_path) + else: + state_dict = torch.load(vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + +# Get tokenizer and text_encoder +tokenizer = AutoTokenizer.from_pretrained( + model_name, subfolder="tokenizer" +) +text_encoder = Qwen3ForCausalLM.from_pretrained( + model_name, subfolder="text_encoder", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, +) + +# Get Scheduler +Chosen_Scheduler = scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +}[sampler_name] +scheduler = Chosen_Scheduler.from_pretrained( + model_name, + subfolder="scheduler" +) + +pipeline = ZImageControlPipeline( + vae=vae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, +) + +if ulysses_degree > 1 or ring_degree > 1: + from functools import partial + transformer.enable_multi_gpus_inference() + if fsdp_dit: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=list(transformer.transformer_blocks) + list(transformer.single_transformer_blocks)) + pipeline.transformer = shard_fn(pipeline.transformer) + print("Add FSDP DIT") + if fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers, ignored_modules=[text_encoder.language_model.embed_tokens], transformer_layer_cls_to_wrap=["MistralDecoderLayer", "PixtralTransformer"]) + text_encoder = shard_fn(text_encoder) + print("Add FSDP TEXT ENCODER") + +if compile_dit: + for i in range(len(pipeline.transformer.transformer_blocks)): + pipeline.transformer.transformer_blocks[i] = torch.compile(pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + +if GPU_memory_mode == "sequential_cpu_offload": + pipeline.enable_sequential_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_cpu_offload": + pipeline.enable_model_cpu_offload(device=device) +elif GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(transformer, exclude_module_name=["img_in", "txt_in", "timestep"], device=device) + convert_weight_dtype_wrapper(transformer, weight_dtype) + pipeline.to(device=device) +else: + pipeline.to(device=device) + +generator = torch.Generator(device=device).manual_seed(seed) + +if lora_path is not None: + pipeline = merge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +with torch.no_grad(): + if control_image is not None: + control_image = get_image_latent(control_image, sample_size=sample_size)[:, :, 0] + + sample = pipeline( + prompt = prompt, + negative_prompt = negative_prompt, + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = guidance_scale, + control_image = control_image, + num_inference_steps = num_inference_steps, + control_context_scale = control_context_scale, + ).images + +if lora_path is not None: + pipeline = unmerge_lora(pipeline, lora_path, lora_weight, device=device, dtype=weight_dtype) + +def save_results(): + if not os.path.exists(save_path): + os.makedirs(save_path, exist_ok=True) + + index = len([path for path in os.listdir(save_path)]) + 1 + prefix = str(index).zfill(8) + video_path = os.path.join(save_path, prefix + ".png") + image = sample[0] + image.save(video_path) + +if ulysses_degree * ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_results() +else: + save_results() \ No newline at end of file diff --git a/VideoX-Fun/infer_axmodel.py b/VideoX-Fun/infer_axmodel.py new file mode 100644 index 0000000000000000000000000000000000000000..0e2939123259cabe2fbce0780ffaa42050fa8bf3 --- /dev/null +++ b/VideoX-Fun/infer_axmodel.py @@ -0,0 +1,12 @@ +from axengine import InferenceSession +import numpy as np + + +session = InferenceSession("./compiled_slice_quant_onnx/cfg_00_timestep_to_model_t_embedder_mlp_mlp_2_Gemm_output_0_config.axmodel") + +input_feed = { + "timestep": np.array([1.0], dtype=np.float32), +} +output = session.run(None, input_feed)[0] + +print(output.shape) \ No newline at end of file diff --git a/VideoX-Fun/infer_split_axmodels.py b/VideoX-Fun/infer_split_axmodels.py new file mode 100644 index 0000000000000000000000000000000000000000..c796f21dec5937431559cc3768102e5615d11a3f --- /dev/null +++ b/VideoX-Fun/infer_split_axmodels.py @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""串行执行切分后的 AXModel 子图以复现完整推理路径。""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +import types +from pathlib import Path +from typing import Dict, List + +import numpy as np +import onnx +from axengine import InferenceSession + +PROJECT_ROOT = Path(__file__).resolve().parent +SCRIPTS_DIR = PROJECT_ROOT / "scripts" +if SCRIPTS_DIR.as_posix() not in sys.path: + sys.path.insert(0, SCRIPTS_DIR.as_posix()) + +from split_onnx_by_subconfigs import ( # type: ignore + SubGraphSpec, + build_graph_index, + derive_interface, + ordered_specs, + sanitize, + trace_nodes_between, + untouched_components, +) + + +def _ensure_numpy_core_alias() -> None: + """兼容 numpy>=2 保存、numpy<2 读取时的 pickle 模块路径差异。""" + if "numpy._core" in sys.modules: + return + try: + core_module = sys.modules.get("numpy.core") or __import__("numpy.core") + except Exception: # pragma: no cover - 极端环境 + return + alias = types.ModuleType("numpy._core") + alias.__dict__.update(core_module.__dict__) + sys.modules["numpy._core"] = alias + submods = { + "multiarray": getattr(core_module, "multiarray", None), + "umath": getattr(core_module, "umath", None), + "numerictypes": getattr(core_module, "numerictypes", None), + "_multiarray_umath": getattr(core_module, "_multiarray_umath", None), + } + for name, module in submods.items(): + if module is not None: + sys.modules[f"numpy._core.{name}"] = module + + +def load_specs(config_path: Path, onnx_path: Path) -> List[SubGraphSpec]: + with config_path.open("r", encoding="utf-8") as f: + config = json.load(f) + sub_configs = config.get("compiler", {}).get("sub_configs", []) + if not sub_configs: + raise ValueError("配置文件中缺少 compiler.sub_configs") + + model = onnx.load(onnx_path.as_posix()) + index = build_graph_index(model) + + specs: List[SubGraphSpec] = [] + covered_nodes = set() + for idx, entry in enumerate(sub_configs): + start = [name for name in entry.get("start_tensor_names", []) if name] + end = [name for name in entry.get("end_tensor_names", []) if name] + if not start or not end: + raise ValueError(f"sub_config[{idx}] 缺少 tensor 名称") + spec = SubGraphSpec( + label=f"cfg_{idx:02d}", + start=start, + end=end, + node_names=set(), + source="config", + ) + nodes = trace_nodes_between(spec, index) + spec.node_names = nodes + covered_nodes.update(nodes) + specs.append(spec) + + leftovers = untouched_components(index.node_order, covered_nodes, index) + for idx, component in enumerate(leftovers): + start, end = derive_interface(component, index) + if not end: + continue + spec = SubGraphSpec( + label=f"auto_{idx:02d}", + start=start, + end=end, + node_names=component, + source="auto", + ) + specs.append(spec) + + return ordered_specs(specs, index) + + +def expected_model_path(spec: SubGraphSpec, model_dir: Path) -> Path: + head = sanitize(spec.start[0]) if spec.start else "const" + tail = sanitize(spec.end[0]) if spec.end else "out" + filename = f"{spec.label}_{head}_to_{tail}_{spec.source}.axmodel" + path = model_dir / filename + if not path.exists(): + raise FileNotFoundError(f"未找到模型文件: {path}") + return path + + +def run_pipeline( + specs: List[SubGraphSpec], + model_dir: Path, + feed_dict: Dict[str, np.ndarray], +) -> Dict[str, np.ndarray]: + tensor_store: Dict[str, np.ndarray] = dict(feed_dict) + for spec in specs: + model_path = expected_model_path(spec, model_dir) + session = InferenceSession(model_path.as_posix()) + inputs = {} + missing = [] + for name in spec.start: + value = tensor_store.get(name) + if value is None: + missing.append(name) + else: + inputs[name] = value + if missing: + raise KeyError(f"模型 {model_path.name} 缺少输入张量: {missing}") + results = session.run(spec.end, inputs) + if not isinstance(results, (list, tuple)): + results = [results] + for out_name, value in zip(spec.end, results): + tensor_store[out_name] = value + shapes = {out_name: tuple(value.shape) for out_name, value in zip(spec.end, results)} + logging.info("完成 %s: %s", model_path.name, shapes) + return tensor_store + + +def load_input_file(path: Path, single_name: str | None) -> Dict[str, np.ndarray]: + suffix = path.suffix.lower() + if suffix == ".npz": + data = np.load(path, allow_pickle=False) + return {key: np.asarray(data[key]) for key in data.files} + + if suffix == ".npy": + _ensure_numpy_core_alias() + blob = np.load(path, allow_pickle=True) + if isinstance(blob, np.ndarray) and blob.dtype == object: + obj = blob.item() + if isinstance(obj, dict): + return {str(k): np.asarray(v) for k, v in obj.items()} + if single_name is None: + raise ValueError("单一 .npy 输入需要通过 --single-input-name 指定张量名") + return {single_name: np.asarray(obj)} + if single_name is None: + raise ValueError("单一 .npy 输入需要通过 --single-input-name 指定张量名") + return {single_name: np.asarray(blob)} + + raise ValueError(f"暂不支持的输入文件格式: {path}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="串行执行切分后的 AXModel") + parser.add_argument("--onnx", required=True, type=Path, help="原始未切分 ONNX") + parser.add_argument("--config", required=True, type=Path, help="sub_config 配置文件") + parser.add_argument("--model-dir", required=True, type=Path, help=".axmodel 所在目录") + parser.add_argument( + "--input-file", + required=True, + type=Path, + help="包含原始输入张量的 npz 或 npy 文件", + ) + parser.add_argument( + "--single-input-name", + type=str, + help="当 --input-file 是单张量 .npy 文件时指定对应的输入名", + ) + parser.add_argument("--dump-npz", type=Path, help="可选，将最终输出保存为 npz") + parser.add_argument("--log", default="INFO", help="日志等级") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + logging.basicConfig(level=getattr(logging, args.log.upper(), logging.INFO)) + + specs = load_specs(args.config, args.onnx) + feed = load_input_file(args.input_file, args.single_input_name) + store = run_pipeline(specs, args.model_dir, feed) + + final_outputs = {name: store[name] for name in specs[-1].end} + for name, value in final_outputs.items(): + logging.info("最终输出 %s: shape=%s", name, value.shape) + + if args.dump_npz: + np.savez(args.dump_npz, **final_outputs) + logging.info("已保存输出到 %s", args.dump_npz) + + +if __name__ == "__main__": + """ + python3 infer_split_axmodels.py \ + --onnx onnx-models/transformer.onnx \ + --config pulsar2_configs/transformers_subgraph.json \ + --model-dir compiled_slice_quant_onnx \ + --input-file onnx-calibration-no-controlnet/transformer_inputs_prompt000_step00.npy \ + --single-input-name timestep + """ + main() diff --git a/VideoX-Fun/infer_split_axmodels.py.save b/VideoX-Fun/infer_split_axmodels.py.save new file mode 100644 index 0000000000000000000000000000000000000000..c796f21dec5937431559cc3768102e5615d11a3f --- /dev/null +++ b/VideoX-Fun/infer_split_axmodels.py.save @@ -0,0 +1,210 @@ +#!/usr/bin/env python3 +"""串行执行切分后的 AXModel 子图以复现完整推理路径。""" +from __future__ import annotations + +import argparse +import json +import logging +import sys +import types +from pathlib import Path +from typing import Dict, List + +import numpy as np +import onnx +from axengine import InferenceSession + +PROJECT_ROOT = Path(__file__).resolve().parent +SCRIPTS_DIR = PROJECT_ROOT / "scripts" +if SCRIPTS_DIR.as_posix() not in sys.path: + sys.path.insert(0, SCRIPTS_DIR.as_posix()) + +from split_onnx_by_subconfigs import ( # type: ignore + SubGraphSpec, + build_graph_index, + derive_interface, + ordered_specs, + sanitize, + trace_nodes_between, + untouched_components, +) + + +def _ensure_numpy_core_alias() -> None: + """兼容 numpy>=2 保存、numpy<2 读取时的 pickle 模块路径差异。""" + if "numpy._core" in sys.modules: + return + try: + core_module = sys.modules.get("numpy.core") or __import__("numpy.core") + except Exception: # pragma: no cover - 极端环境 + return + alias = types.ModuleType("numpy._core") + alias.__dict__.update(core_module.__dict__) + sys.modules["numpy._core"] = alias + submods = { + "multiarray": getattr(core_module, "multiarray", None), + "umath": getattr(core_module, "umath", None), + "numerictypes": getattr(core_module, "numerictypes", None), + "_multiarray_umath": getattr(core_module, "_multiarray_umath", None), + } + for name, module in submods.items(): + if module is not None: + sys.modules[f"numpy._core.{name}"] = module + + +def load_specs(config_path: Path, onnx_path: Path) -> List[SubGraphSpec]: + with config_path.open("r", encoding="utf-8") as f: + config = json.load(f) + sub_configs = config.get("compiler", {}).get("sub_configs", []) + if not sub_configs: + raise ValueError("配置文件中缺少 compiler.sub_configs") + + model = onnx.load(onnx_path.as_posix()) + index = build_graph_index(model) + + specs: List[SubGraphSpec] = [] + covered_nodes = set() + for idx, entry in enumerate(sub_configs): + start = [name for name in entry.get("start_tensor_names", []) if name] + end = [name for name in entry.get("end_tensor_names", []) if name] + if not start or not end: + raise ValueError(f"sub_config[{idx}] 缺少 tensor 名称") + spec = SubGraphSpec( + label=f"cfg_{idx:02d}", + start=start, + end=end, + node_names=set(), + source="config", + ) + nodes = trace_nodes_between(spec, index) + spec.node_names = nodes + covered_nodes.update(nodes) + specs.append(spec) + + leftovers = untouched_components(index.node_order, covered_nodes, index) + for idx, component in enumerate(leftovers): + start, end = derive_interface(component, index) + if not end: + continue + spec = SubGraphSpec( + label=f"auto_{idx:02d}", + start=start, + end=end, + node_names=component, + source="auto", + ) + specs.append(spec) + + return ordered_specs(specs, index) + + +def expected_model_path(spec: SubGraphSpec, model_dir: Path) -> Path: + head = sanitize(spec.start[0]) if spec.start else "const" + tail = sanitize(spec.end[0]) if spec.end else "out" + filename = f"{spec.label}_{head}_to_{tail}_{spec.source}.axmodel" + path = model_dir / filename + if not path.exists(): + raise FileNotFoundError(f"未找到模型文件: {path}") + return path + + +def run_pipeline( + specs: List[SubGraphSpec], + model_dir: Path, + feed_dict: Dict[str, np.ndarray], +) -> Dict[str, np.ndarray]: + tensor_store: Dict[str, np.ndarray] = dict(feed_dict) + for spec in specs: + model_path = expected_model_path(spec, model_dir) + session = InferenceSession(model_path.as_posix()) + inputs = {} + missing = [] + for name in spec.start: + value = tensor_store.get(name) + if value is None: + missing.append(name) + else: + inputs[name] = value + if missing: + raise KeyError(f"模型 {model_path.name} 缺少输入张量: {missing}") + results = session.run(spec.end, inputs) + if not isinstance(results, (list, tuple)): + results = [results] + for out_name, value in zip(spec.end, results): + tensor_store[out_name] = value + shapes = {out_name: tuple(value.shape) for out_name, value in zip(spec.end, results)} + logging.info("完成 %s: %s", model_path.name, shapes) + return tensor_store + + +def load_input_file(path: Path, single_name: str | None) -> Dict[str, np.ndarray]: + suffix = path.suffix.lower() + if suffix == ".npz": + data = np.load(path, allow_pickle=False) + return {key: np.asarray(data[key]) for key in data.files} + + if suffix == ".npy": + _ensure_numpy_core_alias() + blob = np.load(path, allow_pickle=True) + if isinstance(blob, np.ndarray) and blob.dtype == object: + obj = blob.item() + if isinstance(obj, dict): + return {str(k): np.asarray(v) for k, v in obj.items()} + if single_name is None: + raise ValueError("单一 .npy 输入需要通过 --single-input-name 指定张量名") + return {single_name: np.asarray(obj)} + if single_name is None: + raise ValueError("单一 .npy 输入需要通过 --single-input-name 指定张量名") + return {single_name: np.asarray(blob)} + + raise ValueError(f"暂不支持的输入文件格式: {path}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="串行执行切分后的 AXModel") + parser.add_argument("--onnx", required=True, type=Path, help="原始未切分 ONNX") + parser.add_argument("--config", required=True, type=Path, help="sub_config 配置文件") + parser.add_argument("--model-dir", required=True, type=Path, help=".axmodel 所在目录") + parser.add_argument( + "--input-file", + required=True, + type=Path, + help="包含原始输入张量的 npz 或 npy 文件", + ) + parser.add_argument( + "--single-input-name", + type=str, + help="当 --input-file 是单张量 .npy 文件时指定对应的输入名", + ) + parser.add_argument("--dump-npz", type=Path, help="可选，将最终输出保存为 npz") + parser.add_argument("--log", default="INFO", help="日志等级") + return parser.parse_args() + + +def main() -> None: + args = parse_args() + logging.basicConfig(level=getattr(logging, args.log.upper(), logging.INFO)) + + specs = load_specs(args.config, args.onnx) + feed = load_input_file(args.input_file, args.single_input_name) + store = run_pipeline(specs, args.model_dir, feed) + + final_outputs = {name: store[name] for name in specs[-1].end} + for name, value in final_outputs.items(): + logging.info("最终输出 %s: shape=%s", name, value.shape) + + if args.dump_npz: + np.savez(args.dump_npz, **final_outputs) + logging.info("已保存输出到 %s", args.dump_npz) + + +if __name__ == "__main__": + """ + python3 infer_split_axmodels.py \ + --onnx onnx-models/transformer.onnx \ + --config pulsar2_configs/transformers_subgraph.json \ + --model-dir compiled_slice_quant_onnx \ + --input-file onnx-calibration-no-controlnet/transformer_inputs_prompt000_step00.npy \ + --single-input-name timestep + """ + main() diff --git a/VideoX-Fun/install.py b/VideoX-Fun/install.py new file mode 100644 index 0000000000000000000000000000000000000000..5e349b4db1ccf44876c55e910b802875e2016fc4 --- /dev/null +++ b/VideoX-Fun/install.py @@ -0,0 +1,45 @@ +import sys +import subprocess +import locale +import threading +import os + +def handle_stream(stream, prefix): + stream.reconfigure(encoding=locale.getpreferredencoding(), errors='replace') + for msg in stream: + if prefix == '[!]' and ('it/s]' in msg or 's/it]' in msg) and ('%|' in msg or 'it [' in msg): + if msg.startswith('100%'): + print('\r' + msg, end="", file=sys.stderr), + else: + print('\r' + msg[:-1], end="", file=sys.stderr), + else: + if prefix == '[!]': + print(prefix, msg, end="", file=sys.stderr) + else: + print(prefix, msg, end="") + +def process_wrap(cmd_str, cwd_path, handler=None): + process = subprocess.Popen(cmd_str, cwd=cwd_path, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, bufsize=1) + + if handler is None: + handler = handle_stream + + stdout_thread = threading.Thread(target=handler, args=(process.stdout, "")) + stderr_thread = threading.Thread(target=handler, args=(process.stderr, "[!]")) + + stdout_thread.start() + stderr_thread.start() + + stdout_thread.join() + stderr_thread.join() + + return process.wait() + +assert process_wrap([sys.executable, "-m", "pip", "install", "-r", "requirements.txt"], cwd_path=os.path.dirname(os.path.realpath(__file__))) == 0, "ERROR: Failed to install requirements.txt. Please install them manually, and restart ComfyUI." + +nodep_packages = [ + "kornia>=0.6.9", + "xformers>=0.0.20", +] + +assert process_wrap([sys.executable, "-m", "pip", "install", "--no-deps", *nodep_packages], cwd_path=os.path.dirname(os.path.realpath(__file__))) == 0, "ERROR: Failed to install last set of packages. Please install them manually, and restart ComfyUI." \ No newline at end of file diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/.gitattributes b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/.gitattributes new file mode 100644 index 0000000000000000000000000000000000000000..c88e3c116b124ca91c94af1ac21d6d0db7deaa58 --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/.gitattributes @@ -0,0 +1,46 @@ +*.7z filter=lfs diff=lfs merge=lfs -text +*.arrow filter=lfs diff=lfs merge=lfs -text +*.bin filter=lfs diff=lfs merge=lfs -text +*.bz2 filter=lfs diff=lfs merge=lfs -text +*.ckpt filter=lfs diff=lfs merge=lfs -text +*.ftz filter=lfs diff=lfs merge=lfs -text +*.gz filter=lfs diff=lfs merge=lfs -text +*.h5 filter=lfs diff=lfs merge=lfs -text +*.joblib filter=lfs diff=lfs merge=lfs -text +*.lfs.* filter=lfs diff=lfs merge=lfs -text +*.mlmodel filter=lfs diff=lfs merge=lfs -text +*.model filter=lfs diff=lfs merge=lfs -text +*.msgpack filter=lfs diff=lfs merge=lfs -text +*.npy filter=lfs diff=lfs merge=lfs -text +*.npz filter=lfs diff=lfs merge=lfs -text +*.onnx filter=lfs diff=lfs merge=lfs -text +*.ot filter=lfs diff=lfs merge=lfs -text +*.parquet filter=lfs diff=lfs merge=lfs -text +*.pb filter=lfs diff=lfs merge=lfs -text +*.pickle filter=lfs diff=lfs merge=lfs -text +*.pkl filter=lfs diff=lfs merge=lfs -text +*.pt filter=lfs diff=lfs merge=lfs -text +*.pth filter=lfs diff=lfs merge=lfs -text +*.rar filter=lfs diff=lfs merge=lfs -text +*.safetensors filter=lfs diff=lfs merge=lfs -text +saved_model/**/* filter=lfs diff=lfs merge=lfs -text +*.tar.* filter=lfs diff=lfs merge=lfs -text +*.tar filter=lfs diff=lfs merge=lfs -text +*.tflite filter=lfs diff=lfs merge=lfs -text +*.tgz filter=lfs diff=lfs merge=lfs -text +*.wasm filter=lfs diff=lfs merge=lfs -text +*.xz filter=lfs diff=lfs merge=lfs -text +*.zip filter=lfs diff=lfs merge=lfs -text +*.zst filter=lfs diff=lfs merge=lfs -text +*tfevents* filter=lfs diff=lfs merge=lfs -text +tokenizer/tokenizer.json filter=lfs diff=lfs merge=lfs -text +assets/DMDR.webp filter=lfs diff=lfs merge=lfs -text +assets/architecture.webp filter=lfs diff=lfs merge=lfs -text +assets/decoupled-dmd.webp filter=lfs diff=lfs merge=lfs -text +assets/reasoning.png filter=lfs diff=lfs merge=lfs -text +assets/showcase.jpg filter=lfs diff=lfs merge=lfs -text +assets/showcase_editing.png filter=lfs diff=lfs merge=lfs -text +assets/showcase_realistic.png filter=lfs diff=lfs merge=lfs -text +assets/showcase_rendering.png filter=lfs diff=lfs merge=lfs -text +assets/Z-Image-Gallery.pdf filter=lfs diff=lfs merge=lfs -text +assets/leaderboard.png filter=lfs diff=lfs merge=lfs -text diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/README.md b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1b1eb3a1f5b2cb96bb6f9435f7909b0b2311be20 --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/README.md @@ -0,0 +1,197 @@ +--- +license: apache-2.0 +language: +- en +pipeline_tag: text-to-image +library_name: diffusers +--- + + +

⚡️- Image
An Efficient Image Generation Foundation Model with Single-Stream Diffusion Transformer

+ +
+ +[![Official Site](https://img.shields.io/badge/Official%20Site-333399.svg?logo=homepage)](https://tongyi-mai.github.io/Z-Image-blog/)  +[![GitHub](https://img.shields.io/badge/GitHub-Z--Image-181717?logo=github&logoColor=white)](https://github.com/Tongyi-MAI/Z-Image)  +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Checkpoint-Z--Image--Turbo-yellow)](https://huggingface.co/Tongyi-MAI/Z-Image-Turbo)  +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Online_Demo-Z--Image--Turbo-blue)](https://huggingface.co/spaces/Tongyi-MAI/Z-Image-Turbo)  +[![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Mobile_Demo-Z--Image--Turbo-red)](https://huggingface.co/spaces/akhaliq/Z-Image-Turbo)  +[![ModelScope Model](https://img.shields.io/badge/🤖%20Checkpoint-Z--Image--Turbo-624aff)](https://www.modelscope.cn/models/Tongyi-MAI/Z-Image-Turbo)  +[![ModelScope Space](https://img.shields.io/badge/🤖%20Online_Demo-Z--Image--Turbo-17c7a7)](https://www.modelscope.cn/aigc/imageGeneration?tab=advanced&versionId=469191&modelType=Checkpoint&sdVersion=Z_IMAGE_TURBO&modelUrl=modelscope%253A%252F%252FTongyi-MAI%252FZ-Image-Turbo%253Frevision%253Dmaster%7D%7BOnline)  +[![Art Gallery PDF](https://img.shields.io/badge/%F0%9F%96%BC%20Art_Gallery-PDF-ff69b4)](assets/Z-Image-Gallery.pdf)  +[![Web Art Gallery](https://img.shields.io/badge/%F0%9F%8C%90%20Web_Art_Gallery-online-00bfff)](https://modelscope.cn/studios/Tongyi-MAI/Z-Image-Gallery/summary)  + + + +Welcome to the official repository for the Z-Image（造相）project! + +
+ + + +## ✨ Z-Image + +Z-Image is a powerful and highly efficient image generation model with **6B** parameters. Currently there are three variants: + +- 🚀 **Z-Image-Turbo** – A distilled version of Z-Image that matches or exceeds leading competitors with only **8 NFEs** (Number of Function Evaluations). It offers **⚡️sub-second inference latency⚡️** on enterprise-grade H800 GPUs and fits comfortably within **16G VRAM consumer devices**. It excels in photorealistic image generation, bilingual text rendering (English & Chinese), and robust instruction adherence. + +- 🧱 **Z-Image-Base** – The non-distilled foundation model. By releasing this checkpoint, we aim to unlock the full potential for community-driven fine-tuning and custom development. + +- ✍️ **Z-Image-Edit** – A variant fine-tuned on Z-Image specifically for image editing tasks. It supports creative image-to-image generation with impressive instruction-following capabilities, allowing for precise edits based on natural language prompts. + +### 📥 Model Zoo + +| Model | Hugging Face | ModelScope | +| :--- |:--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| **Z-Image-Turbo** | [![Hugging Face](https://img.shields.io/badge/%F0%9F%A4%97%20Checkpoint%20-Z--Image--Turbo-yellow)](https://huggingface.co/Tongyi-MAI/Z-Image-Turbo)
[![Hugging Face Space](https://img.shields.io/badge/%F0%9F%A4%97%20Online%20Demo-Z--Image--Turbo-blue)](https://huggingface.co/spaces/Tongyi-MAI/Z-Image-Turbo) | [![ModelScope Model](https://img.shields.io/badge/🤖%20%20Checkpoint-Z--Image--Turbo-624aff)](https://www.modelscope.cn/models/Tongyi-MAI/Z-Image-Turbo)
[![ModelScope Space](https://img.shields.io/badge/%F0%9F%A4%96%20Online%20Demo-Z--Image--Turbo-17c7a7)](https://www.modelscope.cn/aigc/imageGeneration?tab=advanced&versionId=469191&modelType=Checkpoint&sdVersion=Z_IMAGE_TURBO&modelUrl=modelscope%3A%2F%2FTongyi-MAI%2FZ-Image-Turbo%3Frevision%3Dmaster) | +| **Z-Image-Base** | *To be released* | *To be released* | +| **Z-Image-Edit** | *To be released* | *To be released* | + +### 🖼️ Showcase + +📸 **Photorealistic Quality**: **Z-Image-Turbo** delivers strong photorealistic image generation while maintaining excellent aesthetic quality. + +![Showcase of Z-Image on Photo-realistic image Generation](assets/showcase_realistic.png) + +📖 **Accurate Bilingual Text Rendering**: **Z-Image-Turbo** excels at accurately rendering complex Chinese and English text. + +![Showcase of Z-Image on Bilingual Text Rendering](assets/showcase_rendering.png) + +💡 **Prompt Enhancing & Reasoning**: Prompt Enhancer empowers the model with reasoning capabilities, enabling it to transcend surface-level descriptions and tap into underlying world knowledge. + +![reasoning.jpg](assets/reasoning.png) + +🧠 **Creative Image Editing**: **Z-Image-Edit** shows a strong understanding of bilingual editing instructions, enabling imaginative and flexible image transformations. + +![Showcase of Z-Image-Edit on Image Editing](assets/showcase_editing.png) + +### 🏗️ Model Architecture +We adopt a **Scalable Single-Stream DiT** (S3-DiT) architecture. In this setup, text, visual semantic tokens, and image VAE tokens are concatenated at the sequence level to serve as a unified input stream, maximizing parameter efficiency compared to dual-stream approaches. + +![Architecture of Z-Image and Z-Image-Edit](assets/architecture.webp) + +### 📈 Performance +According to the Elo-based Human Preference Evaluation (on [*Alibaba AI Arena*](https://aiarena.alibaba-inc.com/corpora/arena/leaderboard?arenaType=T2I)), Z-Image-Turbo shows highly competitive performance against other leading models, while achieving state-of-the-art results among open-source models. + +

+ + Z-Image Elo Rating on AI Arena
+ Click to view the full leaderboard + +

+ +### 🚀 Quick Start +Install the latest version of diffusers, use the following command: +
+ Click here for details for why you need to install diffusers from source + + We have submitted two pull requests ([#12703](https://github.com/huggingface/diffusers/pull/12703) and [#12715](https://github.com/huggingface/diffusers/pull/12704)) to the 🤗 diffusers repository to add support for Z-Image. Both PRs have been merged into the latest official diffusers release. + Therefore, you need to install diffusers from source for the latest features and Z-Image support. + +
+ +```bash +pip install git+https://github.com/huggingface/diffusers +``` + +```python +import torch +from diffusers import ZImagePipeline + +# 1. Load the pipeline +# Use bfloat16 for optimal performance on supported GPUs +pipe = ZImagePipeline.from_pretrained( + "Tongyi-MAI/Z-Image-Turbo", + torch_dtype=torch.bfloat16, + low_cpu_mem_usage=False, +) +pipe.to("cuda") + +# [Optional] Attention Backend +# Diffusers uses SDPA by default. Switch to Flash Attention for better efficiency if supported: +# pipe.transformer.set_attention_backend("flash") # Enable Flash-Attention-2 +# pipe.transformer.set_attention_backend("_flash_3") # Enable Flash-Attention-3 + +# [Optional] Model Compilation +# Compiling the DiT model accelerates inference, but the first run will take longer to compile. +# pipe.transformer.compile() + +# [Optional] CPU Offloading +# Enable CPU offloading for memory-constrained devices. +# pipe.enable_model_cpu_offload() + +prompt = "Young Chinese woman in red Hanfu, intricate embroidery. Impeccable makeup, red floral forehead pattern. Elaborate high bun, golden phoenix headdress, red flowers, beads. Holds round folding fan with lady, trees, bird. Neon lightning-bolt lamp (⚡️), bright yellow glow, above extended left palm. Soft-lit outdoor night background, silhouetted tiered pagoda (西安大雁塔), blurred colorful distant lights." + +# 2. Generate Image +image = pipe( + prompt=prompt, + height=1024, + width=1024, + num_inference_steps=9, # This actually results in 8 DiT forwards + guidance_scale=0.0, # Guidance should be 0 for the Turbo models + generator=torch.Generator("cuda").manual_seed(42), +).images[0] + +image.save("example.png") +``` + +## 🔬 Decoupled-DMD: The Acceleration Magic Behind Z-Image + +[![arXiv](https://img.shields.io/badge/arXiv-2511.22677-b31b1b.svg)](https://arxiv.org/abs/2511.22677) + +Decoupled-DMD is the core few-step distillation algorithm that empowers the 8-step Z-Image model. + +Our core insight in Decoupled-DMD is that the success of existing DMD (Distributaion Matching Distillation) methods is the result of two independent, collaborating mechanisms: + +- **CFG Augmentation (CA)**: The primary **engine** 🚀 driving the distillation process, a factor largely overlooked in previous work. +- **Distribution Matching (DM)**: Acts more as a **regularizer** ⚖️, ensuring the stability and quality of the generated output. + +By recognizing and decoupling these two mechanisms, we were able to study and optimize them in isolation. This ultimately motivated us to develop an improved distillation process that significantly enhances the performance of few-step generation. + +![Diagram of Decoupled-DMD](assets/decoupled-dmd.webp) + +## 🤖 DMDR: Fusing DMD with Reinforcement Learning + +[![arXiv](https://img.shields.io/badge/arXiv-2511.13649-b31b1b.svg)](https://arxiv.org/abs/2511.13649) + +Building upon the strong foundation of Decoupled-DMD, our 8-step Z-Image model has already demonstrated exceptional capabilities. To achieve further improvements in terms of semantic alignment, aesthetic quality, and structural coherence—while producing images with richer high-frequency details—we present **DMDR**. + +Our core insight behind DMDR is that Reinforcement Learning (RL) and Distribution Matching Distillation (DMD) can be synergistically integrated during the post-training of few-step models. We demonstrate that: + +- **RL Unlocks the Performance of DMD** 🚀 +- **DMD Effectively Regularizes RL** ⚖️ + +![Diagram of DMDR](assets/DMDR.webp) + +## ⏬ Download +```bash +pip install -U huggingface_hub +HF_XET_HIGH_PERFORMANCE=1 hf download Tongyi-MAI/Z-Image-Turbo +``` + +## 📜 Citation + +If you find our work useful in your research, please consider citing: + +```bibtex +@article{team2025zimage, + title={Z-Image: An Efficient Image Generation Foundation Model with Single-Stream Diffusion Transformer}, + author={Z-Image Team}, + journal={arXiv preprint arXiv:2511.22699}, + year={2025} +} + +@article{liu2025decoupled, + title={Decoupled DMD: CFG Augmentation as the Spear, Distribution Matching as the Shield}, + author={Dongyang Liu and Peng Gao and David Liu and Ruoyi Du and Zhen Li and Qilong Wu and Xin Jin and Sihan Cao and Shifeng Zhang and Hongsheng Li and Steven Hoi}, + journal={arXiv preprint arXiv:2511.22677}, + year={2025} +} + +@article{jiang2025distribution, + title={Distribution Matching Distillation Meets Reinforcement Learning}, + author={Jiang, Dengyang and Liu, Dongyang and Wang, Zanyi and Wu, Qilong and Jin, Xin and Liu, David and Li, Zhen and Wang, Mengmeng and Gao, Peng and Yang, Harry}, + journal={arXiv preprint arXiv:2511.13649}, + year={2025} +} +``` \ No newline at end of file diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/model_index.json b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/model_index.json new file mode 100644 index 0000000000000000000000000000000000000000..570c63a7c5580f63d0a8ed622324f50015dcf914 --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/model_index.json @@ -0,0 +1,24 @@ +{ + "_class_name": "ZImagePipeline", + "_diffusers_version": "0.36.0.dev0", + "scheduler": [ + "diffusers", + "FlowMatchEulerDiscreteScheduler" + ], + "text_encoder": [ + "transformers", + "Qwen3Model" + ], + "tokenizer": [ + "transformers", + "Qwen2Tokenizer" + ], + "transformer": [ + "diffusers", + "ZImageTransformer2DModel" + ], + "vae": [ + "diffusers", + "AutoencoderKL" + ] +} \ No newline at end of file diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/scheduler/scheduler_config.json b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/scheduler/scheduler_config.json new file mode 100644 index 0000000000000000000000000000000000000000..2349bc43351afa730341d07dd44fce4ace1c4257 --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/scheduler/scheduler_config.json @@ -0,0 +1,7 @@ +{ + "_class_name": "FlowMatchEulerDiscreteScheduler", + "_diffusers_version": "0.36.0.dev0", + "num_train_timesteps": 1000, + "use_dynamic_shifting": false, + "shift": 3.0 +} \ No newline at end of file diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/config.json b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/config.json new file mode 100644 index 0000000000000000000000000000000000000000..e49eccdc32f36da9c09cfa0e737084f9e0105e5e --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/config.json @@ -0,0 +1,30 @@ +{ + "architectures": [ + "Qwen3ForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 151643, + "eos_token_id": 151645, + "head_dim": 128, + "hidden_act": "silu", + "hidden_size": 2560, + "initializer_range": 0.02, + "intermediate_size": 9728, + "max_position_embeddings": 40960, + "max_window_layers": 36, + "model_type": "qwen3", + "num_attention_heads": 32, + "num_hidden_layers": 36, + "num_key_value_heads": 8, + "rms_norm_eps": 1e-06, + "rope_scaling": null, + "rope_theta": 1000000, + "sliding_window": null, + "tie_word_embeddings": true, + "torch_dtype": "bfloat16", + "transformers_version": "4.51.0", + "use_cache": true, + "use_sliding_window": false, + "vocab_size": 151936 +} \ No newline at end of file diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/generation_config.json b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..20a8a9156fc8c3f25295ca067f61fdf120d517c5 --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/generation_config.json @@ -0,0 +1,13 @@ +{ + "bos_token_id": 151643, + "do_sample": true, + "eos_token_id": [ + 151645, + 151643 + ], + "pad_token_id": 151643, + "temperature": 0.6, + "top_k": 20, + "top_p": 0.95, + "transformers_version": "4.51.0" +} \ No newline at end of file diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model-00001-of-00003.safetensors b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model-00001-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..546625baab99f4de2b975842100c119fec83919f --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model-00001-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:328a91d3122359d5547f9d79521205bc0a46e1f79a792dfe650e99fc2d651223 +size 3957900840 diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model-00002-of-00003.safetensors b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model-00002-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..3e4d29aad0be87c083e078b50765cd056e793c51 --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model-00002-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6cd087b316306a68c562436b5492edbcf6e16c6dba3a1308279caa5a58e21ca5 +size 3987450520 diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model-00003-of-00003.safetensors b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model-00003-of-00003.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8a83739c1b3794759b2565002478cf8397b4a09d --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model-00003-of-00003.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7ca841ee75b9c61267c0c6148fd8d096d3d21b6d3e161256a9b878154f91fc52 +size 99630640 diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model.safetensors.index.json b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model.safetensors.index.json new file mode 100644 index 0000000000000000000000000000000000000000..95c0a0059df040d75dc6c396b174382cf61d2f91 --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/text_encoder/model.safetensors.index.json @@ -0,0 +1,405 @@ +{ + "metadata": { + "total_size": 8044936192 + }, + "weight_map": { + "model.embed_tokens.weight": "model-00001-of-00003.safetensors", + "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.12.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.12.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.13.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.13.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.14.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.14.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.15.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.15.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.24.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.25.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.25.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.26.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.26.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.27.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.27.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.28.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.28.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.29.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.29.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.30.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.30.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.30.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.31.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.31.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.32.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.32.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.33.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.33.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.input_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.34.mlp.down_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.mlp.up_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.post_attention_layernorm.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.34.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.35.input_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.35.mlp.down_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.35.mlp.gate_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.35.mlp.up_proj.weight": "model-00003-of-00003.safetensors", + "model.layers.35.post_attention_layernorm.weight": "model-00003-of-00003.safetensors", + "model.layers.35.self_attn.k_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.35.self_attn.k_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.35.self_attn.o_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.35.self_attn.q_norm.weight": "model-00002-of-00003.safetensors", + "model.layers.35.self_attn.q_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.35.self_attn.v_proj.weight": "model-00002-of-00003.safetensors", + "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.k_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.q_norm.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors", + "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors", + "model.norm.weight": "model-00003-of-00003.safetensors" + } +} diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/merges.txt b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/merges.txt new file mode 100644 index 0000000000000000000000000000000000000000..31349551d90c7606f325fe0f11bbb8bd5fa0d7c7 --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/merges.txt @@ -0,0 +1,151388 @@ +#version: 0.2 +Ġ Ġ +ĠĠ ĠĠ +i n +Ġ t +ĠĠĠĠ ĠĠĠĠ +e r +ĠĠ Ġ +o n +Ġ a +r e +a t +s t +e n +o r +Ġt h +Ċ Ċ +Ġ c +l e +Ġ s +i t +a n +a r +a l +Ġth e +; Ċ +Ġ p +Ġ f +o u +Ġ = +i s +ĠĠĠĠ ĠĠĠ +in g +e s +Ġ w +i on +e d +i c +Ġ b +Ġ d +e t +Ġ m +Ġ o +ĉ ĉ +r o +a s +e l +c t +n d +Ġ in +Ġ h +en t +i d +Ġ n +a m +ĠĠĠĠĠĠĠĠ ĠĠĠ +Ġt o +Ġ re +- - +Ġ { +Ġo f +o m +) ;Ċ +i m +č Ċ +Ġ ( +i l +/ / +Ġa nd +u r +s e +Ġ l +e x +Ġ S +a d +Ġ " +c h +u t +i f +* * +Ġ } +e m +o l +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠ +t h +) Ċ +Ġ{ Ċ +Ġ g +i g +i v +, Ċ +c e +o d +Ġ v +at e +Ġ T +a g +a y +Ġ * +o t +u s +Ġ C +Ġ st +Ġ I +u n +u l +u e +Ġ A +o w +Ġ ' +e w +Ġ < +at ion +( ) +Ġf or +a b +or t +u m +am e +Ġ is +p e +t r +c k +â Ģ +Ġ y +i st +-- -- +. ĊĊ +h e +Ġ e +l o +Ġ M +Ġb e +er s +Ġ on +Ġc on +a p +u b +Ġ P +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠ +as s +in t +> Ċ +l y +ur n +Ġ $ +; ĊĊ +a v +p ort +i r +- > +n t +ct ion +en d +Ġd e +it h +ou t +t urn +ou r +ĠĠĠĠ Ġ +l ic +re s +p t += = +Ġth is +Ġw h +Ġ if +Ġ D +v er +ag e +Ġ B +h t +ex t += " +Ġth at +** ** +Ġ R +Ġ it +es s +Ġ F +Ġ r +o s +an d +Ġa s +e ct +k e +ro m +Ġ // +c on +Ġ L +( " +q u +l ass +Ġw ith +i z +d e +Ġ N +Ġa l +o p +u p +g et +Ġ} Ċ +i le +Ġa n +at a +o re +r i +Ġp ro +; čĊ +ĉĉ ĉĉ +t er +a in +Ġ W +Ġ E +Ġc om +Ġre turn +ar t +Ġ H +a ck +im port +ub lic +Ġ or +e st +m ent +Ġ G +ab le +Ġ - +in e +il l +in d +er e +: : +it y +Ġ + +Ġt r +el f +ig ht +( ' +or m +ul t +st r +. . +" , +Ġy ou +y pe +p l +Ġn ew +Ġ j +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠ +Ġf rom +Ġ ex +Ġ O +l d +Ġ [ +o c +: Ċ +Ġs e +Ġ le +---- ---- +. s +{ Ċ +' , +an t +Ġa t +as e +. c +Ġc h +< / +av e +an g +Ġa re +Ġin t +âĢ Ļ +_ t +er t +i al +a ct +} Ċ +iv e +od e +o st +Ġc lass +Ġn ot +o g +or d +al ue +al l +f f +( );Ċ +on t +im e +a re +Ġ U +Ġp r +Ġ : +i es +iz e +u re +Ġb y +i re +Ġ} ĊĊ +. p +Ġs h +ic e +a st +pt ion +tr ing +o k +_ _ +c l +# # +Ġh e +ar d +) . +Ġ @ +i ew +ĉĉ ĉ +Ġw as +i p +th is +Ġ u +ĠT he +id e +a ce +i b +a c +r ou +Ġw e +j ect +Ġp ublic +a k +v e +at h +o id +Ġ= > +u st +q ue +Ġre s +) ) +' s +Ġ k +an s +y st +un ction +**** **** +Ġ i +Ġ us +p p +on e +a il +== == +n ame +Ġst r +Ġ / +Ġ & +a ch +d iv +yst em +el l +Ġh ave +er r +ou ld +ul l +p on +Ġ J +_ p +Ġ= = +ig n +S t +. Ċ +Ġp l +) ;ĊĊ +f orm +p ut +ou nt +} ĊĊ +d d +it e +Ġg et +r r +om e +Ġ âĢ +ar am +c c +Ġ* / +E R +I n +le s +_ s +on g +i e +Ġc an +Ġ V +er v +p r +Ġ un +ro w +b er +Ġd o +l l +Ġ el +Ġs elf +at ed +ar y +Ġ . +' ] +u d +Ġ en +ĠT h +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠ +t e +_ c +u ct +Ġa b +or k +. get +Ġ # +a w +res s +o b +N ame +ap p +[ ' +Ġal l +or y +it ion +an ce +e ar +Ġcon t +v ent +i a +Ġw ill +I N +ĠĠĠĠĠĠĠĠ Ġ +re turn +Ġ< / +d ata +) ĊĊ +R e +p le +il d +th er +Ġy our +" Ċ +( $ +Ġ out +) , +Ġh as +S tring +s o +Ġ up +a x +Ġde f +Ġb o +g e +al se +O N +p er +ic h +Ġb ut +Ġ Ċ +Ġ _ +_ m +ad d +que st +od el +s elf +er y +f t +en s +// // +a ke +. C +Ġg o +Ġf unction +Ġ K +iv ate +Ġ im +Ġcon st +. t +Ġ*/ Ċ +) ;čĊ +Ġv oid +Ġs et +ĠS ystem +c ri +( )Ċ +l i +ĉ if +. m +al ly +s et +e p +âĢĻ s +b o +de f +' ,Ċ +Ġm e +Ġ ! +at ch +" > +" ,Ċ +e c +ĠI n +p h +Ġ | +_ f +Ġv ar +en ce +I d +re e +in k +le ct +u g +et h +Ġel se +-------- -------- +con t +Ġs o +at ic +Ġl o +p ro +t on +s s +ow n +ab el +o int +ou s +el d +S T +T he +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +R E +" : +ol or +t p +e g +ke y +u de +ĠS t +ou nd +Ġa r +" );Ċ +en er +s er +b ject +ess age +f er +Ġm ore +ation s +ent s +Ġh is +Ġthe y +. S +Ġ Y +u se +n e +is h +ol d +_ d +i o +i eld +Ġp er +C ont +ing s +## ## +Ġd ata +Ġs a +e f +f o +Ġon e +en g +Ġd is +A T +Ġn ame +Ġtr ue +v al +le d +. f +Ġn e +Ġ end +. T +c re +ar k +lo g +E x +err or +_ id +ur re +ang e +Ġn ull +rr ay +Ġm y +p an +ic t +at or +V iew +L ist +ĉ return +âĢ Ŀ +Ġp re +Ġ x +cl ude +ar g +o v +. h +Ġ > +Ġthe ir +' ) +ir st +ic k +g h +L E +O R +Ġpr ivate +t em +čĊ čĊ +us er +Ġ ) +c om +. A +" ;Ċ +Ġ id +re ad +Ġwh o +_ b +" >Ċ +Ġt ime +Ġm an +r y +==== ==== +rou p +ro p +p ublic +v el +um ber +b le +Ġwh ich +******** ******** +Ġan y +Ġf alse +w e +Ġv alue +Ġl i +" ) +nd er +g r +Ġn o +p aram +f ig +.c om +Ġa pp +_ l +ion s +. D +ĠC h +Ġab out +Ġa dd +Ġs u +Ġstr ing +I D +Ġo ver +str ing +. l +our ce +_ C +] Ċ +Ġ qu +ĠS tring +c a +S E +Ġ ro +s h +u al +T ype +s on +n ew +er n +Ġa g +A R +] ;Ċ +] . +Ġ ? +ic al +Ġd es +ut h +i x +ay s +Ġt ype +' t +a ult +Ġin ter +v ar +. b +Ġp art +. d +urre nt +I T +E N +en c +( f +r a +v alue +ch o +ut ton +o se +Ġ! = +at er +à © +re ate +ol l +p os +y le +n g +A L +us ing +am es +Ġ{ čĊ +at es +el y +Ġw ork +Ġ em +in al +Ġs p +Ġwh en +.s et +ĠĠĠĠ ĠĠ +) :Ċ +t o +qu ire +ind ow +le ment +pe ct +as h +[ i +Ġu se +. F +pe c +Ġa d +o ve +ce ption +eng th +in clude +ad er +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠ +at us +T h +it le +r it +v oid +() . +( Ċ +Ġof f +Ġo ther +Ġ& & +' ;Ċ +m s +Ġbe en +Ġt e +m l +c o +n c +erv ice +Ġ % +** Ċ +an n +ad e +ĊĊ ĊĊ +lo ck +con st +pon se +Ġs up ++ + +d ate +Ġa cc +Ġh ad +Ġb u +ĠR e +Ġw ere +Ġf ile +Ġw ould +ĠâĢ ľ +v en +is s +Ġ our +c lass +r aw +Ġy ear +D ata +Ġv al +Ġs ome +f ter +y s +Ġ// / +rou nd +v iew +Ġp e +Ġth ere +Ġsa id +d u +o f +l ine +/ * +d uct +Ġh er +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠ +R es +Ġc o +Ġcom m +is e +m in +ĠĠĠĠ Ċ +# include +eth od +. P +ut e +Ġas s +I nt +as k +lo c +Ġli ke +od y +Ġle t +lo ad +Ġa m +ro l +Ġg r +y p +Ġal so +ĠI t +ur l +if ic +or s +_ P +_ n +ig h +Ġth an +C om +A N +U L +at ing +ĠTh is +re f +_ S +Ġst atic +ro ll +Ġj ust +Ġres ult +i an +id th +Ġthe m +) );Ċ +d er +re ak +C on +: // +u le +.. . +ar ch +em ent +Ġ< < +us h +en se +ar r +Ġint o +c ess +am p +i ed +um ent +Ġ \ +] , +w o +al s +Ġwh at +an c +V alue += ' +ol um +Ġp os +ag es +ay er +Ġs c +u es +" )Ċ +_ T +Ġl ist +( s +Ġc ase +C h +ĉĉĉĉ ĉ +//// //// +pon ent +Ġ z +Ġk n +le t +D E +re d +Ġf e +Ġ} ,Ċ +Ġ , +( t +Ġf irst +' );Ċ +w ord +Ġ import +Ġa ct +Ġch ar +C T +ĠT r +op le += { +ĉ f +i ent +c ent +. j +le ction +) )Ċ +Ġon ly +Ġpr int +m er +. W +o ck +Ġ -- +T ext +Ġo p +an k +Ġit s +Ġb ack +[ " +Ġne ed +Ġc l +Ġs ub +Ġl a +( ( +. " +O bject +Ġst art +f ile +( self +n er +e y +Ġus er +Ġ ent +ĠC om +it s +ĠC on +ou ble +ow er +it em +ver y +ĠW e +lic k +Ġ Q +ph p +t tp +' : +ic s +Ġu nder +Ġ* Ċ +. L +) ; +ic es +Ġre g +) čĊ +ĉ public +S S +Ġth en +re at +i ous +. G +e k +ire ct +he ck +cri pt +n ing +ĠU n +Ġm ay +ĠW h +B o +I tem +str uct +. st +re am +ib le +lo at +Ġor g +u nd +s um +_ in +.. / +_ M +Ġh ow +r ite +' Ċ +T o +w w +Ġpe ople +ind ex +. n +ht tp +( m +ect or +Ġin d +Ġj av +] ,Ċ +ĠH e +_ st +f ul +o le +) {Ċ +Ġsh ould +op y +el p +i er +_ name +ers on +I ON +ot e +Ġt est +Ġb et +rr or +ul ar +ã Ģ +Ġ Ð +b s +t ing +Ġm ake +T r +Ġa fter +ar get +R O +olum n +r c +_ re +def ine +Ġr ight +r ight +d ay +Ġl ong +[ ] +( p +t d +con d +ĠP ro +Ġre m +ption s +v id +. g +Ġ ext +Ġ __ +' )Ċ +p ace +m p +Ġm in +st ance +a ir +a ction +w h +t ype +ut il +a it +< ? +I C +t ext +Ġp h +Ġf l +. M +cc ess +b r +f ore +ers ion +) ,Ċ +. re +ate g +Ġl oc +in s +- s +tr ib +ĠI nt +Ġa rray +, " +P ro +( c +ess ion +> ĊĊ +Ġs he +" ] +ap h +Ġex p +ert y +ĠS e +Ġp ar +un c +E T +Ġre ad +pr int +Ġre l +Ġfor m +Ġd r +Ex ception +in put +Ġtr ans +#### #### +ord er +B y +Ġa w +it ies +u ff +pl ay +. add +ĠâĢ ĵ +Ġw ant +Ġcom p +ment s +Ġ| | +a z +b e +Ġn umber +Ġre quire +ĠE x +Ġc ol +Ġ key +em ber +Ġt wo +Ġs ize +Ġwh ere +U T +res ult +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ou gh +or ld +o od +u ch +at ive +g er +are nt +Ġ/ * +Ġar g +Ġwh ile +( this +Ġre c +Ġd if +St ate +Ġs pec +r ide +_ F +Ġlo ok +A M +il ity +et er +âĢĻ t +ĊĊ Ċ +ay out +---------------- ---------------- +ag er +Ġc ould +Ġb r +end s +u res +Ġkn ow +et s +ĠI f +ĠS h +. w +b ack +Ġs er +Ġ+ = +Ġf r +() );Ċ +Ġh and +I nd +UL L +I m +() ;ĊĊ +Ġm ost +Ġtr y +Ġn ow +rou gh +> čĊ +ack age +Ġh im +. _ +if y +Ġb reak +Ġ );Ċ +re n +# define +it t +Ġa p +ĉ c +( n +ĠY ou +: ĊĊ +- m +Ġe very +ust om +li ent +oc ument +cri ption +E rror +- b +Ð ¾ +] [ +tr ans +Ġp oint +Ġst d +Ġf il +T ime +Ġm od +Ġ -> +Ġ error +a h +Ġt ext +roll er +lo se +q l +Ġp ol +> < +. B +- c +Ġop en +Ġe st +ĠĠĠĠĠĠĠĠ Ċ +Ġn ext +I M +Ñ Ĥ +O T +à ³ +Ġf ollow +cont ent +ĠĠĠĠĠĠĠĠ ĠĠĠĠ +Ġin clud +H E +ĠR es +Ġh ref +Ð ¸ +Ġc ar +yp es +im age +U n +Ġbo ol +A D +Ġg ame +.F orm +row s +* / +vel op +.D rawing +Ġp ath +is ion +Ġe ach +ĠP l +_t ype +P ath +ne ction +Ġa v +' ). +Ġsup port +EN T +re m +" ). +Ġo wn +Ġc or +c ount +m iss +u ally +Ġm em +st d +i ence +se arch +" ĊĊ +F orm +Ġs ex +en ame +Ġs ign +Ġ et +ĠĠĠĠĠĠĠĠ ĠĠ +', ' +ĠA pp +Ġth ose +o ff +Ġ err +Ġs ystem +Ġbe st +c ode +Ġs ame +Ġd i +us s +Ġc reate +ath er +A rray +. in +f e +S ervice +U N +at s +Ġ Z +al th +Ġm ade +tr ue +A B +Ġm ark +r id +if ied +, čĊ +y n +p ress +Ġg roup +Ġf in +ĠL icense +F ield +eg er +Ġw orld +in ess +t y +Ġpro cess +( b +Ġc re +ar n +iv es +Ġm ain +ide o +_ g +A G +val id +im g +P I +Ġc olor +Ġre port +Ġt ake +ri b +O M +Ġd ay +Re quest +Ġs k +b ers +ĉ s +.A dd +o ot +Im age +Ġcom ple +ol lection +Ġto p +Ġf ree +A S +D e +ĠO n +I G +et a +D ate +Ġa ction +O ver +it or +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +n ot +Ġind ex +h er +ic on +O n +;čĊ čĊ +iv ity +m and +.W indows +O L +Ġre al +Ġm ax +l and +.. .. +r aph +Ġbu ild +le g +ass word +? ĊĊ +âĢ ¦ +o ok +u ck +Ġm essage +t est +iv ers +Ġin put +Ġar t +Ġbet ween +G et +ent er +g round +en e +à ¡ +.l ength +N ode +( i +C lass +f or +ĠâĢ Ķ +t en +o in +Ġ ke +u i +ĠI N +Ġt able +s ub +ĠL e +Ġhe ad +Ġm ust +//////// //////// +. util +Cont ext +Ġor der +Ġm ov +o ver +Ġcont in +Ġs ay +st atic +.T ext +Ġclass Name +pan y +Ġt er +he ad +r g +Ġpro duct +Th is +. âĢĿ +ĠB ut +lo y +Ġd ouble +s g +Ġpl ace +. x +m essage +Ġin formation +pr ivate +Ġo per +c ed +d b +"> +ater ial +ile d +Ġp ut +Q u +Ñ Ģ +un g +m ap +ĉĉĉĉ ĉĉĉĉ +Ġle vel +Com ponent +bo ok +cre en +_ RE +Ġcon fig +ã ģ +O r +. data +Ġd ocument +", " +trib ute +u x +L og +fer ence +p ost +_ e +Ġloc al +and om +ass ert +V al +lect ed +in a +atab ase +A dd +Ġcont ent +.p rint +s igned +r ic +." ĊĊ +Ġf a +! ĊĊ +- f +iv ed +Ġ quest +. ex +Ġf loat +Ġde velop +о Ð +M ap +ad ing +Ġpos s +U E +n amespace +_ O +ĉ b +.G et +> ( +j son +etail s +Ġto o +Ġext ends +ĠN one +Ġf ore +( String +form at +Ġg reat +int er +ca le +Ñ ģ +r on +iv ing +E nt +enc y +x t +o y +Ġmon th +Ġh app +Ġsup er +b ar +def ault +_ de +ord s +l n +( {Ċ +ĠI nd +as es +Ġt itle +Ġcont ext +o h +- p +E m +Ġm et +T est +Ġl ife +_ v +ĠU S +U I +oc ation +m d +Ġ[ Ċ +Ġ ] +s w +Ġin cre +s cript +ent ial +w ays +. de +Ġs rc +Ġc atch +ĠA meric +// Ċ +ĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +Ġp ay +pl it +âĢ Ķ +Ġc oun +ob j +.ph p +Ġch ange +eth ing +' re +ast er +lo s +l ation +ĠĠ Ċ +L e +à ¤ +( { +read y +ĠN o +Ġpos ition +Ġo ld +Ġbo ok +able d +b ug +H and +} ;ĊĊ +is play +av ing +Ġgo ver +Ġv ersion +S ystem +n ect +res ponse +St yle +U p +ang u +Ġth ree +in it +er o +Ġl aw +end if +Ġb ase +em ail +( l +_ V +Ġcon f +AT E +Ġd uring +t es +Ġcon sole +ĠP r +Ġs pe +v es +p ath +ial og +d ition +_t o +ard s +Ġagain st +et work +ĠP h +_ L +c ur +im it +W ith +Ġp ower +i um +' ;ĊĊ +Ġw om +le ft +our ces +at ri +ĠI m +ĠM an +or th +$ { +qu als +es e +_s ize +Ġis s +ot al +- g +i que +r ame +Ġw idth +er g +) ( +itt le +T R +ĠThe y +enc es +r l +on s +Ġl abel +. y +- t +up date +an el +s c +.t o +Ġpro ject +à ¼ +Ġe lement +Ġsu ccess +ĉĉ Ċ +.s h +r am +ch ed +() )Ċ +Ġ( Ċ +Ġd ate +Ġto t +_ ST +A ll +ific ation +ĉ var +Ġt ri +ch em +m y +Ġb ig +ĠA d +ĠA t +ot s +n um +A ct +Ġm ap +er a +co pe +. $ +, âĢĿ +Ġp op +Ġf ew +Ġl en +u id +et ers +u les +Ã Ń +s ource +http s +Ġd em +Ġe ar +######## ######## +Ġm atch +or ies +ac es +ĠC l +Ġn ode +ir c +loc al +un ity +} ;Ċ +Ġan other +< < +og le +Ġs it +ew ork +T E +. I +N S +olog y +ou ght +.C ont +> > +Ġc are +st ate +ĉ private +Ġe ffect +++ ) +_f ile +end ing +L ine +F or +i or +ĠS c +Ġf un +.S ize +ĉ else +] ) +st art +v ious +Ġ} , +our s +Ġle g +Ġs ervice +Ġs ince +ir on +L abel +Ġn on +Ġl os +ict ion +Ġf ull +act er +bo ard +g ress +Ġt urn +ith er +.s ize +Ġb ody +res h +et urn +( _ +y les +orm al +p i +Ġsom ething +! -- +u int +Ġpro du +Ġst and +Ġpro ble +Ġav ailable +m t +ĠB l +Ġ ... +Ġb lock +In put +Ġke ep +C ount +op en +Ġ[ ' +Ġth row +uild er +A ction +Ġth ings +Tr ue +Ġ url +ĠB o +print f +Ġre d +j s +.c reate +ĠO r +St atus +In stance +Ġcont rol +Ġcom e +Ġc ustom +loc ation +m odel +Ġ čĊ +Ġs ource +Ġe as +. out +] ĊĊ +one y +Ġaw ait +Ġpart ic +A P +ub lish +od es +_p ro +p ly +rit er +Ġpro v +Ġm ill +H T +] )Ċ +Ġch ang +Ġas k +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠ +Ġout put +Ġem ail +.p ush +Ġ} čĊčĊ +in ation +atri x +T able +u ccess +] );Ċ +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġdis c +( [ +Ġb usiness +he ight +. html +t a +f ield +Ġrequire d +_ R +Ġgover n +} čĊčĊ +le x +. , +ĠS et +ur ch +// / +t s +a f +Ġm ight +ist ory +S tr +Ġne ver +Res ponse +ar se +ad a +ĠH ow +Ġ* ) +Ġ ; +Ġh ard +A d +Ġinter n +us ed +( data +m od +ann el +Ġn p +ug g +Ġ/ >Ċ +Ġcal led +b ody +Ġch o +( r +_s et +ir d +Ġ> = +Ġ} ;Ċ +Ġo ptions +ĠG ener +Ġhe ight +P oint +Y ou +et y +C lick +Ġsm all +Ġ ide +Ġacc ess +angu age +Ġprot ected +Ġj ob +ĠTh ere +D ef +Ġadd ress +Ġu int +N ot +o o +ap s +< div +ain ed +at ur +Ġs um +- w +ĠD ate +Ġl ittle +Ġf ri +Y PE +Ġp ort +e h +pr ing +_p ath +Ġst atus +a im +bo ol +Ġap pe +Ġo s +. name +ens ion +_ G +Ġup date +Con fig +a ff +ER R +Ġ< = +at ely +# if +u ction +ĠT e +Ġl ink +ĠU ser +.f ind +. org +m e +Ġg iven +O ut +# endif +Ġbet ter +P age +Ġfe el +en n +M L +Ġal ready +Ġinclud ing +o ogle +r u +ic ally +pro p +le an +out er +Ġal ways +ord ing +I f +or age +Ġp arent +v is +ĉĉĉĉ ĉĉĉ +Ġg ot +st and +Ġle ss +/ s +ĠA ss +ap t +ire d +ĠA dd +Ġacc ount +p loy +Ġd er +res ent +Ġl ot +Ġval id +ĉ d +Ġb it +pon ents +Ġfollow ing +_ ex +S ON +Ġs ure +oc ial +Ġp rom +ert ies +he ader +.p ro +Ġbo olean +Ġse arch +k en +Ġor ig +Ġ er +E d +E M +a ut +l ing +al ity +By Id +b ed +ĉc ase +eth er +pos it +Ġinv est +ĠO R +Ġs ays +miss ion +AM E +Ġtem p +o ad +Ġre st +in fo +Ġinter est +A rg +Ġper form +pon s +ĠV iew +Ġv er +l ib +( const +U til +List ener +ar ge +Ġm ult +Ġd ie +Ġs ite +../ ../ +E L +Ġval ues +Ġ} )Ċ +p en +N o +ic ro +Ġbe h +Ġ' ./ +ac y +re c +() -> +ĉ ĠĠĠ +" )) +Cont ent +_ W +ple ment +Ġw on +Ġv ideo +ad i +p oint +% % +Ġg l +erv ed +v iron +I F +ut ed +ã ĥ +' m +Ġc ert +Ġpro f +Ġc ell +ar i +Ġpl ayer +a is +Ġc ost +Ġh um +( R +Ġoff ic +k s +.t ext +at ures +Ġtot al +Ġ*/ ĊĊ +o pe +Ġst at +U M +Ġlo ad +ight s +Ġc lear +u ro +Ġte chn +up port +I R +Ġ row +Ġse em +Ġ q +Ġsh ort +ĠN ot +ip p +G roup +se ction +m ax +ir l +Ġover ride +Ġcom pany +Ġd one +" );čĊ +Ġg re +. Re +Ġbel ie +r ist +Ġhe alth +AN T +() ĊĊ +ĠB e +. value +ĠG r +ott om +Ġarg s +P T +st atus +f unc +um ents +- h +N umber +: čĊ +ĠL og +er ver +Ġ) ,Ċ +am ent +Ġob j +in c +Ġchild ren +ic y +I Z +and s +ab ly +Ġdist rib +Ġc ur +er ial +Ġd ays +re ated +re ct +- l +ir m +idd en +om b +Ġin itial +.j s +Ġ â +Qu ery +Ġon line +im al +. con +a u +U rl +cont rol +ire ction +Ġin stance +OR T +ĠF r +wh ere +Ġjav ax +Ġorg an +ap ter +Ġre ason +o ptions +ĠM ar +( a +Ġwith in +.âĢĿ ĊĊ +O DE +_ DE +ad min +end ed +Ġdes ign +ĠD ata +un e +ĠF ile +ro ot +Ġc ent +Ġa rr +_ add +l en +p age +, ' +_ str +Ġb ro +ab ility +ou th +/ c +p ose +irt ual +ear ch +_ url +arg in +H ttp +Ġs chool +av a +Ġcons ider +.l abel +ĠA rray +we b +o pt +.print ln +ul ation +Ġf unc +P L +Ġ" \ +ĠT ext +act ory +(f unction +n ull +Ġen g +d own +Ġin clude +ĠE n +ĠD r +Ġd b +! ! +s ide +Ġin it +quire d +ĠS he +C olumn +re act +Ġan n +Ġst op +Ġl ater +ĠTh at +ent ion +d f +U G +I LE +Ġc lient +ra ft +ff er +PO ST +el per +Ġlo ve +qu ote +ou d +Ġj son +Ġab le +Ġm en +A X +ĠC opyright +à ¶ +av ig +re q +C lient +} );Ċ +.C om +er c +il t +pec ial +_c om +ro om +. Name +Ġg ive +am b +i ke +Ġcon dition +cl ient +ator s +: " +Ġc opy +ut ure +ivers ity +ern al +{ { +ĠC an +ou nc +d o +Ġo cc +Ġapp ro +th ers +z e +Ġe ither +ĠF l +Ġimport ant +Ġle ad +at tr +AR T +E qual +Ġd a +et ch +ent ity +Ġfam ily +add ing +Ġo ption +Ġex ist +ic a +ĠO bject +' ve +v ers +ition al +out put +ĠTr ue +ĠO F +_t ime +Ġof fer +Ġ} );ĊĊ +H ER +eg in +" " +Ġw ater +Ġc he +ĠM y +ore d +Ġst ep +anc es +C K +A Y +à ¸ +str uction +( C +ou ch +St ream +act ive +am a +Ent ity +pro duct +() {Ċ +Ġgovern ment +ĠI D +aj or +A nd +Ġdis play +Ð » +Ġt imes +Ġf our +Ġf ar +Ġpres ent +ĠN S +Ġ\ Ċ +ue st +Ġb as +e cho +ch ild +if ier +Hand ler +Ġl ib +Prop erty +trans lation +Ġro om +Ġon ce +Ġ[ ] +cent er +================ ================ +Ġresult s +Ġcontin ue +Ġt alk +_ get +Ġg row +.s w +e b +ĠP ublic +O P +ec ute +ol s +Ġ ** +" );ĊĊ +Ġm ass +ure d +.c lass +om ic +Ġme an +ip s +Ġa ut +);čĊ čĊ +Ġun til +Ġmark et +Ġare a +u it +Ġl ength +ĠW ith +struct or +e vent +"> < +ĠS p +I V +Ġm us +if f +Ġk ind +a uthor +ound s +m b +_ key +w idth +posit ory +Ġl ight +u k +R ow +oh n +al f +viron ment +app er +ollection s +Ġs ide +_in fo +Ġex ample +im ary +Ġw r +Ġc amp +cri be +" / +Ġm iss +w ay +Ġb ased +Ġpl an +V is +om ain +un k +Ġaw ay +U P +< T +O S +i od +ĠM on +âĢĻ re +Ġli k +à § +iv ely +. v +im er +iz er +S ub +Ġbut ton +ĠU p +Ġexper ience +C L +Ġre nder +_ value +Ġn ear +UR L +al t +Ġcoun try +ib ility +() ,Ċ +e ad +Ġa uthor +Ġspec ific +b ase +( name +on es +ĠD o +Ġal ong +y ear +Ġexp ress +. ' +en v +Ġbeg in +Ġso ftware +Ġim p +Ġw in +ó n +Ġth ing +Tr ans +ĠT HE +Ġ< ? +Ġwh y +Ġdoes n +i j +g ing +ĉ g +Ġs ingle +off set +ar ning +og raph +le y +_c ount +Ġan al +cre ate +/ m +ĠR eg +un ch += $ +is k +Ġright s +( M +Ġ"" "Ċ +ap er +.m odel +Ġp o +em pty +art ment +Ġa nt +ĠWh en +Ġwom en +ĠE d +Ġse ason +Ġde st +à £ +( h +Ġposs ible +Ġse ver +Ġb tn +Ġdid n +Ġs ent +Ġen c +Ġcomm and +Ġ ],Ċ +_ x +Ġre cent +ol ution +v ector +ĠB y +ĠM ay +ĠA ct +» ¿ +Ġm oney +IN T +bs ite +ĉ p +. čĊ +ï »¿ +s l +atter n +ĠC lass +Ġto ld +ud io +c urrent +Ġe qu +Ġa uto +ĠSt ate +d a +ms g +)) ;ĊĊ +Ġwork ing +Ġqu ery +ĠB r +Ġw indow +a uth +on ly +ĉ t +Ġle ast +ag n +Ġex pl +it ter +ar ing +Ġc olumn +ĠGener al +": " +er al +ri or +Ġrec ord +I B +E X +Ġd at +Ġm aking +u ed +ĠC ar +em p +" . +ĠM ed +Ġc lose +Ġper cent +Ġp ast +( g +: ( +Ġw rite +Ġm ove +Ġp at +Cont rol +.T o +Ġv i +*/ Ċ +in ate +' ll +ag ed +N ull +Ġspec ial +IZ E +Ġc ity +/* Ċ +ĠE ng +ix ed +in ary +p y +Ġe ff +ar io +Ġt ell +av or +Ġse lect +le vel +im um +op er +B uilder +I P +') ,Ċ +es c +Ġf ont +" ;ĊĊ +ĠA m +ish ed +ill s +Int er +O W +Ġcour se +Ġl ate +idd le +Ġam ount +Ġas ync +in o +c ul +Ġ ì +and le +_ user +Ġb en +ĠC al +Ġ$ _ +ĠR ep +Ġen ough +T oken +. user +( j +S c +W idth +n ow +at form +Ġlook ing +Ġh old +M odule +IT Y +v o +is on +.D ata +y c +Ġp ot +ĠTr ump +id ual +id es +r t +Ġprop erty +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠ +am ework +g o +Ġl ow +Ġpar a +Ġpr ice +ur y +Ġto day +ro y +Ġ' / +Ġpol it +Ġ' ' +ym b +P h +Ġad v +Ġatt ack +ĠS te +RO M +an a +Ġme ans +Ġst ory +id s +ak en +Ġme et +Ġm om +ĠâĢ ĺ +Ġ? > +Ġd en +ob ile +ch ange +ĠĠĠĠĠĠĠĠ ĠĠĠĠĊ +ic i +n a +ĠF orm +Ġs ort +Se lect +p are +Ġth ought +_ con +Ġt ask +oc us +ĠD E +ĠM in +Ġo pt +ĉb reak +um er +K E +th en +Ġd et +ĠT est +port s +Ġre view +(' / +m ove +Ġsw itch +ER T +p atch +ann ot +ã Ĥ +Ġab ove +it ive +Ġquest ion +ĠQ u +ãĢĤ ĊĊ +g le +Ġw ord +Ġprov ide +ĠR eturn +Ġre search +ã o +u str +Ġp ublish +chem a +} } +ĠC ON +- in +all back +Ġco ver +\ \ +c olor +ĠI S +Ġwh ether +im ate +is c +B ar +Ġd iv +B e +our n +Ġh aving +le m +pl ayer +ab s +am era +ne y +Ġex c +get her +pl ied +a o +[ $ +Ġ+ + +i pe +sh ow +/ d +[ : +ag ement +le v +_ ID +r ary +ad es +_ se +a use +Ġem ploy +Ġ*/ čĊ +Ġf re +Ġ' @ +Ġcomple t +Ġl arge +r al +\ x +Ġf ac +< String +Ġcre ated +up er +.st ate +Ġh ost +ener ic +/ b +( ! +wh ile +i as +B UG +Ġ );ĊĊ +Ġro le +Re g +ĠC olor +St art +Ġp orn +t op +Ġwe b +Ġde v +Ġde al +++ )Ċ +Int eger +pos ition +. on +Ġ( " +ä ¸ +Ġproble m +s v +Ġp ress +AB LE +AT ION +ĠSe e +an ch +Ġth ough +le ep +Ġ< !-- +Ġpoint s +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠ +. J +Ġ :: +p tr +D B +++ ;Ċ +.p ng +n ode +so ft +pon d +Ġe ver +-------------------------------- -------------------------------- +M enu +(' # +Ġs ervices +p g +} )Ċ +param s +Ġact ually +Ġ" / +Em pty +M ethod +Ġid ent +un ic +Ġmill ion +Ġa ff +st yle +Ġcon c +i os +ign ment +UL T +P r +" ;čĊ +Ġunder stand +u ary +Ġhapp en +Ġser ver +ĠC o +S C +Ġle s +Ġfile s +G rid +s ql +Ġof ten +Ġin fo +_ tr +s rc +on y +Ġsp ace +um b +Ġpass word +Ġst ore +, ĊĊ +ĠWh at +g ed +ĠF alse +U s +sw er +_ index +Ġform at +m ost +s m +N ew +Ġd etails +Ġpro b +ĠAN D +() čĊ +il ar +Ġ$ { +ry pt +.C ollections +$ this +ĠF ree +_ of +(f alse +d ated +Ġ> > +Ġf ace +CT ION +Ġs ave +Ġt yp +de v +(" # +AG E +cont ainer +ed it +Q L +Ġitem s +Ġs ocial +i en +ĠRe act +) .ĊĊ +Ġm ar +Ġre du +ĠR E +.p ut +Ġm ajor +C ell +n ext +Ġexpect ed +Ġy et +Ġin div +trib utes +at is +am ed +Ġf ood +S ource +( string +Ġ+ Ċ +it es +d r +Ġmem bers +Ġcom b +item s +ĠP er +T H += True +Ġb ar +_ SE +com m +( w +)ĊĊ Ċ +Ġs end +Ġin c +un signed +F A +Ġparam s +app ing +ro s +ug in +f a +Ġcon nection +Ġ} ;ĊĊ +Ġbe come +M ode +Ġe v +Ġdif f +ĠUn ited +He ight +ful ly +im ages +Ġm akes +Ġg lobal +Ġcont act +' :Ċ +Ġab s +а Ð +f loat +Ġex cept +ĠP ol +Ch ild +t yp +Ġcert ain +i ón +O UT +Ġim pro +ile s +Ġ-- >Ċ +ĠP art +val ues +os s +/ ** +il it +ĠE vent +cur ity +st er +Ġchar acter +Ġnew s +Ġ" , +Ġde vice +c el +log in +he et +Def ault +@ " +ĉ Ġ +c lick +( value +ĠA b +Ġpre vious +ERR OR +oc al +Ġm aterial +Ġbel ow +ĠCh rist +Ġmed ia +co ver +ĠU I +Ġf ail +Ġbl ack +Ġcom ponent +ĠAmeric an +Ġadd ed +Ġbu y +st it +Ġc ame +Ġde lete +prop erty +od ing +Ġc ard +rop s +Ġhttp s +Ġro ot +Ġhand le +C C +B ack +em plate +Ġget ting +_b y +m ail +_s h +. assert +ĠD ec +( true +Ġcom put +Ġcl aim +' => +ĠS ub +Ġa ir +op s +n av +em ents +( id +Ġent er +ang ed +E nd +Ġloc ation +Ġn ight +Ġdo ing +ĠR ed +l in +}ĊĊ Ċ +vid er +Ġp ick +Ġw atch +ess ages +Ġhum an +Ġd am +p end +d ir +Ġt ax +Ġg irl +re et +Ġbo x +Ġstr ong +( v +re l +Ġinter face +Ġm sg +f ect +_ at +Ġh ouse +Ġtr ack +' );ĊĊ +j e +ĠJ ohn +ist r +( S +ub e +Ġc e +itt ed +V ER +* ) +p arent +Ġapp lication +an y +.sw ing +Ġp ack +\ u +Ġpr act +Ġse ction +ct x +Ġun signed +.P oint +ĠO ne +Ä ± +ip le +a id +Ñ ĥ +V ector +by te +Ġw ait +Ġà ł +à ¥ +Ġto gether +Ġth rows +F O +' )) +h ost +is ing +. view +Ġter ms +fr amework +- r +Ġapp ly +Ġs ession +O ptions +ugg est +Ġo thers +w itter +Ġf und +In it +__ ( +ens or +G ET +Ġsever al +i i +[ j +I O +Ġtem plate +P osition +Ġe con +ach ine +Ġ il +.s pring +m ain +el t +im ent +Re c +m m +ĠUn iversity +urs or +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠ +G L +ict ure +ith ub +c er +c ast +F rom +a les +Ġsub ject +p assword +n y +Ġes c +.w rite +ï¼ Į +Wh at +. H +Ġh istory +ĠF e +Ġindiv idual +un it +Ġ-- > +Ġd u +I ST +Ġus ers +f s +f alse +un t +T itle +Ġm ot +Ġf uture +ach ed +Ġstart ed +Ġm ode +Ġ' < +_ array +Ġa x +'] ;Ċ +i res +Th ere +ug ht +t ml +pos ed +ic ult +Ġto ok +Ġg ames +Ġ} } +Ġ? >Ċ +Ġproduct s +I s +Ġb ad +ĠD es +.p ath +' ĊĊ +ĠP ost +av el +( : +Ġneed s +Ġkn own +F l +Ġex ec +Ġse en +um e +Ġb order +Ġl ive +tem p +P er +Ġvar iable +i et +ĠD ef +Ġg e +em e +_b ack +f irst +Ġprovid ed +//////////////// //////////////// +Ġfil ename +Ġh ope +ul y +a uto +f ind +_ string +b tn +it ude +At tribute +Ġyou ng +.t xt +Ġwe bsite +ĠP rop +Ġe y +> ();Ċ +ion al +AR R +iction ary +ur ther +. +t x +Ġp ur +u el +ymb ol +u ation +ang er +Ġback ground +ec ess +ef ined +.... .... +Ġdes cription +Ġrep resent +") );Ċ +press ion +row ser +Ġser ies +ward s +($ _ +a ise +Ġh ot +ac ity +ri es +action s +C reate +ad io +amp les +Ġorig inal +ens ive +f ont +st ream + using +.spring framework +ser ver +Ġb ill +AC K +il ename +Ġfr ame +Ġ= Ċ +Ed it +adi us +Ġd raw +ank s +Ġd eter +Ġcom es +_ int +Ġfore ach +ang le +Ġe lect +pect ed +He ader +ist ration +F alse +ĠG ame +Ġfil ter +Act ivity +Ġl arg +in ition +Ġ" < +is ed +Ġrem ove +ĠTr ans +m et +se e +Form at +Com mand +ĠE X +N one +Ġfr ont +A SE +ĠR ec +ound ation +Ġv o += \" +( * +Ch ange +.W rite +g roup +i ents +u y +******************************** ******************************** +Ġd ig +h r +( - +Ġg en +n umber +ve c +uro pe +ent ry +L L +Ġst e +Val id +'] , +_p aram +Ġse lected +Ġacc ording +ĠD is +Ġ util +B uffer +_ error +Ġass oci +_S IZE +Ġw or +Ġprint f +r ag + ł +D D +ĠV al +Ġact iv +E ng +et ime +Ġv irtual +a ign +a ur +ĠP res +ĠEx ception +Ġany thing +ĠO ff +Ġh ours +Ġw ar +Arg s +ag ing +Ġmodel s +ĠT ime +O b +am s +j oy +Ġear ly +. read +Ġc enter +ĠIn itial +Ġl anguage +l ength +x y +Ġs n +Ġin f +P ost +Ġag o +Ġeas y +_c ode +ĠAN Y +_ ch +Ġdown load +( T +av ed +âĢ ĵ +Ġstud ents +Ġf ig +l ight +x x +Ġbu ffer +ĠD ep +ĠM ath +IT H +Ġvar i +Ġd ue +F actory +Ġp or +Ġe p +ot ype +Ġcan not +Ġwh ite +< int +ter n +Ġreg ister +Ġpre d +cl us +_d ate +Ġ/ ** +Ġa uth +Ġ[ ]Ċ +Ġper iod +n own +Ġv ot +Ġs creen +' d +T ypes +Ġt mp +е Ð +ur al +Ġben ef +_ y +Ġn et +ĠSt ates +'] [' +ĠN e +ĠN OT +Ġn eg +Ġcomm on +s cope +Ġc red +g es +_T YPE +Ġs uggest +o om +.ĊĊ Ċ +Ġac cept +Ġr andom +er m +ĠV ector +w ith +T ER +( str +Ġres pons +Ġh it +.S et +gr id +ri a +Ġc lick +und le +C ase +ins ert +Util s +Ġ"" " +Ġim plement +at al +tem pt +tem plate +oc r +return s +Ġplay ers +us ers +ed ef +ĠTh ese +Ġam ong +Ġde b +h a +.get Element +Ġc irc +Ġan swer +Ġw alk +Ġt reat +ĠG e +ĠC reate +Ġa ge +Ġre q +O ST +ang ular +Ñ ı +Ġf ive +Ġdistrib uted +Ġfri end +T P +Ġc lean +ow s +.Control s +d is +Ġw ords +. io +z y +Ġhe ader +ĠC heck +âĢĻ m +j ust +h older +=" čĊ +. annot +Ġcol lection +' . +Ġsim ilar +Ġt aken +(" % +Or der +'] Ċ +-m d +ĠT H +ac ed +Ġis n +/ j +Ġs on +gr aph +ĠInt eger +Ġn ecess +re en +Ġ um +Ġ\ < +Ġmom ent +Ġbr ing +Ġind ic +ys is +Le vel +ver se +urre nc +_t est +Ġent ire +D own +Ġ}ĊĊ Ċ +( result +ĠRe ad +à ¨ +M od +Ġtry ing +") ,Ċ +Ġm ember +ĠC or +OD O +- control +un time +ĠS im +D ialog +pl ot +_ on +Ġph ys +} / +Ġn amespace +ĉ čĊ +ac c +Pl ayer +A RE +Ġf oot +Ġbo ard +p art +Ġs us +w ise +ĠM c +Ġp ush +AT A +Ġp lease +ri ed +we et +b it +id ed +V E +ĠS w +U B +Ġt ypes +ed ia +Ġc los +ace book +Wh en +Ġed it +ig ger +Ġen erg +Cont ainer +Ġph ot +ĠC ount +ĠE urope +.I s +ĠR uss +pe ed +ĠS tr +Ġp y +Ġc ult +Ġdef ined +cc ount +Ġob t +.L ocation +Ġth read +il le +Ġinst ead +str ong +ĠS ec +U RE +Ġide a +. se +em y +select ed +Con nection +ac ing +th read +.n ext +Ġc oll +Ġfil m +ist ic +Ġcomp et +Ġcon n +th ough +Ġcom pan +ock et +Ġte ach += ( +Ġph one +Ġact ive +de lete +tr ies +Ġm o +Ġde ath +} );ĊĊ +oc ol +W idget +Ġart icle +ro du +and id +Ñ ĭ +ĠC r +k a +() : +lo od +ĉĉĉ Ċ +Ġal most +Ġs ell +erv let +ri p +Un it +Ġapp lic +Ġcon nect +Ġfe ature +Ġv ia +' ), +Ġl im +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ĠG u +Eng ine +Ġen s +Ġen vironment +b lock +HER E +N ULL +g y +t ag +) ). +ex p +Ġcom pl +Ġinst all +Ġcomple te +que ue +atur al +Ġgener al +th on +Ġask ed +o res +( res +Ġres erved +S P +ĠâĢ ¦ +Å Ĥ +Ġsign ific +O ff +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠ +ĠA g +ĠJ ust +ĠE rror +Ġin fl +ad ata +Ġ icon +ask s +' ' +_ LO +? . +ac count +Ġ( * +' )ĊĊ +r ap +_ var +ĠF OR +Ġpart y +ĠY our +c at +str y +. new +bo ot +ĠN ov +Ġv ector +Ġn ormal +Ġf urther +Re pository +Ġd atabase +att le +Ġmus ic +Ġspe ed +Ġd oc +pro cess +IG HT +.p arse +Ġt aking +Ġvi ol +ce ed +ĠA fter +Ġfor ward +Ġc rit +"/ >Ċ +ro t +Ġfa iled +ef ore +Ġconc ern +o e +b a +Ġs ender +Ġter m +h as +=" # +Ġpot ential +N um +Ġpublish ed +.c lose +ĠIm age +str aint +U D +ĠO b +Ġprob ably +l im +" :Ċ +olum e +Ġcon sum +ag ue +ens ions +Ġinvest ig +- year +') ; +-s m +Ġen joy +or ig +er ing +c p +le ased +ple ments +Ġreturn s +p at +B O +ĠH ouse +.L abel +Ġwe ight +igh b +Ġcondition s +Ġex ception +d escription +Ġtr ad +- to +Ġ{ } +Ġmod ule +EN D +. ap +.p rops +Ġcon structor +av es +Ġf avor +ĠN ow +; i +ĠM ain +_ k +er ies +âĢĻ ll +trans form +imest amp +P re +Ġm er +. res +st ant +L ocation +_N AME +Ġlos s +Ġ ĊĊ +n et +Ġeng ine +B lock +Ġiss ues +Ġpar se +ĠB ar +Ġst ay +ĠJ SON +Ġd om +air s +w ner +Ġl ower +", čĊ +ĠD em +uf act +Ġp s +Ġper fect +R L +Ġed uc +l s +em ory +ARR ANT +u ge +Ġex act +. key +al led +e ch +ie f +\ / +o ke +Ġfor mer +al loc +Ġs ix +id a +Ġm argin +Ġhe art +al d +p ack +.getElement ById +ĠW ARRANT +Ġr ather +Ġbuild ing +er man +lic e +Ġquest ions +iz es +le ge +irect ory +Ġj e +Ġc as +pro ps +ut f +Ġse curity +Ġhow ever +we ight +Ġins ide +Ġpres ident +Ch ar +ĠW ITH +.m ap +Ġgr aph +Ġt ag +_st atus +Ġat tempt +op p +us es +ĉ const +Ġr ound +, $ +Ġfri ends +Em ail +? > +Res ource +KE Y +os p +. query +ĠN orth +able s +ist rib +_c lass +el lo +Th at +Ð º +pecial ly +ĠPres ident +Ġcamp aign +Ġal t +are a +Ġch all +Ġop port +.C on +Ġenerg y +li ke +. string +ing ton +) * +y y +Ġprof ession +ir th +Ġse g +æ ľ +Ġh or +i ers +c an +Ġbeh ind +Pro duct +f g +ĠS k +.j pg +? : +] ;ĊĊ +Ġcall back +ĠH ttp +Ñ Į +l ong +M S +AT H +Ġr aise +Ġwant ed +row n +ut or +l t +] = +el ine +M A +Ġse par +c s +se mb +D is +bs erv +ĠW ill +Ġpol icy +Ġth ird +ph one +Ġb ed +/ g +. __ +ĠIn c +iz ing +.re move +in stance +.t ype +Ġs erv +E ach +Ġh ar +ĠM essage +( key +SE LECT +P os +)) ;čĊ +Ġre comm +Ġtr aining +ĠE nt +ĠCh ar +ic ht +(f ile +Ġp rior +G ame +Ġex it +Param s +.c ore +P C +n es +anc ed +( request +P assword +} >Ċ +Ġm ag +Ġre lease +Ġsh all +ud ent +ĠS outh +and o +: ' +.Tab Index +s k +ann er +is set +Ġout side +led ge +Ġ å +ĠR ob +Ġim m +! Ċ +ĠWe b +D es +B C +anc ial +R oute +D ec +fer ences +Ġp urch +ĠM odel +ct or +g n +_st art +_ un +. * +is es +Ġg round +Ġun ique +Ġbe aut +{ " +Ġp our +ĠO ct +Ġt ree +set s +_ res +') -> +_re g +(" \ +Ġby te +B l +Ġd ating +Ġm atter +ĠR em +Ġ' ../ +ĠA ug +ĠL a +Ġ$ ( +ourn al +i am +Ġshow s +w rite +Ġb all +Ġsim ply +Ġf ast +Ġmem ory +A SS +ĠO f +ov ed +ant e +a ul +ist ry +)) );Ċ +Ġf it +< string +Ġpolit ical +anc el +_ . +c ard +.c urrent +o ch +_ image +\ t +# Ċ +( L +Ġindu stry +com ing +Ġex tra +Ġreport ed +.st art +Ġres ources +Ġim g +fl ow +_E X +(n ull +ĠP re +Ġwr ong +inter face +Param eter +n ers +á » +t ure +ers ist +oun try +Ġseem s +al ance +de st +ĉ String +Ġm aint +Ġun it +act ers +ĠT R +if ul +export s +pro ject +App lication +leg ate +Ġt akes +ter m +Ġet c +ust er +Ġappe ar +add ress +Ġf em +h s +Ġh om +, - +Ġdiff icult +Ġcom ing +O pen +Ġset tings +ĠW ar +ĠTh en +Ġaut om +ĠF oundation +Ġqu ite +D escription +Ġb log +i qu +P S +_f ield +J son +SS ION +ĠS ch +ĠL O +Ġdes cri +Ġevery one +Ġpret ty +Ġlong er +Ġm enu +Ġcurrent ly +se c +Ġrelations hip +################ ################ +ĠM ap +as et +Ġparam eters +Ġcr ush +" čĊ +IL ITY +ig ration +Ġc out +t otal +Ġn ames +nd ef +") ; +ri end +yn amic +Ġeff ort +Ġact ual +Ġfield s +O UN +t ers +Ġf ix +_m odel +Ġc ases +C A +M y +Inter face +ĠS E +] ] +al le +ĠN ational +ĠArray List +in line +. V +ar a +ref ix +as c +Re ader +ĠÐ ¿ +ast ic +( () +C l +.annot ation +Ġperform ance +ail y +.to String +.n et +view s +. end +ay ers +l ate +ĠA pr +ed eral +'] ) +.b ody +Ġhigh er +_f l +c r +al ert +_n ode +ĠG oogle +Ġit self +A uth +urrenc y +Ġsignific ant +app end +Ġres pect +str ap +Ġun a +riter ia +P ORT +.ap ache +Out put +Ġpro gress +Ġm id +ĠM icrosoft +Ġres ource +ab lish +Ġd im +. load +.A pp +Ġd irection +Ġadd itional +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠ +Ġnum bers +Ġcompan ies +.T h +Ġs ound +user name +Ġstat ement +Ġal ert +Ġcon tract +h ome +_l ength +.Com ponent +e v +. Ex +ï¼ ļ +" ; +ĠH igh +Ġ )ĊĊ +ĠP oint +op h +Ġl ines +-> _ +" )ĊĊ +o x +app lication +Ġ ]Ċ +ĊĊĊĊ ĊĊ +Ġso on +ction s +ing er +Ġj oin +ĠP e +Ġ ë +Ġl as +. E +c ss +/ or +ĠSt art +ĠT O +Ġsub s +con n +com ponents +DE BUG +qu are +F unction +end ar +. index +Ġf ill +Ä Ļ +Ġcho ose +h ow +ĠAmeric a +ass ets +-------- ---- +ĠV alue +Ġoff ice +Ġv eh +Ġtrans form +ĠAr t +Ġin de +Ġf n +Ġim plements +ang o +ple te ++ " +t mp +am ily +Ġhas h +miss ions +E ST +g t +Pro vider +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +Ġfl ag +Ġpartic ip +d en +ĠReturn s +Ġnot e +ü r +p m +ide os +Ġspec ified +ĠE N +est er +ol id +Ġup on +( std +ĉ v +Ġ' \ +u z +Ġv ert +Ġv ict +ĉ self +Ġ" $ +. k +Ġgroup s +g ithub +l ang +Ġm ut +T O +Ġv e +ĠP lease +;ĊĊ Ċ +ac cess +Ġ{ " +re a +Ġr isk +ick er +og gle +ĉ while +AN G +.s end +Ġwom an +Ġget s +Ġ ign +ĠI d +_ log +ON E +Ġe vid +ĠH ar +_s ub +Ġend l +Ġinclud ed +() );ĊĊ +ĠA p +ig r +Ġs em +ĠBl ack +d oc +_t able +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +- up +Ġca use +Ġ .. +Ġv an +_d ict +Ġf ocus +IN D +CE SS +.L og +Ġmult iple +id o +Ġreg ard +- M +and ler +our se +Ġde g +. U +Ġadd ition +Ġvar ious +Ġrece ive +е н +ĠH T +Ob j +D F +Ġincre ase +ĠO pen +] ; +Ġcomm it +? Ċ +ateg ories +at ory +sh ip +ĠM ich +Ġh tml +rom ise +Ġle ave +Ġstr ateg +av en +ĠCon sole +k nown +- n +_ LE +.com ponent +Ġb re +S ession +i ance +Ġal ign +typ edef +_ result +ĠW HERE +.s plit +Ġread ing +FA ULT +Ġc lo +Ġnot ice +_p r +ar ter +Ġlo ck +Ġstand ard +et ic +ell ow +Ġp adding +ĠH is +Ġst ates +_c ast +( P +a a +Ġintern al +e an +ĠP RO +ĠK ey +Ġes pecially +m ing +Ġc ross +Ġn ational +_ object +f ilter +Ġs cript +. update +_ i +ĠAss ert +/ core +%% %% +Ġproble ms +ist or +Ġ. = +Ġar ch +Ġwrit ten +Ġm ilit +M ENT +. ch +ca pe +ĠM us +_ config +ĠA PI +fo ot +Ġim ages +end l +. In +F irst +Ġpl atform +.pro t +O ption +st e +ĠT ODO +Ġfor ce +. cont +ĉ echo +ĠD av +P tr +( B +R T +ĠB ase +] [' +Ġann ounc +con sole +ĠP y +d s +. as +Ġpre vent +ap an +Ġ{ ' +} ' +Ġde ad +V AL +Q UE +**************************************************************** ******** +Ġch arg +R eturn +Ġf ul +d om +Ġr ules +Ġmod ify +Ġe val +h am +at ement +\ < +ul a += False +R A +Ġcont ains +Ġst ack +m ar +Ġ{ }Ċ +Ġund efined +A ss +ĠCh ina +ve y +* Ċ +Ġplay ing +) / +act or +Ġb ottom +li er +ĠN umber +Ġcou ple +D C +ĠS O +g or +.set Text +s uccess +com mand +F ilter +ĠO ur +_ item +Ġc tx +Ġro ad +V ersion +c ase +ur t +av ior +y ch +semb ly +ĠPro duct +Ġh eld +a fe +Ġinclud es +< quote +Ġa void +ĠF in +ĠM od +Ġt ab +an o +à ± +ipp ing +- e +Ġins ert +t arget +ch an +.M odel +IM E +\ Ċ +Ġm achine +av y +ĠN O +ĠInt er +Ġoper ation +mod al +T ag +] : +Ġprodu ction +Ġare as +Ġre n +_f rom +n bsp +Ġoper ator +m en +app ed +_p er +z en +(" . +.s ave +=" {{ +Ġt or +( response +Ġc andid +Ġcon v +a iled +ĠL ib +com p +ur a +ï¿ ½ +ĠH ere +Ġarg ument +h ood +Ġest ablish +ograph y +Ġon Click +amb da +Ġs ch +Ġmov ie +Ġse c +Ġact ivity +Ø § +Ġs ql +_ all +inc ip +Ġprovid es +Ġs ys +ack et +Ġwas n +Ġus es +ĠF unction +.g oogle +ĠRes ult +Vis ible +ag ma +el come +ĠS y +ĠC ent +AL SE +ac ión +EX T +Ġl icense +ĠL ong +Ġacc om +Ġab ility +. height +Act ive +olog ical +ol y +)) , +.S e +Ġparam eter +pr ite +AB ILITY +.s ervice +ĠG roup +_ query +ĠI tem +in ing +Ġj ud +im s +f ix +ind er +ag ram +Ġfunction s +Ġexper i +ĠE m +Ġro t +Ġp en +.b tn +ĠA S +#if def +Ġcho ice +ĠP age +_P RO +Q U +å ı +ant ity +Â Ń +word s +Ġread only +Ġf lex +prot ected +ĠAn y +Ġchar acters +enc ed +ĠJ uly +il er +C ard +ur ance +Ġre v +.e vent +al y +Ġwon der +ĠP ort +Ġleg al +ro le +Ġt en +Ġgo es +M P +wh ite +): čĊ +)) čĊ +Ġre ference +Ġm is +ĠPro ject +ick s +> & +C ON +Ġre pl +Ġreg ular +St orage +ram ework +Ġgo al +Ġt ouch +.w idget +Ġbu ilt +d es +P art +( re +Ġw orth +h ib +g ame +ĠÐ ² +ac ion +ĠWh ite +(t ype +( ` +Ġn atural +Ġin j +Ġcal cul +ĠApr il +. List +Ġassoci ated +ĉ System +~ ~ += [ +Ġst orage +Ġby tes +Ġtr avel +Ġs ou +Ġpass ed +! = +as cript +. open +Ġgr id +Ġb us +Ġrec ogn +A b +Ġh on +ĠC enter +Ġpre c +b uild +HT ML +ĠS an +Ġcoun tries +a led +t oken +k t +Ġqu al +L ast +ad ow +Ġman ufact +id ad +j ango +N ext +x f +. a +Ġporn o +ĠP M +er ve +it ing +_ th +c i += None +g s +Ġlog in +at ives +'] );Ċ +Ä ħ +Ġ ill +I A +child ren +D O +Ġlevel s +Ġ{ { +Ġlook s +Ġ" # +To String +Ġnecess ary +ĠĠĠ Ċ +c ell +En try +Ġ' # +Ġext rem +Select or +Ġplace holder +L oad +Ġre leased +O RE +En umer +ĠT V +SE T +in q +P ress +ĠDep artment +Ġprop erties +Ġres pond +S earch +a el +Ġre qu +ĠB ook +/ Ċ +( st +Ġfin ancial +ick et +_in put +Ġth reat +( in +Str ip +ì Ŀ +ç ão +Ġevid ence +)) ; +ĠB ro +Ġ[ ];Ċ +Ġ ou +b uf +S cript +d at +Ġr ule +# import +=" / +S erial +Ġstart ing +[ index +a e +Ġcon trib +s ession +_ new +ut able +o ber +Ġ" ./ +Ġlog ger +Ġrecent ly +Ġreturn ed +č čĊ +)) )Ċ +ition s +Ġse ek +Ġcomm unic +Ġ" . +Ġuser name +E CT +D S +Ġother wise +ĠG erman +. aw +Ad apter +ix el +Ġsystem s +Ġd rop +Ġstruct ure +Ġ$ ("# +enc ies +ann ing +ĠL ink +ĠRes ponse +Ġst ri +Å ¼ +ĠD B +æ Ĺ +and roid +sub mit +ot ion +( @ +.t est +ĊĊĊĊ ĊĊĊĊ +] ;čĊ +Ġdirect ly +Ġ" % +r is +el ta +A IL +) {čĊ +m ine +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠ +( k +b on +as ic +p ite +__ _ +M ax +Ġerror s +ĠWh ile +Ġarg uments +Ġens ure +R ight +-b ased +We b +Ġ- = +Ġint rodu +ĠIn st +ĠW ash +ord in +j oin +D atabase +Ġgr ad +Ġus ually +IT E +Prop s +? >Ċ +ĠG o +@ Override +RE F +Ġ ip +ĠA ustral +Ġ ist +View ById +Ġser ious +Ġcustom er +.prot otype +od o +c or +Ġdo or +ĠWITH OUT +Ġpl ant +Ġbeg an +Ġdist ance +() ). +Ġch ance +Ġor d +c ame +pr agma +Ġprot ect +rag ment +ĠN ode +en ing +Ñ ĩ +Ġr oute +ĠS chool +h i +Ġne ighb +A fter +lic it +Ġcon tr +Ġpr imary +A A +.Write Line +util s +Ġb i +R ed +.L inq +. object +Ġlead ers +un ities +Ġg un +on th +ĠDe v +F ILE +Ġcom ments +_l en +ar row +am ount +R ange +s ert +Grid View +Ġup dated +ĠM o +Ġin form +oci ety +al a +A ccess +Ġh ab +Ġc reat +_ arg +ĠJan uary +ĠD ay +") čĊ +up le +d ocument +gor ith +m enu +ĠO ver +b b +.t itle +_ out +Ġle d +ur i +Ġ? >Ċ +r un +Ġsc ene +( array +de vice +_t itle +ag on +] čĊ +ab y +Ġbe came +bo olean +Ġp ark +ĠC ode +up load +rid ay +ĠSept ember +F e +Ġs en +c ing +F L +C ol +ut s +_p age +in n +Ġim plied +al ing +Ġyour self +.C ount +con f +Ġa ud +_in it +. ) +Ġw rote +N G +. Error +ä » +.f or +Ġe qual +ĠRe quest +Ġser ial +Ġallow s +X X +Ġm iddle +ch or +à ¸ +erv al +.C olumn +read ing +Ġesc ort +ĠAug ust +Ġquick ly +Ġwe ap +ĠC G +rop ri +h o +Ġc op +( struct +ĠB ig +Ġv s +Ġfre qu +. Value +Ġaction s +Ġpro per +Ġin n +Ġobject s +Ġm atrix +av ascript +Ġon es +.g roup +Ġgre en +Ġp aint +ool s +y cl +enc ode +ol t +com ment +. api +D ir +Ġun e +iz ont +.p osition +Ġdes igned +_ val +av i +ir ing +t ab +Ġl ayer +Ġview s +Ġre ve +ra el +ĠO N +r ics +n p +Ġc ore +() );čĊ +M ain +Ġexp ert +ĉĉ čĊ +_ en +Ġ/ > +ut ter +I AL +ail s +ĠK ing +*/ ĊĊ +ĠM et +_ end +add r +or a +Ġ ir +M in +Ġsur pr +Ġre pe +Ġdirect ory +P UT +- S +Ġe lection +h aps +.p re +c m +Val ues +Ġ" Ċ +c olumn +iv il +Log in +in ue +Ġbeaut iful +Ġse cret +(e vent +Ġch at +um s +Ġorig in +Ġeffect s +Ġman agement +ill a +t k +Ġset ting +ĠC our +Ġmass age +ĉ end +Ġhapp y +Ġfin ish +Ġc amera +ĠV er +ĠDem ocr +ĠH er +( Q +con s +it a +Ġ' . +{ } +ĉ C +Ġst uff +Ġ :Ċ +ĠA R +T ask +h idden +er os +IG N +at io +ĠHe alth +ol ute +Ent er +' > +ĠT witter +ĠCount y +s cribe +Ġ= >Ċ +Ġh y +f it +Ġmilit ary +Ġsa le +re quired +n on +boot strap +h old +r im +- old +ĠD own +Ġm ention +cont act +_g roup +od ay +Ġto wn +Ġsol ution +u ate +ell ing +] -> +ot es +ent al +om en +osp ital +ĠS up +_ EN +Ġsl ow +SE SSION +Ġbl ue +ag o +Ġl ives +Ġ ^ +. un +in st +en ge +Ġcustom ers +Ġc ast +ud get +ï¼ ģ +ic ens +Ġdeter min +Se lected +_ pl +ue ue +Ġd ark +// ĊĊ +s i +ther n +ĠJ apan +/ w +P U +ĠE ast +ov ie +Ġp ackage +Ġn or +Ġap i +b ot +" ];Ċ +_p ost +ul ate +Ġcl ub +') );Ċ +Ġlo op +PI O +ion e +sh ot +In itial +Ġplay ed +reg ister +rou ght +_m ax +ac ement +m atch +raph ics +A ST +Ġexist ing +Ġcomple x +D A +.C h +.com mon +m o +Ġ' ../../ +it o +Ġanal ysis +Ġdel iver +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +id x +à ł +ong o +ĠEng lish +< !-- +Ġcomput er +EN SE +Ġp as +Ġr ais +H ash +Ġm obile +Ġo wner +F IG +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +th es +Ġat tr +w d +.t ime +aw n +Ġtreat ment +ĠA c +. View +im pl +m ore +p ass +Ġh a +.f rom +Ġle ading +FF FF +( error +. ui +at ar +ad ers +d ates +Ġz u +Ġfl ow +T arget +Ġinvol ved +Ġi o +par se +$ _ +he st +. int +- item +as y +S p +Ġsh ift +N T +Ġt f +_T R +. web +C S +Ġ} ) +Ġey es +_ z +' );čĊ +if orn +Ġ{ @ +Ġn ice +.l ist +ĠĠĠĠ čĊ +Ġf loor +Ġred irect +ĠU K +( [' +Ġw ish +Ġcap t +leg al +ĠI O +Ġst age +. String +ĠA fr +ig en +ĠS H +De lete +ell s +Ġsol id +Ġmeet ing +Ġwork ed +Ġed itor +in y +Ð ¼ +_ read +. Id +e ff +Off set +ch a +US ER +ĉĉ ĠĠĠ +ipp ed +Ġd ict +ĠR un +.h pp +Ġan g +x ml +im ple +Ġmed ical +_t oken +con nect +Ġh our +Ġcont roller +_m essage +U ID +G r +and ed +_C H +Ġbook s +Ġspe ak +am ing +Ġm ount +Rec ord +ĉ struct +.W eb +ond on +Ġ// Ċ +Ġf elt +.A uto +id ge +_p os +P R +Ġmod ern +C ollection +_m sg +C D +ĠL o +Ġsecond s +ib ly +.e quals +Ġintern ational +# pragma +oo th +W riter +i ate +Ġce le +ĠB it +iv o +iv ery +r d +HE CK +Ġc ache +.c ount +Ġro ll +.Re ad +RE D +Ġset up +izont al +model s +arg v +Ġconsider ed +=" ../ +set tings +ĠR el +Ġgrow th +Ġm ix +ĠWash ington +Ġpl t +ĠI M +á º +Ġturn ed +ĠDate Time +ĠW ed +( url +Ġ" - +Ġlet ter +As ync +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠ +ĠOct ober +_l ine +Ġatt ention +Ġcol lect +ĠH ash +Ġim ag +T ree +Ġsit uation +et te +_n o +IV E +Ġv on +.t arget +Ġknow ledge +Ġdr ive +.p ost +Ġb lood +Ġc it +pr imary +Ġconfig uration +te e +Ġph oto +is ode +Tr ace +Ġg ave +Ġsh ot +ĠA ir +Ġm other +pr ice +Ġmor ning +)) {Ċ +- x +Ġtr ade +Ġdes c +Ġ&& Ċ +Ġparent s +A pi +å Ī +t ed +w er +Ġ æ +Ġs y +ĠK e +Par ser +å ħ +anc y +Ġpie ce +iforn ia +to String +r an +id ing +PT ION +com es +/ lic +.c lient +E l +L ong +Ġprofession al +ru pt +v a +Ġcomplet ely +Ġpract ice +Ġse lection +R em +in i +Ġc am +RE E +Ġsit es +p a +AT US +Ñģ ÑĤ +arr ant +* ( +_ KEY +ĠB utton +ĠF riday +se qu +Ġre ader +Ġm essages +è ¯ +Ġbu f +K e +Ġn ov +H P +M sg +al ign +ar ily +Ġ' , +_w ith +Ġd as +Ġhe ard +at omic +ri al +) [ +Ġdis e +@ end +Ġg old +Ġf air +Ġsa les +. Button +str ict +s ave +Ġme asure +Ġ" + +ec ause +View Controller +ĠT able +.p aram +Ġdec ided +(( ( +IN FO +Ġopport unity +T e +IC ENSE +cc ording +k i +ĠU N +Ġcont ain +Ġman ager +Ġp ain +ĠF ire +rom e +Ġpl ans +F ound +l ay +ĠDec ember +Ġinfl u +à º +ren ch +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ġ +az ing +b rief +c all +wo od +Ġload ed +Ġgr and +/ f +im p +_ U +ST R +âĢ ¢ +Ġcred it +.C olor +or ge +QUE ST +Ġdiffer ence +ĠP C +w args +Ġp ub +und ay +Ġf ra +.m ax +Ġtri ed +ann els +s end +Ġreport s +Ġad ult +ä º +Ġcons ist +ĠSt reet +ĠPro gram +S QL +M atrix +ounc il +- A +ĉ w +Ġwho se +Ġrel ig +ĠS ex +Ġg ives +n one +.m essage +( G +.aw t +- right +ĠNov ember +ell ig +ut ive +Ä ĥ +over n +Ġeas ily +Ġide as +ĠÐ ½ +/c ss +ly ing +el le +C an +_c olor +оР² +Ġp air +ng th +Ġs plit +d rop +art y +on a +Ġcap ital +Ġhe ar +Ġex ists +ĉ log +em o +R un +o i +Ġpar ser +ĠM ethod +Ġeduc ation +[ k +Ġlib rary +> ";Ċ +_ UN +ĉ std +od ed +Ġcall s +h ere +R el +Ġbr and +back ground +g a +_add ress +_param s +C ategory +ĠInd ia +_e vent +Ġ ing +R ender +.c l +ump y +Ġp et +F C +ĠA nt +Ex t +Ġchar ge +en ed +gr ad +E O +Ġdep end +Ġ .ĊĊ +fr ame +Ġd f +Ġh uge +ĠP ART +ed s +; ; +ĠA M +Ġbas ic +ĠL et +lic h +Ġar m +Ġst ar +Ġf ederal +W ork +Ġcar ry +ĠIs rael +( obj +={ { +Ġs aved +Ġs yn +Ġconst ant +V ENT +Ġpos itive +Ġcon duct +Ġsk in +Ġear lier +Ġl ayout +ĠI P +O UR +Ġt im +styles heet +_ cl +ĠC ard +++ ){Ċ +Ġtem per +ĠDav id +ĉ try +.d art +Ġwant s +Ġp icture +Ġv ideos +ĠCom m +is ions +_M AX +M apping +- content +ĠE ar +- de +Ġpre m +br uary +Ġcom ponents +Ġthrough out +Ġp ull +Ġp ages +ent e +res pond +Ġg as +cript or +Ġed ge +Ġb ound +A CT +**** ** +Ġcre ating +ĠC H +Ġnull ptr +B r ++ ' +.c o +> :: +Ġle arning +.L ength +_S H +Ġpat ients +A IN +Ġk ids +Ġcom fort +Ġsh own +ug ins +ĠB ack +ell a +_C L +Ġl at +Ġdis patch +Ġclass es +. at +.b egin +Ġsuccess ful +b an +Ġobt ain +ĠS l +Ġl ack +iter ator +Th read +(s ize +Ġn one +.h as +_ X +s ort +n ap +p et +b in +ĠCan ada +The y +Ġd ans +ĠM at +< td +Ġh air +Ġ' ',Ċ +Ġc u +Ġlaw s +let ed +p ed +Ġp ow +Ġk new +_C OM +_ , +ĠM ag +id ents +( req +Ġ ), +- center +Ġw ide +ĠA uthor +st ants +Ġjob s +Ġm ath +et imes +Bo olean +Ġs cope +_ is +Ġme as +Ġkey s +el ay +Ġexact ly +'=> ' +ĠP aul +m as +ĉ print +(l en +f d +Ġ) ; +. Event +q li +ir it +ield s +om an +ĠT op +Ġv ote +Ġm ask +Ġthem e +- Ċ +Ġpro ps +Ġf ine +Ġwrit er +_ offset +c ar +Ġal tern +Ġc opyright +Ġdest roy +pp er +Ġgener ate +pp ed +âĢĻ d +ĠĠĠĠĠĠ Ċ +m ake +ĠSh ow +Ġb rowser +Ġfavor ite +Ġcare er +Ġhappen ed +( char +Ġrecomm end +Ġl iter +.f ilter +gr ade +Ġ £ +Ph one +om s +Ġn amed +- label +ip o +ĠO ther +Ġp anel +Ġro ck +S cale +ĉ assert +Ð ´ +Ġtr ust +fr ont +Ġdem on +A r +N et +Ġecon omic +foot er +Ġr ace +(n ode +ĠO ption +s plit +Ġphys ical +if est +Ġrem oved +. http +)) ,Ċ +Ġlook ed +' ; +d ing +g est +atur day +/lic enses +Pr ice +Ġd ro +Ġto wards +Ġun s +ĠC L +ĉ static +Ġ rows +Ġdef ine +.re place +Ġf ather +ĠDes ign +ass ign +m ut +De vice +D id +') )Ċ +omet ry +ay load +Ġh istor +ĠP aram +ĠBo olean +Ġn ature +Ġj s +Ġn ation +i h +Ġdis cover +se m +Hand le +ĉ r +ĠTe chn +Ġw all +{ $ +@ property +Ġ" ../ +Ġex am +.d raw +opp ing +Ġnear ly +Ġco ol +Ġinde pend +RE S +Ġhand ler +ĠMon day +Ġs un +St yles +ous ly +Ġ ĉ +v est +D isplay +( y +atic ally +Ġpred ict +y ing +Ġsom etimes +" ]Ċ +Ġdr ink +Ġb ul +ific ations +. insert +.re g +Ġtest s +Al ignment +Ġal leg +Ġat tribute +ĠN ote +Ġmy self +art s +N ow +Ġinterest ing +li ents +Ġpop ulation +ĠCal ifornia +" I +å ¹ +Ġgre ater +ues day +Ġth ous +Ġcost s +Ġla unch +\ Http +k er +b and +ĠPl ay +Ġb and +.sh ape +es ome +art icle +.r f +Ġw er +á s +em bers +us r +B A +ic an +et t +valid ate +ult i +Ġimmedi ately +z er +Ġfig ure +o es +ell er +irc le +ĠS ign +.d b +Ġr ank +By tes +Ġproject s +_re c +UL AR +A PI +ĠL ine +P ort +Ġp oll +Ġg iving +id ence +-- Ċ +Ġpl ot +ic ial +Ġw arrant +IT ION +ĠD ouble +Ġbill ion +gorith m +Ġequ ipment +D ATE +Ġ@ " +E E +Ġp le +i ation +Ġhead ers +Ġpro ced +.Component Model +ĠOb ama +Ġp a +ĠB est +im ately +.get String +. \ +mp loy +Ġr aw +_b lock +und red +" },Ċ +.Group Layout +Ġb rought +NS String +th row +cre ated +.N ew +_ view +C P +ep s +O p +Ġgr atis +Ġ' " +Ġinter view +"" "Ċ +Ġpart ial +Ġa ria +b ing +A uthor +Bo ok +ĠP at +um an +Us ers +pl us +ĠD irect +ven ue +al pha +UC CESS +ĠC all +Ġ );čĊ +im ated +Ġrem ain +Ġant i +ĠL ondon +Ġsaf ety +PO SE +o les +cont roller +By te +ĠCour t +ĠPh il +ĠAss oci +en a +å IJ +_ST R +co in +resh old +Ġb atch +_C lick +entic ation +> ';Ċ +ent y +Ġbegin ning +Ġz ero +ĠCon vert +Ġt err +Ġp aid +Ġincre ased +c atch +-s ize +act ivity +e quals +Ġque ue +Ġ" ' +ĠIntern ational +Ġf ür +urs day +Ġsc ient +all ow +ax is +Ġapp ropri +ed ge +Ġid x +S uccess +ent ifier +: \ +x is +Ġmax imum +ark s +Ġb irth +( index +Ġmay be +.p y +file s +Ġlim ited +_ check +lo ok +pl ies +Ġmov ement +'] . +Ġbro ad +ĠB E +ĠUn ityEngine +.c pp +ĠE very +Ad min +Ġf ans +p ared +Ċ ĠĠĠĠĊ +Ġfore ign +Ġp an +Ġt our +ĠOr der +Ġmov ing +Ġa uf +C all +c b +Å Ł +vent ory +ĠS ql +Ġful ly +Click Listener +W ORD +Ġannounc ed +) čĊčĊ +Ġagre ed +ri e +Ġe arn +_l ink +. array +(t ext +Ġmaterial s +, p +ff ff +v g +Ġ © +Ġun less +aj ax +LO G +Ġsex ual +Ġ\ " +- time +Ġco ach +Ġsupport ed +Ġphot os +if orm +.C reate +) ] +ri er +Ġd ialog +av er +ig e +) + +_id x +: [ +_m in +ĠC ong +Ġpress ure +Ġteam s +S ign +b egin +ri an +NE SS +L S +Ġimpro ve +ĠS unday +Ġdef inition +ig er +roll ers +Ġthink ing +T emplate +- F +Ġem erg +pl ates +ĠUS A +.set State +ĠAl so +re v +Ġen able +ĠC O +PE CT +Ġcon cept +) - +ĠâĢ ¢ +Ġset s +Ġmean ing +em on +ĠCon s +c mp +ed er +ann ed +icens ed +ĠS uper +Ġd aily +Ġmult i +_ u +Ġchall eng +_m ode +ĠP romise +Ġstr ict +j o +int on +( list +On ly +> { +Ġveh icle +í ķ +ĠPl ayer +ĠD el +Ġp ool +. url +nes day +();čĊ čĊ +Ġ" );Ċ +L ocal +. ");Ċ +Ġorgan ization +re nder +ĠApp lication +Ġsum mer +ex pected +N A +Ġr ap +_ obj +Ġsur face +ĠP UR +Ġ}, ĊĊ +Ġvariable s +(m essage +Ġop in +.b ack +а н +Ġwork ers +v m +C o +ught er +Ġm aster +Ġ" ", +Ġst ories +. User +Ġcele br +ines e +B S +ĠCom mand +ash board +Ġo g +k g +. image +.st yle +Ġstep s +ĠB en +( args +ĠP erson +, y +Ġofficial s +| Ċ +Ġsk ills +v c +Ġbuild er +Ġg ar +A ccount +ĠA uth +ç Ķ +'] )Ċ +ĠA T +n n +. Int +SS ERT +Ġeffect ive +LE TE +Ġto ols +AR D +Ġdig ital +D ouble +ĠF ind +R C +Ġin line +/ r +AR AM +AS K +Ġint ent +a ight +_add r +Ġrequest s +.f irst +Ġde bug +Ġsp ent +() ));Ċ +Å Ľ +Ġpr incip +Log ger +clud es +. use +Ġsur v +med ia +ĠFe bruary +ĠM ac +Ġmiss ing +Ġw ife +Ġtalk ing +ĠM ake +Ġc art +Ġloc ated +E nc +- a +ch ron +Ġc ards +Ġgu y +Ġp ers +ĠY es +ate ver +ĠA ng +ol ar +ĠE ven +Ġacc ur +ĠP ower +ĠG old +c lear +Pro cess +Ġrec ords +Ġk illed +.c lear +ĠWARRANT IES +Ġpur pose +pan el +J ECT +ÃŃ a +Ġex erc +W S +/ L +. exports +Ġ__ _ +Ġs in +S ervlet +Ġd é +.de lete +ro ke +S l +ug h +ear s +Ġpoint er +Ġh op +all ery +Ġo bs +co very +ĉ char +ĉĉĉĉ ĉĉĉĉĉĉ +ĉ def +oc ity +itch en +ul ations +ĠF IT +Ġ ). +straint s +vent ion +Ġrequ ires +ĠO per +M E +OUN T +al let +Ġn orm +I RE +ex as +Ġprogram s +Ġwe ak +' .$ +u ing +ĉ ĠĠĠĠĠĠĠ +Ġm il +Ġf irm +init ely +_VAL UE +ap se +atis f +Ġdem and +_m od +Ġdescri bed +Ġpl aces +V ID +Ġal one +Ġex port +Ġv ec +ĠM ax +Ġactiv ities +ict ures +g ener +Ġm a +Ĥ ¬ +Ġexpress ion +C allback +_ content +ĠM ost +Ġtest ing +E C +CH ANT +Ġad just +.Th reading +( ctx +Ġag ree +ig hest +Ġu i +ĠL aw +. Y +> ĊĊ +.ex ample +ber g +Ġmov ed +ĉ e +ĠS aturday +Ġpay load +Ä ĩ +) :ĊĊ +Ġbe y +ur er +< script +Ġs ymbol +Ġass um +Ġp ul +E ffect +Ġh undred +To ol +ak ed +con nection +Ġvo ice +Ġp d +Ġtrans action +Ġlink s +E rr +ĠInd ian +T C +atal og +n i +s ign +<< " +j i +y a +Ġdemon str +ul ated +. St +Ġinst it +Ġbo ost +Ġcell s +ol ic +.P ro +: , +"> \ +Ġth us +ĠReg ister +h ol +ĠCh inese +Ġpost ed +Ġm agn +ab ilities +Ġdise ase +Ġrem ains +ĠPro f +- form +Ġc in +org an +ic ate +Ġst ress +] * +Ġ ---------------------------------------------------------------- +_ context +or ry +Ġd ied +m at +Ġstart s +.M essage +Ġrun s +Ġgu ide +Ġwarrant y +ential s +d ict +ĠS ize +ul er +Ġrespons ible +_SE T +Ġcont aining +ĠPr ice +| | +F S +Ġem p +_b utton +( uint +Ġsu ff +p th +Ġdef initely +put e +Ġmarket ing +ĠW H +ĠS ie ++ = +OL OR +Ġcons ult +Ġs igned +Ġse quence +le e +Ġrequire ments +h y +Ex press +M T +se y +Ġ ult +å ® +ellig ence +Ġanal y +Ġd ress +eng ine +ĠG reat +ĠAnd roid +ĠA lex +m ode +D ictionary +.D ate +ä ½ +V ICE +Ġfam ilies +ĠRuss ian +ĠT imes +.c all +$ ( +Pro file +Ġf older +ch es +Ġleg is +_ row +un es +Ù Ħ +Ġ} ). +Ass ert +ag en +ĠH and +I ter +Ġbig gest +ore ach +Ġpol ic +Ġper missions +Ġshow ed +ĠE lement +Ġtop ic +âĢĶ âĢĶ +ro ad +ĠB ank +rec ord +Ġpart ners +ĠR ef +ess ions +Ġass ess +U ST +ĠPart y +pro du +L C +Ġ ul +. form +h ide +c opy +UT F +ĠSO FTWARE +čĊčĊ čĊ +ĠL in +un a +ug ar +Ġadmin istration +Ġopen ing +Ġsc an +Ġcontin ued +com ponent +.s p +Ġhapp ens +um my +ĠP R +.F ile +ĠDown load +Lo ading +d i +Ġwait ing +_A DD +T ab +.query Selector +Ġecon omy +ĠF rench +t xt +Ġf ant +_ ;Ċ +H older +S H +Ġn umpy +Ġst reet +Ġm ale +\ Model +ang ing +ĠB ill +Ġprevious ly +B I +ĠSec ret +Ġm ist +ĠF ield +up s +ĠPro cess +Ġke pt +ĠO T +Ġtrad itional +. i +am in +Ġhelp s +An y +orig in +ilt ers +j u +d esc +ĠA ccount +Ġ) čĊ +k top +ol ly +Ġf s +Ġ ê +Ġ ut +Ġcent ral +(t est +.A n +Ġs atisf +G R +ĠF ull +Ġhe at +ib er +Ġon to +m os +S chema +Ġfact ory +" .$ +aw s +St atement +(t arget +ĉ new +.b e +Ġg uest +Ġm al +AR Y +Ġre ached +Ġm ouse +Ġchall enge +ĉd ouble +ĠT em +Ġt error +Ġex tract +_T O +Ġsepar ate +Ġm ir +h elp +Ġcap acity +ĠProp erty +k an +_c reate +ĠL ight +.p arent +Ġunderstand ing +Ġeas ier +Ġ| = +Ġen h +Ġf at +Ġprot est +am m +_ AT +- of +il s +ĠO h +Ġps ych +Ġ$ . +ind s +Ġrel ative +sh op +sh ort +ĠS and +uest ion +Ġf ear +/ ĊĊ +. context +Ġschool s +Ġser ve +z one +_d b +Ġmajor ity +ex ample +Ġl ang +ĉ ĠĠ +Reg ister +end o +Ġprocess ing +_t emplate +- user +Ġe g +C OM +ĠBl ue +i ro +Ġrem ote +ĠI T +#! / +Ġred istrib +ra z +ĠS ince +ĠT ur +Back ground +== = +Ġref lect +Ġpro s +c md +Ġwh om +Com pat +ĠA re +Id entifier +ĠTh om +_ port +g u +Ġmon itor +r m +Ġpat ient +ver ter +Ġg ain +- ui +In st +Ġd ies +A rea +_f ilter +Ġgr at +Ġreal ity +ord inate +ol ved +Cont act +Ġcompl iance +_ or +ĠV ar +d l +Ġapp end +G ER +(m ax +.re nder +Ġd ynamic +ordin ates +_ options +_c olumn +Ġb atter +s pace +L a +ĠS ource +/b in +Ġd os +ĠBo ard +ĠTh read +ĠA L +( config +ĠM er +Ġm iles +_ header +ETH OD +iz z +Ġbenef it +Ġinteg r +(c urrent +ul o +. default +ĠD iv +Ġt on +o th +erv ation +ed om +Ġb aby +ce ived +.t op +rior ity +ĠL ocal +ri age +Ġattack s +Ġh ospital +Ġfem ale +ĠLog in +ĠFl or +Ġch ain +ash ion +Text ure +S ave +Ġf arm +.cont ains +.T est +Ġknow s +Ġgener ally +ip eline +Ġme ant +enc ia +Ġn icht +Ġcont ents +P M +ched ule +( line +C G +j ob +ĠRe al +u er +f irm +Ġ Ø +et ro +" `Ċ +Ġspe ech +Ġth r +fore ach +Ġw arn +ĉ l +Ġhe avy +< li +N e +Ġinvestig ation +M ath +- title +Ġch urch +Ġdes pite +ch ain +Ġwh atever +ar ian +f n +Ġm eta +} )ĊĊ +U FF +Ġregard ing +_S UCCESS +m es +ĠInt ent +Ġres olve +pos s +ir a +for ce +o ice +à ¢ +Ġp m +Ġup dates +A rr +Ġ Ñ +test ing +Ġto ward +nt ax +ë ĭ +Ġlist en +Ġgo als +Instance State +D r +Ġr are +Ġtr ail +Ke ys +C al +C ar +ĠPe ople +ĉ local +class es +Re ference +.for Each +em b +act iv +Ġpr im +red ict +Ġr ad +æķ ° +.B ack +Ġsp read +Ġc lock +Ġv ir +ed itor +Ġeffort s +Ġbr anch +Ġind ust +Ġmot or +Ġam b +Ġdat etime +Ġren cont +ĠChrist ian +ĠAmeric ans +f ull +Ġf mt +.m ain +Ġca used +_ update +ĠCont ent +AT CH +Ġb ath +ĠE ach +Ġr adio +ach ment +uz z +Sub mit +Ġre strict +ab in +ĠL oad +Ġext ension +Ġess ay +Ġh at +avi our +to Be +": [ +Ġoffer ed +Ġv ill +(d ouble +æĹ ¥ +b c +_f ree +ĠM iss +ĠB er +Ġ è +ĠL ike +Ġhelp ed +.get Name +_ AL +Ġsp irit +ĠAp ache +w s +Ġthere fore +( params +_ img +Ġpe ace +Ġinc or +ĠEX PECT +Ġmin or +ip es +ĉ data +select or +c ity +tr ie +.b ase +_f rame +Ġopen ed +/ json +L Y +n u +.D e +t f +m argin +.P arse +Ġp i +Ġe q +b d +Field s +ĠT ree +Ġb an +ist an +Ċ ĠĠĠĠĠĠĠĠĊ +ĉg l +Ġprodu ced +s ystem +M ark +_h ash +Ġb g +Ġconst it +ĠLe ague +Ġmiss ion +_ format +([ Ċ +clus ion +! " +Ð · +b reak +ĉs witch +Ġth er +Trans form +Ġfoot ball +- link +r oute +. auth +Ġb ag +ov ers +Ġen abled +Ġr ac +( I +C R +anc ing +Ġman aged +_ q +NG TH +Ġm ac +ĠA uto +ament e +Ġ' ', +.App end +Ġp in +. item +ack ing +Ġocc as +p erson +Ġt i +.Re g +Ġh aven +Ġg lass +Ġ" ) +_ char +res ource +Ġep isode +Ġ' _ +ĠE s +ĠEar th +Âł Âł +UP DATE +ĠS ou +u is +t ypes +Ġm as +Ġf av +Ġcon struct +_r ate +er as +Ġ| Ċ +rop erties +Ġext ernal +Ġap plied +Ġpre fix +ot ed +l ers +Ġc old +ĠS P +ĠCh urch +ĠOut put +los ed +ç ļ +ific ate +oper ation +her it +x FF +. env +_ err +os h +D irection +C ancel +ĠFr ank +Ġfind ing +. )ĊĊ +Ġr outer +ãĥ » +s es +Ġc row +== ' +Ġs and +Ġr id +it ure +Ġent re +Ġo bserv +Ġv ac +ð Ł +- T +A rt +n ight +. search +Ġex change +Ġdistr ict +. os +Ġdep artment +Ġdoc uments +Ġcent ury +ĠN ext +H ost +ĠK IND +Ġsus p +- P +re nd +. em +u ite +ist ers +( json +ĠAn n +w t +at i +ĠHT ML +wh en +D irectory +Ġsh ut +< a +ed y +Ġhealth y +Ġtemper ature +ĠG en +Ġmet al +Ġsub mit +ĠD O +Ġat tract +Ġ{ };Ċ +ĠW ord +Ġl l +Ġseem ed +k o +I ED +Ġl abor +.Cont ext +Ġas set +y ou +Ġc ars +ĠC olumn +Ġr é +Ġs quare +ĠNS String +âĢĿ , +ap es +.. .Ċ +Ġthan ks +( props +Ġt ick +Ġexper iment +Ġpr ison +t ree +- text +ĠIO Exception +-w idth +_ST ATUS +f ast +-b ody +- header +Ġgu ar +cre te +ĠT im +Ġclear ly +ĠRepublic an +Ġjust ify +и ÑĤ +ĉ ĠĠĠĠ +c ache +; // +Ġpres ence +Ġfact ors +Ġemploy ee +] )) +M ember +Ġselect or +b or +ĠM ex +çļ Ħ +ut ex +_t ag +ail ure +ĠN et +Ġre li +E G +Ġf printf +Ġte en +lo ss +Ġle aving +De legate +Ġbe at +Ġmin ute +sub scribe +Ġredistrib ute +Con stants +Ġcan cer +/ { +B L +Ġs pan +ĠCh ild +C enter +Ġear th +Y S +ĠLe vel +Ġse a +.s upport +.in ner +. Item +ill ing +ĠĠĠĠĊ ĠĠĠĠĊ +ĠL abel +ĠE st +( arg +bo Box +ĉf oreach +c os +F ailed +sw ers +Ed itor +r ont +ĠM P +ex pr +ĠL ife +Ġ? ? +ö r +Ġatt end +ĠQ ue +Ġspec ies +- D +Ġa us +Str uct +Ġadvant age +ost on +-b lock +in itial +C RE +Ġtr uly +Ġcomp are +or ney +Ġs pect +F ull +b es +Ġvis ible +Ġm ess +st ances +Ġcl oud +_v ersion +Ġf urn +ic ago +LO W +Ġtraff ic +Ġf ol +rypt o +Ġdecl ar +Ġsl ot +ĠEx t +ĠEng land +ĠU nder +Ġt a +let ter +Ġoffic er +ĠDon ald +Y es +_ json +IT ableView +ĠU SE +mploy ee +Ġopin ion +ĠA ut +b order +Ġad vice +Ġautom atically +is co +Ġm m +. vis +am l +Ġinitial ize +Ġ( { +Ġ ;ĊĊ +Ġgener ation +Ġb its +clip se +Ġun f +ut ors +pl t +Ġdel ta +est roy +is is +< br +Ġlimit ations +Ġend ed +ĠM ad +il m +Th ese +ĠMin ister +Ġch art +F ragment +Ġindepend ent +Y ear +Ġin str +Ġt ags +A VE +ĠAr ch +st op +Pro gress +Ġm i +Ġlearn ed +G e +Ġhot el +S M +T YPE +Ġc y +ERS ION +un ately +l imit +s el +Ġmov ies +Ġste el +o z +g b +ĠC amp +s ite +ĠLog ger +P LE +оР´ +. right +ĠC ore +Ġm ixed +st ep +Ġput s +s uper +R outer +. Http +ly ph +ĠColor s +Ġandroid x +. str +Ġinn ov +Ġde ck +' >Ċ +ap ers +] ( +cont inue +s pec +ĠR oad +AS H +ili ar +Ġcontin ues +Ġapp oint +Ġ# Ċ +ĠV ir +Ġ?> " +Ġb in +} ", +go ing +e ach +B D +ĠA ccess +D oc +ĠMan agement +B ER +ask et +.get Instance +Ġestablish ed +so cket +IN S +ĉv irtual +ĉ result +RE AD +_ height +ĠF ont +Ġ( );Ċ +_ html +Ġneighb or +l or +Ġg ather +Ġ} )ĊĊ +Ġid entity +Ġf ab +p adding +ĠR oute +Enumer able +à ´ +Ġfor ced +/j query +.ĊĊ ĊĊĊĊ +res ents +_ left +.P aram +ĉ throw +ĠH am +Ġevent ually +ac er +p ub +Ġtr a +un ique +d el +ĠFlor ida +ĠC lean +x a +Ġ · +Ġvalid ate +Vis ual +Ex pression +_f unc +m ember +ĉ h +tr l +ĉ G +nap shot +ĠProp Types +v in +] )ĊĊ +ow l +if ies +Ġ$ ('. +ĠCont ext +ĠTo ast +. Key +Ġoffic ers +/ n +s n +und efined +. items +ut ow +am age +Ġaccount s +ook ie +Se ction +ici ans +Ġad vis +( is +[: , +ĠFr ance +F unc +ic ious +Ġto k +Ch annel +ĠA D +_N UM +Ġtime out +lem ma +rem e +u j +.A l +uc lear +( os +(" < +[ Ċ +f etch +Ġb al +Ġgu id +- align +ĠW rite +ĠOn ce +utow ired +OD ULE +Ġp itch +C F +by tes +ĠCom mission +Ġincre d +P ER +_ response +ĠL os +par ser +Ġass ume +. Request +ĠT oken +_p osition +Ġn om +- term +Ġrem aining +i ostream +Ġpie ces +ap y +ĠL ess +r ange +umb n +pr ise +_ option +Im pl +k wargs +Ġbusiness es +Al ert +Ġpart ies +ĠCont ainer +ĠPr ivate +ĠPl an +Ġregister ed +Ġj our +ack er +ен и +/ > +ch at +se ct +Ġcre ation +olut ely +Ġinst ant +Ġdel ivery +ick en +y es +ĠFr anc +bl ing +end a +[ ( +_r ange +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠ +Ġsched ule +Con n +Ġthan k +x d +Ġh ook +Ġdocument ation +Param eters +H ello +v t +Ġart icles +Ġw est +def ined +. select +ok ens +ĠV AL +.f ile +res et +Ġmy s +ĠM A +] ), +Ġc ities +rel ated +å Ľ +Ġappe ared +Ġw id +.p anel +ĠIn s +. entity +Ġde cre +ĠL ou +(t ime +ĠTh ank +.create Element +Ġmention ed +oun ce +ĠT ry +ĠW all +/ images +ĠM enu +' čĊ +ĠE r +Ġcrit ic +ĠY ear +( param +Ġf lo +N N +oot er +Ġ ];Ċ +ĠA ff +" github +room s +Ġh yp +g lobal +Ġa vec +æľ Ī +Ġcomplet ion +Ġcon d +onym ous +( temp +Ġst ars +Ġre levant +Ġcover ed +Ġel im +_t ypes +( bool +Ġt u +_ex ists +Ġsec ure +Ġst ored +] / +x F +ĠCont roller +Ġm igr +M I +ĠD en +Ġann ual +U IL +- and +Ġcr ime +b el +Ġk itchen +@ g +_p h +ourn ament +ĠS ocial +ĠS pecial +log ger +Ġt ail +Ġun known +d ed +Ġapp rec +(d b +c f +Ġass ign +- out +ĠM ont +d p +w idget +Ġst one +- primary +. grid +Result s +az z +Ġda ughter +Ġcur r +Ġl in +Ġs outh +form s +ĠO UT +let te +ak s +ig ure +ĠE U +var iable +Ġb rief +ĠSc ott +Ġcon ference +and a +_ lock +or al +Ġe ine +OR S +//////////////////////////////// //////////////////////////////// +ess o +Ġr is +Ġg ender +est ic +L icense +( out +Ġm s +Se e +Ġwill ing +az e +Ġs ports +Ġy es +l u +Ġp urs +/j avascript +- pro +nav bar +_pro duct +/ bootstrap +Ġdr iving +Ġ Ä +Ġpro pos +ult ip +up lic +. email +Ġappro x +( cl +Ġwe ar +Ġrep ly +ass et +Ġ ice +Ġt x +k r +ĠGerman y +ĠGe orge +Ġc b +ĉ err +M ove +Ġpol y +vo ice +} " +Ġan imal +A v +ĠL ocation +Ġn ative +] [" +< double +Ġm ais +, int +Ġpre par +Ġinter val +plement ation +_ ERR +Ġb ug +> " +st at +Ġ} ,čĊ +< span +Ġfa ith +Ġ rom +pre v +ĠE lect +F ind +Ġg od +ot or +// ---------------------------------------------------------------- +orig inal +C pp +ĠSen ate +Ġposition s +Ġweap ons +Ġco ff +Ġpur poses +p ol +Ġim press +Ġanim als +. Entity +(n p +Ġmur der +Ġ` ` +fl ag +Ġsol utions +ĠAct ive +Ġb right +.d ate +Ġsit u +ï¼ Ī +. ID +Ġs ie +), čĊ +ak t +S pace +.d at +.index Of +h an +az ine +ĠZ e +Ġcr ash +( / +> = +Ð ± +iv a +.Auto Size +ĠL at +_ ext +Initial ize +.reg ister +OP Y +Ġre verse +_d is +'] [ +Ġprom pt +ont o +ĠJ ournal +r outer +Ġmys qli +# else +) " +-x s +let s +ph an +. LE +W ill +Ġaff ord +Ġsk ill +-t oggle +N C +B ind +T S +J ust +iter al +Y P +ĉ unsigned +Ġw ind +)) :Ċ +Ġw arning +ĠW ater +Ġd raft +Ġc m +Ġs am +Ġhold ing +z ip +ĠSc ience +Ġsup posed +G en +Ġdi et +< h +ĠP ass +v i +Ġhus band +� � +n ote +ĠAb out +ĠIn stitute +Ġcl imate +.Form at +Ġn ut +est ed +Ġapp arent +Ġhold s +f i +new s +C M +v ideo +': ' +D ITION +p ing +Ġsen ior +w a +-- >Ċ +_ default +ĠD atabase +re p +E SS +ner gy +.F ind +_m ask +Ġr ise +Ġk ernel +:: $ +. Q +Ġoffer ing +de cl +ĠC S +Ġlist ed +Ġmost ly +eng er +Ġblock s +ol o +Ġgover ning +\ F +Ġcon cent +.get Text +Ġm b +Ġocc urred +Ġchang ing +Sc ene +_C ODE +B eh +" The +Ġt ile +ĠAssoci ation +ĉ P +al ty +_ ad +od ies +i ated +Ġpre pared +poss ible +Ġm ort +TE ST +Ġign ore +Ġcal c +Ġr s +Ġassert Equals +Ġs z +ĠTH IS +. "Ċ +Ġcan vas +j ava +Ġd ut +VAL ID +.s ql +. input +Ġa ux +S up +Ġart ist +V ec +_T IME +.string ify +et ween +ĠC ategory +Ġ[ - +ĠDev Express +ĠJ ul +Ġr ing +. ed +Y Y +L et +Text Field +Ġfl at +_p rint +ĠOT HER +ad ian +Ġcheck ed +e le +Al ign +stand ing +Ġ[ ], +Ġl ab +uck y +ĠChrist mas +( image +.m odule +Ġl ots +Ġslight ly +(f inal +er ge +è ¿ +ĠPol ice +ĠR ight +Ġaw ard +ĠO S +Ġ{ }ĊĊ +Ġp tr +ov es +ic ated +еР¼ +Ġman age +olid ay +Am ount +ool Strip +t body +N av +w rap +B B +Ġwatch ing +ari os +Ġoption al +_ K +ĠL icensed +.M ap +T imer +ĠA P +ĠRe v +( o +, c +um in +eta iled +ĠH y +Ġbl ank +ag ger +ĠS elf +() [ +.m ake +ear n +ch annel +< pre +ble m +_p assword +_s p +ic ing +e z +Ġthe ory +ĠT er +, n +log o +ĠHT TP +() )) +.h andle +> ;Ċ +W orld +Ġpy thon +Ġl if +Ġtr av +Ġcon ven +com pany +ĠCl ub +V er +B tn +Ġz one +product s +ĠE duc +Ġver ify +ĠM il +on o +] );ĊĊ +EN CE +Ġpack et +Ġc er +Ġen umer +Ġpar s +form ed +Ġocc up +t re +Ġexerc ise +D ay +_s um +Ġask ing +apt ion +Ġord ers +Ġsp ending +ĠE RR +.D is +ĠU til +âĢľ I +\ ' +? ) +/ >Ċ +Ġem ot +Ġinflu ence +ĠAfr ica +att ers +Ù ħ +.s ession +Ġch ief +ĉĉĉĉĉĉĉĉ ĉĉĉ +Ġto m +clud ed +ser ial +_h andler +.T ype +ap ed +Ġpolic ies +- ex +- tr +bl ank +mer ce +Ġcover age +Ġr c +_m atrix +_ box +Ġcharg es +ĠB oston +P e +Ġcirc um +Ġfil led +Ġn orth +icture Box +ĉ res +è ® +Ġter min +Ġ[ âĢ¦ +IRE CT +Ġb er +Ġ" ../../ +ret ch +.c ode +_c ol +ĠGovern ment +Ġarg v +ĠL ord +as i +Ex ec +ĉ let +vert is +Ġdiscuss ion +en ance +out ube +type of +Ġs erved +ĠP ut +ĉ x +Ġs weet +B efore +ateg y +. of +ĠM aterial +S ort +ON T +ig ital +Wh y +Ġs ust +Ġ ç +ab et +Ġseg ment +Ġ[ ],Ċ +ĠMus lim +Ġfind ViewById +c ut +_T EXT +ĠM ary +Ġlo ved +Ġl ie +ĠJ O +Ġis set +mon th +Ġpr ime +t i +ĠCar ol +U se +ĠP op +ĠS ave +Int erval +ex ecute +d y +ĠI ran +_ cont +ĉ T +Ġph ase +check box +we ek +Ġh ide +Ġt il +Ġj u +C ustom +b urg +/ M +T ON +Ġqu ant +Ġr ub +ix els +Ġinst alled +Ġd ump +Ġproper ly +( List +Ġdec ide +app ly +H as +Ġkeep ing +Ġcitiz ens +Ġj oint +p ool +S ocket +_ op +Ġweap on +gn ore +ĠEx ec +ott en +ĠM S +Ġ( - +ĠRe view +Ġex amples +Ġt ight +! ( +D P +ĠMessage Box +Ġphot ograph +UR I +é t +l ow +ĠGr and +.p ersistence +Ġmaint ain +Ġnum s +Ġz ip +ial s +ĠG ets +pe g +ĠB uffer +~~ ~~ +ra structure +ĠP L +u en +ob by +size of +Ġp ic +Ġse ed +Ġexperi enced +Ġo dd +Ġk ick +Ġproced ure +avig ator +- on +, j +ĠAl though +Ġuser Id +ac cept +Bl ue +IC olor +l ayer +av ailable +Ġend s +.t able +Ġdat aset +b us +Ġexpl ain +( pro +ĠCommit tee +Ġnot ed +] :Ċ +D im +std io +. ",Ċ +_s ource +ĠWe ek +ĠEd ge +Ġoper ating +Ġest e +i pl +ag ination +Ġpro ceed +Ġanim ation +.Model s +ĠW atch +i at +Ġopp on +/ A +Re port +Ġs ounds +_b uf +IEL D +Ġbu nd +ĉ get +.p r +(t mp +Ġk id +>ĊĊ Ċ +Ġy ang +Not Found +Ñ Ĩ +m ath +@g mail +ĠL IMIT +red ients +Ġv ent +avig ate +L ook +Ġrelig ious +Ġr and +ri o +( GL +_ ip +u an +ici ency +ĠCh ange +> čĊčĊ +ĠEnt ity +Ġrencont re +ĠR et +pl an +é n +BO OL +ur ies +tr ain +Def inition +======== ==== +z z +An imation +ĠO K +_m enu +.b l +_s core +Ġac ad +( System +Ġref resh +'=> $ +.G raphics +ament o +p id +t c +Ġt ips +Ġhom es +Ġf uel +â ĸ +_h elper +ĠĠ čĊ +ĠR oom +.C lose +_ attr +ĠM ount +ĠE v +ar ser +_t op +e ah +ĠDe lete +ãĢ į +u ke +Ġus age +ar ia +_de v +Ġtext ure +Ġconvers ation +e per +Be an +d one +non atomic +ĠSe cond +Ġshoot ing +_p re +Com ponents +Ġ] ĊĊ +__ , +stit ution +.Ch ar +> ();ĊĊ +Ġpresent ed +Ġw a +ok er +- ĊĊ +in er +Ġbe coming +Ġinc ident +At t +Ġreve aled +for c +Ġbo ot +.p age +Enumer ator +_ -> +Ph oto +Ġs pring +. ", +ĠD ictionary +B JECT +Ġloc ations +Ġs amples +Input Stream +ĠB rown +Ġst ats +qual ity +Ñ ħ +-d is +Ġhelp ing +Ġp ed +( se +ĠWh o +al ian +int ernal +Ġf t +> (). +-> { +Ġm ine +Ġs ector +Ġg ro +Ġopport unities +Ġà ¼ +Ġm p +Ġalleg ed +Ġdoub t +M ouse +Ab out +_p art +Ġch air +Ġstop ped +lo op +ent ities +Ġapp s +ans ion +Ġm ental +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠ +F R +Ġdef end +c are +Ġide al +/ api +ur face +Ġe le +ul ator +ĠR ights +angu ages +Ġfund s +Ġad apt +At tributes +Ġdep loy +opt s +Ġvalid ation +Ġconcern s +u ce +.n um +ult ure +il a +Ġc up +Ġp ure +.F ore +ĠHash Map +.value Of +as m +M O +Ġc s +Ġst ores +Ġ ************************************************************************ +Ġcommunic ation +m em +.Event Handler +. Status +_ right +.set On +S heet +Ġident ify +ener ated +order ed +Ġ" [ +Ġs we +Con dition +ĠA ccording +Ġpre pare +Ġro b +P ool +Ġs port +r v +ĠR outer +Ġaltern ative +( [] +ĠCh icago +ip her +is che +ĠDirect or +k l +ĠW il +key s +Ġmy sql +Ġw elcome +k ing +ĠMan ager +Ġca ught +) }Ċ +S core +_P R +Ġsur vey +h ab +He aders +AD ER +Ġdec or +Ġturn s +Ġr adius +err upt +C or +Ġm el +Ġin tr +( q +ĠA C +am os +M AX +ĠG rid +ĠJes us +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠ +.D E +Ġt s +Ġlink ed +f ree +ĠQ t +Ġ/** čĊ +Ġf aster +ct r +_ J +D T +.C heck +Ġcomb ination +Ġint ended +- the +- type +ect ors +am i +ut ing +Ġum a +X ML +U CT +A p +ĠR andom +Ġr an +.s ort +Ġsort ed +. Un +_P ER +it ory +Ġprior ity +ĠG al +ĠO ld +h ot +ĠD isplay +(s ub +_T H +_ Y +ĠC are +load ing +K ind +_h andle +, , +r ase +_re place +.add EventListener +ĠR T +Ġenter ed +g ers +Ġ ich +( start +/ app +Ġbro ther +M emory +Out let +Ġ utf +pre c +Ġn avigation +OR K +Ġd st +D etail +Ġaud ience +Ġd ur +Ġcl uster +un ched +Ġ ], +Ġcomfort able +. values +ĠT otal +Ġsn ap +Ġstand ards +Ġperform ed +h and +(" @ +å Ń +Ġph il +ib r +tr im +Ġfor get +Ġdo ctor +.Text Box +icon s +, s +ĠO p +S m +St op +ĉ List +ĉ u +Com ment +_V ERSION +.X tra +P erson +r b +LO B +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĊ +ĠCent ral +IC K +ra q +Ġput ting +Ġm d +ĠL ove +Pro gram +B order +o or +Ġallow ing +a fter +Ġent ries +ĠMay be +] ). +ĠSh ort +) \ +.n ow +f riend +Ġpre fer +ĠG PIO +os is +ĠGame Object +Ġsk ip +Ġcompet ition +_m atch +lic ations +_CON T +.group Box +Ġal s +" We +_e q +l an +_ search +ĠMus ic +as is +Ġb ind +ĠIs land +r um +( E +Ġse at +V ideo +Ġa ck +ree k +={ () +Ġr ating +Ġrestaur ant +DE X +(b uf +pp ing +ual ity +Ġle ague +Ġfoc used +ap on +$ data +CL UD +CLUD ING +Ġabs olute +( query +Ġtell s +A ng +Ġcomm unities +Ġhon est +ok ing +Ġap art +ar ity +/ $ +_m odule +ĠE nc +. an +.Con fig +C re +Ġsh ock +ĠAr ab +I ENT +/ re +Ġre trie +ycl er +is a +ĠO rgan +. graph +Ġ í +ĠB AS +En um +Ġposs ibly +ÑĢ аР+ĠJapan ese +Ġc raft +ĠPl ace +Ġtal ent +Ġfund ing +Ġconf irmed +Ġc ycle +/ x +G E +Ġhe aring +Ġpl ants +Ġm outh +p ages +or ia +ĠRem ove +_t otal +Ġo d +oll apse +do or +Ġb ought +Ġadd r +AR CH +_d im +dd en +Ġdec ades +RE QUEST +Ġvers ions +f ire +Ġmov es +f b +Ġcoff ee +.con nect +ĠR ow +Ġs chema +S cope +- Type +Ġfight ing +Ġret ail +Ġmod ified +T F +File s +n ie +_com mand +st one +Ġ ÑĤ +_ thread +Ġb ond +ĠDevelop ment +Ġp t +F ORM +ple t +Ġident ified +c pp +Ġc oding +ok ed +ĠM aster +ID TH +Ġres idents +red it +ĠPh oto += - +un te +ate ur +_ST ATE +ĠS ing +Ġshe et +. val +or se +Ġh ers +Ġdetermin ed +Com mon +Ġw ed +_ queue +P H +ĠAt l +cre d +/L ICENSE +Ġm es +Ġadv anced +.j ava +.S h +G o +k ill +f p +_set tings +Ġp al +Ġtr uck +Ġcomb ined +Ġ" ${ +ĠCor por +Ġjo ined +ĠJ ose +ĠC up +un s +est ival +lev ision +Ġbro ken +Ġmar riage +ĠWest ern +Ġrep resents +ĠT itle +Ġs s +.A ss +ongo ose +ient o +< >();Ċ +Ġabs olutely +Ġsm ooth +TER N +ĠUn less +W ord +Ġmer ge +ig an +ĠV ol +Ġn n +.get Id +ĠÐ · +Ġsex y +Ġseek ing +S ingle +. this +Ġk om +b ound +; " +Ġfont Size +_d f +Ġinj ury +( H +Ġiss ued +_ END +: self +Ġp atch +Ġle aves +Ġad opt +File Name +ãĢ IJ +Ġexec utive +ĠBy te +] ))Ċ +Ġn u +out ing +clud ing +- R +. options +Ġsub stant +av ax +ĠB UT +Ġtechn ical +Ġtw ice +Ġm ás +Ġun ivers +y r +Ġdr ag +ĠD C +Ġs ed +Ġb ot +ĠP al +ĠH all +forc ement +Ġa uch +.m od +not ation +_file s +.l ine +_fl ag +[ name +Ġres olution +Ġb ott +(" [ +end e +( arr +F ree +( @" +ĠD istrict +PE C +: - +P icker +ĠJ o +ĠĠĠĠĠ Ċ +ĠR iver +_ rows +Ġhelp ful +Ġmass ive +--- Ċ +Ġmeas ures +ĠR untime +Ġwor ry +ĠS pec +ĉ D +ãĢ ij +Ġ) {Ċ +Ġwor se +(f ilename +Ġl ay +Ġmag ic +ĠThe ir +ou l +st roy +ĠWh ere +Ġsu dden +Ġdef e +Ġb inding +Ġfl ight +ĠOn Init +ĠW omen +ĠPol icy +Ġdrug s +ish ing +(' ../ +ĠM el +pe at +t or +Ġpro posed +Ġst ated +_RE S +Ġe ast +ĠCON DITION +_d esc +Ġwin ning +fol io +M apper +ĠP an +ĠAn ge +.s ervlet +Ġcop ies +L M +Ġv m +å į +Ġd ictionary +S eg +el ines +ĠS end +Ġ iron +ĠF ort +.d omain +Ġdeb ate +Not Null +e q +ach er +l f +ĉf mt +Ġlaw y +Ä Ł +ĠM en +Ġtr im +( NULL +Ġ! ! +Ġp ad +Ġfollow s +"] [" +re qu +ĠE p +.g ithub +( img +et o +(' \ +S ervices +umbn ail +_m ain +ple ted +fort unately +Ġw indows +Ġpl ane +ĠCon nection +. local +u ard +} \ +== " +and on +ĠR oy +w est +ig inal +em ies +it z +') :Ċ +ĠP eter +Ġt ough +Ġredu ced +Ġcalcul ate +Ġrap id +c ustomer +Ġeff icient +Ġmed ium +Ġf ell +. ref +ĠC as +Ġfeed back +S peed +( output +aj e +Ġc ategories +Ġfe e +} ; +Ġde leted +re h +Ġpro of +D esc +B uild +Ġs ides +.Array List +- % +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +Ø ± +.m atch +л и +Ġfe els +Ġachie ve +Ġcl im +_ ON +ĠC D +Ġteach er +_c urrent +b n +_P L +ist ing +En able +G EN +Ġt v +Ġso ck +Ġpl ays +Ġdis count +ĠK E +ĠDe bug +F ore +ĠI raq +Ġappear ance +M on +Ġst yled +ĠH uman +i ot +ĠH istory +Ġs ac +ĠC ollection +Ġrecomm ended +.Se lected +Ġorgan izations +Ġdiscover ed +co hol +ad as +ĠThom as +M ay +Ġcons erv +Ġdom in +ĠF ollow +ĠSe ction +ĠTh anks +User name +Ġrec ipe +Ġwonder ful +.s leep +_ if +ĉĊ ĉĊ +orn o +Ġr u +_t arget +." " +à ¦ +Event Args +Ġinput s +Ġf if +Ġv ision +c y +ĠS eries +) ((( +Ġtr ading +Ġmark er +B egin +Ġtyp ically +Ġca uses +drop down +_DE BUG +Ġdet ect +c ountry +! ");Ċ +ĉ R +app y +Ġc ref +(' < +" => +ĠL E +read er +Ġadmin istr +à µ +uck et +Ġf ashion +. char +iz ar +Ġdis able +Ġsu c +ĠL ive +iss ue +Ġmet adata +fl ags +Ġ ðŁ +Ġcomm itted +Ġv a +Ġr ough +Ġ'' 'Ċ +Ġhigh light +_var s +V O +Ġenc oding +- Z +_s ign +$ ("# +Ġr ain +reate st +ĠEN D +Se lection +Ġcandid ates +Ġs av +. Empty +Ġdec isions +Ġcoll abor +rid ge +fe ed +ress ion +Ġperson s +V M +eg a +_B IT +A ccording +ack ed +Ġdoll ars +_lo ss +ĠC ost +} "Ċ +Not ification +Ġpro stit +Ġauthor ity +.re c +Ġsp okes +ĠT oday +ist ant +ĠHe ad +âĢĿ . +ertain ment +ce an +cul ate +Ġv en +How ever +_ arr +Ġtok ens +G raph +ĠJ ud +ĠVir gin +ĠS erial +un ning +M utable +ag ers +.c sv +Ġdevelop ing +Ġinstruction s +Ġprom ise +Ġrequest ed +_ encode +/ " +ĠI con +u ilt +- day +Ġint elligence +. IS +ĠO bservable +ĠH ard +Bo ol +ident ial +.An chor +Ġsell ing +C I +AG ES +t le +b ur +UFF ER +R Y +Ġbig ger +Ġr at +Ġfam ous +Ġtyp ename +Ġexpl ained +} }Ċ +Ġn uclear +- N +Ġcr isis +ĠEnt er +Ġan swers +/ ${ +/ pl +Ġse qu +_n ext +m ask +Ġstand ing +Ġpl enty +ĠC ross +ĉ ret +d ro +ĠC ast += true +ĠCh ris +ic io +ĠM ike +Dec imal +add Component +L en +Ġco ck +Ġ# { +UR N +< tr +Ġauthor ities +Res ources +- H +B ottom +_ qu +put er +ester day +Dis patch +s ince +Ġfam iliar +, i +V C +Ġm ent +, C +Ġfre edom +Ġr outes +ĠB uy +Ġcomm ands +Ġm esh +/ C +ĠSet tings +- style +Ġw itness +Ġc le +Ġun ion +ef ault +are t +Ġthought s +Ġ ---- +_pro cess +_ us +ing ly +U ES +T ouch +ĠÐ ¼ +_ open +ĠV ec +Ġre ward +.C lick +/ : +Ġn ie +Ch anges +M onth +ï¼ Ł +Ġexec ution +Ġbe ach +( Integer +ĉ a +/ ' +.Font Style +Ġab ort +ĠS ingle +( isset +Ġd p +Ġ}} +Ġ* = +ĠP S +Ġdanger ous +[ p +OM E +O ther +ĠString Builder +Point s +head ing +Ġc urrency +Ġpercent age +_A PI +Ġclass ic +the ad +ĠM O +F E +Id x +aw ait +Ġà ¨ +Ġacc ident +Ġvari ant +Ġm yst +ĠL and +ĠB re +Ġh arm +ĠA cc +Ġcharg ed +ion es +Vis ibility +ar ry +ĠL anguage +Ġwalk ing +" .ĊĊ +if er +Ġleaders hip +.F rom +yn am +Ġt imestamp +i pt +ĠH as +REF ER +ĠIt s +Ġlist ener +UT E +_d escription +Ġexperi ences +Ġcre ates +R S +c art +bl ack +Ġcho ices +w ar +Ġ'' ' +Ġorder ed +Ġeven ing +Ġp il +Ġt un +ĠB ad +( app +r andom +Ġexp licit +Ġarr ived +Ġf ly +Ġecon om +-m ail +Ġlist s +Ġarch itect +ĠP ay +Ġd s +ĠS ol +Ġveh icles +H z +- com +Ġk ing +_e qual +ĠH elp +Ġab use +-- ;Ċ +Ġex tr +Ġchem ical +ä ¿ +Ġor ient +Ġbre ath +ĠS pace +(e lement +w ait +DE D +ig ma +Ġent r +Ġs ob +- name +Ġaff ected +ik a +Ġco al +_w ork +Ġhundred s +Ġpolit ics +sub ject +Ġconsum er +ANG E +Ġrepe ated +S end +Ġ# [ +Ġprot ocol +Ġlead s +use um +E very +Im port +(c ount +Ġchalleng es +Ġnov el +Ġdep art +b its +.C urrent +Ġ` ${ +ot ing +( \ +Ġcreat ive +Ġbu ff +Ġintrodu ced +us ic +mod ules +A re +-d oc +l anguage +_c ache +Ġto d +? > {{ +ĠRes ource +ĠSt andard +ĠP rem +up dated +ival ent +Ġas sets +_t emp +Ġinterest s +Ġhard ware +ĠR om +ĠSh are +Ġ' 'Ċ +Ġ* , +ĠT ake +ĠIm ages +_C HECK +(type of +ĠJ un +\< ^ +Ġli qu +Ġwor st +ymb ols +ĉĉĉ ĠĠĠ +Ġdr ivers +ĠD ocument +en o +ĠTechn ology +Ġappro ved +ump s +Ġs now +form ance +_A SSERT +u its +Ù Ĩ +Ġdiffer ences +. Visible +ĉĉĉ čĊ +ĠP s +_f etch +Ġto do +. ',Ċ +Ġs el +ur ers +in valid +Ġt weet +V EL +Ġresearch ers +Ġs printf +ĠR O +Ġp el +.Tr ans +Ġil legal +d ialog +sm arty +l g +_M IN +Ġher o +f inal +Ġp p +.L e +Ġc i +ĉ RT +Ġsuggest ed +p df +ach ing +ĠR o +ĠProp erties +ĠS i +Ġbuy ing +Ġm u +Ġl ands +if iers +ĠF ILE +RO UP +Ġh older +ĠS on +Ġsym pt +.r oute +) ? +Ġarg c +Ġfor t +Ġcas ino +_c ategory +Ġfor um +p refix +apt ure +T ube +em s +im ize +Ġn ue +a us +c ourse +AT OR +() ), +Ad vertis +ING S +Ġack now +ĠKore a +pl ing +Ġwork er +PL IED +h al +ĠRich ard +Element s +ĉĉĉ Ġ +st ar +Ġrelationship s +Ġche ap +AC H +ĠX ML +, & +ĠLou is +Ġr ide +_F AIL +Ġch unk +[ s +_O UT +Ġch osen +_ [ +/ ( +ĠJ eff +_s l +pr iv +ĠCan adian +Ġun able +_F LAG +Ġn os +h igh +Ġl ift +f un +() { +el ly +ycler View +_ as +_L IST +Ġr adi +.get Value +ĠAnge les +ĠS pan +_in stance +it ors +Ġm igration +A K +O h + ® +. selected +ĠG T +Ġadv ance +ĠSt yle +.Data GridView +e ction +Ñ İ +p io +ro g +Ġsh opping +ĠR ect +I lluminate +O U +ĉ array +Ġsubstant ial +Ġpre gn +Ġprom ote +IE W +.L ayout +Ġsign s +/ . +Ġlet ters +Bo ard +ct rl +" \ +ĠJ ones +Ġvert ex +Ġj a +Ġaff ili +Ġwe alth +ĉ default +Ġsignificant ly +Ġe c +Ġx s +act ual +.p er +_st ep +an vas +m ac +Ġtrans l +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Iter ator +Ġo ch +agnost ic +ĠD uring +ĠDE FAULT +Ġt ill +Ġsign ature +Ġb ird +ĠO l +ĠI r +H S +av atar +ESS AGE +Ġe lev +Ġm t +ĠN av +Ġrel ax +Ġpl ate +IT EM +( date +.n ot +Ġgr ade +Ġ} ),Ċ +? "ĊĊ +i ences +H igh +ĠD IS +dis abled +Q UI +Ġno ise +a ux +ĠU P +os a +Ġv oc +Ġ )) +oc om +_O FF +ĠD b +L ock +.e clipse +, d +ĠD raw +Ġ" ( +Ġvis ited +Ġâ Ī +Ġsuc ceed +Ġim possible +a ire +ĠT urn +Ġd ish +F G +Ġs ensor +AN N +ab a +Ġsur g +] );čĊ +Ġf p +_ an +- J +- G +ĠJ ob +Con vert +ĠKE Y +Ġauth ors +_s erver +\ r +Ġ-* - +f lex +Ġs oc +R et +Ġs alt +ĠâĢ¦ ĊĊ +ĠC lear +(p age +-d anger +Ġroom s +con v +# { +. op +ĠA rea +_S C +h en +Ġbeg ins +- y +Ġexc ited +Ġign ored +Ġbon us +st udent +ĠM ember +Ġrel atively +ĠL ow +ĠPro du +ate way +pos ure +Ġth ick +ani el +( view +ĠCr ush +Ext ension +I l +e ed +LO C +. im +. Items +Ġconflic t +.pre vent +Ġon Create +u v +is er +Ġw ave +M ar +ĠComm unity +ic he +ĠNo thing +[ m +ĠLe e +ri ends +è re +!! ! +an z +. result +ĠS K +_P ARAM +Ġdem ocr +Back Color +.ex ists +" It +( options +ra zy +as er +\ Database +al endar +_ ass +; }Ċ +vert ex +ine craft +W arning +arg o +Ġact or +ĠInst ead +ĠUs ing +S elf +@ interface +Ġspe aking +ĠPar is +ĠL ICENSE +.n ode +ĠF ood +E IF +ĠB i +. Start +ĠI B +Ġun iversity +ĠHe ader +.pro duct +C opy +et c +r ical +Ġ> >> +book s +Ġal gorithm +Ġ' __ +(j avax +Ġnumer ous +Sh are +H ave +Ġrec ru +Ġpro ve +.sub string +he alth +е л +Ġdec imal +Ġcomm ission +s cription +x C +Ġsum mary +att ed +Ġclo ser +fin ished +() ){Ċ +ĠW ood +_field s +k u +_ items +Fl ag +Ġconf idence +ĠF ederal +du x +Ġcomp at +Ġvert ical +Ð ¹ +è s +; ">Ċ +_m anager +() ))Ċ +ID E +: ", +__ Ċ +ĠW ay +Ñ Ī +T emp +ĠS TR +rit ten +S ync +ĠA V +ĠC EO +ĠG uid +Ġenvironment al +Ġcorrespond ing +ĉ console +Ġjust ice +ĠJ S +Ġl ived +g ar +ĠG raph +ĠSt at +Ġi Phone +. al +ĠH D +Ġocc ur +Ġth reshold +Ġon click +RE G +.Graphics Unit +M eta +Å ¾ +Ġc um +.g nu +à « +Ġobt ained +Ġcompl aint +Ġe ating +Ġt ar +_t ask +Ġopt s +( to +P ass +Ġpl astic +t ility +ĠW in +.prevent Default +p ile +ĠG ar +Ġqu antity +_l ast +Ġg reatest +D ao +_D IS +ĠUs ed +ĠH P +rit ing +S ION +bl ue +d omain +Ġs cores +N ormal +_ admin +ĠA SSERT +Th en +** * +d ist +l on +Ġh ate +sh al +Image View +d atabase +Ġp and +Ġlog ic += false +b g +ĠConfig uration +Ġn ur +O G +Ġmar ried +: + +Ġdro pped +Ġreg istration +оР¼ +ult iple +iz ers +sh ape +.c opy +Ġwe aring +ĠC ath +Ġded icated +Ġ.. .Ċ +Ġadv oc +ĠF amily +Ġstat ements +em atic +ampions hip +Ġmot iv +ĠH ave +Ġbl ow +J ob +c ert +_v ector +inst all +ĠC OPY +em bed +D IR +ĠS pring +Ġex hib +cd n +ĠCom ment +ĠOption al +. player +ĠD ark +( pos +ĠSh ould +Ġcent re +ĠGu ard +ó w +Ġtr ouble +EN ER +( unsigned +_s ervice +Ġn s +ul ing +ĠMex ico +ĠN Y +mys ql +Ġl ic +å ľ +M r +- fl +ĠC ustomer +id i +Ġ? >ĊĊ +ri ble +Ġп ÑĢ +Ġs izes +_STR ING +valid ation +ĠJ on +( Http +add Class +N odes +Ġfrag ment +Ġsp oke +Ġw aste +J oin +Ġill ustr +el i +c ient +Ġa id +Ġpro sec +') {Ċ +Ġpass ing +Ġf aces +Sh ape +_ Z +it i +Ġal le +Ġro bot +ĠĠĠĠĠĠĠ Ċ +ĠS pe +Ġrece iving +ĠD etails +Ġ" ) +m g +_RE F +Ġcompar ison +* , +ĠF ound +_s ession +( U +/ F +Ġx xx +N etwork +d ers +Ġcap ture +Ġcor re +ĠL td +ĠAd v +[ @ +Ġcl ip +M ill +ĠPro file +Ġend if +Ġob lig +des cribe +.e lement +riter ion +L D +er ed +Ġfav our +s core +ĠF ilter +at tributes +Ġcheck s +In flater +ĠPl us +Ġscient ific +Ġpriv acy +He ad +Ġfe at +Ġdeg rees +ĠP ale +; "> +Ġfil ms +ĠA udio +ĠT ag +ĠE nergy +it ar +par ator +Ġf ellow +Ġev t +ĠT ri +ĠD AM +cl oud +ĠP assword +ĠDemocr ats +ĠAc ad +$ lang +Ġre b +() )ĊĊ +н Ñĭ +ĠB ur +read cr +Ġh ex +Con sole +ct l +ous el +ĠWill iam +Ġa z +_P ORT +Ġpract ices +Ġany where +ĠP osition +Ġ- >Ċ +i ams +.user name +place holder +Ġo der +ĠSecret ary +Ġi T +mon d +event s +? âĢĿ +.S ub +Ġatt ached +Ġn ão +Ġest ate +. action +Ġfig ures +Ġ} );čĊ +Ġsubs cri +.t ag +n am +. plot +no on +li ament +Char acter +.t ab +Ġw inter +ĠVar iable +Ġtre es +Ġpr oud +( V +_ load +Ġh ier +ĠE con +Ġf d +Ġvict ims +R est +ian a +Ġf ake +.Print ln +Ġstr len +Ġs ad +Ġb le +Pro t +Ġbutton s +Ġte levision +Ġlog o +ext ension +ĉ j +ste in +acion es +Ġ"" "ĊĊ +Ġsim p +Ġrecord ed +Ġbr ings +Ġprincip al +Ġfe es +(s ource +k dir +Ġutil s +Ġcorrect ly +f il +Ġw el +P air +-b utton +s cale +ver ify +[ c +Ġ-- - +Ġes cape +ik es +Lower Case +ic ian +Ġch apter +ĠT YPE +Ġsh adow +Ġaw esome +W E +el if +Ġl ambda +Ġdist inct +Ġb are +- off +Ġcol our +.append Child +ole c +ag a +.f ill +ĉs uper +Ġad j +( position +.get Item +Sh ort +Ġtot ally +V D +ĠT re +_ ep +v ements +ĠS olution +Ġfund ament +F ollow +Ġfac ility +Ġhappen ing +O F +.text Box +S pan +Ġ « +id en +Ġex ceed +(p arent +Ġc p +ç » +Ġhas n +Ġp ri +Ġcon sequ +n en +ĠIN TO +I gnore +ĠF uture +Ġcar bon +ĠSte el +f mt +ok ie +Ġs pl +(t itle +- info +Ġde als +Ġfix ture +e a +D iv +Ġtest ed +_ return +)ĊĊ ĊĊ +upport ed +ĠC ook +Ġpay ing +ĠI ll +Ġarrest ed +ĠPr ime +_c allback +> ,Ċ +dr iver +On ce +ab b +_by tes +ĠS ets +( Object +Ġc c +Ġsh ell +al o +); // +( log +ct ors +) +Ġ$ (". +.p os +Ġbo ys +Ġwed ding +Ġag ents +=" _ +ĠAr my +Ġh int +v ision +Ġte ch +ĠCon nect +Ġleg end +ĠB et +.B ase +Sub ject +Ġl it +Rem ove +Ġ" : +ĠF inal +pear ance +ĠiT unes +Ġparticip ants +ĠPy thon +Ġbus y +i el +vert ices +Ġtemplate Url +ĠC lose +Im g +ĠCorpor ation +t imestamp +Ġext end +Ġwe bsites +Ġposs ibility +о ÑĤ +Ġk ö +Ġme at +Ġrepresent ation +Ġ ĉĉ +_ST ART +.app ly +ĠVal ley +ĠS uccess +H i +Ġn ob +ĠI Enumerable +_ select +ge o +. ")Ċ +Ġturn ing +Ġfab ric +(" ");Ċ +Ġpers pective +é Ĺ +ĠS n +Th ank +; j +.Param eters +ĉ ĠĠĠĠĠĠĠĠĠĠĠ +Ġfact s +Ġun t +.in stance +################################ ################################ +- end +ĠJO IN +ĠH en +Ġur i +åIJ į +Ġн а +ĠIn fo +Ġconduct ed +Ġà ¥ +OUR CE +Ġw ine +J ohn +.Error f +ĠA ge +ound ed +Ġreal ize +Ġ] ; +Ġsub sequ +, m +( User +ian o +Ġaccom pl +is p +.st d +é ĩ +ĠB ed +.set Attribute +B R +ke ep +ĠA LL +Ġis ol +am ma +P ackage +Ġoccas ion +-s uccess +еР´ +ĠLIMIT ED +st rip +() ĊĊĊ +istrib ution +Color s +Ġ+ :+ +Did Load +al er +Ġt id +ĠL ED +ĠLink ed +ĠC art +() )čĊ +_RE AD +Ġkill ing +ĠP HP +fe ction +Ġinst ances +c v +"/ > +Ġs f +Ġtax es +_ location +ĠBit coin +u able +r ank +ign ore +tr ack +к а +Ġshould n +ĠO P +=> {Ċ +Ġk m +Ġh elper +_ head +ĠWh ether +oc o +_b l +Ġstat istics +Ġbeaut y +Ġto g +t ip +ëĭ ¤ +Ġc sv +(s ql +std lib +we ak +Ġlik es +Ä į +Ġrepe at +Ġap artment +Ġem ph +_ edit +Ġv it +ĉ type +E ven +ut en +Ġcircum stances +b ian +Ġs ugar +W indows +ì ŀ +Ġobs erved +/ data +Ġcal endar +Ġstri ke +ĠR ES +_s c +f ony +ore m +( z +p ower +et ect +ĠS at +.d escription +Ġg ang +ĠS ports +ong s +ĠB undle +.s um +on ce +Ġacc used +Ġexplo re +Ġapprox imately +Ġlos ing +thes is +ĠF und +Ġdi agn +A utowired +prop erties +Ġ_ . +Ġc nt +ced ure +Ġy y +Ġgr ant +so ck +.inner HTML +Ġ] );Ċ +ĠCON FIG +=' $ +] ];Ċ +UN D +Ġg lob +Ġd ire +uff le +_M EM +Ġauth entic +> (" +Ġdec ade +ĠIm port +Ġorigin ally +Ġj Query +Ġindic ate +Ġours elves +S w +.l bl +ener ate +Ġbas ically +ĠH om +Ġ+ #+ +ĠBrit ain +ĠK ar +to Equal +.st op +Ġmod al +is i +Ġsuggest s +Ġd type +Ġt ur +b f +Ġconnection s +ĠB efore +ist ed +m ouse +Ġpul led +.b uild +Ġlegis lation +Ġfor th +p ad +eg o +.N ow +Ġexc iting +}ĊĊ ĊĊ +Ġcom pr +Ġsh ares +Ġr ig +g reen +_ vec +Ġenumer ate +A uto +ic ator +ĠR ay +as se +Ġh oliday +Ġnull able +g un +_d etails +Ġwr apper +se q +ĠYou ng +ju ana +Ġ" __ +lic ense +ser ve +^ ( +id ers +.Rem ove +rop down +' S +p in +(t oken +.D efault +Ġreason able +amp ion +ĠS ociety +Ġbe i +erv es +r ad +ĠF ox +_ images +Ġw heel +') [ +Ġc fg +( By +Con structor +Ġv ary +.sw ift +Ġpro xy +ĉ H +ĠAn other +ĠP en +Ġcheck ing +Ġj est +man ager +Or igin +ug s +o ir +>< !-- +Ġexpress ed +Ġmod er +Ġag encies +Ġi h +-h idden +ious ly +ĠR od +Ġso le +M ed +.A ny +Ġp c +b al +Ex ample +ĠS ale +Ġst rip +ĠCom p +Ġpresident ial +M ost +put ation +( ref +ĠF our +_f ilename +Ġen forcement +Ø ¯ +ĠGe org +we ights +/ l +Ġag gress +Ġd rawing +and y +< I +- j +ak a +h ref +Ġteach ers +_ Q +( it +ĠM B +Ġtemp orary +ire base +str a +æĹ ¶ +è ´ +( label +ou p +Ġtop ics +Ġport ion +id os +ĠJew ish +Ġre covery +Ġstand s +# [ +Ġafter noon +ĠArt icle +_ att +Ġexpl an +ĠP ak +.setOn ClickListener +. children +Ġi k ++ ( +l ag +Ġdis k +Ġcont rovers +"> & +as p +Ġw ie +ĠAustral ian +ĠYou Tube +At tr +cont ains +du ce +ĠM att +at ern +Ġvol unte +Ġnew sp +V P +olt ip +Ġde legate +_m eta +Ġaccur ate +ĠEx ample +% , +ĠD aily +Ġc abin +ĠS W +Ġlim its +k ip +Ġar my +Ġend ing +Ġb oss +ĠD ialog +Al so +="# " +ord an +row se +- min +Ġ" & +_ loc +U X +Ġdevelop ers +Ġaccur acy +Ġmaint enance +Ġhe av +Ġfil ters +.T oolStrip +Ġn arr +ĠE mp +ORD ER +ĠM obile +.S erial +.out put +.c ol +M aterial +um a +Ġconsum ers +sh ift +Ġp ued +Ġmin i +c ollection +Ġk an +.c enter +H istory +Ġben ch +() ); +itor ies +Ġcrow d +_c all +Ġpow ers +- E +Ġdis miss +Ġtalk s +ĠCh annel +for ward +_ control +/s rc +i est +**************** ******** +Ġbet a +(c olor +_O BJECT +ĠA pi +Ġeffect ively +C amera +s d +uss y +D ict +ĠE ffect +ib ilities +Ġreturn ing +ĠF ar +Ġ' ') +Ġmod ules +il ation +Ġ( % +TR GL +Ġst orm +on na +ĠEX P +Ġs pons +Ġdis pl +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠ +f all +å Į +ign Key +_ US +et rics +Ġhand les +T L +_ amount +ow a +br and +ĠT ool +Ġus ual +. Z +cre ment +ad ium +st ock +Ġserv ing +ĠB on +Ġline ar +ĠT arget +ĠR adio +H L +Sh ader +om atic +ag ues +in ity +d iff +_ iterator +qu ot +Ġ ,Ċ +c allback +Ġsympt oms +[ _ +ĠB ul +ĠF eb +und o +_ account +Ġtyp edef +и Ñģ +tr as +User Id +ĠP enn +ĠSup reme +} > +user Id +ĠK im +Ġg a +Ġart ists +å ¸ +ĠAb stract +ok emon +Ġh am +o val +Ġch a +at en +å Ĩ +F ixed +Ġvul ner +ĠParam eters +qu antity +.C lear +Servlet Request +Ġy a +Ġsou l +trans action +Ġsol o +Ġp airs +æ Ķ +ĠG re +_ word +ĠC C +Ġg i +z ie +Ġsched uled +rot ation +gy pt +ul ous +:: _ +ĠE ll +< ! +ĉĉ ĠĠ +l p +ah a +C opyright +Ġdr am +Ġdi agram +ĠM em +Ġg arden +Com p +Ġattempt s +uff ix +> () +Ġphil osoph +_re l +å ¼ +Ġs v +.se cond +ant o +.J son +ĠTe le +_ local +_s end +Ġas pects +ì Ĺ +IB LE +Ġr ail +Ġwid ely +ash ed +i ar +in f +up per +d jango +_result s +iss ing +Ġequ ivalent +OUN D +Ġt y +Ġpotential ly +Advertis ement +ĠRec ord +resent ation +_w idget +ound ing +Ġrelig ion +Ġcons c +ĠL im +. am +H tml +Ġ' : +P ATH +_s pec +ort ed +id ades +_sh ape +Ġkeep s +.S ave +ĠL oc +or i +ĠT EST +unic ip +Ġreg ions +Ġbelie ves +/ en +pos ite +{ ' +pre pare +_ const +s ample +ĠWill iams +Ġstr t +_ Get +ĠAnd rew +. active +Ġl ayers +Visual Style +az y +ĠK n +Ġac id +ĠAs ia +Ġex cess +ĉm y +Ġkey board +ens us +Ġcre w +Ġmiss ed +m aster +ĠW ild +Ġnew ly +Ġwin ner +Ġst ub +ic ode +.m ove +D omain +ĠS ar +Ġfore st +LE D +claim er +.ex it +ĠW indow +Ġres istance +ĠC HECK +(" - +ĠR yan +Ġp ipe +Ġco ast +DE F +// ! +_ off +ex it +Ġult imately +imit ive +ĠKe ep +Ġhistor ical +Ġany way +ĠJack son +ock er +ER N +ĠU INT +y ntax +ER Y +is ms +Ġc n +Ġocc urs +Ġ; ; +Text View +A E +/ img +Ġy esterday +- default +Ġt iny +Ġpro c +Ġal ive +ĠRE G +. th +ear ing +.get Logger +< link +_ login +F older +ab c +lyph icon +н о +Ġnot iced +od igo +Ġed ition +im ator +. Enabled +.parse Int +Ġy ards +ĉĉĉĉĉĉĉĉ ĉĉĉĉ +Ġver bose +л Ñı +_B Y +.log in +.* ;Ċ +ĠM id +é es +Ġg lo +Ġbuild ings +Ġz e +ĠI ter +Ġt ube +ĠP ot +\ M +< th +br idge +ĠS cript +ĠM odule +Ġv acc +Ġinstall ation +v y +VisualStyle BackColor +ĠS M +.t otal +b at +Ġfind s +Ġat mos +Sub view +iz ard +Ġrepl acement +lic ated +ap is +Ġlog ged +ĠLe ft +G ui +_ Type +t m +P ad +Ġhouse hold +Ġre le +Ġpropos al +_CL ASS +:: :: +Ġinf rastructure +In ject +/ html +Ġad s +iz za +Ġm g +ctr ine +% Ċ +< html +- image +Ġatt orney +< m +(' , +Ġcan n +Ġprint ln +o ose +Ġy ellow +.ex p +p ayment +Ġtable View +aw ay +Ġopp osition +ĠAg ain +ĠH andle +Ġex clusive +in ar +é r +оР± +ĠC ODE +emp orary +Ġre act +pi pe +c z +. activity +Ġlarg ely +Ġdis s +ax y +es is +ĠR en +Ġc orn +.Use VisualStyleBackColor +d ays +Ġfr uit +In sert +_ enc +E st +_de c +ĠL uc +Ġü ber +param eters +P ERT +ex press +_pro file +Un known +Ġrev olution +.add ress +_re quire +Ġun iform +ĠP ack +l ar +ĠU ITableView +Ġdep ends +Valid ation +conf irm +O wner +Ġt rib +h et +ĠI de +ans as +L anguage +u et +ĠP o +ĠSte ve +Ġcont est +_DE FAULT +Ġapparent ly +RE EN +Ġfrequ ently +Ġtrad ition +ocol ate +S I +ĠArg ument +F ocus +ert e +ĠL ayout +Ġd x +Ġgener ator +ĠW ait +P olicy +l ights +.Ex ecute +P y +Ġbed room +ed a +ra id +ĉs ize +Ġan cient +Ġp ump +Ġd w +Ġ(! ( +Ġspec ify +( status +ĠF BI +.ex ception +Ġrem ark +ly mp +ant ee +Up load +ern et +é ¡ +in ent +ĠR ender +d m +ĠM emory +r ich +ĠT ools +Ġk ne +Ġper m +b ad +Ġd inner +.res et +Ġj Label +Fe ature +.S ervice +Ġ( {Ċ +Ġre ferred +.class List +Ġinit With +ĠText View +Ġne ither +Ġcount y +Ġ" { +ç § +Ġt ack +class Name +ĠUS ER +Ġre new +` ` +get Name +Ġb rown +Err ors +ert o +Ġsust ain +S O +let es +ĠIn valid +Ġen emies +un ge +Ġexist ence +err a +Ċ ĠĠĊ +utor ial +# a +p ay +char ge +ĠI re +ate st +Ġexp los +Ġf ired +N ER +ĠT y +ic ion +U ri +Ġobvious ly +ĠC olum +Ġ' + +ĠDe vice +- related +_ ARG +Ġv or +ĠLess er +_O P +Serial izer +Ġup grade +L ight +Ġc odes +++ ;čĊ +Ġwrit es +fo od +Ġé t +@ section +Ġtrack s +Ġserious ly +ch t +(size of +Ġimmedi ate +Ġscient ists +Ġ{ $ +_ ne +.Anchor Styles +Ġaccom mod +ĠHar ry +Ġs ight +ĠPale st +ersist ent +Ġ Ñĥ +- input +Ġco ordinates + · +W elcome +.con f +Ġgre w +Ġb old +ĠC PU +(m y +Ġperfect ly +Ġmom ents +ĠM ovie +- data +yst al +_W IDTH +ĠS creen +æ Ŀ +Ġdis ap +Ġredu ction +.Get Component +_M ODULE +Ġgener ic +Ġd y +all er +Ġc url +ĠB ody +Ġb anks +, t +av g +Ġev il +Ġmanufact urer +Ġrece iver +Column s +Ġing redients +ĉ out +qu es +.L oad +Ġslow ly +ĠT own +ĠC ell +_n ormal +_p refix +ĠAl ert +(" { +ä r +âĢľ The +ĠM D +Ġcour ses +ath an +é Ļ +oc c +ĠS ER +es ign +Add r += [' +(" ./ +] } +.f ont +ĠInst agram +ĠB order +od a +Ġh all +Ġr um +_b it +Ġs aving +_d own +R andom +_reg ister +( Context +Ġoppos ite +R oom +Y ES +ан и +Ġenjoy ed +_r un +C lear +âĢ ĺ +ĠF ord +on ic +ost en +"] ) +_ auth +// čĊ +Ġsuff icient +LE S +Ġph en +Ġo h +_c sv +Ġrout ine +.Are Equal +ay lor +Ġb asket +_COM M +rypt ed +S im +ĠSh op +Ġstud io +at os +( W +[ string +ä t +og a +Ġsh r +Ġs ick +An other +Ġdo ors +_N E +ĠTH REE +. order +raz il +Ġmap s +_TR UE +trans late +Ġnear by +Ġn ach +LO AT +b atch +Ġl ux +ash es +ang ers +âĢ¦ âĢ¦ +_E VENT +_ UP +Ġact s +in v +_M ETHOD +cc ion +Ġret ain +ut ch +ĠÐ ± +Ġknow ing +Ġrepresent ing +N OT +p ng +Con tract +Ġtr ick +ĠE dition +uplic ate +Ġcontrol led +c fg +j avascript +Ġmil k +Wh ite +Se quence +aw a +Ġdiscuss ed +ĠB ush +ĠY ES +.f actory +t ags +Ġt act +Ġs id +$ $ +ĠE num +Ġfr ames +} ); +Ġreg ul +'] ;čĊ +Reg ion +ff f +Ġc ro +( com +=" + +St udent +Ġdis appoint +RES ULT +Count er +Ġbut ter +ĠH a +ĠD igital +Ġb id +"> {{ +ing ers +ĠC ountry +_t pl +"] )Ċ +/ k +d ating +: # +ĠD ATA +yn chron +_b ody +olly wood +Ġval or +ip ient +o ft +UB L +doc s +Ġsyn chron +Ġform ed +ru ption +Ġlist a +Request Mapping +Ġvill age +Ġkn ock +oc s +" { +_fl ags +Ġtrans actions +Ġhab it +ĠJ e +ed en +Ġa ircraft +ir k +ĠA B +Ġfair ly +. inter +.A ct +Ġinstr ument +remove Class +.com mand +Ñ ī +ĉm em +( min +Ġo t +Ġcol le += s +time out +Ġid s +ĠM atch +ij n +z ero +Ġnetwork s +.g ov +Ġint el +Ġsection s +out ine +(c md +(d ir +ĠLI ABILITY +ĠB log +Ġbr idge +ĠC V +con vert +Ġ" )Ċ +ĠB ern +_P O +e val +( set +to ol +Ġpay ments +Beh aviour +Ġcon crete +Ġel ig +Ġacc eler +Ġh ole +_ o +TE GER +Ġgraph ics +O wn +Form atter +on der +Ġpack ages +/ a +ĠK now +Or Default +Ġdut y +W ait +н а +_rec ord +[ t +M esh +Ġon going +.be ans +Ġt an +Ġinter pret +ast ers +QU AL +Ġleg s +\ Request +- file +_m utex +ĠS aint +// # +Ġpro hib +( info +: = +lin ux +Ġb lo +ot ic +ĉf inal +_ex p +ĠSt op +ap ing +(s aved +_p ush +Ġe ase +_F R +pons ive +str cmp +: ĊĊĊĊ +ä» ¶ +ol i +Ġextrem e +Ġprof essor +Im ages +.IO Exception +Ġaddress es +plement ed +Ġincor por +Ġuse Effect +_O F +ĠD a +n ombre +IR ST +Ġdisc rim +Ġcomp ens +greg ate +anc ell +ach es +ĠC riteria +$ result +D estroy +Ġsecond ary +W atch +ĠS em +ĠMc C +Ġacad emic +U pper +:: ~ +ut ral +ĠD og +ad ed +Valid ator +Ġder ived +Ġset Timeout +ĠK en +Ġtyp ical +ĠB ob +Ġb ounds +ĠSe ason +Ġc razy +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠ +-r outer +itt est +ĠM ir +Ġemot ional +, v +c n +/ st +å ½ +on om +Ġdecl ared +> . +ail ing +Ġ/* <<< +Ġnorm ally +(M e +ev in +lik ely +Ġpoint ed +ĠSt ack +Ġw alls +. Vector +me an +] ]Ċ +Ġlist ening +ad v +Ġsw ap +IF T +Ø ª +. argv +ul s +< option +not ations +Ġemail s +ĠU kr +ast a +ĠTh us +ĠSt one +Ġappe al +. âĢĻ +Ġreg ulations +Pre ferences +ĠPh one +ul f +ĠD R +Ġtechn ologies +Ġpar agraph +Ġnecess arily +.e ach +< float +res a +Ġunder st +Ġf inger +press ed +-b y +if fer +w atch +ĠB a +A IM +Ġwe ights +ĠR on +') }} +[ self +-------- --Ċ +per iment +Ġto String +x ic +ĠC amera +! ĊĊĊĊ +aur ant +P refix +Ġinstit utions +: int +Ġex posure +p attern +ĠLin ux +.n umber +red ient +Argument Exception +ĠCh ief +" }, +Ġelect ronic +r ong +er d +sp Net +ra it +/ ', +ĠOh io +Cont rollers +Ġcontin uing +ĠT emplate +ĠE th +s z +/ env +En v +% . +art ers +) (( +ĠT ABLE +Ġà ® +per ature +pro gress +P res +ê ° +im plementation +Ġb ien +Ġstre ets +_M SG +New s +## # +: / +Ġcut ting +x B +ress ed +_EN ABLE +l ab +Ġca using +] ));Ċ +b ra +x FFFF +il ly +plet ion +w ill +_b ar +Ġstruct ures +ĠI mp +Û Į +Ġ< > +Ġ ---------------- +_B UFFER +.d ir +Ġpl ain +Ġpe er +g g +oint s +Ġsomew hat +Ġw et +Ġemploy ment +Ġtick ets +ir ms +Ġt uple +s is +$ sql +r ig +Ġcon version +Ġg es +Ġconfig ure +eg r +ĠC a +Ġ__ (' +ou ston +.t oken +Bl ack +Ġmag azine +A W +. IN +os ing +Ġbro ke +ĠC ru +DE LETE +Ġdestroy ed +(M ath +Ġappro val +-d om +ĠI II +table View +Ġdesign s +Ġcrush ing +Ġcons ent +dir name +om p +Ġc rypt +? ( +or ough +. o +ĉ list +ams ung +."" "Ċ +err ing +G oogle +_p air +_IN IT +rem arks +Ġg ear +F ill +l ife +} ")Ċ +Ġsuit able +Ġsurpr ised +_RE QUEST +Ġman ifest +att en +Ġfr ustr +ov ement +.c lick +Ġi i +Ġexp ansion +ig s +P arse +.Reg ular +R ob +_l ayout +ì ł +Ġtrans lation +ĠBe aut +B est +_C OLOR +< label +Ġliqu id +IT S +Ġpro d +Ġoper ate +UI Kit +Ġn atur +arg ument +_d etail +ĠCent re +Ġ" -- +Ġ}} " +lo cale +.t v +_se q +Ġup coming +Ch art +ĠDiv ision +Ġclin ical +Com pany +S epar +l as +ĠH un +: s +Ġhead ing +оР³ +Ġ" ");Ċ +[ id +b ia +Ġst retch +ic ide +Ġre produ +.pro ject +leg end +end ers +Ġrespons es +Ġon t +rit ical +Ġref uge +ĠL i +Ġ: ĊĊ +ĠTh ree +.cont roller +_IN DEX +_F OR +\Model s +j ax +ĉex it +Ġâ ĸ +Ġc overs +ĉ y +- . +IND OW +Ġfail s +in cludes +Ġf ault +Ġl y +ñ o +.s lice +ILE D +ĠP ur +ĠAs ian +_b atch +.M ax +v l +ĠCOPY RIGHT +Ġg iant +ĠMan ual +ĠC opy +Class Name +He alth +C ursor +IB Outlet +Ġt we +æ ³ +_label s +Ġcol lected +Ġfurn iture +Ġdeal ing +Control s +ĠHot el +ck s +Ġch ose +âĶ Ģ +od d +S R +Ù Ĭ +ì Ħ +Ġacc ord +ĠM ove +ĠM ode +ĠM ock +Ġthread s +++ ++ +ĠO ptions +Ref resh +ĠD id +'] -> +u cc +_ch annel +. abs +Ġ{ },Ċ +ĠW al +er ior +Ġmain ly +ĠDr iver +NotFound Exception +Ġcount s +e am +Ġ& = +Q uestion +ĠA li +Ġany more +d etail +t ail +Ġm ile +ĠF air +Ġs orry +Ġsurround ing +Ġad m +De v +Ġmari juana +ĠS ound +ĠA sh +F D +Te am +. port +Ġ[ ]ĊĊ +ub ble +Ġas c +Ġint ention +A cc +ch i +ust ers +Ġins pired +se g +CL U +Ġman ip +M etadata +Con nect +ĠB eh +Ġfind ings +Ġas sembly +w orld +Ġrem ained +Ġu id +( . +Ġm x +Lo op +ĊĊĊĊ Ċ +Ġfant astic +wh o +ak i +ĠB asic +ĠY et +ĠUs ers +ik ip +Ġhead s +ĠMich igan +_ it +ĠTor onto +Ġrec ording +Ġsub mitted +_var iable +medi ate +.graph ics +Ġst ood +Ġre ar +vel ocity +_M ESSAGE +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ro les +ĠT our +_ year +end ment +amp s +ĠIre land +m al +Ġyoung er +Ġstrugg le +Ġc able +ĠSD L +(' - +an es +ĠNe ed +.R ow +P ol +ĠP H +_s cript +ag em +ĠB as +_s pace +. loc +: i +ad r +Ġengine ering +it en +) & +Ġu k +ĠL ittle +_C OUNT +x A +Array List +æ į +Ġ" ")Ċ +An chor +Ġh ang +t witter +Ġcompet itive +.s rc +ãģ Ĺ +Ġtrans late +ĠCre ates +ook s +ĠR oll +'' 'Ċ +/ sh +s ome +Enc oding +.res olve +Ġdesign er +ĠSt orage +Ġz a +ĠN ever +Ġsomew here +Ġbox es +.s ource +Ġpy game +Ġgrow n +.t w +() ),Ċ +', [' +Ġoppon ent +(s rc +.l ayer +AP P +ĠAct iv +Ġguest s +ĠVAL UES +};ĊĊ Ċ +.n ative +Ġamount s +. RE +Ġcl one +Ġwer en +Ġ" << +_ ac +Ġbreak ing +Ġreli able +.P OST +ĠSk y +Ġ' & +Ġsaved InstanceState +ast ing +ill ion +com ments +ult y +.m enu +/ config +Ġ ĊĊĊ +T ODO +Ġpurch ased +_c or +ĉ auto +Compat Activity +com plete +_ graph +is odes +Ġsitu ations +ĠH or +Re ceive +âĢľ We +Ġent ities +.assert Equals +оРº +ĠS ans +v ince +rom pt += Ċ +Ġ/ . +.Se lect +yl v +Ġb att +A udio +Ġincreasing ly +.B undle +Ġexpl ains +the ast +. offset +Ġh al +Ġtechn ique +_l imit +Ġdraw n +AY ER +Ġfeature d +yy yy +at in +ph en +ach el +! \ +l ower +ĠG R +Ġp ag +ĠP arse +Ġt ou +ä¸ Ģ +D istance +Index Path +Ġh ell +s im +UT TON +Us age +elen ium +ĠF all +Ġ" .$ +ĠM u +Ġcr uc +Ġs ont +REF IX +Ġinter ior +ĠO lymp +.Auto Scale +par a +Axis Alignment +Ġr iver +D to +Ġwith draw +Re act +- class +b efore +_ alloc +Cont ents +ĠW as +I CT +Ġform ula +Ġindic ates +ĠĠĠĠ ĊĊ +_st ore +it ting +ĠIt alian +_S et +_re port +Ġp id +_V ER +Ġw ins +ĠCl oud +") {Ċ +ch ester +Ġden ied +Ġw ird +ĠSte p +Ġinvest ors +b old +_d isplay +ou ver +or er +Res et +Ġsurg ery +Ġstrateg ies +/m aterial +_ unit +Ġc ouncil +.P er +ĠâĢ ŀ +Ġre form +F ramework +Ġlist ing +_b tn +Ġb is +% d +eg as +Ġsudden ly +_S ER +Ġa o +_d irectory +f as +Ġprem ium +Ġtrack ing +ĠB L +Ġm ature +Ġbath room +Ġ'/ ' +ĠÄ ij +Per formed +Ġsold iers +arn ings +Ġwalk ed +- con +b ottom +Ġsurpr ising +Ġg ene +Us uario +.DE FAULT +ĠM IT +C ODE +ĠE gypt +p icker +ys ql +AT URE +d etails +ĠCon ference +In formation +ĠM ail +-d own +r aries +b ro +Ġsubject s +Ġ' * +è¯ · +or ient +: @ +ver bose +E F +Ġto ler +eng ers +Ġend point +Ġstr ange +Ġcol on +Ġpre ferred +de p +ĠE V +ARR AY +Ġw he +Ġp up +_n odes +Ġtalk ed +Ġinstit ution +db c +Ġex posed +te en +ĠFr ont +T T +_N ONE +\/ \/ +pro gram +Ġencour age +. ` +sh ire +ĠIsl am +e en +N I +' " +.W idth +Ġlik ed +Ġ{ ... +ĠSystem s +Ġvot re +Ġmanufact uring +Con verter +ĠIn f +ì ļ +D TO +Ġin ches +Ġ ठ+à ¹ +ĠChar les +B U +")) ;ĊĊ +ĠL abor +un n +Ġest im +m obile +ĠL earn +_C ALL +â Ħ +Ġind ices +Ġt ub +ikip edia +C ost +row able +ë ¡ +g age +Ġfunction ality +uzz le +em os +.l ib +Ġd ass +еРº +enn a +Ġsh ots +Ġrest ore +/ D +For Key +], [ +al ias +l int +.st ream +æ ł +_FORM AT +Ġsil ver +.re pository +Ġlegis l +.B order +_fe atures +Per mission +Ġhous es +ĠW ars +_COM P +Ġinj uries +Ġconstant ly +fl utter +EN U +ĠCon f +Ġrecogn ized +Ġpract ical +Ġde cent +B J +] ); +ast y +ĠAct ivity +-m ode +Ġsl ide +.IsNullOr Empty +ĠY OU +P ower +ind ices +Ġqual ified +Ġthrow n +h ello +ĠN ick +l ah +as sembly +ĠSm all +old ing +Sh ould +ĠSil ver +(saved InstanceState +Ġtog gle +.N ot +C trl +: nil +ĠCont inue +ĠB oot +æ ī +ĠM ur +d on +ĠF A +S napshot +Ġassoci ation +fo x +, a +az ione +] )čĊ +CT YPE +Ġf ade +ĠD ar +.n avigation +Ġl uck +SC RI +ĠDe ad +Ġterm inal +_LE NGTH +Ġeff iciency +Ġun w +Ġn arrow +iment o +( Color +ĠSe a +_ area +, A +_ opt +ĠHill ary +.t ask +ĠJ ac +ast ed +ĠAd am +ĠIl legal +Ġsearch ing +Instance Of +J ava +ĠForm at +Ġreal ized +ĠChild ren +Ġk il +(f rame +âĢĿ .ĊĊ +Ġscen ario +"] );Ċ +Ġincred ible +li x +IO Exception +ĠQ uest +il ty +Ġun lock +â Ĥ¬ +Ġre ferences +ĠV ert +B inding +eg ative +Ġwr ap +.d atabase +( content +B uf +ĠTr ad +ĠA ud +tr ace +.m ock +Ġther apy +ĉ L +.To Int +ĠKing dom +B us +ha ust +"" "ĊĊ +( end +.draw able +[ ];Ċ +ĠH ospital +Ġph arm +---- - +ĠA G +é d +> ");Ċ +Ġw allet +at able +) $ +Ġmonth ly +Ġdi agnostic +S ymbol +Ġiter ator +un finished +Ġimm igration +s r +RO W +(g ame +Ġclo thes +ĠU nt +Ġactiv ation +_C on +.h ash +Ġinitial ly +.H ash +Ġcut s +f ound +ĠSt ory +ÑĨ и +ac ao +_T YP +pro to +est r +-p age +ah r +Ġincor rect +ĠJose ph +TextBox Column +_st yle +ĠD aniel +s heet +Ġl iv +l ined +Ġr a +R untime +_ empty +sl ug +_ struct +ë Ĭ +m u +Ġper mitted +Ġreg ional +Ġsob re +ĠS uch +Ġ[ _ +Ġro of +.Al ignment +t imes +.m sg +Ġche st +ĠT ab +Ġest a +ä n +Ġsubs cription +( command +s pecial +Ġme al +") :Ċ +_ ctx +Ġclos ely +et ry +- be +ad el +ĠR am +ig est +ĠSpan ish +Ġcommit ment +Ġw ake +* >( +P HP +_ { +ck er +< List +_n ull +ĠRes erved +Ġin her +.Column s +.A spNet +_IN VALID +ĠParam eter +Ġex pr +} { +Cell Style +Ġval uable +Ġfun ny +In v +Ġst able +* t +Ġp ill +pl iers +ĠC SS +ĠCon dition +ĠS peed +ublish er +Ġoff ensive +ce st +ic as +Ġsp ark +ĠPro te +set up +IF Y +ĠT ax +Wh o +F amily +- for +. uk +Ġf asc +sv g +") ). +Ġbirth day +âĸ Ī +ve h +el led +Ġimport s +ĠIsl amic +T A +ĠSt an +we ather +Ġsus pect +e ature +enn es +W M +.m inecraft +av id +è ½ +.se curity +in os +G ood +Ġm arch +Ġposs ess +us uario +Con s +am ber +ched uler +Ġhor se +ç ½ +(b ody +ĠTrans form +_de code +.s vg +Ġf oo +Ġd ella +ext ends +am er +Ġprocess ed +ĠH arr +ĠA I +Ġk o +CH AR +( % +Ġt ap +({ ' +c roll +D OM +Ġte a +Ġre in +Ġworld wide +_f n +sh a +Ġb ir +ç ões +="# "> +Ġrepresent ed +ill er +(ex pected +Ġd ance +Ġvisit ors +.con cat +-b it +UR RE +ĠR og +v p +ip h +ĠL LC +it led +iam i +C oll +_re al +_sh ow +_f older +Ġd ar +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġl atter +arch y +Ġb ow +Ġout come +ĠPost ed +Ġris ks +ĠThere fore +Ġowners hip +Ġpar allel +Ġp ending +ge ometry +Ġrecogn ize +ST EM +ĠC P +Ġimm igr +IT LE +ĠĠĠĠ ĉĉ +conn ected +Ġsm ile +(d ocument +\ Component +vert ical +Ġconsum ption +Ġsh oes +. impl +un ks +. ";Ċ +Ġfood s +_ );Ċ +.assert True +Ġp ipeline +Ġcollection s +Ġearn ed +ĠC ert +Ġpartners hip +( action +Ġc d +ĠV ery +Option al +Ġscre ens +Ġtit les +ener ator +Ġab andon +k ind +IL TER +Ġclos ing +lic a +_ inter +Ġcamp us +set ting +S prite +ãģ ¯ +_re ply +To List +: \/\/ +ed e +Ġfol ks +Ġbo at +( argv +Ġperman ent +Ġcarry ing +Ġconserv ative +import ant +. img +ĠIm m +Ġdim ensions +al and +s ingle +Ex it +-------- -- +ari ant +tern al +Se conds +ĠIt aly +ot lin +.Res ume +=' " +) == +cept or +Ġs ca +/m ain +Sec urity +_d at +Ġlet s +Ġa qu +Ġwhen ever +b erry +Ġact ing +ant i +p d +& gt +æ Ń +Z one +T oday +! . +To Props +ab is +it able +Ġg al +] { +iz ona +Ġin contri +N ET +/// Ċ +[ in +_s ave +Ġex em +ĠK enn +Ġev olution +var s +_st ats +- only +ĠColor ado +Ġwatch ed +b our +Ġsever e +Ġprofession als +port ion +Ġguar ante +Ð ³ +Ġpush ed +ĠG i +ï ½ +Ġt um +ĠA z +ĠEdge Insets +")) ;čĊ +is se +. ac +Set ting +Ġapprec iate +ĠValue Error +Ġsur ve +ĠR ole +. Inter +plot lib +j et +d am +Ġplatform s +te le +UT O +ĠInt ernal ++ : +} ;čĊ +Gener al +\ Entity +Ġlawy er +qu iv +ĠPost s +is o +Ġacc um +ob e +Ġmark s +Ġ] ;ĊĊ +ĉ text +.s uccess +cur r +as a +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠ +Ġth in +_ over +are st +ĠO s +( address +Ġvel ocity +Ġ[] ;ĊĊ +=" ../../ +ĠPr iv +b ow +Ġguar antee +% ĊĊ +Ġeval uate +.LE NGTH +Ġin ventory +q a +_de bug +.On ClickListener +Ġl ies +Ġassess ment +dat etime +.background Color +Ġ*/ čĊčĊ +ra f +un wrap +ĠF oot +Ġnot ify +Ġlow est +DO CTYPE +Ġl anguages +ex tra +- back +Ġein en +tem plates +_p ass +ĠM ust +Ġest á +_c ore +ĠSc ot +A I +Ġb ias +ations hip +Con stant +Ġprogram ming +In s +uspend Layout +ĠPRO VID +ant es +Ġsh irt +in ated +. OK +[ a +Ġthink s +? ĊĊĊĊ +Ġregard less +ĠMag ic +ul ating +ĉ class +add Group +RE ATE +ĠS U +Ġsim pl +c opyright +Ġb unch +Ġun iverse +ĠE rr +Ġpresent ation +c ategories +Ġatt ach +.s ign +_A C +Ġdisc ipl +Ġregular ly +Ġprim arily +ink s +[ [ +.r and +.sh ould +ownt own +=" ' +Ġs ans +Ġsupport ers +se quence +G O +. .ĊĊ +ĠS pr +Ġcare fully +U IColor +dest roy +Ġtod os +ĠOR DER +ott ed +Ġd ont +aud i +_ player +g re +ĠO il +< body +_st ack +.P adding +ĠProduct s +Ġpriv ile +Ġinj ured +ĠF urther +Ġal ias +.Resume Layout +_LE N +Ġs es +'] ;ĊĊ +cre ens +Ġdirect ed +.S uspendLayout +od ge +.A t +mark s +ĠUn ivers +ert s +ĠE sc +Ġnav bar +Ġutil ity +agnost ics +Ġin ject +ĠD NA +Ġ" ," +am ar +Ġe u +Ġrestaur ants +_p ut +ut ers +Tool Strip +t w +ist ro +Ġz oom +Ġleg it +pec ific +ĠC ome +Ġlocal Storage +Ġabs or +.P anel +ĠDesign er +Ġo w +IC AL +_ uri +(f ield +Ġsup erv +Ex ists +Ġrespect ively +ĠSt and +Con f +uss ian +Ġar c +Ġ nd +uck s +Ġre str +Ġseason s +ĠCh apter +ĠSw itch +p ic +Ġh i +load ed +Ġfl uid +-b tn +Ġrun time +. it +B N +Op acity +as ant +ry ption +-n ative +Ġta ught +å ¯ +ag ment +Ġm ul +Reg istry +_ grid +ĠBro ok +: Set +Ġm ongoose +AM ES +inner HTML +Ġs oci +ĠInt el +get Id +C md +Ġaccess ible +r ames +le ton +Ġ__ ( +ĉ delete +ĠS quare +" ĊĊĊ +Ġbu cket +avor ite +ĠB reak +++ ] +Ġbr ush +Ġt ensor +/ http +T ile +Ġfunction al +Ġ" * +wh el +Ġt ent +ĠChar acter +Ġse es +. ST +B ig +Ġext ern +Url s +)) )), +ĠJ r +.B uilder +. ; +n l +_ Init +ĠH ER +ż e +mys qli +_ icon +v an +Ġfeel ings +Ġle an +Ġhop ing +T V +="čĊ +b est +all as +ent ed +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĊ +_con nection +Ġrep o +en abled +аРº +Ġsh a +Ġmembers hip +Status Code +in ating +_s m +_c ustom +_ weight +Ġc ss +St at +_ env +link s +TR L +ĠH it +, r +up id +Ġop ens +Ġg ent +_v is +Ġj oy +< w +_c ost +ĠPy Object +ren ce +ĠGeorg ia +ĠBro ad +m ma +â Ĥ +p f +Ġ" \" +Ġ( & +om o +Ġliter ally +Ī ĺ +met ric +Ġb ars +z ed +(w indow +ĠIsrael i +Ġform al +ident ifier +.d ao +ĠDe ath +% ;Ċ +Ġdecl are +ar ms +RE AM +PERT Y +Ġconsequ ences +to ols +Pe ople +ĠWh ich +> ();čĊ +.de code +_A CT +Button s +.f loat +.F irst +ë ¥ +ĠPol it +ĠX CT +T ags +ĠCG Float += str +Ġle af +- check +ĠI ss +.s ystem +log out +ach t +Ang le +s in +ch art +INT ER +ĠN UM +B asic +.P roperties +ä¸ Ń +_ change +ĠB razil +Ab stract +Ġ: +: +_ use +а л +ĠL y +IB UT +Ġout er +Ġ-- >čĊ +Ġrel ief +l ap +qu er +_p arent +he ap +LO SE +Ġcomb ine +ĠR ose +ow ers +Ġproced ures +ĠS ort +an im +var iant +eh icle +Ġsign ing +Pr imary +c urrency +Ġsex e +o en +th eta +em an +Ġimpress ive +(' _ +ĉ U +ĠText Style +_c nt +Ġs lice +(' : +Ġunderst ood +H is +Ġinform ed +Ġn ick +(T AG +h d +Ġelection s +est ure +ĠS anta +ĠCo ast +.p df +inc iple +.cl one +b orn +ut a +Ġl icensed +C r +Ġb read +ĠH ouston +Ġn od +Ġhop es +ĠCG Rect +Ġgu ilty +.g if +Ġro se +.Com mon +T ip +AN K +ĠF C +D uring +ĠSym fony +Ġdef ensive +k m +) > +arch ive +ĠU RI +ycl ing +- o +ĠWe bsite +AM P +ish ment +Ġdo ctors +D irect +AR I +ĠRed irect +ier en +_d ist +y o +ĠPro gress +Ġz um +Ġmem or +ĠE D +Ġj ur +æį ® +_T ABLE +Ġu uid +Ex pr +. head +(' % +point er +Ġest imate +ĠG reg +Ġlo ader +Ġi OS +Ġm ens +[ y +Ġref used +Ġprec ision +is ch +ĠA CTION +Cl oud +s With +( ret +_ADD R +_con f +(d f +Ġlock ed +Ġr ising +ãĥ» ãĥ» +ĠM s +Ġscen es +_EX T +_ raw +_ the +pe ople +Ġre con +ĠF un +Ġb less +ĠUp dated +ü n +ĠĠĠĠĠĠĠĠĠĠĠĠ čĊ +pe ction +Re lease +.log ger +ĠS Y +Ġcoun sel +ur d +_ true +Ġevery body +iv ot +Ġh ence +ĠN AS +Ġoppos ed +unk nown +ĠDES C +ĠCh air +fa iled +ĠIN CLUDING +Ġwrit ers +{ }Ċ +ÃŃ t +_c opy +} : +ĠB at +Ġconvert ed +ed ing +pl acement +ĠH ost +S ound +и м +Ġs ought +m id +Ġsal ary +og g +âĦ ¢ +b ul +Ġw ir +valid ator +_ST AT +.st ore +ĠB attle +ı n +Ġ-- >ĊĊ +Tr ump +d ot +ĠCON T +.f etch +Ġcontin u +w as +Ġfra ud +_t mp +mit ter +.p ictureBox +G A +Ġt ournament +. Input +[ r +ex ion +cent age +ĠKore an +und ef +ĠAv ailable +resh ape +Ġk it +ĠStr uct +ĠS UB +An swer +_l ib +.t witter +Ġo re +ĠDr agon +.Ex t +, k +Ġexplan ation +ref s +ĠDr ive +ĠTr aining +.H as +int age +b ig +olog ist +enn is +Ù ĩ +Ġch icken +ĠĠĠĠĠĠĠĠĠĠ Ċ +ç Ľ +ãģ § +Ġpe ak +Ġdrink ing +Ġen code +ĠNE W +m alloc +ĉf printf +Ġ= ================================================================ +in cluding +Ġprincip les +ĠM ah +st orage +- key +Ġkey word +% ; +Ġtr ained +.con trib +Ġk v +__ ':Ċ +ĠB oy +param eter +Ġsu ite +Ġthous and +Ġco ordinate +-g enerated +íķ ĺ +gener ated +Ġad mitted +Ġp ussy +# w +Ġsw im +un ion +N a +ĠRoy al +.ch annel +Up dated +_RO OT +Ġv ital +ra ction +ĠCrush er +Ġpre ced +Ġhor izontal +Blue print +Ġattr s +Ġsm oke +Ð Ĵ +. Equals +F B +ĠRes ources +roll ing +Ġpass es +ĠN um +rot ate +et ype +\ ", +Ġsens itive +Ġt all +? âĢĿĊĊ +Pro xy +i y +_ section +âĢĶâĢĶ âĢĶâĢĶ +br id +Ġcirc uit +at an +EN C +Ġdr iven +Ġvot ed +Ġeduc ational +Ġinter action +abet es +Ġt one +ĠInitialize Component +Ġmer ely +Ġì ŀ +co okie +_ div +ĠUIL abel +vel y +} );čĊ +_ ENT +#+ #+ +art icles +ĠSou thern +Ġstrong er +ĠG iven +ĠE ric +ĠI R +ab stract +U nder +n able +Ġincre ment +ov en +Ġco in +_t imer +Ġsuffer ed +ĠF REE +'] ." +ĠQue en +st ats +Ġmeet ings +Ġenter ing +Ġalong side +(s ession +it als +Ġfound ation +ĠC redit +. div +_ ALL +pc ion +_st at +ick ing +Default s +_s rc +Ġoutput s +/ B +Ġent hus +-b l +.Fore Color +ĉ temp +F ace +Ġinter act +Ġwe ird +M ount +re ll +ud ents +Ġrequire ment +ĠS us +I ER +Ġe lected +re ference +ĠM E +Ġserv ers +.w ait +Ġsnap shot +il ton +Ġtri es +Ġt ipo +.T ime +> w +Ġmount ain +Ġp ounds +Ġ[ ... +ex ists +Ġng On +_M AP +Ġf lying +xi ety +ĉ value +_D B +un o +Ġse ats +T URN +. author +! ) +or ce +Ġindic ated +.s in +Ġass ignment +im iento +ĠF rame +_g en +in ery +_ ) +m essages +.set tings +ĠMe an +ĠM useum +ir q +att ach +ĠPalest in +_ QU +_t ags +Ġcas ual +em en +ASS WORD +$ s +ĠC irc +оР¹ +et ric +/ P +Ġep och +< head +_C MD +Ġg it +Ġpen alty +or ph +_ users +ours es +.Date Time +atern ion +_pro ject +Ġsuper ior +ĠD am +ĠSe attle +X Y +> The +ĠA k +Ġgr ass +/* čĊ +(d is +Ġgun s +Ġt b +ĠK evin +. args +ĠA h +op ed +( J +column s +arg uments +ĠWith Events +_f ull +ĠDef ense +S imple +Ġdeath s +Ġext ensive +ĠSt ill +ĠEx pression +ĠAg ency +Ġperform ing +F X +Ġus uario +U AL +S ide +od os +apt op +Ġcred entials +_c ap +at ient +ĠDis ney +Ġa i +Ġch ip +Ġvol t +.make Text +%%%%%%%% %%%%%%%% +Ġbelie f +_LO C +ĠC ivil +N avigation +Ġreve al +Ġviol ent +ĠF il +Ġc atalog +em ed +sc an +. control +Ġconstit ution +C ountry +Separ ator +_A PP +top ic +uet ooth +M IN +Ġdes criptor +y t +ET HER +Ġdistrib ute +' }Ċ +.tr im +.L ine +Ġl bl +assert Equals +ĠD et +omb ok +( width +Ġt ort +ĠEXP RESS +ac o +Us ing +ĠBr and +w all +EM ENT +ĠComm unic +< uint +ĠG UI +EG IN +ĠR ange +/ i +ĠT aylor +c ost +Ġrespond ed +ĠTh eme +n ce +IS H +Ġfeat uring +Return s +ĠK r +Ġ .Ċ +Ġn am +_c b +Test ing +Ġ{ }, +y al +.f ield +Ġ/ = +_SH ORT +m ates +Test Case +ain less +Ġeval uation +_ ITEM +ĠPac ific +ĉ k +Ġc ant +ĠR os +) s +Ġf et +STR ING +ĠDis pose +g al +ĠJ oin +ĠP orn +ĠCath olic +AR GET +cp u +ç łģ +.sc roll +IS ING +ifest yle +anc ement +Ġm erc +ĠB rowser +eter min +Ġover flow +Av ailable +Ġbott le +: UI +ific ial +Ġco ord +clar ation +Ġcon j +G LOBAL +ok u +Ġk wargs +cond itions +ul um +Ġg enu +ĠH ero +å İ +Ġun expected +ĠDAM AGES +Ġk a +ĠC ould +UP PORT +ĠPh otos +Ġconf ident +Ġdet ected +de g +rg b +Ġstrong ly +Ġ} ;čĊ +Ġ) : +Ġle ct +urs ive +RO L +ĠWe ight +Ġent ertainment +Ġ) );Ċ +Ġg onna +Ġb b +.d o +G S +Ġmist ake +D L +ĠPROVID ED +ear ning +L imit +iss ions +[ v +ä¸ į +ir ty +D el +Ġunder lying +pre ne +Ġj aw +ĠD I +pe er +Ġobject ive +Ġde posit +Ġk on +Ġes p +.set Visibility +/ login +< typename +Ġfr anch +/ e +Par allel +Ġsc ored +ĠH on +ĠV ill +ig a +Ġant icip +_ assert +ĠO pt +Ġdescri bes +w an +m ount +Ġmonitor ing +Ġt out +ëĬ Ķ +}, { +................ ................ += int +Ġc ust +---- -- +Ġatmos phere +P AR +ort e +IS IBLE +ĠI ron +ĠNot ification +.log ging +ĠBO OL +-p oint +Ġaf raid +ent a +Ġtom orrow +@ implementation +Ġeng age +ĠAn th +ĠF loor +ĠU l +To ols +Ġb ab +Ġcare ful +ãģ Ħ +Ġcruc ial +Ġcalcul ated +ĠS A +Ġw y +D X +_T AG +ind ed +Ġj et +ĠEngine ering +.M AX +en z +v d +Ġpublic ation +Ġ## # +Ġfac ed +ra ham +ĠC apt +As set +ĠCon stants +Ġlo ans +_ IP +ĠF ish +Red uc +_m at +Date Format +_m e +[] [] +Ġintegr ity +ĠC ourse +lob als +Ġfac ilit +Ġem br +ĠN g +.S ystem +Ġmanufact urers +Ġpro ven +.on Create +Ġal arm +Ġ § +Ġcomm only +ic os +æĸ ° +ĠSt ation +} ). +ĠF ilm +w i +ç ī +Ġeng aged +St ats +Ġgovern ments +Ġafford able +_p roperty +Ġag es +(' -- +Ġf ör +ĠProf essor +Ġhy dro +P ush +Ġorgan ized +Ac cept +é m +_c ell +Ġn b +p b +Art icle +Ġrem oval +Ġauth entication +ĠF R +l ide +Ġple asure +ap ol +Ġpart ition +ĠS ide +Ġcr imes +Ġdem o +hold ers +ĠPak istan +In struction +Ġexpect ations +.sc ene +Ġ' ) +h es +ino is +_P ro +Ġm olec +and al +_sh ort +Ġdefault s +Ġn ations +in en +Ġr t +O CK +P acket +S B +ĠSH ALL +_cont ents +ise conds +vert y +á t +G uid +n om +Ġcon clusion +. Update +Ġlo vely +Ġem it +b ec +ĉĉĉĉ Ġ +Ġintel lect +Ġb rew +ec ycle +F ire +Ġad mit +Ġar bit +Ġarr ang +ĠM IN +M ail +ĠN ative +C ur +Ġcon vent +.R untime +" }Ċ +.R un +Ġprint ed +Ġconven ient +. ar +m ock +ĠAdmin istration +ãģ ¾ +Ġelect ron +fl ate +Ġl ombok +Ġjava fx +n h +Ġsup plies +Ġvisit ing +ah l +Ġpow der +Ġult imate +Ġorient ation +ut as +_s cale +Con firm +ph ones +ĠOper ation +/ T +_IN TER +Ġair port +Ġmet rics +Ġphen omen +a udio +Ġm ai +( K +h u +all ing +rodu ction +ĠTrans port +ĠNOT E +æĸ ĩ +Ġfew er +_T IM +ì § +к и +A ge +F IN +Ġì Ŀ +ĠAt tribute +group s +er k +at to +. define +.AspNet Core +ategor ia +ĠS ir +( form +< User +. round +_d ay +.A ll +Servlet Response +.N o +l arge +IG H +qu ent +Ġvir us +Ġret ro +Ġim per +Bit map +Ġv ice +Ġoff ense +ist e +ĠA UTH +Ġê ° +ToolStrip MenuItem +G u +Ġr ape +ĠDav is +Ġover whel +: flutter +- table +ĠCon structor +Pr ivate +e ven +ch r +Ġap plies +_at tribute +Ġcon tribute +E VER +L ines +ĠAf ghan +Vis itor +ĠS L +se ason +C U +Ġintrodu ction +Ġmat plotlib +Å ij +Ġnewsp aper +âĢĶ and +< tag +Ġin i +Ġd iverse +Ignore Case +ĠU r +Ag ent +Ġb ull +.em it +( Exception +ar Layout +Ġincred ibly +ĠTr ust +={ ( +- nav +Ġe quals +Ġl ady +ĠP od +d isc +al am +ĠI V +â Ļ +iv idual +ph i +add ed +Ġdifficult y +Ġcomp act +ĠAction Result +c ers +_class es +Non Null +Ġqu it +Ġp ou +S witch +ir s +- test +ĠK ind +ĠCal endar +Ġstream ing +} ', +S W +Ġst ead +oc a +Ġprov ince +Ġcol span +Ġperson nel +ĠE mployee +Ġprodu cer +Ġevery where +od b +Ð Ł +bs olute +act ivate +Ġgr inding +ĠBuild ing +ĠSand ers +(s c +ĠOff set +//////// //// +} ;čĊčĊ +({ " +Ġscan f +ĠY Y +ĉdef er +Ġj ew +Ġrestrict ions +.m p +[ l +ä¸ ĭ +label s +red icate +aw esome +Ġw aves +Ġcon front +Ġmeas ured +Ġdat as +_ex it +ot ton +Ġshould er +ask a ++ # +ĠĠĠĠĠĠĠĠĊ ĠĠĠĠĠĠĠĠĊ +Ġtro ops +ĠU nd +_c ard +w ich +Ġn ous +Ġ"/ " +s b +Ġcommunic ations +Ex port +Ġdec ode +th s +inter pret +By Name +ĠSp irit +ed ges +O LE +ĠE M +t it +ĠTh rough +Ġb io +ĠP ackage +or ne +Ġ} . +` ;Ċ +Ġok ay +ĠZe aland +ident ity +(n ext +ĠB ang +Lib rary +Ġheav ily +il on +Ġdi pl +Ġrot ate +put s +) ',Ċ +ĠData Table +Ġmay or +.to LowerCase +Ġsome how +ĠNor thern +al c +Ġcap abilities +Ġv ibr ++ Ċ +ĠS u +ĠRes et +_m ean +Ġc ig +.cl oud +ĠB and +ĠF actory +ĠAr izona +_ io +op her +Ġconsc ious +Ġà ¶ +\ Controllers +_s peed +ĠF ac +_C om +ĠB ible +w en +ED IT +Ġun n +ĠSt aff +ĠIn n +Ġmechan ism +ĠM embers +Ġmigration Builder +'] .' +.get Int +< void +ĉf ree +oid s +\ Support +Ġautom atic +Ġch ances +Ð ¶ +Ġcomp licated +[ row +ah oo +Ġ}ĊĊ ĊĊ +Model s +W in +Ġt ape +ir us +iz on +on omy +(" _ +: . +.st ereotype +( env +_re ct +(w ith +Ġassert That +Ġcon straints +put y +E mployee +T D +Ġgu itar +ĠJew s +.pro cess +Ġf iction +ĠSh ared +âĶĢ âĶĢ +Ġprop ag +.N et +Ġachie ved +ĉ Q +Ġn urs +Sh ared +_FAIL URE +Ġbeh aviour +Ġcol s +ism o +Ġfem in +Ġchalleng ing +Ġpost ing +enc il +Ġcapt ured +ĠD ou +( word +ĠTur key +pan ies +Ġre putation +ORM AL +Ġelig ible +prot ocol +id as +(f rom +Ġfin ance +- per +Ġg otten +H A +d uration +ĠP arent +Ġin vent +Ġre start +ол ÑĮ +r ition +(r s +< bool +i ert +Ġmod ification +ĠT X +readcr umb +b ank +$ / +ĠMill er +] ),Ċ +.Check ed +Ġsac r +se curity +Ġp ose +ĠBr ad +Ġfit ness +Ġannounc ement +ation Token +Ġserv es +ne ed +Ġge ometry +AR S +æ Ģ +andid ate +Ġs prite +_s plit +We ek +ad ies +> (Ċ +?> " +Ġ/// Ċ +Ġein er +Ġweek ly +ĉlog ger +_p op +_m an +Ġmigr ations +Ġask s +Ġb s +Ġfall s +.W here +- height +_fe ature +.M in +Ġhy per +Ġvol atile +Ġtw enty +Typ ography +Un able +D et +, f +-m od +Ġsett lement +Ġcontract s +n ome +B ad +ĠB rian +(user name +!! !! +Ġh ack +.F ield +H R +ĠJ ordan +iz a +Ġ ł +ĠSh er +. header +( other +ĠD ub +( op +ĠR ound +Ġv ie +Ġap pl +ĉ J +ĠIn sert +ĠL P +reg on +ĠM PI +Ġan chor +ac a +ø r +Ġa de +anch or +que e +ĠTree Node +Ġtarget ed +Ġla id +AB EL +v et +ĠOr igin +A nt +. ');Ċ +ex pect +ed Reader +ĠM ajor +Ġin ch +Com par +Ġpre view +Ġill ness +ĠCONTR ACT +ĠInd epend +u uid +Ġn ome +Ġt c +ĠA venue +is an +Ġph rase +_m ove +") [ +Ġprov ision +Ġconcent r +_ IR +ĠU t +() + +Ġn as +! , +ĠRob in +i ations +at itude +Ġp x +ĠWith out +/b ash +ek t +re ement +Ob server +ĠReg ion +UBL IC +Ġ{ // +K N +å · +Game Object +å ¾ +enc oding +Ġ** * +project s +Ġt k +Ġche ese +EM PL +ar o +Ġا ÙĦ +Ġcons ists +ref resh +ure au +ĠSc anner +Ġso il +Ġfl avor +Data Source +Ex ecute +ени е +Ġsh it +åĪ Ĩ +< any +Ġretrie ve +Ġbelong s +.st rip +abs olute +Ġexp anded +bo y +): - +Ġresc ue +.J Label +Ġre ly +Ġal ignment +-f amily +Ġre nd +OLUM N +Ġb orrow +Ġqu otes +ĠL ew +Ġsh ower +ĠDE LETE +_lo op +! "ĊĊ +ĉ re +Ġattempt ed +aver age +ĠP aint +quis ition +ol en +Ġliter ature +ĠRe ference +_TEXT URE +ĠS eg +ĠInd ust +ct ype +D UCT +_H OST +ĠTr ade +Ġpl ugins +Ġbre ast +ul se +Ġcreat ure +ãģ Ļ +ĠW i +Ġsup plied +c oll +! (" +Ġfuck ing +ĠCh rome +ĠU ri +ĠN ation +Ġvert ices +T HE +ĠOr iginal +on de +Ġsh arp +Ġcook ing +Ġ{ /* +ĠPs ych +ĠH ollywood +=$ _ +.D ock +Ġg er +Ġb one +_con n +_se c +ys ics +Ġ= " +S al +s f +Ġdeep ly +ang les +T erm +b ell +ĠQu ick +ener ation +adio Button +åħ ¥ +}čĊčĊ čĊ +Ġcapt ion +l c +ĠE L +, [ +ĠĠĠĠĠĠ čĊ +ret t +(m ethod +ĠFl ash +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +W ISE +.s cale +Ġrough ly +_ child +m emory +ay ing +Ġinitial ized +in ator +а ÑĢ +Ġsc alar +ĠH o +ai res +(c olumn +.de stroy +P ACK +Ġh em +ang el +_S UB +. qu +Ġ × +DE FAULT +pos itories +ĠL ength +ĠF ast +Ġsign als +Ġ// $ +ri ers +Ġd ummy +AN Y +Ġperson ality +Ġa gricult +Pl atform +ER O +ĠT ra +Ġen orm +ĉ W +Action Result +Ġa ver +[ str +Ġ' -- +.S printf +Ġdeb ut +Ġ Ñĩ +h ex +_ utils +Ġp b +U ITableView +Ġz ur +. encode +Ġv ag +.error s +о н +Ġm r +ĠA ward +Ġc pu +Ġpress ed +' est +ĠF estival +' T +Ġa k +res olve +.m e +Ġn ic +Ġgen re +Ġat trib +ĠMo on +Ġarr ive +ĠD ating +Ġt m +.Config uration +. red +Ġgl m +Ġst ations +sw itch +Ġt ied +äº º +Ġ/ >Ċ +Ġsubsequ ent +pos able +-fl uid +Ġth orough +Ġpublic ly +apt ers +ĠWil son +_P RE +y ard +ä ¼ +ĉ in +Ġre vers +Ġbul let +cri bed +nes ota +Ġ($ _ +ann on +c ursor +Ġclo thing +ĠM ulti +: ', +Ġv ess +ordin ator +Ġein em +C annot +Ġar med +ĉ V +ä¸ Ĭ +.F lat +ĠS ep +ĠSub ject +_f ont +Ġcharacter istics +D one +el n +######## #### +PO S +Ġd ensity +ĠPl atform +- items +Ġo vers +Ġpush ing +ç ¤ +.Con nection +_ term +Ġinitial ization +________________ ________________ +ç ¬ +.d ocument +les h +ĉd ocument +ĠP in +ç a +Ġdefinition s +.P ath +_W RITE +Ġ ĉĊ +? >ĊĊ +Ġter rible +be an +ick ets +ĠS V +B uy +(t ask +Ġreg ime +g oogle +Ġcr ack +.vis it +N UM +ener gy +Ġstr uck +_s ample +.p ayload +Ġre vis +ĠSc ene +Ġp g +Ġbreak fast +URRE NT +.char At +_ex ception +ĠAnt on +Ġguid elines +Ġex haust +ĠFin ancial +Ġind ent +Ġdes ktop +H idden +F ailure +Ġpr inciple +Ġ iv +Ġse ks +n etwork +Ġnumber Of +ĠAl bert +ĉ long +, . +Ġz eros +f ade +ĠT yp +ĠT erm +ĠAr ts +.App lication +Ġbeh alf +æĪ · +Ġm ere +(` ${ +Ġaware ness +elp ers +f lix +Ġwe igh +Ġestim ates +. child +/ O +ĠBit map +.b ottom +Ġ************************************************************************ ** +Ex pect +ent o +ĠFor um +ver al +Ġj ail +Ġab ilities +ĠH OLD +ĠC it +Ġd ynam +Ġgr ay +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉ +.next Int +ant ly +ĠAR ISING +( private +Ġreject ed +ĠN ic +Ġle ather += {Ċ +aly tics +th etic +.T op +.P age +={ ` +Ġ ;čĊ +de pth +m ann +W D +ĠS om +.R ight +Ġ) }Ċ +Ġtr ait +Ã Ĺ +i ac +Ġr v +S ample +.X ml +opp ed +ĠÑ Ħ +list s +Ġt ear +ivers ary +.c ollection +ĠCon stitution +ĠHttp Response +Ġbr ill +ĠP rom +h over +ĠM iami +Ġarg ue +_f loat +Ġ ãĤ +Ġn at +ĠT al +Ġinteg ration +(c ur +Ġrem oving +Ġco eff +ĠTh ough +Ġfore cast +ĠV egas +S ite +Ġtr ab +ĠHen ry +- i +Ġinvol ves +B T +Ġs lo +In voke +Ġl ucky +r at +Ġ? Ċ +Ġhand led +(f d +cont ents +ĠO FF +R F +Ġst y +ĠM otor +ter y +t ax +M AP +ĠMr s +Ġph ones +ĠUI View +")) );Ċ +( dev +ĠIr ish +Ġw s +D I +_OFF SET +ĠEvent s +Ġst ages +Ġ} // +Ġhab en +ST ANCE +ĠS in +ĠM oney +(t op +Ġappoint ment +VER SION +met adata +_com ment +Ġcolle agues +map s +â ĺ +Ċ ĉĊ +( al +_re q +Ġf ut +Ġarchitect ure +ĠWH ETHER +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +_s creen +Ġstyle Urls +Ġmon ster +. up +ph ia +Ġprocess or +ĠT err += ', +ĠMan ufact +ĠN T +k el +ib ern +ĉf ile +A li +rient ation +Ġ// ! +ap ore +ane ous +ĠC reat +f older +Ġh ay +Sup press +( left +Ġe uro +Ġdis claimer +ustr y +sh ips +_f d +ĠF a +_in sert +Ġro l +if ting +ĠCom ments +_b r +Ġloss es +ĠAdd ed +ch arg +Ġп о +_s ystem +ĠS ometimes +ĠSp ain +(g roup +ial is +Ġdoll ar +ĠAr gs +qu ires +ĠT en +.s css +Ġsurv ive +us age +Ġj un +im iter +ï¼ģ ĊĊ +Ġfif th +t oggle +Ġdecl ine +($ " +(L ong +ing e +Ġpil ot +-l ight +-r adius +Ġpod cast +Ġnatur ally +P ages +ä¸ º +ĠDes pite +Ġlight ing +Ġcr ate +ĠB inary +Ġredu cing +Ġe leg +ĠM ouse +ĠTest Bed +Ġbefore Each +_ ARRAY +Red irect +Ġf lood +Ġsh ips +Ġelectric ity +)* ( +ê ¸ +ĠV iet +her o +Ġd ia +ĠK ent +he art +Ġthreat s +_ acc +Ġs ymbols +is chen +_in st +C riterion +ĠT IM +. Height +Ġ âĢĻ +();ĊĊ Ċ +Product s +_S P +ĠC y +Ġdepend ent +est e +Ġdat os +d it +аР² +IGN AL +Ġless on +"> ' +ĠC over +ĠH ope +ĠT imer +Ġd ad +vid ers +ĠPh ot +/ ? +rop y +om ing +as ion +Ġ\ ( +ĠE T +ĠRe ading +Ġep isodes +l m +ech a +Ġne uro +Ġhar mon +Ġlib eral +- ind +D ATA +Ġevery day +Ġdiv ided +ĠActive Record +fig ure +U A +ä ¹ +riend ly +te ch +.game Object +иÑĤ ÑĮ +Ġmo on +ft ime +Ġno ch +ĠT ORT +ĠV M +.in itial +( child +Ġmus ical +Ġo c +b as +ĠH ay +_l ong +Ġmem set +ile y +adel phia +S V +ro at +_t x +Ġl on +ĠngOn Init +b p +ĠGold en +AC HE +Ġwor ried +az i +E ar +T ake +(f p +bur gh +_ Data +g res +ĠO nt +p us +Ġtrans parent +Ġp ocket +Ġr am +igr ations +. čĊčĊ +Ġ[ ( +Ġadopt ed +Ġreported ly +ĠD ream +Ġ} ));Ċ +los ing +Ġte eth +ĠBook s +", & +enn y +LE MENT +Ġg el +ĠPl ant +! âĢĿ +.h ost +ĠRep ly +re ngth +Ġrecogn ition +Ġ}} >Ċ +L A +Ġmir ror +Ġassist ant +( device +Ġspirit ual +b uilder + § +Ġou tr +Ġt t +ĠP ER +Ġrad ical +Method s +Ġp ace +ud y +Ġg ut +ĠG reek +Ġnon atomic +ĠP aper +_G PIO +Ġob st +.A d +viron ments +ĠS ov +( con +ĠTrans action +. assign +ĉc atch +el ter +Ġbit coin +_G R +ĠčĊ +met ic +Ġtrans formation +åı · +Ġr gb +istrib utions +Ġimp licit +/ in +dest ination +аÑĤ ÑĮ +Z ero +Ġun set +. where +.g o +Ġform ation +Ġdeclar ation +() čĊčĊ +ĠEx pl +ĉĉĉ ĠĠ +/ pro +.J SON +Ġdes k +.sub str +//---------------------------------------------------------------- ------------ +ly n +p son +dis able +ĠF unc +ĉ Assert +ĠM ARK +Ġdefe at +Ġbl ind +Ġconst ants +. headers +UIL D +Ġexp enses +P ixel +Ġh r +Ġf el +ĠEast ern +_d el +ĠC ub +Ġs q +ĉc ount +ĠD irectory +Ġex clus +Ġhistor ic +Ġ ------------------------------------------------ +Ġcom position +Ġdata GridView +ĠB urn +ĠB C +M aster +Ġsp awn +Ġbe aring +.Set Active +il o +Ġg allery +Ġfound ed +Ġav ailability +.s qrt +Ġp es +ĠD OM +m ate +O ct +Ġmatch ed +it ivity +Ġan xiety +.pr ice +ĠIn stant +ì Ĭ +Ġt ut +IC ollection +.sh ared +_s ql +t bl +lib rary +_de stroy +erm al +ĠNot es +ĠE in +Ġsou thern +ĠOTHER WISE +Ġmac ro +.l ower +cl s +Content View +.l ink +const ant +ĠB es +Ġsome body +n b +"> { +( local +.. ... +ĠN ull +m x +Ġà § +Ġp ause +-------- --- +_M O +ĠC M +Ġfor Key +ĠD VD +Ġclose st +_DE VICE +ĠSte phen +ĠB BC +ĠTr avel +P aint +ĠResult s +ĠR ule +Ġt p +Ġrat ings +c in +c sv +> / +ĠG OP +l ad +Ġ ÑĢ +Ġindex Path +m atrix += f +ars ed +Ġ} ); +ĠC os +ĠS core +Ġt ak +ĠE SP +ĠIN C +_N ULL +-f lex +"] [ +int o +el and +Author ization +_F ALSE +Ġg ate +Ġv id +ist ent +T IME +Ġre write +Ġt ie +Ġarch ive +.event s +.get Parameter +ĠPer mission +Ġprogram me +Ġ é +j ud +Ġcam eras +(s ys +ĠSy rian +Ġimpro vements +Ġh ip +Ġsu icide +Ġsch olar +Ġcompat ible +rem ote +.d own +F UNCTION +Ġman aging +ĠUI Kit +. raw +>> >> +Ġdem ands +ell ite +Ġd ent +ĠM icro +åı ĸ +'] [$ +ĠI E +im ension +Ġt rem +Ġg ained +.w ith +. ok +h ou +Ġb om +amp aign +Ġjoin ing +f ish +Ġadd Subview +Ġnor thern +.c or +ore t +D ie +in ish +_com p +Ġatt ended +Ġcoll apse +ĠS S +ac ent +_E QUAL +ĠDe ep +R GB +ĉ test +ol ves +us et +Un ityEngine +w riter +Res olver +, % +if ference +_re move +ond a +Ġfem me +de code +Br anch +Ġfl ush +Ġinnov ative +Test s +Ġ[' ./ +Ġcover ing +. admin +ultip art +(l ambda + namespace +ĠS port +Ġ! ( +ac les +Ġde pression +ĠK ong +Ġp ert +ĠCon n +ĠOther wise +/ home +s upported +Ġp ink +Ġinv ited +ñ os +_en abled +Ġ- Ċ +F W +en ers +ĠM Y +Ġsuggest ions +Can vas +Ġf er +ĠMarket ing +@ Test +unt u +ĠV en +ĠC ou +iv als +Don ald +lim ited +ĉĉĉĉĉĉ Ċ +Ġanal yst +( entry +Ġrepresent ative +_at tributes +Ġf ur +.h ide +res p +ado res +rid es +ĠJ osh +ro bot +ĠN AT +Ġs esso +Ġintegr ated +: true +part s +Ġst upid +: event +@end section +Ġp u +.T able +ĠY ii +` ;ĊĊ +Ġcl ang +=" "> +eng an +_param eters +.int ernal +ĠMod ern +Ġmet ric +Ġsem i +={ {Ċ +.am azon +ĠB B +aint y +view port +Ġstart Activity +dis patch +**** * +Ġfl av +iffer ent +[ this +Ġst ake +Ġarg ued +vious ly +.w ork +ĠO ak +O ld +( async +not es +Ġfl ip +Ġdis ag +ĠT E +ĉ error +< ' +Ġ» ĊĊ +Ġfilter ed +ĠM ach +Ġh ung +_d ump +_s amples +-dis miss +Ġr ay +Im plemented +D K +Ġj ed +Ġbreak s +Ġf its +. gr +ĠZ ero +or o +Ġequ ally +Ġ' [ +Ġconcern ing +< meta +play ers +_P OS +_s im +J an +Ġyour s +ĉ N +Ġsp ir +Ġch ampion +ĠAn alysis +ap a +ĠNS Log +_l ines +ñ a +ĉĉ ĠĠĠĠĠĠĠ +.S c +Re p +etro it +ur able +M IT +com pat +own ed +_ind ices +], čĊ +Ġdis covery +ĠDie go +ob i +. Index +Ġtrend s +PL AY +.n o +Ġl ens +_c fg +Ġan no +ag an +Ġperiod s +ter ms +y z +Ġattack ed +ib ration +PEC IAL +_ grad +Ġaccord ance +.Read Line +.de vice +ri x +. container +m ay +erc ise +ĠL u +Ġr g +ĠÑģ ÑĤ +ĉĉĊ ĉĉĊ +( un +TERN AL +Ġless ons +Ġalleg ations +Ġtrans mission +.Re f +M obile +ĠT ournament +ĠN ut +ĠG a +ĠCap ital +def inition +- exp +c lean +Ġfant asy +Ġenh ance +ent ence +'] :Ċ +ack ets +Ġcelebr ate +@ ", +Serialize Field +Ġarray s +t b +ĉ st +[ assembly +( reg +.c ategory +Ġimpro ving +Ġsal ope +Byte Array +Or iginal +Ġ[ {Ċ +åĽ ŀ +ĠCl in +oen ix +ĠS amsung +Ġmaint ained +Ġag enda +f ail +Ġpres ents +Ġtim ing +.m ark +' >< +Ġprom ot +Ġin cl +_ only +ë¥ ¼ +ĠAtt orney +- date +Ġlands cape +Ġf u +S Y +.p rop +ĠA rr +p ag +Parallel Group +': čĊ +Ġlog s +a unch +unc i +n ama +Table Cell +iss ues +. { +ec urity +_ex ec +old s +Ġhost s +Ġpro to +_ import +_s ort +ĠB ow +ĠN ormal +ĠF arm +.create ParallelGroup +R otation +. err +Ġp leased +it age +.W h +ĉĉ ĠĠĠĠ +M R +ĠM ORE +ĠN atural +_ transform +B ASE +ener al +ut down +.common s +W T +Ġa an +. Result +d og +Ġclick ing +), ĊĊ +# line +Oper ator +Ġc iv +Ġm erg +ob uf +ng then +Ġ[ { +Ġcan cell +tr igger +. : +W ORK +decl are +Ġdecre ase +ÅĽ ci +lo om +.N one +ĠM I +ĠJ ason +Ġhealth care +iam ond +s ylvania +* x +ĠR a +[ b +Ġprint ing +ph abet +ĠLab our +op per +Ġz ijn +-t arget +_F UNCTION +Ġo ct +ени Ñı +åľ ¨ +Ġwest ern +Ġcomput ers +ĠR ET +Hash Map +[ String +get Value +_D ATE +.N ext +ĠF if +é l +ick ed +æ İ +-M M +Ġ{ ĊĊĊ +Ġcontact s +Ġdig its +Pro du +Ġunus ual +Ġrapid ly +t ures +Ġang ry +c ancel +xx xx +_p arser +id ity +_P REFIX +Ġme hr +Ġrare ly +et he +op es +Ġ% . +work s +Ġthe ta +Ġcontrib ution +ĠT ony +Ġsqu ad +аР¹ +Ġî n +th ere +out ed +ĉ q +Ļ Ĥ +g ood +L I +é¡ µ +ĠL iving +iz abeth +Ġk t +ĠD allas +] ],Ċ +Ġ/ >ĊĊ +Ġrais ing +/r outer +_g ame +ĠC UR +z ens +. es +Ġfont Weight +(f unc +not ification +Ġ'../../ ../ +Ġbl ame +ãĢĤ ĊĊĊĊ +an co +Id entity +f ollow +Ġart s +x s +Ġofficial ly +ĠSt udio +Ġrecommend ations +Ġloc ale +Ġam ateur +ĠEn able +Ġcap s +. End +- add +_g shared +ĠC T +For ce +Ċ ĠĠĠĠĠĠĠĠĠĠĠĠĊ +Ġor ange +Ġl p +Ġanswer ed +.G rid +Ġd ual +Ġstrateg ic +Ġnob ody +Ġf atal +_ est +( el +Ġì ł +ĠB udd +A IT +_f actor +- one +ĠH AVE +" čĊčĊ +Pro f +Ġä r +str ings +Ġdir ty +ĠF ace +ĠB egin +ĠB us +Ġw is +åŃ Ĺ +Ġspe aker +Ġcar rier +ĠO m +Ġhad n +All ow +:: __ +Ġver b +ĠCom plete +ĠE asy +Ġb ills +ĠĠ ĊĊ +Vert ical +Ġpr on +ĠDef ine +Ġlook up +variable s +Ġpand as +um es +Ġinn oc +Ġset Up +ĠCh ampionship +art ist +ĠC Type +F oundation +๠Ī +ĠSet up +Ġrec ipes +ĠU IColor +ĠF ight +Ġauthor ized +_c lick +_s uccess +ang an +ĠMount ain +ĠDo ctor +Ġeg g +ĠMedic ine +c les +` .Ċ +[ int +d ashboard +ĠApp ro +-d r +Ġprodu ces +Ġrent al +Ġre load +Ġarr ival +sp ot +Ġund ert +Ġequ ipped +Ġpro ved +Ġcent ers +Ġdef ines +al so +Ġop acity +ĠUn fortunately +ĠIll inois +Ġн е +ĠTem ple +ĠTr ail +ĠK elly +Ġmeasure ment +Ġsepar ated +-c ircle +H ey +ĠRE AD +ig its +Ġ ib +ĠM OD +atter y +аР· +Ġv end +ен ÑĤ +ĠHttp Client +s afe +_A SS +ic it +ĠCon struct +ĠC lo +ĠS ix +_T OKEN +(b lock +Ġwarn ed +/* ! +! Ċ +Ġinnov ation +_ " +Ġ );čĊčĊ +Ġsp ots +Ġcho osing +.c s +Ġflex ible +U Int +Ġscr atch +- al +Ġf estival +Ġout standing +================================ ================ +M ean +ĠO regon +s ymbol +. account +d ney +'' ' +! ", +Ġpart icle +à ĥ +[ MAX +IV ER +ER ENCE +NS Mutable +ĠColum bia +_ ĊĊ +.f r +Ġc ogn +V R +ĠMethod s +ĠM ade +ĠB R +ĠEl se +Ġeg gs +Ġsw ing +ĠIn v +Ġdise ases +Ġf irms +Ġle mma +}` );Ċ +l ings +Ġg ym +umin um +.T rim +M em +Ġcritic ism +ibern ate +_T X +ion i +Ġguid ance +Ġrepeated ly +Ġsup plier +Ġpaint ing +.F ragment +ed Exception +Ġw iring +Ġcour ts +W EB +æľ ī +\ . +ill ance +Ġb rows +ĠP attern +PL ICATION +ĠSum mer +Ch ain +Ġc ute +mer cial +Ġd il +ĠFrank lin +ĉg lobal +IN CLUDING +h istory +Ġl st +Q t +SD L +al ia +i ere +( ... +ĉc in +iff s +vel ope +ĠR oot +cl uster +User Name +ign e +< S +Ġf est +Ġindic ating +ke eper +Ġc ada +é g +cons in +ĠG B +Ġl b +em ony +-icon s +_d oc +Act or +e lem +.De lete +Ġin fection +ĠPriv acy +Ġgreat ly +ĠP os +ĠT reat +Fl ow +Ġattract ive +ĠMar c +s udo +tes y +- an +ab ama +ĠW ould +Ġsu ck +index Path +ĠE t +T imes +Ġclub s +_ass oc +Ġac quired +(" : +Ġint ense +.m aps +Ex pected +T oggle +Ġa y +Ġl ifestyle +-c alled +ĠS now +V olume +Ġcann abis +ĠD irection +ĠLim ited +-s pecific +Ġd owntown +/ icons +Ġre ven +L eg += null +Key board +') ). +Ġ"" ;čĊ +Ġatt itude +.n avigate +- error +AM PLE +ĠJ ay +v r +c ow +.com pile +Ġmem ories +_m ark +ĠMin nesota +Ġk osten +Ġprob ability +w arning +Ġgen etic +F ixture +ĠHash Set +N ombre +_m onth +Æ ° +- start +xy gen +ĉ ft +i agnostics +ĠMat thew +Ġconcept s +Ġcon str +. State +и н +N ov +Î ± +ĠP anel +ä¸ ª +com pare +> ()Ċ +Ġapply ing +Ġprom ised +Ġo x +nc ia +ĠValid ation +ort s +_c ur +e lect +ey e +( Data +Ġreport er +ĠB uff +Ġs r +Ġ" ; +ick y +Ġtemp or +S N +Ġres ident +pi res +ys ical +Ġend orse +ĠS ong +is Empty +le et +_ util +Ġdist ingu +ĠT alk +ĠM ot +( default +.A rg +gorith ms +_ words +im mer +_res et +f amily +W W +Ġsav ings +ĠâĢ Ŀ +_en able +side bar +Run ning +Ġal i +Ġtest im +Ġwarn ings +ĠCh em +ĠEx it +Ġfound er +pect or +Ġr m +_d ataset +ĠD as +Ġh an +Get ty +á l +Ġn y +Ġpo verty +Ġresult ed +.b y +ĠVis it +Ġobt aining +/ '.$ +ĠĠĠĠĠĠĠĠĠĠĠ Ċ +sh all +_LE FT +UI Image +_ Name +h ave +ĠN ob +l r +- footer +Ġn aked +ĠG arden +\F acades +Ġgrad uate +Ġfranch ise +pl ane +Ġcontrib utions +Ġstring With +Ġc rypto +Ġmov ements +ath ers +Ġlif etime +Ġcommunic ate +j ar +ĠFr agment +_ IF +ĠN avy +ĠF igure +Ġsim ulation +_st op +Ġreport ers +Ġvers us +aj a +ĠÎ ± +Ġgovern or +List Item +Ġse aled +.Back ground +ed i +ash ing +Ġl ip +ĠI h +mer ge +Ġn ec +el ocity +ATE G +Ġse eds +Ġflo ating +_F A +w alk +ĉ user +_de pth +Ġw age +@ app +N il +( [" +( vector +Ġsecret ary +Ġj Panel +ve z +³³ ³³ +d irection +ĠE P +Ġh unt +Json Property +ĠP ORT +] ", +аР¿ +ĠFore ign +pan ic +Ġtri als +ĠA le +Ġr ural +- value +author ized +ĠScot land +.d rop +ĠM T +ç ± +row th +File Path +Ġrec all +if le +Ġc el +ĠSE LECT +k n +_c ase +Ġc rop +s ure +p ot +IC S +Ġst em +Ġindust ries +P ut +Ġa ber +road cast +Icon s +) ")Ċ +æĪIJ åĬŁ +g ui +Ġassum ed +Ġr x +E A +è § +EL L +Ġdo se +Ġin e +Ġde eper +l ider +Ġord inary +Ġg olf +_IM AGE +ĠN AME +(m odule +Ġat om +Ġbel t +Ġoff ices +b eta +Ġphilosoph y +( JSON +-f ield +Ġintrodu ce +Ġconven ience +opt im +> "Ċ +ath y +Ġemploy er +qu ate +Ġed ited +Arg uments +ĠN ations +__ ) +Ġno se +ĠS ample +' )ĊĊĊ +Ġc ake +.get Attribute +H D +Mod ified +Ġpredict ed +Å Ħ +an ie +S orry +(d oc +w ind +ie ve +Ġprov isions +AT ER +OT E +M Y +.A utowired +ĠB ath +. Boolean +Ġback end +.M ouse +ater al +p aper +Con st +ĠV R +_ entity +_C TRL +ĠProte ction +ĠG M +ĠStud y +Ġsou p +ot ime +' use +] " +/ users +a ug +ĠH ong +_n orm +ãģ ¨ +Ġse cre +(B uild +ĠCon tract +ol as +Ġsa uce +Ġaggress ive +Ġrac ial +char acter +@ @ +Ġcomp ile +ĠV oid +_re m +_m emory +k k +Ġm ic +S ame +U tility +ĠH tml +ĠX ml +Read y +Ġg all +Ġalleged ly +ĉĉĉĉ ĠĠĠ +ĠMet al +ĠPerson al +Ġborder Radius +rx js +object s +Ġwant ing +Ġb owl +v endor +offset of +ĠR s +ĠR ating +Ġr ally +_N ODE +ĠM ix +Ġadvert is +Ġnarr ative +s al +Ġm c +SE rror +Ġf ingers +Ġaccom pany +Ġt ired +Ġstr ide +Ġgu i +el ist +Loc ale +Ġrele ases +ik ing +Ġan ger +)) )ĊĊ +alle st +Sum mary +( O +(f or +Ġbasket ball +Ġroad s +ĠInst all +ĠF ab +it map +Ġ) )Ċ +Ġinter section +ighb or +ĠB ry +ĠHER E +So ftware +elf are +ac s +Ġtrail er +.get Class +ch ars +Ġreg ulation +Ġref ers +Ġde struction +Ġcontin uous +ĠAust in +é ¢ +ak an +.w indow +ĠTem plates +Ġabs ence +: n +Ġdis order +fl ash +Ġde let +bo ards +ĠĠ ĉ +RO P +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġac qu +Ġlaws uit +ĠRe views +Ġgar age +t imer +Ġe j +ĠRect angle +Ġflow ers +il st +ĠIn stance +S uper +d et +dis posing +ĠE S +ĠI C +ver e +S k +_ch annels +put ed +/ null +nn en +ĠG allery +_g lobal +Auth entication +ĠR ank +Ġblock ed +Ġcal m +mark et +ĉ val +Ġa ug +per iod +ĠCon stant +Ġ?> ">Ċ +Ġl obby +p al +Ġs ink +ia h +Ð ¡ +urn ame +Ġcon ver +Ġinvestig ate +Ch rist +H ub +ĠIN D +ĠP ed +ur as +ĉ url +ĠT ro +Ġpre ferences +Ġguarante ed +` ĊĊ +Ġport ions +Ġeval u +' > ;ĊĊ +.AutoScale Mode +Ġc ats +Ġreg istry +ul us +F I +p ayload +- search +Ġstay ing +ac ious +Dec oration +Re view +In f +Ke ep +it is +, String +Co ord +Ġper o +S ex +ĠAtl anta +uest a +Arg b +> * +} _ +F ooter +Ġemploy ed +_b ound +v ide +.f unc +$ scope +Ġsp o +ĠAn al +ounc ed +ar ound +Ġrestr iction +Ġsh ops +å Ģ +ĠLat in +-c ol +Ġbare ly +ĠE uro +E r +Ġfa ire +_d istance +_un lock +Qu ote +IV ATE +Ġå Ī +Ġaim ed +ĠRet rie +. iter +Ġwr apped +Ġagre ements +str ument +( product +Ġstud ied +.set Value +Ġy e +ĠC ache +MB OL +Ġquarter back +Ġsy ntax +.getElements By +.v ersion +we bsite +Run ner +_s ingle +at iv +ĠAl tern +ĠBeaut iful +right arrow +Ġd iversity +pl ash +( co +.F ill +Ġtyp ing +Ġcl ar +H it +O O +ac co +w orth +Ġscript s +ĠMuslim s +ĠL L +erv ing +( boolean +Ġbase ball +ĠC AN +MA IL +de pend +Ġrespect ive +Ġconst expr +.* ;ĊĊ +'] ))Ċ +Ġy ard +Ġident ical +if ecycle +US H +up iter +. validate +cl i +IST ER +Ind icator +F ail +Ġdemocr acy +. var +Ġsatisf ied +------------ - +enc er +h or +Ġr ounds +DA O +o a +Ġfl ask += c +[ ]Ċ +/d ist +Ġpart e +Ġconfirm ation +er on +aw are + +Ġdepend encies +ĠV ideos +- row +Ġ** /Ċ +Ġn ou +Ġh over +æ ŀ +Ġn in +ĠUS D +M ac +_L oad +Ġout comes +_s ocket +Ġqu eries +w m +Ġhit ting +in ux +M ich +ud ge +AT AB +Ġvulner able +ä ¾ +Ġport folio +: YES +ĉm ap +B ound +Ġiter ation +in cess +Ġact ors +ĠQ ual +_c lean +ãĢij ãĢIJ +MS G +G reen +ĠOff icer +Ġsm oking +> ', +ĠF lo +++ ; +oly gon +Ġbul k +Ġdr ama +Ġexception s +os ed +Ġ+ čĊ +Ġleg acy +C V +Ġcontrib uted +ĠTer ms +Ġb t +Ġunt uk +Ġal ien +=== Ċ +ĉ Vector +Ġl s +On line +.f acebook +num eric +ock ets +A ut +b ury +-re dux +ĠRed istributions +GLOBAL S +urrenc ies +Ġt ons +âĢĻ , +Ġà ª +(c ol +ĠS ymbol +Ġstay ed +ĠM L +Ġm unicip +Ġsex o +S en +n r +Ġg ains +Ġshort ly +.M enu +à ½ +KN OWN +Ġoper ators +- V +ĠPat rick +/ add +_C O +ir ation +(p ost +Post s +/ _ +Ġpl ug +Ġintellect ual +Ġmet ab +Ġpregn ancy +ĠPrem ier +n m +Ġpred iction +ĠMin istry +Th ree +val uate +ĠMin i +b u +оР· +< ul +Ġd d +ol ving +ĠC ut +Ġs chem +.tr ain +it ate +Ġr ice +Ġbird s +ãģ « +m iddle +struction s +Ġn erv +a que +Ġfl u +Ġsurv ival +ĠGal axy +ĠF ant +. Order +At trib +irt s +é c +M ovie +Ġcon ce +qu arters +Ġm ood +.Add Range +Ġres olved +ãĥ Ī +Ġburn ing +ĉĉĉĉ čĊ +ĠW E +Ġhost ing +L AB +Ġman agers +Ġstre ngthen +< const +ĠFire base +on ed +ĠJ ean +' ";čĊ +ĠS av +.B old +Ġen ables +ĉt mp +Ġman ually +ĠS qu +user id +.f unction +.c ache +LO PT +.S ervices +dd it +t im +< img +ĠTh ings +ĠEvery thing +Ġa pt +em and +Ġroll ing +ë ¦ +. level +Ġst om +ĠW inter +Ġview ing +( values +ocom plete +v ia +up o +Ġabort ion +i ère +ï¼ ij +_B UTTON +_d omain +Ġb ra +ĠA st +in as +Ġstat ist +c od +L R +Ġdr ives +Ġfollow ers +Ġall ies +ĉc urrent +ecess ary +Ġdam aged +_ pt +and les +oun tries +Ġsim ult +e u +Ġcontrovers ial +_G ROUP +Ġr ib +. Info +: mm +.n ormal +_ADD RESS +Ġ íķ +add le +ĠD ur +. Element +W arnings +Ġcred its +Ġin hib +Ġem issions +Ġh az +.y outube +ugg ed +Ġbo ther +ĠK ansas +ĠF ixed +ĠTest s +ĠF IX +Un iform +Ġk ont +>> > +st ation +lo re +at ype +ish op +/ **************************************************************** +Com boBox +Ġvac ation +Ġiniti ative +Ġdefault Value +con cat +ĠK h +ĠW elcome +ized Name +M igration +Ġgrad ient +H ot +Ġhard ly +el o +ĠStud ents +Ġlo ose +at z +.S end +' / +Ġunivers al +Ġenter prise +Ġreg ex +Ġvis itor +ĠF ly +Se q +ภĻ +ĠVis ual +Ġlib raries +ato es +P ayment +Ġp ent +Ġgather ed +VRT X +ĠD M +S plit +Ġlet ting +Ð Ŀ +_error s +ep och +P ARAM +c u +ÑģÑĤ в +ol utions +Edit ing +font s +Ġalloc ated +ĠB ased +( Y +ĠJud ge +Ġbro thers +FILE S +ç o +w b +_P I +' ^ +Ġs word +.s ervices +Ġn l +T im +ig g +ĠMo ore +Ġcrypt oc +åĩ º +_post s +ot ate +? ' +... .ĊĊ +Ġk l +=" $ +Ġdec oration +Ạ¡ +ĠD IRECT +G UI +) =>{Ċ +Ġnews letter +Ġprec is +(p oint +ĠEqu ipment +ut y +ĠD ave +Ġparticip ation +u arios +x it +.A s +ET ER +or ous +Ġsh ield +[] > +ilit ary +. origin +Ġprom otion +U nt +Ġc t +TR A +View Holder +Ġsig ma +d elta +are house +con tract +( Vector +Ġcompet e +/ form +/ components +Ġn r +ĠInd ones +Ġо ÑĤ +ĠV olume +.f iles +(res p +/ models +Ġsur f +stand ard +/ o +ĠXCT Assert +V ICES +.C ode +SE D +Ġact ivate +D elta +Ġlimit ation +ri j +Ġpregn ant +: ^( +Ġs our +p ie +Ġexp ense +ic ation +ĠL arge +Ġ ± +ĠB owl +(model s +/ N +P a +.re load +Ġwonder ing +Exec ution +ĉ ĠĠĠĠĠĠ +ĠG raphics +ĠCont in +_j ob +Ġget Name +ĠM agn +ĠD WORD +m ad +Ġn h +fe atures +} ");Ċ +he ets +(tr ain +z n +Ġrecru it +.con nection +Ġbar rel +Ġste am +_set ting +Ġang ular +ane ously +Ġb il +ĠN orm +(! $ +ib t +% ( +Ġpos it +ĠF ather +int endo +L ive +Ġport s +Ġme j +Ġland ing +pon der +Ġc od +_HE ADER +.M argin +Ġball s +Ġdiscuss ions +Ġbl end +H ex +Ġfarm ers +Ġmaint aining +ĠĠĠ čĊ +s yn +[ T +r us +uff ers +Ġcontrib utors +_s ys +.De bug +Ġconstruct ed +om es +? id +sl ider +Ġsup pliers +scri ber +p es +Ð ŀ +": čĊ +\ Controller +)) ĊĊĊ +Ġl ua +M ulti +EN S +S rc +Ġpet ition +Ġsl ave +look ing +V ERT +ĉ vector +S pecial +h h +an ne +ĠN iger +/ views +z ing +end ant +< C +s peed +Ġ{ };ĊĊ +Begin Init +Ġf open +@ RequestMapping +End Init +Ġp unch +S ender +é Ķ +get Message +/t ypes +.P I +(' ');Ċ +oc used +( all +Ġdrop down +). __ +ĠV in +.Fore ignKey +can f +ou red +ĠOrgan ization +ĠÐ ° +ĠC ulture +(cl s +, _ +rg ba +ìĿ ĺ +.data GridView +Ġdo zen +ĠG es +_sh ared +n ick +Ġh osp +om eter +Ġclaim ing +ib les +ri k +æĺ ¯ +en ario +Ġd engan +ob b +m ont +_r ank +('/ ', +Ġap olog +P s +_p ower +ĠG ree +Ġful fill +Ġfire base +Ġf are +ĠH im +Ġbe an +âĢ¦ . +ĠS PI +_R X +Ġper ception +rel ative +comp ile +u um +ut os +a uc +ĠAs k +Ġindic ator +/ th +.set String +ĠWis consin +.D omain +Ġart ificial +De velop +ĠSar ah +Ġl ying +( search +ĠEmp ire +urr ing +æŶ éĹ´ +=" ${ +Ġget Id +ĠP ayment +trans ition +Ġ ]. +ix in +V T +- select +Ġdemonstr ated +Ġlast Name +employ ment +.get Property +Ġf ought +file Name +ĠP ers +-c ard +a str +attr s +Ġprom inent +Des ign +anc ouver +ãģĹ ãģ +ard o +se cret +Ġr ag +Ġpo ison +-m an +, omitempty +ĉ un +it zer +ĠCas ino +ĠR oss +- foot +(result s +Pl an +Ġlas er +ê¸ ° +_D R +F acebook +Ġbo ards +st a +] ], +Ġt iles +S IZE +Ġ= ~ +Ġprem ier +oc ab +Ġenc oded +Ġres erve +ĠAfghan istan +ĠList Node +url s +Ġsub mission +Ġne u +Ġ# +# +_P OST +Ġmo ist +ell i +ellig ent +. alert +ó d +b re +ĠCol lect +Ġgraph ic +Ġlong itude +ĠPro vid +ĠCal culate +x ffff +c riteria +Ġw aters +ro ck +lo quent +ĠT rib +Ġbur st +Ġsuff ix +.Ext ensions +ish es +iv el +ĠLI KE +ĠGet ty +.Action Event +.s lf +ĠH AL +up al +E AR +ud i +_time out +U F +ĠSing apore +ĠAd vent +_int erval +cha ft +ĠE mer +Ġtele phone +ĠTur k +_ interface +ĠO wn +Ġencour aged +< Object +_T ext +ĠOnt ario +ĠApp ly +.f irebase +Ġant ib +P riority +ene z +D ays +c id +urre nce +; / +inn ed +Ñģ Ñı +Ġve z +f w +// $ +att ack +Ġstart up +ain ers +.f ragment +op acity +( conn +he im +.n etwork +( stream +ĠN ON +t ol +ĠX box +ĠD S +Ġc ached +Ġprostit utas +ĠB alt +(' [ +Ġno except +" ' +Ġs d +. valid +_ ag +Ġr aces +Ġro d +itud es +< >( +.Pro duct +Form s +NE W +P ay +ĉ boolean +_ contact +ĠElect ric +sk ip +Ġw ur +Ġch ronic +_d river +ĠS ab +ĠU lt +ĠR ad +ST ATUS +ĠLew is +O B +Ġgift s +.Re c +TR UE +Ġint ensity +Mark er +.com pare +ff ic +C ookie +ĠB aby +ĠBig Decimal +ile t +ĠHOLD ERS +ĠL ady +Ġl ung +ĠAl abama +Ġd ess +` );Ċ +ĠB uilder +_reg ion +Ġne utral +Bo th +Ġh p +Ġh orn +Ġseg ments +ĠE C +"=> " +( rec +ĠP i +G M +Ġl aptop +Sc alar +is d +-d ialog +ĠAnd erson +Ġmist akes +ĠH an +j es +est ination +Ġprom ises +b id +ĠSc ient +G IN +ĠPer formance +b age +. users +le ading +Ġor al +G raphics +_P TR +h ang +Ġin ev +process ing +F actor +ĠN A +$ string +Ġground s +.Save Changes +c lock +cri pcion +ĠNew ton +g c +.in cludes +Ġbl ast +Ġ'- ' +Ġpued e +.S ession +Ġgre p +_f inal +ĠG ay +ĠG ive +ir i +-st ar +ĠUI Image +_ep och +ub b +ent h +Ġel ite +Ġcampaign s +ĠP orno +_ assign +Prot ocol +ĠBe ing +ĠAir port +Ġconvent ional +ĠW at +ĠC I +ET A +ĠAnth ony +Ġtable t +( format +Ġconsist ently +ĠI owa +Ġav atar +.c ursor +! [ +Ġh anging +H er +S uch +';ĊĊ Ċ +orge ous +() == +Ġview Model +Ġ ãĥ +Ġel s +ĠAg ent +F etch +ap or +Ġc x +p read +ĠP ier +oe ff +S n +ĠV irtual +A pr +.Wh ite +_M OD +ĠPoint s +å¤ ± +Ġgen es +Ġv endor +Ġmain stream +< src +ĠEl izabeth +Dec oder +- state +ĠG lass +nc y +adi ans +_m on +ĠRem ote +Ġwire less +ĠM i +å ī +è¡ ¨ +st age +ĠT ile +ll ib +V ariant +== Ċ +Ġgold en +(Q String +.put Extra +ĠD om +ĠAn imation +Ġinter active +if act +éĻ ¤ +LE T +Ġfrequ ent +Ġ< >Ċ +F ilename +Ġs ne +ĠFoot ball +Ġr ival +Ġdis aster +ion ic +ĠD amage +. Resource +- en +ĠT ypes +get String +( board +Ġb ol +pl ain +z ym +ภ² +Ġsc anner +ild er +_msg s +æ ı +(int ent +Ġde struct +Ġb ust +ĠE mploy +on i +ĠUI ViewController +Ġodd s +ear er +Ge ometry +Ġy ii +_EX PORT +ĠAtt ack +Ġn iet +Ġim pression +ĠG il +_pro b +ĠC F +ĠEx perience +/pl ugins +.M ethod +Ġbelie fs +N ative +_b uild +Ġv ig +Ġr anks +cover ed +s uch +G uard +.p ack +add er +iv ia +l ng +Ġв Ñĭ +T imestamp +_n ow +Ġp oker +Ġun c +Ġsh apes +-t ypes +_per iod +p k +Ġveter an +Ġson o +Ġappoint ed +over flow +.d river +_c at +ut t +pl ant +im b +ĠAc cept +Ġconc ert +ĉ node +ĉ z +? >čĊ +Ġb anned +ĉ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġto xic +Ġdisap pe +È Ľ +Ġgr ace +ate ful +Re ply +ĠCru z +Ġsc rap +Ġkey words +s imp +Ġmort gage +Ġcy ber +ĠEx ecute +Ġlat itude +if u +.C OM +d bo +Ġsort s +ĠG as +om ial +.L ocal +Cell s +.Re place +String s +.f it +ĠTh ird +% ",Ċ +Ġ{} ". +ĠS ony +Ġ[ : +Ġfall en +. ')Ċ +in h +ĠM C +Ġred is +C odes +Ġprofile s +h ook +Reduc er +_F UNC +Ġn avigate +str len +Ġh orm +á ŀ +ĠS R +. boot +Ġdig est +ĉ header +.find One +æ ģ +Db Type +n ia +_m erge +Ġdon ne +/ Getty +_CH AR +Ġb ands +. URL +art ial +Ġf req +Ġs ist +N g +Ġrender ing +\ Core +Widget s +ĠV A +Ġactiv ists +St e += _ +all a +St amp +Ġload s +Ġx x +ĠL earning +.M vc +u ir +(" $ +Ġconnect ing +Read Only +ur u +ĠE ag +B IT +_DE L +å § +arr ass +ext ernal +ĠY OUR +ĠB rew +ĠF ive +Ġres ize +ig id +er ation +ĠÑ į +åĬ ł +ĠC atch +Ù ģ +ĠLe on +am il +.B ody +Cl ip +/ list +.b r +Edit Text +ĉ db +.G ame +(Build Context +back end +.R ed +face book +.url s +m r +rol led +---- --- +Ġinter vention +Ġretire ment +ĠK it +ĠP RE +Upper Case +ĠS ocket +Ġ: - +Ġstudy ing +ĠMet ro +ard ed +Ġconvers ations +C alled +Ġexam ine +ert ificate +.g z +-res ponsive +Ġref und +_n etwork +allow ed +em pt +Ġme als +C ategories +Ġtravel ing +Ġk g +Ġsh ame +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġexplicit ly +Ġmath ematic +ĠS uite +ĠR GB +****** / +Ġmix ture +lear ning +.t emplate +att s +w x +ĉ ctx +.p roperties +Ġdrink s +ĠE ither +set Text +.get Data +.z ip +Ġreve als +< table +.Hash Map +ĠH ur +) ");Ċ +.f ramework +ĠST ART +feed back +Ġsaf ely +. icon +config ure +. lock +.l ayers +/> .Ċ +Ġrank ed +_ impl +ĠHand les +Ġhost ed +Ġup dating +al bum +é Ŀ +Ġsh ader +Edit ors +- round +[] { +Ġse p +ĠH i +TE M +look up +.m an +_IN PUT +Ġthreat ened +_IM PORT +Ġd rops +ru it +s id +bo th +ĠEx cel +Ġj er +ord inary +еР¹ +V IEW +re ply +Ġ) :Ċ +color s +ver ified +_T r +_p arse +Ġcon gress +P romise +int s +ĠM other +.A pi +ĠD uration +Ġfirst Name +inherit doc +ĠM ars +Ġa pr +OD Y +Ġvis its +Ġhe aling +let ters +)) );čĊ +f uture +.F ramework +Ġk iss +Ġinv olve +Ġsil ent +ad ows +Ġany body +s ch +Ġsole ly +- img +Ġprop ri +Ġin struct +Ġlic enses +Ġm eth +Ġcond em +ĠD omain +ĠHarr is +Ġs Ã¥ +CE PT +B atch +@ extends +ĠCONTR IBUT +.Data Frame +_p acket +rec ision +Ġfoc using +. ht +__ ":Ċ +: Get +ĠK C +Ġpass age +Seg ment +_c enter +-z A +_B L +Ġconv in +Ġclass ified +ĠNS Mutable +_ ap +t ile +Rect angle +(n ums +v ens +ĠUI Button +ĠF eder +am o +Ġout line +ĠPar ser +Ġâ ī +ĠWork s +.S chema +Ġeng ines +_com mon +_ old +Ġset ContentView +Ġ/// < +ĠB T +f m +Ġd ivers +_ weights +em ark +ĠA CT +Ġpro portion +over lay +.dir name +ĠG it +_REF ERENCE +< > +l b +_r ule +è´ ¥ +ĠPut in +Ġsleep ing +() :čĊ +Ġpres erve +Ġpar liament +ĠLook ing +Ġpick ing +ĠDis patch +Ġsl ip +ë ĵ +ĠL yn +_sign al +config uration +ĠP itt +ad en +pro cedure +Ġenthus i +f ight +ĠCons ider +Ġt orn +Conn ected +.c os +_group s +ĠTh ink +Ġdel iber +Ġres id +work ing +.column s +ĠCal led +Ġes lint +> ", +_D OWN +h ist +ĠAdv anced +Ġre wards +act ors +Ġsil ence +Ġmy th +Ġne ur +Ġa uction +.Get String +ek s +( project +ĉ msg +ĉ output +Ġcomplaint s +, S +Ġt bl +Ġ, ĊĊ +ri ors +ah ren +Ġlawy ers +re dux +_s ymbol +off ee +_RES ULT +( Name +UT C +.current Time +Ġorgan is +. arg +Ġmin im +w ick +Ġrece ives +B alance +Ġspeak s +ĠD ays +ĠBel ow +t ipo +P resent +Ġres erv +h p +Ġr it +_R IGHT +-- ) +Ġchair man +D IS +ĠBO OST +Ġexper iments +__ );Ċ +Ġst amp +Ġf ert +Ġf ond +T er +el ve +ure n ++ i +end ency +Ġvirt ually +... " +ï½ ŀ +- cent +_un ique +Ġpr icing +m ic +RES H +Ġ:: : +Ġan notation +ĠC ircle +ong odb +it as +Ġ% ( +( component +Ġо б +( port +-h our +. obj +L BL +Ġj ury +GB T +Ġsp y +ĠProf essional +Ġ"" ;ĊĊ +Ġstri king +Ġdiscrim ination +Ġp ays +lic t +ent es +Ġthrow ing +ĠPl ugin +( def +ĠRuntime Exception +ĠM igration +Ġd ic +b ag +on ia +Ġcor ruption +( Map +Ġpr z +.d to +Ġac quire +State ToProps +Ġlo ving +оР¶ +_p attern +Ġemot ions +Ġpublish er +_b e +Ġcoup les +o j +ĠCh art +Ġt rop +.t ool +Ġestablish ment +Ġd ol +Ġto wer +Ġl ane +ĠSy dney +Ġfill ing +claim ed +Ġdialog ue +Ġcon vention +book ing +pare ncy +æ ± +ĠGener ic +\ Schema +Ġr anges +/ ch +Ġpan els +Ġr uled +çĶ Ł +.t s +_s ets +Ġclean up +Pre vious +ĠAn imal +($ ( +ĠA ve +oll ar +_e val +ĉ Name +(t ree +Ġ" ] +Ġdut ies +=' / +Click ed +Ġdifferent ly +ĠCl ark +Ġd it +olog ists +Ġsy nd +Ġs ends +- known +k b +ĠMod al +it ative +Ġr acing +Ġhigh lights +ĠSim on +ĠCapt ain +ä¿ ¡ +ĠC B +cont in +ar an +Ġphys ics +ret ty +et al +.m d +ax ios +Ġspeak ers +Ġpre p +Ġaward ed +ì§ Ģ +ĠC orn +ĠN ature +UD IO +Ġpro j +- pre +[ u +Fe atures +Ġis Equal +B inary +s ig +Ġconf usion +ĠH at +Ġkt ó +.config ure +M ON +/ edit +_A dd +, true +Ġc li +Error Message +- loader +Dim ensions +ultip ly +Ġ{ !! +ĠSql Command +Ġsp oken +Ġp ics +Ġto y +( Key +ĠLo op +Ø ¨ +E ATURE +in ction +_set up +w rapper +Ġt ong +c ular +O pt +.P l +=" , +(l ength +um n +Ġch rom +Ġse vent +ĠIllegal ArgumentException +ĉ start +Ġbeg un +CE PTION +dat aset +ĠF ailed +col s +Ġkne e +im ore +.sp lice +sh ell +ig gers +Ġthem es +ĠD J +ĠAss istant +- $ +May be +Ġorder ing +ĠInt elligence +ĠMass achusetts +Ġfail ing +el son +G reat += i +.re st +Ġinv ite +-dis able +.Group Box +âĢĻ est +Ġtack le +g v +et ter +Ġ), čĊ +_r ules +.w arn +function s +ĠChrist ians +Ġback ed +Ġsl ider +Ġenjoy ing +n est +Ġh ij +_m s +// * +An notations +ĠVariable s +< V +( server +ĠOr acle +element s +Ġorgan isation +_point er +ĠHe aders +[ d +Ġdead line +iss a +Ġkn ife +ĠNAS A +ĠHe ight +ĠAs ync +Ġven ue +.d om +bour ne +ĠHaw ai +Ġmem o +ict ions +Ġsurve illance +om i +/ assets +Ġed u +Ä Ľ +Ġro ster +Ġh ired +ĠT ok +Ġpl acement +ur ations +Ġset State +ĠMag azine +Ġhor ror +T ry +Ġl ag +ĠEvery one +th ur +)) ;čĊčĊ +. return +Ġsy mp +âĸĪ âĸĪ +Ġn ights +work er +Ġa le +ennes see +.st ep +Ġsynchron ized +our i +Do es +. change +f on +.set Background +irc ular ++ - +ĠC IA +ĠJ ane +ĠSim ilar +- I +level and +Ġpros pect +_f ound +ĉc olor +.D iagnostics +Ġann ounce +Ġassum es +/ tr +Ġb d +ĠCar bon +Ġanal ys +.de st +n ik +ĠL ie +- index +Draw able +ĠT AG +Ġtri angle +_F LOAT +ĉĉ ĠĠĠĠĠ +.bl ack +v ue +cur acy +Ġaffect s +Ġsure ly +Sl ider +uk i +c ery +Ġun ter +.pro file +ord on +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +le ave +Ġsmart phone +g ie +Ġcons pir +Ġt utorial +ç± » +Ġc ab +ĠSum mary +* ĊĊ +ä h +" This +Ġsl ides +" +c ycle +ĠB ull +path s +Ġun p +Ġview DidLoad +_M odel +Ġassert True +Ġr ated +De cl +vert ed +ĠD at +b rew +Ġpoint ing +M s +ĠPoint er +) ' +_n on +ĠSE C +Ġy eah +g ency +initial ize +f ly +[ pos +, g +Te le +Ġj oke +Ġcl ause +.find ById +en es +( instance + £ +Ġs lic +_h ome +Ġ*/ }Ċ +_p ages +(s ervice +R P +ĠAm ong +.get Current +ãĤ ¹ +Ġs lee += [Ċ +ol er +Ġlib ert +Ġ` Ċ +Ġw enn +l ated +Ġimm une +( Node +ĠPro blem +ĠA bs +log s +Ġ ../ +ĠA DC +Ġ}} ">Ċ +> ');Ċ += b +ĠW ind +lah oma +Ġalloc ate +or ian +Ġpres cription +- quality +ĠMay or +in ely +end foreach +ĠCom plex +k om +T Y +] ]. +. Style +_m any +',' $ +Ġbar rier +ĠF etch +ĠMar vel +Ġres ist +ог о +b idden +ĠRun nable +: false +Ġbuild s +ĠSt age +Ġd ub +emp o +.s ite +;ĊĊ ĊĊ +ĠDen ver +Ġre vel +Ġtrigger ed +Ġd ice +_f ail +Ġg c +ĉ X +ĠTh rowable +.r outer +ĠRev olution +ÑĢ а +_N ON +Ł ¥ +Ġel der +Ġab road +ĠÐ µ +ĠAd ult +bl r +g lyphicon +Ġprom oting +Ġ iz +ĠS olid +_lo ader +ear ly +.en abled +- edit +ĠU L +_ play +ĠInt errupt +Ġadvant ages +uc le +Ġmechan ical +.table LayoutPanel +ĠWork ing +Ġan onymous +R ating +ig ious +_ph one +.addAction Listener +Ġfr an +und en +Ġ*) & +_ bool +ul ative +Ġcon e +ĠM ult +Ġm ö +ĠFor ward +] ):Ċ +Ġconvin ced +act ed +ãģ ĵ +ĠConfig ure +Ġce iling +D er +Ġpass engers +Group s +Ġsoc cer +/ W +avi ors +sw ith +ĠZ one +. Options +ĠM om +ied er +Array s +Ġtreat ments +Ġprotect ing +f ac +Ġpick le +Button Item +Ġblock ing +str ar +à ² +ĠEx port +Ġth rew +ott a +ĠB ASE +.w s +.LE ADING +order By +_d elay +ĠP u +.d ll +ĠCh oose +Pol ice +ĠBE GIN +box es +Ġdiam ond +, l +Ġ ĉĉĉ +Ġcur ious +t v +Ġerot ische +ack ages +ĉ Set +T ick +.b order +static method +Ġch er +in voice +Ġcr u +Ġdef ect +_m etadata +re lation +ik an +[ N +(Q t +( Base +æģ ¯ +be at +ĠEm pty +ĉ o +_sh ift +Ġreg ret +Th ose +C ent +ĠPort ug +ĠIs lands +ĠT IME +Man agement +-s p +ê me +Ġnot ion +un ifu +P K +è¡ Į +ĠCUR LOPT +\" \ +U V +ç º +d ra +c ou += ` +ĠD estroy +r p +.c ancel +G G +r untime +ĠV ue +Ġprogress ive +/s ervices +Ġrun ner +_FR AME +.ToolStrip MenuItem +Ġ' ,' +d elay += utf +Ġscreen ing +Ġpull ing +om as +Ġan th +- new +/ local +Ġi Pad +Ġt witter +Ġd ying +Ġhe aven +ĠU Int +ĠSen ator +Ġpres um +ĠWalk er +Ġover come +ete ction +Ġemb arrass +Ch ina +In clude +RO LL +Ġdata Type +D avid +ภ£ +lo p +-m onth +Ġsc ar +ĠS afe +Ġ **************************************************************** +Ġaccess ories +Ġr amp +_U SE +Ġcontr ad +)) ]Ċ +Ġpre st +ĠH R +ĠR ap +Ġus ize +Ġcap ability +Ġc ort +- next +Ġbur den +_read er +Ġ@ @ +reg ular +ĠK a +M AN +Ġa str +Ġ' ')Ċ +Ġf ed +Ġpars ing +ĠY ears +Ġbro ker +": {" +Ġa kt +In ventory +abe led +Ġarg parse +****** *Ċ +vers ation +Ġc ord +ĠT i +Ġhope fully +Ġa h +ver b +Ġst olen +. Entry +Ġexpect ing +O rientation +Ġpower ed +Ġp ersist +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +'] ); +')) ,Ċ +ĠC ash +ĉ item +gr ades +rop ol +b asic +Ġ" );čĊ +Ġaw ards +(r ange +- all +ĠIB Outlet +ĠInd eed +---------------------------------------------------------------- ------------ +Ġstom ach +Ġfl ower +Ġs ew +_t imes +av is +Q String +ĠR outes +_pro t +Ġcom edy +Ġlog out +Ġwood en +Ġpost er +p iece +.J oin +ĠP ok +cel ona +mut ex +;čĊ čĊčĊ +Ġstri kes +Load ed +) arg +es a +Un ited +E p +PE LL +ĠAtl antic +ul let +app le +Ġsett led +a con +Ġprint er +ĠG C +å® ļ +Ġrender ed +, âĢĻ +he it +s ocial +. ge +ĠR ick +ĠUt ah +g ot +on ical +ĠSc roll +ĠSc iences +Ġj ug +Ġam pl +ent i +LE FT +Ġt abs +Ġenorm ous +.get Key +loc ate +. EX +.st orage +.W e +Ġto ast +ĠAdd itionally +ĠN OW +_ UPDATE +Ġtrans ferred +th a +.D isplay +_ ui +ID EO +Ġmeaning ful +ĠMos cow +, this +ĠVict oria +æĶ ¹ +ĠÐ Ł +.st ack +ĠB arn +pared Statement +: string +Ġb ij +ĠST ATE +Ġemploy ers +ĉ input +( | +Ġle x +in voke +ĉ num +++ , +at ial +ors es +Ġfor k +_t xt +ĠAnton io +Ġ( < +aver se +Ġdev ast +ãĢ Ģ +.D ec +ĠG ard +/ ui +. % +tr i +Ġrol led +Value Pair +itt en +ĠTh er +Ġv rou +ĠFl ow +ĠFin ance +ĠCom b +H C +.set Visible +is l +Ġp k +Ġup set +( raw +ĠV ice +e atures +ĠL ang +Look ing +ĠA ST +Ġtri ps +ĠJust in +b rowser +=" '.$ +. vertices +- co +}/ { +Ġ? , +ĠD omin +ĠBel g +" < +Ġsup pose +add y +Ġwalk s +ERR U +_f ilters +Pre ferred +sc ene +е Ñģ +ĠAff airs +Ġ"# { +Ġon Submit +Ġstock s +/ view +g ree +- get +h it +J o +.get C +Initial ized +ÑĤ и +c uts +( Type +ĠAg reement +ĠViet nam +Ġ/* ! +Ġp izza +- view +_ em +Ġl hs +Ġm uy +ĠId ent +ĠF riends +Ġab und +_A D +.t imestamp +- ' +Ġd uplicate +Ġhun ting +Ġregul atory +ia o +am ous +ĠEnt ertainment +[ A +iat ric +_CL IENT +ĠK ids +/p kg +B reak +)) );ĊĊ +ĠSh ape +Ġrel ating +Int errupt +able Opacity +emb re +Ġmyst ery +Ġjournal ists +rit able +.L ink +Ġstop ping +CRE T +.D B +Ġpopular ity +Ġg ew +Ġim pr +set Value +FL AG +ĉm ax +Ġb ake +w y +ĠEcon omic +Ġen contr +Ġf name +/ de +R ank +Ġbug s +.s m +Ġmed ian +D OWN +ĠS ure +At Index +ĠD ick +Ġ( __ +.d elta +F r +Ġsuggest ing +ĠRec yclerView +, e +ST ART +/************************************************************************ **** +xf ord +Ġrece ipt +CL AIM +read only +Ġeng aging +C a +as ma +Ġens uring +Eng lish +ĠV ancouver +hy th +Ġpurch asing +ĠP I +. word +(s p +.h ome +: def +Ġg ig +ĠV e +for um +ĠM itch +B ay +_F L +Ġs oll +_column s +Ġminor ity +b ird +Ġhand ed +SS L +ST AT +Ġnerv ous +ĥ ½ +Ġfile Path +CRE ATE +A w +Ġp ens +se ed +ĠCom pute +ol k +ĠAs set +re ach +'), čĊ +n avigation +L F +/ util +ĠP ub +Ġâ Ķ +c ion +## Ċ +II I +Tag Name +Ġam id +per mission +if iable +xFFFF FFFF +н и +.B uffer +_ irq +d ark +Ġret val +.f ire +produ ction +.list en +ĠWe ather +Ġbuy ers +. ne +er p +ĠP ent +Ġw elfare +Ġpage Size +ĠSt adium +ert a +Ġle v +amp a +P ager +Ġcharg ing +ĠNet flix +| null +_r andom +.x path +Ġst ere +ĠIS IS +pons es +( loc +ey ond +ĠOff icial +ĠMary land +Data Type +_p ar +{ }, +ĠEn joy +_SH IFT +ĠA wards +_ENT RY +Ġseem ingly +entic ate +Ġheart s +_ ;ĊĊ +ĠH IV +Ġindiv id +ĠFl ag +_ ctrl +ĠC allback +, z +ĠG PU +ĉ obj +ĠPh oenix +ĠB US +Ġrub ber +_A UTH +ĠSol utions +( location +Variable s +.set Enabled +_h igh +W O +G esture +Ġre try +Ġobject ForKey +allow een +Ġm os +ĠC ele +Ġik ke +(c ell +ĠM ODE +ren a +Ġdescri bing +Ġph i +Ġr d +Ġdes erve +Ġwhe els +å¸ Ĥ +Ġcrit ics +N amespace +ĠF ra +Ġ ĊĊĊĊ +Ġall a +Ġrequ iring +æľ Ł +ut ation +Ġdelay ed +Ġadministr ative +Ġb ay +.h idden +T ex +Ġbound aries +Ġ] );ĊĊ +ĠFollow ing +~ / +F i +_con v +_T ITLE +Ġdes de +ICollection View +Ali as +Ġb ite +pat ient +_COMM AND +Com pleted +ĉ elif +( < +B usiness +ĠP ool +Ġpurs ue +ĠB an +_st eps +_DE CL +um ble +Ġcom bo +ĠL ayer +.x r +Ġd up +-------- - +Ġmod ifier +ro b +re z +Ġath letes +Us ed +w ear +Ġlegit imate +Ġ" ĊĊ +Ġh v +St d +ĠH old +Ġsurv iv +ĠAll iance +ĠEar ly +Beh avior +(f ont +/lib s +Ġrect angle +Ġs inger +Ġam p +Equal To +Ġ" ." +Ġgirl friend +å ± +line ar +obs erv +Ġpi ù +Ġcomple ment +With Value +(p assword +t ake +Bl ank +ĠCom par +' ", +_p olicy +m ongoose +_FA ILED +.re port +R atio +.Perform Layout +us able +m ers +_re nder +PE ED +Ġles b +ĉ E +_t ool +Ġl adies +о Ñģ +)) ))Ċ +;; ;; +.d ot +Ġn est +pe ak +uk kit +ec a +_S W +Ġ& ( +ĠOk lahoma +Ġbank ing +ĠN intendo +Ġreprodu ce +_element s +_m ac +pro xy +Ġremark able +}/ ${ +Ġout s +.has Next +M ODE +Ġan ime +.con n +Un ique +D om +Ġimportant ly +itt y +Ġju ice +T w +ĠPart ners +Ġattack ing +Ġport able +am iento +.P ictureBox +.g en +Ġopt imal +Ġre cre +Ġjournal ist +ĠEx tract +ĠMore over +Ġmargin Top +.A p +Ġf iring +Na N +ĉ template +аР´ +. En +Ġdef ence +ĠT el +il en +j an += data +ĠU rl +ĠRe uters +(t otal +ĠFif th +Ġess ays +Ġinterpret ation +Ġchar ity +ĠR ules +Ġsub section +st yled +az er +l ags +L IST +Ġupload ed +Ġtr ash +Ġreg istr +Ġsell er +>' ;čĊ +Ġstart Time +ç Ļ +s y +(Http ServletRequest +Ġtr ap +G C +Ġembed ded +Ġsurround ed +im its +T X +yl inder +ĠF al +Ġsent ences +ĠJ a +IF ICATION +we apon +ov ation +Ġco at +Ġinter pol +Ġl ips +ĠK y +Ġv ectors +_ am +Ġint ake +.w orld +Ġin box +ĠM AC +_ ab +(name of +Ġent ert +Ġgather ing +ĠS IM +++ . +ny a +' }} +ĠUP DATE +Ġp ac +( html +ĠS ant +i ating +ĠIde as +Ġspr ay +ĠH art +Ġver ification +ades h +/ modules +ĠM ind +ĠSized Box +Ġsh elter +Ġher oes +att y +Ġcert ified +s j +Ġê tre +ÅĤ o +Ġpublish ing +ĠMal ays +.get User +ĠPro vider +ĠLinked List +ĠB or +RO UND +d id +t ain +p ire +ĠJ enn +t el +and e +_f ront +ĠMc G +Test Method +à¸ Ń +Ġoccasion ally +ĠW ales +Ġexerc ises +ĠÐ Ĵ +- plus +Ġvalid ator +Ġpr ayer +L ATED +_ author +Ġlab our +++ Ċ +-e quiv +ĠG PL +Ġface book +s imple +g ly +Process or +ip y +Ġ* > +Ġcle ared +ĠP ush +Ġpen is +Struct ure +li j +ĠM organ +Ġhand ful +" .Ċ +| \ +Ġ ******************************** +ĠA qu +_ IC +.load s +Ġm eter +ĠMar ine +:: { +ĠT S +ĠArray s +.T itle +GR AM +ter min +Ġco inc +El se +_st ates +-r un +m embers +ast ro +Ġon Press +Ġbe ings +Ġabandon ed +Ġtax p +own ers +.m ode +Ġdiagn osis +Ġ_ Ċ +ĠK night +ĉ A +Ġob serve +), ' +! ")Ċ +ĠPar a +Ġvari ation +( False +ĠAnt i +Ġg ri +Ġhome less +? v +Ġbe z +.S erver +re lease +ĠP atri +Ġchar s +Ġrank ing +activ ation +Ġw ides +q r +.S ql +ac ular +ĠB ot +_s ync +Ġhapp iness +Ġvolunte ers +Ġs its +/ < +[ e +(file Name +Ġcap ac +ĠMar ia +f ather +Ġgr am +* i +Ġcas o +_d raw +ĠR aw +ĠIter ator +ĠP adding +P D +BO X +ĠS PECIAL +Ġfe cha +Ġv ide +ĠLe ader +ä» ¥ +$ (". +Ġdiam eter +Ġm ild +Ġrock s +app ings +d irectory +.fl ush +ĠJ ess +UN IT +ĠP ear +Ġmand atory +S ur +q t +Ġstream s +Ġco operation +ĠS ac +Ġche aper +ĉ ch +an imation +f are +( height +( True +N Y +Ġw rest +Ġpoll s +Ġencounter ed +ĠMarket able +_P ASSWORD +_SE LECT +ĠArab ia +_c lock +Ġv oy +Ġи з +Ġst ir +is ible +-e ffect +.c reated +Ġto ys +ĠTrad able +Ġr ust +Ġstr cpy +_t imestamp +Ġtalent ed +, null +ĠJ obs +ĠPort land +Ġweak ness +Th row +ĠAng el +ä¿ ® +Ġun cert +ï¼ī Ċ +ĠìĿ ´ +Wh ich +Ġ[- ]: +S omething +Ġconv icted +k le +ed ium +Ġbranch es +Ġb ases +ç ® +Ġcomplex ity +ĠF ig +. reshape +$ db +_CON ST +ĠT es +.r untime +Ġden y +ĠB SD +Ġk r +h att +ĠSt atic +Ġunivers ities +Re place +Ġdro ve +Ġad oles +_pl ugin +ĠL GBT +Ġt ex +du ction +ED I +ĠT ed +_ URI +Ġre ception +art en +.S ingle +r ice +sc ious +_b g +Ġw ages +ĠS ervlet +UIL ayout +Ġform atted +.M od +< class +is en +Ġrepresent atives +"] = +Ġport al +ĠHun ter +Ġh iring +__ )Ċ +ric ulum +u o +li est +Ġt ears +L at +Ġliter al +.In sert +Ġc urs +ĠCom put +Ġterror ism +Ġswe ep +Ġ[] čĊ +Ġpass enger +Ġeast ern +Ġtwe ets +Ġoper ated +w nd +ĠS yn +.t ools +ĠW M +ul ates +Ġbacter ia +( bytes +.set Data +Ġvis ibility +// ================================================================ +el m +Ġgener ating +Ġm v +Ġk h +j en +/ search +Ġaccount ing +se gment +act ic +. ip +Ġdeploy ment +Ġfoot er +> ',Ċ +Ġexpand ing +ĠHam ilton +ĠCon trib +.T ables +Act iv +H H +ocom merce +_ ; +Ġamong st +ow ing +ĠC old +AP H +Ġpsych ological +_t ensor +Ġpack aging +ĠSw eden +Ġp are +Ġag gregate +Ġmoder ate +_h and +Ġdesign ated +Ġdr um +Ġget User +ĠC reek +_s cope +ĠTrans fer +ĠM arg +Ġfight ers +W nd +ĠS el +ĠLa unch +Ġemerg ing +if rame +ĠAdd itional +Ġf ears +Ġsat ellite +_ : +Ġdis posing +Get Value +Http Post +AT IVE +ul ary +View s +Ġatt ending +ĠT ennessee +ĠM ission +Ġmedic ation +ĠW y +ĠAn na +Ø ¹ +ĠVert ex +.t ypes +O rgan +.DataGridView TextBoxColumn +ĠR S +Ġtemp o +( App +Version UID +.p oint +ĠD utch +H ours +L U +Ġqu oted +.b uilder +ĠPer fect +ĠAl ways +_t wo +Ġexclus ively +ĠC ra +ific ar +ĠA WS +ing ham +com plex +k ernel +Ġgr avity +Ġw i +Ġover view +ĠW ant +ĠW P +( sh +. rotation +St ates +ĠTe en +_com ponents +ì Īĺ +Re ceived +Ġly rics +rit es +ĉĉĉĉĉ Ġ +-A merican +[ num +/ python +ĠU ART +Ġapp le +ĠJon athan +Ġmoment um +ภ± +Ĥ ¹ +Ġm ich +and ra +Ġbi ological +ĠM ens +Ġ% % +else a +ĠMex ican +.rand int +Ġt ale +ĠValid ate +Ġdefe ated +.ht m +Ġcop per += / +cos ystem +Ġr ip +dec imal +.V ISIBLE +ĠT a +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉ +Ġdownload ed +en vironment +Ġnom ine +build ing +ĠSp ot +ipher al +Ġal to +qu et +ĠF T +/ get +/m aster +W IN +åħ ĥ +W est +arg c +Ġprodu cers +ĠM uch +_st orage +cred it +CON T +Ġv et +Ġvo ices +(' ', +Ġinstr uments +ĠM SG +es se +re pository +om ics +Ġdeal er +St ill +Ġb anner +asc ii +Ġrem arks +[ js +Ġshort er +g ulp +Ġmyst er +Ġk un +ĠB ird +Ġti ene +n ut +ĠU m +Ġw ise +Y eah +INE SS +_b egin +- heading +C ourse +Ġ čĊčĊ +omb ie +grad ed +ĠG PS +Ġ że +F it +c aption +ö n +/ image +l ia +(m od +Ġle ak +en za +/ H +ĠH appy +D ist +n x +ĠGovern or +(l ast +te acher +ĠS ent +s upport +ject ory +Ġ Ùħ +Reg istration +ĠGr ay +, false +Ġadjust ed +( settings +< R +ĠM age +Ġpl aint +_ )Ċ +ĉ it +omet ric +. bootstrap +Ġcar ries +I p +Ġ! $ +Ġswim ming +ĠMar io +ĠQuest ions +P ACE +æĸ ¹ +e or +}} " +Ġo ven +ĠK on +Ġwis dom +Ġac quisition +ess ment +ag ine +Ġexpress ions +Sequential Group +F ront +ul pt +aw k +'] )ĊĊ +_ AR +Ġanal og +ul in +_PR INT +ĠL G +Ġb lob +ĠFurther more +_com ponent +ĠC ole +L AN +SCRI PTION +Ġl ap +icens ing +_TIME OUT +ĠF ro +Ġli ability +Ġcom posed +.create SequentialGroup +_p erson +Ġbe am +ĉ ĠĠĠĠĠĠĠĠ +ĠNot Found +. 'Ċ +ÃŃ s +.Text View +P DF +Ġk ar +__ (' +Ġ" :" +_m essages +Ġhar vest +.h istory +> 'Ċ +-f old +æ Ĭ +ĠBet ter +Ġ"\ < +sp acing +Ġfurn ished +os er +] }Ċ +Ġ$ " +p ull +.P ost +( ip +Ĺ ı +.f ront +nt e +ĠF M +g uid +Ġnegot iations +agon al +Ġtrem end +unge on +Ad v +car ousel +ÃŁ e +_DE SC +Ġham mer +áº Ń +ĠĠĠĠĠĠĠĠ ĊĊ +-c ore +-s ervice +Ġcorn ers +ĠS F +p red +> A +ĠJ Label +Ġrom antic +Ġtestim ony +os c +ĠGener ation +as ures +_int ernal +Ġprint s +Ġ] )Ċ +ĠC leveland +re po +D isc +Ġ" >Ċ +�� �� +Ġne arest +_t b +( require +EO F +- child +Ġbu dd +.Xtra Editors +alt ies +\": \" +W ords +Ġloc ally +Ġpurch ases +Draw er +ex tract +Ġexec ut +} '. +user data +Ġfocus es +-min ute +ĠP ublish +og o +Ġmount ains +B ot +} >{ +Ġt ension +ro d +m esh +Ġtransform ed +, R +() }Ċ +.l ong +Ġg orgeous +ĠS chedule +Ġol dest +Ġsub process +( IN +y ect +ĠCo oper +arn ess +ĠMon itor +.p art +ĠN BC +Ġc otton +Ġh ol +Ġrg ba +ĠB io +Cont inue +P od +Ġparticip ating +clus ions +(By Val +à ¬ +ĠH OW +_set opt +Ġaccompany ing +at on +Ġ/ \ +ĠAuth entication +i én +ĠBar ack +/* . +Ġe ager +ĠC ancel +< lemma +ep h +ĉ window +Ġinc idents +), ( +.D es +ib e +ĠFunction s +Ġhosp itals +Ġo xygen +root Scope +Ġd rew +ĉ request +not ice +ak u +am ents +f ar +Ġprec ise +_w rapper +Ġlisten ers +A Z +.b ounds +ĠA verage +field set +_ axis +Ġexam ination +' .Ċ +mon s +++) {čĊ +ĠForm s +íķ ľ +Cpp Method +_tr ace +Ġengine er +ĠFl at +Ġrev ision +Ġhe ating +/ profile +.r u +p riority +Ġin fer +_ST REAM +Ġ* )( +> $ +OLE AN +OK IE +IB ILITY +U AGE +ĠSur vey +Ġres ign +w ing +Ġsecre ts +Ġch ips +JSON Object +Des ktop +_SY MBOL +(res ource +ĠĊ +Ġnew est +ul i +Ġdes ert +Ġd ip +ĠP ow +Ġequ ation +Ġposs ibilities +ĠF ed +os ph +Ġ[ % +Ġb ubble +ether lands +Ġc ement +. auto +_ AN +âĢĻ . +se lection +ĠB ond +D en +- O +.get Type +.W indow +p res +Ġsw inger +" })Ċ +Ġp ip +Ġm ice +Ġcomp ound +- plugin +ik o +Ġcent uries +ic ular +-in line +ĉ key +> \< +EN SION +Ġ[ čĊ +Ġprecis ely +Ġét é +ĠP ast +ĠCam bridge +-f ull +Ġanaly ze +ĠSte ven +Ġn em +d ue +ore n +Ġmus cles +ij ing +/ - +ĠKenn edy +R M +oss ible +Ġact ress +Ġd olor +å½ ķ +Ne ed +.t oggle +ĠR ace +w ers +.m aterial +ĠD ue +ĠP el +# print +Ġindepend ence +ex us +Sh adow +Ġenc oder +( level +ĠSw ift +.d oc +_se lection +Ġserial VersionUID +Label s +Ġperform ances +.T ag +ĠN HL +iz en +/ UIKit +_CONT ROL +Ġearn ings +ĠAl t +_H ANDLE +C tx +Ġpers u +Ġtr an +ç ¨ +_CH ANNEL +Ġsatisf action +ĠG P +io x +m itt +land o +Ġp ig +inal s +ê ncia +S urface +ĠU UID +Ġbenef icial +Ġsequ ences +ĉmem set +Ġmag ical + « +Ġw orn +AS C +pop up +COM P +_b efore +en ess +U i +L es +.re quire +.Serial izable +add Gap +Ġauthor ization +.py plot +urr ay +lat itude +fr ames +aj s +Ġcomp ass +Ġobserv ations +_s up +.en viron +Ġtri ple +ĠRub y +Ġdr ain +_F ILTER +S an +UM P +Null Exception +ĠG ab +ow e +ĠTurk ish +_se quence +ĠGr ant +uel a +Ġw o +Ġc ube +i q +Ġdis orders +Ġextra ordinary +Ġc trl +ĠSe q +ent r +Ġsan ctions +uts ch +Re ports +Ġin herit +Per iod +Ġphot ography +ĠF ramework +Ġspecial ist +Ġ? ĊĊ +_ selected +.P layer +Ġal location +( account +Ġstruct ural +v able +- offset +.App CompatActivity +аР¼ +.Add WithValue +Ġicon s +Ġshut down +_l ow +ĠCom pare +ĠC e += head +l am +.p redict +_DE C +ĠS leep +ĠGr atis +Ġsuggest ion +ĠD EL +ca ff +av irus +No thing +ŀ ĭ +Ġwides pread +Ġmechan isms +Ġtext Align +occ up +ĠR ail +: NS +Ġf iber +Ġm k +Ġv intage +-l ong +.re duce +. Entities +( record +Ġple asant +FR ING +.C ells +OT T +ĉelse if +_con firm +ĠView Group +s ym +Ġpr ay +Ġsus pected +Cont ains +Ġb orders +Ġcomponent Did +ASS ERT +Ġinf inite +- order +Ġh ello +ĠGr ade +.currentTime Millis +apol is +z h +ĉ Object +: \\ +H O +val uation +Ġvoc ab +Ġcou pon +atab ases +.Get Type +L earn +] =" +ĠG ary +ot ive +Ġas h +Ġb ib +XX XX +Ġbal anced +VAL UE +ĠN at +_A d +< E +åĮ º +ĠMethod Info +L IB +Ġconsider able +ĠInd ustry +test s +.set Title +ĠBl uetooth +Ġm apped +ĠBru ce +ĠMain Window +ĉ status +Ġr az +ĠM and +Ġclass ification +Per missions +Ġ---------------------------------------------------------------- ------------ +Ġcontain ers +: set +_x ml +Ġwh ilst +Th rough +Ġval ign +Ġworld s +C ORD +ED IA +ÑĢ ов +Ġsp are +ĠH ad +ĠDE F +(p tr +Ġwarm ing +ठ¾ +Ġcons ensus +ag ne +CT L +Ġì ķ +.M ain +web Element +Ġp ist +Fl ash +App end +.tw img +T ap +Ġveget ables +al g +.s ample +Ġcoach ing +( ind +Cell Value +Check Box +ĠH ell +RO OT +Ġst adium +Ġinvestig ating +) % +st ed +ĠW riting +Ġê ² +Ġun o +Ġ{{ -- +Ġco ords +Ġun ser +organ ization +ĠCr ime +ĠDemocr at +Ġv in +/ file +- api +ĠA y +Ġfund ed +ĠBre xit +ĠG h +ent ina +c ases +Ġd ash +Ġ!! }Ċ +H I +Off ice +Ġcapt ain +Ġwor ship +\ C +Ġglo be +_ board +Ġbab ies +Ġconsec utive +Ġenh anced +ere um +ĠAd vis +Ġgr ain +Ġc raw +ancell ationToken +. alpha +_W ITH +ĠO tt +ĠC ool +.b atch +Ġver ified +(c allback +Ġreg ards +ĠInt Ptr +ouch er +Ġk in +Ġtou ched +it Ãł +ath on +Ġadj acent +Ġaccom panied +LE AR +Ġim plies +Ġh ill +ĠBalt imore +=" - +Fin ally +S am +ic opt +Ġs od +Ġm aj +ĠSh ipping +Ġget All +Ġcoach es +Ġdon ations +il ot +ĠT ar +c err +Ġbad ge +Ġmark ers +ĠR and +ais ed +iss ance +Ġexpl oring +uc ed +ĠIndones ia +Ġbene ath +Ġmagn etic +Ġm useum +match Condition +Ġdis rupt +Ġrem ind +ĠT M +Ġ/ >< +Ġf ool +Ġes k +.N ull +ĠD ies +_OUT PUT +_TYP ED +Ġpaint ed +Ġsoph istic +ĠB ear +* n +_P ACK +Ġdeliver ing +ĠC OUNT +åį ķ +Ġj eg +-c ar +f name +Ġr anging +ĠN eg +/ ******/ +ĠCH AR +Ġul tra +Gr ad += t +Ġjud ges +ĠD ise +ann ers +Ġsc al +_c al +ĠCON NECTION +_ embed +(f n +ĠC raft +ĠP as +") -> +.con vert +.res ource +ĠST ATUS +ô ng +ĠT it +Ġclass room +ĠArch itect +ĠK ings +Ġstead y +/* !Ċ +ĠG ene +) ";Ċ +ic ia +st an +ĠCon struction +um per +w c +ĠC BS +ing ing +-p arty +(d river +M ARK +Ġn ested +ew ard +Ġdepend ency +Ġm ales +ĠO NE +ĠProdu ction +][ $ +ãĥ¼ ãĥ +_LO AD +ĠB ol +el ry +ł éĻ¤ +ĠRe quire +Ġpl acing +xx x +CA LE +Ġth umb +Ch oose +Ġprot otype +VO ID +Ġles bian +Ġtra its +Sh arp +Ġconsum e +Tr uth +Ġaction Performed +ĠEnvironment al +ĠDe an +Ġest ado +s ame +Ġnumer ic +Ġtrans it +. Email +-s ide +_R UN +ĠVill age +_OP EN +è ¦ +.re m +-w arning +any a +Property Changed +Ġ(! _ +( check +il ia +ĠSo ft +st eps +ĠMad rid +Memory Warning +Ġhand lers +Ġexperi encing +Ġins pect +button s +Receive MemoryWarning +chem y +Link s +Ġur llib +.System Colors +ĠE igen +Ġpun ishment +:UI Control +bar a +- set +Ġ}čĊčĊ čĊ +Ġtoler ance +Ġinter faces +. redirect +ighb ors +cs rf +_back ground +. Utils +_H T +ĠInter est +im os +Ġgr ants +Ġexam ined +Ð Ķ +Ġc f +for ge +back s +ĠObject s +_s ent +. entry +ĠTH EN +ell ido +c ia +, res +/std c +. nd +( Int +ĠAuth ors +ĠApp CompatActivity +' { +Ġmed i +M usic +ig m +ce ipt +Ġa uss +Ġtarget ing +ĠKe ys +h n +: ]Ċ +Ġmin eral +à ® +.c a +om ed +Ġshe ets +Ġc amb +Ġdead ly +.in ject +( unit +ĠSe lection +.g ms +( connection +Ġ$ (" +é mon +ĠCurrent ly +pt e +_path s +le af +Ġimp lications +pos al +ä½ į +[ / +anc ia +é Ľ +m ul +c ie +Ġge ile +im als +UI View +Ġs urre +serial ize +IS O +Ġarbit rary +Ġsock addr +.f n +ĠM erc +Ġcast ing +Key Down +Ġnew Value +op ens +T odo +Ġflex ibility +ĉĉĉĉ ĠĠ +V elocity +ú n +row ing +Ġcomput ed +` )Ċ +st atement +Ġr i +_c art +L ow +trans fer +.n av +Ġgr ave +ĠDo or +ĉ alert +.sub scribe +- profile +ĉb ase +ĠâĪ Ĵ +__ ĊĊ +Ġengine ers +Ġexplos ion +Ġd ari +ĉ Log +on al +Ġisol ated +{ i +ĠM sg +F uture +Ġrac ist +-w rap +ĠV ers +b org +IS ION +Ġ ÑĢаР+ĠY an +init With +Ġn omin +( empty +ÃŃ n +ãĤ ¤ +ĉ width +Ġch amber +/ ajax +EM P +Ġnec es +iv os +log ic +*) & +cript s +Row At +ib lings +Ġe ars +Ġcomput ing +Ġm aker +ĠNe ither +b readcrumb +Ġserial ize +ĠWith in +Ġd ell +_TR ACE += a +Ġwish es +-in ch +ĠD or +Ġinnoc ent +ĠD ol +Ġint ens +for ced +ĠB IT +Ġphotograph s +Ġcas a +ĠL en +\F ramework +.S imple +Ġde ar +)/ ( +ip pi +Ġown s +Pl ayers +Ġpropos als +.p i +us alem +D amage +Ġcal ories +ĠCreat ive +Ġ[ $ +Ġ// čĊ +And View +è me +.c ustom +_f actory +command s +_lo ok +Ġstr cmp +Y N +a ired +Ġaud it +о ÑģÑĤ +ĠRe verse +ropri ate +et ics +< vector +.s elenium +. or +Ġpred icate +Ġfinish ing +Ġk le +ĠRep os +ĠK han +ĠM aking +ĠF S +Ġp ute +ĉ state +_S UPPORT +' - +orient ation +Ġexist ed +atur a +Ġexpect s +ĠSh adow +Ġorgan iz +å ŀĭ +Ġsusp ension +Ġu it +Ġsimult aneously +ĠAff ero +: ");Ċ +Ġro cket +c as +eter mine +ace ut +x l +ĠA MD +( graph +ass oci +_C R +.ar ange +(j Label +Ġbe ef +Qu ick +.c ard +] ): +- gr +.G ONE +_C LOSE +ĠNe v +ÃŃ as +Ġste pped +ĠFre edom +ĠW R +NS Array +_r x +_d ialog +Ġhot els +Ġ( \< +ĠD iamond +Ġassum ption +um i +( items +č ččĊ +æ³ ķ +Ġn el +Book s +åİ ¿ +us b +ĠF IN +æ ¬ +Ġcorpor ations +US A +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +.p roperty +ew ise +_ plot +"> ';Ċ +Ġpe pper +Ġsh ed +ĠMed ium +ĠC ookie +Ġoverse as +ed or +asure ment +åŃ ĺ +Ġ' .' +Ġph p +ĠPRO C +Ġexception al +( th +ĠJ et +Ġoccup ied +.set Image +ĠRel ated +uck er +M embers +PR INT +ĠG lo +_V IEW +} ",Ċ +Ġad option +[] )Ċ +ĠMiss ouri +ĠLin coln +eral d +Pop up +Ġf ate +- bootstrap +fe ctions +ĠP oll +_ARG S +in ance +-h ome +. ), +_d one +: ĊĊĊ +Ġdiscuss ing +ĠSQL Exception +Ġelect ro +ĉ req +Ġz w +Ġl ui +Ġover night +$ user +ĠW AY +Ġall erg +Ġdisappoint ed +Ġradi ation +Ġimpress ed +ific ates +Ġto b +CL ASS +Ġc uda +_d et +- post +ul u +Trans lation +-h and +.y ear +ĠM ongo +Ġun clear +. engine +WEB PACK +r ices +_AC CESS +Ġh olidays +per cent +.Id entity +ĠG ov +Ġpassion ate +!! . +ĠGree ce +plus plus +')) ; +G P +Ġexc it +.tab Page +_ cond +Ġspons or +M ODULE +_pro c +Ġ$ Ċ +Ġr ational +.T ool +Ġi hr +cc a +åĵ ģ +ĠE state +IB UTE +Action Performed +ĠS olar +¦ Ĥ +Ġequ ity +t id +Ġrec ip +.s imple +m k +ĠL uke +ĠGuard ian +Ġenc rypted +Ġdomin ant +. place +ĠN V +Ġtong ue +( Get +Ġst ainless +.P lay +Ġe b +ac i +.b uffer +readcr umbs +Ġvacc ine +p rom +Ġuser Info +Ġsl ug +Serial izedName +-w ide +Ġre actions +ĠY ang +ĠAdd s +(user Id +Ġpl ates +ĠM EM +Ġb ail +In side +et ed +Ġels if +Ġs ake +Ġc ycles +Ġì Ĺ +ĉ I +-c ollapse +ĠG MT +De claration +Ġg ros +Ġreach es +Ġcust ody +Unt il +t u +ĠCh en +Ġn x +( addr +ĠO ffer +Ġcol leg +ass ador +Ġm apper +ĠS IGNAL +ĠB loom +ĠH oll +ĠIm per +-d es +_s ite +Pro c +E qu +Ġat omic +ĠW oman +s ent +sc ar +Ġint elligent +ĠGet ting +ĠReg istration +ĠPh ill +Ġkill er +unic ode +Ċ ĉĉĊ +ĠJac ob +ĠCon st +Ġloc ate +Ġca us +ĠSch olar +Ġconstitution al +Ġinfl ation +ĠG ot += array +end um +Ġtransl ated +Ġdiv orce +En tries +Ġs or +ĠQu ote +irl ines +U K +Ġexc el +( opt +ĠAD V +,: , +Ġcontact ed +ĠD A +Ġr ings +ĠIndust rial +.get Context +Ġforg otten +ĠT an +Ġp ants +Ġo v +Ġdec oder +ĠPart ial +Ġv c +Ġbatt les +A rial +FRING EMENT +ir ates +, w +aint enance +ĠO d +ĠTechn ologies +åī į +ĠCar ter +.find All +N ome +B en +ĠUs age +ĠP icture +Ġbad ly +_p anel +Ġpat ent +ĠProt ocol +lot te +ĉ player +je ctions +Ġd ou +_re lease +urn iture +_t ax +ĠF ields +.d ataset +_m aster +CLU DE +ĠPh arm +b st +Ġoper ational +.c ell +Ġident ifying +Ġj wt +t uple +ĠT C +ĠC ro +ix map +- components +gener al +Ġo z +_D e +_d ouble +ĠTo o +.View Group +g ate +d ings +ph otos +Ġgrand e +ol lect +_l in +Ġaw ful +f ilters +Ġaltern ate +es p +Ġcomp ress +e o +ĠS cale +Ġind irect +Ġinv oice +ĊĊĊĊĊĊĊĊ ĊĊĊĊĊĊĊĊ +Start ing +ĠPl ayers +ie le +. then +Or d +ĠT uple +Ġb out +ĠStat istics +Pre view +Ġp uzzle +ĠW idth +ST ATE +Ġover lay +ĉ on +Ġin fr +Ġsm allest +lock ed +ÑĤ о +ss l +Ġde emed +Ġs co +re ck +Ġj Button +Ġmiss ions +ç§ ° +.Selected Index +T ABLE +Se pt +Ġacknow ledge +Ġstrt otime +ĠT ell +ĠD ak +Ġal uminum +Ġf ence +ĠSt ars +CON FIG +Ġretro fit +Ġemph asis +/ header +ĠS omething +in ished +=' ".$ +ĠValid ators +Ġpol ar +section s +.as px +Ġas pir +.M ock +Code Gen +Ġpe ut +Ġaccept ing +Ġback ing +P icture +/ ap +еР³ +_SE C +- use +annot ation +Ġcogn itive +Ġg rip +h our +ĠLeg al +Ġep ic +.t oolStrip +.not ify +.L ast +OR IZ +M iddleware +cri ptions +l ash +_F OUND +ĠLiver pool +Ġ{} ", +Inst all +Ġn it +Ġfig ured +[ len +.W in +.pl atform +Ġgam bling +(d t +av ery +ĉ include +Wh ether +R outing +Ġther ap +Rem ote +ĠL oss +y ll +Ġappro ached +ĠV ehicle +ĠAl pha +Ġvoc ê +ans wers +NS Dictionary +cons ider +un used +ĠF an +or able +f re +ĠDIS CLAIM +ĠAct or +. ] +to Have +.user Id +Ġspeed s +ew ay +Ġrec urs +ĠÐ ³ +_pr iv +! âĢĿĊĊ +Ch oice +Ġsett le +Ġplan es +' }, +T om +IT ER +! "Ċ +å » +achel or +Ġsepar ation +Ġd al +ad j +Ġreg isters +r iz +ĠNot ice +Ġl u +Ġcour age +Ġax es +cell ent +.as ync +Ġcompat ibility +ç « +Ġ! ĊĊ +ĉ title +Y LE +ĉ message +U UID +OLD ER +ĠH H +ĠStyle Sheet +Ġaccess ed +. validation +t asks +Ġpoll ution +.c anvas +Ġing redient +ĠC abin +A h +old own +ĠNO I +ĠÃ Ĺ +[ f +ed uc +y alty +(n ot +_ State +am en +Ġda o +ud ad +ell ers +} & +lic ity +_W INDOW +Ġt atto +val or +.R ange +Ġrefer enced +ĠRes erve +M oney +SCRI PT +/ product +cho ices +Ġt in +ãĤ ĵ +Ġsepar ator +Ġp kg +am med +ĠM AT +! !ĊĊ +Ġr aid +Ġmotiv ation +ĠX P +ĠBack ground +ĠQu aternion +.define Property +ik er +ĉp arent +ĠOrigin ally +ant age +ĠH ans +Ġtim eline +.c ur +op ic +ĠSe qu +m ust +ĠCo al +Ġform atter +_R GB +Ġ_ (" +'} ),Ċ +Ġ= ================ +ĠF UNCTION +Ġl ng +ic ates +l ive +_ engine +Ġtown s +')) ĊĊ +ĠP K +( api +ĉs canf +pack et +.ph one +á Ģ +ĠAnd y +_N AMES +PL Y +Ġmin s +im i +Ġbr ick +Ġbl ade +.std out +}` ;Ċ +Sh ift +ĉs b +ĠCheck s +Ġphenomen on +Av atar +Ġmin istry +ro se +ĉ File +Ġtit led +( LOG +Ġg an +des ign +(), čĊ +Ġb ones +st m +ÅĽ Äĩ +ĠInput Stream +Ġvol unt +ĠSerial izable +Ġfight er +ĠDr ag +T witter +Ġsubs id +ç ¼ +Ġfor ums +.load ing +log ged +_ this +Ġterr ain +Ġir re +ĠIn g +ĠC N +_object s +. uid +Ġconscious ness +T INGS +ĠG all +Ġport ray +ĠDevelop er +Ġparticip ant +Ġ" ;čĊ +/ model +ĠOper ations +^ \ +ĠL ater +Ġrais es +-n one +.m eta +=' .$ +Fin ished +Ġrepl acing +Ġsam pling +ĠJ en +" There +RE AL +A LE +ìĬ ¤ +Or ders +_param eter +ĠOlymp ic +Ġtr ès +Ġare na +i ol +; ?> +Ġimpact s +ĠW S +: get +Ġfl ights +ĠRuss ell +c amera +F n +s igma +Ġfor cing +Ġloc als +Ġdepart ure +Ġcelebr ation +ĠS ay +ï¼ Ĵ +ĠH ills +.has OwnProperty +Ġtyp ings +.A PI +Ġdon ation +Operation Exception +.Act ivity +c plusplus +ĠChar lie +Ġimport ed +Ġd ann +Ġoccas ions +Ġimplement ing +Ġpur ple +.d ialog +SQL Exception +ern o +Ġw ars +Ġpast e +Ġdecre ased +Ġhar sh +Ġel abor +input s +ĠView s +Ġerror Message +_m ul +ĉ write +ĠC op +ĠAnn ual +(b utton +Ġv ida +b ars +ĠHar vard +ĉex pect +Ġindex es +Ġdocument ary +Ġf lesh +OR LD +ĠD elta +M AND +Br ush +-c olumn +Ġdevelop ments +method Visitor +s lice +ĠP DO +Ġinvest ing +ir able +Ġxml ns +ï¼ Ľ +art a +Ġthe ories +_c ity +Ġ$ __ +Cre ating +( pr +D ropdown +ism atch +ĠN ET +'] )){Ċ +ĠVal ues +ĠSE O +ĠST AT +Ġe cosystem +Ġtem pt +Ġ\ \ +Ġ// {Ċ +ĠChrist opher +ĠKent ucky +ĠHttp ServletResponse +Ġhy brid +y on +Ġfeed ing +ĠEx tra +N orm +IT CH +ĠSe an +ĠUp load +m un +p ur +Ġp ersistent +ĠID C +ĠPer form +.m erge +_ room +Mean while +! =' +ĠW el +Args Constructor +.D atabase +Ġcount ing +() * +Ķ åĽŀ +ĠT OP +m ill +ĠD T +IGN ED +ĠK B +Ġcomp ly +S outh +_c ollection +Ch apter +Ġexpl aining +_ AM +_t s +c ards +Ġqu el +Ġp ole +Ġtouch down +ĠO thers +Ġpe ers +ĠType Error +Ġsix th +Ġche er +Ġdis pute +us c +) ], +th umb +Ġh iding +ĠS IG +lik es +ĠP AGE +.Ref lection +Ġhead quarters +T ING +ĠG host +M LE +$ Ċ +Ġcontr ary +ext end +'] ). +FF ECT +ĠP interest +úmer o +ric ane +ĉs ession +Ġcr ystal +- Control +overn ment +og raf +- action +v olume +ft en +Ġun con +Ġan imate +Ġle ase +sc r +Ġref use +ãĢ ĭ +ft p +in formation +Ġeval uated +Ġin jection +Ġj ack +Ġwork shop +æ³ ¨ +PT H +ĠT s +off er +ĉ os +Ġking dom +M issing +Ġlaw makers +ext Field +Ġsing ing +ab i +/ client +.m edia +ATEG ORY +Sign ature +% ',Ċ +ĠF uck +][ : +Ġsens ors +/ com +ĠPr imary +.S QL +_pro gram +Ġp ills +Ġinteg ral +Ġfle et +Ġdro pping +.s l +Be en +Ġp ets +Ġadvis ed +Ġdr agon +_ EDIT +( im +F ER +ĠDr ug +(r andom +Ġcomp ression +ou st +[ % +Ġbuy er +h op +R oles +man age +Ġpain ful +ĠBr anch +-mod al +en ant +ĠM esh +/ font +ĠG raham +Ġâ ĺ +Ġn c +ĠFranc is +Ġspec ification +Ġdam ages +- config +Ġthe oret +sec ure +_m ulti +aceut ical +Ġdemand ing +en ne +IST S +() ));ĊĊ +Re ason +Re cent +ph ase +Ġps y +_M AN +Ġvolunte er +å ¿ +istrib uted +li o +Ġproduct ivity +_com m +S pring +n is +. weight +ĠC ancer +Al loc +ĠT weet +Ġsepar ately +ĉ check +_p roperties +. Unit +_CL K +Ġg t +Ġ( );ĊĊ +Ġhand y +ĠThom pson +Ġunn ecessary +ĠRe ader +G N += request +ĠU tility +.Re pository +ĠA x +hy dr +ie u +Ġth y +Ġl t +_m ail +ä¿® æĶ¹ +ail and +ĠPhil ip +Ġbit ter +Ġbet ting +Ġtim ed +ock s +' a +Ġal gorithms +Ġre interpret +Ġto ss +ro gen +Ġhop ed +( selected +Ġvent ure +TE X +ĠLe ave +.Sub string +Ġgr ateful +uk a +ĠCon sumer +Ġag greg +C ircle +ภģ +_block s +Ġleg ally +Ġ" | +ãĥ ĥ +. board +.A b +Function s +rec ipe +è ĩ +ĠO xford +Ġwho les +.B uild +_ch anged +h ai +Ġdepart ments +I mp +Ġcoal ition +IN FRINGEMENT +Ġemp ower +itch es +N orth +Ġinfl amm +ON SE +Ġmiss ile +ĠR aj +ĠIss ue +Ġat oi +ca led +.Cont rollers +ĠW olf +Ġcrush ers +á» ĩ +.A uth +.add Attribute +h is +Ġbo ots +.c lean +c amp +Ġten ant +Ġt une +Ġ{} '. +Ġwork out +Re po +Ġpartial ly +MI SSION +j amin +ĠS B +Ġdetermin ation +Ġ' ');Ċ +ĠB eng +Ġv os +Ġin hab +/ lang +s burgh +Exec utor +h one +ĠCh allenge +_link s +.Le vel +Ġunder ground +-c ode +Ġoptim ization +log ging +_de st +Ġsn ake +Ġchemical s +_IMPORT ED +ado op +ĠTH AT +man aged +Ġredu ces +ĠRE AL +ĠG uy +_GENER IC +/ ******************************** +. amount +Ġd ere +get Time +Ġp ant +an onymous +Ġharmon y +ĠAl an +Ġscen arios +Ġd irt +ht ags +M c +Sh ell +r in +{ čĊčĊ +.p ow +ĉ client +Ġconspir acy +Ġad mission +ĠReg ional +ĠView Controller +ĠPhilipp ines +Ġde pos +Ġp ap +ĠP ad +P aul +.Com boBox +Ġt utor +ĠRec ipe +w riting +Ġcontrib utor +OT H +Sm all +V I +Ġh acer +e qu +ĠEx amples +h uman +.m essages +ĉt yp +Ġ( čĊ +ĠS SL +LE N +ĠRom ney +( grid +ĉ min +Ġ> ĊĊ +Ġfr uits +Ġvot er +In line +pan e +ĠC ollections +char set +Ġsp am +z b +item ap +Ġsucceed ed +_C OL +Ġel apsed +im eter +Ġrecover ed +T ensor +hatt an +.set up +ist o +( head +ĠS IZE +Ġtact ics +Ġdist ur +Ġpre val +ici os +( Value +_c ols +ĠF at +Ġse al +Ġs ons +Ġens ures +Ġpress ing += & +igen ous +Ġharass ment +_ JSON +Ġign or +yn omial +om er +_st atic +Ġsignific ance +Ġcirc les +_S ystem +Ġdiscipl ine +Ġdress ed +Ġs phere +Ġclim b +_ actions +ĠB ab +Ġ' =', +_s chema +" use +Ġund ers +Ġc ups +.s creen +/ new +Ġappe aring +T OP +vis ed +cl ang +Ġinvestig ators +Ġmyster ious +Ġprom ising +Ġqual ify +Ġc ave +Ġequ ip += x +G T +( link +. velocity +. erase +ot er +++++ ++++ +pro fit +Ġz ones +_ uid +- ser +Ġobject ives +Ġmil f +web kit +(m atch +ne h +ĠAssoci ated +ĠT odo += d +C am +Ġv ocal +Ġs udo +( EX +Ġtr ou +AB C +.b ean +ĠG round +ĠRE ST +we ets +In g +im on +_b us +ĠC OLOR +un to +Ġf oss +ĠLink s +ä ng +/ forms +pr ises +Ġachie vement +C ALL +ел ÑĮ +ĠVer ify +_S OURCE +apt cha +ID D +_re ference +G old +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĊ +Re ceiver +Ġa j +_d irection +} ] +ĠCom pet +Ġb ang +ĠC ass +- url +te chn +ĠJer usalem +long itude +' );čĊčĊ +Ġwin ners +T asks +ĠD MA +Ġtool tip +İ · +ĠB ra +_d uration +cur y +parent s +---- >( +ĠK ir +Ġint ros +Ġsk etch +Ġsk illed +Ġim mer +Ġade quate +_re p +( header +_ like +Ġper ceived +ss h +Ġassum ing +Ġf f +_u uid +ul as +Ġdemocr atic +. entities +S eries +aph ore +Ġnew er +} ( +SE C +ai ro +Ġcomm od +Ġprivile ge +Ġde ux +ĠH op +.' / +ct ic +. ';Ċ + C +ĠWar ren +Ġoptim izer +ĠSER VICES +_ oper +get Attribute +ĠMc K +_s elf +.r s +" )ĊĊĊ +Get Component +er ce +Ġt ous +un its +'] );čĊ +Z oom +/ E +Ġobs c +Ġfast est +on line +Ġpeace ful +ff en +Ġc argo +ĉ pr +Ġseek s +z u +Tr im +Ġw ard +Ġver d +Ġblog s +.exception s +ĠPrem ium +ĠN etherlands +S afe +Fin ish +ĠAl bum +_A CC += this +v irtual +] > +_L ABEL +ĠN ich +_w in +ĠA aron +W P +; $ +aim s +ĠImage View +Ġend less +ER A +_DIS ABLE +Ġcancel led +- us +Ġins pection +em in +ĠG rey +- open +Ġiter ations +. owner +Ġk eras +.P assword +ĠR y +ĠIN S +A ir +ĠSe veral +.Tab Stop +ING LE +ĠH air +ĠCan vas +AA AA +Ġfl aw +ced es +.Re port +í Ĭ +ĠT ips +cript ors +.trans action +.S pring +Ġview er +Ġins ights +è¾ ĵ +ord ion +U INT +se ek +ĠA uf +ìŀ IJ +Ġstr ain +To oltip +Ġd z +ign al +ad t +Ġu c +fin ite +Ġn m +.c md +ĠMy Sql +[ data +.j ackson +.t ree +Request Param +_ agent +") ]čĊ +Ġass ass +( Constants +: ss +ĠM AN ++- +- +ĠB ottom +print s +ĠS ame +@ Autowired +sw ap +ici ón +Ġprotest ers +Ġh oney +ĠV eter +(C alendar +- ad +ĠBrook lyn +L ife +_V AR +ze ch +ĠC ALL +_C AST +ĠE lection +Ġthick ness +V ery +_IN TEGER +- dev +)) )) +ap at +oo oo +d emo +Ġparse Float +ĠR ather +ST IT +m aker +[ current +chron o +Ġch rist +ãģ ª +ĠD etail +Æ° á» +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġs ul +id ency +Q ue +Ġeleg ant +ap ons +Ġdish es +Ġinteg ers +( read +find ViewById +ĠAm ount +ĠSk ip +Ġhab its +* )( +Ġmon sters +M AC +: end +Ġfr ank +As sembly +Ġd fs +Ġne ut +_TYP ES +e qual +loy d +( uri +Ġch i +Ġdefend ant +Ġconflic ts +Ġv il +- js +ĠPe ace +Ġmut able +) sender +ĠF ocus +å» º +Ġapprec iated +s leep +ĠR ED +C ulture +Ġdesign ers +_g enerator +c odes +/ ex +.Get Value +umb led +.scal ajs +per or +Ġveter ans +Ġ} )čĊ +Ġun fortunately +_C REATE +M ass +ĠCL AIM +ĠMe et +_s upport +B ank +() .Ċ +D ark +_LO W +ĠMin ing +ĠO wner +ier a +Client e +Ġencour aging +> S +Ġboy friend +ĠH alf +ĠA CC +A ff +_ ar +-l ife +c x +.J Button +iz ado +.z ero +.open qa +ot on +.text Content +Ġto ll +at ie +Ġball ot +- number +. Exception +ĉ params +c ircle +-m ap +Ġn ap +ĠRob ot +ĠI ch +reg istration +Am azon +roll ment +( exp +Ġt anks +ĠG ordon +Ġmach inery +Ġbas eline +æ ĭ +Ø © +ĠCon vention +ĉ config +ook ies +m ult +Rec ords +ĠE ST +Ġgar bage +Ġcon form +id al +Ġb arg +Ġsurv ived +Ġinvestig ations +.contains Key +---------------------------------------------------------------- ----------Ċ +ort ion +Ġhor r +_ http +Ġm ant +] ;čĊčĊ +b inary +em pl +Ġin quiry +ĠMean while +Ġcollect ing +.Entity Framework +", ĊĊ +ĠP ic +@ Inject +ick ness +ĠB inding +Ġcont rolling +re verse +Ġch airs +semb led +( add +Dis abled +an as +.trans late +-------- ---Ċ +Ġref lected +"] ĊĊ +Ex ternal +Ar row +Single ton +% x +Ġ Å +Ġan cest +ĠOr leans +ĉc md +Ġprohib ited +ith metic +(ch annel +_c ss +For ward +.s ocket +Ġl uc +â Ĩ +ĠFire fox +ĠM ovies +) _ +. ends +( shape +Ġde alt +Ġs aves +Ġgl ory +Ġmej or +Ġbreath ing +Ġ eller +get Data +Ġang les +Ġtool bar +Ġsp acing +IP S +Ġflo ors +_ACT IVE +Ġsh uffle +/ shared +ĠE le +ed ish +Ġweb cam +.ex pect +il oc +ĠIn cludes +Ġtweet ed +Ġ: ) +ĠEss ay +F ix +-b etween +_ web +.con v +Ġrac ism +Ġreflect s +um m +иÑĤ е +_f ooter +/d ocs +ĠP our +Ng Module +.initial ize +pattern s +_ In +ĠAb b +* čĊ +Ġsent iment +b uff +_count s +Ġre use +ch unk +Ġim posed +Primary Key +Fore ground +Ġconsum ed +? ! +Ġd ick +Ġch ron +ĠF ern +Ġrespons ive +Ġin sect +icult y +Ġr w +Ġal ike +Ġsub set +ĠCook ies +ĠP air +Ġt ier +IF O +av our +ĠQ U +, sizeof +Ġmerg ed +m v +it ol +yl on +Ġjump ed +. role +ens aje +R ules +Ġb rowse +An imator +Ġy oga +Ġvari ants +Ġcour tesy +ur an +p bs +else if +Al t +ĠL ane +CL K +IM ARY +_PRO PERTY +ï¼ IJ +Ġch an +Ġgrad ually +Ġsh ake +Ġbl onde +... ");Ċ +-se x +Ġgame play +ac ies +.ref resh +US B +ĠPl ot +W as +iss ippi +ĠT ensor +Ġcryptoc urrency +Ġdifficult ies +De leted +With out +_ append +_ ver +")) čĊ +Ġhonest ly +Ġp ivot +Ġtem ps +_p s +ĠUn like +[: - +V S +_in f +Ġjun ior +Ġanim ations +Ġfile path +? {{ $ +Ġun icode +pl aces +ĠC offee +.S E +ĠP AR +(t xt +ge bra +Ġf ires +Main Window +med ium +Ġ( âĢľ +Ġl g +Ġc mp +/ base +_l ayers +_ entries +Ġadmin ister +ĠSU CH +B P +ĠScott ish +ĉčĊ ĉčĊ +gu ard +ĠStr ong +In sn +ĠC AP +as ury +ĠSE E +C lock +er ie +\ models +Ġ$ $ +ĠC ab +Ġwur de +Ġsold ier +Ġcl ips +Ġarrang ement +ĠW onder +ĠH orn +Ġsc ared +Ġc ure +m kdir +Ġal igned +ĠP ink +Ġland ed +Dim ension +Scroll Pane +.ch at +.W ith +ĠTr ain +] .Ċ +Ġth irty +Ġdur able +Ġl d +Ġlate init +Ġch arts +Ġins ult +.F atal +_ ct +Ġm asks +CLU DED +Pres ident +Ġcol ours +g ments +.at tributes +ĠF lex +ĠC lock +ÃŃ cul +im en +J O +ĠReg ex +_L INK +Ġc ouch +ĠIN PUT +Ġbe ating +b usiness +pre ced +. unit +ĠF el +N ever +osp el +.start swith +ĠE PA +. only +Ġprevent ing +y er +Column Name +Ġelev ation +fl u +icy cle +Ġoff line +Tool bar +Ġcompet ing +) ]. +Ġm og +Ġis Valid +As k +_ av +_l at +AN C +ĠJ oh +k ers +Ġgu ards +Ġch ains +ĠSimple DateFormat +.st atic +Ġvess el +Ġm ud +Ġst abil +Ġst ret +g m +am ation +ç ľ +-w ith +Ġro s +_P A +Ġresult ado +Ġconf idential +ĠTok yo +ĉ using +ĠMath f +omb ine +ĠESP N +Ġdeal ers +Ġdismiss ed +TR Y +Ġte ens +rec ords +Ġw ings +g allery +account s +_L IB +Ġj acket +ĠNS Object +Ġst ones +ĠDel ivery +ĠD iet +/w atch +Ġto ilet +ĠG uest +.d ay +Ġint val +Vis it +Ġinvestig ated +Ġpent ru +ĠThe atre +andid ates +L ang +ĠS erv +Ġcont rollers +Ġset Title +N P +am y +fl at +( ui +_d ocument +è ĥ½ +ĠC oin +ĠAd ams +pt ic +Ġproduct ive +Ġaccompl ished +čĊčĊ čĊčĊ +Ġdefer red +ient es +Ġs inc +ol ars +Right arrow +Ġvari ations +( offset +.Layout Inflater +Ġsus pend +Ġprevent ion +_pr ivate +_ js +âĺ ħ +Ġw ieder +at um +Ĵ Į +Ġappear ances +.D ocument +Ġvalid ates +cal endar +} ";Ċ +.d emo +con ut +Ġcorre ction +ĠDe al +Ġbatter ies +.d uration +, \ +_m arker +m ulti +Ġh alt +Ġc ms +Ġsh aped +B ro +re duce +Ġ #### +CT OR +ĠBen ef +Ġicon ic +Ġp iano +Ġeffect iveness +| .Ċ +Ġa jax +Ġv olumes +ภ¡ +Ġcl js +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ath s +ra its +å¤ § +Ñ ĸ +_m ult +Ġfasc inating +A verage +Ġpr é +ĠChair man +.find Element +_p in +Ġcomp aring +Ġdark ness +-F i +- server +Ġselect ing +ster dam +ĠPart s +FORM ATION +Ġnot ing +Ġp ile +og s +Ġpa lette +_d o +it ize +() ( +Ġdef ining +Ġremain der +Un its +_T ASK +Http Client +S ocial +Ġfund ra +N R +ch est +C urrency +.ad apter +Ġd op +un ting +ANG UAGE +" He +ĉ index +_p ackage +.I con +Ġrep et +m ass +=" .$ +ĠS ud +Ġl id +pro vince +ì ľ +G PIO +Ð ļ +ĠMy SQL +Ġdoc s +ĠG A +Ġip sum +K ernel +Ġaccept s +Ġfit ting +Ġcu ando +Ġd uplic +ĠBro ther +ĠK le +num s +Ġmor ph +Ġ ######## +ĠCG Point +< unsigned +ä¾ ĭ +ĠD uke +.set Bounds +q s +or ic +j er +Ġregard ed +Http Request +Ġbond s +Ġthorough ly +enc ent +Ġhighlight ed +Ġac res +Ġwork place +ĠL ux +Ġqu ot +.in flate +Ġdocument ed +Ġadd iction +Ġmut ation +.c ity +Ġbott les +ĠRepos itory +on n +err no +ARI ABLE +åº ¦ +_B EGIN +gl as +' })Ċ +ĠMass age +ĠWh it +reg ex +W A +Ġout let +- head +Ġexp ired +ĠTh ai +/ include +grad ient +scan f +Ġse am +w al +ĉb uf +B earer +Ġprec ious +if acts +co ord +Ġexpl oration +.get Y +(h andle +Top ic +ĠV ent +r hs +---- --Ċ +ĠB right +Ġg uild +m other +st orm +Ġmunicip al +Ġin k +.T YPE +w l +... manual +ĠTechn ical +Ġcorpor ation +ĠH W +ank a +T AIL +ist as +Ġperform s +ĠBeh avior +.F or +_ ORDER +ĠK ick +Ġcallback s +_d r +ue go +h ub +uff icient +sk y +Ġb p +ht able +ĠON LY +ĠAUTH ORS +.Arg ument +" };Ċ +ĠTh under +ĠK om +.Sh ould +A UTH +ah u +_p ayment +Ġst arter +ìĦ ľ +ìļ © +B log +.p atch +Ġgovern ed +ass y +-f ound +Ġthe ater +ĠFont Weight +ĠBat man +" If +.R andom +_d elta +ĠC E +Auth enticated +Ġdr one +Ġc ous +r adius +M er +( None +ĠN J +_ headers +Ġam er +py test +ĠA ctions +ĉĉĉ ĠĠĠĠ +Ġet t +Ġh oly +Ġun comfort +ĠN in +ĠDec imal +ĠM essages +.s ender +] ])Ċ +Ġembr ace +Th ough +/ sp +Ġcult ures +Ġhigh way +t ar +.f ail +_h idden +ĠcomponentDid Mount +ĠW right +Ġj ag +_ il +../../ ../ +ig u +F ood +Ġa ce +Ġa ños +US D +Ġmut ual +Log ic +Ġtem ple +Ġbrief ly +ĠT rip +class method +default s +Ġch unks +,, ,, +ĠRe ason +$ id +-up s +Ġdam n +Ġtruck s +Ġun limited +Ġsc ulpt +ĠC ards +Ġaut or +ĠTest ing +Ġdies e +sh ops +ç ´ +(p ayload +ĠP ATH +ĠMem orial +Ġridic ulous +eg ree +-w inning +Ġre hab +Ġsophistic ated +wp db +ĉ path +! ";Ċ +_S YS +.s peed +Ġso ap +s uffix +W rap +Ġenh ancement +à ī +ú b +Ġplay list +Ġmix ing +ant idad +=" ";Ċ +ĠRev ision +ĠBe at +.in c +-w ay +enc ias +ul ers +C at +id el +ĠSh ip +.set Color +Ġthreat ening +.mod ules +Ġafter wards +ĠD ashboard +Ċ ĠĊ +Sign al +Ġpr imer +orne ys +ici ary +Ġl igne +_p redict +Ġa est +_ https +> : +ĠL ex +Ġrencont res +eg ral +sc ala +_f amily +ÃŁ en +_s ym +Ġuncert ainty +ĠVAL UE +Ġ} ;čĊčĊ +Ġbro ader +Ġh orses +ãģ Ŀ +ĠK al +ob a +_IN ET +ĠK ill +j query +am ination +[ @" +Ġm uj +## #Ċ +First OrDefault +then Return +C he +/ footer +Ġpark s +as je +ĠG ulf +Ġmod est +. Init +ï¼Ł ĊĊ +Ġpros pects +Ġs vg +Ġå ı +.D ialog +_N ET +Ġ( ($ +Ġe k +ĠW arning +ĠM K +< LM +Ġ' čĊ +i em +h etic +Ġi x +th ink +-sh adow +ĠE ld +ĠNev ada +ĠLe af +ĠG ROUP +Ġprom o +ent ine +ĉ Map +ĠModel s +ĠK rist +_k ernel +-m ade +Ġc err +As sets +ell ar +Ġinv oked +.v ue +Ġcult iv +C losed +Ġgener ates +ffff ff +thes ize +s qrt +ĠCast le +.c ar +Ġke en +und a +ĠC row +ĠSing h +y thon +Ġbe ans +l arg +æĸĩ 件 +Aw esome +unc ate +Path s +o ji +(c urr +CON DS +Ġm im +Ġshould ers +H ard +ast es +а еÑĤ +Ġconv ince +de cess +m ade +ĠC MD +. Im +Ġcha os +ens ively +Ġcool ing +Ġbur ied +(' @ +_S e +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉĉĉ +.com pany +.sub mit +ph ant +Ġboot strap +_h elp +à § +.d ump +Ġdif er +_m apping +Ġcirc ular +Ġescort s +Ġb ere +Ġgrad u +ĠLeg end +im edia +ĠBar celona +Ġbed s +åĪ ° +ãĢ Ĭ +_v olume +Ġtremend ous +Ġsc aling +Ġp ins +en as +type param +D ashboard +render er +Ġsp i +Ġ& $ +ĠSk in +alm art +Ġh ockey +Ġ'" .$ +Ġerr no +Ġb ew +Follow ing +.M odule +er able +ĠM ilitary +ĠR io +_ available +ĠSur face +Ġst ab +IF IER +ĠL IST +Ġd ashboard +Ġcl usters +.pl ugin +Ġj ou +ĠDec or +F our +Ġdel le +****** /Ċ +ia z +in de +ch ing +Ġget Item +.Add ress +ment ed +A meric +Pl ain +Ġus b +ĠPract ice +_ ment +.bl ue +H int +ÑĢаР² +Ġconn ector +Ġinher ited +и в +Ġinterval s +Ġc ere +Ġu d +Ġin con +.Ex ists +ĠM ic +F K +(c ard +.Set tings +Ġexhib ition +Ġon Pressed +Ġrest ored +eng u +. def +Ġrec v +." );čĊ +enc oder +ather ine +( dest +az ed +# endregion +sem bl +, M +ob y +Ġп еÑĢ +.C all +Ġattend ance +-b order +Ġaddress ing +ê n +ĠLe v +Ġb ash +ben ch +C redentials +Sp acing +( of +_RE SET +ig uous +Ġcr uel +Ġcross ed +Ġle ur +ĠG olf +or rect +Ġpack ets +ĠData Set +Ġpart ly +SEQU ENTIAL +Ġindic ation +ĠS alt +ac ia +Ġ* );Ċ +ĉ info +ĠView Bag +on z +Ġeditor ial +ĠA rena +Ġs ir +_ Static +( socket +s u +cho ose +.m onth +.M y +é ri +; font +do es +Ġcon verter +Ġsal v +Ġl r +Ġinflu enced +(f eature +ĠQue ens +let t +_M ON +& amp +Touch ableOpacity +O FF +Ġmetab ol +( iter +Ġvit amin +ĠIND IRECT +aut om +_p ublic +Ġadjust ment +Ġspecial ized +w indows +.add All +Ġaccording ly +ĠJ OptionPane +Ġcell spacing +Ġqu ad +Ġcre ep +Ġout lets +}` )Ċ +Ġpri est +_TH READ +ĠMar x +ĠBy Val +Ġc ual +éĿ ¢ +Ġtempor arily +An n +ke leton +å ¥ +ĠLO C +au er +der ive +Ġbeh aviors +as ename +ĠCent ury +Ġhor rible +ME SS +_ List +we i +P at +ĠCh oice +_F ROM +ĉ line +.in voke +.B ottom +Ġnow here +." ĊĊĊĊ +_ export +Ġstrugg led +.Ap pearance +ĠJ Button +ĠJer emy +([ [ +Ġkick ed +mar shal +st aff +es ity +Ġqu iz +_e ffect +Ġ} ));ĊĊ +m el +b anner +ĠP IN +Ġin vention +Ġcons olid +Ġop s +ĠB etween +j ack +ern ational +Ġsacr ifice +ag ation +ĠJ oy +Ġam endment +ĠS old +Ġprison ers +ан нÑĭ +Doc uments +) ])Ċ +ust ed +ĠLine arLayout +os o +_E M +.s elf +.M iddle +) // +Ġ\ ' +Ġfuck ed +ĠM urray +Ġprof ound +_E LEMENT +ult a +il ers +port folio +J une +t cp +mod ified +ĠTr ace +ĠK el +aly zer +) => +ĠRep air +_B E +Br and +u art +pre view +Ġiniti atives +run ning +b ang +ĉ update +ĠCo ach +R ich +Ġy outube +Ġrit ual +app a +ĠRobin son +prec ision +//////////////////////////////////////////////////////////////// //////////// +=[ ]Ċ +Ġcelebr ated +OT O +Ġin clusion +J P +' ;čĊčĊ +Ġnot able +(_ . +Man aged +Ġgu ides +& nbsp +ated Route +ĠAd just +Ġcol ored +_s cores +ĠTes la +_pro gress +.in st +[' _ +.fl ags +Ġf close +_O PER +ż y +_n ote +Ġtrans gender +å ķ +RI PT +Ġabs ent +Ġam et +Ġoper and +ë © +Ġh ood +to LowerCase +av o +ĠCirc uit +ĠL ind +-- }}Ċ += m +Ġsup press +ĠM AP +i ang +- admin +Ġside bar +ĠB u +ĠH ex +, F +ĠSign al +Ġtrans parency +ĠFeder ation +/ V +Re q +Ġpul se +Ġt ends +Num bers +% ' +Ġde port +dat as +_U INT +_ tra +ok o +Ġ" ? +comp et +sole te +und ry +Ġover lap +}` ,Ċ +. ly +_sum mary +ĠL ost +.C enter +Ġdis ability +.Serial ization +Ġge om +Ġ? : +ĠW o +Ġsh ipped +Ĥ æķ° +Ġu gly +Ġexcit ement +Ġext erior +Ġcheck out +Ġk ur +, D +ĠAl aska +Ġsyn thetic +ĠB udget +ĠSub scribe +Ġ& Ċ +ÈĻ i +ĠY u +ĉ query +} .Ċ +Ġtr aged +ass en +Ġaccommod ation +Ġphys ician +Ġren amed +Ġtid ak +z Äħ +Ġmin us +ny ch +_EX CEPTION +thread s +Ġt ire +_c reated +ens ure +Ġworth y +Ġexc use +Ġclo th +.parent Node +/pl atform +ĠU FC +ĠG tk +un ny +Ġg ibt +ke ley +h um +(t x +ĉ dev +Ġout fit +do ors +Ġf on +ic ut +vol atile +Ġhom osex +Max imum +Ġexp end +Ġ});ĊĊ Ċ +E q +ond ers +dep artment +ĠPhys ics +" });Ċ +Ġpar ad +.S tr +Ġse le +IF IED +Ġdel ivers +iv an +Ġrespons ibilities +Ġadvoc ates +è µ +ĠR ID +.param eters +M etrics +ron ics +ĠUITableView Cell +A bsolute +ip se +yl um +MLE lement +_VAL ID +< title +D lg +p aces +Ġsynd rome +be ans +_d atabase +oz illa +ĠM eg +DB G +Ġl ub +Bag Constraints +ab ad +Ġproject ed +_BY TE +.Size F +st reet +ĊĊĊĊ ĊĊĊĊĊĊ +ĠLO SS +Ġdirect ors +/ news +Ġnurs ing +ĠD one +. HTTP +dis count +ĠR ot +To Many +Ġen abling +Ġauss i +ost a +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ čĊ +è½ ½ +Ġhel icopt +ĠIn side +ä¿¡ æģ¯ +is per +ĠAll ah +ARCH AR +Ġroll s +Com pare +X P +Index Of +S UM +Ġass ured +ĠPhys ical +End point +.G lobal +.d etail +Ġthe ft +.j upiter +Ġhum or +.R ender +A lex +.c ap +Ġbuff ers +Ġdis pose +t ion +.p resent +z el +, P +Ġdesper ate +.get Column +Ġtw in +ì ĸ +.c an +Ġf lee +ĠIran ian +Ġstick y +ĠU TC +L T +//////////////////////////////// //////////////// +Ġl icensing +_PO INT +ĠM aps +Ġl ol += models +-t ab +ĠN ash +_log ger +tor ch +ĠCON SEQUENTIAL +Not Empty +/ react +Ġp f +Ġassert ion +Ġsubsequ ently +_c an +Ġpand emic +og ue +"+ Ċ +_ ent +_P aram +.ĊĊ ĊĊĊĊĊĊ +Res earch +C apture +Ġbel oved +d em +Ġextract ed +Ġf ights +ER C +(a uth +position s +Ġrevers ed +(st ack +Ġ_ ) +uto ff +_fl ow +ç Ĥ¹ +( Game +Ġex cluded +ĠCS V +c g +ĠT itan +p ause +Ġcer ca +Ġdump ster +L ess +Ġkotlin x +aster xml +Ġpoint ers +Ġfl ows +ĠT un +ĠMain Activity +Ġdis cret +Ġcomb inations +vis it +_b ind +oot ing +d ater +_look up +.n io +Ġswe at +ĠR d +Ġscient ist +ĠP ixel +@ NgModule +Play ing +Ġunf old +Trans late +ĠLaw rence +ĠFIX ME +B ill +ĠR IGHT +Ġwhere ver +Ġo ok +vid ence +Ġ] ]; +ĠSk ill +unist d +ĠðŁ ĻĤ +Ġfem ales +-- )Ċ +İ· åıĸ +ĠF red +Over all +Ù Ĥ +Ġess ence +Ġthere by +Ġw ounded +ĠD OWN +les son +text ure +R ound +Ġautom ated +ĠÐ ¡ +ĠUp dates +Ġsh ade +p ublish +ĠG ear += lambda +Ġle ver +) +" +h ill +Ġrad ar +ry ing +Ġ" ). +f illed +Ġline up +Ġd l +Ġworks pace +V o +_d t +ë ² +_ Item +NS URL +. verify +ĠHawai i +G od +M arch +Ġ[âĢ¦ ] +Ġpel o +ur ious +ĠPitt sburgh +. It +C lean +> \<^ +Ġi os +s ound +"] ; +Ġfre ed +rot tle +ĠL ower +[ count +å Ŀ +Ġp ale +ĠWay ne +ear th +_c ategories +U CK +.m etadata +Ġsum mon +H OME +олÑĮ з +Ġmanufact ured +Ġdo ck +Ġcompet itors +_MODE L +ok ia +ĠH ey +Î ¿ +Ġback ward +ĠPO SS +rop a +Ġc ri +_O BJ +Trans port +-h igh +Ġerot ik +_s lot +Ġart ic +_f ramework +-ser if +ĠSql DbType +') ( ++ "/ +Ġw ore +S il +Ġst oring +ĠPh ase +u ant +Ġb ump +in ho +Ġd ign +Ġback s +q q +(h ash +Ġge o +Ġt ender +Log o +! )Ċ +ĠM X +ĠAr thur +esso a +_C h +Ġbed rooms +="# ">< +Ġth roat +ins ic +.int eger +Ġpr imitive +Truth y +Ġfacilit ate +Ġcreat ivity +ĠD NS +Ġg ra +ue z +Ġcount less +ĠPol and +' M +ĠD ist +Ġv est +Ġcert ification +á» ij +h eld +ext ensions +( static +Ġgr ades +ĠU ber +ãģ Ł +Ġ[ ])Ċ +dat os +Ġget Data +ĠCh arg +ĠB S +.m icrosoft +.v ideo +.d irection +->{ ' +l ua +ape st +Ġbo iler +ere k +Ġdec ides +.j ar +IS C +ĠW ords +(C ON +EMPL ATE +ree ze +sh ots +app s +unt ed +.set Name +:: < +-b old +ê ² +å¯ Ĩ +Long rightarrow +Ġunf air +Ġear ning +Ġsh elf +URE MENT +Ġid le +_M ENU +.C ustom +AG ER +- " +_s witch +b ecause +) view +m are +_ condition +ĠStart ing +M vc +(p re +d ump +_LO CK +at etime +.c allback +ĠC er +op ol +ib rary +Ġres ervation +ĉĉĉĉĉĉĉ Ċ +lect or +grad uate +Ġgener ous +Ġ ion +ric ao +m q +_com plete +(c ursor +ĠForm Control +: center +Ġsub stitute +ĠPl anning +Ġp ension +Ġrecommend ation +ĠT ags +Ġg ef +Ġalbum s +Ġwash ing +ro c +Ġtr ains +at ings +Ġex ponent +ack bar +- ln +á g +.Data Annotations +ĠE IF +ĠMalays ia +ĉ PORT +on us +Ġcle ver +Ġpe u +> ĊĊĊĊ +ĠArg uments +Ġdebug ging +( right +' D +com pute +Ġfin est +OR AGE +Ġspect acular +ph rase +Ġind ia +Ġlegend ary +b irth +Ġcom posite +Ġg rows +ĠT D +Ġep id +Ġlaunch ing +] ][ +Min utes +ĠCh a +Ġclean ed +Ġwitness es +uk an +ĉ Type +Ġhab e +par agraph +ĠJ Panel +ĠH ann +Ġvar ied +ĠP okemon +ĠM UST +åĬ ¨ +.vis ibility +op up +^ [ +.exp and +Ġ" ', +.f asterxml +_ auto +ĠShe et +mark er +Par cel +ew s +ĠStr ategy +-m aking +Ġun ve +Ġtrail ing +Ġclick s +ĠGet Component +ĉ content +IG ENCE +ERN EL +NSMutable Array +Ġb reat +Ġharm ful +¶ Ī +Ġbes ides +Ġb oring +Ġbrut al +v ang +(p arse +qu ick +Ġpy test +Ġswitch ing +() ]Ċ +Ġì Ħ +L ER +ĉf ont +Ġnet t +) ]ĊĊ +(/ \ +æŀ ľ +to Array +Ġbre ed +ĠC AR +ĠWe apon +A bs +t ot +Ġset Name +apt ive +Ġ: , +Ġesc aped +ord en +ĠP ri +th umbnail +Ġdescri ptions +/ styles +ĠPC I +Ġal phabet +astic search +NOT E +Ġc ialis +ĠGr iff +Ġpor que +Ġprote ins +pl ays +Ġst ating +Ġimag ination +Ġfac ial +ĠMe chan +Ġarr anged +_ used +Ġarrang ements +ĠP ipe +host name +Ġprov inc +T it +.Flat Style +ĠS plit +ĠLo ader +.c c +Ġclin ic +---------------- ------------ +Ġb aking +ĠEN T +ne ath +ãĢģ ĊĊ +AN E +.EntityFramework Core +app ers +. ic +ĠNg Module +ĠF ORM +Ġ' ; +-pro fit +h w +en emy +ĠE ye +Ġca ution +t own +Ġur ged +ĠJim my +ynchron ous +-s ized +m aking +, { +] ', +_ Object +ah oma +Ġactiv ist +IN VAL +ĠCom mercial +ĠOr lando +(t ab +ĠØ ¨ +Al gorithm +Ġher itage +Get Mapping +Ġfail ures +ri os +at iva +Ġt et +Ġcar pet +( Z +th ree +Ġdisc losure +. ERROR +_c alled +Ġd ial +Ġoccas ional +.E rr +Ġfunc ion +caff old +Ġrele asing +ï¼ī ĊĊ +_ Value +ĠV ari +y ellow +Ġstrugg les +.c al +ĠDak ota +ĉc lose +Ġsand wich +Ġanaly tics +Ġ** ) +& # +ĠJ os +Ġpass ive +AT TR +Th rowable +ĠM un +ĠU int +(dis posing +ar ak +ĠLe aders +Ġaffect ing +Ġitem View +Ġeconom ics +f v +๠Ģ +.r b +ĠOver all +Ġwealth y +Ġev olved +nd a +ĠH us +re strict +um en +ĠA gricult +! ĊĊĊ +Ġexp ires +Ġspokes person +int erval +Ġà ¢ +Ġque en +(n il +ing o +He ap +Ù İ +Ġcompl ain +S ym +ĠCl one +ĠR u +ĠW ILL +ĠCr ystal +/ content +ing en +oint ment +Last Name +av icon +ĠIB M +ĠDim ension +an h +icip ants +ĠAn ne +.pro gress +Ġal go +ob il +ĠV oice +ĠF E +Ġg li +Ġv ed +Ġprevent s +\ Column +Ġfol k +ett i +Ġm n +ĠCL ASS +Ġdisplay ing +ĠK l +ĠF err +d uto +. ib +Ġd ados +' name +-s pace +Ġit alian +Ġin verse +Ġd ense +ut er +ĠI Enumerator +-s ign +Ġnation wide +Ġperson a +Ġsol ved +Ġdram atically +Log out +Ġgr av +Ġanalys es +ol lo +Ġl amp +. team +ĠE rot += [" +Ġd ancing +Ġ?> / +Ġc ater +ff e +ĠSh a +ĠB os +ĠRE QUIRE +ĠMon ster +ĠR B +ĠI DE +Ġsu its +Ġform Data +( theta +Ġsp atial += NULL +ĠSql Connection +Ġ à +ĠV enez +ĠMor ning +Ġpublic ations +ĠNON INFRINGEMENT +first Name +ud s +W ould +_HE AD +Ġinvest ed +st able +f red +Ġcommand er +SE S +âĢĶ a +an che +ĠM ovement +ë ³ +S uite +Ġjur isdiction +ë¦ ¬ +ĠB eth +j Query +ĠIs a +Ġd ental +, * +ĠL imit +ili ation +=" { +b ast +Ġt urb +is y +O OK +Ġadvoc ate +im ag +LE CTION +л ÑĮ +(c ategory +.de c +Ġun iqu +_s n +Ġattract ed +Ġà ī +ĠRun ning +_ edges +ĠDis able +_A S +åĽ ¾ +Ġnetwork ing +_br anch +H aving +toBe Truthy +G I +Ġcamp s +se p +-p art +Ġ)ĊĊ ĊĊĊĊĊĊ +ustral ia +ĠRe ports +rit o +Ġwa ist +_pl us +ĠW W +-p erson +Apr il +Ġs ar +.t ar +Ġagricult ural +t ic +Ġt cp +Ġset Value +agent o +ĠAp pe +p iler +CA DE +Ġan che +atch er +Ġcom ics +Ġl bs +_se gment +'] =$ +itt ers +ich er +G INE +Ġutil ize +ĠC ursor +_ex pression +Ġd ag +< long +Ġr hyth +æı IJ +Ġconsult ation +Y et +")) ĊĊ +_M AC +c ould +Ġ' \\ +ĠV o +ĉ http +Ġg s +ph er +- grid +J ames +J ul +Ġsch on +Ġtensor flow +ĠLOG GER +am as +Ġsc ipy +Ġconv iction +. ag +Ġadministr ator +)) {čĊ +Ġn un +" group +P or +Ġnur se +ex pression +ak y +ĠHe avy +. opt +.get All +Ġover l +/ ", +_c ountry +ç İ +ĠG ENER +_r oute +ĠD al + ´ +ol oad +Ġuncomfort able +(m enu +Ġhost name +' ");Ċ +Ġcalcul ations +-c lick +Ġprotect ive +ãĤ ¯ +_F orm +ung s +Act ual +m f +ĠProcess ing +ĠIn ventory +(m atrix +app ropriate +w eg +ij a +Ġch r +Ġr ifle +-w sj +k ar +Ġindepend ently +I OS +Ġconsist ency +v n +/s ystem +ĠCh anges +Ġexp ose +ici ents +Ġrel ate +ĉ next +è ¨ +ud es +Ġglass es +F XML +.... .. +ĠP df +Ġappro ve +Ġ{ \ +Ġexist e +)) ( +ARE NT +оР¿ +ĠL atest +ĠNiger ia +.Inter faces +Ġrem oves +En emy +Ġen force +vert s +ĉ pos +_text ure +W ARD +ĠINC IDENT +( container +Ġdef ending +ĠR X +ĠH ook +br is +ĠFl ask +Gr ay +. )Ċ +vis ibility +ĠRedirectTo Action +err al +_e lem +Ġres on +front end +_variable s +ater ia +Ġ+ " +ave led +RI X +Ġdef icit +_C heck +YY YY +To One +sp y +Ġun ited +end ent +Ġp ode +ãģ Į +C AT +(f mt +ĠBon us +Ġre ck + º +Mod ules +Ġvac uum +R adio +ĠDAM AGE +P en +ĠPark er +; ;Ċ +ĠRe ally +_n eg +p ending +Ġnomine e +ĠC ategories +ĠUl tra +We apon +Ġdef ender +I ss +ĠG ender +ĠD ress +Ġimpr ison +Ġbank rupt +imension al +PH A +ĠStr ateg +ĠPROF ITS +Ġp atri +//////////////////////////////////////////////////////////////// //////////////// +de legate +Ġfor State +Ġdev oted +_m ake +Ġterror ists +ĠS nap +_n av +ĠA A +ĠI an +ĉ app +Pl acement +_h dr +< K +Ġs ang +st roke +- Q +> x +.T ask +m oney +ib aba +' });Ċ +ĠSpec ific +ĠLine ar +_O PT +Hash Code +( Player +.Contains Key +Ġcoll apsed +trans parent +_R ANGE +View er +(c fg +Ġsort ing +Ġinf ected +ĠN ach +Ġaccommod ate +.element s +_P ART +ĠSex y += get +( year +Ġx hr +: ] +ows ki +Ġsum mar +Ġ ¿ +Ġint e +Ġwork flow +ĠTai wan +vers ions +åı ij +Ġsurprising ly +Ġopt ical +Ġpro ces +Ġdisag ree +Ġnue vo +ĠC AM +sort ed +le ases +ist le +Id ent +ĉ event +ject ed +Ch unk +V ars +.pro vider +Ġproceed ings +Ġin clusive +Ġart work +end ants +ï¼ļ Ċ +se en +Ġl ig +Ġm akers +_f un +Ġlength s +Path Variable +[ item +ภµ +De ad +FFFF FF +ĠUr ban +up les +ich en +(null ptr +.s pec +, System +UR ATION +(j ob +å¼ ı +Ġtrack er +Å Ļ +ĠM R +ĠSQL ite +Ġd to +Ġ; ;Ċ +Ġm int +ĠInt roduction +ca o +Ġquestion ed +Ġf itted +rev ision +s q +Ġm ig +_un its +_ async +Ġf lick +});ĊĊ Ċ +Ġnot re +}` , +F ilters +Ġm undo +_d ays +Ġfr m +ut c +Ġval s +ew idth +ĠGener ator +ĠArt ist +ĠID s +ĠArt icles +re ater +ĠComponent Fixture +. = +Ġr ou +- no +.b ukkit +eg g +ĠD iff +atic s +Ñĥ Ñĩ +âĢĶ ĊĊ +ĠChar lotte +by e +Ġ} );čĊčĊ +ĠV ik +ĠB row +Ġl v +ĠG ib +-w ing +GL IGENCE +(I l +ĠEngine er +.W ait +ĠP ictures +Ġr het +Ġth ermal +Ġpr aise +< >();ĊĊ +ĠSp ider +P ause +ĠB aker +Ġsl ower +Ġ} ]Ċ +_en queue +Ġdisappe ared +ĠT icket +IN UX +_LOC AL +аÑģ Ñģ +@Inject able +comm unity +Gesture Recognizer +åĽ ½ +Ġsca les +Ġ- ( +/ '+ +ĠS it +Ġexecut ives +ard ing +Ġad vers +Ġback wards +ĉ context +ĠH amp +ĠP F +ĠDe ck +ĠCra ig +A merican +Ġb ell +Ġpro l +uf en +Ġr ng +ar shal +ĠSim ply +first name +sh ore +J uly +Ġmort ality +ĠâĨĴ ĊĊ +Help ers +Ġbench mark +em ade +Ġorganis ations +.g son +ĠText Field +Ġciv ilians +.Array s +ĠMiss issippi +Ġinter mediate +get User +_cl uster +Rel ative +fore ign +.querySelector All +Fore ignKey +Ġreason ably +-------- -Ċ +C ards +ĠK am +ĠTh or +Ġroll er +-e lement +ĠC urrency +dd ie +ALL Y +ĠR A +Ġper met +aa aa +Ġhom ework +ĠV it +Ġm old +ĠF er +[ start +Ġstatist ical +Ġsc ary +_H OME +.B egin +Con struct +ogen ic +ĠDEAL INGS +Ġtamb ién +ix on +. ind +ac re +Ġtransform s +ĠN ap +.B lock +uss ia +pir ation +ul ent +Ġce il +Cl ause +na ire +T ES +Ġne at +ST D +ĠReg Exp +per form +: ) +Ġun ions +Ġs ublic +Ġw inds +lo ating +g lich +Ġp agination +S kill +App ly +ĠOper ator +ist ogram +Ġqual ities +C ross +Ġde com +], " +ĠJ uan +.mod al +.Ch ild +ĠRog er +STIT UTE +:CGRect Make +a lette +Ġst a +as ide +Ġbl ur +ĠW a +if etime +re ed +control s +Ġb ins +Ġп ол +*/ ,Ċ +U IS +ĠR ou +ĠDem o +- awesome +ĠCh ain +Ġh asta +ĠB art +. KEY +Ġvend ors +nof ollow +ĠD est +_b uilder +Ġarg ues +_ answer +g oto +ĠRES ULT +ĠM ON +Ġp oder +o ons +_C ASE +Ġrep lic +Ġfin ancing +ĠD ATE +c ern +_tr ack +t ies +/ logo +ĠNE GLIGENCE +get Type +> T +b et +g irl +ĠINCIDENT AL +-s ite +.tr igger +ĠL isa +_input s +Ġrel atives +Logged In +Config ure +I K +. accept +Res ume +ĠD raft +Ġ* >( +ĠW A +ed ian +ern ess +ĠLayout Inflater +*/ čĊčĊ +oth y +Ġoblig ation +Sub scribe +Ġth umbnail +ex ist +Ġins isted +ĠU ICollectionView +ĠAng ular +Ġtable ts +ĠImp act +ãĢį ĊĊ +ah o +Ġcharacter istic +g d +Ġ= ================================================ +our t +` . +App ro +Co ordinate +Rem ember +Ġmar ine +] ==' +ĠAdmin istrator +.get Default +Ġforg ot +ĠStruct ure +V ue +ars ing +m oment +k w +_c ursor +Att ack +Ġath letic +Ġdiagn osed +Ġend e +åĪ łéĻ¤ +H ouse +ĠP ARAM +Ġw iki +ĠO pp +Ġcons ervation +Ġs nd +_t em +sub str +ĠC ape +.s im +UT ION +an an +âĢĻ un +Ġg y +- work +Ġcomp elling +=' # +ĉs ub +Ġdirect ories +íĬ ¸ +Ġtouch es +out ines +.C ollection +s chedule +.l at +ĠDo ctrine +CA A +ĠRe fer +Ġshift s +Ġlik elihood +pre ter +ĠF emale +Ġinter cept +Ġl ou +çĻ » +Ġr ug +ĠC rown +Ġ************************************************************************ **** +- product +Ġprompt ed +ung le +d ocker +ĠT u +ĠUn ique +_ Error +ul os +Ġâ Ħ +Ġ( ` +Get ting +_s cal +ĠEn h +ü t +Ġsust ained +Ġp atches +Ġpros per +ĠG aza +_l ight +Ġin cons +-------- Ċ +ĉĉ ĠĠĠĠĠĠ +S F +C N +: ";Ċ +ĠColl ins +( *) +Ġcomp ilation +'] čĊ +Ġcon sequence +, ... +Ġd m +ĠB LOCK +Cl uster +Ġsk i +(arg c +T uple +Ġjo ins +ĠSher iff +W ar +ind i +Ġcomment ed +H OST +Ġinv itation +apan ese +Ġperm its +preced ented +_z one +ĠA my +_R D +Min imum +Ġinv ocation +.en able +icht en +- owned +" id +_PO INTER +F ac +Ġspecific ations +Ġnom ination +Ġg p +< ( +Ġrob ots +ĠJ erry +Ġhold ers +Ġw and +c ms +Ġ} ))Ċ +.To ast +ĠI List +B ased +z oom +/ style +ĠBe ck +M en +Ġcontrib uting +Ġund o +ĠO H +Ġadd Object +Ġe igen +sign up +éĶ Ļ +Ġdist ant +PAR ATOR +ĠM ari +Ġm á +E mp +ó s +Ġì Īĺ +ev t ++ j +p ark +ĠSt ay +ĠD un +Ġso y +> % +az ines +Ġti empo +(m e +p resent +.Th is +Ġedit ors +F IELD +.W ork +ĠUn iverse +Ġdr unk +.t imer +Ġalter ed +ĠN ar +ëł ¥ +.Act ive +id or +ç Ń +.delta Time +Ġawk ward +& quot +ĠSaf ari +Ġtr icks +MENT S +div ision +Ġvary ing +ĠHigh way +Ġphotograph er +ĠSt ewart +Ġlast ing +.P re +.amazon aws +ĠL uck +.D escription +ĠN az +n eg +Ġc ó +<<" \ +ĠSur v +ĠU nc +Rec ipe +.Border Style +Ġmod ifications +- at +AT FORM +h dr +ak o +Ġsublic ense +ĠJ ump +Ġbe im +ĠMan hattan +. bool +_h w +ÑĤ ÑĮ +B in +Ġg ateway +" ": +ĠU IS +:" + +- def +ĠReg ular +/ testing +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +string stream +Ġdis par +Ġmob il +- read +ĠAd apter +ĠCh ampions +Ġsched uler +Ġk ills +ĠM ultiple +ir ror +Ġgod s +AD O +ak te +ĠUs uario +.c ircular +Ġre cept +ĠEx pr +Ġelder ly +Ġnic ely +Ġbest e +W ant +Ġclass ical +.s prite +obj c +ĠM ason +Ġsist ema +.Bl ack +es o +ĠZe it +Ġdiv id +Ġent ers +_sub ject +ĠPlan et +.w arning +ĠG ram +_t okens +Ġhousehold s +_c ustomer +user Name +c ross +Ġp ione +Ġass ists +_S M +ib o +Ġlo yal +Ġuse less +# elif +ĠUlt imate +C ome +g el +Ġd ich +xy z +ik el +ob ra +_s can +ĠInter ior +ĠN ice +Ġpl ac +ĉt arget +Ġvir al +ass o +() / +und e +ĠAd obe +O s +vis ited +ĠO W +ĠFe ed +ĠSe quence +Ġman ages +in son +ĠLouis iana +{ }) +ĠH ab +ĠL D +Ġb ip +pr ites +(e lem +.h ibernate +él é +Ġoh ne +_trans action +Ġann unci +P ublished +ĠH onda +ĠT am +ĠP acket +_ selector +Ġchalleng ed +Process ing +-h over +Ġtr ainer +_c ancel +ĠNS Dictionary +ab ric +ĠM LS +_s ensor +Ġshr ink +ĠF X +th reshold +ĉH X +-m ark +` .` +S cheme +(f ull +_w riter +ĠS ys +Ġf led +ĠC in +-w idget +ĠPre vious +G ender +_ question +Fe ed +Ġscr ut +(p refix +ãĢĤ ãĢĤ +Ġin fections +Part s +Ġhier archy +_DE LETE +ĠPat ient +_p ay +Ġprom oted +Ġì ĭ +Ġcivil ian +Ġagricult ure +ĠP iece +Ġst ance +uts che +Ass ign +.A CTION +F ig +_r adius +ĠS ync +du cer +f ailure +ens ed +pt ime +B M +_dat etime +qu ivo +QUE UE +èĢ ħ +Ap pear +Ġsum mit +: void +Ġv ine +è® ¤ +on ne +_TR ANS +.g reen +_ cc +Ġhung ry +Ġ" > +() );čĊčĊ +Ex tract +iz ens +Ġsol ver +Not ify +Ġeng lish +ĠSh opping +inter faces +RE Q +Ġil leg +ĠUI ImageView +Ġdis connect +ĠUnt il +ĠConserv ative +@ Column +Ġshift ed +Ġ: čĊ +Ġf ich +Ġd la +Ġsh oe +"), čĊ +ular ity +_RE SP +We ather +UI Application +. iterator +Ġag ing +.P arent +ow ie +(e qual +ĠCon v +/ default +Ġmeas uring +.pre v +.Is Valid +.F at +Ġs Äĥ +key words +with out +Ġso vere +Ġex changes +Ġm elt +Ġis lands +ĠInt egr +Ġjump ing +Ġg le +Ġjournal ism +Ġd ated +Local ized +ĠRef resh +Part icle +Ġa a +ĠSTR ICT +Ġb od +.Pro cess +_A UTO +ĠP ublished +e very +Ġtechn ological +ls x +Ġir rit +Add itional +Ġdel imiter +_l anguage +- area +bo ys +ĠT ube +Ġw at +Ġmechan ics +_ owner +Sp ell +ĠSt ories +.Append Line +Table View +h em +st ick +oll ower +I FF +ĠU V +oll ision +S UB +Ġcompar able +Ġdon de +s ales +ll vm +Ġ} ],Ċ +OTT OM +ĠPur pose +L ab +Ġinterview ed +o is +as il +.set Id +ĠIn struction +-- > +ĠMod ified +ation ally +ĠMe eting +è¯ ¯ +# region +Ġrout ing +.f ocus +ĠYou th +< D +ĠN ag +contact s +Ġform ing +Ġm ie +',[' ../ +ĠB P +Ġapp et +ĠTe acher +ĠT P +Ġann ually +outed EventArgs +ĠSpe aker +Ġre name +CF G +(" // +æİ ¥ +/p ages +Ġpr és +ĠSp ell +.All ow +ĠINT ERRU +Ġ( # +âĢĻ ĊĊ +_G eneric +.im show +_t im +- face +(& ( +atin um +Ġrevolution ary +ĠH ours +r ain +Ġany time +Ġab b +.j sp +Scroll View +ĠTr uth +Ġanticip ated +Ġacc ent +. checked +Ġspec ifies +Ġca f +Ġcell padding +Ġcook ed +ĠH ugh +pe ek +_R ATE +Ġd orm +/ čĊ +IV ITY +.Cont roller +(p art +.con straint +Ġinv asion +MO VE +Ġgl uc +l ename +Ġam en +eng lish +ĠSw itzerland +";ĊĊ Ċ +pe st +.col lect +N ib +ĠD ict +ĠE mb +(sub ject +Ġoutr age +Ġdec iding +Ġsent enced +F echa +" A +Ġqu er +Ġfont Family +Ġqu adr +- Y +_C ACHE +Ġanaly zed +Ġg aining +ĠAgain st +ĠSou l +ta u +Ġlight weight +ĠT F +ĠEffect s +.T ypes +.add Class +Ġv egan +é ģ +.' " +ĠExpl orer +.d etect +.sh ift +Ġoblig ations +last Name +Ġassoci ations +ĠTime Span +un ter +ĠF resh +Compat ible +P ub +id ges +. option +var i +.hash Code +Ġg eb +. section +- not +ĠSub mit +T N +reg istry +_m edia +Ġn aj +ff t +Ġm ate +-th ird +Ġp ockets +est a +Ġb ent +ĠN ord +Ġretail ers +ĠMor ris +."" "ĊĊ +W rong +Ġ ÅĽ +R ay +. ec +ĠB ind +_H AND +(n on +is Valid +Ġsimilar ly +_L IMIT +Ġdynam ics +Ġdist inction +ãģ Ĩ +< N +Ġor th +ĠToy ota +ĠK ate +ĠL S +or ie +ĠSpr ings +Ġf reak +last name +_M ULT +-st ep +" ( +AD DR +Ġentert aining +_CON F +Ġdec oded +Ġst reak +Ġwait ed +Ġnot ified +rodu ced +vis ual +.Layout Params +æ ° +es ian +f its +s pring +ĠBern ie +User Defaults +Ġped est +Ap pearance +ĠW iki +ĠNOT ICE +Ġs sh +Ġdur ante +ĠZ ip +ı r +ĠNAT O +Ġtw elve +Ġro yal +ï ¸ +Ġmer chant +ĠF urniture +'] ),Ċ +, X +Ġfold ers +ĠG ate +ĉf unc +p ick +_us uario +ĠV erm +ment ion +ur pose +Ġalert s +x ious +_s ig +ĠF u +Ġ( : +Ġd umb +åħ ³ +Ġaccur ately +éĩ į +R B +-s creen +ĠV ER +j our +Ġrom ance +uc ceed +. choice +Ġad ip +_d ims +Serial izable +ãĤ ĭ +.j ob +Ġpro g +uch ar +Ġg ently +ĠR SS +ict ured +_ENABLE D +ĉ label +aw ks +ĠEn sure +rem ember +ìł ķ +Ġtrans mit +{{ $ +.Trans action +ur se +_rel ative +Ġs ized +ĠX X +ĠPr incess +ĠL arry +Ġpr ó +ĠÑģÑĤ ÑĢ +Ġs isters +estr uct +Ġcheck point +: length +ĠCar los +/ icon +_T ARGET +T okens +Ġpat ience +ĠSe lected +q ty +.show Message +Ġwild life +ĠP rops +b m +- arrow +Ġpar cel +fire base +ĠBen jamin +cess o +.t im +ĠG arc +. any +ĠHOW EVER +ĠK o +Ġgrab bed +_f rames +Ġobject AtIndex +ĠADV ISED +Ġsub ur +ĉ GL +Ġ}) }Ċ +-l ength +ìĭ ľ +ĠPot ter +_b uff +.g ui +ĠEnc oding +E lect +-m essage +Ġ � +Ġ ÈĻi +ĠArgument NullException +а ÑĨи +Ġmin imize +Ġrespond ing +$_ [' +ĠInd ividual +á c +ĠIN TER +Ġmast urb +ĠB in +(' $ +ëĵ ľ +Ġopen ly +Ġ> < +Ġun to +olog ically +ĠM ul +VID IA +Ġsl im +ĠCommission er +( on +Ġunder neath +/ db +v ote +( Message +ĠP ope +Def ined +Ġsw ift +ur f +Ġadapt ed +SE L +Ġreven ues +Ġdiv ine += y +Grad ient +_ act +Ġ/*! < +Ġpoly gon +ĠF DA +ĠC arr +at ables +(std out +Ġrefr iger +Ġco ordin +avor ites +ÑĪ и +Ġcompass ion +ĠPOSS IBILITY +- secondary +ur acy +Ġcomp romise +_A V +_ os +Ġbes ide +ĥ Ŀ +Ġl n +.pl ugins +Cap acity +al ah +.b in +ĠC RC +_b alance +Ġflex Direction +Ġam bit +Ġnick name +ĠFor ces +C LE +ĠSh ell +Ġs ail +ĠW riter +ĠA lice +d w +ĠInd ians +ĠMar shall +_S RC +Ġnormal ized +ĠJ ag +ãĤ Ĵ +ze it +r pc +ÃŃ c +.in line +Ġtrav ers +_n umeric +Ġutil ities +Ġev ac +IN PUT +ĉ register +M X +ĠCamp bell +Ġdatas ets +Ġdem anded +Ġinitial State +g an +Ġe i +Un expected +- web +tr ait +, Y +ĠT odd +Ġske leton +Ġoptim ize +ç¬ ¬ +ĠU pon +ĠSt Object +Ġap lic +.' P +v ron +. UN +Ġpaint er +izar re +Ġl av +Ġp om +p reg += function +( serial +ific a +um ing +åľ ° +ãģ Ĥ +- op +U CH +ĠH end +.prop Types +Ġy o +Ġrout ines +Ġcar ing +S em +Ġres erves +Ġprior ities +red its +IST R +Content Type +ĠSch w +/ media +Ġe str +Ġclim bing +- week +cher che +s ensor +To Array +ĠMont real +Ġcloud s +ĠInject able +ĠR ice +Ġpropag anda +_pro vider +Ġind oor +Ġin aug +Ġdipl om +Ġmess aging +_m ut +å ¦Ĥ +Ġk w +ON S +ari ans +R PC +) ]čĊ +-r ay +ĠS or +m all +Ġmarket place +Ġv tk +M a +og an +ig i +Ġspons ored +ĠD ani +.S EVER +>' .$ +m ultipart +ĠW ol +Ġtable Name +ĠUser name +Background Color +Ġf right +_E MAIL +Sept ember +_val s +op ia +Ġsp otted +- Ch +Ġdata Source +/ "Ċ +ек ÑĤ +ĠRequest Method +ĠRe place +-d o +ah n +ĠPh D +] .ĊĊ +N ON +g ement +ĠTh r +Ġquiet ly +Ġtort ure +Ġte as +ĠC Y +Ġa tr +develop ment +-d etail +Ġlight er +Ġarg uing +Ġdes erves +Ġcur riculum +_CON TEXT +ÅĤ y +H ITE +ĉ ID +/ uploads +Ġt its +re o +_d rop +. UTF +Ġpick up +Ġgro cery +ĠP ure +Ġeas iest +Ph il +.f eature +(" * +Ġinvest or +t ok +Ġj ar +L os +âĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶ +. queue +-s peed +M al +um blr +ĠCON ST +ĠH RESULT +ĠD ance +(file Path +Ġattrib uted +ॠį +ĠB und +co ins +Ġs ão +Ġp ir +person al +Ġpre lim +Ġprop ose +ĠT L +] ]) +ĠSub scription +ĠK re +, len +.First OrDefault +) -- +_product s +.Get Bytes +Sh ip +Ġenc rypt +ĠS G +ĠM yst +h ir +Ġiter ate +Ġint end +.mock ito +Ġch apters +( angle +ĠV lad +è® ¾ +' .ĊĊ +Response Body +ĠAb d +de al +Ġbar riers +-out line +b ill +ĠF alls +_se cond +. include +. ceil +Ġoccup ation +ph ony +.move To +ĠJenn ifer +AST ER +; ">< +ĠEn abled +Ġtermin ate +ĠI o +l ations +ĠTHE ORY +Ġear liest +Ġr ack +ĠSc ar +sh ake +ch ip +Ġu v +Ġall iance +п иÑģ +ĠGOOD S +z ione +ĠV I +Ġ{ - +Ġfilter ing +Ġmis con +.Dock Style +Ġb ush +Ġj unk +æ Į +ĠQ UE +Ġhook s +Ġfirm ware +Ġmiddle ware +d ic +ĠOak land +Ġarr ives +P ayload +p ixel +] | +Ġstart Date +.P RO +_a udio +Ġmid field +igid body +ĠSw iss +ĠCl ip +ĠD ump +ĠText Box +Ġg eh +y ield +od s +Ġrefer endum +Back end +ĠC ream +Ġdomin ated +ĠArch ive +Ġrid ers +.prepare Statement +Ġqu ando +Ġche f +w iki +in el +am pling +(" \\ +Ġs ag +_pro xy +ãģ ķ +p do +.getElementsBy TagName +Ġdemonstr ation +ĠN PC +Ġarch ivo +end ance +Ġefficient ly +( actual +.t ableView +Ġm ush +Ġbe ars +_thread s +j as +ah un +Ġne ural +Ġdesign ing +ĠG DP +Ġlift ed +çĽ ® +ĠJ oint +ĠIn clude +ĠGi ants +Ġwithdraw al +ĠR ent +n ative +ĠSe ek +gress ion +_C PU +\ S +ĠSh ield +Ġsol ic +Ġbo om +yect o +Ġmanufact ure +ĠâĢ ĭ +Ġb box +Ġearth qu +ollect ors +:@" % +Ġlo ops +J e +alk ing +ĠWh ats +ĠBo ys +. book +ARG E +_p ixel +Ġsus pects +Î ¹ +us p +ĠBM W +ie ces +(p erson +å¼ Ģ +é » +ĠPod cast +Ġb ou +( Item +à » +( Input +Http Get +Ġb urg +) ^ +BO ARD +*/ , +Ġg ulp +ĠB enn +Ġdeck s +.status Code +Ġac ute +Ġh ug +ug u +Ġp led +," % +h ape +Ġз ап +ĠMain e +.re al +Ġd alam +ĠMin or +.F loat +dis p +Ġt l +Ġen count +=> $ +Ġf g +te es +ĠRec omm +ä l +Ġchem istry +Block s +O ID +Ġfore x +ĠApp end +Ġ{ * +ĠSup ply +CG Float +(b l +Ġat e +ador a +Ġg ust +Ass oci +> .Ċ +F ETCH +.s erial +widget s +ard less +ie fs +_F ULL +ernet es +ĠP red +Ø Ń +äº ĭ +ub ernetes +ĠL aura +Ġl abeled +High light +Ġanno ying +/ update +(d escription +Ġintim id +$ c +")) )Ċ +.A P +Ġ[] * +ĠEX IT +.H ost +ĠOP EN +.send Message +_c amera +_t ile +Ġth erm +onom ous +Ġdis adv +Ġna ar +index Of +ĠP P +.prot ocol +AF E +Ġtext ures +################################ ################ +umb ai +.st ats +ĠG E +Ġi e +ĠST D +ĠM ann +.ref lect +K B +Ġd ive +.w av +/* ---------------------------------------------------------------- +/ settings +.l ifecycle +Ġda ughters +or us +ub er +N ING +st ri +ĠT ip +Ġz n +Ġswitch ed +in et +uff y +ĠTransport ation +( conf +fr ica +ĠX L +ĠLe ad +_per cent +< Map +Ġthr ust +or b +ik k +Ġtra uma +Access or +ĠF it +ĠString Buffer +ex pl +(s creen +Ġaud iences +ĠO PTION +_ round +[ node +be h +-> __ +per missions +ĠD etermine +.M an +Ġadv ances +. InputStream +Ġstrong est +Ġe Bay +Ġ# - +Ġdir name +ĠS MS +Ġmedic ations +Ġam ended +Ġchurch es +ĠImper ial +$ row +ĠMad ison +ĠIn sp +Ġaff air +Ġpsych ology +v h +Ġsever ity +âĢ IJ +Ġstri ps +A H +vert ising +Ġcon se +IM AGE +ĠSt ats +ĉs c +.C ursor +Ġfree ze +ss on +(x ml +ĠSus an +.t ile +ed ed +ĠĠĠĠ ĉĉĉ +uel le +ĠMitch ell +b ased +Oper and +½ æķ° +ĠF F +ĉstr cpy +ounc es +ild o +.execute Query +Ġapproach ing +ĠSe ven +Ġn uts +Ġr ic +ass ignment +Ġcalcul ator +ĠMur phy +ĠB ou +í Ħ +Ġbut t +Ġt icks +Project s +il ib +.text Color +m ov +_log o +( template +ĠIN IT +Ġimage View +scri ptions +OR ITY +Con sumer +Ġun precedented +Ġtour ist +Ġbr on +Ġcontract or +Ġlic ence +ĠN am +æ ¯ +( transform +_AT T +P ref +ĠG am +Ġvess els +Ġh av +L ater +.To Lower +Ġurl s +Ġbreak down +Ġpen alties +Ġf oster +ĠU E +Ġcl ue +com ed +åIJį 称 +-m ain +Ġp ts +Ġcount ed +ict s +/ post +Ġget attr +Ġp ing +ANCE L +Ġp ec +Ñħ од +ant om +ĠBlue print +ĠEvent Emitter +Ġl ä +æ ² +Ġstr aw +( comp +' une +> N +- client +es Module +-b ase +Ġret reat +_s imple +ĉĉĉĉĉĉ Ġ +fe e +') čĊčĊ +Control Item +Ġsubscri bers +ple ase +ĠE ff +Ġp ound +ĠBy tes +ĠTe a +_ activity +Ġmax im +Ġop code +B SD +. constant +; } +omb res +Ġcare ers +) .ĊĊĊĊ +Ġsp reading +-exp anded +ĠOr d +amar in +Ġmob ility +Un fortunately +ak k +N L +_ redirect +ĠP G +ĠS ensor +b ol +t ap +_MEM ORY +ĠUI Alert +plit ude +We bsite +ĠLog o +lo ve +[ ind +Ġalto gether +Ġwonder ed +Ġes per +ĠLib eral +Ġo ss +Ġel it +Ġst iff +od ox +_ment ions +ĠDou glas +_p id +ĠC K +ĠinitWith Frame +.b log +p kg +ang hai +QUI RED +u u +Ġm kdir +AT AL +Ġun h +in ces +st h +Ġhypo thesis +Ġc ata +ĠT B +ĠCl ar +Ġpre decess +Ġsitu ated +-w orld +)) / +Ġhead lines +.st at +Ġout break +sp ath +_FLAG S +ĠServlet Exception +S un +F ROM +ĠD ir +ãĥ»ãĥ» ãĥ» +_co ord +ĠOpt im +Mon itor +.b it +XX X +Ġtod as +f eld +ÑĢ и +im ir +Ġpolit ically +Ġmolec ular +Ġtrad ed +Ġ{{ $ +ĠSw edish +Ġ'@ / +_RE AL +Ġw arehouse +t oday +, L +or p +< section +- br +ym e +ĠUser Service +Ġlib erty +Ġmoment o +( Image +< size +S ch +Ġj og +i ology +arent ly +Ġquant um +ĠAb u +Ġr im +Ġman a +Font Size +Build ing +st airs +AIL ABLE +Ġ& ' +Ġs ect +Ġs igh +(b atch +.I Container +p oll +ĠCor ps +Î µ +ar u +ĠK ay +.r ange +_click ed +ĠRobert s +.N etwork +fin ish +- Man +Ġcolleg es +ĠF ine +")) ,Ċ +f ilm +Ġrem inded +Ġgest ure +out il +Ġthread ing +Ġobj et +Ġt ours +activ ated +.m kdir += user +Ġre de +f ü +_SY STEM +p v +Ġcon gr +Ġmass asje +Ġpract ition +Un iversity +Ġtab index +Ð ĺ +S ets +Ġcount ies +g uest +f an +Ġword en +.d i +на Ñĩ + ¿ +ig Decimal +Ġsh ore +Ġg ö +Ġrep airs +Ġhelp ers +Ġcenter ed +OL LOW +Ġmap StateToProps +Ġc ents +< A +Ġexpect ation +Oct ober +Ġbg color +ca les +.C ON +ĠV el +Ġcry ing +-se ason +Ġfunction ing +_LOC ATION +ü ss +ber y +Par a +omin ator +- le +Ġeth ical +has htags +emp lo +Ġn úmero +( activity +.St op +.str ftime +IL D +Ġto e +ĉ Node +") čĊčĊ +ĠPu erto +Ġexec uting +ĠG UID +Ġoppos ing +al ph +Ġexhib it +_fl ash +Ġme ille +Ġjson Object +H ero +aint ed +_D OM +Ġw il +Ġslo pe +Ġm Ã¥ +ĠIraq i +Ġorgan ize +ĉj Query +H UD +sh ine +. we +ĠSk ills +pons or +Ġcon clusions +Ġre forms +Ġrel uct +n amed +ĠOl iver +Ġ// }Ċ +- looking +Ġf og +ĠH O +ĠF ried +Ġinev itable +ĠData GridView +H our +il les +log ical +Ġconnect ivity +.tw ig +ĠK yle +(d st +- Sh +ĠStud ios +( Level +.j et +_PRO TO +-de coration +OT HER +Ġread ily +.Param eter +Ġmultip ly +ĠL IB +ar med +Ġsoon er +æ Ħ +_ ES +Ġfoss il +ĠA nc +âĢľ This +l odash +Py thon +Ġhist ogram +west ern +Ġinf ant +Ġco ordinator +Ġn ib +: m +Ġres pected +Ġdef init +& T +_p ad +ĠTr igger +th al +Ġimage Named +Ġbeat en +ĉ rc +ĠPal ace +Ġhaz ard +Ġisol ation +_ rc +cont re +OUT PUT +Ġre ign +ĠPl ate +AT ES +Ġfl ux +Ġpack s +.get Selected +Ġparticip ated +Ġneed le +-de pth +:::: :: +-l aw +ins pace +on itor += no +ĠAt omic +ĠBr ain +Edit able +-s c +red ential +ĠP erry +k ie +Ġ ----------Ċ +.st roke +( Intent +Ġun ity +um lah +F urther +Ġpr ze +Ġs ø +ãĤ Ĭ +ĠPROC UREMENT +ĠH ousing +Ġatt orneys +Ġcomp ose +atter ing +" What +dra ul +Ġstraight forward +In stant +.J TextField +Ġtr ades +л а +Ġ{ ! +Ġl ately +IM G +ĠA ld +ĠIN NER +Ġcart oon +.S ource +F ALSE +Ġd ough +f en +( rect +Data Table +N ick +ĠBut ter +read s +_com ments +EN V +ĠConnect icut +-F IRST +ĉĉĉ ĠĠĠĠĠ +ach i +.M sg +re ction +Ġrelax ed +Ġsha ft +Ġe f +ĠAdd ing +Ġbre ach +Ġ ï¼ļ +ram a +Ġconduct ing +Ġ( ; +(g l +ĠCA USED +ash i +ĠF LAG +ĠCom merce +ĠIN TEGER +h ours +ĠSchool s +Ġn ucle +Ag ain +pro j +Ġsevent h +EMPL ARY +(m ock +'] ,čĊ +_S PEED +> false +Ġsp a +ĠN ear +ì ķ +Ġintr ig +_m embers +w ave +Ġanalyst s +_O S +ed in +ĠF ri +Ġretrie ved +Reg ular +_ obs +EX PORT +')}} " +" class +__ (( +b ucket +Ġst ro +ĠP atch +yst ick +ful ness +ap os +D a +ĉĉĉĉĉ ĠĠĠ +Ġen rich +un ordered +h ole +C ong +< Product +ĠC urt +( the +_l ower +Ġavoid ing +Ġbu zz +Ġv iable +ub a +- is +are l +Ġact ed +-d etails +ภĩ +ĠThe ory +ĠP un +ĠAn onymous +... "Ċ +è res +åı ¯ +ĠV ision +_se m +ash a +Ġcelebr ity +Ġend Date +Ġpop ulate +Ġcu is +qu ant +f loor +Ġglob ally +Ġcru ise +ĠStan ley +Ġb ikes +.get Connection +Ġpoor ly +_ other +amp ing +." );ĊĊ +od i +_A DMIN +.color s +ĠG aming +> ';ĊĊ +STR UCT +Q R +ID s +(arg uments +_a ux +( Event +_PR IVATE +ĠTre k +Ġdownload s +m utable +_STR UCT +(w x +Ġdom ains +js px +ĠVi agra +Command s +J s +.c fg +Content Pane +ĠEdit Text +à¥į ठ+Att ach +ĠAR M +posit ive +ĠGener ated +Ġse ized += : +Ġelectron ics +ĠApp Component +/ ',Ċ +.equals IgnoreCase +Do ctrine +d isk +ĠPolit ical +CH O +< F +ĉ height +ĠB ug +. le +ik h +Ġmill iseconds +Ġconstit u +m ag +.n l +-r ange +ang gal +', [ +ropol itan +Ġà ľ +ĠU C +.d esc +-L AST +f stream +ib il +Ġf ier +VER Y +Ġë ³ +IR T +_ UI +( abs +Ġkne es +Ġro okie +ĠV ac +are na +comm end +- \ +ĠSUB STITUTE +So ft +Ġpart ir +we alth +è¦ ģ +(d ataset +ĠCl imate +- show +Ġreli ability +_ch unk +ä» £ +_st ock +ĠEX EMPLARY +ï¸ ı +Ġv ÃŃ +Ġsm iled +Ġdr ill +.F unction +ĠS I +Ġreg ression +- X +ĠJ ar +p ref +ĉs uccess +ĠHit ler +Ġinst inct +Ġfem mes +Ġlo ver +< Ċ +Ġmulti plier +r il +Res ize +ĠAuthor ization +ĠK an +Dispatch ToProps +Ġc rops +t okens +ec n +ential ly +ĠINTERRU PTION +f ake +Und efined +ĠA K +ĠTest Case +Ġr ab +Ġtor rent +ĠO t +B ars +Ġlect ure +Ġen jo +Ġrespond s +Ġindex ed +Of Work +_ch ain +)) -> +ĠBeaut y +Ġ` < +Ġtouch ing +Ġ| -- +ĉf lag +normal ize +Ġtr apped +Ġestablish ing +/b uild +A J +f y +- react +av n +RI PTION +Ġk ut +ĠF ashion +ĠIn form +cur ities +< byte +ĠUkr ain +Ġs ug +Ġconsist ing +ood le +. ctx +.To List +Ġcomment ary +Ġtransf ers +Ġn ost +ih ad +ĠU pper +Ġconf using +miss ing +- cl +Ġbound ing +Ġcongress ional +Ġreve aling +d h +r up +Ġt res +re peat +, ĊĊĊĊ +_t ac +Ġexp ed +G irl +h orizontal +Ġ"../../ ../ +( option +Ġwe iter +ĉs ql +Ġ=> {Ċ +Ġgar lic +Ġre pr +Ġrepl ies +( prop +Ġspir its +Ġins pire +Ġbas ement +.re ject +Ġhint s +Ġpoll ing +ĉ ĠĊ +_r ating +Ġc ath +av ier +Ġcomp ressed +ĠV S +] ' +Ġjud icial +ĠT rend +tr aining +EST AMP +ogn ition +Ä ģ +SE NT +vent ions +Ġconsult ant +um ph +Ġuser Service +, NULL +k h +D ear +_B AD +it ations +Ġmet aph +' é +and ise +-f ont +.ch art +Ġs g +_ Controller +.j peg +ĠUL ONG +ĉg ame +( ss +ĠM aj +ĉg o +ĠS ad +ĠB erg +ĠM ine +P ack +Ġres istant +ĠR OM +Ġp eg +ĠStan ford +ĠY ahoo +Ġsca led +Ġl an += [] +"/ > ččĊ +Ġs ud +ĉ background +Ġsch olars +-m uted +ar á +Ġ= ==== +Ġ__ __ +C reat +ene ver +/w p +ĠV PN +Error Code +) ],Ċ +(b uilder +ĠEn emy +S ensor +us a +Ġtr iggers +Ġplayoff s +_RE Q +Ġ( ~ +ĠBar ry +Ġperman ently +ĠR UN +Ġb ure +.Fat alf +Ġch ick +ĉ panic +ps i +ok a +éĢ ī +> [ +Ġunderstand s +ĠJun ior +ĠIN FO += mysqli +ust ain +-s ource +s erv +ĠC REATE +. au +Ġsell s +ĠĠĊ ĠĠĊ +E urope +z w +pre h +ĠNS A +Ġx y +ภ´ +ĠB eyond +Inst ead +Non Query +Ġar ise +Ġavoid ed +.em place +_model s +} ),Ċ +Ġh id +Ġ& _ +.p oints +.get Width +.Ex ec +Ġ// // +ĠS essions +... \ +ĠCol omb +Ġacceler ation +rest ore +Ġ ile +ob ic +< Node +ĠD X +ĠBes ides +. age +ĠCont ains +N ational +ĠIm plementation +Ġeff ic +ĠR M +H y +ĠWed ding +ok ies +Ġrec ursive +Ġprosec utors +.Se lection +ĠForm ula +Been Called +[i i +ĠFr an +Ġtraged y +_F EATURE +Ļ ¨ +comp ass +ĠB h +? ĊĊĊ +.w riter +ĠH our +Db Context +io v +am on +re pr +é ĥ +ĉf i +'] ] +ĠD ry +. ro +ĠO bserv +æł ĩ +Form er +ĠB alance +ĉ json +Ġpr zy +I SS +( sock +ĠL INE +Ġde ce +Ġal ly +Ġtend ency +F un +Ġschem es +Ġinter ven +æĺ İ +Ġad verse +quote lev +Ġsacr ific +_s ide +Ġmut ex +AG IC +Ġocc urring +ĠCommunic ation +um ar +ç¼ ĸ +ĠTreat ment +.p erson +ĠL C +Ġe ch +( (" +ĠDise ase +ä d +ĠA Z +.A ccount +Ġcontinu ously +END ING +ĠRET URN +- string +.f ilename +syn thesize +Res ponder +( opts +reg s +Ġn uest +Pe er +// ------------------------------------------------ +Ġg auge +ĠK in +.s chema +Ġarr ange +ĠBl ake +_Type Info +C over +ĠHamp shire +P aper +-in ner +util ity +Ġcross origin +F OR +Ġign oring +ĠD D +av an +Ġtrad itions +Ġget String +Ġeth ics +ĠMaterial s +DE SC +Ġen zym +io let +ĠCh ip +ĠMc Donald +Ġn erve +ç Ħ +") ] +æ± Ĥ +ĠS ugar +_S IM +j peg +Ġdiscret ion +ĠT N +bo ve +ĠMin imum +ĠForm Group +Ġwork force +ĠExec ution +err er +ĉ ĠĠĠĠĉ +Ġpres cribed +.Text Align +OP EN +ĠP B +im ity +ĠEx ternal +° C +ĠApplication Controller +Ġb arr +imp licit +_d ot +ĠCol on +C OLOR +.Pro ject +* }Ċ +pl aint +get Text +Ġindivid ually +Ġcheck box +U Y +ĠL amb +Ġdys function +ĠL ar +à ° +ĠCre ating +');ĊĊ Ċ +" They +loc ations +_C ORE +Inter action +umbn ails +ĠPart ner +b rit +Ġless er +ĠSl ot +set Attribute +ĠW ave +.p o +/ store +Ġbrows ing +_p d +sum e +s ed +Cur ve +Ġpl asma +Ġsusp icious +ìĿ ¸ +ĠB ah +ĠExp licit +_C C +.Client Size +\ View +Ġsub stit +lo on +ĠG AME +ĠB rid +Ľ 建 +_ User +Ġsqu ares +f one +Ġsac red +ug hs +] interface +ĠTh row +ĠK irk +Ġemp ire +Ġassess ed +T ax +ĠHe aven +-b uffer +_STAT IC +én é +-b ordered +Ġpun ct +(m ode +Ġke ine +S ent +ĠCal cul +ĠE ve +Ġsty lish +Ġoil s +.Test Case +Ġtrad emark +Ġliter ary +Ġconcentr ations +ĠRel ations +( Class +Ġstd in +Ġv æ +back up +. VERSION +.AutoScale Dimensions +st arter +Transaction al +- panel +St udio +k c +ĠCh amber +ĠSpi el +Ġr ho +ا ÙĦ +! ' +.At tributes +Ġmurder ed +apeut ic +Ġint imate +Ġtext Field +ĠBuff alo +d ummy +" % +ĠLib erty +ob ar +ĠT ank +ĠPop ular +erv isor +ĠIn iti +ĠM all +ĠP rior +C AP +ĠCl ay +ĠCert ificate +.L ock +-st rip +-dr iven +/ all +ĠMessageBox Buttons +_SE CRET +_p b +Ġr ats +ा ठ+Ġn t +.R outer +_top ic +Ġt ennis +ĠP UBLIC +ĠActiv atedRoute +Ġ' ,Ċ +Ġcost ume +Ġj okes +. Handle +ĉ byte +Ġflav ors +( cc +Ġperson as +ĉ image +ĠN azi +Ġgram mar +Ġú lt +Ġval ve +Ġv ic +ĠR achel +_in valid +P refs +std int +(r oute +Ġhtml specialchars +Ġpe oples +pl ine +Ġn v +ĠQu ant +opp ers +Ġcurrent User +ĠC atal +Ġrecon c +Ġconj unction +l x +amb urg +Ġinflu ential +d anger +ind ers +Ġ% @", +.config uration +os ome +. identity +Ġpick er +n ost +ĠDI Y +Aug ust +ab lo +Le af +ĠRec o +ck o +DO C +ĠH erm +: any +ĠInt erview +ĠT ex +x fe +( work +Ġle ap +He ading +Ġqu arters +\ Bundle +re b +Per haps +ĠG mbH +B irth +ĉ sum +ĠWat son +.n il +ç ¡ +{ }ĊĊ +ica id +Get ter +" name +Ġ" čĊ +_n one +z m +ac ute +uest o +Ġs ous +Ġre build +Ġnewsp apers +ĠH az +Ġk its +if o +Bl ur +Ġsu ited +- In +à ¯ +ĠKe ith +ĠNor way +IN IT +ire ccion +iet ies +_us age +ĠDou g +r ise +Ġtr illion +im ited +ĠR EL +al ic +Ġcritic ized +the orem +Ġce ase +Ġsid ew +ĠT erry +Ġsubs idi +Ġfirm ly +Ġaw s +Ġh ott +Ġdress ing +bad ge +ĠApp lications +è¿ ĶåĽŀ +Ġlaugh ed +Ġh obby +Ġmus icians +Ġ* . +. placeholder +Ġcount ers +ĠCap itol +SD K +Ġhel met +and box +qu it +Ġcriminal s +Ġteen ager +( update +G l +.se lection +Ġdis charge +Ġpresent ing +ufact urer +_UN KNOWN +Ġstress ed +å Ļ¨ +Pro to +_cor rect +ha us +Ġren ov +Ġfire arms +Ġtechn ically +-b rowser +Ġc andy +St roke +Ġexec utor +Ġocc urrence +ĠIP v +_INTER FACE +ĠRetrie ve +.b ad +Ex change +Nav bar +ĠK id +(get ApplicationContext +_ST OP +ĠB oss +List eners +Ġshoot er +ĠAl b +ä ch +Ġp ix +.key Code +al one +Ġabs urd +ĠC um +ĠNewton soft +ik t +Ġlaugh ing +Ġcapital ism +ree Node +T x +_QU ERY +.S leep +( login +Web Element +Ġcelebr ating +Ġde precated +Ġma ar +Ġart istic +_ASS OC +ĠBorder Radius +ĉw p +Ġsurviv ors +In ner +- red +Ġprosec ution +_ pp +(" $ +Ġcomm a +un checked +graph ics +r ors +G ROUND +( public +Ġcustom ized +ĠArk ansas +ĠR ew +Ġexp iration +× ķ +ĠC ul +Ġn ons +.F ilter +Ġsen ator +_def inition +ash ington +ym ph +/ J +Ġf use +ram id +ĠSup plier +Ġaut ocomplete +Ġ} ), +." ĊĊĊ +_function s +ĉ to +.e val +ĠT Object +Re ferences +Ġhe ated +H AL +Ġ)) }Ċ +} $ +ĠB arr +_UN IT ++ $ +Ġget Value +ip ed +ch ied +(v m +c ue +_int eger +_c ourse +th ird +Ġrevis ed +** /Ċ +_D IRECT +Out Of +(" ( +ĠFe el +Ġre ass +Ġsub title +per i +n f +Ġenjo ys +Ġtreat s +) this +-t abs +anc ers +Ġcontin ent +Ġcard io +S er +. question +Ġph rases +Valid ators +Ġpop ul +Ġl ÃŃ +s ong +_IN TERNAL +Ġadvis er +Ġp uzz +Ġambit ious +ĠT ob +ĠD P +Ġpres idency +Ġsurre nder +Ġwatch es +_b inary +ĠSo on +Ġcan ada +(" ")Ċ +] =' +ĠBr andon +eps ilon +r w +.add Child +.C opy +Pr incipal +Ph otos +Ġmarg inal +Ġbas ics +e ing +M ust +_ String +Ġo le +M agento +.c ustomer +(p rev +ภ¥ +Ġlo yalty +C og +Ġprot ocols +ĠCom panies +Ġtheoret ical +Ġaccess ing +ĠZ en +. ones +att ice +_w orld +z es +Ġtatto o +Ġmen os +Ġinter sect +"] ;ĊĊ +bel ie +Ġin active +.read line +-label led +.d one +lick r +ĠW ORK +Ġderiv ative +Ġd atabases +âĤ Ĥ +Ġs x +.is Array +Ġy s +Ġp ada +ĠBul let +(` / +is Active +ĠCG Size +(equal To +ĠColum bus +Ġmar ry +DE V +_l imits +ron es +I AS +Ġt au +min o +_W rite +ĠW ine +Ġ[ [' +ĠP ull +rit ers +ri ents +Ġsh ifting +up p +_TIM ER +ĠCondition s +Ạ¥ +ĠOr ders +ĠSt rength +æī Ģ +Ġvalid ity +Ġf ot +et ur +Ġb olt +åĨ ħ +ĠAl ong +os hi +Ġassum ptions +Ġmag azines +_S PI +Ġp unt +_PRO DUCT +Ġrel ay +ĠJ avascript +. te +- es +Ġwidget s +(f s +< Item +_ex tra +Ġrecru iting +E t +Ġnecess ity +p w +Ġnov els +uss els +Cre ator +ĠM VP +ĠO C +th ood +cl ients +)) * +Ġcharacter ized +_SE ND +ut i +T y +.from Json +@ Service +ãĤ Ĥ +Ch ris +_ Is +ĠJohn ny +Ġclean er +ĠInitial izes +UN K +( axis +еР· +ie val +ĠWar riors +} )( +DM I +âĻ Ģ +ĠTre asury +Ġfe as +Ġsl a +_EN UM +l hs +ĠIn stit +ipp ers +Line ar +Re ading +quir ies +-c ell +ch rome +.S earch +IN A +ç±» åŀĭ +ĠĊ ĠĊ +ĠSam uel +Ġmill s +Ġdon ate +ĠGe o +( rows +Ġshe ep +Ġé l +ä½ ĵ +Ġb em +_UN USED +ĠR CC +Ġintrodu cing +att a +ĠP riority +ĠF B +ĠSer ge +> "; +atch ing +ĠKnow ledge +ĉ The +; margin +less ness +op ard +um atic +() ));čĊ +Ġf als +(c ache +Type Id +éĢ ļ +_ choice +ĠGo th +ĠS ites +M G +_b order +Ind ices +Compar er +ĠRed istribution +Ġclo set +Ġvers atile +Input s +**************** **** +Ġob esity +qu iz +gr a +(g lobal +åĬ ¡ +Ġcollect or +Ġk or +ov able +AD C +ĠEvent Handler +. nc +Ġplay back +ient os +_p erm +_W ARNING +ĠOlymp ics +.n orm +ĠBroad cast +_sm all +dr ive +. iloc +Ġtyp ed +M EM +_con s +DM ETHOD +Ġl un +.d istance +(p ar +po on +Ġb ast +activ ities +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +: čĊčĊ +S ER +) && +_l st +ĠPol ish +Ġknock ed +Ġfrustr ation +au kee +Ġph osph +iqu id +_c oeff +æŃ ¤ +L atest +ĠD ust +T ipo +Ġmaint ains +Ġmar sh +inc inn +l bl +C are +Ġneighborhood s +_g pio +ĠAr senal +D em +ĠW he +_h ook +Ġl dc +ĠHar per +ĠBer keley +Ġgrad uated +Per cent +Ġarr iving +ĠAdvent ure +(s cope +(' * +qu arter +ĠMar ie +Spe aking +_code gen +Ġimm un +c aster +ãĤ Į +åķ Ĩ +ĠDim ensions +.rec ord +Ġtext o +ĠMich elle +P ending +( by +_P AR +uch t +be e +.Th read +amp ire +k now +ĠClin ical +Ġmargin Bottom +Ġdistingu ish +.F ull +. undefined +ĠSequ elize +################################################################ ############ +Ġeduc ated +_O VER +åº ı +ĠÂł ĠÂł +_e ach +Ġur ge +de part +Ġdon ors +ĠA u +Ġbill ions +Ġbelong ing +_ age +_ Int +Ġsub stances +m achine +!! !ĊĊ +Ġjson ify +ib bean +ĠC ad +Ġend Time +Ġc ycling +ĠUIT extField +Ġle verage +Ġvan illa +e at +La unch +( pt +st ates +ĠControl s +ĠRes pons +ĠJ ake +Ġas leep +fort unate +.next Line +Size Mode +ìĿ ¼ +Testing Module +G erman +ĠInvest ig +.re verse +ĠB ACK +( DateTime +Ġnon profit +ĠEx pect +Ġt anto +'] ), +ĉ the +M ultiple +(get Activity +_W AIT +Ġj á +de cor +lev ance +ĠGit Hub +min ation +_qu antity +.Sc anner +ĠL ion +éĶĻ 误 +Ġd re +Ġtan tra +Ġcontent Type +Ġf id +_ alt +NS IndexPath +- pl +åĮ ĸ +Ġantib iot +table s +ac ial +ĠReg istry +Ġol ive +ig ers +Ġsubscri ber +_p res +ĠSy ntax +Ġlo vers +. Byte +old ers +_for ward +al ways +C aption +Pr iv +ĠT ampa +is ateur +-labelled by +ĠTo String +Ġì Ĥ¬ +Ġinit iated +W F +Ġinstitution al +in ject +ĠSc r +Ġdo ctrine +Ġsp acious +is ure +ĠAn a +" time +ess aging +Ġc id +ĠN an +Ġin complete +T AG +-b uild +Dec ember +Ġres idual +(P DO +ĠList en +Ġg lyph +Ġg aps +ne a +.R ect +Ġsa u +ĠPhot ograph +Ġexec utable +ĠExp ert +Cor outine +_s izes +ĠN L +.is Valid +); }Ċ +- reg +Ġc iting +c wd +ĠOtt awa +ĠB att +Ġrenew able +Ġprelim inary +Ġas ylum +Ġw rist +Ġutil iz +Ġdet ention +F ast +Ġan ge +incinn ati +Ġste ering +ĠNa N +ios ity +/ page +Ġè ¿ +ster ol +Ġdis g +( DB +ĠDESC RIPTION +Ġ_ $ +Ġobst acle +Ġb izarre +Ġextr action +_ex pected +Ġlos es +ĠCele br +Ġhtml For +Ġexplo it +олÑĮз ов +XY Z +Ġmagn et +amp ed +Ġat oms +S ources +pect ives +Ñģ ли +Ġ= čĊ +Ġd are +ĠWal ter +Ġbright ness +Ġan notations +ë ı +is ke +S chedule +. images +ros so +Ġ" .. +g amma +Ġin structor +Ġover write +- am +Ġdevast ating +ĠSaint s +Ġh s +Ġbon uses +$ output +ij d +(Action Event +mon itor +Ġmatt ress +Jan uary +.j p +Ġcar acter +Ġim pose +_re st +ĠSign ature +Ġcoron avirus +ãģ Ĭ +_com pare +Me asure +it ated +el ijk +ig os +es ar +Ġrush ed +met ry +_SE PARATOR +_W E +_ATTR IBUTE +Ġy aml +Ġspec s +ĠR ah +ph eric +ĠInvest ment +ä ll +Ġappe aling +Ġview port +ç © +Ġmargin Left +Ġsub tract +ĠED IT +ĉ ArrayList +gr ading +ĠF ailure +as per +EE K +(n ow +< object +ĠAl ignment +ple ado +q tt +( ERROR +ĠIN VALID +Ġuser id +ra ises +ID I +Ġvari ance +ĠN il +/ delete +_M AIN +.T oken +.C ategory +> )Ċ +Coll ision +ĠGre ater +ĠR acing +al an +Ġmon etary +, new +ĠS orry +. Enable +ĠInstant iate +oll en +ë© ´ +ĠCall ing +_h our +AD A +Ġsh y +) ** +Ġ== > +Ġes pecial +Ġinterpre ted +! =" +Ġpharm acy +.s ingle +ĠC ialis +Ġpar as +.to UpperCase +ĠDem on +Pr ime +Ġrank ings +Add ing +_H ASH +ĠEx am +Ú © +ĠVict or +Ok ay +"] ;čĊ +Ġfort une +ĠF ETCH +exp and +.Inter op +Ġb arn +æ ¶Ī +ue vo +Ġspec ulation +âĶĢâĶĢ âĶĢâĶĢ +ĠN u +ĠBl ues +(f name +Ġinhab it +Ġ\" % +C ES +ular io +_c r +Ġvalid ated +Ġmid night +ank ing +Ġincorpor ate +Ġpurs uit +EX P +pr ime +P id +- US +ĠN urs +ĠW heel +é ĺ +Ġin p +Ġsupport ive +.m ember +ĠSh ot +.Check Box +Ġaff irm +T or +Full Year +Ġconsider ably +cred entials +_ opts +R oll +( round +Ġcom ent +_U ART +Ġext ending +R G +result ado +it u +.get Session +Ġattr action +& D +$ html +ĠJess ica +ĠAssoci ate +a ñ +_ ed +ĠL ag +Ġorig ins +()) -> +add EventListener +IAL OG +åIJ ¦ +.Com pare +Al bum +ĠK u +< Q +arg est +Ġpro long +Ġconfig urations +Ġaccident ally +_ph oto +Ġ'' ;čĊ +Ġver se +B ob +Ġfarm ing +del ivery +ĠM ack +Ġuse Selector +.bootstrap cdn +keep ing +en y +. upload +ĠM ETHOD +cre ator +< _ +ĠE aster +. -- +UI Button +ãĤ ī +om eters +Ġsh ine +Ġh ogy +\ s +Ġh arness +.C ell +Ġlif ting +Ġcomb ines +ĠOcc up +ex clude +pat ial +Ġres pir +_f it +Ġfif ty +ĠM ol +Ġtun ed +-d imensional +Ġq s +Ġto ps +> ";ĊĊ +quis ite +ch annels +/ res +ĠAn alytics +.app compat +/ to +Ġon Error +( attr +IR M +Ġrag az +- as +.Se cond +orient ed +Ġdon n +Ġlight ning +f id +ĠP le +ãģ¾ ãģĻ +t ro +.Tr ue +O bservable +× Ļ +umb ing +Ġpros pective +-f ilter +Ġpurs uant +(p oints +.B ind +Ġp alm +clear fix +ö s +ĠG onz +Ġwe aken +Dr ive +en ido +l ld +ob ox +ane an +G ot +ä¿ Ŀ +Reg ex +æ ĥ +Ġsal ad +ass is +" net +inherit Doc +ĠR V +qu ier +Ġcl azz +ı ÅŁ +oster one +Ġair line +.list dir +Ġdownload ing +ĠP alm +w aukee +& lt +.B L +_IN LINE +off s +<< ( +_new s +Ġch ase +/ >< +Ġeuro s +ĠEgypt ian +ĠSt ainless +_BO OL +ĠG uild +ĠD ynam +[index Path +Ġ ï +Ġmemor able +ĠCh ampion +Resource Manager +.Log in +ĠForm er +yp ed +Ġl leg +; ", +D WORD +Ġtax i +Ġbom bs +ra h +.t ags +_test s +st ones +âĢĿ ) +[ g +r type +Ġv u +Ġhost ile +Ch ars +ĠPatri ots +/ status +< B +ĠIn come +ĠD ad +Ġpat rol +_CH ANGE +Ġup graded +Ġch ina +set q +Start ed +.U ndef +Ġcheck sum +Ġfrustr ated +{ o +Ġen f +Ġwood s +ĠAny one +Enc ode +ĠQt Widgets +are as +Ġshe er +sk i +end point +_T est +S oup +~~~~~~~~ ~~~~~~~~ +(f iles +ĉĉĉĉĉ čĊ +.sp ark +Ġval ued +Ġ% Ċ +.control s +ĠXCTAssert Equal +Ġf ame +ĠR ic +D OT +ĠAlbert a +ä½ ¿ +os al +.Web Controls +Ġ ------------ +ĠM is +ĠS YS +Non null += item +Ġexp ire +Dec ode +_ operation +ĠValid ator +.C ENTER +uff s +* m +Ġav ant +æ¬ ¡ +âĢľ You +.per mission +... ) +ĠL ic +_co ords +.n ombre +c lo +.Int ernal +ĠCh o +_s w +ĉ Il +cl k +Ġcast le +(l ayer +p it +Ġgu ided +Ġâĸ Ī +Ġsuper b +Ġsup plements +_c ent +Ġpe ek +IN ARY +.Content Alignment +f alls +")) ; +W all +). čĊ +ĠD anny +irm ingham +IAL IZ +( create +" In +Service Provider +Ġpr iced +mac ro +am ac +. box +---- Ċ +ãĥ « +ĠS uit +ur st +br u +ourn als +num ero +__ ()Ċ +D as +ĠM itt +ud er +? \ +f u +[ B +Ġ: )ĊĊ +(int er +br ains +Ġatt itudes +Ver ify +Ġsign atures +ack Bar +Ġg d +J ack +.c at +Ġz z +war f +FT ER +");ĊĊ Ċ +Al ive +IC LE +ĠWh atever +Ġout lined +s prite +еР² +_A B +_DE PTH +Ġcrush ed +aa a +(e v +æľ º +Ant i +IC O +is EqualTo +.s un +ic ulo +s ale +_h ex +ĠV k +apt or +Un ion +ĠDis count +list a +.Undef Or +Ġautom ation +N or +å¯ ¹ +åı Ĥæķ° +Ġref lex +ĠLa ure +.showMessage Dialog +.t emp +Ġa kan +Ġ__ ____ +.Is True +ARE D +ag le +E nergy +Ġquant ities +âĢĻ é +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġcitizens hip +m outh +Ġin appropriate +ĠOut door +White Space +An onymous +load s +webElement Properties +T en +Ġacc idents +Ġadvertis ement +ĠY emen +(c all +Ġsl avery +Ñģ п +ĠL am +_BIT S +ome ga +ĠO le +Ġkid n +_A n +ĠR aid +Cre ation +s aved +Ġpro port +W ARNING +\ P +Ġp wd +Data Reader +is cher +ade on +ĠP redict +Ġreason ing +Ġdestroy ing +H el +* d +ĠLeg isl +_P r +ĉĉĉ ĠĠĠĠĠĠĠ +Ġsymp ath +Ġch ess +Ġm am +: hover +Ġconvert s +Ġp ela +Ġprogress ion +Ġ"_ " +ĠG ill +ĉ show +Ġsupposed ly +ac curacy +el in +Ġunf olding +ĠHy per +Ġw anna +Ġup s +( # +ĠCr iminal +( Point +at Lng +act ly +Ġcontract ors +'] } +draul ic +ód igo +ĠT T +ĠW ide +ĠAR G +_ ic +FLAG S +S chool +Ġclear ing +-be ing +={ [ +, const +man ent +Over lay +(' " +éĩ ı +ĠT imestamp +Ġmail ing +ĠC ake +.Th at +Ġmed itation +q p +Ġemp resa +ĠL ions +Ġw eld +ĠLinked In +Ġc ush +Ġgen ome +.Index Of +ag ain +Ġf allback +Ġcamp ing +re dd +-strip ed +Ġd v +Fe bruary +ĠPro xy +us k +Ġdies el +W RITE +RE AK +L orem +.In voke +- div +Inter ceptor +ĠD H +ia les +Ġvill ages +Ø ´ +ĠEN V +S ys +.X R +Ġpo em +à Ĥ +c ade +pl ots +Ġ{ ( +.g it +/s vg +nc mp +ĠÄ į +ain es +åĩ ½æķ° +Ġ( )ĊĊ +ops is +ĠRel ationship +_ aut +ĠB omb +ĉ com +* sizeof +off icial +_p ayload +ĉĉĉĉĉ ĠĠ +.m anager +ĠA round +ĉs end +ĠEx ercise +ĠB illy +iv i +Ġneed ing +_url s +_t asks +ĠH em +Ġtear Down +enc rypt +.t ie +Ġas m +IC H +ĠCGRect Make +ìĦ ± +ul ong +Ġit r +ĠG ST +Ġoffer ings +ro be +EE E +oper ators +_PRO P +ind ent +A DE +or f +ë IJ +Ġbless ed +vas cular +Ġcon oc +H appy +B ridge +ilit ation +j oint +ĠAdmin istr +- transform +Ġmeant ime +/ K +ĠBed room +Ġrig id +Ġbrows ers +EM PTY +.S erialize +_ ED +Ġst itch +Ġj an +ell t +Ġbr ace +Ġtr ails +p ublished +å¯Ĩ çłģ +} ')Ċ +Ġac ids +Ġ! !! +_d irect +> ());Ċ +aj Äħ +_O CC +Ġplan ets +æ Ł¥ +ĠDub lin +Ġser ie +.print f +de ep +` ) +Ġ\ $ +ĠÎ ¼ +_V IDEO +end ors +ĠC rypto +F ar +.Trans parent +.T R +ias m +_tr aining +Ġteach es +ĠB elt +Ġlimit ing +ĠK ath +ĠIndex Path +Ġachie vements +Ġser á +interop Require +Ġdis se +.I f +arm ing +uls ion +P o +_DE TAIL +Prot otype +ĠC AL +Ġagre es +.v o +.Execute NonQuery +ĠTop ic +Ġ' {} +Ar m +Ġe cc +M ag +Ġserial ized +ĉ conn +c ached += tf +ĠByte Array +prot obuf +var char +ĉ ASSERT +Ġlist e +_tr igger +· ¸ +Fe el +T ahoma +ĠL ik +Ġstruct ured +erg us +.In itial +_ ge +cl js +.cont act +Ġand ere +$ stmt +_C URRENT +ĠDis cover +$ res +form atter +H a +vang st +Ġem erge +ãĢĤ âĢĿ +ĠCabin et +-s quare +éĥ ¨ +Ġr age +ĠA J +ĠV T +sh adow +ĠFa ith +en ames +pret ty +has il +part y +Ġvar char +Ġf otos +Ġal um +ĠBelg ium +.y label +Ġde j +_num bers +Ġh u +.set Adapter +ĠUs ually +(s ample +.Sh ared +Ġbook ed +Ġ>> = +Ġmin erals +"> +pro g +bo o +_m d +_p ack +(ex press +ut z +\ Auth +, id +ĠCh ile +act ice +Ġrecruit ment +Ġpos es +Ġvulner ability +inst anc +or um +d ess +Ġx l +%%%%%%%%%%%%%%%% %%%%%%%%%%%%%%%% +( fig +Ġdelet ing +.d el +) ')Ċ +ĠWeek ly +?? ? +(str cmp +sm ith +Ġpurs uing +- so +ĠApp s +/ 'Ċ +Ġdec is +FO RE +Every one +Ġl anes +V irtual +. attach +( Log +ĠMed icaid +( Path +ĠTurn er +/ application +Ġport rait +Ġopp ose +check out +Ġfinish es +_M E +Bar rier +S ong +V AR +Ear lier +rell a +Ġh ast +az ar +Ġpull s +ng x +Ġinspir ing +Ñĥ Ñİ +-d irection +Ġexplos ive +Ġcreated At +st o +Ġwhe at +ĠB uilt +' ai +Ġtrack ed +ham mad +RowAt IndexPath +_ heap +D ue +Ġconnect s +.p ublish +em u +Ġbul lets +B AR +ol ate +Ġintern ally +Ġcatch ing +-p assword +ou ched +æĢ § +e ous +Ġx range +Q uality +v v +Man age +( ($ +ac ements +ĠBro thers +ĠHE AD +ĠUn supported +s an +es i +** *Ċ +Ġadapt ation +ĠWork er +'] / +.save fig +( trans +Ø ¬ +ne e +Cor rect +... ")Ċ +Ġsubmit ting +-p ath +ĉ last +iss an +.x label +ĠS epar +/ no +_b est +ĠM ills +_s ock +(f lag +Ġdest inations +em ption +ĠF AIL +å ĴĮ +Ġr p +f act +ĉ len +D AY +Ġse iz +_d st +l ip +.Line ar +ĠB asket +$ t +$ i +- brand +ĠNe il +ĠE q +Ġth ou +og ene +Ġscholar ship +æĽ ´ +Ġs wo +ag inator +en i +( book +Ġbl ink +th us +Ġcancell ationToken +ĠPalestin ians +Ġprofit able +Ġback pack +ens on +< Long +Ġp ools +Ġst icks +Ġspokes woman +Be ing +ĠHer itage +ĠN ike +SH A +ĠNotImplemented Exception +$ core +ĠR ico +/ latest +ĠC zech +ner Radius +(l ines +Ġsem ester +Ġw ounds +Pro cedure +.m ail +() ):Ċ +Ġcor rid +ter ed +ĠN CAA +Ġgal axy +_k ind +il k +Ġtr as +_P OL +ĠH et +Ġrefuge e +Ġteen age +.b inding +post al +Ġiç in +ĠData Type +é ĸ +ycl erview +, value +_id entifier +< b +Ġout file +čĊ ĠĠĠĠčĊ +Ġcr é +Ġrespond ents +ĠBe ast +ce led +Ġinter f +-th eme +g if +ĠR angers +IT AL +Ġauthentic ate +Com pletion +urs ors +Ġcin ema +Ġdisc our +ĠJ aw +OCK ET +Ġpr ayers +ĠL uis +fr ag +=[ Ċ +Ġbr ave +_p ose +C ertificate +- fe +ifer ay +ĠFl ags +Container Gap +ĠC rit +Result Set +ĉc ur +Ġcorrespond s +St aff +.Http ServletRequest +Ġneur ons +ĠMain AxisAlignment +ed ar +Ġg ad +_p arts +ĠÎ ² +Ġf x +/ files +ĠB ros +hip s +Ġgluc ose +Ġfar ms +Ġment ally +rest aurant +Table Name +ĠMer cedes +. Visual +Ġan ch +inal g +_r untime +Ġpropri etary +Ġintent ions +iz i +S lice +; "> true +ĠNY C +Ġb ored +ĠD etect +Ġapp ar +Ġje ans +ĠT ak +I OD +ĠH orse +( FILE +( ? +ri que +optim izer +n at +lo ys +ĉ Token +oub ted +u ess +oco a +Data Member +_P OWER +class List +Push Button +ĠWi Fi +. Stream +.g uild +Ġn og +ĠPortug al +ĠUnt er +Pr imitive +b oss +ĠDe utsch +Ġerot ic +Ġstr conv +.Try Parse +Ġgr ams +.S uccess +_p k +ĠHar vey +-m inded +.c ountry +[] " +Ġang el +Ġbe ats +ĠV or +il io +.m aster +s omething +ĠP ACK +( if +Request Body +Ġant es +/w idget +Ġmod o +ĠA W +find er +Ġoptim ized +Ġmiss iles +N B +ĉint ernal +t ex +ĠS ri +Ġdam aging +ĠM ais +- Allow +ĠZ h +- alt +Ġ ));ĊĊ +è ī +Ġinflu ences +Ġc atal +_REG ISTER +ĠAPI s +-cent ury +Ġbi ology +ĠAct ual +Ġhe els +TR ACE +_D IG +D ataset +ĠM atter +Ġclass ifier +.w ikipedia +ĠRog ers +Ġdon ated +raw ler +en en +Ġcas inos +ort al +Ġpr ive +s pe +duc ers +. ep +Ġgr asp +ac ji +Ġd airy +Ġb uses +.com m +. ins +ĠI RS +ĠBe er +ad c +o ard +_M ET +Ġ' +' +r ans +Ġkind a +ĠâĶ Ĥ +ĠM aur +аР³ +Ġband width +ib us +ĠD ifferent +(m at +ĠRes ume +_UN S +est ablish +Ġfon ction +Sub scription +_com pany +Ġlight ly +.con firm +.y aml +ĠBo ost +Com merce +- template +_DEL AY +ĠH I +Ġn avig +(S ender +ĠH S +_ "+ +ĠRE QUEST +Ġw ifi +=" "Ċ +]) -> +Ġro pe +Ġviol ated +Ġgl ance +ĠK urd +Ġè ® +de ck +ĠIS BN +Ġin fect +ĠF oo +Ġget ter +Ġt ener +ap pe +.h h +_h ot +< AM +p oly +! ",Ċ +Ġconver ting +ĠW WE +RO S +(' { +Com mit +) L +ĠO re +Ġsp arse +Ġdis posal +Ġcan celed +åIJ İ +Ġa er +Ġvin yl +á» ĥ +rec ogn +ark ing +Ġtrick y +* s +Ġproceed s +Ġis o +Ġco conut +Ġcraft ed +IEL DS +Ġquest o +Ġcomm un +_CON NECT +Ġtraff icking +De ep +a ções +c odigo +ve au +Ġbet ray +int a +T ED +æ r +m art +_B US +/ sc +ial ly +Ġcigaret tes +è¯ ģ +(n n +Ġmodel ing +/ products +w arn +Ġmet ro +ĠI v +& ) +ĠC able +Î » +Compar ison +g ary +ĠB A +P ART +Ġp v +_up dated +C redit +orth y +observ able +Ġthe atre +B LE +; }ĊĊ +la unch +_str ings +ug o +ĠR PG +- auth +Ð ł +hol m +ĠP and +U id +Ġim ply +ìľ ¼ +'] =' +/ User +Ġstr cat +нÑĭ й +Data Adapter +Ġland sc +Ġdipl omatic +ï¼ ĵ +************************************************************************ **** +ĠCh icken +Ġbc rypt +.In f +[ col +ĠQu antity +- position +Ġdiet ary +Ġfil mm +Is rael +Pre v +ĠMill ion +Ġrem ed +Ġbill ing +Ġout doors +.t m +Ġn ad +F org +Z Z +Ġs sl +], ' +K T +f req += document +bl ur +¬ ¸ +ĠJeff erson +C s +(s ave +Ġstr ap +Ind ia +Ġide ology +BO SE +ĠF P +( ans +Ġfe ver +ĠY am +K ing +à ² +AT ING +bo hydr +roll back +Ġnew Node +ĠN VIDIA +Ġhon our +ĠCon firm +xb d +Ġsuccess or +/ u +l iv +ourn aments +Att achment +Ġgr up +Ġtri be +Ġca res +e ft +_s ame +' label +Ġ ãĢIJ +M otor +Ġin exp +Ġ" (" +_POS ITION +Ġval ley +ĠResult Set +Ġpres erved +Ġmut ations +Ġquestion ing +mun ition +parse Int +ĠS r +ĠMet adata +âĢĿ ï¼Į +timestamp s +Ġtrans itions +í Ļ +Ñ Ĭ +i om +.D o +Ġp ine +Ġf ung +Ġtrans mitted +ct ime +ĠF am +Re vision +B as +UP ER +D estination +toHave BeenCalled +Ġun fortunate +IN ES +_pro f +Am ong +ĠCy ber +ĠB attery +gen re +ĠView Model +- = +Ġutil ized +p aint +.Integer Field +ern ity +comp iler +âĢĭ ĊĊ +ĠM asters +.To Array +Ġstrt ol +ĠUkrain ian +} ));Ċ +Ġsh emale +" That +for all +/ download +Ġrhet oric +.l atitude +ĠWH EN +Ġshock ing +IF IC +.N ormal +_F OLDER +Ġdr ift +Ġmount ing +- book +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ĠWire less +> ".$ +Ġrel ies +( Console +Int ernational +-> {$ +M id +Ġdis sert +dd s +Ġdepos its +ĉd river +# ga +pr ising +print ln +Ġpres enter +Ġmin es +C SS +ĠD ual +(! ( +Ġk am +Ġis Loading +ĠProt ect +. upper +ar ium +]: ĊĊĊ +Y ii +-sh irt +ĠIM AGE +_color s +Ġur gent +.Cont ainer +! (Ċ +S aturday +Ġsoci eties +ĠTh an +ĠC od += @ +Ġattach ments +.m obile +Ġsp ite +Ġb ounce +raw l +instanc etype +ĠTr uck +Ġmanip ulation +( Config +-in st +Ġst or +it ution +Preferred Gap +Ġmain AxisAlignment +Ġlist ened +'' 'ĊĊ +ott age +- project +.AP PLICATION +ĉ root +Ġwh it +Ġb ilder +Ġk er +Ġappl iances +row ave +ìĿ Ģ +ematic s +ĠO rg +op ing +_SE ARCH +Ġch am +add ContainerGap +Ġ( ). +ĠAr row +Il legal +Current ly +Ġus a +Ġpassword s +Ġre nown +av ern +ĠEv il +Ġconc at +Ġdu o +Ġv ale +ĠBe an +Ġindic ators +cm ath +ĠP ump +Nov ember +ific ant +_DOM AIN +reg ar +ĠPort al +" $ +Ġformer ly +"] :Ċ +ĠVis ibility +.getElementsBy ClassName +_RE D +Ġch ampions +à ´ +Val or +_ es +* a +-re peat +B and +.st age +Ġbure auc +C nt +et en +- function +Ġm uito +P ID +_ editor +Ġcrash ed +de ad +k at +ag h +ĠEX T +ass er +-sm all +Ġreal iz +( Entity +ú s +ĠAct ually +ĠEl ite +Ġhel m +(non atomic +ash er +Comm unity +all eng +ir y +ĠG rowth +Ġs ue +Ġfrequ encies +_des criptor +.At tribute +Ġrecip ients +_N S +/ "+ +ib an +Ġath lete +ĠI gn +_D MA +(d s +ĠRequire ments +AD I +ere z +\ Admin +br aska +ĠR ust +Rel ation +C OD +ĠV ERSION +em ma +)) { +.D uration +ĠC amb +- logo +Ġread able +Ġcre ators +() ];Ċ +Up Down +-h alf +.get Month +(s f +P ic +Ġhun ger +.t x +Ġexceed ed +_se ed +( ^ +_s k +.per form +Ġ> :: +Ġm ongo += float +bind Param +Sm art +if a +Ġse curities +Ġpre jud +Ġ, " +Ġcor ps +Ġv ra +amac are +it err +(M edia +uch e +Ġc ob +Ġlib er +. geometry +Loc ator +Ġsl iding +Ġsurg ical +_C UR +Ġcon sect +[ * +ĠRes ort +St ub +_DO UBLE +ĠS oph +Ġelect oral +_dis able +ĠÑģ о +ĠLight ning +Ġment ions +oc y +Ġle aked +Ġrelax ing +Pres enter +v sp +Ġgu ilt +=- =- +.re ply +ĠMir ror +C amp +Ġ+#+ #+#+ +Ġ+#+#+#+ #+#+ +.A uthor +Ġdirect ive +-h ook +íĦ ° +}ĊĊ ĊĊĊ +@ pytest +_r and +m is +Ġcolor ful +u je +lass es +ĠClass es +.h ave +% ), +é¢ ĺ +Ġdistur bing +sub string +ĠK oh +In vest +p urchase +Ġrec ycling +ĠA RT +ier archy +Ġf ps +.check Box +íķ ´ +_m aterial +duc ation +Ġf w +ud it +Ġreview ing +ĠS id +S yntax +ĠW ritten +arg ar +UM E +/ q +Class ifier +Off icial +Ġj azz +Ġom ega +Ph ysics +Ġl ugar +_access or +.command s +Ab ility +ĠB atch +R AM +Ġencount ers +. Qu +BY TE +ĠD istribution +Ġus o +ĠReco very +appro ved +Ġden ial +/sh are +Linked List +)čĊčĊ čĊ +udd y +Ġf ines +Ġr y +Un icode +ĉ render +Ġprem ises +Ġp on +ali ases +/F oundation +c uda +ĠC ock +,: ) +(f older +Ġm éd +dr ag +Ġtal ents +ĠĠĠ ĊĊ +е ÑģÑĤв +m ob +.y ml +Ġa ster +Ġdis cre +go al +ĠGT X +ĠS UCCESS +ĠL ONG +(f ind +Ġsing ular +_s z +ĠEth ereum +.. Ċ +Ġir res +')) {Ċ +Ġmin isters +St eps +ivers al +ĠNever theless +- led +Ġ( %) +ç¡ ® +Ġtime zone +Ġstr anger +(re nder +Ġsh util +Ġm ph +Ġtri o +pp y +Ġpred omin +Ġend ors +ĠRuss ians +ĉ row +Ġw izard +.s erialize +Ġcompl ained +Ġs ido +Ġdelight ed +-m e +ĠR av +H uman +ad ays +rec v +Work ing +J ump +ĠÃ¥ r +ĠAut omatic +_B ase +æł ¼ +aur ants + ¯ +æ ¸ +(C Type +IF I +( amount +Ġbelie ving += mysql +Ġf ir +Ġrest oration +ere co +Ð ¢ +_ '+ +Ġe book +Ġde bris +(input s +AY OUT +Ġscre aming +av ia +land er +Ġdist ress +Ġas sembled +ĠA void +( thread +ĠR PC +_EX IT +( queue +и ÑģÑĤ +D ll +Ġsk ull +_p ub +che z +min ate +ens en +Ġins ane +b ounds +ĠR osen +Ġcondition ing +process ed +v ideos +f our +.Con v +| ;Ċ +Person al +cer pt +:UIControlState Normal +Ġdos es +ĠKar l +ĠFre qu +.B ASE +ĠV ote +Ġcon current +ĠMessageBox Icon +Ġà ĸ +ĠDub ai +ĠR etail +: number +ĠOb server +ĠBig Integer +_ origin +_W ORK +F rames +Ġnot ably +. âĢľ +Ġtrop ical +Ġn iche +am ina +.s ys +(t okens +mod ify +os it +st rom +ĠCom ics +O PTION +T icket +Ġfact ories +Ġdis put +_F ile +ĠFin n +ee e +ĠDisc ord +_m oney +.t pl +_s afe +L B +Ġgl ut +J K +.fl ow +- cont +g os +Ġhor izon +ĠR ush +:: * +P ipe +ull a +bor ough +he imer +(m ove +( Text +} );čĊčĊ +w elcome +ĠCom ponents +Ġgovern ance +c losed +ĉm argin +Ġla undry +ĠTerm inal +iz ards +. âĢĶ +.rem ote +.r adius +ĠQue bec +Ġd h +T ech +ĠM ist +s eller +_l iteral +Ġgen ius +Ġbr ains +g em +ĠMe asure +Ġcata st +r ance +.Text Field +Ġconsum ing +Ġ'\ '' +oubted ly +ĠC ertain +E v +ert i +be ing +Ex perience +Ġ// [ +ĠArab ic +ĠC rist +ĠAz ure +Ġhor a +l adesh +\ Blueprint +d ar +.re l +Ġsup rem +ĠRe agan +ĠAt tributes +-s idebar +Ġuse Styles +ĠA irlines +Ġh ills +/x html +v inc +_m ock +Ċ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ +ĠP ill +.Layout Style +ĠCommand er +] < +sign ature +Ġ{ }čĊ +Ġhat red +Ġë ĭ +ole sterol +Ġ ******** +ancell or +c rop +T IM +ĉĉ ĊĊ +ys qli +uit ive +ĉun set +_s el +Ġmen us +t ick +Ġconstit ute +ĠElement s +ĠRed is +agg io +_f p +_de pend +em as +CA ST +or ange +j on +ĠEm ily +Ġpot atoes +Ġre ceptor +ĠElect ronic +ĠL ights +Ġcomb ining +ĠSome one +Ġ######## . +ĠT OD +/ show +X d +." ' +af x +Ġtr agic +St yled +ĠMar co +G allery +d ale +.âĢĿ ĊĊĊĊ +é rie +/s ervice +äº Ĩ +Ġamb ient +_SET TINGS +.Ad apter +l ene +Ġtrav els +Not ice +Ġcle ans +ĠF em +ch air +Ñĥ н +/ my +_b ad +ĠEcon omics +IS A +_C NT +(M enu +äº İ +ĠR idge +Ġlength y +D ot +Ġjump s +Ġhe y +$ pdf +Ġw orm +Ġs ut +Ġsh er +iam o +ĠCal c +trie ve +Ġc ops +ĠCh rom +Ġreg ulated +reat ment +ĠHigh er +ok s +Ġde ze +LOC ATION +ongs To +Ġfin ite +Ġvar ies +Ġposition ed +' il +éĩ ij +Ġh ike +(d one +play list +Ġad a +Ġcoast al +ĠN ancy +.DateTime Field +Cpp CodeGen +ĠSimilar ly +re ur +ĠCon tr +ĠH idden +ĠB eta +atch ed +_inst all +. Output +Look up +ĠRich mond +qu ared +Ġm anga +-control s +ĠBern ard +L arge +Ġslic es +Ġoff ence +ĠM ega +Ġest ar +Ġjoint s +Ġsum m +_pl atform +B uff +.add Subview +Ġret ained +Let ter +.d im +Ġess ere +ĠS caffold +EX PECT +ĉ RE +.long itude +ü nd +Ġstat ue +.add Widget +ĠCar ibbean +add PreferredGap +il de +UIL abel +ĠOp port +Ġimper ial +urs ion +Ġmand ate +Ġpromot ional +Ġv k +ia ÅĤ +Ġp yl +ĠCre ation +оз д +Ġsim pler +. what +ĠRec ent +St orm +. quantity +ĠL ov +" - +ubb les +_not ification +(w orld +ur ger +* (- +: "Ċ +h m +ans hip +ĠAl most +Ġmotor cycle +_f ee +Ġabsor b +ĠVin cent +Ġsound ed +ÃŃ st +Ġpharm aceutical +ht ag +ĠKind le +ital ize +ĠEm peror +oust ic +Ġspecial ists +åħ ¬ +Border Style +/ \ +RE LATED +(', ', +(ex pr +Ġh t +åį Ī +_C reate +Ġspecial ly +Ġ[] ;čĊ +Ġhe el +Ġse pt +_ arch +(in itial +% .ĊĊ +\", \" +Ġdiscuss es +Ġu pt +Ġ[ & +Ġman us +.h and +ĠM AIN +ĠDen mark +Ġ], čĊ +Ġcr yst +Ġn ack +Co ords +_in ner +Ġmid st +Ġaw ake +ĠÐ ŀ +-b reak +ÃŃ vel +_P ASS +ĠParam s +Ġdet r +Ġsp ider +ĠCon cept +Ġpre nd +CH ED +.Ex it +Ġpop ulated +Ġvirt ue +_SE SSION +Ġnou vel +o auth +Ġд аннÑĭ +r ink +.Header Text +atur ated +Ġer st +Ġå ħ +ॠĩ +_vis ible +ey er +Ġli able +Ġde be +Ġb w +{- # +_W IN +df s +H over +ĠP UT +- angle +Ġnob le +Ġtr aces +enc v +Ġuser Data +_in s +ĠS uz +Ġnews letters +ĠMod i +Ġentreprene urs +Ġtrib ute +Ġrum ors +Ġr r +ĠQu arter +ê³ ł +Ġfeed s +ó g +Ġen velope +Ġle ar +Ġk ø +develop er +Sim ilar +: ")Ċ +sub scription +Mod ifier +ital ic +Ġn asty +Ġtermin ation +Ġchar ming +Ġâ Ł +ton s +.tr ace +h ots +ĠU R +M ont +Ġjust ified +ĠG ang +ine a +Ġb og +( ap +_ $ +Ġcont amin +.D ot +ĉ Debug +( exports +Ġpa ired +ĠAss ignment +Ġautom obile +ĵ į +Ġph ases +v w +@ SuppressWarnings += \ +r ant +- ed +ĉ await +Ġcert ificates +'> " +Ġint act +CT RL +M ike +greg ation +AT TERN +Ġre public +_up per +ili ary +Ġcomput ation +h ire +ĠSh in +_ ANY +ĠManufact urer +ĠC arm +Ġbear ings +_c omb +c ad +ur istic +Ġwholes ale +Ġdon or +.inter faces +press o +ĠBr un +-c lose +pro ve +_S K +ĉf rame +et ros +ĠP ain +_EX P +ĠL T +_f s +.dat as +ĉ ss +vo ir +ĠA xis +M ajor +=" < +[ h +Ġprof ess +igr ate +(s core +Key word +" os +ĠĠĠĠ ĉĊ +an alysis +Ġre play +.p ass +\ d +t ls +Ġsan ct +.l ight +_m obile +ÑģÑĤ ÑĮ +ĉt otal +u ity +Ġpa used +N AS +Ġen core +lo e +Ġ-* -ĊĊ +.h igh +am pler +ĠSec ure +Ġfrag ments +_ vel +ill ary +ĠSte in +ĠD awn +Ġmax imize +ภ¢ +Ġ/ ^ +Ġcontin ually +Ġsh adows +ĉ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +ĠI ActionResult +Ġinform ación +C HECK +.Selected Item +b undle +ol ley +< Int +AIN ER +ĠW ing +tit les +ount ain +C Y +ĠLoc ale +form er +< context +R adioButton +_s chedule +Ġfab ulous +Rob ert +_PRO FILE +Ġg ates +IM P +ĠPent agon +g old +b ach +employ ees +R otate +Ġch amp +Ġsel bst +Al tern +Ġconvert View +/ , +Ġ~ ( +St reet +_ place +Ġpersonal ized +P ublisher +ĠSO CK +_NAMES PACE +ĠStand ards +so ever +_C ENTER +Inter est +ô t +tem perature +View port +get Resource +Ġeat en +Ġsem pre +Ġab normal +Ġc ylinder +Ġtroub les +n od +Ñĭ в +g ames +_g l +Pl ane +g rey +_t bl +.Component Placement +ĠCh ase +Log ging +man y +ì Ĩ +Ġfl ame +="< +Ġtra jectory +_r ing +Ġhydro gen +tr on +Ġstat ute +Ġcondition al +Ġtr ay +-s chool +(w idget +$ config +Ġrequest ing +. uint +et on +brit ies +Of Type +AD MIN +p redict +Ġg egen +ĠH app +OC UMENT +ĠA part +Ġ---- - +ro e +u ide +just ify +ĠSqu ad +Ġprof es +.b ot +_c urrency +inn en +ĠM umbai +ĠNum bers +avana ugh +agn itude +âĢľ There += http +çī ĩ +Ġv b ++' {{ $ +Ġin ode +s il +Ġh ace +Ġsever ely +ĠOver view +Ġspr aw +Ġbeach es +: left +· » +($ { +ĠF IRST +ĠSp a +- ass +Ġb aise +ĠN ODE +ĠP izza +P et +(se q +\ ">Ċ +CppMethod Pointer +Ġv p +Ġi a +_se conds +em et +/b lob +_TH RESH +... čĊ +D est +ĠN H +.data Source +it és +ĠJ ak +s ell +Ġwork shops +< u +Ġr ivals +ĠEX ISTS +h om +-t oken +compat ible +.J Panel +Ġphys icians +art in +Ġdes irable +Ġdistinct ive +.D ep +g id +ili ate +, max +Ġprem iere +Ġq Debug +Ġadvoc acy +Ġwh isper +P t +Ġun changed +_q ty +请 æ±Ĥ +Se ason +avel ength +ĠP ul +Ġd ÃŃa +'] ]],Ċ +al is +(" & +bor o +Ġb m +ĠR adi +w rong +ĠGo ing +ime Type +ij i +- feedback +ĠN ames +ĠB apt +Ġprob able +ĠE ther +ĠPolit ics +_prot ocol +lin ing +S at +Ġcor rel +.Pr imary +(null able +RI ORITY +Ġcolor ing +Ġutil izing +d as +Ġexport ed +Ġcar riers +Con v +. editor +i ó +(h andles +Ġapprec iation +. import +ĠAust ria +ĠStr ip +il ight +Ġappropri ately +ĠP rest +ĠW ir +ĠUI Application +al chemy +ĠM ob +ĠD etermin +ergus on +register ed +_con vert +ĠVlad imir +.Show Dialog +ref lect +Ġsh ook +Ġass ure +ĠO ften +Ġcivil ization +Ġvocab ulary +fore ground +ĠS cope +Ġunw anted +act ing +Ġ( [] +Ġmark ing +. original +ĠMO VE +Ġsport ing +ception s +NS Number +S izes +Ġprovinc ial +_Tr ans +Ġproblem atic +d igit +ĠEm ma +lock s +ĠC rew +ib a +') : +ish a +Ġm amm +Ġocc ured +w cs +(r ule +Ġmerch andise +es pecially +ĠT win +Ġn aming +Ġs log +Ġimpro ves +Ġad her +: text +.h adoop +_HT TP +.to List +.dis abled +Ġl enses +.in i +ĠR are +ĠUb untu +Ġsc ram +ol ation +tit ulo +Every thing +Ġnod ded +icht ig +_const ant +z c +l ift +ĠNot ify +ond o +ĠIN F +(" + +ĠK az +Ġd read +.m apper +le ur +ĠCome y +ĠN B +ic ers +.P ush +ĠH ack +ĠBrazil ian +_pro d +Ġ// ĊĊ +Ġb icycle +Ġun available +Ġadoles cent +bl k +Ġmit ig +_bl ue +ì ĺ +fade In +ĠUtil ities +ĠM N +; k +< style +- status +ind o +Ġinn ings +Ġg j +Ġ|| = +.e u +: Number +Ġcuis ine +ĠURL s +ie k +Ġw ires +ĉ ps +ie g +.m k +so ap +Ġsom etime +Ġst ap +_s eries +.T arget +æ º +.dest ination +OUN TER +R aises +& A +Ġsmart phones +NI Env +.s dk +Ġhelicopt er +Ġim pe +ĠB irth +A U +b readcrumbs +co ords +Ġexplo red +Ġl od +ĠI p +g able +ian e +Ġart ifacts +Box Layout +ا ر +list ener +.c art +ĠH uff +ĠHind u +ĠData Types +ĠDr upal +IGN ORE +Ġoffset s +ĠR TC +- login +æ ® +ĠQ Object +Ġprosec utor +R ock +_ch at +W ay +ì ² +Ġneg lig +Ġd ude +; < +Ġdeleg ates +_f ailed +/ dev +/ work +( New +et able +() " +( Icons +Ġp ork +ĠModel AndView +ĠV IP +ĠK or +m ix +Ġox id +ĠSC REEN +ĠFour th +/ ",Ċ +Ġte e +ĠSte vens +t icks +Ġp ledge +ib bon +ĠLo an +Ġne o +n umpy +ĠShared Preferences +- oriented +ĠLogger Factory +ĠGraph QL +zen ia +" _ +W omen +.c ast +Ġdeliber ately ++ b +ĠAr n +font Size +Ġm aze +Ġbl amed +.m as +} )čĊ +eler ik +Ġsc anning +ĠWork shop +Ġfind en +Ġca ut +UI Font +( return +al in +cast le +//////////////////////////////////////////////////////////////// //////// +Ġincent ive +op ath +b lob +Ġcigaret te +Ġfert il +*/ ĊĊĊ +ĠSh ar +Ċ ĠĠĠĠĠĠĊ +Ġunc ertain +ĠS ton +Oper ations +ĠSp encer +Ġdef in +ĠS olo +on est +·» åĬł +Ġu omo +G ive +Ġdent ro +; padding +ent ai +ĠC ars +Ġenthus iasm +ĠOper ating +S kip +par ation +Ġprotect s +Ġre ver +d g +ĠC incinnati +Ġconsect etur +Ġm uss +employ ed +a uses +ink le +. Values +£ ¼ +lo v +_W ARN +Ġbook mark +ĠAp ollo +. axis +Ġm ét +Ġop ener +Ġtum or +d an +Ġelement ary +Ġsk ipped +ĠK er +as ia +_res p +Ġdem ol +ĠCan adians +Ġt astes +U Integer +Ġ' ${ +.aw s +RO ID +ri ans +M Q +ord able +Ġcous in +Prop agation +(S ession +ph alt +UL D +ĠSc alar +Ġblo ody +Ġ ঠ+.m ask +, q +ĠUn its +Ġcent res +ĠPr im +. ]ĊĊ +ĠSh aw +P rom +ĠTh ought +Check er +_output s +( chan +E INVAL +Ġb ob +_c mp +P ed +Ġmat rices +Ġvrou wen +Ġgenu inely +high light +(d isplay +) != +Ġdel icate +ĠL uther +ĠM iles +Ġuser ID +% = +ate urs +_B UF +---- ---Ċ +imit ives +Ġsh elves +sl ow +_in formation +LE G +W r +.form s +cel and +/ un +: & +.âĢĻ ĊĊ +=" % +Ġpro st +Ġfont size +uc ión +get ic +am t +=" . +Dec or +B rit +Ġ"" ). +Ġfound ing +.File Name +ĠT ier +Ġdisc lose +á m +.s yn +.View Holder +lic ant +_st age +Mon day +Ġdes erialize +t alk +Ġtradition ally +æĢ ģ +Ø ® +LE X +Ġe h +ĉ ROM +Ġ{ })Ċ +Quest ions +nc py +Ġfix ing +к Ñĥ +_ Key +: x +ĠSTR ING +ĠÑĦ ай +ĉ left +ĠBen ch +ell ij +UR RED +ĠDi agram +} catch +/ time +ĠMiss ing +db name +Ġs ore +ĠW alt +ugg ing +rep resent +ĠG S +ne ys +ĉ page +Ġvol can +(b tn +Ġexceed s +Ġ erg +Ġpil ots +ĠS ed +ers ions +Ġpat ron +R V +/ top +. asset +_c ross +. Editor +.t b +Ġwel coming +SC REEN +) findViewById +C oder + ",Ċ +_P in +ues e +Ġover rides +_ ready +Adv anced +Ġop i +-c art +("/ ", +ĠDe b +CR Y +ĠVert ical +ĠO VER +ĠCorpor ate +Ġ"" ; +Ġste pping +e j +Ġaccus ations +Ġor az +_t ail +Ġindu ced +Ġel astic +Ġbl own +, // +Ġbackground s +âĢĻ une +-s dk +Ġset Interval +Ġincent ives +Ġveget able +_ On +exp anded +p ix +_sh ader +ĠSP DX +@ example +ĠW rapper +.Z ero +Pos itive +Ġsp inner +Ġinvent ed +ĠG ates +оÑĤ оÑĢ +Ġcompar isons +è · +.pr imary +data Provider +add itional +ĉ options +s napshot +.set Horizontal +Ġ" {} +ĠFish er +hal ten +< Type +Ġmax Length +ĠM t +Ġê° Ģ +.jet brains +Ġident ifies +Ġflow ing +ĠDisc ussion +ats by +Ġsch w +ught y +Ġr ivers +.un ique +_PH Y +ed ral +( ll +Ġcs rf +pp ers +ü l +ĠEs pecially +port ed +ĠHarr ison +****** */Ċ +Text Color +ìĬ µ +w ire +Ġstatus Code +ĠFin ish +c ence +ĠMcC ain +ĠW or +( await +Ġ) -> +ĠRegister ed +IN ED +k al +par ison +Ġobj eto +V i +mand a +Ġrenew ed +ĠS of +ess el +.nd array +Ġcr ap +ç® ¡ +.ab spath +( up +Ġclear ance +ĠT W +_C OPY +ĠĠĠĠĠĠĠĠĠĠĠĠ ĉ +Ġforest s +Ġarg uably +ĠA SS +he y +am el +_f ore +ĠSou theast +Ġab used +Ġpract icing +aked irs +ä¸ » +_res ources +Ġp ond +.F ixed +Last Error +ĠPsych ology +Ġ" // +! : +Re usable +Ġmens aje +Ġro spy +Ġb our +Ġvar ieties +Ġem path +(( { +_ org +ĠM es +ĠMag ento +IST ORY +Un less +Ġh j +ĠD uty +J un +, size +Ġpaint ings +Ġdisp ens +d art +Ġbehavior al +Ġr pc +cal culate +fr uit +_m m +ĉp thread +Max Length +Ġc urrencies +_cap acity +ĠO z +Ġfire arm +Ġcoeff icient +Ġbankrupt cy +w art +Ġfat igue +AV A +Ġes pa +_p c +ĠQu otes +_L IGHT +ĠT ickets +Ġrel ates +Ġpublish ers +Ġunlock ed +Ġ// ---------------------------------------------------------------- +ĠInterrupt edException +Ġout look +r n +Ġreb els +W ritten +Ġas ian +ot to +Ġ ĉĉĉĉ +_g pu +T xt +.Image View +Ġsu is +_t ables +.Rec yclerView +Ġwhat soever +è ģ +] ++;Ċ +assert True +_ verify +ĠR ivers +Ġ ][ +J et +id ian +S ibling +Ġgen res +.A ccess +OP S +Ġtr ivial +ภª +al en +в ед +ĠS word +Ġscrut iny +(c b +Ġcomm erce +Ġguarante es +_ad v +ĠL ET +rec io +Ġh ilar +Ġback yard +ãĢ ı +Ġillustr ated +/v endor +. Util +Ġw ow +LO Y +ĠMar shal +"> '.$ +ĠB ak +Ġmod ifiers +d ictionary +ĠSt re +m ultiple +")) , +ĠC ort +'] "). +( admin +ĠCre ator +Int ernet +( ms +log y +DECL ARE +ĠMarc us +<< << +ãģ ł +_m y +(in st +Ġsc iences +ND ER +. enter +Ġit u +Ġbeh ave +P an +omb ies +=' < +')) ;čĊ +ĠM ENU +ĠWork ers +.No Error +Ġbind ings +Ġdis abilities +{ \ +ĠM unicip +Ġco res +ur ple +ĠN okia +us ions +ĠF itness +.handle Change +Ġjav ascript +ìļ Ķ +( dec +Ġpack ing +-de pend +Ġtrans cript +z eros +_ alert +? ",Ċ +lib s +± оÑĤ +Ġ| ĊĊ +tr ained +ĠG ent +ĠR ab +x p +_config uration +å¤ © +_ accept +.rec yclerview +: url +ĠMu hammad +Ġprivile ges +_b ank +uk u +w allet +ĠRO OT +Ġenc uent +? family +ĉ position +Ġc g +Ġprec ip +method s +_f ast +in crement +ĠT iger +_OCC URRED +qu ip +ĠH AS +_d om +Ġw reck +b j +Ġd ern +Ġorg ans +. entries +Ġ_ (' +ram ento +ĠJam ie +Ġp unk +IP P +Ġprogram a +Ġatt ain +Ġpro ves +/s ign +Ġanswer ing +Ġl adder +************************ **** +ĠW almart +ĠCONT ENT +duct or +Ġver bal +ĠP ID +c rypto +_CALL BACK +Ġ= ================================ +Ġpot ent +Ġshort s +.U ri +.un iform +; border +ĠW er +Ġhere in +ll a +ĠI hr +P ixmap +l iteral +! )ĊĊ +g eneric +r ust +_script s +ost o +it us +ĠCoal ition +Ġrem ot +de ploy +ĠEag le +ãĢģ ãĢĮ +Ġimportant e +ĉ object +Ġseason al +ne j +aid u +Bind View +ĠSi erra +-b g +Ġmake Styles +[ offset +G ames +Ġhorm one +AR IO +head s +( select +ĠStart ed +@ param +_de cl +_b log +Ġa ño +\ Api +ĠMil waukee +Pro vid +An imated +Ġcool er +ĠSe ed +. Edit +Ï Ħ +ĠT aking +Ġborder Color +-found er +.Logger Factory +Ġ"" ĊĊ +AL T +ĠL ate +EDI ATE +Ġ);ĊĊ Ċ +af a +Ġcancell ation +At om +ĠB irmingham +emp resa +HE MA +asc al +Ġup side +.V ersion +ĠF older +ĠE ight +ĠV intage +ĠApp Delegate +ĠPre vention +.se parator +ST M +( room +gener ator +Ġc attle +ĉ Z +ĠPart icle +' };Ċ +Ġneighb ours +ĠState less +Ġalt itude +Ġsa int +об ав +Ġconv inc +ĠCont ents +Ġje une +(t s +Serial ization +(c ollection +ĠJ azz +ĠD od +ĠR och +ac io +comm ended +DEF INE +.on load +Ġspecial ty +PL ACE +_MO VE +Ġaccount able +Re uters +Ġf icken +Ġde pr +W ow +V oid +.s pace +à¸ Ĺ +Ġt q +ĠP ets +< $ +(C urrent +ber ries +plan ation +Ġlist Of +ĠTh u +ĠPR INT +Ġm ismo +Ġdo i +ch k +ĠUn icode +( role +Ġvir gin +< Point +_RESP ONSE +-h ouse +ĠVenez uela +EM AIL +Ġp úb +_ex ist +B all +.C L +re ferences +ĠBeautiful Soup +ĉ Expect +TH IS +Ñĥ д +b ane +Ġtemp oral +ER IC +et as +Ġrefresh ing +Ġsec ular +@ synthesize +ac cur +Ġn ella +ĠS OL +.p ipe +Ch annels +èĩ ª +Ġinsert ion +á» ĭ +el ia +Ġadjust able +Can ada +ĠI TEM +Ġcur ves +ĠChe ap +let ing +Ġoptim istic +al lo +Ġpolit ician +_down load += edge +ORT H +Ġmodel o +art o +. rotate +Ġs elenium +æĪ ij +_al ias +Ġrenown ed +.' . +Ġc zy +Ġal les +.Com piler +ĠB ass +Conn ector +.R ole +L INK +Ġc riterion +lem etry +Success fully +/p ng +Ġey eb +asp berry +( gr +Ġd angers +Ġcorrect ed +Ġgl ow +Ġelabor ate +ĠB ears +aw ai +=" '+ +Ġpromot ions +Ġmathematic al +Ġ" ` +_Generic Class +ĠChe f +.S ort +table Name +R IC +Ġvolunt ary +ĠBl ade +-e lect +ĠCom bat +ĠAb ility +Ġab dom +Ġd uck +T mp +åħ ¨ +Ġer ase +.P h +ĠDefault s +p artment +_US B +ê te +; ' +Ġp ads +ĠOb amacare +.T otal +Ġdiv ert +Ġcr icket +Ġrecre ational +( red +ĠC le +R U +Ġmist aken +ĠMont ana +Ġstr ive +_sl ider +ĠPl astic +Ġdecor ated +ĠV P +lic o +ĉf alse +Ġpre fs +( \" +_f alse +i endo +Ġ@ $ +B ucket +act ical +ĠZ hang +.c ols +.B inding +Ġw ax +_ST ORAGE +Ġlaw n +Ġr f +.Sc ene +ĠCal culator +.d esign +Ġres il +л ем +E mploy +ĠPr ices +ĠP WM +ag i +.e valuate +ĉ param +Ġbr ass +bb en +Ġinflamm ation +ull ivan +Ġan not +Ġp H +iam eter +ĠB TC +( box +Story board +Ġcl ay +.assert Raises +| string +.App ly +Ġmatch er +und ed +Ġsatisf ying +Ġìł ķ +Render ing +_app ro +ind rome +AN EL +_f ix +br ush +.M atch +Ġsm iling +on aut +S unday +Ġdelet ion +Ġencour ages +P ull +Ġreven ge +Ġqu arry +tr ade +Ġc ables +(d elta +ites pace +Ġf h +.b unifu +Ġvi el +_IN CLUDED +ĠT ail +ad ar +of s +Ġmet als +g om +_method s +Ġn j +.St d +(w in +$ (' +Ġt urtle +ur on +Ġen rolled +ĠH z +ĠBox Decoration +Ġp ont +rel ationship +B i +³ » +Ġmas cul +Ġsh ades +Ġv r +ĠLog ic +Ġa in +ĠD IST +Ġcoll ar +" profile +Generated Value +ĠP ossible +Ġe ines +ĥ ģ +.time out +ĠE c +Ġjer sey +.D ouble +Ġqual ifying +v or +CRE EN +_A pp +_rec v +Ġali ens +It s +E sc +i ator +ĠE clipse +Ġg h +V ict +ĉ html +to o +. const +Ġant erior +ĠW u +(key s +Ġul tr +_p oly +ĠT ap +ĠB ud +A WS +Ġcrash es +_t ot +Cont in +-h anded +alth ough +ภļ +ific ent +Ġde ve +ut ory +ĠW orth +_M S +Ġfloor ing +Ġsell ers +ĠThank sgiving +Ġp ng +Ġval ores +Ġslee ve +Ġfil le +Ð IJ +Ġappoint ments +Ġv im +User Info +BO OST +Ġpos ed +initial ized +.product s +ĠLeaders hip +man uel +' % +em arks +Per centage +(d ist +. avatar +(h Object +ä» Ĭ +_ iff +ic one +; ) +_n il +Ġab ol +е ÑģÑĤ +Ġven ues +.Con vert +! ')Ċ +.B itmap +sk in +_C OLUMN +Re v +G RESS +g ow +Ġw ished +tract s +.assert False +Ġscreens hot +Ġfo is +Com b +Line Width +ĠGr ab +Ġint ensive +ĉ sh ++ ) +.first Name +_PRO CESS +Ġt ilt +it ored +.L OG +Ġb ak +Ġintention ally +.play ers +(c anvas +)) )čĊ +.Pro vider +_P UBLIC +T alk +ĠL iv +ched ulers +Ġl c +ad ic +feature d +.res ources +Full Name +Ġmean while +B uffers +Ġres olver +ĠS AP +_T E +G NU +ĠForms Module +_ wh +ĠS we +.widget s +Ġcabin ets +Ġsus cept +ĠB ott +activ ex +av ar +ant ics +Ġ" =" +_k wargs +Ġgame Object +ĠAng le +.I ter +mar sh +ĠB irthday +ĠC MS +request s +ĠPear l +_E OL +Ġlin ux +( org +_M ouse +.con structor +Ġz d +Ġk icks +art isan +Ġe ax +K n +pon ge +ĠFin land +Ġmet res +ĠAss essment +part ner +/ pre +! ',Ċ +[ Int +Ġos lo +date picker +/ String +op lay +ĠHe brew +, double +Ġtrab al ++" \ +ĉ EIF +/ text +_F IRST +ĠP ete +Ġe go +Ġextr as +P DO +Ġreg ulate +ĠQ Widget +st s +ĠSh ows +ĠN HS +.c ourse +p thread +ĠF uel +.t imes +Ġ ° +Ġstr ides +($ ('# +( words +Ġrhyth m +Ġsp ont +Ġsens ation +Ġsp ike +C losing +页 éĿ¢ +N umeric +Ġbreat he +Ġfin ale +_F ACT +in ion +Ġch ill +Ġform ally +ANG ED +Ġ' :' +ĠпÑĢ и +a q +ĠFab ric +(l at +ĠPr incipal +Ġer ro +oc ale +N om +Ġf ost +_C USTOM +.int ellij +ert ools +Ġcl asse +adi ents +Ġfundra ising +EN E +_OPTION S +_ ob +// }Ċ +Ġprote ctions +.se ed +N V +term inal +;; ; +P redicate +Ġì ¶ +Ġbomb ing +G F +Ġch ew +)) ). +qual ified +] ={ +list en +C ENT +d igest +E ast +Ġd iver +Ġend points +Ġe e +Ġcolle ague +Ġdissert ation +_com mit +_D AT +. rc +Ġbre asts +ĠR ug +ĠP il +Contract s +ĠBry an +Web View +Ġconcent rate +ĠIn ner +Ġ' | +std out +_S ub +> -->Ċ +V ol +ĠS SD +)) ), +. Optional +Ġnurs es +Ġor b +_ pe +);čĊ čĊčĊ +pl aced +ess er +Ġther apeutic +Ġwhites pace +Ġa ston +Success ful +Ġpr aised +ĠW es +Ġe ighth +ir al +Ġvrou w +Ġf action +_b ias +Ġw itch +Ġnp c +(s b +ĠRod rig +_b ig +Dep endency +ĠAb raham +ard i +C AR +n os +Ġabund ance +Ġnut rients +in stein +.V ert +ĠI SS +< U +Ġsum s +_h ist +Ġfar mer +ĠA br +Sh ot +ĠBad Request +Ġh ass +ĠR ails +Ġaffili ated +æĿ ¥ +Ġer f +IN F +ĠView Holder +min i +ĠR oth +Ġfaith ful +ĠPhill ips +AND OM +]. [ +_P AY +ĠAr ctic +f aker +D igit +M ale +std err +se ys +Ġ Å¡ +_rem ote +li que +Ġin def +ĠIndust ries +it ra +_p airs +< iostream +Ġsal aries +ik en +.F rame +PL IC +_S PEC +ĠMed iterr +Ġsystem atic +Ġinter rog +Icon Button +se a +int ro +ĠIss ues +enc rypted +Ġintern ationally +Ġsn printf +Ġpast a +ĠBrad ley +_ Status +AL K +_P AD +.l aunch +< select +Ġhar dest +Ġph y +Ġ(( * +-s lide +ĠNob ody +S u +Ġas ÃŃ +close st +_initial izer +Ġsupport er +-g en +Ġt ales +Ġcor p +_f u +s at +ne ighbor +.M igrations +Ġal gun +Ġsin on +.S pec +? ,Ċ +.G L +m ale +Ġmon itors +yl an +-L icense +.m atches +ĠA BS +ĠM ast +ĠW allet +($ ("# +Dir ty +Ġco pe +Ġinterpol ation +ous ed +ĠJ ets +.F LAG +.C ancel +.Event s +ne ver +ĠM Hz +> D +Ġs ervlet +bast ian +Ġ> & +S ID +_cl k +Ġdiv isions +} ',Ċ +Ġd ildo +Ġpar ade +m ajor +Ġab oard +; ++ +Ġf usion +"}, {" +ĠDialog Result +ĉ arr +- em +_n r +(h andler +.N ET +.Xtra Reports +ĠSh ah +ĠB rief +- , +Ġprec io +ĉĉĉ ĠĠĠĠĠĠ +Ġt ant +ĠGrand e +/ xml +_IC ON +ĠR etro +un que +Ġn ag +to Fixed +X L +Ġdecl aring +ĠCon crete +ĠAm azing +ĉprint k +Ġdeb ates +D ATED +Ġaest hetic +emet ery +Routing Module +ĠNash ville +W AYS +Ġw olf +Ġobserv ers +OT A +ans on +Ġe a +Ġgreen house +ĵį ä½ľ +Ġst air +Ġimmigr ant +_app ly +pe are +ĠBloom berg +_PL AYER +Res p +æŃ £ +Cho oser +ĠI Collection +P eter +Er ro +.detect Changes +Map s +Ġs queeze +ĠHom es +weg ian +Ġformat ting +Ġnegot iate +ul d +ĠN ep +ĠQ B +Ġeconom ies +Ġ*/ , +Ġredu nd +ĠA ber +.IsNullOr WhiteSpace +yc led +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĊ +_S h +Ġske pt +Ġre created +Ġget Type +Ġmarg ins +Ġcolon ial +ch arts +// @ +Ġprocess ors +è¯ ´ +b atis +æĦ ı +ator io +mention ed +P atient +Ġpre y +Check box +_x path +.s kip +ĠMorm on +ĠMemory Stream +CRE MENT +Ġk u +m eld +\ Data +ĠK ernel +il tr +éĢ ģ +( profile +Car bon +RO LE +( pl +] *( +.m emory +Ġmed al +Ġadvis or +it ät +Ġh dr +ier ung +ĠProvid es +( alpha +Ġteen agers +- parser +.L atLng +] ()Ċ +Ġfel ony +ĉĉĉĊ ĉĉĉĊ +BO OK +Ġsl ash +Ġclear fix +ĠPro phet +å® ¹ +right ness +-f i +.k ind +ert on +J im +Ġmanip ulate +Ġworks heet +ol in +st ars +Ġart ifact +_EM PTY +ĉm ain +------------- ' ; +Ġexpress ing +ĠI Q +ĠF act +/************************************************************************ *******Ċ +_m ass +)) : +Ġcon dom +Ġcreate State +omet own +Ġir r +Ġ> ( +> B +iter ation +ãĥ ª +Ġshirt s +ount y +-> $ +_S IGN +ĠD ale +Ġj j +E asy +F re +ĠN y +Ġch lor +match ed +ĠG erm +- UA +ĠN athan +educ ation +-y ard +- che +h ouses +r itional +Ġprox imity +Ġdies em +áºŃ p +Ġd rought +.a udio +ĠLe o +Ġfavor able +in ch +ĠD aw +rib ly +_st udent +id able +O VE +Ġlack s +ounc ing +.b usiness +Ġre open +may be +_G LOBAL +Ġdress es +ĠEd wards +ens ible +ĠHard ware +ĠEx cellent +ĠTime Unit +CTION S +Ġsched ules +Ġseg ue +Op ens +am men +- Identifier +Ġst aring +Ġhapp ily +ĠH ob +' _ +Ġ" ); +ament os +et ched +Ġ/> }Ċ +. Users +Ġinterrupt ed +Contact s +Ġreg istro +in burgh +CH A +_ imp +ph is +s ay +Ġretail er +.N ODE +/ maps +_L AST +ĠCh arge +_g uard +Coll ider +ĠStateless Widget +": [" +(" ../../ +iox ide +ĠS und +Ġ'' ; +un set +add Widget +л Ñİ +el les +alk er +A rc +Ġded uct +G UILayout +ĠV illa +Ġfor bidden +_ where +Ġ\ / +ĠT ib +_A X +] čĊčĊ +ĠB ir +Ġb end +ĠMA KE +ĠM ET +Ġfut ures +Ġweight ed +"" "čĊ +Ġauthor ize +(pro gram +}, {" +Ġcoeff icients +ê s +Per Page +ĠBath room +ĠPublish ing +G PL +Ġsub missions +ĠNUM BER +j Äħ +Ġaddition ally +em pre +ĠSh el +ot yp +S olution +Ġth under +_ ec +ĠĊ ĠĠĠĠĊ +ĠF ellow +Ġk ay +Ġnew State +ONT AL +Im plementation +.L ook +Ġ ents +Ġl ors +ĠB IG +f ab +Ġaver aged +ĠFe edback +ĠW ells +Ġm artial +Ġind ul +ĠComm unist +ĠFore x +ĠAgricult ure +" [ +Ġqu ar +ĠK ont +ĉ view +. Bytes +des ktop +ĠM akes +akes peare +.Null able +Ġspot light +V B +ow y +(t orch +tr idge +_b ounds +Ġapolog ize +.add Item +ant d +* );Ċ +, u +(g en +ç» ĵ +re ator +ĠC ord +ou pper +.m etro +Ġ ew +ĠW ORD +.A fter +Ġdet ained +ĠHam mer +ex isting +Ġo st +Ġmon ument +-c ustom +User ID +ĠN om +Ġre jection +(d im +Ġsingle ton +ĉd ie +ari ance +re ports +] != +eld a +Ġpreval ence +_reg s +." . +Ġfemin ist +Code c +Ġ **Ċ +(label s +_M ARK +FA ILED +Ġadminister ed +W N +ĠĠĠĠĠĠĠĠ ĉĉ +Ġn oun +w ig +Ġg otta +Ġr if +- im +ĠPaul o +ĠCommand Type +] ))ĊĊ +-z ero +Tr aining +Ġl ord +_ art +re ddit +C ert +Ġpes o +R ot +Ġend anger +.d r +user Info +un ts +n v +ĠTrail er +-f irst +(m ake +Ġbenef ici +-bl ack +i ÃŁ +Ġund oubtedly +Ġm ex +ĠAnc ient +( as +Ġdes cent +P ick +Ġrep lica +$ obj +ä hr +Ġar rows +ft y +ĠLib ya +ug a +charg ed +T ur +Ġh omic +iss en +ĠF ake +Ġbe ers +Ġsc attered +( Time +UT IL +Ġbureauc r +/pl ain +Ġstick ing +FA IL +ĠC ovid +Th ird +_p resent +ĠPier re +Ġë ª +Ġ[... ]ĊĊ +Pro b +ĠTra ffic +ica o +do ctor +Ġ), ĊĊ +T abs +al u +ï¼ļ âĢľ +Ġinher ent +_N o +rit is +ĠPro of +.b asename +ä¼ ļ +Ġch im +ĠProt ected +c rit +Ġpr one +Ġк он +ĠHero es +Ġan xious +Ġan os +Ġweek ends +Ġs ext +Ġredu cer += UTF +h alf +ĠS aw +.m m +Ġnue va +.current Target +.l ua +_EXT ENSION +ĉ reg +ĠC trl +_ align +accept able +Ġrush ing +fr ac +Ġbo asts +F ive + ± +ĠTem perature +> ): +Ġchar ter +RE ATED +Ġsubject ed +Ġop c +health y +使 çĶ¨ +ĠScient ific +Ġfra u +ri ages +ภĶ +.in ventory +ation ale +M ad +min utes +>> ();Ċ +ĠEn v +Ġrecord ings +Ġsusp icion +sql ite +ĉ read +ãģ ¦ +Ġwor ries +.put String +ĠSh anghai +( uid +r er +ĠvÃŃ de +") : +Ġmethod ology +Ġк оÑĤоÑĢ +cc c +av ad +Ġindu ction +ĉ Thread +, string +ạ i +neh men +u ition +Ġ* __ +.em f +Ġì ľ +/th emes +ĠN ine +. One +ĠEm bed +Ġf az +u ations +Ġpriv ately +Ġl ing +[ F +ush i +Ġlaunch es +( KEY +G MT +Ġaim ing +pat ible +ĠB iden +i w +ĠD egree +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġ$ ('< +á rios +to UpperCase +ìł ľ +ĠE UR +Ġovers ight +Ġtable sp +Up dates +.m akedirs +Ġhum idity +/ template +Al ways +( IS +_c ert +D ig +Ġunder way +ort on +ĠHur ricane +Ġsp ends +ĠSeg ment +Ġfl ies +ĠT oggle +ĠLyn ch +Ġs enses +ĠK os +set Enabled +ist ically +Ġtest er +Ġadministr ators +Ġtag ged +Ð ĵ +Ġshort cut +ĠRes olution +Ġsuperv ision +ĠAsh ley +Tr acking +ul atory +and el +ist en +Ġun re +(d iff +ANT S +Ġr ider +Ġs Äħ +.S eries +_ orders +ORIZ ONTAL +Ġret ention +ãĢĤ čĊčĊ +Ġdi agonal +ĠC ancellationToken +_ Internal +Ġru in +.Q t +ocr atic +T el +ĠAn swers +m atic +Ġx p +at em +_j obs +_ any +Ġsen iors +Ġland mark +ĠQ List +Ġman eu +ot ify +/ ";Ċ +/ server +ĠPhil osoph +uten ant +( io +h z +Ġauthentic ated +d v +- Compatible +Origin ally +, function +ãĢĤ čĊ +ĠRepresent ative +as ily +irc uit +.d t +(m ath +.M arshal +[ , +ĠC ities +_ turn +| )Ċ +Ġcant idad +al ter +ĉ ui +ĠNe braska +Ġsk irt +.b g +Shared Preferences +( style +Ġg rief +g ew +Ġsaf eg +ol ang +_l ists +ì Ľ +Ġgran ite +Ġhott est +.j dbc +.C ustomer +Ġâī ¤ +Ġwa ar +_sc ene ++' / +ĠJ TextField +Ġse ating +Ġwe ars +Ġ` / +C ases +ĠY outube +ı m +Ġbal con +, G +Meta Data +- price +SC R +Un ity +Ġtr unk +={` ${ +Ġearthqu ake +Part ial +Ġsub st +Ġelim in +=" '. +//* [@ +Ġsuperv isor +vro let +_ article +Ġp ane +b io +Ġmot ors +N M +F rank +Ġon ion +- word +Item ClickListener +Ġb rit +end encies +Com puter +_r unning +( day +- he +(n amed +ĠS ach +о Ñĩ +c ampaign +.Ab stract +(w rapper +.p ay +Ġu w +Ge o +r ails +/ select +icht e +son s +E VENT +Ġal iment +Pro viders +A wait +_INTER VAL +. off +Ġgl uten +_cl oud +Ġw en +.ex tract +ĉ button +/ MM +Part y +Ġdem ographic +_err no +Ġh iking +(' ')Ċ +", @" +Ġw it +r á +olog ie +ĠSt yles +ĠBrowser Module +.Request Mapping +ic ans +P AGE +cre ation +ĠF erguson +ud ed +num bers +ĠGT K +Ġpresent ations +ĠB obby +_s pan +est yle +Ġilleg ally +abel a +Ġbattle field +cap acity +ter ror +] ");Ċ +Ġwar rior +le ader +ĠDB G +ĠRe venue +Ġvig il +Ġcounter parts +( Error +ACT ER +Ġhe eft +Ġselection s +ze ug +t om +-t wo +. ;Ċ +_st atement +ĠA id +ĠV ul +_r gb +Ġpr izes +Ġedit able +ĉ form +ın ı +.de cor +D emo +lic es +Ġen ctype +rat ulations +ĠR OS +_ch ars +ĠJ ahr +part ial +Ñĥ ÑĤ +ĠRe ceive +ĠL ands +AP TER +Ġch opped +.. " +ĠAn aly +ĠU ID +ĠR adeon +ĠB ee +Ġun m +> M +.find all +Token izer +ĠWH AT +Ġs j +D rawing +E ss +ON D +Ĭ ¶ +(p acket +âĢĶ but +Inv ocation +ĠN uclear +? ;Ċ +Ġgrand es +ĠC rypt +rem ark +Ġ'../../ ../../ +Ġin ability +m agic +c ats +Ġsim ulate +: ${ +in flate +Ġen er +: NO +ip les +Ġmer it +ĠR ated +Ġgl ue +/b log +Ġg ren +Ġthr illed +.C H +unc an +ĠPR IMARY +Ġper sec +Ġfe ared +.M IN +ĠThe ater +é Ĵ +ategor ie +æ® µ +Ġappet ite +s quare +ĠAlex and +.User Id +_g t +_ enter +Ġgradu ates +Fragment Manager +Author ize +-N LS +(M y +Ġtri umph +ust ing +_PARAM S +Char acters +(: ,:, +_B UILD +M Hz +Ġwash ed +Ġun cle +Ste ve +ard own + ${ +_confirm ation +Ġtro phy +Work s +ĠElect ronics +ĠMediterr anean +_m etrics +Ġannounc ing +ĠD AY +_pro to +Ġp ear +base Url +ĉĉĉĉĉĉĉĉ Ċ +Ġcoord ination +: N +.an imate +ĠC otton +_h it +â ľ +Ġjet zt +if ter +(f ields +own load +ific acion +.c uda +ĠLi u +> equals +ĠA ce +ÑĢаР¼ +ĠSuper man +ĠGarc ia +Ġarrest s +ag ar +Ġ{} ) +Ġmac ros +rou pe +ê tre +Ġtw isted +str uments +_ (" +_ vertices +ĠTrans ition +и к +[ max +m ind +Ġaccess Token +Ġun le +m us +c op +ĠF actor +Ġcon ced +Ġre tr +.l inalg +-s lider +ob l +_Static Fields +Ġz ombie +s elling +Ġch ap +Ġsh aking +ĠTrans late +ĠAm sterdam +ĠE TH +_EX TERN +k d +_d isc +Ġpreced ing +Ġpri x +Object Name +_mod ified +ard ware +Ġ?> "> +ĠD W +` ${ +Ġ?> ">ĊĊ +Ġspin ning +_p ending +Match ers +. Keys +ĠP V +en us +ant is +Ġdisc ard +Ġh aul +Ġem pir +Ġpath way +Ġo ak +м ен +-ind uced +Ġimp air +ĠCal gary +.is Hidden +d z +_ include +Ġg m +Ġ' (' +P Y +uggest ions +Ġcommod ity +c ro +/ sub +Ġget Instance +ĠLeg acy +ĠK il +B al +( short +In form ++ x +* r +ĠHope fully +or ate +Ġmach en +Ġtreat y +ĠO ri +.p ublic +-h orizontal +Ġtact ic +Ġb ord +w ares +Ġam mo +ĠL ists +Ġequ ations +/ her +ĠNS W +B ounding +_C ollections +Ġav ail +.Drop Down +è ° +Ġh h +Ġl Ãł +.p b +Ġmemor ial +ĠAT TR +Ġexhaust ed +Ġt sp +ĉ redirect +Ġlik ewise +ST ER +L java +Ġcondem ned +oca ust +(str ict +Ġexem pt +Ġs ms +Ġex agger +S YS +Ġl ounge +: ^ +Ġto dd +de b +ator ial +ĠPort er +Ġtu ition +Ġexem pl +Ġp aren +.line To +Ġkid ney +Ġç a +Ġc ui +ï¼Į 请 +X C +Ġmo ż +Ġnomin ated +l ung +Im Gui +ĠB uzz +Ġstere o +port al +res as +Ġk lass +Ġdraft ed +Ġproject ile +/g pl +(param eters +* )Ċ +Ġassist ed +ĠNS Integer +s itemap +:n th +.View s +.Argument Parser +Ġme er +z ier +ĠD ig +Ċ +Ġpl ag +p ine +Ġblank et +Ġ: - +Ġl cd +------------ --- +(" " +Ġtact ical +ĠRon ald +ex tr +ĠF est +Ġf uer +-n avigation +Ġk b +gh ost +Ġhandle Change +_cl s +() != +Com parator +.v m +ĠCo x +_re view +/ @ +_c ookie +Ġrecogn ised +ld ap +Thread s +ĠSex ual +ĠB earing +(S QL +Ġx r +Ġth igh +URL Connection +ĠSU V +Ġm Context +Ġinc idence +ĠE ste +.s up +_t e +(EX IT +C MD +/ "> +Al most +ĠU ne +Ġand eren +ĠSingle ton +Ġb ore +Th ink +Ġn arc +] initWith +_sh op +(str ategy +! ', +her its +ĠDes k +_m achine +.net ty +ı nda += < +ĠQ R +ĠS idebar +.split Container +Ġon Success +Ġmon key +En joy +(n odes +pect rum +Ġ(* ( +ĉU INT +, height +ĠNetwork s +.t ail +.l inspace +Ġ" ... +List en +Æ ¡ +.Ch annel +- defined +Re peat +ad just +ER M +_ application +.assert NotNull +- stream +Ġr abbit +Ġposition ing +Ġw oke +Ġf ing +Ġmulti player +Ġregister ing +un til +Ã¥ n +( :: +uss ions +Ġpot ato +ĠE quals +.S up +/ap ache +Ġ( = +. ") +.p tr +ĠSpe ech +.cl ip +ĠGab riel +Ġmusic ian +/ issues +.sh op +ĠH ier +_RE T +_b ucket +ãĥ ¡ +av s +Ġro z +fl ower +Write Barrier +ĠMil an +Ġlegisl ature +ĠD oll +Ġprov ing +.concat enate +âķ IJ +Ġg char +cdn js +b les +ĠList ing +л о +.xr Label +ĠS ak +just ice +ĠVal entine +un less +Ġp iger +(r un +Ġtest ified +AN A +ĠRem oves +)) ));Ċ +rec ated +ĠRuntime Method +Ġcon qu +ãĤ ¢ +Ġt issues +ail er +ét é +- Star +Ġfl ames +.set Icon +Ġsup ern +Ġvag ina +- variable +Ġwell ness +C UR +Ġbel le +.get Request +Ġp oco +ben h +ag ens +Ġsp ill +ĠJ ur +Ġdispatch er +н ого +emon ic +(dir name +ĠÐ Ķ +Ġpas se +Ġg anz +ric ing +E U +Ġmuj eres +ess en +.at tribute +j j +ĉĉ ĠĊ +[ ^ +Ġstrtol ower +lex er +ect ar +hot el +.s quare +Ġr all +Ġlower ed +hand led +Mark et +ĠUs es +iv as +.B usiness +ãģĹãģ ¦ +D IV +Ġw asted +Ġav oir +ê m +_ACC OUNT +. et +ĉ SDL +k ap +Ġf ox +up pet +{ },Ċ +", ' +F avorite +P END +ĠA ES +} ), +Ġded uction +Ġpol ÃŃt +Ġcomponent Will +ĠT elerik +_SE LF +Ġm use +C raft +Ġd ens +ठ¿ +( tp +Ġt asty +Ġbal ances +Ġded ication +ĠWall ace +Ġun law +\"> \ +Ġm um +- update +ement e +Ġs oda +Re public +as mine +é ric +( Status +ĠJson Convert +ĠD isk +.Red irect +Ġfilm ing +/m ol +R o +Ġv ille +Ġtrab aj +Ġsyn thesis +reg a +Ġr l +S cheduler +ISH ED +current User +(error s +' h +_b ot +x imo +ĠUS ART +_s uper +_DEC REF +н ой +_RO W +Ġprom otes +ĠT A +Ġhor as +ĠRep resents +Ġname of +ĠEx c +ĠGar age +Ġse ine +, # +Ġher b +/ resources +Ġple aded +.r adioButton +Ġæ ĺ +O ps +ĠN est +c string +ĠDef ence +Ġref ere +_le af +Ġrevel ation +ë § +.execute Update +_W ORLD +Ġexp ans +(" \" +j ab +Ġdoub ts +ĠGe ometry +Ġintrodu ces +Ġsen ators +Ġcan al +.h elper +ĠBi ology +_SE NS +.pre vious +-t ouch +ab it +Ġimpact ed +Ġbr ackets +.d irect +acc um +Ġtest osterone +ĉ action +ĠCh ance +Ġpe aks +CppCodeGen WriteBarrier +Ġun belie +_p ress +.R el +ang led +/ templates +-- >čĊ +l ime +Ġsufficient ly +_ nt +Exp and +.is file +Ġis Empty +Ġq t +Ġmul her +ac ob +Ge orge +å¸ ¸ +Ġass im +as o +Ġcompr ised +O V +(CON FIG +ĉw riter +Ġdes p +Ġten ure +(c r +.p ool +ĠB rend +Ġc ensor +(time out +Ġple a +.W rap +Ġtight ly +ĠW ere +ĠI gnore +abe i +Ġbr idges +Ġcondem n +Ġsimp licity +Ġrout inely +Ġblack s +j b +ĠP it +U tf +Ġ/ Ċ +re load +Ġset Object +/g lobal +Ġf atty +Ġsock s +Could n +Ġerot isk +æĿ ¡ +ĠPress ure +ĠM az +n pos +tol ower +ĠE Q +ute ur +ĠM oment +Ġet a +{{ -- +Ġgraph s +ĠGu ar +r ine +( -- +ĠHttp Status +(st udent +* np +Ġrail way +Ġas ynchronous +_v m +'] ,' +, text +mer chant +(G uid +ĠG ra +ix er +fetch All +.add Listener +fl ip +* $ +> (), +Ġsun light +ass igned +Ġab c +ĠC OLUMN +ĠðŁĻĤ ĊĊ +) ... +Ġen semble +Ġnew line +_S INGLE +ied ad +Ġdark er +orm ap +Ġl ion +pl its +Ġillustr ation +ĠI EEE +Ġv ista +ous ands +****** * +ĠTom my +Ġh ue +S el +Ġa ura +ĠTher apy +Ġanim ator +.con straints +Ġv ague +(" ") +Ġvill ain +Ġbless ing +Ġstring Builder +ĠM isc +ĠD IR +f ax +- node +ĠWalk ing +ĠA U +s ess +Ġgr ill +VERT ISE +ĠF oods +Ġt ournaments +à ĵ +ĠMar sh +Ġw onders +Long itude +.Command Text += input +_enc oder +page Size +Ġget State +> >Ċ +.g rey +p od +Ġread ings +Ġre consider +Start up +Ġexc er +.b alance +_c ycle +_T ime +LOC AL +ĠE FI +ĠRe yn +.set Foreground +by n +Ġdis connected +ACT IVE +Ġembed ding +ick ers +Ġsurround ings +* c +Ġgar ant +Ġb f +Ġw ipe +Ġ ä¸ĭ +_T RA +ado x +ç ķ +Ġsu cks +ĠS ongs +ĠAssoci ates +ĠB ald +ĠB rett +ven ile +Ġv t +Ġin ade +Ġres igned +ĠGl enn +.p attern +.Data Bind +Ñĥ м +Layout Inflater +ch et +ĠTest ament +.m s +Ġp av +ĠReact DOM +ur dy +AD ATA +M u +/ actions +ĠJ s +_ex tract +ĠBr ing +: id +str t +iv ation +Ġoutr ight +az u +loy ment +и Ñı +al do +ĠP ublisher +E ducation +Pa lette +_d rv +Ġ($ ( +ĠAnd a +Ġrem edy +Ġincons istent +te ction +Ġregul ators +Ġshort est +(p air +ĠInstall ation +Ġdefend ants +Ġ( ); +-l arge +M el +Ġthreat en +н Ñı +Ġfet ish +ot ine +_d ic +Ġ< $ +Ġst agger +sp i +$ response +S erv +-b orn +j os +ĉ img +ĉW HERE +_l t +å½ ĵ +.c ost +ĠT ue +.label s +ĠL V +wcs store +ĠJes se +ภ« +Tr ade +Ġpredecess or +ë Ĥ +fin ally +_g eneral +ogg ler +_REG ION +n ement +Ġblog ger +ĠHar bor +ĠD ataset +[ w +Ġattend ees +. ico +max imum +.Un lock +_SY NC +ág ina +Ġdown s +ĠW ii +]) / +Ġkick ing +unic ation +ĠD AC +ĠID S +ĠR ental +Ġcurrent Time +Ġvacc ines +ĠDev il +Ġn ors +_m ouse +urre ction +(n o +Ġ> čĊ +Ġaggress ion +Ġbre eding +.s ymbol +im an +Absolute Path +ĠWH O +_fl ush +- root +arn a +& M +Ġf athers +ĠR ocket +ive au +Ġw ander +Ġcom pos +ĠWar rior +ĠSe at +ĠClin ic +_in voice +(dis patch +Product o +at uring +oss ier +ĠM AY +Ġd agger +Ġsanit ized +ĠR FC +Ġpro ph +Ġur ine +Ġgr ind +ĠExp anded +des cripcion +-f w +ĠK erry += name +Ġch k +Ġnation ally +Ġthe e +In c +Ġ? >> +.R adioButton +.Http ServletResponse +/ Y +ĉf ield +Ġhom me +y per +Ph ysical += v +Ġdr iv +ĠErr ors +Ġc Äĥ +De ath +ĠW INDOW +Ġpo et +ĠSh arp +ĠImm utable +ĉ create +Ġge ht +ĠRe form +ais er +ĠInitial ization +Ġimm unity +.com pose +Ġlat ency +ĠLeban on +ĠPar ad +Ġfu els +ĠEx hib +co h +% ">Ċ +ĠCL I +) initWith +-Z a +_C LEAR +reg n +Ġfin ances +.st andard +_C ATEGORY +.lib rary +Ġtravel ers +_w p +ĠE valuation +start ing +Ġ )),Ċ +ep isode +ĠV ariant +Ġda emon +ĠJul ia +ĠN R +Ġdoub les +< v +/r untime +Ġinterpre ter +ĠIN DEX +ĠHol mes +_D IM +Ġp addle +_ex ample +Ġfore ground +.r outes +Ġs owie +S UCCESS +ĠC DC +ĠB D +_ - +as ured +W riting +Ġcurrent Page +( answer +ĠASC II +à ¨ +Ġsocial ly +yy y +ĠSpecial ist +(c ustomer +ist ani +ke st +ĠM ak +Ġth o +. pt +( comment +ĠCon verter +g am +b ins +. tele +ĠVeter ans +_AL LOC +олÑĮзов аÑĤ +inn amon +; width +oh l +Ġfant as +Ġs ung +ĉ K +( Json +Ġneighbour hood +Ġv ow +Ġs ins +on acci +Ġepoch s +im agen +.Ch ange +.my batis +Se ek +W ER +管 çIJĨ +Ġinter ess +_ Event +eder land +Ġterr itor +Ġci udad +uck ed +Ġsn ack +Ġtransport ed +ĠMan ifest +ĠD AT +_th eta +Ġw ont +.ĊĊ ĊĊĊĊĊĊĊĊ +Ĭ¶ æĢģ +ĠEp ic +De ck +l tra +_Z ERO +Ġ[] ; +/ scripts +Ġ---------------------------------------------------------------- ---------------- +æĥ ħ +Ġwe ed +N BC +Ġrap ed +ĠG ateway +[ M +ĠTime out +ench mark +.View Model +Ġporn os +ĠY a +th ritis +ĠFly nn +Ġme ga +ac in +Ġtrib al +.app le +ĠB lo +â n +ib i +ro v +ĠL ives +^ . +get Request +ĠEst ablish +cont ainers +Ġst arring +Ġcele brities +ĠRel ative +ĠHe ights +Ġtq dm +ĠNorth west +iv ic +ĉ cl +Ġautom otive +ent ric +Ġfort unate +Ġfire place +se ud +nick name +; s +_C AL +h alt +(n s +_de leted +Develop ment +m ovies +Ġident ities +Ġprompt ly +ا ÙĨ +Ġant e +Ġ" ',' +åı £ +imp se +Ġy ap +Type Name +Ġb itch +Ġassoci ates +HE ME +- empty +ĠØ ª +ol vers +Ġpist ol +Sc oped +ag ner +'] ==' +ĠI MP +ex c +Ġo mitted +Ġmind set +Ġ[] ( +Ġor n +_C AM +A vg +Localized String +ĠN atur +Ġcom poser +ĠPlay ing +Ġover d +_ utf +.s k +ĠF ol +$ page +, Object +Ġbe es +al ary +bul let +_lib rary +O ffer +loc ated +Ġ(_ , +âĢľ He +ĠOwn ers +) ).Ċ +Ġb ri +.Ad min +kt ion +лÑİ Ñĩ +Ġerot ici +Cancel led +Ġa gr +re views +_d ma +RI CT +Ġg fx +mp i +pp o +Ġ// @ +Ġupper case +Ġcommit ting +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +User Data +Ġv ai +ĉs ort +Ġcongr at +Ġd ioxide +д а +. area +ĠJosh ua +ĠK och +_b reak +az ure +ist ical +_AL PHA +_ views +Ġelim inating +OM B +en umer +ĠHy dro +(* ( +ERT ICAL +Ġinev itably +Ġst ole +-e ast +ier on +Ġl inger +/d oc +Å º +ĠAl ready +as io +Ġ-- Ċ +Ġabb rev +ĠAt om +h im +ĠINS ERT +s un +âĻ ª +CON NECT +er ator +ĠM anning +Ġ: ( +g as +=> ' +Ġquery set +; }čĊ +ĠPop ulation +uted String +res ident +_F ONT +ĠRes pond +Ġobsc ure +Ġo bservable +ĠContrib utors +k on +ĠMus k +ex ao +ĠT ub +Boot Application +S OR +.H orizontal +.find By +.p ower +Ġposit ively +ven ience +ĠJ ong +Ġwh istle +Ġз наÑĩ +Ġl ending +Ġdestruct ive +Ġon Delete +author ization +(); ?> +_ original +sc ience +at ra +?, ?, +ĠAs c +Ġconvinc ing +$ a +org en +_D ate +ĠPro vide +Ġlon ely +) 'Ċ +ex change +; ?>Ċ +.f ast +S amples +L ondon +'] )čĊ +ĠI onic +Ġp esso +ĠKn ights +ĠR af +_attr s +Ġrepe al +> Main +ĠOrder ed +_N ew +=" "> ";Ċ +ĠS ERVER +ĠHE ADER +_ velocity +ĠIn voke +.timestamp s +Ġs ulf +I QUE +Ġinhabit ants +ph ins +azz o +Ġmon o +Leg end +Ġnon ce +IF E +; ";Ċ +- create +" ",Ċ +per mit +ĠImm igration +Ġpath name +ffect ive +âĻĢ âĻĢ +Ġex ams +- event +ĠT ill +[m id +F IX +; color +( Order +_tra its +Ġorder By +Ġs unt +ĠNich olas +Ø ² +Ġsun ny +in ers +Ġaccess ibility +ĠH B +.com p +ĉ op +Ġminor ities +ethe us +Ġcollabor ative +pr it +H IR +Ġwr aps +ĉd raw +g od +ĠI X +.app s +ĠN M +Ġirre levant +ĠT igers +Ġdi ag +G V +ĠAccess ories +k ont +Ġsimpl ify +ĠF avorite +_t ools +([] );Ċ +Ġtow ers +B es +Ġhun ter +Ġsal on +(b uff +ĉ debug +Ġmal ware +M oving +- options +) +' +ĠLO VE +_S OCKET +_f in +ĠDel aware +Ġsher iff +-in valid +ĠF ULL +Ġп од +el as +" strings +ĠRepresent atives +s urface +res olved +ht docs +)) :čĊ +Ġpress ures +Ġnorm s +Ġpl a +Ġs urname +Ġpost al +ĠDep art +Ġsla ughter +or ida +Ġhe bben +Ġdes ar +comp act +_L ANG +åIJ Ī +op oly +_r ad +ĠST DMETHOD +L azy +ĠĠĠ ĉ +... , +( web +ĠP ont +Ġet was +Ġup ward +_h at +Ġ], ĊĊ +Ġbase Url +Ġworry ing +-add on +(get Class +S PI +Ġcapt uring +) },Ċ +Effect s +Ġcompet ent +Ġf oul +Ġsubscri bing +ĠO BJECT +IX EL +b ucks +( edge +(p ass +ĠPet erson +Ġbo obs +ĠD elay +_s quare +el im +ot ers +_P C +% E +on click +ĠSV G +Ġto pped +Ġf ist +sm art +ĠR alph +( owner +j ours +Ġbron ze +ĠArgument Exception +( original +_S CALE +_c p +Ġrecomm ends +.set Style +S ure +L AND +Ġrepe ating +M att +. Visibility +Ġenter prises +.Set up +(sc ene +ĠRe active +ur ge +b w +.P ut +p ersist +.c ookie +ĠAud i +` s +sup plier +( Form + ¡ +_s o +Į Ģ +ĠLeg ion +t te +N d +L oss +( attrs +.sc atter +Ġg room +Ġgl impse +Ġn ails +Ġcum ulative +Ġf azer +_s ervices +.N um +ib ilit +_res olution +ĠT x +umin ium +op a +.s chedule +sm tp +ภķ +ur ry +ü k +go og +_sign ature +.int o +ĠSte ps +Ġhome owners +ĠNS URL +ĠP AC +ĠĠĠĠĠĠĠĠĠĠĠĠ ĊĊ +> ')Ċ +en h +Ġinc ap +$ MESS +Ġmo ins +ĠF i +Ġoff season +press ions +> .Ċ +ĠGr ass +ĠGo al +_p df +Hand lers +Ġstack s +.get FullYear +=[ ];Ċ +è½ ¦ +, V +(s plit +Ñĥн к +Ġbake ca +Ġ~ /. +pe z +t ails +ĠG len +Ġset Image +ĠCom ic +B LOCK +ĉ This +o ader +Ġcapital ist +_ST EP +( Boolean +ĠCor rect +r ina +Ġconc aten +å® ŀ +() :ĊĊ +Ġun anim +ll i +al ars +- ne +Ġdiv or +ĠKick starter +]. _ +< number +/m enu +GR APH +vis itor +Ġimpro per +_N EXT +Ġb isa +background Color +/ input +Ġmo i +Go al +li qu +Ġmiscon duct +Ġcompr ises +aw ns +ĠP ie +ra is +role um +Ġcur se +y u +_p oll +.current User +ES H +]) [ +Ġstory t +)? ;Ċ +* = +ĠB urg +/ layout +_back end +; ?> * '+ +åĿ Ģ +ac ency +( URL +_h alf += l +Ġlist View +( section +.to Array ++ / +ĠRodrig uez +ist ream +Ġelig ibility +:: - +.new Instance +P B +ĠAs sets +ĠCom posite +ĠL abs +ĠHam as +++ );Ċ +Ġbl k +ĠNe o +L uc +@ login +Ġun aware +.m et +_RE LEASE +( ST +AM IL +ri ke +Ġ( ){Ċ +(s printf +ĠAccount s +ĠV IEW +ĠA j +ãĤ ° +Ġwh isk +Ġid i +Ġro de +Ġih n +ĠElement ary +Q ty +Ġintrig uing +Ġå ¤ +J obs +ĉ offset +ĠAh med +ĠTal iban +Ġè İ·åıĸ +Ġinject ed +.Auth entication +_line ar +.Dec imal +Ġapp les +Ġshare holders +Ġb aked +.d iff +ĠE ddie +ok ers +Ġconfront ed +vo ices +Ġt us +ĠSp in +N ODE +_ Un +CT X +/g oogle +Tem perature +Ġ' '). +Ġmagn ificent +Ġstart Index +semb les +Any one +z k +eh en +ĠD ame +. strict +Ġrepl aces +Ġline back +Ġpush es +Ġche ek +ĠSh i +_BY TES +RE A +ả n +_CON NECTION +G ateway +ĠTr avis +ĠA X +ĠBas ically +ĠUp grade +à ª +th emes +erm o +k or +F emale +_att ach +ĠìĤ¬ ìļ© +Ġpo z +============ ==Ċ +(s ymbol +ĠS ector +__ )ĊĊ +_p adding +ï¼ļ " +Ġf abs +Ġr anged +set Name +Ġp error +â Ĺ +ĠFile Reader +Ġful filled +_C urrent +Ġdom inate +Ġsm ugg +Post Mapping +_for ce +Ġb loc +ĠG iant +(v ideo +ĠC U +System Service +Ġ elf +Ġkont akt +ë ª +ke es +gt k +Ġparam Int +Ġmark up +u ales +Ġaccount ed +Ġgang bang +RY PT +ĠW rong +Ġcred ited +ĠM ESSAGE +Ġfl aws +Ġbb w +Ġmetab olic +ĠO EM +/ event +(C ollectors +mont on +ap pear +Ġopt ed +Ġche at +Ġd av +ĠPro ceed +Ġê ¸ +ank ed +и з +ans k +ĠH ang +ĠC ler +Ġdis gu +Ġc map +.cl js +Ġa ument +le z +ĠJo ined +_re ceived +Ġa erial +ot el +Ġgre et +" s +ĠGen esis +ĠCal if +pan ion +Ġtail ored +m apping +and Expect +.tr ack +at omy +ĠO w +ull ah +.Y es +ĠSimple Name +db h +' en +Ġnons ense +Ġphilosoph ical +(get Context +Ġis so +ĠA CE +start Date +Ġb ÄĻd +ĠAUTH OR +ĠGlo be +Ġinsect s +_A l +ush ing +è® ° +/ Home +ĠLocal Date +need ed +hes ive +Ġill usion +äº Į +Ġtr at +x o +/d etail +_M ATCH +Ġbroad band +Ġw al +ĠIllegal StateException +IRE CTION +Ġnor theast +es ium +ĠClient e +ul ance +nt y +Ġt ecn +Dev ices +Ġgr ains +ĠO g +ĠS EL +ud iant +Ġ++ ;Ċ +Ġexplan ations +oc co +Ġdi ets +Ġco hort +( controller +.Iter ator +-r ich +ro cess +G D +Ġcar bohydr +Ġfri ed +ĠEmploy ment +ìŀ ¥ +ĠLeon ard +_ ${ +qu ares +Ġcompan ions +Ġpar is +Ġstim ulation +ĠZ oo +Ġre levance +ĠCol our +Ġspe ar +ot ional +ĠL ite +ĠK osten +Ġà ³ +_att achment +orph ic +Ġdam it +Ġd lg +Ġthr ive +CH ANGE +ĠApp arently +Ġat ual +Ġroot ed +( images +aw i +ari at +Ġch erry +STAT IC +m nt +ĠUser Id +il let +ĠHis panic +Ġn ak +Ġcent ro +Ġdim s +_initial ize +ı k +ĠCent ers +RE N +Ġevolution ary +ĠTop ics +_d amage +em er +Ġr und +Ġpun ished +Ġcub ic +f air +[] ;ĊĊ +Ġinstant iate +Ġover see +- delete +unte er +start Time +ĠP ipeline +_G AME +ĠC ir +ĉ Null +.Format ting +uc umber +ĠR ide +Ġz oo +Ġcheck er +åIJ Į += C +Ġg rit +"); // +_x y +ĠDe claration +Ġcall able +F oo +ĠList Item +Ġin accur +ml in +ĉ Data +Ġev olving +aw an +Ġca fe +fol k +_ID X +ĠAny thing +ĠPalest ine +ĠGrid View +Ġcol ony +ĠGerm ans +( + +.p id +.js x +ĠSuper ior +Christ ian +ĠL ect +ĉ Game +Ġinstrument al +Anim ations +д ал +ĠMos es +ĉĉčĊ ĉĉčĊ +z s +k te +ä¸ ļ +_D IST +bit map +d B +Ġp ersistence +ÑĢ оÑģ +$ l +B ron +Ġ{ | +_ch art +ĠCon sum +Ġh emp +Ġ" ))Ċ +Ġattack ers +Ġknowledge able +Ġc et +Ġvir uses +' I +Ġpitch er +Ġsweep ing += list +apt ops +.de pth +Ġinstruct ed +ĠR us +benh avn +Ġи н +S ports +Ġon set +æĿ ĥ +. RED +_s i +ĠP ST +.on Change +> tag +ĠR oh +_char acter +ĠLaw s +ĠB achelor +_s wap +.re activex +Ġreward ing +Med ium +- [ +ĠRec ently +J oint +part ition +ĠMin utes +Ġind o +Ġabsor bed +ĠG N +_IN D +Ġsab er +Sp awn +output s +ĠJeff rey +Ġmed ieval +h ed +Gu ide +Ġpsy cho +Ġgl am +E lim +äd chen +_pl ain +ĠS au +-f our +Ġanaly zing +QU ERY +Ġtom ato +_button s +V EN +.set Status +. Url ++ ĊĊ +Ġcompl aining +deg ree +conf irmed +Ġsub t +p arsed +Ġtor que +Ġtroub led +ĠT ARGET +Ġtrad emarks +ĠCo ordinate +ĠV iv +Ġ// }ĊĊ +Ġapr ès +.get Position +(Key Code +ĠSil va +Ġmet eor +Ġendorse ment +Over view +ĠP oss +.In ject +Ġeven ly +Ġvisual ization +Ġw char +ĠH DMI +Ġfun ct +ick name +',' ',' +Ġfor wards +Managed Object +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠ +ĉ server +ĠOut look +ĠChron icle +Ġdub bed +Ġd ok +ĠW ear +.A L +pare n +. Interface +Inter faces +.c od +Ġd ib +.Global ization +ĠAcad emic +Ġass ms +Aut om +Ġl w +ĠN W +Ġ&& čĊ +Ġproble ma +ĠManufact uring +lim its +-m obile +Ġfil me +/ map +Ġdo it +ĠIn k +Ġsu ed +. arr +Ġunder min +ĠPro c +croll View +__ $ +Ġsidew alk +( that +ภ· +[ q +gram mar +Ġt ë +qu ito +Ġspir al +ext ended +Ġf ocal +Ġdig ging +p as +ĠT all +.pro xy +it ures +TR ACT +ĠRe alm +Ġf eder +Ġorient ed +ĠAltern ative +Ġo we +Ġsour ced +ink er +.d et +S ep +ĠQ ui +ĠPal mer +(_ , +s amples +oy er +ull an +que z +Ed ges +Ġsh out +ĠA chie +Ġha ar +_Con struct +Ġprem ature +Ġre vert +'). Ċ +Ġs chn +filter ed +null ptr +S aved +itect ure +CL A +Ġv l +st ell +ĉ Me +ĠL ip +n ational +Ġwh olly +Ġspr ings +.T imer +ĉs rc +els en +åħ ¶ +Ġcommunic ating +ĠQu iz +Ġt eng +Ġge z +ĠOut side +.S ign +(c s +Ġdisput es +ĠWe iss +ann es +> No +ĠB ach +.remove All +re fer +/d ashboard +ĠA jax +Index Changed +ĠWe ak +' "Ċ +Ġs ights +access Token +ĠJ oi +(d omain +ĉc v +Ġcontin uation +Ġpl um +ad ir +.set Message +Ġ ï¼Į +Ġsw allow +ĠL amp +Ġq w +Ġu u +C oin +ub ic +ĠDe als +r ace +Ġdict ator +Ġmem e +turn ed +ĠJul ie +.grid Column +Ġpup py +Ġp am +Ġ) {čĊ +Ġinv iting +Ġf rench +v im +Ġwr apping +Ġ#- }Ċ +([ - +Ear ly +Ġsh iny +.f aces +Ġreb ell +abc def +ä lt +Ġest imation +ph ys +los ures +_RE L +Ġex clusion +ĠSk ype +we ise +-st op +no thing +ĠE gg +is ors +Rich ard +Ġcounsel ing +Ġcomm em +ĠQ MessageBox +ĠSy nd +ĠFro st +ĠCompet ition +ĠAw ake +Ġt ed +ic iones +ĠDev Components +VERTISE MENT +ott i +.run ner +Ġuniqu ely +.fl ag +ĉ rs +_g eneric +Ġ`` `Ċ +ACH INE +Ġme in +( Application +( br +Ġrat ios +: , +ĠXCT est +ustain able +- www +it les +_T EMP +Ġs yst +umeric UpDown +ĉassert True +Ġw f +. peek +ĠBul g +Ġterr ifying +.M ODE +ĠG W +á r +Ġf ic +Ġcommit ments +- tech +ĠL iquid +ope z +z heimer +a ña +-m edia +( animated +_go al +Ġg um +yst one +.S ET +ĠW end +set CellValue +Ġmsg s +c ash +AL LOC +/ aws +Ġmic rowave +.Point er +ĉ Console +_s orted +ĠFil ip +Pro d +Ġ//! < +ing roup +Ġk s +_T RI +Ġteas poon +ĠAT T +Ġrecover ing +ĠG LOBAL +.P ar +Ġ/> ;Ċ +Ġmar ble +ul ators +ĠC ycle +Ġher bs +_m etric +) ! +_C LOCK +_ Button +H arry +è¿ Ľ +Ġstr ains +ĠApp Bar +ĠCh an +/v ideo +Ġb am +.Pro gress +$ f +lem en +Ġir regular +ĠD uncan +ĠM int +-v ideo +ঠ¾ +ó wn +ĠEM PTY +Ġstack ed +ĠH A +_c ut +Ġwhere in +ĠW ays +(count er +è¯ ķ +Form Group +Ġble w +c ourses +Ġproduct os +ry s +ĠRest r +Ġsty ling +> s +Ġp iv +Ġit ertools +get Repository +ĠI k +_dev ices +lay ui +Ġhalf way +Ġfran ç +Ġtun ing +O A +_N ode +ar de +Ġfier ce +lic ted +# čĊ +Ġbreak through +ĠE rik +Ġb ride +Ġ. " +cul us +ins ide +ĠIndian apolis +ĠE E +Ġy og +urre t +.f s +. grad +_c ards +_ac curacy +_ep i +qu eda +/ org +é ªĮ +Ġcom pte +)) [ +Out side +G reater +ĠRender er +. actor +Account s +Id le +_h ours +ern er +Jo ined +Ġmen j +requ ires +ĠO PER +.remove Child +ĉs p +Ġes se +r ift +xF E +ĠSh akespeare +________ ____ +Ġbudget s +Model State +fill able +- component +oc os +ĠBUT TON +/ io +, out +s ms +Th omas +ĠAr med +res ume +Ġrot ating +ĠV ault +Ġse us +. (* +Ġa mino +Ġ[] );ĊĊ +Ġprov oc +no x +.Get Enumerator +==== ===Ċ +æĸ Ļ +_sc roll +Ġfil med +ĠS oci +g ap +g ro +V ote +" But +_R C +An imal + Ģ +ib ile +Ġaw aken +ore st +in ja +ĠI van +( Command +Ġ ***** +Î · +Ġkv inder +/h elpers +_c ases +t g +ìĦ ¸ +Register ed +ĉp ass +_d igits +Ġcont our +Ġinf ants +Ġjust ification +ĠFort unately +Con tr +ĠonCreate View +_S AMPLE +Ġallow Null +Ġn ud +Ġfet ched +_e qu +ĠUn able +=\" " +> {Ċ +Ġcommit tees +ist ema ++ ". +ÃŃ an +m ant +Ġsou theast +ï¼Į Ċ +dialog s +PRO JECT +charg er +- port +(u uid +. export +S ix +ĠR P +P rem +Ġconsc ience +Ġmargin Right +_d istribution +y aml +res izing +D ock +ĠLoc ations +G Y +Se ed +B UFFER +oss ip +ull en +Th ings +- self +.p oll +PL AYER +Ġå ® +G ROUP +ĠA way +Ġg ospel +xf d +M ary +ĠPort able +T URE +Ġutil is +Ġse it +Ġstr and +Ġtrans c +Ġ( ^ +ĠAl fred +.m em +.c ircle +Ġ~ / +for cing +Ġr iot +pro x +TH ON +iz ación +ĠN I +ro st +Ġdis pro +_in stances +ï¼Į âĢľ +ograph er +end as +ĠIsa ac +ĠP ine +/d is +Ġcolor With +iter ate +_str ide +Ġpun to +.Event Args +( center +Ġneighb oring +ĠPr ison +ĠMess enger +Ġepid emic +da o +_com plex +Ġgr avel +_D IP +é ment +ĠA ri +_bit map +.qu it +( valid +Ġp end +Ġrespir atory +Ġre bound +Default Value +ãĥ Ń +Ġcomm its +.test s +_f r +it et +.s f +Ġspace craft +c ritical +Ġde pressed +ĠAny Object +Ġun b +Ġdisc ern +(m ysql +L atin +ĠB og +ĠWild life +To File +iox id +@ RestController +Ġ"$ ( +Ġ<< " +Ġdefect s +Ġdat um +h in +Ġreal izar +any ahu +ĠS ig +@ Data +ad aptive +ĠC atherine +.c r +ĠCO OKIE +Ġp ictured +ĠFight er +Query able +ĠAny way +ĠGL FW +_n amespace +_ ft +Ġ] ) +Organ ization +Ġconstit utes +Ġqu and +(ch unk +"/ >čĊ +ĠL akes +main window +Car thy +sp in +(c sv +: red +-com merce +ภ¹ +Ġdiscover ing +Ġe co +_f ac +inc eton +ĠGre ens +j wt +Ø µ +ĠBron cos +ĠGood s +(G TK +Ġreturn Value +Ġsi empre +Ġneut r +w ent +ĠN atal +Ġenthusi astic +á» į +F N +/d atabase +C atalog +Ġbr un +ĠK ash +_P l +isc rim +, width +Ġin mates +Ass ignment +ĠH aven +Ġplay ground +ex am +@ Controller +ul iar +.get Parent +Ġ" ;ĊĊ +: size +iss ors +Ġf is +Ġal c +ens ation +ĠN ixon +Ġmight y +- str +_s pecial +_A DC +ĠTw ig +um bling +- address +Ġher oin +Y TE +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĊ +F riend +Ġa ve +ĠP NG +ĠKurd ish +DataSet Changed +Ġbl ades +br al +St eam +Ġsig u +IRT UAL +ac os +UD P +(d atabase +he c +ĠString s +_scal ar +ĉd esc +ĠT LS +; "Ċ +ĠCor byn +Simple Name +u ell +ĠEnt re +ell ites +- place +Ġfrank ly +ĠE rf +CE L +Ġpa ÃŃs +Ġh edge +Ġlat ent +ĠIR Q +ĠH erald +ĠP rec +ë³ ´ +.T EXT +Sal ary +Ġaut umn +Ġtrav ail +.S um +Ġc ared +M or +Ġint uitive +Ġj ournals +_ IT +ĠT rou +ä¼ ł +Has ColumnName +Com posite +Ġsp ice +_d isk +_CODE S +ĠInt roduced +ion a +Ġnue stra +o ct +ĠĠĠĠĊĠĠĠĠĊ ĠĠĠĠĊ +(param eter +Ġstud ios +Ġproject Id +Ġbd sm +.Sql Client +im izer +ĠC ARD ++ t +a an +.s ol +_Ad just +Ġright eous +ĠLog ging +.f ilters +_T AB +ĉs ys +roph ic +other apy +ĠB rowse +key board +R ON ++ \ +ro pped +Ġext ensively +f k +Ġl ime +year s +Ex c +Ġs ph +Ġche ating +and ro +ÃŃ o +Ġpr ince +o ire +ĠD estination +ĠConvert s +Ġup stream +o led +Ġserv ants +Ġsem antic +Ġcr unch +Ġevent ual +run ner +/ error +Sp in +Ġsecret ly +Ġas semble +.P erson +end error +_ < +Ġp endant +S leep +ĠChem istry +Ġboss es +l k +)) ),Ċ +Block ly +DE VICE +Ġreflect ing +Ġam ple +Mill iseconds +ĠPresident ial +Ġus uarios +ĠN Z +ĠSal ary +ĠA manda +_n p +j ury +Ġkö n +Ġtherap ist +Ġhomosex ual +ĠDr ake +-w indow +ĠLoc ated +.D river +ĠV IDEO +Ġmerch ants +ĠC hest +- lock +/ php +Ġmil ano +_ST YLE +arg er +ide a +G UID +adv anced +me al +Options ItemSelected +=' % +ĠCh am +: data +(st at +Will Appear +Ġinform al +aj i +Ġre productive +ĠC AS +ãģ £ +F UNC +ĠR uth +)+ ( +CON ST +ĠF ans +Ġgroup Id +xffff ffff +Ġsam pler +Ġ}} "> +. the +Ġh ollow +W AY +ĠFac ulty +Attrib utedString +ĠLook s +ĠR ex +j k +ĠM IL +Ġb ard +.L ong +Ġliv est +Ġsk al +ic ism +MA IN +Ġmu cho +B ODY +Ġes e +ĉ use +F oot +.SQL Exception +Ġinherit ance +re ceived +Ġput as +ed is +als a +ĠError Message +Book ing +Ġtr act +ac z +ĠC ant +_reg ex +Ġide ological +Ġj ihad +h os +/s ys +col m +(p ool +Ġest án +ĠP ending +em ás +Ġktó ry +));ĊĊ Ċ +trans actions +Ġw ield +it ere +ert ure +_s s +Ġstretch ing +Ġprison er +.Read All +Ġbes ch +-- ;čĊ +Ġcr isp +_SC AN +Ġa e +Str ict +ĠMin neapolis +ĠBo eing +ar is +re k +_p ipe +Ġpri ests +(E IF +eh icles +ĠInter active +b etween +ĉNull Check +ĠBl air +ĠL t +_in line +eth yl + ¼ +_p ackages +Ġbarrel s +_ he +Ġreg exp +_ pts +_H andler +ing ular +ĠN issan +ĠR anch +Ġper ch +Un supported +Sm ith +ĠLeg ends +M i +Ġg f +st eder +Ġacqu iring +Ġsim ulator +() ," +re ceive +Ġin place +A CTION +ĠWeb Driver +files ystem +< Order +lo pen +ĠHE IGHT +.set Border +į ° +__ [" +Ġcl amp +Seg oe +b ands +to List +amb a +>' +Ċ +Ġcred ible +am at +play ing +.setImage Resource +qu el +Ġpod r +ge om +E k +ĠQ atar +Ġg eld +? ',Ċ +Ġc yl +( ax +ĠW I +ur ally +ĠBr asil +Ġsen za +ale y +on en +Ġb ah +Ġmolec ule +R ad +è¿ ° +AN CH +- background +- agent +Ġprol ifer +: boolean +Ġt ide +erial izer +_ ;čĊ +F ee +** ) +erg y +ĠHon or +.Log ging +ir is +Ġunder mine +ĠD y +Ġt yr +Ġde que +Ġdam er +([] )Ċ +.layout ControlItem +pe ated +C AN +rag ments +L and +) ]);Ċ +ĠS ah +ĠDE CL +With in +ĠN amespace +an other +sem bling +.des cribe +Con sum +ĠF ear +g iven +Or ange +< boolean +Ġstead ily +pa Repository +Ġresult Set +_ ENTER +_re peat +Ġt ones +ĠPRO P +n al +part icle +Ġsign aling +Ġaccess ory +ĉĉĉĉĉĉ ĠĠ +Ġvie le +ĠNo ah +- ag +Ġmur ders +Ġa ired +ĠPL AY +ĠS ullivan +_C ore +Ġul ong +Ġblog ging +> This +Ġdata Index +Ġprint able +ĠE yes +_target s +(P y +. over +Ġbr u +am pton +Ġplaint iff +< Key +b ull +Ġ⣠¨ +Iss ue +.cor nerRadius +C ritical +_p hi +. angle +Ġdynam ically +! ");čĊ +> );Ċ +in vest +.* ĊĊ +Ġt élé +Ġsuper f +Ġcas cade +DT D +Ġviv id +Ġsubsid ies +ĠH ass +Ġcoll aps +Ġcer amic +{} ". +ĠLeak age +-tr ash +coll apsed +-s ocial +ĠCh ad +Ġincl ined +Ġst o +Ġstory board +.p ayment +stack overflow +ĠRaid ers +Ġ# ' +olic ies +ìľ¼ ë¡ľ +em ap +Ġk j +Ġqu ota +ĠGard ens +ë² Ī +ĠAng els +Ġof t +Ġlower case +Ġi Param +Ġche apest +un ta +_p kt +ic ators +Ġle urs +Ġdecre ases +ĉ define +PRE C +amm ers +ĠPre paredStatement +(d irection +Ġcre ws +ark ed +ĠMem phis +ĠS ell +G TK +Ġm aid +: disable +éĽ Ĩ +ĠP f +Ġal beit +open h +?> ">Ċ +.get Source +(s cale +D u +ĠP IL +_ref resh +Ġbet s +(c ar +ĠV on +| --------------------------------------------------------------------------Ċ +ĠGr at +M uch +( Dialog +.stop Propagation +Ġte k +Ġex its +'], $ +Ġphone Number +uc s +ec imal +------------ -- +in p +.po jo +Ġcor pus +Ġpractition ers +.p ic +" testing +Ġstring By +.Not Null +Ġr ang +.D ynamic +_R ender +аÑĤ а +Wait ing +ĠW ik +Ġoverwhel med +% "> +ĠA E +}} >Ċ +u w +_t yp +Ġbuck ets +Ġgre eting +Ġla ughter +Ġant agon +uggest ion +- email +ĉt op +Ġer os +_tr i +Ġiss uing +Ġh á +Ġisol ate +Over flow +, E +Ġnut ritional +ĠAbb ott +Ġn f +.t ouch +.fetch all +_z ip +") }Ċ +Ġam at +ĠC isco +Ġn Ã¥ +PLE X +Ġse i +f oto +.to Json +å¤ ļ +ĠKle in +Ġlib c +Ġmin ers +å ¢ +- print +ĠP ride +T odos +Ġmask ed +Ġset Data +Ġtele fon +Ġunh appy +ĠT ables +ge b +( debug +_all owed +- access +Ġlog istics +Ġg ems +ĠM ature +Ġr sp +ĠAl le +.get Bytes +\ web +ynchron ized +Par agraph +Ġth rottle +.sql ite +cons ulta +ĠSe ah +C e +Ġsub mar +ER E +V ous +Ġre ddit +Ġsql alchemy +-m ile +oc ide +P our +}} ">Ċ +st ead +Ġ@ ( +Ġ[ ]) +ĠAd s +Ġover load +r idden +ĠDes ert +ĠW rap +ĠPortug uese +et z +ĉf irst +Ġmile stone +æĹ ł +Ñĥ Ñī +(s uccess +< Vector +co ol +Ġ[ ]);Ċ +erv als +Ġin vert +" io +cur so +fr agment +Ġfeas ible +.set Position +Ġel m +Ġimag in +@ Spring +Ġb ats +pu és +ga lement +ns ic +gi ene +ell ation +ĠBa iley +Sh ar +ĠT ul +ĠH K +Ġfree zing +gl m +ce ans +-c ut +_c ircle +åij ĺ +n egative +Ġind ian +s alt +Ġt ing +ĉm od +Ġs int +ak in +um l +ĠText Input +Ġpop ped +T MP +Ġpark ed +×Ļ × +ĠF usion +Ġhe ater +ET F +ro zen +h all +ĠM ik +lev ard +- heart +ĉ order +M aking +Ġpled ged +Ġdir s +$ post +ĠH err +stant iate +, "Ċ +.get Color +ĠS AT +Ġtimed elta +ĠM ai +ĉm ethod +Ġid iot +ĠTr av +ident ified +ĠDiv ine +.get Path +D ash +Ġinf iltr +Ġhandle Submit +bro ok +.g eneric +.short cuts +................................ ................................ +Ġdat ings +ĠM V + # +} "ĊĊ +Ġimprison ment +ason ic +rou d +uc ion +æĬ ¥ +Ġdia lect +Ġon Mouse +const expr +.label Control +Ġwe aker +Ġman kind +ĠRE CE +Ġd iz +Ġapp Bar +Ġqu é +f ra +_default s +Ġal iqu +_at om +: indexPath +Ġmiss es +Ġvis ually +ĠH ands +STR U +i ates +_ asset +F inder +mid t +Ġsn acks +(__ (' +. uri +ĠIn strument +ven ir +($ __ +.Dot NetBar +Ġconfig s +Ġguess ed +ि ठ+Ġinitial izer +Ġ? ", +ĠVer izon +man ifest +ge ben +.d etails +G ate +pons ible +ĠEl im +, str +Ġwrit ings +ĠD erek +ĠCo ordinator +Ġpill ow +Ġnotice able +R s +Ġduplic ates +ern els +k J +.z z +oll and +ĠSE CTION +_f name +uff led +'].' ")Ċ +ĠD ollar +Ġem oji +Car ousel +- player +Ġadjust ing +Ġjug a +alleng es +g ene +(body Parser +lop edia +ĠBeh ind +Ġslee ves +Ġdrag ging +ĠChe vrolet +Ġb iz +iv ities +ĠFrequ ency +, char +.W HITE +_pre view +) ';Ċ +_ ax +ION S +.c pu +.input s +UB E +_fe ed +ĠSup plement +! ). +es us +ĠU DP +Ġmicro phone +Ġconf irms +.is NotEmpty +":" ",Ċ +_S CREEN +ĉ expected ++-+- +-+- +ĠH ait +fast call +Ġdep ict +v b +_p icture +ĉd escription +ĠW ife +uc i +Ġv icious +ä» ĸ +ue ba +Ġset User +ãģ ¡ +Ġd iving +Ġoper a +user content +ar ah +) }, +y un +vel t +Ġun covered +Ġh ips +Ġosc ill +Ġassert ing +ĠX i +.re store +ke a +Ġsp elling +Ġder ive +ab we +ĠD ow +.set Type +_v s +Ġco zy +.c ategories +O rg +_m gr +Ġd ungeon +collection View +ĠBl ank +ac ias +ä ä +_clean up +_ACT IVITY +Ġtri angles +.Menu Item +Ġip hone +ĠW on +] ]ĊĊ +ĠCompar ison +.D oc +Ġcan onical +ĠSud an +') { +Up Inside +b uiltin +ENC Y +x be +Ġch uck +Ġcontrad ict +Ġnuest ro +Ġarchitect ural +ĠF ib +Ġcomp ares +* k +C fg +çĦ ¡ +nt en +Match es +ĠDOWN LOAD +_HAND LER +man agement +[ S +EN G +ÂĢ  +f ang +Ġsl ipped +ĠL anka +esc aping +Ġtack les +ĠPed ro +.P rop +.' ' +.G enerated +.New Guid +at rigesimal +ill on +Ġstat istic +spec ies +hold ing +Dr upal +Ġfundament ally +Ġbond age +Ġres olutions +Inline Data +\ Type +est ion +.w rap +Ġwar riors +ĠLOC AL +Arch ive +Ġembr aced +á» § +.V er +ĠAff ordable +oles ale +ĠAp plied +ĠCon version +m ega +_c am +Ġcer emon +aur us +ĠVol k +.op ens +/ about +ĠSt d +j ournal +()) {čĊ +," \ +( Arrays +ĠD ense +ase ña +än ner +/ stat +user Data +Ġg erman +Ġt z +worth y +Format Exception +ph erd +Ġsm iles +ĠWh enever +( adapter +.bad logic +Ġbrief ing +.Grid Column +- char +dim ension +ĠC opper +Ġnin th +Ġ' {{ +Ġr av +_T able +Ġderiv atives +ĠR aise +ĠF ut +arm or +-p adding +Ġre min +ĉ style +ĠMembers hip +Ġspread s +Ġgall eries +ĠClar ke +Ġcon ception +min ute +Ġab usive +_ad j +Ġterr ific +Ġover t +our cing +Ġentr ada +level s +Ġcrit ique +Ġrespect s +ĠM MA +i ene +Ġenc aps +ĠRay mond +Div ider +iv able +b az +Ġ@ _;Ċ +ĠCl aire +Ġur ging +CE E +Ġtransform er +disc ord +ĠJ ourney +t os +Ġcompet itions +ĠO BJ +ĠB is +Ġrelax ation +id y +_IN STANCE +ĠP ref +d ados +ici encies +ĠMedia Query +ĠC ube +ĠStr ange +g pu +(d ays +_Init Struct +Ġfinger print +em at +ĠGe cko +Ġr ails +ĠL um +str action +ig ung +(m ovie +_d ictionary +_int errupt +ĠQ C +ik ed +append Child +rec ipient +r é +V e +Ġtow el +.last IndexOf +Ġplace bo +ĠW ie +.es p +( Debug +oper ative +Ġdece ased +& id +ĉm utex +el ic +Ġb apt +ĉ čĊčĊ +Ġfar ther +H alf +.dis able +.menu Strip +le ccion +Ġresult Code +Ġc ans +-e lection +f emale +_F IX +aus ible +ĠP OWER +Ġrecon struction +Ġsc ans +.Xtra Bars +âĢĺ s +Rem oved +Ġparagraph s +_m argin +Ġl ymph +Ġb os +ling ton +ĠBapt ist +Ġadvertis ements +ĠMan age +/ yyyy +IO US +ENC ES +ĠF iction +ĉm enu +ĠFile OutputStream +ov an +ĠF eng +Ġsk ipping +get Class +ann i +Ġreb ounds +Ġpublic ity +Ġing res +use ment +Ġthought ful +.Ch art +Ġhat te +pass port +Ġhook ed +ĠL ens +Ġflag ship +Ġst ip +ĠG EN +Ġcl ues +ip v +ĠR ise +ĠG ew +tab lename +Ġfore most +_ validate +_an alysis +oll a +Ġqual ifications +Ġdistrib utions +ĠFl ower +Ġt ense +Ġthank ful +Ġcl utch +Ġun ified +ro ads +Ġsit i +Ġst all +_P RIORITY +c stdlib +_USER NAME +.by tes +? page +ermal ink +ĠVe get +/v nd +- author +.N ONE +ĠCon current +ĠC ry +Ġstart ers +ĠInter action +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠ +ĠLE VEL +E ll +Ġcom boBox +ĠTh eresa +te k +_H andle +Ġab y +.g dx +, end +(L ocal +O l +kn ife +ar ial +ĠH off +Ġprostituer ade +Do ctor +Inst ances +.Set Value +ĉf rom +Ġlux urious +Ind ent +Alloc ator +_D RAW +(", ", +ĠFr ances +Ġgroup Box +(s chema +Print f +OR IES +- gradient +Ġre put +ar in +_D ONE +in cre +ig nty +Ġex ert +Ġ- . +/ App +-th rough +Ġdecl ining +Ġdess ert +Ġinc umb +Ġdesign ation +.P ORT +, strong +Ġsand box +Ġw ines +ĠP av +$ str +ask ell +Ġh ö +ĠP Y +Get Instance +Text Input +game Object +/ events +created At +Ġlocal Var +ĠWH ITE +per ed +ile ge +eff icient +, color +c ate +ĠC afe +Ġsimilar ities +Ġp umps +ĠHung ary +.User name +Ġsk ate +Ġtouchdown s +Ġacceler ate +ĠH elen +OM EM +ĠK un +_v ol +Ġfind All +ĠMens chen +a head +); " +kom men +Ġpossess ed +.arg max +.trans ition +AR P +OLUM E +(s cript +ĠÐ ĺ +ĠF inding +on ces +I o +B old +Ġrenew al +_D IALOG +Ġdis reg +INT ERN +Ġt oute +Ġelect r +ĠG ross +ĉ true +.F ields +ĠW IDTH +ĠD ent +Ġà ģ +NS Notification +Ġa os +Ġme lee +. Validation +ĠDE C +-depend ent +Ġsu ic +T raits +$ message +ĠD ear +ĉ FILE +l anguages +.P rot +.add r +-g eneration +IC ON +Ġtrans plant +-d escription +Ġch asing +Ġche es +Ġ} */Ċ +Tr ad +qu eries +/widget s +sub package +Ġes pec +Ġcr acked +Ġcompet itor +P urchase +- team +olec ular +or Thunk +& P +Ġrel ent +/ #{ +Ġproduct Id +Ġè ¾ +ĠL av +ĠAl ter +.M ode +AD IO +gr p +æ ·»åĬł +Qu it +Ġdepth s +-c ategory +ĠD ATABASE +S PELL +ĠFal con +ĠQString List +Ġ'' . +ĠIn stitution +d amage +az or +bel ongsTo +ver ages +ĠN ONE +ipp ets +, \Ċ +Ġfoot print +_ archive +n ak +.get Field +ĠRef lection +Ġ' ] +ĠH BO +_dis count +Ġin cest +ĠD odge +ĠW ade +.N O +" encoding +ĠBlock chain +Ġlaws uits +ĠM aint +ch ten +Ġét ait +Ġktó re +_ ctl +(t imer +B attle +iz o +ay ed +I OR +ĠGlas gow +Ġsyn th +_log s +.p ose +_Adjust orThunk +(( & +Ġuns ure +yst ate +íķĺ ëĬĶ +O ULD +. ng +Ġdefault dict +work space +Ġselect ive +Picker Controller +YNAM IC +.method s +Ġpath ways +ĠF ew +K G +CRY PT +follow ing +ĠD LC +ĠS ara +Ġpres et +estruct or +ĠK urt +Ġair plane +Ġo mp +ĠParent s +ĠMart inez +.com plete +Ġbroad ly +Ġsc are +ĠM é +Ġelim ination +Ġpou red +/ sw +Ġcom un +Ġm asc +ĠOrgan ic +ĠString Utils +il ateral +Ġreluct ant +- age +Ġn z +." \ +Ġpast or +ale z +Ġe fect +pro v +/ init +Ġp enn +und s +Ġs size +ĠPro j +bas ename +Ġsh ells +ĠNe ck +ĠEn forcement +vid ed +st own +S phere +$ r +uss en +af il +ĠTele gram +Ġanaly tical +нÑĭ е +us ually +x n +Ġhistor ian +ĠGreg ory +ol ph +ĠUn a +Ġcon tributes +% - +anti ago +ÑĢ ед +.reg ion +Ġab rupt +ĠUnsupported OperationException +ĠT ASK +_f inish +Ġnot orious +ĠV s +ĠM Q +Ġsun set +Ġun acceptable +ar cer +Ġill umin +ĠOr b +Ġb h +E ste +_dis patch +Ġr ipped +Ġtou jours +ĠPar cel +_ ll +.user Name +.class es +S OURCE +( Number +ел Ñı +Ġhead phones +(s ide +const itution +ann ah +čĊ ĠĠĠĠĠĠĠĠčĊ +Ġcl iff +- ref +Ġmo strar +ĠPow ell ++ y +ĠB G +_f ragment +.P ort +Ġreal izing +param ref +Ġh ometown +@ Table ++" --}}Ċ +F rench +Entity Manager +ĠPl ain +//////////////////////////////////////////////////////////////// //// + ³ +( RE +c apt +Ġorgan isms +Ġj ets +ol ocation +ĠApp RoutingModule +Ġgl orious +æľ į +Ġdisc arded +ĉĉĉĉ ĠĠĠĠĠ +ĠArn old +l ug +Ġpar l +Ġhorm ones +Ġm ah +ĠSon ic +Ġorgan izers +_PL ATFORM +.in v +Ġch ord +vent ional +ĉ of +Ep isode +. Enum +unk t +ĠD h +ĠJ ared +ĠN ak +Ġint ends +End ian +Ġa ustralia +_c v +(res olve +Ġclin ics +lik ed +ASH INGTON +in ha +' * +ĠN P +_b eh +Ġh f +Ġw ür +c ategoria +$ form +Ġsub way +Ġis Active +pop ular +C our +Ġco oldown +Ġa insi +ĠGL uint +ere al +Ġarray Of +Ġh atch +======== == +ress es +_P P +. ^ +_dec ay +ĠB less +met rics +ĠCOPY ING +ĠDump ster +ĠJos é +ĠDesign s +< +Ġ" }Ċ +time zone +Ġe er +max cdn +ĠE SC +ig aret +_conn ected +_re verse +Ġquestion able +ĠUS C +Ġtut ti +Ġdrop out +ĠActiv ities +ĠW inds +')) );Ċ +Ġcon gest +ÄŁ ı +Ġprolong ed +è¿ Ļ +ĠCross AxisAlignment +LE EP +ĠVAL ID +ĠG az +Ġdepend ence +ĠP rix +.Compiler Services +j ump +Ġstr at +c irc +ĠC USTOM +x aa +Ġb mp +Ġb ureau +Ġw aren +N X +( Window +ĠChrist ie +_F E +Ġt n +ĠOm ega +communic ations +Home Page +com pletion +Ġsupply ing +YP ES +á vel +åĪ ¶ +(c lick +\ Contracts +/ questions +Ġe z +AM S +.m esh +Ġ' \Ċ +Rob ot +Json Object +ĠD F +ĠProcess or +_sh ould +.prot obuf +- users +Ġemb ry +F ONT +Ġstart ups +ĠData Source +) # +uro s +_C olor +Ġstand alone +} [ +j d +Ġforg ive +Ġng x +ĠGener ally +Ġconfig urable +/ order +Ġv as +') ";Ċ +ĠR R +ĠT roy +Ġcomprom ised +ĠSw an +int endent +Cent ral +_ keeper +Ġar quivo +ĠRead Only +_cur ve +k v +ent in +è ± +ĠE y +.im read +ĠP am +if fe +at ivity +xb c +Ġgr im +-f illed +names e +'] : +Ġa ur +ĠGib son +.Mouse Event +Ġl ado +avad oc +Ġfam il +ĠM oder +f ps +ãĢĢ ãĢĢ +- example +ĠAl zheimer +ĠU tf +_arg uments +Con clusion +text Content +rem aining +Ġinterrupt s +ĠBack up +ĠM ong +Ġrecept ors +h istor +.cor outines +Ġsh outed +Al arm +Ġcomb ust +Ġg rote +ult ural +( ids +---------------------------------------------------------------- ---------------- +ipl inary +O pts +ĠY ale +local Storage +Ġequ ival +ĠF leet +\ b +* pi +ĠQ Label +æ ¡ +Ġv x +ĠA CL +Ġsu cesso +Ġper c +ĠNot re +Ġan arch +R ing +sp b +Ġstr pos +st ores +ĠMap le +(Main Activity +(" ")) +Ġview Holder +Qu ad +Ġig ual +ors che +.m argin +Ġind ie +Ġfr anc +ĠForm Builder +ĠPart icip +.fl ash +Ġstorm s +U lt +Ġf en +[ new +E ver +=" Ċ +Ġlocal ized +_f ollow +Ġn ave +Ġdomin ance +(t ile +J ournal +ĠV C +Ġpenet ration +ï¼ ķ +Ġcomp artment +Ġb ids +Form atted +****** /ĊĊ +(c ity +âĢĶ it +[ C +Ġuse Callback +a ub +) ?. +ĠV AR +ĠSe bastian +ĠM oss +Ġabund ant +G reg +ÑĤ а +_c i +Ġbib li +CR M +ĠAt tempt +ism e +d ash +ãĢ İ +_m u +.Formatting Enabled +Ind eed +-d irect +Ġsuck ing +Ġp ne +ocab ulary +ĠPack ers +.N avigation +Ġp ied +cri bing +ĠSt uart +.To Double +ĠSecond ary +S aving +ĠD ut +ĠM add +M agic +, H +.document Element +ĠB ST +Ġdiff ers +Ġmore over +_ nd +SE ARCH +п ÑĢав +æ ´ +to Match +Ġdecre asing +-m ember +amp us +( boost +D aily +Data GridView +ĠHttp Context +Ġh ipp +_work ers +-l anguage +é ĵ +Ġconsist ed +ath ing +ĠMer cury +$ content +Ġpract iced +ĠMod ules +_D AY +Ġweakness es +ĠL odge +Ġn ar +ĠM ate +Ġj p +ĠHttp Headers +Ġsm o +ĠT OKEN +] )( +Ġaqu i +sw agen +Ġs rv +ĉ ans +A round +ĠMan uel +Ġfiction al +ĠIM G +Ġ. ' +ĠB erry +Ġwall paper +sex ual +ier o +Ġ çļĦ +ìĨ Į +Backing Field +ĠAd rian +BASE PATH +Ġrepe ats +Ġbl ues +Ġunp redict +_c oll +st acle +ĠT umblr +ĠEl f +Ġass urance +Ġc ensus +ĠIM PORT +END ER +an os +Ġ= ( +ĠEll is +" ĊĊĊĊ +.w in +ĠA bove +al on +_t ick +Ġrepresent ations +Ġæ ķ +w id +ĠAr ms +List a +_f ailure +_c m +.Flat Appearance +Ġthr one +P atch +ĠV oy +eng l +Ġnegot iating +> ` +Ġshoot s +ĠF PS +.Y ear +ĠK iss +enc ión +reet ing +From File +Ġresign ation +Ø · +Ġtw ins +Æ°á» £ +Ġge bru +.get Content +.T ree +ĠEmploy ees +ĠF IFA +Ġcert ainty +(C l +Ġtot als +edit able +ॠĢ +.Report ing +M as +qu iet +.r ules +ĠV O +con exion +, K +Ġalloc ator +ĠPow der +\ Repository +Be at +_t ipo +Ġ[' ', +_IN TR +Ġ<< < +< hr +") == +ugg age +ĠC raw +Ġé galement +Ġg inger +Ġprim era +Ġprod uto +lt k +.User Name +Ġstr error +m ith +_n b +Ġdis comfort +']; ?> ");čĊ +drop IfExists +ĠB eg +_H AL +Ġcross AxisAlignment +ĠE vidence +Ġpec uliar +Ġinstit ute +ve is +Ġf ft +à ģ +Ġzo ekt +an aly +ĠHom eland +Ġpen etr +udden ly +ĉ element +ĠB ren +ĠTr udeau +ĠCub an +j am +us lim +_e v +Ġst ems +} % +Ŀ å§ĭ +Ġbrand ing +Ġcorrespond ence +.j query +¢ åįķ +ĠRead s +(Http StatusCode +ass in +(s lot +ĠGrad uate +/// < +Ġinform ations +EN ABLE +Ġp uis +Ġfind er +ĠBr is +Ġnett steder +_m id +Ġo gs +ĠSter ling +Ġar rog +str ftime +| ĊĊ +Ġvo x +ĠReg ardless +Ġes o +ĠCom fort +.Boolean Field +Ġu h +AC Y +Ġsque ez +ĠV ic +cont ro +. lo +Ġ ire +ĠCom edy +ë ¶ +Ġorigin ated +Ġsh ipment +| max +_g uid +lev ation +на Ñı +( undefined +ĠD DR +Ġshoot ings +ĠLat ino +END OR +Ġaver aging +Ġgre eted +Ġthe aters +о е +Ġd B +Ġg st +Ġdef inite +. Storage +.h er +Ġa fore +ĠRe ality +ĠGod s +vers ed +Ġhands ome +Ġex cluding +( ad +Qu otes +ĠS cheme +? q +ĠT amil +T icks +Ġp est +' n +Ġporn ography +_mod al +Ġ ---------- +Ġdis posable +F REE +Ġsh ark +C HE +Ġdep icted +Ġdemonstr ations +ĠK illed +ĠR ULE +Ġobs essed +Ġsimpl ified +Post al +Ġconcept ual +Ġp st +L as +_PRO JECT +ucceed ed +ol u +ÄŁ i +Ġpersonal ities +Ġres hape +Ġenc losed +ĉp tr +Ġtutor ials +Ġexpl oded +_DIRECT ORY +åĨħ 容 +Ġcan on +Ġrecogn ise +P AD +ĠAppro x +ĠRest ore +ĠImport ant +Ġheav ier +.Se quential +Ear th +ĠMil k +.set Request +.t em +Ġre construct +Ġskept ical +_Pr ivate +BU F +qu a +: a +Ġse k +Ġd well +oss a +Ġreward ed +и й +(top ic +_part ition +Ġ__ ________________ +Key words +ĠFr anco +L ite +Ġn aken +Ġз а +O BJECT +Ġcraft s +ĠSw ap +.X na +.Con nect +Ġbalcon y +(re al +ĠBarn es +b ir +ĠTw enty +ay an +at ars +ĠProp el +ĠIh nen +Up grade +Ġcur b +- second +Ġn eph +.p res +ìŀ ħ +.se q +Ġp added +" ? +j l +ãĥ ¬ +') a +Co ordinates +Ġen acted +ENT S +Ġl ac +.f inal +ĠPhp Storm +c alled +Ġin quiries +.m iddleware +ĠD owntown +/ ';Ċ +Ġkil omet +ac cel +Ġqu ien +w string +set Data +Ġman era +Ġmod ular +rim p +Ġtar iffs +âĢĻ il +_TH ROW +/c olor +ĠHT MLElement +Ġcar ro +Ġpr ere +Ġplot ting +ĠPos itive +ĠMach ines +OT ES +á» Ľ +ple asant +Ġal te +Ġa inda +th ese +Ġc ors +ip ay +ĠAdvis ory +ĠRub io +j q +Ġl imestone +Ġdet ached +设 ç½® +ten ant +ĠDep th +al ore +ĠÑģÑĤÑĢ ок +ĠF ORE +ĠL ay +p resentation +) ');Ċ +.sub plots +Ï ĥ +N OW +G ar +hand les +ab ra +put ies +ĠElect rical +M iddle +rop ic +ĠJ D +ĠD yn +ĠB ristol +ĠMc Carthy +Ġstri ker +Ġenumer able +ĠEv an +.default s +qu ences +) || +ĉt oken +â Ĺı +-d ropdown +ST ORE +ĠGraph ic +( pp +Ex pl +Ġup wards +ĠD istributed +ĠW EB +J er +is NaN +çĶŁ æĪIJ +> R +üss en +ef s +Ġun cover +Ġl ud +.cal culate +Ġint ptr +Ġmidfield er +. Headers +Ġm f +ere f +.M etro +ĠSpe aking +: b +Ġcryptoc urrencies +Ġdem ons +ĉ EXPECT +Ġw icked +y outube +: Int +ĠHind i +ĠC AT +ĠØ ¹ +r ar +om ore +/ per +/lic ense +Ġre im +Ġawait ing +Ġle thal +ĠE F +round ed +ĠPl atinum +ĠвÑģ е +.co ords +.De vice +/ item +ĠW enn +compile Components +ĠK inder +.remove Item +Ġand a +bn b +Ġpr a +( transaction +Ġembarrass ing +ĉ BOOL +.content View +Ġevent data +at ore +Ġprovided In +ir ma +Ġz ona +_H W +æ Ļ +Ġst ove +Ġcounter part +_Pro duct +_MAN AGER +Ġinfr ing +ĠE RA +_p arty +Ñ ij +Ġin ici +_ Request +Ġmir acle +Ġcancel Button +S py +at ó +Ġpol ish +ĠNic ole +.display Name +\Request s +Ġuse History +Router Module +Ġst ared +ID ER +Ñĥнк ÑĨи +Ġnot a +$ arr +pec ified +Ġto pp +_DR IVER +/ ng +å ł +_t m +% timeout +< s +Ġ( *) +ĠHttp Request +_TR ACK +(n ote +ĠExp lore +_s erv +Ġç » +B inder ++ ", +. att +ĠEth i +Ġc ódigo +=' \ +.l ines +( Of +å° Ĩ +miss ible +Ġv é +Ġac oustic +Ġcraft ing +n it +.b a +ĠLuc y +Ġi Pod +Ġpup ils +-m ax +_w r +(c p +ĠRE PORT +Ġd ns +ĠRe ferences +Ġundert aken +Ġkø benhavn +Ġch ai +ĠC roat +_ Log +rown ed +_m ed +ĉ date +# __ +Ġcost umes +ĠRe quires +aff le +ç Ĭ¶æĢģ +-S emit +ela ide +еÑĤ од +Ġp estic +Ġd ra +DOC UMENT +Ġ... čĊ +}` }Ċ +ĠA uction +ĠD ock +xxxx xxxx +(get String +ħ į +Ġborder Width +ĠMach inery +Ġpredict able +.S H +Ġam plitude +.for Root +IN avigation +Table Model +at trib +Ġmaneu ver +Ġexc av +B ERS +Ġd apat +Ġinstall ations +.A sync +Ġr ays += âĢĿ +; ččĊ +.c rypto +_db g +ĠEnum erable +Of Size +_epoch s +m w +M ENU +out line +ĠP apers +============ Ċ +Ġuniform s +ĠG ig +- package +ĠJen kins +ĠHome Page +.is Selected +Ġmechan ic +M K +ĠS ounds +//---------------------------------------------------------------------------- -Ċ +Ġresearch ing +Ġinf os +ograph ics +ers et +([' / +ĠTim ber +. agent +.to JSON +_command s +par ing +_ad just +.n ome +(g lm +Status Bar +file path +? âĢĻ +Ġdetect ive +Ġunser er +ĠTib et +EN DED +(se ed +Ġsne ak +Ġam or +=" // +ĠPan thers +all ax +ĠL IVE +ĉD WORD +]= - +Ġtorn ado +/ min +Ġlung s +-c urrent +ĠBook ing +åĪĹ è¡¨ +Ġenjoy ment +ठ° +J A +typ ed +.B tn +f at +ug al +ĠSh ares +Ġdis gr +ĠB AR +ĠFO X +Op code +ĠS z +key down +iction aries +Ġdetail ing +} ))Ċ +Ġp ok +Ġdemonstr ating +Ġnot ation +l ayers +@ if +ĠN PR +.strict Equal +ĠRec ipes +.T ensor +Ġliqu or +Ġdeb ts +.ends With +W heel +.P os +CS V +$ arity +Ġun stable +( loss +ENS OR +Ġele ven +ĠL opez +ĠHop kins +con om +ĠS eth +Ġpo ems +Qu ant +Ġg sl +Ġsy rup +Ġs ibling +Ġc ass +-v ous +ö t +_P ATTERN +_SE CTION +est imated +up grade +.m ongodb +ĠBo at +_C TX +Ġfetch ing +ust in +pi el +M arg +Ref lection +Ġd uct +ĠMunicip al +Ġb x +.Get Current +ml ink +ĠAccount ing +ĠGene va +_P os +Ġpass er +Ġhear ings +com pan +Ġfrag ile +Initial izer +walk er +.M aterial +ĠHun ting +trys ide +Ġk at +Ġcl erk +á Ł +do ing +ĉg roup +Ġsan ction +.l b +ĠL azy +ĠCon straint +P agination +Ġpou vez +ĠInd icates +M ER +Ġcour s +Ġyear ly +Ġgros se +abb rev +ĠD ON +Ġproceed ed +ent lich +Ġproperty Name +ĠTe aching +st adt +Ġc utoff +orn ers +Ġa frica +Ġrend ers +ĠYan kees +ĠTool bar +sp aces +.fill Style +Ġseg undo +_str len +.F irebase +å¤ Ħ +Ġmention ing +\ ( +ĠVal ve +Set ter +Ġsp ans +ĠAl cohol +ĠLet ters +\x e +ĠT K +_B LE +.get Result +< Player +ĠP att +Ġeas ing +Ġtur key +ĠF en +') " +Ġconf ined +Ġin clus +Sup erview +(with Identifier +enc ial +Ġstuff ed +Th eta +Ġeconom ists +} ));ĊĊ +co okies +ĠRo ose +ĠChe ese +Ġfich ier +Ġen forced +AB B +no ÅĽci +_AL LOW +Ġrecru ited +Ġexpend iture +-n ight +Ġassert NotNull +_ex ecute +ĠØ ¯ +IN DEX +_F MT +Ġresc ued +ĠMonth ly +ĠCons ervation +ĠG eb +Ob ama +Ep och +ic ies +ĠOr t +Ġso it +( icon +F riends +m ol +Ġground ed +ĠC ause +ad ena +WE EN +ĠL un +IT IVE +. loop +_un til +Ġcor r +.ed ges +Ġhyp oth +ched uling +trans lator +ĠÐ ľ +R om +ãĢij ĊĊ +ĠX amarin +Ġviol ating +. anchor +--- ĊĊ +Ġtr ader +AD VERTISEMENT +Ġuns ere +ĠD AO +Ġbl ond +ĠP AT +.g lob +Ġè¾ ĵ +Ġsplit ting +Ġun subscribe +Ġatmos pheric +ĠTr im +Ġcit ation +Ġin ference +ĠF t +ĠDar win +find One +ĠG el +( Convert +Ġaccess or +; text +(s orted +Ġjud ged +); \ +: p +Ġme ine +ĠS lim +.Command s +Ġper ceive +coh olic +< Data +.entry Set +Ġassert False +ĠPat rol +ense m +ÅĤ Äħ +¨ ¡ +W IDTH +ĠRes cue +ĠU IF +_THRESH OLD +ĠMich el +ATER IAL +opens ource +ĠD iana +Ġinv ites +_B ODY +Ġreserv oir +Ġro i +c ust +(t c +ï¼ģ ");Ċ +Ġfest ivals +Ġperform ers +Ġclim bed +Ġj ungle +String Length +Ġunlaw ful +ier re +vertis ement +Ġst akes +Ġh ats +Mod ify +ĠLET TER +.H ide +Ġstat utory +_ white +ĠPer l +uten berg +em ple +.W orld +Ġoverlook ed +Ġcon cludes +/* ================================================================ +-w ise +ĉ stream +pop ulation +Ġevent o +Ġillustr ations +ft s +Ġaut of +ĠPro cedure +Ġdes erved +-t imes +Ġg ol +N SError +cre st +ĠPak istani +any ch +get Current +Ġl ar +nt l +ĠRe becca +Ġm ateria +Ġfind By +/ ad +Callback s +ĠAl s +ĠKat ie +ĠObservable Collection +ĠDocument ation +Typ ed +ĠCulture Info +ĠTim othy +Ġlater al +" type +Ġun authorized +Ġteach ings +Ġdebug ger +[ value +Ġal ors +Ġu z +Ġsc atter +Ġdown ward +Ġmig li +status Code +Ġ( )) +ĠM W +Ġм ож +RO SS +.b uf +Ġfair y +ĠInf rastructure +=> " +t lement +$ (" +From String +ĠB ild +Ġconvent ions +_n ative +ĠIns pector +ĠP ist +ub ar +Ġreg s +ĠP ilot +Th us +>' + +Ġc ela +.new s +( Product +L iving +R ussia +Ġfac et +et ical +Ġ[' $ +/ [ +ĠD ire +Ġg ases +ĠIN FORMATION +ĠE at +ĠFor ums +ĠChar acters +_m et +Ġìĭ ľ +Ġk ings +ach ie +ĠL ambda +Ġtim ers +ĠLight ing +ĠCase y +add ir +and ex +. answer +ĠH ip +ĠPr incip +Start Date +Ġ ãĢĮ +t res +Ġ& # +.Max Value +ĠPro blems +Ġlat ex +Of Class +ĠLyn n +// ' +Ġvoy age +Ġshut tle +ĠRoll er +ĠRuntime Error +uy a +D ic +ĉb uilder +Ġbul lying +Ġsimple st +.c alled +ĠL R +Ġmor ality +Ġst urdy +tr acking +.sw agger +_B IND +IT OR +-url encoded +ĠÑ ħ +ĠTr inity +Ġtr aps +Ġ| - +Ġset Text +Ġbarg ain +Ġbr akes +.get Code +Ġmigr ate +Ġrib bon +) return +Ġcharg er +ac om +ADI US +ĠAmb assador +-a fter +Ġann i +ĉs pin +Con cept +ĠHend erson +ĠH OST +.r ank +ĠNor theast +Ġber lin +Ġrequ is +.f eed +Ġsource Mapping +ĠRen contre +. ajax +nest js +Ġtre k +ĠN acional +Ġ& [ +Ġpay able +ort ex +Ġde pt +field Name +Ġcomple tes +ĠR VA +Ġon ions +al ignment +Form ats +Ġ' {$ +Hash Set +ĠB od +.Invariant Culture +Ġsettlement s +Ġhy dr +. updated +vent h +( seconds +="/ " +Ġweb page +( ĊĊ +Ġt ir +Ġto es +ĠBr ick +Ġamb ition +P ot += max +ET IME +Ġdep ot +c alls +ĠNor wegian +` : +Ġbur ger +Ġprofess ors +ĠAl locate +-third s +-ch art +Ġfor d +* N +.k otlin +Ġpaper work +ĠDE VICE +% @", +res pect +(m p +é «ĺ +- if +Ġcush ion +ob ot +Ġpar c +SP ACE +ĠNet anyahu +Ġself ish +fe at +Ġclient es +-to ols +Ġpor ch +Ġj q +. verbose +Ġlib erals +] )ĊĊĊ +p ies +Not Blank +( term +ÈĽ i +_Param s +.normal ize +B ullet +AS IC +(h ex +_client e ++ , +_D I +Ġforth coming +} ")]Ċ +se o +U m +> Name +Ġcomfort ably +irection al +W ITH +/ pr +ĠP oor +ĠVit amin +v ic +G H +Ġprior it +ĠN N +ĠC losed +¤ í +Ġis Open +\ Console +And Feel +.S UCCESS +_OPER ATION +pol ation +ĠT as +ps z +> '. +C URRENT +V endor +host s +ĠE rd +>tag ger +ĠsourceMapping URL +Ġmar athon +_c losed +Ġexem ption +Ġrecogn izes +ides how +' $ +('/ ');Ċ +m its +war z +ĠCh erry +µ ¬ +n or +port e +Ġw l +_back up +.get Boolean +.get Resource +Ġdefinit ive +. EditText +Ġs ÃŃ +.C ONT +ĠPL AYER +.c ards +ĠSh ore +('/ ')Ċ +cl uir +Web Driver +(m onth +-re lease +Ġins pector +å £ +ĠN F +_cl ip +åŃ IJ +Ġinteract ing +.t mp +Ġ'' 'ĊĊ +Ġde e +Ġfro st +"] ))Ċ +ĠPl aces +Th rows +f ork +/ day +i Phone +ĠM IC +Ġfold ing +Ġcro re +ĠCh iefs +pher ical +( price +.Write String +Ġexit ing +] ',Ċ +ight ing +Ing redient +( vertex +Ġscroll View +h f +: new +SE N +se ctor +Ġsp ins +ĠS cheduler +ote chn +sem icolon +Font OfSize +ĠSpecific ally +fl amm +.Object Id +Ġcont a +_per missions +ĉF ROM +IC ODE +/ kg +ĠHot els +-m ed +ĠD in +Ġn avy +get Param +Ġm end +Ġportray ed +ĠMet ropolitan +Paint er +Ġref erral +_g ood +Ġmar vel +osa ic +> (& +. ur +Ġest os +Will iam +Ġtim ber +Ġquel ques +ĠDoc uments +.X aml +Ġbatch es +éģ ĵ +ĠRe leased +T ail +CO OKIE +he id +_st ation +ĠV ia +S ale +ĠRe peat +Ġprom in +ĠZ o +- forward +ĠI on +it ary +Ġj us +- request +Ġproud ly +ĠStream ing +(Mouse Event +ĠS print +_ rotation +Re positories +Ġt art +ĠÑģ в +Ġm appings +è ª +C u +C ycle +Ġb un +ĉl ua +ãĥ ī +Ġ(( ! +Ġcollect ively +ĠCon d +Ġwsz yst +(l ib +openh agen +_s kip +.Column Header +é Ĥ +peri enced +ı è¿° +_p rops +Ġcontr ace +Ġmatch up +ab etic +.m embers +RE CT +(d at +Ġs og +ren om +_M ethod +Custom ers +full name +Z N +re try +Ġk ap +ĠNe u +è Ĭ +add Child +will Return +_p ermalink +Ġener getic +ĠW et +ĠMor r +Ġg cd +count s +, type +d ig +( Login +Ġcr acks +Ġbacter ial +ĠMe at +ĠArm strong +ĠBron ze +Ġapprox imate +_dir s +lig a +ÅĤ ad +Ġkind ness +Ġcont re +ĠE VERY +M ET +Ġannounc ements +g pio +ĠWaitFor Seconds +ĠPhotos hop +Ġdis contin +/ dd +Ġtop ology +an ical +. interface +auc oup +.Hash Set +ARI ANT +(r outes +ĠT eh +Ġh ype +] "). +Ġsl am +Ġbro th +- inter +ĠR id +-m anager +Cancel ar +ĠP agination +Ġsound track +Ġpost erior +Ġscr ub +cre ating +- * +ir teen +.d y +.s ymmetric +Ġ"" . +============ === +Ġch assis +ĠnumberOf Rows +Develop er +_b ins +ĠO UR +ri eb +Pro s +Ġwi ÄĻ +" d +Ġasync io +ze igen +_s pi +.A LL +Ġscre ws +Ch inese +Ġapi Key +Ġun successful +ĠSeah awks +OR G +ç« ł +Ġprofession ally +ĠCou pon +åŃĹ æ®µ +Con vention +Ġpol ym +æī ĭ +Ġsalv ation +Ġengine ered +ĠW rest +ĠG CC +Ġwar mer +Layout Constraint +Ġag grav +Script s +vent ure +Ġrefriger ator +Ġinnov ations +ĠRun ner +N IC +ĠRoll ing +Control Events +Ġlo os +p ac +ĉ panel +ef e +ĠBudd ha +------------ --Ċ +åº ĵ +(for Key +Ġl umin +Ġ( ? +ĠA IDS +, user +im ientos +content Type +ant lr +é ¦ +ĠW elt +Produ ction +m ight +ĠV II +", ( +Ġobserv ing +Ġdeliber ate +( control +Ġwith d +Ġsem ana +ST ACK +uch en +N ice +ĠDeutsch land +ĠSpec ifies +d ma +iz io +ĠF acts +_pop up +ĠDirect ors +{ : +[ R +ĠÑį леменÑĤ +Ġpl at +Ġdirect ing +ä¸ ī +ĠGil bert +âĢ¦ .ĊĊ +.q ml +Ġthere after +Ġdis position +d raft +Ġsurge on +ĠIns ider +Bl end +ĠT rev +tr insic +Top ics +rie ve +_FILE NAME +Ġaut res +J ose +Produ cer +er us +Ġpet it +ĠN EXT +ĠF ilters +Ġreplic ate +"] ). +Ġl enders +] ",Ċ +; charset +Cpp Object +Ġfl oral +ĠT ipo +Ġcirc uits +e asy +(& $ +itt a +ery l +_COMM ON +'}} >Ċ +-back ed +(var iable +( Index +Ġvo ir +_loc ations +++) { +ĠLouis ville +Ġgrat itude +.Mock ito +ĠP owers +ie urs +Ġge ographic +ra le +Ġc ra +ĠSp urs +iph ertext +AC ION +- common +Ġvict ories +ĠFinal s +.sh uffle +-m illion +_PRO C +ass ume +Ġil s +DB C +Boot Test +Ġl avor +.test ing +. ast +"] / +m oid +Ġqual ification +ges ch +ĉ put +Ġair ports +J I +Te acher +_un iform +Ġn ama +ĠB ast +ert ype +c apture +get All +ĠReyn olds +oo led +.com ments +Ġch in +). * +Ġи ли +t gl +ud os +Ġd ÃŃas +ch ai +.pro gram +Ġps z +ĉ icon +ph il +ent ral +_WR AP +ov i +Ġnost alg +In finity +ĉy ield +Ġvit amins +Qu aternion +S ink +_g oods +Ġ ........ +ĠW ings +ur idad +-st ory +"] )ĊĊ +idel ity +Type Def +G tk +Ġí Į +_M ain +Ġche z +ĠR aven +Ġpay roll +Ġfreel ance +LL U +ĠM end +ed ay +Api ModelProperty +.Form BorderStyle +Ġeconom ist +stan bul +Ġfre ight +-A gent +(m eta +Ġsym metry +Ġ' .. +.C alendar +- aut +g f +p ent +yc lopedia +Ġwish ing +ĊĊĊĊĊĊĊĊ ĊĊĊĊ +Ġgentle man +Ġê ³ += # +Ġlect ures +âĢľ In +Ġ! _ +Ġh b +ĠV endor +Recent ly +_n otes +æıIJ 示 +" My +Headers Height +_S O +Ġunw illing +Ġsuper hero +g io +ps y +ĠPe er +j avax +& apos +ĠCr isis +ord inal +Mem cpy +++++++++ ++++++++ +- val +Ġwork book +- ap += k +Ġmetal lic +_ peer +By PrimaryKey +_S D +u ator +_SH ADER +) Math +.Trans form +Ġc ows +Ph i +ĠC lem +(_ (" +ĠL ud +-d elay +ĠSec urities +ĠOrth odox +Sym fony +(re port +Ġent ertain +E PS +iz oph +ex ual +IR D +ä» İ +Ġl ith +Ġsanit ize +Ġfemin ine +IS BN +.auth entication +_p ipeline +/ constants +ĠCON F +Ġluc r +ric ia +.t tf +.set Content +Ġst an +ore an +ĠL loyd +.raw Value +Ġg or +ĠBrow ns +Re gression +Ġlower ing +na issance +Ġbl ows +Ġam azed +Ġun related +Re views +Ġrub y +ĠMod ifier +Ġgi ants +. thread +Ġcontain ment +ĠStart Coroutine +um at +ore lease +ĠR andy +@ endif +D igest +Ġsubur ban +=" );Ċ +Ġann once +. variable +\F oundation +Ġa cre +V an +Ġt uples +d ns +ĠStand ing +_l arge +Ġbox ing +Support ActionBar +ĠFort une +ĠR um +_m ultiple +arch ical +Ġf write +_ quote +Ġfool ish +Ġcompr ising +Ġо п +- selected +v f +ma id +N ama +(d atetime +Ġindirect ly +g art +fix tures +ch os +ĠH alo +Ġrec urring +- news +v il +ĠNurs ing +- produ +ĠH Q +\Http Foundation +enc i +au en +Ġv y +ocr acy +Ġdeleg ation +Ġas phalt +Ġset Selected +k ok +/ rest +met ics +ĠNS Date +Ġtravel led +Ġrec ib +Ġm ime +CL IENT +ĠG U +ĠH ANDLE +/ Q +[ z +Ġbother ed +ĠBB Q +ç as +_ex amples +_F IN +Ġwhite Color +Ġastr onom +-d ir +Ġsovere ign +Ġb reeze +Ġin ning +ĠEd monton +g li +.blog spot +js x +Ġvers a +ĠMoh ammed +.J ob +-t oggler +Ġп олÑĮзоваÑĤ +ard on +Ġnew born +Ġnav al +note q +Ġtum blr +Ġh entai +ĠTyp ically +Ġlo ot +.S prite +Fl ight +Ġw avelength +-s k +ĠEl le +_ exports +Ġ Ñı +ĠI H +izoph ren +Ġí ģ +_pr imary +Ġmo is +ĠB N +Ġsystem ic +Ġdifer entes +IN CT +Ġ'' ĊĊ +$ q +Widget Item +cl ide +$ file +L emma +/ table +ag rid +ĠMongo DB +int e +Ġapp rent +ÂŃ ing +.D b +Ġà Ĥ +ham mer +=' ';Ċ +Ġbro kers +it lement +sembl ies +E le +{ x +Ġlast name +< - +Ġfl atten +_b and +.R oot +.read FileSync +==== == +.r x +? čĊ +Ġmetaph or +T i +con te +Ġdeb it +Ġcont empt +Cpp Type +æĶ ¯ +Form Field +r atio +os opher +Ġimpl ant +P URE +Ġal ta +_man agement +Ġref ine +ĠCheck Box +ĠChar l +- version +cond itional +ven ues +Ġrif les +Ġoff spring +Ġmill ing +Ġshar ply +Ġunder water +( origin +_ Control +Ġ. $ +Pl ugins +Ġdry ing +Ġillustr ates +- u +Ġveget arian +n pc +He art +; ',Ċ +com ma +te enth +as an +/s pec +_m oves +-m argin +Ġing en +³³ Âł +Ġpro jet +Ġo tra +Ġbr as +. utc +Ġsle pt += sub +ab ilit +post er +Ġs dk +ounc ill +Ġw d +Pre paredStatement +ĠDr um +( attribute +ĠEther net +ĉ DB +Cal ifornia +c ube +[ I +.C reated +ĠH M +Ġtr acing +Forms Module +- you +.c urrency +feed ing +Ġt body +L i +acc ion +n as +Ġtr ouver +N ONE +"} ,čĊ +Ġf tp +With Identifier +pol ate +File Info +Ġpurs ued +ĠĠĠĠčĊ ĠĠĠĠčĊ +DE SCRIPTION +} */Ċ +From Nib +Ġdecor ative +_S SL +(ch at +T LS +Ġsurpr ises +al culate +ĠS plash +( Configuration +ĠS EM +im son +/lib rary +< Double +. robot +³³³³ ³³³³ +ĠCP F +ĠUnder standing +Ġcos metic +ĠX t +t ips ++ k +(" ' +ĠP DT +W AR +.get Object +ĠTrad itional +.sl ug +ĠDi pl +=" ", +ĠFil ms +ĠAn im +.h elp +Ġemb assy +ĠBoot s +Ġb unk +-r isk +Ġp ci +Ġ/ \. +ĠI PT +Ġcrash ing +Ġip v +_ ke +ĠRES P +.Log Error +Ġinade quate +I on +ĠF ür +ric ula +Ġshould Be +al ready +']." +G ED +fa q +Ġoption ally +_D is +ĠSuccess ful +ĠC ensus +Ġinc arcer +_C ARD +Ġav iation +ĠG ym +Author ity +.B ean +sh ader +Not Exist +_Text Changed +ĠST OP +( team +" H +w g +Ġgr inder +Ġstri pe +Ġpres ervation +Cl aim +avers al +ware house +target s +Tr ust +Ġal lev +, www +ous se +_ch an +_S ize +system s +Ġobj ection +ĠK ane +Ġcor ros +ĠD SL +Ġu a +ĠM H +ĠStrateg ic +_t cp +Ġê° Ĵ +Ġborrow ed +ĠA ch +ĉ command +Ġg ps +le ston +iche ver +ĠU A +Ġassault ed +Ġspecial izes +ĉ search +Hot el +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ čĊ +ĠP itch +Ġ Ùģ +READ Y +Ġparent al +Ġg éné +Ġdonn ées +Ġdet ain +T ARGET +Ġprotagon ist +Ġclear Interval +ĠIcon Button +ĠGet All +Type Info +E H +âĢľ They +Ġ{ [ +Ġg ag +Ġ Ú© +ĠD ropdown +.f ree +g one +im ens +Ġinst al +ĉc url +_C AN +ĠB one +ï¼ Ķ +ony ms +-g overnment +.binding Navigator +ĠD ans +ĠMc L +( en +>( _ +ÐĴ Ñĭ +.* ;čĊ += j +-c or +S on +.ToolStrip Item +- around +_X ML +end Date +Ġsl ack +Ġrot ated +Ġno qa +Ġc ottage +Ġencontr ar +_s kill +hou ette +! čĊ +. weather +Ġemphas ized +å® ¶ +ĠÑģ пиÑģ +ĠComp iler +( android +ĠâĢ º +. turn +Ġsup pression +_c alls +Ġ* @ +(str len +.h ex +ĠB ills +ĠR SA +Ï Ĥ +ĠEs cape +ement ia +Ġfront end +Ġp int +_ex c +zz o +[ ],Ċ +Ġ"',' " +. Environment +Ġafore mentioned +Ġend ure +prot otype +ther apy +ss i +D eg +_pl ugins +.user Info +Print er +ĠPRO GRAM +Ġru ins +Ġempir ical +Ġcraw l +ĠBo iler +- comment +.sub plot +_ et +Ġ'. ', +min or +ĠCustom s +Ġy aw +under line +ĠCom o +( (' +(m ean +Ġcha que +ĠBlock s +.r ad +ilib rium +Ġweb driver +Ġmel hor +d ana +ĠAb use +ĠSouth west +ĠP aren +PERT IES +ĉ IL +Ġscre am +v u +Ġin comes +Ġn im +Ġl ace +Ġcompens ate +Re verse +D at +_att ack +Ġn our +ach en +ce k +< Func +w ie +com pressed +-m atch +(" ")]Ċ +im ized +. orientation +.compare To +Ġmass aggi +Ġìľ Ħ +Ġel bow +Ġant ioxid +undred s +/ tools +ĠR OW +an mar +ĠW ow +_t icket +Program ming +Ġthe or +-re view +() )));Ċ +ĠRichard son +ĠP ocket +] [] +am pp +_ health +ĠP OP +ĠNav al +Gu ess +Ġancest or +.Get All +.local Scale +ĠM apper +Ġaccum ulation +Ġsim ulated +ĠDr ivers +Ġd és +cur ring +Ġele phant +Ġadvert ised +Ġmail box +SH IFT +ĠMon ica +Ġan c +Ġward robe +Ing redients +Ġ|| čĊ +ipp y +Ġantibiot ics +av ings +(c x +ĠFerr ari +ĠAn imator +.d type +rem oved +order by +Ġc res +oc ê +Ġp ym +ĠCirc ular +@ index +ĠW arm +S ay +ĠAss istance +Ġcur tain +ĠMont e +IL ER +ĠC VE +ĠD uck +ĠAll ows +_f ire +ĠDer by +Ġre pos +Ġhttp Client +Ġpsych iat +Ġnow adays +Ġcaut ious +ĠComput ing +Ġcompletion Handler +ĠWel sh +ĠB EST +Ġstress ful +_P E +æĹ¥ æľŁ +ĠData Frame +ĉ Integer +_P rint +M oves +Ġtransform ing +.B atch +y ahoo +Position s +ze j +Ġno od +io res +_ * +Ġcl k +ĠF loyd +Ġh ap +font size +Ġn az +.not ification +ĠDep ression +Ġac ne +*** ĊĊ +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĊ +.cont ents +yn th +ĠStra ight +')}} "> "+ +Ġtoken izer +Ġsovere ignty +ĠP ence +() ");Ċ +Ġpesso as +.G e +ĠIn cluded +Ġpag ina +Ġex posing +е ÑĪ +_SC RIPT +/$ ', +Th umbnail +× Ķ +webElement X +webElementX paths +press ure +ĠCur ry +_C P +OL UTION +ILE S +prot ect +ool a +Work space +{ };Ċ +ĠU NS +Ġsymp athy +ro ker +Ġrem odel +ĉc ell +Ġat op +.Full Name +Ġfa ut +ĠE asily +_d ynamic +Ġfr amed +Ġmot ive +è· ¯ +s am +Ġmar ca +ĠText EditingController +Ġde structor +cre am +Ġr ude +ĠB old +ĠInd igenous +Ġg ens +Ġrel acion +(s ystem +ĠUIF ont +_char ge +UST ER +E V +.N amespace +Ġmer ger +Ġcal loc +g ang +Bad Request +Ġs per +-d esign +Ġâ ĩ +Ch an +Ġorgan ism +, ) += id +_pl ane +ĠC ases +elf ast +ĠLegisl ature +ĠF aker +Ġinv oking +- utils +(). ' +.f ace +Ġguard ian +my Modal +Ġclip board +ĠAT M +Ġpe as +ĠS ylv +.c alc +ĠContact s +int Value +Ġmodify ing +ĠBar b +. loss +_per centage +Ask ed +(l st +ategor ical +- files +ĠRoman ia +.A c +Ġh ai +ĠF lying +Ġ ż +j p +ĠTr ainer +. arc +_de g +Ġtrace back +Or Fail +F LOW +. old +oy a +g mt +is empty +Ġvacc ination +Ġob solete +recogn ized +Ġru ined +ĠRe in +ĠTr acking +xf b +ا ÛĮ +Ġvæ re +Ġbr yster +ĠIT S +Ġdest iny +Ġsw ear +Ġred es +Ġcl f +Ġfl ipped +ĉ head +Bl uetooth +ĠOver rides +: Boolean +_ = +_l r +sp awn +: index +VAL UES +is key +? ");Ċ +.syn thetic +ĠCheck ing +struct ures +ip ing +Ġvoc als +- Up +ĠManufact urers +ĠMar riage +代 çłģ +Ġgar ner +_C lient +par allel +RI END +Ġvine gar +seg ue +J B +Ġcontact ing +ĠCar roll +Ġout reach +t ensor +_var iant +Ġthe at +lic able +{ | +t iny +_ letter +Ġp encil +HeadersHeight SizeMode +ilt ro +.auto configure +.d rag +.use State +ĠB MI +h int +Com pile +* \ +en ary +Ġl vl +.C ache ++ =" +_t v +ruit ment +Ġf read +Art icles +f ila +Ġpack aged +âĺ Ĩ +AT HER +ĠPl anned +s cheme +Ġdi ary +Ġoff enses +/ F +ĠSt ick +Ġc erc +ĠS lee +ĉĉ ĠĠĠĠĠĠĠĠ +< Image +Ġè® ¾ +- editor +pie ces +ĠD rama +Ġ// //////////////// +ĠT asks +AR C +g ateway +.get cwd +.M etadata +Ġguess ing +åľ° åĿĢ +Ġsm arter +ĠGet Enumerator +Ġe fter +/ operators +ĠGL float +Ġf ør +Ġop aque +ä¿Ŀ åŃĺ +Sp read +SY STEM +Ġinv ersion +ĠBasket ball +Ġsim ulations +Ġden ies +Ġa vez +_list ener +Ġenh ancing +ĠMy th +ĠL akers +_M D +Nd Ex +D ATABASE +Ġt á» +ar th +[ left +Ġcontest s +st ile +(K ERN +_f c +_p m +Ġpres idents +Ġhospital ity +Ġfade In +RO PERTY +_m aps +ĠDefinition s +Ġassess ing +Ġus ar +Ġquant itative +mo z +Be autiful +[ (( +b ons +f requency +Cont ain +Ġpuzz les +ĠCast ro +Ġv illa +Ġkind ly +Font Awesome +ern a +epoch s +_dat as +ĉ ip +.p adding +ĠCont est +Ġed itions +Ġdispro portion +ĠI CO +Ġcome back += value +ri ad +-s ort +Sub mitted +(n etwork +ĠC el +Ġinstall ment +l ashes +.List View +ĠV atican +(Media Type +IV ED +reach able +: Is +ĠC ITY +äº ¬ +ĠHelp ful +Ġba ÅŁ +% čĊ +Ġpsych iatric +Ġrec ycled +FORM AT +ĠG row +b ine +G it +.s s +ĠWe apons +ĠSt y +_ arrow +* self +ire ment +Ġdeg li +App Delegate +_b anner +Ġcoordin ated +ĠWeb cam +Ġcelebr ations +. act +******************************** **************** +( show +Ġweek day +Ġconc erts +ол н +cl in +Ġcr on +ĠN im +.set Vertical +ĠEll en +س ت +ĠS AM +E ff +g z +ste am +Ġant ique +ph ysical +ĠForm Data +.set ter +ĠPO INT +B on +Ġflav our +erv ention +_ENT ITY +ĉ ĠĠĠĠĠĠĠĠĠĠĠĠ +Ġintr insic +Ġæ İ +append To +aram el +) ]) +ĠRecomm end +) m +OutOf Range +Ġkn ight +Ġsat ellites +ĠTit ans +Ġweigh ed +ĠD ana +e ase +Ġs ip +S IM +ĠDevelop ers +mal ink +/ check +_P LL +n ung +Ġdry er += A +.d w +_S QL +Ġsub plot +D ROP +Ġprot otypes +Ġhour ly +display Name +Ġas i +ĠViol ence +Ġastr onaut +Ġdat atype +Ġinformation al +Ġinvestig ative +etermin ed +ren al +; '> +ĉc ol +V G +_ boolean +re cent +Ġ* )ĊĊ +ĠRain bow +om men +Ġl ur +Ġopp ression +(", ");Ċ +ĠFac ility +DEF INED +Ġne on +Ġoff ender +AF P +ĠClean ing +[] ): +Ġund ocumented +.Re positories +ĠG uitar +аÑģÑģ ив +Sk ills +Ġtestim on +rypt ography +ĠAm ber +ĠSt alin +Ġl one +Ġap enas +Ġdies es +ĠAr duino +è½ ¬ +== - +_A ct +Ġc oded +âĸ ł +amb urger +-link s +Ġarm our +.H igh +get Content +st ag +Ġhe ck +ĠìĹ Ĩ +ĠMc Connell +ĠCon cert +ĠAl loc +ä re +.replace All +Ġpart itions +rot t +ĠF le +_T REE +reason able +ĠReport ing +Ġbillion aire +s cores +min s +- eye +M ORE +ab ort +ĠSW T +Ġin verted +ĠTe achers +; n +Ġast ro +н ов +ани ÑĨ +product o +c ountries +ĠO wen +Ġcont amination +Ġv ibe +ĠEll i +.s cript +ĠOl ive +D MA +v ier +: semicolon +-m odule +gress ive +ag u +_ players +Ġresult ados +start ed +scroll Top +==== = +Ġweigh ing +Ġ[[ [ +z ahl +( NS +ĠAssert ion +le ague +.setText Color +ĉ Message +Ġmom s +_A F +. wh +AL S +Ġaut re +] ĊĊĊĊ +.op acity +ĠBudd hist +Ġde af +ĠOrgan isation +(G lobal +ens ch +Ġhead ache +ĠAli en +_in ode +ĠSt ark +Ġæ ī +-l nd +ore f +_fe at +Ġpedest rian +Ġnom inal +Ġbal loon +Ġspr ites +Prototype Of +ĠA post +ĠF EATURE +O H +Ġre cess +ĠDon na +con sumer +$ GLOBALS +ĠG IF +- frame +In icio +Ġpass ages +Date String +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠ +.by te +B ug +initial izer +p kt +od ium +ĠD ER +. ops +ler i +Ġgift ed +Ġdet ach +ter rain +elt ers +ãģ ı +. loader +ĠN GO +str ncmp +K h +(font Size +ro cket +Ġpreced ent +ĠAur ora +ĠEx periment +is phere +Enc oded +ĠâĢĵ ĊĊ +Ġpy ramid +ĠAnn iversary +of il +ë Ł +( plugin +C oeff +Ġcooper ate +Ġpredomin antly +IS M +Ph rase +_DEF INE +Fl ip +AMIL Y +ĠMark ets +ĠStream Reader +ĠComb ine +Ġmanus cript +z za +, tp +Wh atever +IT ICAL +ighb our +Data Provider +.Text ure +priv acy +.S DK +Ġre charge +Ġc pp +ĠC FG +(h older +(p y +m ot +Ġsav oir +ĠR osa +ĠPC s +Ġí Ļ +.her oku +Ġf ren +ĠR iley +ag ate +Ġs ond +.x lsx +Ġh acked +st ad +G i +Ġsan ity +ĠSql DataAdapter +... ", +ĠP ussy +Ġ **************** +Ġhass le +_P ARENT +ĠU AE +Ġbegin ners +( Client +Ġstatist ically +.h our +ed elta +Ġtr action +uel ve +ar at +Ġsa una +IN VALID +Ġindict ment +AL LE +Ġdiss ent +ĠTyp ography +Ġintention al +s it +ĠAn imals +Ġcoun tryside +Ġu art +} \" +Ġseam less +¾ 示 +Ġaut os +Ġ"' ";Ċ +Fl ush +ANN OT +Ġal gebra +ass oc +ĠW aters +Ġprepar ations +ron ym +[, ] +S ans +Ġarm ies +ipe g +Ġcream y +. art +et re +ĠAn imated +Ġun pleasant +eme an +g reat +i Äħ +ĠEar lier +Ġch ic +Ġpres erving +(ex ec +ĠInvest igation +ĉG PIO +Ġrig orous +ij o += num +Ġtool Strip +) set ++" & +ĠAcc eler +Ġdevelopment al +is posable +Ġflaw ed +re ne +Up dating +Ġwatch dog +Ġden ominator +Ġsubur bs +Ġ... ) +Ġconv ictions +c losure +.I P +Ġtransl ates +.sw t +.Tr ace +Ġmet tre +.is Enabled +ĠEffect ive +.to Int +Ġen chant +Ġst unned +Ġpo i +/ code +ad m +.datab inding +ĠL orem +________________________________ ________________________________ +Ġled ger +Ġcar a +ĠG ir +Ġwa its +Un o +Ġc wd +è¾ ij +ĠT Result +Ġre jo +Ġem itted +ĠWest minster +ä¸Ģ 个 +ne k +_T is +Ġen act +ĉ with +org ia +Ġj ue +Per form +SP ATH +.top ic +ĠD aten +Ạ§ +Ġsit io +_M M +" So +b ial +Ġsc oped +Re quires +ĠT OTAL +ĠCh ancellor +( contents +Ġste alth +dev ices +-p ass +ili h +ĠMal colm +ĠDep ot +Ġconfig ur +a ussian +_con straint +в еÑĤ +G RA +ĠR ates +.dataGridView TextBoxColumn +ĠNob el +it ics +Ġignor ant +ĠReport er +ĠEb ola +ĠSh ock +_re lation +ĠNin ja +) c +Ġt icker +.is Checked +ĠSup pliers +ĠRap id +Level s +âĤ¬ âĦ¢ +ĉ queue +Ġch op +ĠUn ix +re ject +-c alendar +(s ort +è ne +erc icio +Ġh ect +CALL TYPE +rou pon +Ġrent als +auth ors +{ name +ĠF IFO +Ġl assen +ĠN ous +Ġsn apped +Ġfert ility +" log +click ed +Ġplant ing +Ġg b +/ output +PE AT +Ġc ategoria +Ġb ach +Prof essor +in th +"] čĊ +Rec order +ser de +ĠTrans mission +tr ad +Ġtur bo +_VER TEX +\ Event +il ver +Ġbod ily +ĠS ources +Ġkill ings +.xr TableCell +Ġfold ed +/ legal +un er +ĠR ifle +ĠM IDI +_Selected IndexChanged +.Size Type +ĠWeb Socket +Ġsele ccion +S and +ot ros +Ġenv ision +/ etc +ĠMel issa +Sp ot +но е +_ ARM +At tempt +ĠB I +ãģ Ķ +ĠD U +Ġback lash +str ide +/ classes +Ġtext Color +_st aff +ob lin +agent a +.c ollections +ill age +' čĊčĊ +fl atten +_s ales +_M ASTER +T W +_d a +P itch +ph ies +Ġz ombies +ĠV ERY +ĠPharm acy +Ġprogress Bar +Ġhas htag +S idebar +@ stop +(p c +ол ж +MA KE +ĠCor on +Ġkv inner +ĠM aid +b ob +.title Label +Ġsuccess es +ĠDemocr acy +ĠSurg ery +Ġcou gar +Ġcur so +Ġl oro +ist ency +Sen ior +æ k +ĠA AA +ĠBO OK +к о +W STR +Ġ*/ ,Ċ +oy al +.v ector +ĠS PEC +SS F +Ġcomp uls +ĠAppe als +ĠW inston +ĠMock ito +con trib +. available +entity Manager +ari as +_s ale +_r s +Ġdec oding +Ġloc ator +ol ith +Ġk ol +Ġasc ii +ĠR ut +/ interface +ĉĉĉĉĉĉ ĠĠĠ +ĠN umer +.fl ip +-d el +Ġbol ster +on omic +Ġz m +L G +Find By +Ġadapt ive +lo o +Ġv ue +(re verse +_c anvas +. roles +ific ado +ven ient +" As +ĠEn tr +al igned +Ġbere its +/// ĊĊ +.g wt +. employee +_cl i +Ġanticip ate +éĻ IJ +Ġp ik +Ġmush rooms +(t t +Ġo ma +ĠSan chez +_g oogle +. Valid +ĠFile Name +iv ative +k ed +-w ar +Ġm aturity +и д +Ġmin er +Reduc ers +ĠLat Lng +_ST D +D igits +Cal c +-up load +Ġhand ic +ี à¹Ī +egr ated +ĠST M +C lients +ĠTur bo +SY NC +Ġphotograph ers +. Out +.char acter +B UILD +.un lock +Ġar ises +ĠCommand s +(" ");čĊ +_F ORE +; ', ++" ' +. Images +") { +ĠM eyer +Ġneg atively +ĠD LL +Ġex e +Ġdef iciency +Ġwild ly +-s witch +con struction +Ġexception ally +ĠL iz +/j ava +Ġtheir s +ĠCont emporary +l is +.fill Rect +ĠN FC +Ġre he +(num bers +Ġr aster +Ġfig uring +Ġshow c +ĠJ ill +Ġarc ade +ĠConstruct s +md l +(' | +Ġident ifiers +Ġst ellar +( Connection +Ġ" {{ +y or +(m ysqli +Ġdo ve +Of Birth +.dis connect +_h i +Ġzw ischen +ĠGr und +i ros +_A rray +.on click +ans om +An swers +ĉ remove +F a +Ġhur ry +-in f +Ġget Class +ĠReg ulation +ĠFLAG S +m isc +K en +_ heading +G Hz +- entry +Ġbi ography +S ig +-m f +Watch er +âĢľ A +} px +Ġsp icy +_s q +L ost +(tr ack +а ли +Desc ending +< bits +qu ine +ĠAdv oc +_S N +ĠHann ah +PO P +Ġem itter +Ġc yn +ĠC AD +? ). +/ set +ĠS ister +ĠEnd point +Ġmen or +Ġinter p +r k +id le +Ġout fits +. vertex +Ġc lic +ARE N +Ġpost ure +ĠOpport unity +v x +ĠFor bes +.D irection +Ġres ide +Ġremember ing +nest y +Auto resizing +pro viders +ĠA H +Ġhur ting +ĠL ily +eval uate +lij k +p apers +ĠSm ash +ĠL AST +Ġwell s +w asher +_RO LE +ĠD anger +* (( +_re pository +ĠRes olve +ĠRoom s +_R G +ĠQ T +o op +ĠHe ap +Ġslow ing +Ġgrat uite +_c atalog +Ġpol ynomial +L y +pc s +F ox +ĠC yr +Ġdim in +/ month +S alt +Ġh ind +.P ER +For um +c en +_p ol +íĺ ¸ +Ġin ser +( ~ +@ test +ĠGold man +Ġupload ing +F c +Ġkom mer +Ġm itt +_log ged +Ġbu cks +-l ayer +) };Ċ +ĠO M +Ġv eg +col our +Ġоб ÑĬ +Std String +_ que +ĠT ian +Ġspecial ize +и п +Ġк л +tr ial +- edge +Ġm ars +OG LE +Ġempath y +ĠB om +Ġcoll isions +Ġcart e +ĠTe il +ĠM PL +Ġporn ô +Ġa irlines +A ws +N s +ĠSp awn +( use +é» ĺ认 +Ġy acc +st or +Ġconf ess +Ġpe que +r age +? "Ċ +/dat atables +ĠSh ower +__ / +Ġcryst als +Ġbus car +ĠH aus +iz ação +_ entities +ķ Į +ļ Į +x cc +v irt +-che vron +( Result +c ake +COM E +Ġprohib it +ĠCh ess +Ġbe aucoup +ĠÑĩ ÑĤо +R UN +ĠI K +ó ÅĤ +_ Update +Ġsle ek +ĠSpec ify +_c redentials +ÅŁ t +ĠUser Name +ĉ Value +Ġarray List +Ġex changed +ips is +.re lated +ĠSe ite +_B AR +ĠL em +ĠW ATCH +ĠC lients +Ġ. * +ĠEar l +-re port +Ġforeign ers +Ġstrengthen ing +ĉ Description +(g o +.tool bar +Ġcalcul ates +ĉs ource +Ġcz as +Ġre cl +ab o +Ġlocal host +Ġ^ {Ċ +.P op +ĠDes igned +\ Abstract +H old +ĠGuid elines +ipl ine +Ġc aching +.Re ader +_ext ernal +.str ptime +ĠWeek end +-M ar +ĠBe i +Ġ{* } +ĠR ud +Ġexpl or +ĠBou levard +C ash +Ġprep ares +Ġserial ization +ew ater +Ġad c +: ĊĊĊĊĊĊ +Re fer +Ġsc anned +} }ĊĊ +ĠF ul +Ġtour ing +ãĥĥ ãĤ¯ +> (( +sur vey +Ġí ĺ +... ')Ċ +ĠDiv ider +os l +_C ANCEL +_pre pare +st in +ĠHe ath +.Primary Key +ĠâĨ IJ +ĠLocal DateTime +Ġcooper ative +L earning +.en queue +Ġgo og +ĠReg ression +im ates +Ġvoy eur +ĠDr ink +pl ug +Ġl ender +man a +Ġperson nes +yp se +Ġun link +ĠRav ens +Ġhur d +Ġperiod ically +ARG S +ĠG H +char acters +... "ĊĊ +- establish +Ġd n +( condition +ĠGr avity +Ġest as +_f ocus +Creat ure +(s ite +Ġc arr +ĠR L +ĠR I +ĠM oto +AS F +ĠLuck ily +ĉ Route +Ġent ropy +(" ," +Col lect +( contact +ĠFlo rence +Ġpremium s +Ġlif ecycle +Ġb ans +x ef +Web Kit +ĠFlo ating +Ġcos a +Spec ific +ĠLo ans +b read +Ġdes criptors +Ġ{ :. +TH READ +ĠT rent +Ġsc op +Q A +ĠAnt ar +p el +_d ifference +_ch anges +(... ) +ĠR otation +ĠLG PL +ĠJ UST +(T ask +_sub set +ĠTR ANS +åĬ Ľ +ĠSc out +-p opup +Ġsm oked +_C lass +Ġturn over +br akk +ĠRock y +t as +.Regular Expressions +ĠElli ott +ĠSp inner +DU CTION +Ġlib re +Ġmol to +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠ +ĠF TP +m peg +(f eatures +Ġb ald +ĠV id +Ġsh outing +L int +Ġsock ets +Ġpro w +Ġnouvel le +isc ard +ĠS ponsor +Ġconsult a +)) ); +Ind ian +ĠR aspberry +Ġteam mate +ĠJ WT +ĠGh ana +Ġc akes +pr imer +form a +erg arten +_M anager +Ġpre season +G AME +| " +ĠBro ck +Ġoccup y +Ġdecor ations +á nd +Ġc ot +Ġpar an +D isk +rem ain +> ? +Str ong +Ġfr ance +ĠE ra +-c r +.Buffer edReader +ĠParad ise +ĠV AT +ĠAnd ers +Ġlim b +amp oo +Ġimper ative +UT ILITY +ĠRec ognition +Ġragaz ze +Ġpop s +yp ress +Ġemb argo +// {Ċ +Ġsy ll +P TR +åŃĺ åľ¨ +Ġdid nt +Mail er +Ġacad emics +ĠFra uen +ne ider +- rel +Ġrain bow +( In +Ġslic ed +============ =Ċ +(s end +NSMutable Dictionary +v os +(p ackage +Ġord inance +view er +ĠSant os +-s elling +Ġgo v +ett le +Ġfound ers +Ġw aking +sl ashes +-p ound +re cht +ا ت +.on Click +Ġn ord +st änd +_ when +UT ERS +ic c +Ġcaps ule +ĠW id +M arc +ภ¸ +ro red +UG E +LO UD +ĠAud it +ip ients +op ian +ĠS ue +Ġwur den +.H elpers +Ġf actions +[ np +-th an +Ġre co +Ġk as +Ġcmd s +/n etwork +xb f +get Color +Ġbi ased +ĠL ak +D atas +vent s +Ġë ² +_P S +. Validate +Inv oker +Ġne uen +Ġju venile +V ISION +Ġdev ote +Ġlin ha +Ġdiscount ed +\ Config +Ġworth while +Ġskin ny +ĠC ourses +le ys +ĠMort gage +K evin +Ġannounc es +]) * +res ervation +Ġæķ ° +Ġprejud ice +ĠString Comparison +Ġbe ard +-w in +ĠS ão +ĉ ms +j al +ĠE arn +_ ports +ĠN ombre +_C OR +ĠB UILD +.s ound +Y ellow +Ġlineback er +Ġchar itable +j ug +_NON NULL +ĠD ental +"> ${ +ĉm atch +R ussian +Ġvers ch +Ġp inned +Ġadopt ing +Options Menu +P ag +Ġpair ing +Ġt read +erc ises +ĠSp read +) i +ĠB AD +_t f +UI ImageView +pop ulate +b ab +ĠÏ ĥ +[ ++ +Ġopi oid +Ġ## Ċ +d type +ĠStart s +('/ ') +Ġperson als +-mark et +Ġredund ant +ĠEss ential +Ġscrap y +Ġи м +a cl +Ġcre ar +ĠB end +Ġrel ieve +- room +w ife +Ġv Ãł +ĠQ Point +Ġqu asi +Ġmethod Name +\x c +ĠPer u +/ The +. orm +Ġv iz +/p df +Loc ated +Ġconfront ation +ĠChampionship s +Ġhyp ert +Ġd j +ĠUser Info +ĠåĪ Ľå»º +\x b +(s im +Ġ== Ċ +Ġst aging +Ġdr astically +åŃ ¦ +l ords +. less +вед иÑĤе +ĠB ucket +ĠM am +. term +_p i +c zy +.p ub +prec io +ĠV irt +Ġrom an +it at +L ex +_inf os +Ä ° +. other +VE LO +Ġp onder +Ġh anno +( Page +do i +Ġpol ite +Ġprogram mer +D ies +$ d +Ġrep lication +add Column +fr ican +Ġl eng +be er +o it +Ġw asting +yl im +me asure +N eg +Ġpart ie +.con sole +ĠGu inea +TE L +_f act +.ch unk +Ġl ent +Ġall er +Ġठķ +_id le +Ġad missions +JSON Array +Ġv ibration +.h elpers +å¤ ĸ +Ġh en +j ohn +Ġì ĥĿ +Ġjud gement +Ġge en +ter ra +^ { +ĠI z +Ġc â +inst ances +Ġthreat ens +Ġm üssen +Kind OfClass +Ġstoryt elling +_d emo +ri as +Priv acy +h ift +ĠY i +es or +íķ ł +ens itivity +.W riter +ภĤ +D istrict +.get JSONObject +Im pro +(get Resources +ĠS PELL +rodu ce +Ġslow ed +Ġlin ewidth +Ġhonest y +ĠCo ord +ĠF ork +ĠDispatch Queue +ĠCl iff +ĠW iring +_TIM ESTAMP +oll ah +av oid +++ ];Ċ +sem antic +-c ss +Ġv eto +ĠM err +Ġlegisl ators +CEE DED +Ġquestion naire +ĠP ills +Cal culate +(c ore +' e +Ġdis like +ĠPre ferences +_EX TERNAL +è° ĥ +Ġd odge +æľį åĬ¡ +.n ames +.draw Image +_p rom +uck land +Ġ<$ > +ı z +/s ite +é¡ ¹ +rop he +Ġcomp elled +Ġl aptops +Ġun i +C LOSE +Ġcasual ties +ĠUn iform +Term inal +. "," +D AT +(T reeNode +ĠGand hi +(st mt +AX B +* M +Ġumb rella +an imal +Ġgr pc +Ġwhere by +Ġfloat s +ĉ arg +Ġdb g +Ġexceed ing +Event Type +.SaveChanges Async +Ġ{ {{ +Ġow ed +ahren heit +Ġì § +Ġequ ipo +ur ai +Ġid ol +] ")Ċ +_m ajor +Ġentire ty +inger print +ç os +/ account +ĉ right +urs os +ĠE DT +_INS ERT +Ġsh ining +Ġ< : +Edge Insets +Ġcolon ies +. IM +ĉĠ ĉ +RO AD +CC CC +pl acing +Ġget Activity +em acs +' %( +.click ed +ĠTh em +is ia +Bus car +.re name +Ġo ath +Ġafter ward +ĠU FO +AP S +ĠJackson ville +.s ome +Conf irmed +.s can +ig Integer +Decor ator +sh ield +ress ive +.d id +请 è¾ĵåħ¥ +Ġsh utter +D am +Ġparent ing +ey ed +$ item +-de velop +Ġextract s +Ġdecentral ized +ĠEl sa +_sp in +]) + +-in itial +Ġmult itude +Ġsens ory +ĠMODE L +Ġsafeg uard +ì ¹ +Ġhunt ers +ĠT iny +IN O +decor ate +ĠNo Such +H o +( Response +Ġr uler +ĉ short +Ġc aster +Ġclient Id +Ġp db +ëı Ħ +it ic +ĠGame State +Ġnew Item +)ĊĊ ĊĊĊĊ +ou is +n oc +.BL ACK +_V ECTOR +---------- (); +.get P +any e +Ġneur on +if old +ĠK nown +Bit coin +Any way +ay ette +Ġ' [' +Ãł nh +m gr +Ġcor related +Ġn ause +Ġment ality +has Many +ĠF G +amp ie +IT U +F s +.S p +_b etween +Dep endencies +ou g +Place holder += text +ĠMan aging +ocal ypse +åĮ Ĺ +_m ag +f ld +â ij +C AM +ĠHelp ers +Ġd ost +/ out +Ġassass ination +.get Image +ĠKenn y +.' )ĊĊ +){ // +ĠR anger +Ġg ek +Ġsinc ere +< Value +ĠD OT +ĠVict ory +Ġleg ends +Ġpr isons +(ex pression +ĠR abbit +_s entence +Ġbit es +Ġon Failure +ĠâĪ Ī +K im +.g ender +ĠÎ » +Ġ[ . +"] ); +land ing +-d igit +TE MP +ĉ entry +Ġstrt ok +Ġdesc endants +um no +Ġlean ing +Ġspecific s +q n +ĠSp art +Ġpor r +EDIATE K +Ġse per +' aut +ĠSTE P +ĠBorder Layout +Ġret ros +ĠSalv ador +ĠEN GINE +x dc +T weet +v k +Ġì ² +] << +het ics +c oding +Re ach +.re q +gu ide +.s cope +sh irt +rog ate +SET TING +ĠProte in +Ġe ing +. EMPTY +.d f +Ġclear er +Ġc rossover +ĠTo ys +Ġco ated +.M onth +ĠAtt ach +/ run +.t abs +Ġogs Ã¥ +B rown +.D ATE +Ġf os +åŃŠ符 +W ood +-th ree +her ited +Ġ rop +( ac +Ġembod iment +ĠKenn eth +Ġcan non +Ġb idding +čĊ +.get Resources +Ġl ump +_const s +( ext +ĉd ir +â Ŀ +Ġpadding Top +Ġobs ession +Ġb anning +ĠApp Module +Ġpart isan +Ġcatalog ue +Ġmin ors +Ġpitch es +we ep +Ġundert ake +Ġthem ed +aud it +.scroll Top +Ġr er +Ġsympt om +Ġopen ings +.block s +open id +Ġas sh +-s ave +ĠP ig +Ġreg ain +Ġin icial +/f avicon +ĉ exp +Ġsp ices +isk a +claim s +m ak +definition s +Ġcorrespond ent +ĠCann abis +__ ,Ċ +ĠL ucky +ĠGa ussian +ĠN early +C AD +'] ]Ċ +Ġadequ ately +ĠT ITLE +constitution al +-m m +_ override +Ġbl as +.ready State +Ġremin is +Ġrein forced +ĠColl abor +Ġdecor ating +Ġb achelor +ERRU PT +Ġup right +ip ation +ĠNob le +Ġvalue ForKey +Ġset Loading +.I gnore +å ģ +G lobals +ĠM ent +AS SES +Ġlim bs +ĠH UD +inc i +. iv +ĠQ ModelIndex +F use +Ġped al +_F REQ +( verbose +Ġlong itud +ĠChar ter +ê ·¸ +Ġbund les +. ignore +um bo +EM A +.... ... +s x +.C ard +Ġhe ute +Ġste er +j umlah +Ġ{ _ +_Check ed +Ġf ax +ĠG ust +itch ens +Ġ ))ĊĊ +Ġremark ably +/ XML +- remove +_b t +Ġinc ub +.p ackage +.current Thread +ĠHigh lander +.s ide +s plash +Ġ ici += D +Ġp uck +Ġball ots +Ġhug ely +co eff +Ġp Data +.C OLUMN +ĠHe aling +Ġord in +! ), +Ġ' ',čĊ +(m d +ĠS ask +< strong +Ġsurviv or +.s eries +Ġcaffe ine +Ġ` ( +.TRA ILING +_ Input +(" ^ +z d +& );Ċ +ĠP ing +Ġv oucher +.r ating +-sh irts +ĠRetrie ves +.al ibaba +Or acle +_MO V +Old Data +Ġ/* čĊ +Ġg boolean +Ġ=> čĊ +Ġr á +Ġbl unt +ĠImage Icon +if ik +RT C +Ġfib ers +Ġto ile +.s ent +ĠPy Qt +$ app +Ġmed io +Ġgrant ing +Ġtsl int +ĠM ö +(fig size +Ġhur ricane +Ġlif es +Ġà Ħ +rocess ing +_st andard +- option +')) ) +Ġvac ant +å· ¥ +ĠH ollow +handle Change +Ġdiv ider +ĠEngine ers +Ġsv ens +Ġcompl iant +t anggal +ĠC redits +ĠEm irates +Rule Context +Ġreal ization +Ġdistr acted +]+ = +Ġaug ment +ĠD w +ot p +or rent +Edit ar +.st ock +St udy +pe ctions +ĠGame Manager += cut +Ġf lock +ĠRom ans +th em +-h op +Ġscreens hots +Ġ/* !Ċ +Ġconvers ions +Ġnormal ization +(config uration +Ġa eros +_se curity +! 'Ċ +B onus +ĠDR IVER +ĉ Date +t ie +ĠWy oming +St and +it re +Ġsh oppers +Ġdisadv antage +Ġlik ing +ç¬ ij +Ġunderstand able +SE E +Ġh oy +Ġnin ete +Ġcon fer +Ġnow rap +ĠV ern +, čĊčĊ +imest ep +Layout Manager +à · +ĉw ait +PLE TED +J apan +Ġindu ce +Ġå ¯ +оз в +_END POINT +.h orizontal +Ġacceler ated +rim on +IV ES +Trans actions +Le an +ĠSO UR +wh ether +y g +Ġo id +ĠEntity Manager +OUN TRY +Ġfil a +OLUM NS +IN UE +ĠAn chor +TR AN +wo o +block quote +ĠN urse +ĠCar p +Ġrede em +. try +ĠJ P +Ġtimestamp s +Ġ?> ">< +ĠREM OVE +ĠStar bucks +Re ally +Ġflood ed +.C allback +Drop Down +ip ro +Ġt ended +l te +Ġproport ions +- te +ĠR ena +lic ate +for ces +.ex tra +.auth enticate +в од +¡ ° +Ġfor ControlEvents +Ġsen ha +Ġke in +Ġmin ist +ĠPre ference +ĠTele graph +Ñĥ п +str pos +Ġillness es +Ġp igs +Ġget Intent +S ol +Ġ ¡ +(c pu +[ prop +s creens +'); ?> +ĠAct s +Ġstr dup +Ġaver ages +an al +ĠCas ual +Group Box +ĠHand book +/ comments +Ġnumber ed +Ġbroadcast ing +çĽ ij +.native Element +.m u +Ġupdated At +ĠDoes n +.A C +.c oll +Ġrec order +_sh a +B g +b il +Ġbol ts +Ġç ¬ +Ġim posing +ĠInformation en +_flash data +e conomic +Rem ark +uc as +ĠOff icers +ĠT ER +W alk +Ġmerc ado +_g enerate +H Y +Call ing +s nap +script Id +. operation +ĠFl ame +l iness +Ġrent ed +_t oggle +-ch anging +ĠT Y +' util +EE P +Ġgraph ql +ĠUn i +Ġimp ulse +.B asic +Ġenerg ies +M ARY +ĠMar cel +Ġmort al +Ġf res +m ens +m otion +Ġsample d +âĢľ That +id ay +qu ipment +get Int +ĠA bsolute +,' " +un ed +.sh are +Ġ} )( +mm m +ĠR ising +ä» » +Ġun employed +x fa +.f ollow +ĉĉĉĉ ĠĠĠĠĠĠ +sl t +.P hone +Ġkn ives +Ġe ve +on Click +] ))čĊ +ĠW itness +ĉ NS +ĠE OS +ĠSte fan +ĠPri est +âĢĶ which +Get String +. By +Ġup stairs +Ġdetr iment +bro ken +emb ro +Ġnic otine +il ion +Ġaston ishing +_ aff +ĠLess on +Ġaccident al +od or +Ġdec ir +Ġnew Name ++ . +çĽ ¸ +igs list +ĠG ithub +Ġsuccess ive +rac ial +Ġen viron +éªĮ è¯ģ +Ġredirect ed +T OTAL +Ġgrab bing +ĠL ance +Ġfor fe +_C B +å¾ ® +El apsed +_w ay +(Dialog Interface +_me asure +x bb +D og +Dep art +-s rc +res olver +with standing +_sh ell +ĠLast Name +ĠAv iation +Ġbegin ner +("% . +(to ol +Ġн ов +: init +(A PI +ĠMorr ison +vt Color +Ġstap le +/ INFO +Ġsupern atural +Ġste ak +tim eline +zz le +" `ĊĊ +Second ary +ĠNep al +.String Utils +Ġad am +Ġ( ... +Ġsub stitution +Ġboard ing +ĠKey word +ĠAss ault +dbc Template +Ġorder Id +( engine +.assert That +ĠVen us +Ġhomic ide +ĠA val +Ġg utter +ĠSupport ed +/p art +Ġac claimed +H istor +Ġmes es +ü ber +ĠRen ew +Ġgr as +ĠE k +Ġin file +ind y +.m usic +.S croll +ĠA ges +ĠNar uto +ĠG ather +Ġconfirm ing += (" +Ġpitch ed +ole y +Fr ance ++' " +$ total +Ġon de +Ġd itch +_s igma +Ġcontinu ity +re ward +- load +Ġproces o +Lock ed +st aw +Ġsp inal +l azy +! == +j est +Ġd un +ĠRod gers +ĉ grid +Ġlog os +ĠBeng al +.s uper +Provid es +Ġnut rient +.T imestamp +IZ ATION +åĨ Į +Ġf ats +ĠX xx +ct ica +Target s +Ġcont ours +Ġre ordered +: Array +Ġtoler ate +V ir +Ġter ribly +Ġbr icks +(& _ +h b +Port al +ĠB read +. which +ÂŃ t +as InstanceOf +Ġj object +ĉ length +_M T +; ">čĊ +_EX IST +Ġmat ernal +RE L +Ġê²½ ìļ° +he e +Ġlayout s +ĠL ap +ais y +Ġst umbled +ĠU IG +ĠS co +Ġimp aired +RES SED +Ġab uses +V F +AR B +.N AME +r ch +prim ir +_com pleted +Ġp enny +Ch rome +(b egin +ern en +- checkbox +Plain OldData +ĠL PC +r ade +sp ir +Ġcon ceived +T ips +ĠIo T +ĠG an +èģ Ķ +Ġbi ases +Ġconsult ants +ple d +_ ht +associ ated +], ĊĊ +Ġdelight ful +ĠÑĤ ек +Hel vetica +( load +-exp and +_W IDGET +to a +ĠA kt +Ġom n +Ġcl auses +Int el +*/ }Ċ +_reg istration +Ġold Value +Ġrest oring +Ġun real +O VER +ĉĊĉĊ ĉĊ +AT S +_pro be +Ġdiv isor +.update Dynamic +å¹ ³ +Produ ces +st amp +.j boss +ĉt ask +! (: +Ġpsych ic +@ class +M artin +ĠPass ed +clar ations +h el +а Ñĩ +ĉc opy +-b in +z an +ig ram +া ঠ+(s ig +ĠC aval +_ ## +Ġ% = +out lined +ĠAc id +Ġunpredict able +-d ashboard +Hex String ++ c +.P ublic +Ạ© +Ġconvey or +ĠE B +Ġselect s +Ġknock ing +ĠC ec +IBUT ES +owa Äĩ +g atsby +* v +ent ropy +Ġdispatch ed +Ġcam el +ĠSat urn +Ġover weight +( phone +par able +% B +_v ectors +Ġbrew ing +ĠT k +ĠDownload s +ĠS aved +.Pr ice +Ġcur ved +ĠParen thood +è ¶ +.p nl +plet ely +.D ay +Ġadvertis ers +Ġej ec +Ġpr zed +ë ¯ +! ';Ċ +ĠK ush +ĠT AB +Ġquest s +Ġcoinc idence +umm ies +ĠKash mir +ĠEth ics +_g rowth +Ġakt iv +Ġgroup ing +å¢ ŀ +_tr uth +åIJ ¬ +t odos +is et +Tex Coord +ä tt +ĠZ ur +ro ys +_M AGIC +Ġbrew ery +( State +ĠSM ALL +ĠPl ants +it bart +each er +ĠAd elaide +L u +Ġf ick +und les +_load ed +и е +P oll +rit ic +EL Y +Ġ+ ' +ĠProf ession +Ġst amps +ĠS ew +scroll View +Ġcomm unist +/pro blems +}čĊčĊ čĊčĊ +, o +Ġu dp +Ġob ese +appro ve +ancell ation +_G ame +ĠHas htable +adaptive Styles +Ġpossess es +.match er +function al +M rs +ĉs ave +ĠDb Type +Ġk en +get Context +Ġm ans +( rel +ĠBrother hood +) `Ċ +è§ £ +.In formation +OutOfRange Exception +ĠS ek +C as +Ġblog gers +E ither +(" "" +Ġpin ch +Ġco arse +) p +ĠP ulse +Ġlear nt +Ġdent ist +Ġon change +Ġdirect ives +( actions +ny der +ĠSh ir +T rait +_de p +ĠP ET +ĠRE P +.App Settings +cu ador +iden av +Ġenv i +Ġsl ammed +ĠSh oot +Ġdate Format +.j oda +ve ys +Ġ) .ĊĊ +Ġcare g +ĠPar allel +_ translation +.function s +. obs +Runtime Exception +[] = +over view +ĠSch l +Ġno isy +ĠOn PropertyChanged +S ending +Ġunf amiliar +U pon +ĠPrint s +.t yp +Ġflee ing +ĉm ove +( Un +Ġq r +× ľ +_b eta +Ġsk ies +ĉm e +W ND +Ġstick ers +bl as +Ġinsert s +Ġvers es +ĠD ew +Ġtang ible +Ġhe cho +P OL +Ġte ardown +om nia +IB E +.c over +_str ategy +^ - +set Position +u ale +S igned +Ġif ace +as eline +.set Time +ĠMin eral +ĠFight ing +sk ins +Ġdiscrim in +Ġdans k +ĠPr inceton +ac ist +Ġ( ));Ċ +tr acks +imon ial +ad ecimal +EP ROM +ugg le +.Not ification +$ mail +c antidad +ĠJ ung +Ġseek ers +Ġpl ausible +t ier +еР¶ +Ġr apper +ĠMan a +ĠHttp StatusCode +Ġburn t +los es +ĠF oto +ĠJson Object +Inst agram +Ġsys call +Ġreal ities +ĠMAT LAB +:^ {Ċ +TER M +ĠC bd +ĠPar agraph +Ġtrav és +Ġconstruct ing +Ġsw al +Ġp ige +LL LL +-ex isting +G ets +Ġmelt ed +Ġmitig ate +H en +Ġh m +im as +ĠA o +ĠP erez +ĠD AL +Ġëĭ ¤ +Ġdiv is +Storyboard Segue +ĠMod ify +ĠÃľ ber +_O VERRIDE +.p em +unt os +Ġespa ñ +Ġ{ ? +ĠP AY +_ip v +ĠF ury +__ .__ +el ow +-center ed +check s +_ Reg +-J avadoc +ĉ load +ĠLik ewise +ا Ùħ +UN E +.se m +x cb +ĠC ave +_s leep +Ġsil ently +ĠExt reme +.To Upper +ĉC HECK +Ġc ue +ĠQ ByteArray +Ġcorrupt ed +ĠD é +Ġimp ed +Get Name +Ġinaccur ate +Ġso ber +е е +Ġbar code +-- ){Ċ +ink i +Ġé p +Ġd ri +ĠAL T +>>>> >>>> +ont a +[ L +Ġinter es +ver ting +Ġdi agnostics +p dev +è © +ĠIntegr ated +). ' +_g c +$ text +.g ames +ĠT erra +' Re +.trans fer +_F IFO +get Model +Ġbl and +ĠCole man +Ġpr imes +Ġæ Ī +Ġcross es +n k +G ING +Ġ' ^ +ĠB lob +Ġinter course +ĠBl vd +Ġweigh s +_reg ular +ĠPer th +Ġsepar ating +Ġb illed +.tab Control +Ġpup pet +Ġutil ization +Ġâĸ ł +Ġsucc es +Ġl amps +_pro j +E ric +Ġren ovation +ĠFam ilies +ĠB its +part ials +-M en +s olution +Ġd warf +.IN TEGER +ĠLO CK +. ct +Ġexcer pt +ĠP ix +ĠFirst Name +ANT ED +ĠAd mir +-h elp +P rior +ĠAl ign +.IN STANCE +Line Edit +('/ : +Ġin et +od us +.p kl +ĠK Y +up ert +Ġn erves +_grad ient +} ',' +_un ref +Ġs aturated +ĠConn ected +ĠF N +EX IT +Ġtele port +Ġav ait +Page Route +Ġdivor ced +(l ang +f st +ĠT yr +Ġmess enger +if stream +X S +ĠBank ing +Ġinfect ious +ĠM ons +_LO OP +Ġzur ück +Ġobt ener +/re pos +V el +ac ro +Ġuser Repository +style Type +ĠS RC +VML INUX +rec ursive +/ bar +_ch ip +omin ated +ĠN it +âĢĶ to +ĠBudd h +ом еÑĢ +ĠM AG +ĠC HE +_d en +. raises +_de gree +Ġpump kin +_tem plates +_M EDIA +ĠTim eline +Ġb ots +Object Type +Ġbu ys +.post s +C AL +wait ing +ĠDani els +Ġd abei +ĠS igma +il or +ig el +, W +AD S +( panel +ì² ´ +it ating +.p alette +Ġmos quito +Ġt ego +(parse Int +Ġdes pués +p romise +Ġw ij +types cript +ĠT v +_IDENT IFIER +).ĊĊ Ċ +_fl at +its u +US R +ex perience +-f it +ph inx +_th resh +Ġide ally +ĠFre eman +, DB +_r w +çŃ ī +U b +_stat istics +=" ">< +Ġch ore +Ġy ork +inst alled +Add itionally +Ġp stmt +yl ko +:: Ċ +Fore st +Ġhead set +Ġgall on +ÑĢ ем +Ġwithdraw n +ĠC andidate +Ġmel ting +Ġfree zer +Ġh l +_HE LP +m ime +( /* +Ġth irst +$ return +member of +еР± +ĠHttp ServletRequest +( ob +_ Result +Ġassert ed +Ġfulfill ing +Ġstret ches +par ated +-f unded +Ġå Ľ +ing les +_c a +. condition +ĠDis plays +Ġor ang +ĠC RE +Ġgl Bind +ĠSelect or +/ type +ĠAlex a +ched ules +ĠPen insula +Ġpar ity +ĉ dest +ĠDo ors +čĊ ĉčĊ +_dim ension +Ġa load +.St oredProcedure +(p aren +ĠBur ke +') ]Ċ +- engine +Ġqu ir +ĠHy brid +ĠDo e +Ġout lines +ĠTrend s +_N V +per iments +ĠH in +? ', +ĉ Text +F UL +Ġsm ells +Ġs lick +Ġmis erable +ĠArray Adapter +Ġparam String +H om +_l iterals +us uarios +Ġprompt ing +_l azy +ĠActiv ation +_ oc +We ak +Ġan ecd +ĠU CLA += re +isse ment +ĠEsc orts +Ex cellent +ĠP ause +Ġre positories +T OR +ari ate +_is o +up dates +hal b +udi ante +ë¡ Ŀ +Ġna ive +ĠP eg +ĠL ounge +ARG IN +(b in +On ClickListener +ĠFA ILED +Ġl ite +Ġd zie +ĠL iteral +iv or +fc ntl +Ġe ats +Ġq ed +Un lock +rid ing +und ai += M +AT TER +Configure Await +ici as +ustom ed +Ġsuccess ion +end Time +ĠJ upiter +Ġjud ging +d ration +_d ocs +.m o +Ġeduc ators +ĠV ine +Con d +[ out +q b +\ Validator +Ġmean ings +Ġpresent ly +Ġdiv iding +otten ham +asc ular +Ġtrail ers +ĠC LOSE +ам и +âĢĻ ai +ĠG ain +w or +Ġpl anner +Ġdistrib uting +v at +month s +x label +H F +V iol +.BASE LINE +еÑĤ ÑģÑı +ĠR otate +Ġtx n +: bold +Ġb loss +Forg ery +( embed +Ġjak o +s printf +the ir +Ġexhib its +- static +he cy +get ActiveSheet +.c lients +ãģ į +_h ide +[ word +C b +add Item +ax e +_r adio +al ion +mod ifier +Ġsat uration +Ġden om +_p ixels +m ess +(f l +at if +Ġse cs +Ġpro stitution +Ġgrand children +Ġparad ise +ĠF eld +_B INARY +it ous +๠Ħ +Ġflash ing +-s ided +Ġcontrad iction +/* ĊĊ +y label +ĠT et +Ġadm ire +res o +Ġlet z +ĠSE ARCH +sl ots +ĠRew ards +ĠH og +ĠNS Data +st ash +F all +ĠA mer +Line arLayout +/ photos +Ġfe ather +Ġ| čĊ +Download s +.Start sWith +Ġ// # +ine Transform +Ġaff id +V tbl +ĠRog ue +scri bed +Ġfa uc +ĠMon roe +Ġdecl ares +mod ern +re on +ay be +P ASS +f ers +_MULT I +ĠMath ematics +Ġsud ah +_ATT ACH +Ġnumber With +ĠSol omon +j in +ograf ia +ö l +_d esign +cul ated +ĠL una +ies z +Ġ=> ' +Ġrevel ations +Al ong +( ed +ĠF ilename +Ġy label +Sec ure +Ġbus ca +agn osis +_RE CE +Ġoverl apping +Ext ent +Ġanticip ation +Check s +ĠALS O +or c +iling ual +it ational +Ġadv ancement +ou ro +ĠP redicate +å¾ Ĺ +er ia +ĠPier ce +or io +Ġmer its +Ġpe anut +.P ackage +ĠCon duct +_SENS OR +Ġbo iling +Ġin tra +ĠI GN +ĠF ur +.Ref resh +ĠRe ach +_dec oder +.Ex p +ĠÑĤ ак +p ill +, Q +ĠGr ill +Ġpop ping +.A g +Ġpro yecto +Ġmile age +Ġec ological +] ]);Ċ +ĠÂ Ń +sub plot +ac ad +ĠTry ing +rec ipes +$ criteria +ĠPers ian +-b ound +M ASK +ĠG esture +Ġk k +ĠP VC +Ġprohib ition +Ġcom ando +ĠLO OK +Sh opping +Ġdist ortion +< Boolean +.Get Length +um pt +\ Product +ell ery +Ġfire wall +form atted +.red is +Ġes a +ĠRh ode +S om +.n on +Ġ' ). +Ġget View +ạ n +pr us +Mat thew +Ġs ia +ĠF ors +G PU +ient ras +_IN ST +Ġol arak +Ġimport ing +T CP +/ ");Ċ +e ither +Ġfresh ly +c ascade +(char acter +ĠJe ep +ot ics +_ UTIL +.Xtra Printing +.first Child +ĠEx cell +Ġd vd +Ġt aller +Ġr as +yp ass +Ġassign s +Ġgri ev +-m ore +J D +ĠBurn s +' >čĊ +.D ependency +.Query String +.O wner +Ġexp iry +Th u +( Vec +Ġhazard ous +Ġr pm +AP ON +Ġadd Target +sv ille +p Net +ĠIm g +ĠTIM ER +.An imation +Ġbe k +Ġass ort +Ġle bih +Ġbody Parser +Ġvibr ating +ID L +Ġbutter knife +int ers +Ġpersu ade +ĠLGBT Q +è ĭ +.s oft +Ġbe ams +_s ur +.D ef +Ġl abs +ĉ plt +Ġsk ins +Ġtransf erring +Ġimag inary +_E nd +; background +Ġl aps +_COM MENT +(S DL +ond s +.Rec ord +ĠIm plements +_t icks +() ))ĊĊ +Ġa rose +] ? +ĠM p +ĠI Command +Ġsculpt ure +Ġcontract ed +< HTML +Ġcal end +at y +/ Sub +Ġkv inn +_ IGNORE +ĠSh ane +ML S +Ġstim ulate +Part ition +Ġm un +ó m +eral a +- account +.B inary +c é +Ġse ize +connection s +ĠĊ ĠĠĠĠĠĠĠĠĊ +ĠDi agnostic +V ISIBLE +ĠRun s +Ġimpress ions +s uite +ob le +~ - +ak ukan +< Person +ĠN os +ĠG ui +.wait For +RE SET +Ġpost pon +Dis cover +arr ison +sh aw +b lood +AJ OR +æĽ´ æĸ° +ĠM use +æĶ ¶ +Ġret aining +ot te +Ġmos que +ĠS ne +Ġstandard ized +Ġmain land +_th ree +unge ons +get Doctrine +Ġwh ale +Ġag g +ĠP orsche +now led +lat ent +ĠRel ation +Ġ// ' +Ġshut ting +ĠRem ix +_c ov +Ġs ailing +Ġv owed +Ġp ots +out u +Ġhair y +cast s +Rel oad +Ġre connect +ter a +.child Nodes +ĠR ack +Ġcurrent Index +Ġall en +Ġ çĶ¨æĪ· +ĠC ubs +[ X +_SE Q +_RE MOVE +.get Action +(/ ^ +err ar +Ġ ether +cur ve +Ġsl ap +Ġu om +O thers +Ġen gr +Dis position +Ġst aged +E ye +ĠA ux +auth enticate +Ġ$ ? +ĠAndre as +Ġset w +.A rt +Ġforecast s +Ġa unt +-m iddle +Ġmis d +des k +Ġescort e +ĠCas a +rop ical +Ġexem ple +plan et +(U INT +Ġwh ip +ĠPC B +clide an +=" \ +Ġox ide +Ġsucceed s +der ived +ĠEcon om +_co ordinates +ir as +D raft +Ġvisual ize +B rian +_ASS UME +ĠObject Id +Ġtrain ers +_FOR CE +Ġcon soles +- process +lic her +ĠSim mons +T aking +ĠCl aims +Ġdiffé rent +Activity Result +Ġsn s +éĢī æĭ +ĠCr us +Ġll am +r ab +ĠJo an +AA A +ĉf ilter +ish ops +get ting +à µ +Ġquant o +P ast +ov ich +Ġin justice +ĠF LOAT +Ġal right +\ DB +( GameObject +u ish +(b ot +Ġgall ons +ĠR é +ĠS aid +ĠSTDMETHOD CALLTYPE +ais ing +_process or +ell idos +ter dam +ĠBe am +Text Area +Ġret orno +.M ake +Ġ$ ("< +Ġlock down +Ġremed ies +Ġve el +x ee +do ctype +F il +ĠExp and +Ġemp loys +Ġsession Storage +Ph p +P ublish +Ġret al +f abs +ynam ics +Ġtoss ed +ĠnumberOfRows InSection +x path +\ modules +Ġdis astr +ĠM ULT +.M esh +-st age +Ġs df +it ung +ug es +Ġ?> ">' +kin son +Ġк ол +ogn itive +_ li +Ġim minent +Ġaff inity +.sign al +Ġnot ch +ĠSteel ers +max length +K K +ĠEug ene +_P WM +ro i +Ġâ Ĺı +ĠH amburg +.M ust +Ġax e +en ef +Ġamb itions +ĠSpec ies +ĠSt ress +Ġa while +Ġб Ñĥд +Ġwith stand +ĠDec oder +_in ventory +Ġ{ ččĊ +Ġt gt +Ġrail road +W ASHINGTON +Ġnegot iated +N ST +- phone +, U +Ġexerc ising +á» ¥ +_P IXEL +av ors +iter ated +Ġv ampire +ad al +In grese +Ġun g +ject ive +.c ells +Ġn ano +Ġmark down +_R ULE +(event s +Ġl uggage +MESS AGE +ig keit +$ count +Attribute Name +IG INAL +_E nt +ĠB F +ĠCOM MENT +_in i +ĠEurope ans +ĠB elle +åij ½ +) [' +åº Ķ +ĠUse ful +.re ference +() ", +_ grade +ĠK aw +Ġsent encing +Ġsocial ism +mon ster +_L AYER +Ġdee pest +w k +ĠNo ise +### ĊĊ +Ġpr éc +ot le +ÑĤ е +a uf +ib al +Ġcon quer +> Email +Ġamb ulance +O AD +Ġ(" % +ĠF I +.f ixture +Ġter se +ĠĠĠĠ ĉĉĉĉ +Ġsanct uary +ug i +ĠCom parator +Definition s +Ġast hma +Ġl act +Ġhard wood +.c lock +Ġattract ing +ĠM our +(d istance +ic its +Ġbon ne +ĠAC CESS +.Deserialize Object +ĠTyp ed +Ġje u +Ġapp Id +ĠCl ara +ĠH F +ĠRe ich +ipp les +//---------------------------------------------------------------- ---------------- +_del ivery +erial ization +Ġplaint iffs +Sc ient +sh opping +ĠD ummy +ĠW ald +Group Name +Ġins cription +el og +:::: :::: +_ ld +Back Pressed +.R aw +ĠOn Trigger +Ġmuse ums +ĠBe en +ĠAdvent ures +Ġsl ate +Ġlet t +Ġsu nd +ĠG in +ĠMechan ical +.s hip +App Component +Ġdest ined +Ġdw elling +Prof iler +Pre pare +ze ich +Ġsil icon +(h as +Ġ# % +VID EO +Ġcollabor ate +L in +Ġsc opes +( className +(s d +and in +.h am +Service Impl +-des cribed +Ġiron y +st ial +ĠHu awei +(re po +Ġunexpected ly +ĠK ai +.inst all +\x f +Ġexhib ited +_T CP +ĠO x +_CH O +Ġprostitu erte +Ġv ä +Ġsit o +Ġconstitu ents +ĠContin ued +ĠS AVE +r ss +/ message +ub es +Ġmisd emean +Ġtax ation +Ġstory line +h air +ĠFind s +S IG +ver ification +~ = +.h p +Iter able +Ñĭ е +ator i +Ġc tr +R x +_ );ĊĊ +d ag +.p in +Ġp seud +Ġinv o +ÑģÑĤ ÑĢ +_p ix +为 空 +Ġsw orn +âĢĶ or +_reg istry +Ġdis asters +ĠRO I +ĠâĢ ķ +akt u +fore st +be iten +âĢĶ I +ue va +eg t +Ġsp ikes +URE S +ĠRecomm ended +Ġexplo ited +ĠFreder ick +_COMP LETE +ĠDr ugs +!!!! !!!! +ĠR iv +ST OP +RO OM +ĠP ASSWORD +C ookies +.E l +á» Ń +ĠB ert +Ġhash ed +ic ester +Ġdecor ator +Ġquery String +: ;Ċ +Ġ" [" +oto pe +-A meric +ĠMatthew s +UR AL +âĢľ , +Sum mer +f os +_CONT AINER +_A CK +Ġfil tr +_dis p +_ Re +Ġfac ile +а ÑĪ +Ġìķ Ĭ +Ġe ben +Ġspr ink +ĠQ uint +> V +Ġhistor ians +our met +ĠMonitor ing +led ger +c ott +Ġw are +GG LE +c ars +ĠM EDIATEK +Ġvol upt +_ View +HE L +(c opy +(st ats +Ġchrom osome +ĠCurt is +- conf +( asset +Ġhv or +File System +< >();čĊ +oc oder +ĠC annon +) x +ĠSm ooth +ĠS AS +_ ce +ĉ prev +_m ovie +E c +_w all +< Button +ĠF AST +Ġon View +ul an +ĠS UPPORT +Ġgesch ichten +ĠS ons +Im m +$ IFn +Ġfair ness +Ġd pi +ats u +J osh +Equal ity +Ġ} ()Ċ +_ less +ĠR atio +ĠC ats +ĠS tern +Mon ster +Ġmer cury +ü hr +Ġplus ieurs +.des erialize +sc opy +.F alse +) animated +ĠExp erts +Ġ"") {Ċ +.W hen +see also +.un pack +LE M +.select All +Ġperception s +ud ing +ir ling +ĠPrint ing +gram s +ĠFile Stream +erv ille +il og +ic mp +_C ount +Ġlivest ock +- ca +doc uments +Ġpo les +ĉw ant +Ġflu ores +Ġstand point +ĠH uge +Ġradi ans +ĠUIB ar +EDI UM +ĠHistor ic +_h older +ĠMar ines +Ġt ä +.L ight +quir er +ason ry +div ider +ĠFl utter +_f b +restrict ed +ĠEvery body +N ão +Ġkn ot +ĠT witch +Ġhall way +(C ollider +Input Element +? )Ċ +/ off +/ ) +play ed +[ OF +Ġbat ting +_d l +Ġcom edian +Ġé v +ĠD EM +ĠEd en +: white +' ', +Con struction +acer b +Ġtask ed +.man age +Rel ationship +Ġph on +n z +_B GR +Validate AntiForgeryToken +_ air +âĢľ When +Ġgl fw +ĠCon versation +_T OTAL +, Z +Ġg raz +Ġiter able +ĠP ASS +Ġadvert ise +Ġmö glich +/ train +ĠVolk swagen +Ġcreep y +Ġ" )čĊ +QU ENCE +Ġalt ar +Ġed its +comp iled +aw ning +ĠD ungeon +Ġo sg +Navigation Bar +Ġtrend ing +ĠE co +ogg les +cd ot +| - +S ie +ec ret +ĠN egative +ĠL ing +ĠD IM +ĠC WE +ĠCar rier +Ġcar tridge +_us b += os +ĠJack ie +Ġo tras +Ġcommod ities +ĠP resentation +)&& ( +ĠMar tha +ĠCath olics +ĠM ond +об Ñĭ +_ absolute +Ġash amed +pons ors +t al +Ġsad ness +Ġpu ò +F ade +-pre view +ĠRequest s +ĠCal vin +h orn +Reuse Identifier +(pro vider +/app s +ime o +ĉ Class +S amsung +ĠW ORLD +Ġc innamon +dot env +ĠI User +ĠDE V +_C har +.ib atis +et i +/ me +s st +.s ym +ĠRug by +-m aster +aj ar +ĠY EAR +Ġo dp +ĠR oles +Ġbip artisan +ail le +Ġblock er +Ġgre ens +.SE CONDS +Ġbelie vers +ĠL ikes +F LOAT +Ġm ak +Ġg cc +âķIJ âķIJ +(" ~/ +SCRIPT OR +Ġton nes +ĠS ang +Ġtrans pose +enn ai +P red +Ġsoll te +.github usercontent +( print +ĠH ole +çľ ĭ +ad get +Ġprompt s +Ġgen etically +ĠH od +Ġvert ically +_control s +ÑģÑĤ ан +") {čĊ +$ title +Ġ} ),ĊĊ +Ġstate wide +ĠCor respond +ĠAt tr +it ant +Element Type +Ġout ward +Ġfam ilia +( article +Ġbl at +Âł Ċ +Ġgl Get +ĠRe ceiver +Ġ% - +ad am +W inner +Ġtail or +_p wd +ert en +St an +ĉ all +al ive +strt otime +� s +s essions +$ conn +ass ist +Ġchat ting +ĠM ant +Ġ% @ +Ġ"" );ĊĊ +Ġd gv +Ġíķ ¨ +.re peat +_M essage +Ġadvis ers +/ path +Ġk es +) } .ĊĊ +ogen esis +ĠOPTION S +upt ools +Ġmilit ant +Ġex ited +ig ar +ĠCOM M +ĠDis posable +ay cast +Ġrow span +Ġsyn thes +Ġsond ern +ĠĊ +ĠJ acket +R ATION +.getSelected Item +- init +ĠReg isters +_se p +ĠTool kit +.d ict +Ġx label +\ Table +t oc +_com bo +ĠComp act +Ġr ugged +à¥ĩ ठ+-man agement +')}} ">Ċ +ĠSt amp +ı l +ro x +Ġlandsc apes +_NOT E +mon ary +c ab +Ġmo et +x af +rc ode +- cli +_g ate +[ event +SP ORT +g ia +ĠS UPER +/ Login +_sh utdown +int errupt +Ġpret ending +Ġfr inge +ĠRed s +ĠC UDA +ĠUN IX +v it +Ġbr ig +dr v +ĠConn ector +There fore +Ġl ia +D etection +_ actor +Ġtemp file +Ġecc entric +- role +Ġpad x +d ent +West ern +Ġê ·¸ +ĠApplication Record +Ġcampaign ing +_run ner +ĠC ivic +ale igh +Ġdire kt +.s ul +ĠĠ ĉĉĉ +ant en +Ġiss uer +Ġassert ions +( orig +AT IO +Ġlean ed +ä s +.D TO +expl ode +.O bservable +Ġstagger ing +Ġkidn apped +Ġprogram mers +ĠInn ov +.param eter +Ġdom ination +Ġske ptic +Ġæĺ ¯ +Ġavoid s +.Ver ify +ub by +ĠAS N +Ġformat o +ĠBeat les +_b rand +Ġin set +y outu +Ġto c +-f inal +Show ing +ĠD oub +ĠM esa +Ad j +_m edium +Cre ates +(end point +ĉ UP +bb ie +Ġst alk +.datab ind +.S can +ag ents +$ , +ind ividual ++ )/ +ĉv m +(not ification +Ġin ex +ĠClass ification +ren o +Ġo lig +-r ated +Ġform ulation +', { +Ġa cept +_un pack +_C A +.P ow +ĉ im +Ġal uminium +AN O +Ġx n +Ġcó mo +ĠIng redient +Ġseiz ures +åħ ± +ific ador +Ġsigu iente +ĠIn fragistics +Ġduplic ated +ĠDe e +Ġn ø +ĠAC CEPT +(c rate +иÑĤ елÑĮ +- less +Ġinf inity +An alyzer +-D ay +rit t +(c in +ĠG y +Ġmulti plied +uch i +ĠBald win +/ ip +Ġshort cuts +.A DD +Ġvig or +_in struction +( ; +_ eta +è¿ ŀ +utor ials +Ġboost ing +b v +Ġacknowled ges +List ening +FA Q +; b +(( - +Ġarchitect s +Ġz we +Ġpul s +Ġget Count +ver bs +ãĢ ľ +(C ollection +k re +Ġjuris dictions +_b ridge +ĠCr ack +ĠDiff iculty +K O +Res ervation +_re quires +T our +ãģĹãģ Ł +.set Current +Ġk y +ĠAlb any +Ġè § +ll er +agn a +work ers +.bl ank +ĠPr ayer +M IC +Ġresil ience +Te X +ĠL anguages +st udy +ĉc urr +Ġenzym es +Sl ug +ĠíĮ Į +str al +Ġtum ors +Ġseg unda +=' { +in struction +ĠL isp +/ info +Ġ" {$ +,: ), +Ġg v +( ErrorMessage +Ġ' = +}- ${ +.Doc uments +" Well +Ġreminis cent +Ġg az +iro pr +eh r +Ġsup pressed +ers h +.scroll To +Ġcad ena +Ġgame State +ÃŃ m +( conv +ĠTom orrow +ĠC CT +M ongo +ul g +.C amera +.hand lers +m ph +Ġst k +Ġgen etics +AC ING +Tr ivia +ĠB am +(m arker +.St retch +ĠSun ni +ĠBet ty +.t olist +un likely +.Rect angle +ob solete +IL ON +inner Text +emb ourg +a N +ĠV ehicles +un lock +: utf +n ob +ĠSee ing +ĠNE VER +Ġt ls +Ġfil les +Ġbenef ited +ĠCl int +*/ ), +.f old +Ġpos ible +A DED +th ouse +.D AL +ĠO dd +ro kes +ĠSun ny +ĠPartial Eq +_B uffer +ĠLe vi +long rightarrow +eld on +g ages +_w arn +.Create Table +ĠD ip +_ questions +.log ic +Ġ# " +={() => +Ġt ep +Ġju icy +ì Ĥ¬ +en ko +ia lect +Ù ī +Ġon board +Ġæ ı +ĉ rt +_ UTF +ĠQ Action +âĢ ŀ +( Component +(a udio +.h it +g te +Ġprogram med +state Params +Ġpoly ester +f ires +by ss +] =( +_ quality +Of Day +ĠFair y +Ġy elled +op l +(user Name +ĠD ifference +Ġevalu ations +iff any +Ġcycl ists +Ġc idade +Ġtext book +Ġprof iling +__ ), +de a +. activate +Ġindic ations +Ð ķ +Touch UpInside +Ġinval uable +ĠM ASK +Ġcont end +F req +Ġrecru its +(int erval +ĠUser Profile +Ġ'./ ../ +ed u +_C allback +Ġanal ogy +ĠTro phy +app hire +V ideos +ĠCh er +ĠH av +âĢ¦ " +. validator +g fx +ĠU Object +class names +tri angle +ĠEnc oder +.s py +Ġpred ators += status +-s afe +: ",Ċ +ĠIn cluding +Ġ{} ;čĊ +* cos +Ġend ured +.sul ake +Ġnurs ery +Ġfrag rance +Ġre building +Ġn th +ĠFr aser +.set Date +ĠV ince +_RE ST +Ġvent ilation +æµ · +cri bes +.as m +lp Vtbl +ĠA be +uis ine +, array +ĉ className +err als +Ġ' ĊĊ +Check out +Ġsol icit +A ux +_c apture +Ġrib s +rag on +vi ol +top ics +Function Flags +ĠM arty +b ike +ĠT ucker +(k ernel +ĠO ps +Close Operation +/d emo +ild a +ĠlÃŃ nea +APP ING +Ġsu ites +.visit VarInsn +ur us +ĠMin ute +(m anager +Ġbutter fly +Ġap are +Ġw olves +J WT +ĠSal on +ĉd elay +-es lint +is ations +.r pc +)| ( +ĠSnap chat +/m m +M N +cer ies +.text Alignment +ĠFrank furt +Ġad o +(new Value +( access +( Expression +ĠSign In +ĠHait i +_t p +.set Parameter +Min ute +Ġmanual s +ric anes +ĠP TR +ĠOut er +Ġget line +oc ations +_C D +ĠLy on +/g ui +_l ive +id an +.ge om +Ġborder Bottom +im uth +_check point +Ġme u +ĠIr ving +Ġpeu vent +(M AX +ĠAR CH +Ġp ov +.source forge +Ġjam ais +Ġar k +ĠBaghd ad +ĠC LEAR +Menu Bar +Ġtro is +CHED ULE +Ġ# čĊ +(C all +$ order +(M aterial +Ġencontr ado +$ list +ĠMETHOD S +.begin Transaction +_M AG +Style Sheet +Ġmaj ors +Ġindef initely +clean up +Ġhom eland +(d to +D ates +P resentation +ĠD K +={` / +ĉ Key +( Block +_check box +ne eds +Ġon Complete +ric o +Ġgle ich +Ġx m +O OD +B etter +ĠSQL ITE +. Book +x ad +ĠG one +ĉd p +Ġdev otion +Ġst m +Ġobs ess +ĠBack end +Qu eries +I k +// **************************************************************** +Ġdivid ends +.parent Element +} ")ĊĊ +ĠMaterial PageRoute +: num +Ġexp lic +ĠO L +le ast +O ops +iment os +Ġins urers +Ġhero ic +ĉf ields +.img ur +.btn Cancel +ĠDetect ive +(s m +ĠMutable LiveData +.l ab +(( [ +Ġha irst +ĠTrans actions +å¼Ģ å§ĭ +Ġstd Class +uent o +G IS +_c od +Instruction s +C alls +Pointer Type +ĠR w +Ġassort ment +ĠD IG ++ r +_C ERT +Ġinst ability +Ġv ib +on as +Ġro ku +ap ellido +Ġan gl +prene ur +Ġfluid s +ise ase +Ġde ed +qu ist +_CONST ANT +Ġequ ilibrium +_de legate +ĠQuant um +re i +Cap abilities +rect angle +? >< +al ien +ĠJ ug +D NA +T ickets +Occ urs +ĠHaw k +.setHorizontal Group +\ Collection +ff iti +Ġre arr +.setVertical Group +Ġc avity +Ġadult e +Fac ade +- wh +ĠL OL +Ø ° +Ġgrand parents +Sw ift +ĉw x +æīĢ æľī +if en +ff set +B eyond +// }ĊĊ +Ġw ager +Ġb ury +Ġcomm ence +reg istro +sc ient +ĠPer cent +Ġд олж +( identifier +.set Model +Ġs eldom +nt on +Ġappl iance +am us +rys ler +Ġpant ies +engu ins +Ġmim ic +Ġon Changed +Ġal coholic +.reload Data +Ch arge +ĠF ax +Ġj ScrollPane +Emp resa +Ġsh attered +x ba +Font s +? s +Ġpost season +ret ain +_r ates +Ġrequest Code +.t odo +´ s +CH K +ĠKeep ing +enge ance +Ġvs code +IPP ING +Default CloseOperation +_ raise +ĠO culus +ogram s +ra j +pc i +Ġcorros ion +.handle Submit +Access ible +ĠP iano +l ittle +AC L +Äĩ e +.un wrap +ĠCon vers +ĠLe ben +ione er +ĠMer chant +ĠJ orge +Ġembr acing +Ġvent a +á st +Ġvi ene +< QString +Ġexplos ions +Ġdistur bed +." < +m emo +ĠAb original +Ġcomple to +Tex Parameter +Ġuom ini +( agent +Ñĥ ÑĢ +ĠWh olesale +/ am +ĠBook mark +dr agon +Ġglo ve +Ġ" "));Ċ +iv ariate +now rap +In Children +.B r +Ġcon exion +Ġback bone +Ġe clipse +Ġpersec ution +': ĊĊ +/ link +ĠP ero +and as +ĠT ek +. "); +-an alysis +Ġer ad +Mar shal +Ġanch ors +og er +Ġconver gence +st icky +Ġnave g +int ern +_DE SCRIPTOR +ĠConsult ant +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ĠA uch +Ġer re +ÅĽ li +ĠHor izon +col a +Install ation +hot mail +C NN +.C ollectors +ch s +(tr ace +ĠEnc rypt +Ġ---- -- +ĠBase Controller +Ġag ua +Ġre active +id l +Ġclass Names +ĉ Session +ĠDod gers +H ad +_l v +Is Valid +ĠHEL P +ut to +ĠVer ification +Ġget env +_p a +.b mp +: f +ĠLou ise +(' ; +/ socket +Gr anted +.c alendar +( IP +ĠP X +.R oom +Ġprogram m +ens i +Ġtablesp oons +Ġle ve +Ġmo str +.t ipo +/ an +(d i +Ġb iod +Ġdb Context +ĠJS X +ĉ results +. END +ht e +l ify +P recision +èĬ Ĥ +ARS ER +)did ReceiveMemoryWarning +at tempt +IS P +& a +_P OP +ĠT ac +Ġprepared Statement +Ġзап иÑģ +Ġow ing +, start +Ġreview er +Ġr st +Ġprop Types +Ġrock y +_lo cale +ĠStrateg ies +ĠWe ber +.C ascade +_equal To +Ġcos as +ĠDe letes +ĠMax im +Ġsh rimp +re trieve +.In clude +IG IN +ĠO E +] );čĊčĊ +.en umer +Ġco ef +_N ull +R a +ty ard +ĠSh awn +keep ers +Ġq q +_s b +om ens +ĠExec utes +# " +TT Y +ĠValue Type +); */Ċ +ĠAbs olutely +ĠT ottenham +/ art +Ġbless ings +Ġswift ly +b uster +Ġa vid +COM M +, temp +Ġ} ?>Ċ +-g rowing +Ġdeep copy +A ck +egg ies +Ġ__ (" +Ġno ir +terror ism +Ġanth em +ag ency +_PACK AGE +ĠC losure +.reg istry +Ġmamm als +< L +U ICollectionView +ĠLED s +Ġvol ley +( Buffer +_N ATIVE +lib c +impl ode +Scroll Bar +ĠMar ion +.Con tracts +_A t +ĠWe instein +compare To +ĠH ose +en ity +.create Query +_r outer +Ġstim uli +Ġ++ ) +ĠCh amp +ĠBay ern +ass a +.v a +Ġdistrib utors +Ġfile private +Ġdepart ed +cc cc +@ click +ĠL unch +> L +Ġbl uetooth +.De ep +- standing +ác il +Ġro oft +ĠPath s +_iter ations +Invalid ArgumentException +.s pi +ĠUIAlert Action +uy e +sign in +.p riority +ĠEss ays +=' {$ +Ġè¿ ĶåĽŀ +_s igned +.p ersist +Ġred esign +To Lower +ĠNew man += start +ĠIsrael is +asis wa +Spe ech +Ġnum eros +hand lers +ĠW ong +Ġм еÑĤод +We ights +ĠGu jar +te il +ĠNon etheless +_E FFECT +Ġv ect +ĠO sc +Ġco ats +ĠW heat +Ġge ek +ĠPRO PERTY +w orm +_const ants +ĠB oulder +ĠP arm +co le +Ġdefault Center +ĠRou ge +: A +xc f +ĠVen ice +med ian +Ġred emption +F resh +Ġcos m +Ġfig ur +Ġref urb +CO PE +.c d +Ġch ords +ĠS gt +Å į +VP N +ĠS END +ain en +_account s +Ġtent h +Ġdiss olved +< App +ĠCover age +use State +é ro +.. < +Ġì £¼ +Ġdream ing +ĠFore cast +.C ursors +Ġvis as +/ script +_start ed +Ġga str +(P RO +]; // +.T ile +* sin +( Adapter +ĠSand ra +_S IG +ard ash +ĠO val +Ġdescri pcion +(s l +ĠDes criptor +Ġ` $ +/f ree +ĠKey words +Ġt udo +ion ale +(f ound +.x yz +ĠGeneration Type +_DISABLE D +( area +Ġel ites +Ġh ombre +(m essages +ĠR ac +Ġext ingu +ĠEst a +op o +. vel +mouse out +Ġconv olution +ĠHand ling +Ġceil ings +T ek +ĠAre as +.writer ow +< View +ĠCorn ell +_B IN +.in valid +'' 'čĊ +ie ż +_P osition +Ġk idding +PC ODE +Ġwatch er +lo x +Ġâ Ĺ +D ave +_all ow +Ġbis exual +Ġun ordered +ĠSch we +_se gments +Ġt earing +IN LINE +Ġund es +.g oods +.c am +ĠL W +ĉ where +Cal culator +-th reat +- alert +ĠSuz uki +ĠIP A +ĠAtt achment +AC CESS +(d type +O pp +_s ymbols +Ġdans ke +l age +or get +res olution +е Ñĩ +ĠQ Color +ĠBar rett +аÑĨи Ñı += \' +ĠNav Controller +/ ref +(c ountry +_H DR +Ġterse but +pet ition +Ġsu f +cred its +๠Į +x m +ĠDav ies +.re ddit +Ġw oven +ĠO bl +ĠK M +ĠConsider ing +ens ored +.per iod +Ġd dl +$ wp +Ġextrem ist +; \Ċ +Ġk im +al ers +Ġspan ning +Ġco herent +Ġconse gu +.text Label +.g eneral +_d ashboard +л ение +k ick +_P ID +ĠExt ensions +reg exp +ĠCl ause +_m ov +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠ +ĠR eward +ĠLEG O +A k +=-=- =-=- +ĉ parser +Ġon ze +éĢ Ģ +âĢĿ ãĢĤ +_b all +(r hs +Ġch orus +< count +as urable +Ġwirk lich +ĠEr in +ĠMS NBC +Ġet ter +ĠC ron +_F LOW +Ġ, čĊ +Ġcal idad +ĠFile Writer +ĉ stmt +( Byte +_p at +Ġte lescope +Ġgre ed +ĠT ort +(w rite +\ application +ĉRT LR +ĠConfiguration Manager +Un ix +End Time +In cludes +ĠHar vest +en berg +ĠAustral ians +Ġë ĵ +Ġr n +Ġreput able +Ġbl ending +UL ATION +ĠBrend an +d ad +Ġm ø +ĠW oo +_d c +U ne +Ġr ue +with in +ang ep +Ġp ouch +\" ", +ĠS ic +âĢĿ ), +aly ze +ĠG ef +c overs +Ġd bo +replace All +ĉ Logger +Try ing +[ state +-p iece +éĸ ĵ +beh avior +all ows +l rt +_p ython +ert ura +-c ountry +ĠT G +.UI Manager +b ens +ale x +ĠBre itbart +b ac +Ġpredict s +Ġg ab +Ġcard inal +.Time Unit +ĠVis itor +ĠM ing +Ġliv re +Ġparent Id +port un +Ġdimension al +ĠV est +en ic +à ³ +Ġ Ùĩ +ĠBL UE +Ġitem Count +Ġfe athers +ĉp stmt +ĠPol ar +{ // +und i +Ñĥ ж +z ar +Error Response +ì ĥģ +Rep resentation +* _ ++ ] +pre pend +Ġ' > +Ġlegitim acy +Ġo o +S linky +Ġnation als +. words +; p +tr ap +oman ip +Ġc ues +Ġgradu ating +Ġsem aphore +"] );ĊĊ +ace y +RE ET +Gr ab +ĠFel ix +( Id +_ne ighbors +Ġmeaning less +(d el +Ġj eder +ĠContent Values +.abs olute +/ cl +Ġx b +dat um +Ġtort ured +Ġrub bing +S cores +ĠðŁĺ ī +Ġav ons +Ġam sterdam +E OS +H al +Ġtrust worthy +# = +.EX TRA +Ġman o +is icing +-s upport +ĉc ursor +ĠSp o +aim assage +M ission +[] {" +Ġprint ers +G REEN +Ġt eg +Ġabdom inal +! ĊĊĊĊĊĊ +.Sh ort +аз в +ĠGift s +} ") +(b inding +x ce +âĢ ij +inf os +Form Data +Ġd art +Ġele ms +(in v +Y L +t in +GEN ER +á» ¯ +ĠT aken +uck le +: e +Ġspect ral +.b aidu +/ ');Ċ +Ġgre edy +es ion +,,,, ,,,, +Ġ/> ,Ċ +Internal ServerError +NSNotification Center +ĠA i +Ġsp it +Ġaug mented +Ġstandard UserDefaults +FIN ITY +R ace +: C +ĠRE CORD +ĠHigh light +Ġ' ` +Ġdef icits +Ġne i +Ġresearch ed +T a +Ġc opp +.Get HashCode +): čĊčĊ +On Click +ĠWell ington +Ġrev ival +æ¯ Ķ +éĹ ® +ĠN SS +Ġfor n +Ġint é +ĠKu wait +_fl ip +_ bo +_ \ +Ġocc urrences +ĠScient ists +S RC +og ens +igr ant +RE MOTE +ĠS ID +. opts +u ve +() ])Ċ +Ġlibert arian +ĠGl ide +les en +Ġform e +ow ania +Ġannoy ed +Def s +ĠExec utor +Ġcast s +.set Checked +ĠSh aring +.Serialize Object +Ġselect ors +_ OTHER +ë¯ ¸ +(s uper +( OS +_VER IFY +id unt +< header +Ġ/> ';Ċ +Ġvidé o +ĠNeg ro +ĠL ords +ĠT ours +Ġsoft ly +.re ceive +ĠE RC +Ġdata Set +Bad ge +ĉ Event +Ġper l +Ġ{} \ +(s entence +Or Update +Ġdim inish +P IN +(d raw +.To DateTime +.Equal To +(p in +-p encil +lu ent +ĠCall er +Ġplay ful +- '+ +x ca +sw ick +){ }Ċ +}: ${ +ĠM eth +.get Cell +.b reak +Ġy max +=' Ċ +ĠH iro +( TRUE +as urer +Ġcu er +U ber +. Operation +Ġol an +Ġthr illing +< Response +ĠF emin +Ġtravers al +Ġp oc +Ġset Status +decl ar +std afx +Ġaddict ive +ĠB tn +Ġexplos ives +ĠCook ing +ĠPl aint +Ġaccum ulator +ĠApp ointment +, password +ĠF AR +lu et +Further more +decl spec +_Static s +.D ictionary +"> '. +ĉ valid +" ", +In strument +> J +Ġno str +ĠR ift +_P ort +Ġvec es +[ [' +Ġrall ies +- series +Ġv v +. uc +Ġr tn +State Changed +( ins +ĠCl a +------------ Ċ +c us +ĠRel oad +//---------------------------------------------------------------- -------------------------------- +.se conds +_dest ination +Ġscrew ed +> c +Th ickness +Design er +Ġgr ids +n Äħ +( cookie +T rip +-M obile +Ġv oll +Ġgen ital +Ġconf isc +ĠConfeder ate +Ġweb View +Ġm ise +Ġcl er +(se lection +$ date +Ġshar pen +rag en +And Update +Ġrem ix +Ġh tons +R W +M PI +Ġretrie val +Ġric hest +.Dec ode +:init Components +ĠT Value +S aint +@ include +ĠPER SON +.se p +ĠLD AP +g ba +Ġgro ÃŁe +Ġreli ably +ĠD FS +.getItem Id +Ġprés ent +.get Token +Ġch inese +ĠMe al +Y OU +"> >ĊĊ +b ower +Ġsw apped +/ install +Ġs inks +etr ize +Ġdecl ines +ĉm ysql +ĠC String +ĠMotion Event +.L anguage +R oad +ÑĤ еÑĢ +asc imento +')) -> +. about +( editor +ĠR atings +in come +Å¡ e +.de queueReusableCell +ĠAust rian +Ġs ulla +ĠTrib unal +ĠDid n +ов аÑĢ +Ġins pections +B oss +Ġcock tails +Ġapolog ized +_sub plot +op al ++ =( +Ġreson ance +ib u +Ġë ¦¬ +rom a +res erve +pl s +ĠT ah +ax ies +OP LE +ĠDar ren +ĠZ ombie +_M ap +Ġ] )ĊĊ +ĠQ i +ĠS ail +Ġrestrict ive +Ġeros ion +- par +WH ITE +Ġold u +Ġap erture +Ġbit coins +text o +ĠCom cast +Ġtime less +en kins +Ġfeed er +/ tmp +res den ++' _ +.D estroy +Ġç ok +ĠD OCUMENT +.l ng +.tag Name +Ġk ullan +eg rate +Ġ(* . +ç¼ĸ è¾ij +Ġhand shake +s oc +_ geometry +ĠDam ascus +Min or +ĠK afka +ìĹ ¬ +Fl orida +_com pute +.ex pr +Ġpar alle +ĠD iaz +c ir +[ target +Ġj oking +Ġgl or +(set q +_hand lers +H ang +Ġf err +rim inal +ĉĠĠĠĠ ĉĉ +ent ies +def ines +-t ax +json p +ĠU PS +met ro +__ ;Ċ +ĠUg anda +])) :Ċ +_t d +x ae +l w +. OS +ĠLog ged +ac id +ĠMay o +as pect +Ġvag inal +Ġinitial izing +Ġster oids +f iction +G RE +g end +Ġli abilities +ĠL ets +M ech +( nc +( change +Ġconnect ors +: k +Ġt ast +! ");ĊĊ +th ings +ro phy +luet ooth +ĠSign Up +. ctrl +Ġthere in +ord a +. escape +ig ator +Ġpet rol +Ġspec imen +Ġdeb uted +- Pro +Ġcr ises +.add View +ëı Ļ +-d oor +Ġmon et +Ġmill is +Ġv ier +Internal Enumerator +Ġadmin s +ĠL air +z in +get Query +umb les +L IMIT +ĠV ig +_s ong +< Character +:: . +_h om +_b p +ĠSup ervisor +sub mission +ab ile +Ġno i +Or Create +Ġpe el +Ġon Start +Ġsent iments +veh icles +Ġclass rooms +Ġs zer +Ġb ending +Ġlong evity +Ġa cl +ĠAle ppo +ĠU M +ĠR icht +Ġmultip rocessing +DOM AIN +"," + +_Y EAR +Ġsc rape +Ġsol itary +Ġ"] ";Ċ +/ errors +ìŀ ¬ +ľ ëł¥ +b etter +ĉ number +ĠL F +ĠAc ross +Pub Med +\" " +ĠExcell ence +Ġus ando +ĠU IP +Activity Indicator +_V OID +Ġbre eds +ï½ ¥ +uest as +ĠTre asure +ustral ian +(f ace +ĠT ennis +ĉ Int +ĠHans en +ç µ +: I +Ġâľ Ķ +GR AY +O USE +Ġhe pat +ł í +A IR +ó ż +Ġque ued +vinc ia +ĠChrom ium +Ġcompet ence +ung al +ill i +Ġget By +ĠF inder +Ġincap able +Ġs add +Ġc ites +ĠChurch ill +S dk +More over +As pNet +( Float +$ password +ĠConn or +-s ession +_d m +* )) +Ġde utsch +ĠN X +Ġper ks +_S ORT +_TO OL +_V ISIBLE +.as p +æĪ ĸ +ĠBre ath +D etect +ĠD uel +.c mb +[ it +.Set Bool +Ġnarc iss +Ġab ide +Ġej emplo +ĠâĦ ķ +Ġm ornings +Ġcomput es +.s sl +j t +Ġmuch os +_S S +[ end +Ġbas in +Ġalgun os +ĠCroat ia +lin ewidth +(t ags +(h idden +ÃŃc io +Ġap ar +ĠÐ ¶ +ä¸ İ +. food +ĠR ural +Ġbread th +å½ ± +(s ess ++ ") +ĠP aste +Ġserv idor +ĠBit Set +ĠTr an +la us +v ette +ey es +ĠCL ICK +ĠV III +ĠTurn s +ĠLe Bron +ĠM uj +ĠD eg +ĠAdult s +_s uite +process able +ĠPH Y +g hest +.F ail +ĠSl ack +ce j +\ Carbon +Ġsuper star +Ġhold ings +( forms +Ġ'# ' +M ultip +("[ % +-s olid +/ url +-t ier +[ length +ĠStream Writer +ĠMarket place +get text +_T ICK +ĠFor ge +Ġblack jack +ĠDO ES +ĠM atters +w aves +Ġwhisper ed +Ġl ush +ìĺ ¤ +d igital +Ġwr ink +ĠH ogan +Ġrust ic +.Apply Resources +ĠHard y +os omes +A UT +.ST ATE +Ġnarr atives +ĉ store +b ib +ĉ Scanner +ĠC ody +\ Repositories +Ġre union +and um +âĢĻ h +Ġsn iff +NS Bundle +Ġcompreh end +_US AGE +_ occ +URRE NCY +J NI +Ġspecial izing +Ġvis ions +Ġdol ore +Ġv á +ĠChe vy +ĠSt yled +imp act +all en +Ġk art +ĠTable t +st uff +re esome +аÑĤ оÑĢ +//---------------------------------------------------------------- -----------Ċ +_Ad min +Ġcell phone +Ġaut oplay +Ġcamb io +Ġmar itime +_BO OT +- quarter +Ġlat ina +ĠAJ AX +e quiv +ĠFront ier +ĠX Y +} ]Ċ +ĠR ough +.pro to +Ġcorrect ness +Ġfac il +ĠRe ached +ãģĿ ãģ® +V IS +.p s +Ġstr ncpy +Ġdiff usion +.start Activity +�� � +Ġaccom p +AMES PACE +imon ials +ĠBl ast +aby rin +Ġd ome +Ġextr av +Ġy en +Ġcul inary +P RI +ĠComm unities +n id +_oper ations +.h s +ĠMil ton +Ġno ises +Autoresizing Mask +(c id +}ĊĊ ĊĊĊĊ +] },Ċ +ĠD etection +tab la +Ġlib erties +_D YNAMIC +w get +ĠT ür +ĠP ascal +Trans parent +Delay ed +] () +ĠHer bert +< ActionResult +ch allenge +Ġmush room +.insert Before +ĠR in +Ġhum our +Ġf ø +api Key +alloc ated +Ġconf ession +. ",čĊ +ĉassert That +ĠS ORT +ĠL ORD +Ġexport er +.set Level +p okemon +ash tra +Ġf é +ur ator +(M SG +Ġt up +ĠH ull +Ġyield ed +.Sub ject +\ Route +! ? +ĠÑĥ дал +\ Security +- ar +Ġalleg ation +( Settings +ä nder +Ġell ipse +ĠRetro fit +Ġregul ating +ĠM olly +ĠL ok +_C ustom +ĠProm o +is in +Ġres umed +Ġmet ropolitan +.error Message +: ------------- +Ġpas ado +th ank +_De lete +ĠBright on +, unsigned +ä½ľ èĢħ +Ġaspir ations +-h ow +R ose += (( +_ne eded +_pl ural +< Application +ĠW EEK +ĠUn lock +ĠT EMP +S ou +Ġschizophren ia +Ġt roll +Ġcomplement ary +ĠNET WORK +Ġbl ir +Ġprogress Dialog +" %( +ĠAttribute Set +ĉ ts +.iter items +è¯ Ŀ +Ġesc rit +v ous +_pl aces +H K +Ġseg uir +_f w +ĠR ounded +Ġdis posit +è§ Ĩ +par m +w ow +STRU CTION +. allow +ĠChar Sequence +ĉ extern +Ġprosec uted +Ġmort ar +ĠJ uda +- msg +Ġest ud +.get Description +Ġs ow +amb re +Ġrom a +En h +bon us +Ġsqu at +Ġdist ra +ed Image +Ġpe ppers +-per formance +, ĊĊĊ +, file +ĠM IME +_con cat +AB S +-f ashion +Ġunder cover +One ToMany +Ġre claim +C OPY +Ġb inds +ĠT ape +Ġg ossip +ĠEqu ity +/ Card +. activ +' am +Ġdrain age +< Scalars +ĠonBind ViewHolder +() ?. +Ġs orrow +ĠI b +up y +_U UID +ĠCh arm +ĠElection s +.on Destroy +ĠInterest ingly +ounding Box +_d etection +-h eld +_ unknown +Ġrefr ain +Ġmét odo +Ġe Book +EN OMEM +Ġd ang +Prof essional +Ġd ictionaries +/m ysql +ĠST UD +Ġmas se +s cape +Ġdre i +: name +.log o +Sign Up +Ġt ahun +( theme +ĠFem me +Ġbom ber +ĠJ ade +ĠT ay +Ġsubmar ine +_cl ause +zy ch +Ġsimult aneous +Ġcas os +. boolean +(l hs +Ġcontin ental +-s ale +ĉ env +ĠC ute +ĠFactory Girl +ab us +/ value +Ġj adx +Ġst ern +> >ĊĊ +Ġsurf aced +Ġìł Ģìŀ¥ +pl atz +ĉ email +cept ors +"> ( +Ġep ile +è¯ » +ĠDe bt +åij Ĭ +N OP +" https +: j +Form Item +_L ICENSE +.get Double +ĠAg enda +ĉf inally +(f ilters +( av +ç¾ İ +AP ER +Ġl ava +еÑĢ ж +)) ))ĊĊ +Ġfault y +_n m +Ġtr ava +(B itmap +Ġspeed ing +> '). +Ġscreen ed +_ roll +ĠMac Book +ĠA UD +Ġdiagn ose +.G enerate +Ġ^ ^ +Ġstr s +[ Test +Ġr ansom +ĠDH CP +eld en +Ġinterpret ations +() ]. +flat Map +Ġline Height +_m ount +ĠW izards +Ġsl uts +eh ler +od al +Ġmilit ia +å ² +earn ed +Ġmis ery +int val +f und +Ġh ides +Ġdi arr +ĠWes ley +Ġx mm +Ġqu em +ĠAr abs +if th +ategor ized +Dis posable +P ure +_NOT IFY +sn ippet +ĠGar rett +.run ning +. weights +Ġ( -- +Ġin variant +äºĭ 件 +ĠAll owed +dir s +Ġpass ions +Ġl ad +ĠFl ush +men us +: block +Ġcompr a +.ch omp +alloc ator +Ġcur ated +ĠKnow ing +ĠPatt erson +Ġtel ah +' ex +Ġdo omed +Ġphil anth +ott y +.st yles +Own ed +Ġallerg ies += params +oc ese +it elist +ĠS ending +b ef +orr ar +ĠN ão +ĠF argo +ĠL ub +ĠComb ined +_g iven +ĉĉĉĉĉ ĠĠĠĠ +Ġreconc iliation +Pattern s +az ard +Ġbiom ass +ĠH ouses +resp uesta +cc o +/top ics +ĠY uk +Ġweaken ed +_c alendar +Ġmulher es +ĠMar l +Ġs ine +ĠT il +ĠSou ls +ĠDe utsche +ĠF OLLOW +Ġpip elines +ĠBever ly +_DIP SETTING +" # +ĠPro to +.b ig +ĠSav ings +ĠT anz +j un +ĠG amma +ĠS add +Ġadvis ors +Ġro ast +Ġun ters +ud ies +_l on +-point er +ĠElement Ref +\ Builder +example Input +.web driver +data Type +ĠQu ite +ĠCelt ics +u il +-def ense +b ish +ĠUI Window +ĠS uddenly +.h ot +.re ason +Ġg ör +AM D +.M ulti +auth enticated +reg ions +; ( +а ÑĢам +ĠKir by +$ route +PREC ATED +ĠDur ham +ow o +ĠPer forms +Ġdisreg ard +n st +ĠP ols +Ġget P +"] : +-col ored +( Keys +ĠAl leg +_mod ify +_ loading +str ained +Ġat roc +_p hr +< Sprite +Ġsatisf actory +m anship +.p ipeline +T ony +Ġth ief +pol ator +( lock +bur st +ĠOptim ization +Ġsurf ing +" Yes +Ġdesc ended +æ Ĵ +_C lear +Ġc ries +ĠFro zen +D IRECT +- Con +ĠLe icester +å¥ ³ +O OM += db +Ġget Message +< Student +_b atches +.M ask +_ eth +\ ) +Ġsom a +C atch +[ ch +Own ers +ind le +: auto +. vert +iv r +.set Location +Ġfl uent +_END IAN +ĠCar lo +cept s +add Action +.o auth +< UnityEngine +re ements +.S kip +? )ĊĊ +.default Props +Ġc abe +ĠSh en +eros is +ĠPro fit +Ġpo is +_C REATED +Ġremove From +(w s +? action +( Field +Ġerr one +.min imum +ĠRetrie ved +Ġd ado +ĠPR IVATE +-s pec +Ġg zip +p data +Ġpos Y +(l ow +Ġqual quer +/ cloud +ê² Į +( common +ĠAr beit +organ isation +Ġtid y +ĠRol and +( ph +.z one +Ġgent lemen +ượ c +å± ± +Ġenc losure +ĠMan afort +ĉ Color +St encil +N ic +Ġthe orem +ĠV G +Ġcol oured +V BoxLayout +uls ive +Drag on +c ff +et est +ens a +of day +.A zure +:UIControlEvent TouchUpInside +_up dates +Ġtrend y +ug as +weak Self +Ġr idge +ib ri +Ġì¶ Ķ +(C G +ĠMon key +.write Int +.tim edelta +ViewController Animated +ĠProvid ence +ãģ Ī +Ġbl ends +/Sub threshold +ĠAp pl +Ġat an +Ġreload Data +umb otron +st üt +O Auth +ĠG iving +ĠìĦ ¤ +ĠFinn ish +check ing +. Embed +sequ elize +Ġinitial izes +ĠOs lo +Ø ¶ +get Extension +_AL T +(bl ank +Ġfatal Error +Ġdem ise +**** *Ċ +ĠX S +(A F +ĠEn s +an tha +ĠP OR +Ġn ich +.N amed +Ġgig antic +ĠObserv atory +.Res olve +ĠPay ments +g uild +Ġcurrent State +============ ===Ċ +ĠS ey +p Data +Ġdead lines +Ġcentral ized +ĠScholar ship +_s upported +.ch rome +() ]);Ċ +Ġc yan +ĠC age +Auth ors +_ čĊ +/ os +k im +de e +.t ex +Ġyours elves +Ġm gr +Ġal k +-inst all +Ġdraft ing +Ġrum or +Ġstat ues +Pool ing +ol ina +AAAA AAAA +/* ---------------------------------------------------------------------------- +Ġextrem ists +Cal cul +ighth ouse +In set +(IN PUT +Ġsynchron ization +iv irus +. axes +ĠG ap +- An +_T emplate +Ġgam er +ĠCr icket +Ġl int +Ġauthor itarian +NS UInteger +Ġred o +Ġadip iscing +_F ETCH +che id +ĠF ang +. indices +t one +д ел +Ġ{{-- < +bra him +Ġsal a +get Code +Ġcommunic ated +start sWith +ert z +Read able +Item Id +oref errer +cred ible +á ria +Ġcombine Reducers +** /ĊĊ +Ġbl iss +Ġad orn +dep ends +ĠRO OM +Ġfr aming +Ġ? ', +aut y +_p ot +_t abs +Ex act +, ", +Ġ'} ';Ċ +Ġarbit r +ahr ain +.getString Extra +Ġ$ \ +Ġoutput Stream +Ġcomm enc +an us +ch y +< Employee +Ġhex atrigesimal +Ġn acional +(serial izers +_put char +_S AFE +ential Action +ItemSelected Listener +.Dis patch +Conf lict +_ about +os aur +Bound ary +Ġclear Color +( Location +ĠMON TH +ĠT aste +- General +ĠW AR +Ġer halten +-s aving +Ġcou pling +-tr igger +m otor +Ġy yyy +ĠPat ent +pt o +Ġmisdemean or +vas ion +ĠAdmir al +à¹ī า +_P WR +Ġdevast ated +fol ios +ITU DE +urre ct +Ġrobot ic +ĠSan ct +ĠHawai ian +.R oute +- condition +Ġr k +/**************************************************************************** Ċ +create Element +ĠK op +ign ant +. rollback +Ġsal ud +_ ', +ĠAN SI +Ex cept +ĠDraw able +.Utc Now +":[ {Ċ +Ġk ole +L ua +ĠBel ieve +Com put +Ġhall uc +ĠSign s +r st +.h u +ĠKN OW +W i +ĠBr ass +ĠR as +@ hotmail +Ġsed iment +Ġap k +Ġì ĥģ +_reg ions +Ġpod ium +< Book +ж е +Ġsix teen +ĠAli as +Ġinfr ared +ĠV ander +ĠLe ading +uc ing +,: ,: +_h or +w at +Ġdé cou +_W idget +S ounds +_n avigation +Ġschn ell +(g enerator +uc ene +Ġrem ake +IP v +Ġré al +_IN CREMENT +Ġhypoth etical +_ ang +Ġof s +Ġ! Ċ +.com pleted +Get Type +Ġkom men +ál ido +add On +Ġz ÅĤ +UL A +_ind icator +'] ĊĊĊ +ap ache +_S elect +ĠGre ene +Wh ats +_an im +Ġrepet itive +m uch +ĠTh reshold +Ġl f +(C ategory +con e +M ix +_MET ADATA +ays ia +Ne ighbors +ĉĊ ĉĉĊ +IP HER +ĠFr ag +ĠC ells +Ġnames paces +( back +ĠRest aurants +sv c +Ġл и +ote ch +-s l +¥ ¿ +ĠW T +ĠRed uction +Ġd otted +ĉf ound +ĠTE AM +B orn +ĠM ush +ĠCompar able +Ġh itch +AT O +Ġmax Height +begin Transaction +ÃŃ v +_b n +Ġher d +Ġrevers al +ĠH ond +del imiter +Ġconf use +Ġh ops +Ġcent roid +Ġcourt room +.decor ators +Ġm pi +ĠImpro ved +IN NER +ĠBang alore +ĠT amb +Ġbo ast +() ))čĊ +Ġil licit +ĠMor occo +greg ator +_res ume +Ġcrack down +Ġport raits +/h igh +( \' +Ġay ud +_fe edback +Ġc ate +/ avatar +Ġhe b +Point Cloud +Ġå ĴĮ +Ġ< ![ +Ġget Resources +} :{ +Oper ating +ĠF og +ĉt ab +ĠResearch ers +Ġfabric ation +.datas ets +ĠCamp o +ĠKa uf +Ġd ll +lig t +] ));ĊĊ +st ellen +ACK ET +l vl +ĠGl ory +.date Time +Ġcomm ute +ĠonCreate ViewHolder +ĠX Element +ĠT okens +< thead +_p ick +ì ¤ +v on +depart ure +(render er +phone Number +(P erson +gen es +ĠL ars +Ġ) {ĊĊ +ĠJson Result +Ġmet odo +VO KE +.get UserId +Acc eler +ĉ required +Ġchampionship s +Build Context +/t ask +/re leases +C ategoria +_over lay +Ġscar ce +_l im +n gr +ah len +ĠArt ificial +sp read +Ġbow ling +.an alysis +SM TP +ĉp assword +Ġbath s +] )){Ċ +current ly +ac iente +_se parator +Ġde ber +ĠDis abled +i ères +Ġâ ķ +_process ing +Ġprotest ing +ĠR OT +gr ab +Ġз ак +Ġpro active +word press +ĠSe ver +ind en +Ġw ikipedia +){ čĊčĊ +_w indows +is lation +Ġun rest +Ġdismiss al +.N UM +_F AST +iss ued +ĠF ACE +_u nder +Ġpl ugged +Ġå ° +ĠbÄĻd zie +ĠI CC +Ġcombust ion +Ġkiss ed +Ġstar red +ĠW atts +Ġspi elen +-p urpose +ĠE val +arg es +, result +techn ology +Ġnational ity +ic us +ĠN ug +ĠÑĤ о +ĉĉĉĉĉĉĉ ĠĠ +col o +Ġg astro +ante ed +OL ID +.b ias +_t ele +.ins pect +Ġve il +. footer +Ġneglig ence +Ġjud gments +Room s +yn n +ĉcount er +occup ation +Ġ çĶŁ +un as +Ġ(^ )( +L ambda +f el +.Param s +Ġд обав +set Layout +Ġdeport ation +Ġlocal Object +ĠPharm aceutical +cept ive +ĠN ome +Equ ipment +F an +Un iversal +ĉ socket +Ġgr in +Ġex poses +Ġhab er +Ġsincer ely +Ġc ams +Ġm ü +en ia +E mer +C rypto +Sl ow +(x hr +! =( +-s ervices +ĠP W +Ġprend re +Ġm ädchen +em ons +озв ÑĢаÑī +.M anager +ì Ļ +Ġg raf +- ra +met rical +/ fl +Ġc emetery +g ens +Ġp ÅĻ +ĠMySql Command +- To +Ġv Ã¥ +Ġa irst +oment um +Ġserv o +m illion +ĠMir anda +" She +Ġadvoc ating +-c aption +ĠAt tribution +Ġwel che +_v endor +ĉ Status +arr is +Ġprint k +"," # +Ġrel ativ +if ferences +izz es +Ġdec imals +ĠPro v +.max imum +Ar n +Ġhelicopt ers +_B OTTOM +ch ure +od ings +' ( +")) );čĊ +( bean +.f d +F und +Ġhang s +app id +/k ernel +.p oi +.Min Value +- validation +L uke +c df +ĠFun eral +ĠS amples +ĉ de +Ġto astr +Ġtax able +Ġcl ustering +Ġ'\ ' +Ġre straint +ec ed +ch ains +ãĢĤ ï¼Ī +_GR APH +Ġfue led +éľ Ģ +H p +å¤ į +T iles +Ġa unque +J C +Ġhost age +ĠE sk +Ġm av +Ġgest ion +Ġb anners +} {$ +.int Value +.' "ĊĊ +_M ATRIX +Ġce ased +ĠG OD +_CAM ERA +.Allow User +tr acked +C ook +b airro +( company +Ġview point +.get Writer +ĠN ets +w ives +Ġ( ))Ċ +example Modal +ĉ child +Ġmyth ology +Ġ// " +_ axes +ib old +.D ark +ĠMax well +Ġg pointer +olic itud +B at +ul ner +bal anced +mail er +Ġcont empor +æīĭ æľº +(" __ +Ġ" )" +re ar +ĠHu ang +] ')Ċ +× © +FT A +ĠCalling Convention +ĠOutput s +P k +.Re ference +lect ual +Ġ) :ĊĊ +Ġbrace let +ug er +ĉ Error +S weet +("/ ");Ċ +h x +Ġun reasonable +Inter preter +Ġlo ft +_product o +Ġsoci etal +.P arser +ĠAd apt +. foo +( where +.F eature +ĠYam aha +g lass +For ge +Ġprohib its +Ġcapac ities +Ġíķ¨ ìĪĺ +Ġper mutation +Ġih m +F ld +el ial +======== ===Ċ +@ Configuration +Ġge ared +ios o +iest a +trans lations +Input Change +Pop ular +ĠPL US +Ġv f +_F ree +b box +Ġcaus al +PI LE +Ġsch ö +Ġiron ic +M ir +. @ +åį Ĺ +Ġè ĩ +R ew +ul ence +fl en +Ġcan Activate +- response +Ġacc ents +ign ored +° F +.Dependency Injection +ĉ point +Ġconting ent +Ġsqu ash +Ġpar ms +ĠC emetery +Ġdelta Time +ĠD OS +Ġvan ished +аÑĢам еÑĤ +ĠD PS +t foot +ĠZ us +_IN STALL +G AN +Ġar b +Ġmunicipal ities +Into Constraints +AutoresizingMask IntoConstraints +, image +_ ignore +Ġdanger ously +quis a +pl uck +Ġhar us +up pe +Http Exception +Br acket +.' 'ĊĊ +ĠT ol +ĠView er +zb ollah +.Code Analysis +ì nh +Ġcorrect amente +.d a +ĠAl ger +× IJ +ba um +ĠPan ther +part icipant +å¿ ħ +-s up +Ġem ulator +Ġf ading +ĠW olver +cre ates +Ġbook ings +.Q uestion +§ è¡Į +Ġstress es +Ġre written +.PI PE +ed es +Ġc bd +": "/ +Ġenh ancements +_s y +B IN +ĠSl ip +Ins pect +ĠW eg +Ġcon gregation +Ġ_ : +_r m +Frame buffer +Ġ'& # +ĠFall out +Is Required +ĠPear son +ĠF ACT +Ġrel ie +ĉ box +ĠShe pherd +ĠWiki Leaks +ĠCollect or +Ġres ized +method Name +Ġevent Type +ĠA then +Des criptors +Ġb ers +- oper +ĠInitial ly +å ¡ +_B TN +ĠĠĠĠĠĠĠĠĠ čĊ +á b +_c ampaign +_w atch +F ord +-date picker +Ġvis c +Ġsat u +_s ms +Ġcont ador +-s vg +ĠDO I +$ args +Ġkn ob +.B OLD +Ġdeb ated +img s +sock opt +tr uth +ĠFe es +Ġh Wnd +_f ood +Ġab ras +Ġnot ions +ĠT od +: create +ĠConf lict +Us uarios +OT OS +Ġm sm +K HTML +([ ( +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠ +Ġ} ] +w izard +Ġm ientras +Ġdata List +Ġemerg es +Äĥ ng +.Read Int +PG A +ILL ISE +I Enumerator +(t uple +Christ mas +Look AndFeel +og enerated +Ġ# ĊĊ +control led +Ġex quisite +Ġa cest +Read Write +G ain +ãĢį ãĢĮ +Ġcopyright ed +Ġdo om +.Table LayoutPanel +ĠD ort +Ġch ili +Ġwer k +ĠEVENT S +ĠBe acon +Ġship ments +Ġse bagai +up on +ut om +.con verter +.Drop Table +={ }Ċ +f ic +~ ĊĊ +Ġlesb ians +_n a +Fore ign +ĉ then +/ ms +Ġor i +get Property +ĉsn printf +hes ion +ãģ ¤ +"} ," +Ġac rylic +P ers +@ Enable +I sl +(C ard +. Stack +L icensed +_G UID +: title +Ġh ust +Ġprincipal Table +an itize +/ embed +Ġens ured +ĠE GL +ÙĪ ر +ĠåĪ Ĩ +/ ,Ċ +Ġfundra iser +Key Name +Ġmarch ed +_VAL UES +ĠSc enario +Ġmet ic +_ass oci +ĠPast or +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉĉĉĉĉ +er ate +Ġinv itations +quo ise +Ġbl aming +Ġd aring +UM MY +Ġrich er +em aker +ĠIdent ification +ĠìĿ ¸ +ĠBinding Flags +ch as +Ġresil ient +_p g +Ġre leg +ĠI RA +ST E +Ġtr actor +- loading +ĠPre viously +ĠV acc +/ be +Ġn Ã¥r +Ġurl encode +ĠNor folk +.Re lease +ĠNe utral +ä¸Ń åĽ½ +ĠAr lington +Ġalleg es +ĠW riters +Test er +ĠR ally +Ġc á +ĉ Print +Ġâĩ Ĵ +ĠUser Controller +ĠSeek ing +.V AL +List Node +_ ff +ĠPhill ip +FA CT +Ġc aramel +ĠM ultip +ĠCom pared +ĠSer bia +Ł ³ +Ġrev ive +ĠK anye +Ġver ge +ĠBulg aria +get Body +Ġ| > +ce ph +.DateTime Picker +." ;ĊĊ +ĠT ie +, item +Ġm enn +G as +och a +_v irtual +Ġmaster piece +_se quences +L TE +ĠSub mission +Call er +$ \ +S port +ag us +Constraint Maker +Ġcol oc +Ġw ig +ĠÐ £ +ĉ Array +Look s +ĠGT A +.st eps +atch ewan +_r anges +ext Alignment +ĠBren nan +Ġab straction +uler Angles +.m isc +Ġantib odies +Ġexponent ial +ĠCH ANNEL +exp ense +' y +Ġdetect ives +Ġpur ported +Y STEM +Ġradio active +ĠLat ina +.Enc oding +.T AG +x in +D egree +ur acion +pr ices +ĠRefer entialAction +Ġr arity +Ġp iles +g ende +_project s +_g lobals +.start Time +Ġê µ¬ +SE CTION +_p ublish +F ault +DD L +_p rior +M om +Ġth icker +Ġsequ elize +Ġessential s +str as +in tr +>( () +.man agement +e il +éĹ Ń +A ware +.C ity +ĠAr bit +_D M +_key board +L Object +- webpack +ĠNew port +Ġprincipal Column +leg ant +Ġp allet +Ġfract ure +Ġg mail +.M eta +A bove +.Key Event +j it +_mac ro +_P USH +á» © +/ controller +åĬł è½½ +Ġsuperf icial +exter ity +Ġmens agem +W ind +ist on +.open api +и ÑĢов +ĠSerial izer +uct ive +Ġz ar +Pl aces +.St atic +B a +Ġin advert +ĠIndones ian +_IP V +(h orizontal +Ġget Title +ide press +ĠConsole Color +ip ers +$ out +Ġfest ive +Ġeven ings +.Get Data +uit ka +ĠManual s +uss ed +_M ax +.Ch at +ĠA ircraft += com +FO UND +ap ro +Ġtre asures +_al ive +Ġgad get +ek ing +Button Down +B rowsable +.PER MISSION +P ASSWORD +ĠH ASH +f é +\ TestCase +LO SS +o thers +, J +Ġassh ole +wer k +Ġm ã +. ie +ev il +kont akte +//////////////////////////////////////////////////////////////////////////////// Ċ += sys +ĉ lock +-- ;ĊĊ +_F UN +Fill Color +ó a +pre nd +Ġcompress or +M other +ĠAr cher +.g oto +Ġwür de +Ġbam boo +ï¼ İ +ĠT rees +Ġb umper +Ġsa usage +ĠEl asticsearch +Ġhor izontally +ĠG ul +Im mutable +Ġlos er +Ġabort ed +-d emo +ĠH atch +Ġund e +Ġprocess o +-c all +In come +å ĥ +_ returns +']." ' +(s w +C BS +am ilies +ĠYour self +ĠH olt +.M ON +ৠĩ +ÑĪ е +an on +ĠFont Awesome +produ cer +j r +Ġm au +ĉint er +Ġdish onest +Ġmagn a +ĠCollect ive +Ġvra iment +Ġcho ix +st ay +Ġweld ing +r ising +, min +ĠF ate +g lob +RGB A +Ġdet te +V en +Ġembarrass ment +.DE LETE +greg ar +-re nder +(b ucket +"> ĊĊĊ +.wait Key +Bus y +Ġdifferent iation +ĠC ST +.Con stant +Ġline Number +(m atches +Ġweb socket +Ġbar red +Ġpued es +M ono +C ORE +I ID +ĠĠĠĠ čĊčĊ +Ġpúb lico +lean ing +Ġcleans ing +Ġcr is +ĠDev ils +_SET TING +unt ary +. );Ċ +Ċ ĠĠĠĊ +[ curr +ts y +ĠAlex is +rit el +Ġpet roleum +.pre processing +m atter +For Result +- license +Ġtrav ellers +ĠDispatch er +enn ifer +Ġdigest ive +P ED +hib ition +MAS ConstraintMaker +ĠW att +Ben ef +.set View +d to +TE E +ĠPel osi +_EX TRA +Ġmed als +x hr +fore cast +Ġn argin +oun s +-f ill +_CUR SOR +Ġsuperv ised +Ġtur f +ĠEd gar +POS ITION +Ġcategory Id +â ī +_ ER +ủ a +Sh own +. ll +_POL ICY +(), ' +ĠPre v +ĠString Field +ĉG lobal +ass ed +Through out +o stringstream +.awt extra +Ġslo pes +ĠSe quential +Ġgi orn +Ġz elf +Ġvers atility +lene ck +.c gi +Ġdou bling +ĠBang kok +Ġbu urt +Ġusu ário +st udio +Ġje unes +Ġm uted +Ġ ips +_f raction +&& ( +Ġst unt +'); ?>čĊ +Ġev apor +b able +ĠPR ICE +Ġæ ³ +lu cent +Ġv amp +ĠTechn ician +Ġuniqu eness +M es +ur ban +.param etrize +ĠRe play +S essions +em br +-Americ ans +_PRO XY +Ġp ian +Ġtri e +ĠD estructor +Game State +ĠIM F +ch in +Ġport e +ĠSw al +åŁ İ +Sub string +im ing +/L ibrary +Ġfright ened +w rites +Ġrecurs os +ar Result +_INIT IALIZ +ĠBad ge +_c rc +E ight +ĠDIST INCT +Ġth ro +@ Xml +ĠLegend ary +-t witter +_e asy +Ġ+ ++ +(D ATA +.L ocale +Ġk ä +Ġn urt +Ġcr uis +_ ios +Ġsens ing +_L ine +Ċ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ +pon g +ole on +Ġwild card +çĶ¨æĪ· åIJį +Ġbeg ging +R od +ĠÃ İ +_C ELL +Research ers +. selector +_ ing +Ġaspir ing +Ġimm ortal +Ġy min +_ robot +Ġpl ur +B TC +ĠD ID +Ġpier cing +* u +_DEFIN ED +ĠTh i +ita ire +(m edia +- ons +Ġche fs +Ġ"* . +/ AP +Ġraz or +Ġsearch Data +Ġ= & +Ġ ãĢĤ +Ġm ourn +ting ham +Ġo li +ĠVern on +_R S +ŀ æĢ§ +Ġf ácil +ang en +cel ain +Ġa il +le st +ĠQ COMPARE +g ain +ĠÎ µ +ĠK ob +ĠF ault +_config s +ç»ĵ æŀľ +. + +cal ar +(color s +M ul +_ ART +Ġexperiment ing +erm en +ĠAng lo +.Fixed Single +Se a +Ġc txt +.s lider +C ollapse +G rey +Ġf ld +-pro of +.cap acity +get Parent +ĠCom pliance +Ġburg l +- rec +Ġover written +M U +Ġrout ers +ĉ Model +Ġfantas ies +av ian +_p rec +ĠSc andin +Ġ// < +/o ct +Ġceremon ies +Month s +und y +Ġqu ed +ĠN ou +ĠV ibr +.r gb +Ġcit rus +Ġbr aces +-upper case +get Table +Ġdop o +ĠK err +_CH ILD +- cloud +ĉ Matrix +Ġgard ening +S ing +al most +Require ments +ugu ay +( Property +sub scriber +FA ST +re action +(l p +) })Ċ +` ). +.w allet +_ex change +.Max imum +ĠVer b +âĶ ģ +() < +ï¼Ľ Ċ +RO T +C ARD +ub it +{ @ +_k el +ĠTool tip +My SQL +Main Activity +ar f +Ġm align +Ġse inen +ap ist +Ġ< % +Method Impl +M il +ĠM ick +.de pend +< ID +Ġpredict ive +ĠAP PLICATION +le f +dim ensions +Ġconoc er +/ conf +ĠTr acy +F oto +_rem aining += file +Ġpage Index +ĠPar ish +Ġt exas +ĠM AGIC +ĠH ew +d ifference +Ġalt ura +c um +ĉdata Type +Ġcaracter es +avi ours +ĠV OID +è¿ ij +P UBLIC +B io +ĠstringBy Appending +Parse Exception +ĠS uff +ĠN orton +/d etails +.n ull +>> & +ĉ ok +-l ow +. usuario +n ested +X B +OUR S +.Border Color +Ġb row +ĠÐ ķ +cor r +ĠRed skins +.get Tag +.get Transaction +Ġst igma +hard t +ĠPlayer Prefs +als y +uc son +L anguages +ĠOl ivia +Ġt ac +Ġb li +Ġc aval +Ġconsolid ated +Ġper il +Ġde le +Ġform ulated +Ġhigh ways +.sp awn +== $ +ĠN iet +Ġv eggies +yp o +-r ule +ĠV ie +/e pl +Ġenf ants +string Literal +Ġtou ghest +buy er +Ġcov ariance +Ġil i +ĠSoph ie +ĠB AB +Ġ" ), +ĠU k +current Index +_user data +.code c +ĠPun jab +ĠSN P +l ol +adv ance +Ġcom fy +Json Ignore +Ġfashion able +ĠI CON +Ġor a +ĠP ricing +< num +ĠI RC +ER V +ĠMe in +ĠID ictionary +AD OW +is New +ĠDev on +at l +(request Code +ĉ PreparedStatement +IM PORT +Ġmar ital +_SELECT ED +get Response +ar Down +B V +ib Name +ĠP ATCH +ä än +Ġda ar +ĠFile Mode +Ġm arty +.Spring Application +c ene +amp oline +get Size +Rest art +æķ Ī +.project s +ĠEthi opia +Ġstatus es +T ION +(b g +ĠX unit +Temp orary +ĠEng agement +Ġx f +Ġprox ies +Ġgen esis +Pager Adapter +ĠSl ave +Ġsung lasses +ĠCh loe +Ġko ji +ad em +ĉ JSONObject +Î ³ +Ġh ors +* w +ó r +es ch +Ġcritic ised +z ial +ĠSale m +.Vert ical +ĠR ash +> E +ter ing +/s creens +Ġheight ened +аÑĢ ÑĤ +Author ities +_b box +ün st +.font Size +ĠBO OLEAN +div ide +ĠSlo ven +uc er +Ù Ĵ +st ub +Ġnavig ating +: animated +_N OW +_v ect +} {Ċ +@ ( +Ġtele com +Ġcontract ing +ĠAss ange +Ġextract ing +Ġgr ö +c obra +.D IS +Ġcr ab +Ġtw itch +Ġvert s +Ġreject s +ĉ format +Ġreg eneration +.S ys +s olve +ĉd ialog +sh i +m eter +(b est +valid ators +Ġon wards +Ġg uru +Ġmoder ator +ow ied +ex periment +r ub +Ġm qtt +ĠCa ucas +Ġnational ism +Ġm ange +ĉ ImGui +/ Edit +Ġin h +Ġint ellig +ero kee +ĉ export +Ġdiscrim inate +sub tract +ĠM oodle +ens er +ĠGuid es +R AP +-h ot +_gr p +.p icture +X A +Ġinit View +_Com m +Ġoverd ose +Ġ+ ĊĊ +ĠSil ent +show s +Ġinterpol ate +Form ation +Ġb isc +mark ets +( SC +Z e +ĠNetwork ing +Ġad renal +ĠG uns +ete or +Decl ared +orget own +Ġk arena +/ password +_address es +ITER AL +B uzz +ĠCon way +(c ase +P WD +he iro +( act +** čĊ +());ĊĊ Ċ +Ġan v +Ġ. .ĊĊ +(Menu Item +(m ail +_section s +ĉ net +Ġpl ut +Ġw rench +/ object +ĠI st +ĠV IS +/p ub +al ten +Ġguit ars +Ġantibiot ic +ï¼ ĸ + ¹ +Ġ" +" +form ula +Ġbab es +ĠP rompt +Ġen im +/ player +ĉ ref +Ġby Äĩ +Ġconsum es +ĠH ast +ĠT ao +Ġ' ))Ċ +Ġcl am +Ġthigh s +Ġmot if +Api Operation +ĠW L +get C +ĉf lags +oint ments +Ġeconom ical +need le +x ls +pr actice +ut zer +time ofday +- output +Ġfind ById +ĠBudd y +Ðŀ ÑĤ +Se ven +ĠB ark +Ġenv oy +_al gorithm +åĪ © +Ġball istic +ç§ » +r ades +ĉd oc +rodu cing +ĠE ating +Un mount +/data Tables +_b onus +Ġl itt +pp s +) localObject +per f +ĠHel vetica +sh utdown +/ ml +.t okens +ĠHard core +, row +/b g +Sc aler +âĢĶ as +_log its +âĢĻ int +ĉ App +Imp licit +.F printf +ET O +Ġterr a +Ġpossess ing +.r strip +, ), += yes +ĠStr ipe +? = +ne utral +.g ood +Ġk ennen +ĠS ung +f ault +ystate change +Can adian +',' ".$ +ĠM its +æ nd +ĠSTR UCT +ĠURL WithString +ĠCom pass +Ġ-- ĊĊ +ĠNS LayoutConstraint +| min +-ad just +Ġreb uilt +L IGHT +/ se +-m ount +vp n +valid ated +(Q Object +Ġign ition +ĠCharg ers +RYPT O +]initWith Frame +ĠFl uid +Ġcad re +Ġnomin ations +Ne ill +ĠH ou +Ġcurrent s +_g ene +(in p +Par is +z ÄĻ +ag gregate +Ġass oc +weet ed +err at +âĢĵ ĊĊ +Ġ'/ ',Ċ +fix ture +ĠH ighest +amb ient +Ġch mod +Ġcon te +Ġsens ual +Ġgar ment +z ers +ĠPower ed +dom ains +R eward +i omanip +Ġcock pit +out file +Ġbuilt in +Ġins isting +. vars +zip code +Ġ ���� +f ails +Ġconsolid ation +_ oid +Plan et +Ġ= ", +ĉ el +UIL T +ät z +af ari +ĠMc Cl +Tim eline +Est a +Ġfr am +Y E +Ġcere bral +Of Month +ĠP regn +Ġкл аÑģÑģ +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ +ĠF res +Appro ved +.S pecial +ĠProtest ant +Ġallerg y +_p cm +ĉC opyright +Ġsuper Class +" strconv +ĠMoh amed +Ġ' // +Fore Color +Ar thur +ĠJ ungle +Ġve ins +S ad +Ġback ups +ĠOp inion +û t +Ġinter mitt +ody n +ĠChrist ina +Ġand re +Ġevac uation +pa lette +h orse +ĠRes ident +ĠHass an +.N il +Ġa isle +ĠG rowing +Ġblog info +/s ql +_io ctl +Sc aling +ĠMon ad +_c pp +ĠH utch +ĠApple WebKit +Exp ense +_J OB +Ġpoint less +From Body +ant al +Ġdepict ing +ĠC ELL +Ġref in +ĠC NC +ì¹ ĺ +_dim ensions +ĠS AN +Ġa ft +Ġfoot steps +cc oli +_PH ONE +/m ath +-k ind +ĠMe ans +ich ael +.g una +Ġinaug uration +-dr iving +( delete +Ġtotal Count +_M C +.Ext ension +Com mercial +Ġz Index +< Customer +" g +-sh are +Ġp act +ag ara +ĠS IL +_m odes +ĠM olecular +Ġsystem atically +< G +_s cr +ĠO ro +as ers +Ġb ic +Ġdest roys +PI PE +.Start Position +Ġc ủa +ire z +.B unifu +_F unction +Ġs ü +_f uture +ĠWe alth +ĠNatur ally +æĢ » +_y es +Ġabrupt ly +String Encoding +ĠCGPoint Make +Ġz h +Ġimp erson +Ġpiv otal +ĠSom alia +Ġsegment ation +_AN AL +ĠLogin Component +Cons ult +Ġtr uncated +] ";Ċ +.get Config +Ġintern ship +B aby +ê° ľ +Ġstrengthen ed +_M I +b asket +Ġnicht s +ĠTV s +ĠSh an +ãĤ µ +rac use +.Re LU +/ interfaces +ĠgetItem Count +Ġret iring +Ġspecial s +Ġentity Manager +bel ief +Ġs older +da ughter +ij kl +Ġutil izes +.f ixed +S U +Ġdr astic +Ġh acks +gr und +ĠM U +ĠSt arter +.Com ponents +_m otor +Gold en +Ġl odge +Ġ )); +ĠCor inth +иÑĩ еÑģÑĤво +ón ico +gre SQL +ĠFl uent +Ġmar c +.Load Scene +.Group s +Ġer h +ĠAut umn +St opped +Ġitalian o +Ġmin ions +ĠAssert ions +Ġm ux +B u +Ġ---------------------------------------------------------------- -------------------------------- +ĉ up +read ystatechange +_M eta +Ġcurrent Date +ĠChap man +Und o +Se an +ap r +Ġpar m +_ icons +ĠSt a +á z +Ġsub division +Ġalter ing +P NG +ponent ial +Ġpost gres +ĠB DS +-ex istent +ĠBrad ford +ĠO MX +_W HITE +_PRO GRAM +q c +Ġtypings Slinky +ĠP ics +_M ETA +IT TER +_sub scription +IRON MENT +ĠHy undai +();ĊĊ ĊĊ +ĠØ ³ +Ġj ac +Ġelimin ates +) });Ċ +Ġcomp rend +ĉ insert +_f aces +"> $ +Ġeb ay +Ġcapt ive +pl iant +ĠCalcul ates +ol ta +est ing +_re vision +Ġm ús ++ m +"," "," +WH AT +Ġcompassion ate +h arga +[ random +Ġmod ulo +(s n +Ġoccup ations +//// Ċ +ĉ board +ĠB alk +wi Äħ +ĠW ifi +.Pro file +:m aj +ĉm at +LOCK S +(j Button +Ġ(' $ +M ur +æĮ ī +b ble +Ġf rog +-h ide +Ġbroad caster +ภŀ +ha led +Ġam using +_predict ions +_in tr +Ġe agle +аÑĤ елÑĮ +Ġget List +ps ilon +Ġcharacter ization +AR DS +Ġre location +Ġr ulers +P AY +ĠDef initely +_A ction +Ġclos ures +Ġfact ual +odyn amic +Ġpreca utions +nie j +ĠPart ies +ĠSub aru +Ġcous ins +ar beit +.m oney +gun ta +( and +get item +.Style Priority +Ġsl id +single ton +Ġg arn +ĠP AS +Ġd azz +a ż +Ġbog us +ĠM og +Ġrival ry +is ol +Ġland marks +ñ as +B ern +ĠSach s +Ġ" )ĊĊ +Ġhost ility +_m ex +m ere +M ot +p ictureBox +Def ense +Ġaffid avit +other wise +.d irectory +_ UnityEngine +-b log +.s kin +ph em +Ap ellido +er chant +[ class +Ġw art +." [ +ale ur +/ back +ĠĠĠĠ ĉĠĠĠ +Ġprecip itation +Ġob struction +Ġp Obj +Ġr upt +UCK ET +ay e +æİ Ĵ +g x +Ġe cl +Ġsecre cy +/ Header +ĠLes b +Ġle i +ĠBullet in +Ġgive away +.H ome +_RO OM +" W +Ġcow ork +_ ra +ĠC ycling +ĠP aw +Ġpup il +/ arch +ĠFile Utils +é¦ ĸ +r sp +Ġfreed oms +ĠL ear +}` ). +Ġbow ls +/b lock +_log ging +Ġmeth ane +Ġhorn s +Ġwonder fully +Ġalter ations +Ġex ile +ls en +_p ause +_L ANGUAGE +ĠUS DA +_m ysql +_AM OUNT +ĠL IFE +Ġyoung sters +Ġri ots +[ E +Ġun forgettable +, },Ċ +Dis posed +ĠAss assin +UN G +ĠNew sp +User Service +: aload ++ ', +Ġsett lers +Ġscre ams +Ġincon venience +.R otate +Ġj ars +ĠP uzzle +Ġm est +ars i +ĠSh arma +| ( +.d s +ĠSac red +_e vt +Ġexpress es +Ġh och +ĠD uch +.c alls +th r +ĠShe ffield +.Alert Dialog +Ġrad ically +Ġtr ous +Ġprev ailing +ĠWW II +âĢĻ n +ens ely +ĠY esterday +ĠSir ius +Ġkill ers +ĠF FT +Ġo val +') :čĊ +Ġìłķ ë³´ +our age +ĠCheck box +Work book +.def er +_f loor +Ġc ouncill +Ġnors ke +mo il +ore a +Ġmarket ed +_S UR +x AA +Ġst ained +e ut +ĠM eng +Ġi eee +. extern +eg ie +Ġr app +ĠPy ongyang +' class +M ob +Ġinitial Value +_w ave +Ġj ab +Ġmascul ine +Ġampl ifier +Ġt ty +Path Component +_ xt +ĠG FP +/ sec +ĉdis patch +mark down +ĠS chn +bo le +· · +mouse move +Ġerr Msg +Ġas ign +_m ono +To Selector +ĠZ u +(R ect +ĠError Code +lat in +ang ible +v tk +CG Size +P okemon +Ġclass mates +Ġattract s +ĠT atto +ult an +ol óg +Ġhalt ed +ठ¨ +ĠK art +Ġ ue +_Init Structure +Test Class +ĠAir bnb +_ ", +Ġchar coal +Ġip c +ĠSt retch +.g lide +lates AutoresizingMaskIntoConstraints +Ġpot ion +ITT LE +Ġcount ert +_h d +pre pared +Ad s +ĠV ampire +rob ots +.Create Index +Status Label +Ġt ucked +af ür +U t +Ġswe ater +_F N +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĉ +ata ka +Ġeyeb rows +ac oes +ud en +.LinearLayout Manager +Ġsw ay +Ġmult in +() )))Ċ +ĠNS UInteger +ĠMy Base +Part ner +uts chen +ĠC ater +.setBackground Color +Ġaccompl ishment +_pro blem +.d td +Ġpage Number +Ġj ackets +Ġcro pped +u els +ĠH ep +Ġc apped +* Math +_callback s +Ġpub b +ĠBrun swick +.res pond +[" _ +Ġbed ding +hyth m +O X +(s peed +Ġpestic ides +Ġ---- --- +.Bl ue +Ġnood les +ĠGo es +Ġs aver +o xy +_com pletion +ĠSw inger +Ġget Date +Ġmind ed +int egration +ĠLot us +(st op +(', ');Ċ +Ġflood s +ĠWork flow +Ġerupt ed +Mac ro +ĠSau ce +Ġevent Name +\ Input +Break ing +ĉ when +_p w +IND ER +ĠWell ness +Ġvox el +ĠM ell +ĠM EDIA +SE NS +ĠFund s +ĠM ild +< Array +- this +ump ed +/f w +ĠDb Context +W I +girl s +H OW +'); ?>Ċ +Ġtempt ing +Ġtest ament +Ġb ible +Ġconsult ed +ĠIndex Error +è¨ ĺ +Ġkey pad +izz o +( ok +Ġwhats app +ĠRemote Exception +Ġteam ed +âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ +» , +Ġget Time +di ag +iss y +Ġh ed +Ġkn ots +j om +Ġfun nel +-m ails +Ġexport ing +ĠV L +ĠK arn +ĠBuddh ism +ĠAll an +_R ADIUS +Ġw ording +ĠFor get +ĠCor ona +ip hy +Ġlim burg +ugg y +ĠUser Repository +im in +(e le +Ġlabel led +ç¤ ¾ +ĠH erman +.q q +Ġ" ));Ċ +ie ber +.Trans late +ry n +Ġdes env +um d +Sim ply +ĉm ode +R pc +ĠVal encia +Ġstaff ers +Ġsel v +ĠSpi ke +Ġdel ic +Ġer u +_D T +J udge +á» ķ +ĠBas in +.m utable +" url +Ġtar iff +ĠSlee ve +Ġfl are +.drop out +Ġbr ides +)) ,čĊ +_con straints +de struct +Out line +Ġdisappe ars +_lock ed +ĠNS LocalizedString +ck e +ĉ null +ad resse +Ġto pping +ĠJ oker +b ishop +но ÑģÑĤÑĮ +and ering +_ amp += time +_S pace +_P ULL +' = +Ġant iqu +Ġc ach +___ ĊĊ +ON ES +о Ñı +Ġun read +.p olicy +oooo oooo +ëŁ ¬ +Ġu sted +ĠRe ce +Ġal lem +ãĥ¼ ãĤ¹ +ĠThought s +ve illance +istr ate +_l ane +Ġfam ed +.Get Name +Ġsmo other +ĠQual ified +az ers +_ geo +F ax +ĠM inds +ĠR aises +Ġtrans cripts +Con versation +Ġremark ed +ëĤ ĺ +d ling +Ġdeploy ing +Ġshared Application +Ġk p +FontAwesome Icon +_d ummy +reib en +ĠJane iro +Direction s +.get Bean +s ass +Ġcommand ers +v ation +error Code +ĠAl loy +.local ized +Ð ij +Ġdish washer +ĠSou p +N u +_D efault +Ġune ven +Ġ/> ";Ċ +-B ased +Ġseam lessly +- null +ĠX C +Ġst ew +(d elay +AT ORS +ĠWhe eler +" H +e ast +. air +âĢľ But +Object Context +success fully +_l and +Ġfold s +_CO ORD +Ġsub po +.get Address +in str +Material s +Ñĥ ÑģÑĤ +de posit +-l ast +_GR AY += find +Ġmut ant +Ġlesb ienne +let cher +RO UGH +ure ka +.c apture +Ġen n +Ġ([ [ +ĠFl u +Ġtask Id +ĠHus sein +.f older +Ġa usterity +ISTR ATION +_ Impl +注 æĦı +Ġdec ree +- chat +Ġimp lication +Ġguess es +ul kan +An alytics +. plus +COM MAND +е ли +» ĊĊ +_S ITE +Ġequal To +Support FragmentManager +ĠRec ording +å®Į æĪIJ +Ġbag gage +Ġpitch ers +ĠE h +o que +ĉc nt +Ġ=> $ +/ foo +IR A +ĠSat ellite +bor ah +Ġ}} "Ċ +ĠEnd s +ĠSpr ay +, param +.Ch rome +* q +th ought +ibr ated +Ġth ieves +Ġbenefici aries +Enter ed +ottes ville +Ġveter in +By ID +qu ipe +um ption +- unit +Execution Context +@ s +ĠG iov +.Tool Tip +_f riend +( attributes +Ġdump ing +ĠJ C +_D OCUMENT +ĠArm our +( insert +.Horizontal Alignment +ĠQ ed +ãģĦ ãģ¾ãģĻ +/g it +ĠY YYY +ĠCard iff +Ġap a +organ ic +ĠWhere as +Ġæ Ŀ +ĠM ia +Ġdemol ition +Ġsc ars +Ġp ai +Ġre tries +Ġr q +ĠDen is +( Utils +Ġallev iate +ĠP IC +id ue +Ġacknowled ging +Ġ// //////////////////////////////// +ç¡® å®ļ +Ä « +\ Json +.b inary +Ġx type +sign als +ĠAp pearance +& r +} s +C i +ĠI llum +por ate +h og +Ġindex Of +\ Command +_par allel +ĠSher lock +í ĥ +Ġ" ")čĊ +//////////////////////////////////////////////////////////////// //////////////////////////////// +Ġcritic ize +ĠSo ap +ĠMatch er +Ġgr illed +* T +Ġad ore +ull ing +Ġjed och +_ref s +lean up +ĠJ AXB +Ġro ses +ĠL iam +size i +Ġget char +Ġtar de +-to oltip +Ġqual ifier +ĠInter mediate +_W indow +ĠMal ta +Dis connect +ew here +Camp o +Ġirr ational +led o +ĠD N +ARG V +Ġout ro +Ġth irteen +Jose ph +M AR +/g l +J ess +ĠPsych iat +Ġpadding Bottom +- loop +/ fonts +_se en +Te ams +React DOM +(m an +(x path +.get SimpleName +>( * +ĠP vt +Ġel ders +Ġp ies +.user Agent +- region +ĠGree ks +(f ragment +st u +Ġcouncil s +Ġst amina +ĠGod dess +è ¥¿ +Ġphilosoph ers +Ġpers one +ĠL ose +ĠCL R +ĠD ocs +Ġso ak +ĠHOLD ER +Ġb ells +hash Code +R ATE +_WE IGHT +in ous +end ra +oph obic +Ġpro se +Ġfin ely +/o auth +(s pace +ad ge +ĠM ama +Ġstring Buffer +Ġst int +Ġmis ma +Ġvill ains +ĠCrime a +Ġdipl oma +Ġпо Ñģл +ĠBe a +(j oin +Ġíķ ´ +CH AT +per ing +ĠC ros +Ġmon keys +Ġpred s +yl a +,, , +Ġvibr ator +ĠN U +åħ Ī +f ant +z et +Ġb ietet +un ft +sw orth +.F low +Ġpsy ched +ĠContin ental +> t +Ġqu ilt +. UP +Ġexpans ive +Dis pose +(l anguage +C aps +_Z ONE +Ġrec ycle +ĠMan aged +current Color +.b roadcast +sign In +.p rom +ll u +ue blo +Ġpunch es +Ġautom at +Ġassign ing +Ġcreate User +ĠAll ied +Ġconduct or +Ĥ ¨ +Ġs addle +Ġd ni +omed ical +-W est +Positive Button +Ġit alic +? [ +(tr igger +Ġele phants +":" "," +Ġcal iber +raft ed +d igits +Ġmar shal +mill iseconds +mark ers +m om +/ place +Ġhol istic +: t +# , +Ġb oto +Ġnause a +ĠSh ooting +ite ch +Ġtext Status +< Class +ĠDes cribe +Ġbuff et +g il +Ġlog its +std call +mod s +ĠSk ull +ĠB are +h ope +ĠIn tr +F air +ĉ pt +Ġacompan h +Ġf kk +_r pc +Inst alled +_ ans +.get Minutes +âĢ¦ "ĊĊ +- thread +Ġpres chool +AIL S +Ġdiff ic +( convert +ĠN ath +ĠDO J +Ġreg imes +Ġenthusi ast +Ġwarrant ies +Ġfasc inated +_b inding +_N ot +oft en +_R W +/m ail +Ġtitle Label +Ġvill agers +ĠJ iang +Ġsw agger +.Row Index +_img s +rap y +VER AGE +. Up +Ġno op +c io +ĉ ST +Ġdecre ment +Ġmagn esium +_ rotate +S it +Ġnieu we +Ġter med +íķ ©ëĭĪëĭ¤ +Ġur g +_t ouch +Ġsw arm +Ġcl ave +th est +ĠL af +H X +ĠH ulk +Ġplaint ext +ĠSof a +get Session +L ed +Ġecosystem s +he i +ĠK ills +Ġhus bands +Ñħ ÑĢан +(d om +_t iles +Nib Name +Ġdon ating +. acc +Ġlifes pan +.b n +_RG CTX +æ ¥ +ans en +Ġmod elling +Layout Params +ĠonChange Text +rs a +- location +.P e +(b us +(s ong +Ġprodu k +ĠSH OULD +ĠC J +Ġs os +ĠHome Controller +.load ed +(D ocument +.s ocial +t iles +Ġl ame += df +.parse Long +Ġpr ac +Ġdet ox +ĠV E +Ġpunt os +Ġdo ctr +Ġan cor +CA PE +Ġc mb +çĦ ¶ +*) " +:// / +Value Type +Ġmort gages +; q +ĠRock ets +s port +UG C +ct s +ãĤ ģ +ie ur +ĠAppe al +(n b +//////////////////////////////////////////////// //////// +IM ATION +ĠC res +ĠMan ip +C ause +at ypes +man ufacturer +# ---------------------------------------------------------------------------- +Ġsp or +es on +Ġpun ched +Ġbook marks +ĠBul k +Complete Listener +ĠTalk ing +ĠEr nest +Ġrub bish +k ills +ĠDE FIN +Ġneighbour ing +ar lo +ĠP CA +ĉm atrix +lo k +Ġat las +ĠG ur +Ġw yn +-n egative +Ġt ul +Ġre lic +ĠV oltage +ĠPre is +ĠJ NICALL +ĠPM ID +ak et +ĉ attr +Ġet iqu +ĠM J +ĠG mail +cl r +_exec ution +éĶ ® +pos itor +. af +N r +Ge orgia +Top ology +Ġperch é +Ġmus lim +Ġepid emi +Ġsab ot +act us +Ġë ĮĢ +ĠIO Error +. est +p refs +ĠKr ish +.Read Key +NAS A +u ção +_D b +umer ator +W ide +(st atement +.end point +.... ..... +Ġ[ * +stream s +m time +P x +at r +Ġt pl +R oman +Ġscen ic +.n z +ĠSe conds +sub menu +Ġìĭ ¤í +_b undle +Ġde ÄŁ +ĠS isters +pre ferences +Ġport a +Ad visor +max Length +ĠG REAT +__ (Ċ +ole st +ĠLabel s +Ġen fer +ĠĠĠĠĠĠ ĊĊ +ĠThe ft +_F ILL +ĠW ise +) application +un ami +> ())Ċ +ADD RESS +B ST +et zt +ĠQ gs +S ense +Exception Handler +ĠCh u +.get OwnProperty +Ġexerc ised +iot ic +ĠRe leases +Ġp interest +ol ie +is oft +Ġsequ encing +Ġpad re +] ));čĊ +(r adius +.m ed +aint ies +.Object Model +Ġem ple +Ġseg uro +St ars +Ġqual itative +lem n +á» ± +> "). +Ġg x +-c ert +ĠAST M +Ġfull name +Ġte lemetry +ĠCamb odia +_ ul +ĠCl are +C USTOM +Q C +ĠUn s +ĠHTTP S +ĠPark inson +ancy box +',' . +T ue +.get Last +Ġab i +Äħ d +A st +ĠEd iting +.Un ity +j mp +Ġm ats +Ġshared Preferences +Capt ain +.page Size +Ġr tl +Ġan meld +Runtime Object +Ġdemand e +(" ; +se ite +-head ed +ĠK ra +ĠF ONT +` \ +Class NotFoundException +. avg +atic al +A j +Ġpermit ting +Pro j +ERR Q +Ġcre ampie +ĠBuy er +-mod ules +ĠSund ays +| `Ċ +Ġday time +Ġ+ ( +Ġgl itch +ĠOper and +Ġtox ins +iny a +D NS +ĠS as +C ake +ĠNation als +.add To +Ġs inking +Ġcompreh ension +Ġsc or +ag ements +Ġt ard +Ġmarch ing +ĠM TV +Ġs ane +Create Info +Ạ¯ +Ġend Index +ĉ layout +ĠåIJ į +S ITE +ĠT HERE +Ġ[ {' +opath ic +Ġtrans mitter +/ body +Ġp und +ĠC losing +Ġset attr +Ġbound ed +At las +sum ing +(t imes +par er +yn om +fe it +Ġf rem +- leg +ĠBr as +> # +Ġì¶ ľëł¥ +ĠIN STANCE +ĠC ouch +_host s +lik elihood +.M arker +ĠM asks +Ġcere al +util ities +Ġelement al +Ġdist orted +in active +c ry +W L +UPPORT ED +.Th rows +/s chema +ser ie +." ', +ĠBened ict +-p icker +ig gs +ĠPir ate +åij¨ æľŁ +ĠTh ema +ĠSouth ampton +Ġarray With +ĠPaul a +Ġpredict or +- Ass +.user id +Ġper i +Ġexagger ated +ur ate +arse ille +ĠCon cent +ĠP ik +Ġ@ _;ĊĊ +Ġform ations +Ġden omin +"/> .Ċ +ended or +Ġpan cre +Ġam t +Ġon Resume +on Delete +ĠB CH +) (" +m ovement +Ġpot assium + čĊčĊ +ĠMah m +} ";ĊĊ +Ġd q +ĠPublish ers +ĠAm pl +ĠDani elle +Ġt ern +èµ · +no ÅĽÄĩ +e in +ĠAsync Storage +un ger +rou w +Ġsc issors +/ assert +.b ucket +/ archive +_M an +Ġint oler +Ġ() => +ĠÐĴ Ñĭ +Ġsa i +.x y +." čĊ +Ġur inary +es ub +IST ICS +ĠÎ º +Ġcompl iments +Ġtypings Japgolly +ih ar +Exp ansion +ĠS erving +_st udents +ĠX BOOLE +( il +Ġì² ĺ +Ġj ó +(t ol +( JS +ĉC G +ĠD RAW +tw ig +Ġo at +_sm ooth +ĠC SL +Ġos ob +Ġens uing +Ġbank er +ĠBack pack +_p ing +Ġwish list += ax +ĉĠĠĠ Ċ +Dis ney +stead y +"> % +Ġproph ets +ĠZ X +Ġminimal ist +.PL AIN +Se attle +. ordinal +ĠPI PE +Ġret orna +Ġjug ador +ĠB ret +ĠâĶ ľ +Ġpl ush +UL ATOR +Sort ing +.grid y +ect omy +_ activ +r ack +Inter active +ĠAntar ctica +Ġv engeance +en so +_k nown +up plier +.Mod ules +ĠConnection State +éļ IJèĹı +@ FindBy +Ġpl acer +\ model +< ()> +.is Successful +-g ood +b z +ĠDr aco +Ass istant +-ex tra +аб лиÑĨ +Ġhyp ocrisy +Ġt st +ĠA gr +$ txt +Ġlog istic +lic ensed +ĠH of +Ġt at +( iv +Ġinto xic +post Id +_st rike +Ġhum iliation +pc odes +" sync +(rec ipe ++ N +rent e +ĉ Client +ycop g +ĠZur ich +ĠPro files +C ountries +Ġp ict +Ġroll out +requ encies +Ġpatch ed +Ġcar tridges +Ġsh ading +J ar +Ġsalv age +ĠTax es +Ġstand by +apor an +E igen +. angular +ĠN ested +äº « +Ġis Visible +ĠDw ight +_BR ANCH +.D elay +Ġk end +Ġfacilit ated +.flat Map +Ġs anta +ĉS end +/m essages +Ġof Type +ĉs wap +# plt +ĠTur ks +N ES +Ġprogress ively +ĠRes idence +ĠT REE +Ġno en +d io +Ġn elle +Ġsog ar +itt i +week ly +Ġambigu ity +_Set tings +W are +.ne o +_D ST +Ġæĸ ¹ +pre p +lob by +@ email +/m ovie +Ġfun kc +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ Ċ +ÂŃ s +Ġguard ians +- pos +Ġconfig uring +ĠC PS +ĠDe us +Ġvidé os +_ empresa +Ġsl apped +< Model +Ġunders cores +U h +.access Token +SET S +ĠS parse +ĠCal d +: path +ĠS ervers += batch +Ġkn itting +Ġx a +Ġsearch Bar +Ġsn ag +Ġinf used +.b am +le ver +Ġtax onomy +Ã İ +Ġatt aching +Ġh ern +_N OP +Click able +(P arse +ĠDynam o +-b uilder +Ġdere g +Ġsc attering +è¿Ľ è¡Į +an zi +ĠShe pard +"> ',Ċ +_X DECREF +ĠBuzz Feed +_M ARGIN +P LOY +.sm all +Ġm imeType +Ġh olog +ĉc amera +li as +Ġsusp ense +ody nam +b au +Ġgrave yard +_n amed +":" ' +Ġ******************************** **************** +Ġgame Over +ĠLENG TH +ĉs creen +Ġdo InBackground +_depend encies +Ġr tc +/ up +_ ROM +H all +Ġdef iciencies +( te +' # +_e quiv +Ġpre order +ĠA xe +ом Ñĥ +.send File +Ġfil t +ĠLim its +ĠCaval iers +.dis count +âĨ IJ +ĠW it +QRST UV +Ġi j +Ġt egen +Ġ: ", +diff iculty +p unkt +ĠEmail s +ch lor +(f un +.U int +ĠSt all +_ verified +u D +File Type +Ġple asures +Ġjud iciary +Ġsh am +ip ur +_PL US +off ers +( foo +_G T +ĉc ore +ENT ION +ĠLib eration +Command Line +_de partment +.A r +_ne ighbor +ĠSub mitted +ĠĊ +Ġdro its +Ġhomosexual s +Ġab duction +ĉw idget +$ headers +ĠD AR +Ġfl a +th reat +Ġlou is +.Get Property +" Just +(f rames +ry o +prof ession +| i +íķ´ ìĦľ +(s v +Ġun recognized +I onic +F ashion +Screen State +ĠIn coming +Not Nil +Ġsync ing +em ie +Ġtherm o +_pro cs +Ġincons istency +rel igious +.m j +Ġperson n +Ġmoment os +or arily +Ġæ Ĭ +_ne urons +Ill ustr +im oto +il ik +ĠW oj +Tr ading +Ġapp are +Ġentre prises +ach at +Ġ ¬ +Ġne igh +BUTTON DOWN +ĠMah er +ag han +-h ash +" f +Ġclient ele +.add Button +ĉ SP +Q i +Ġgr ated +POS ITE +: > +ĠHow ell +ĠCompar ative +ĠIS C +ÂŃ i +O cean +D avis +ĠFil me +W ins +ĠJ IT +oc cer +ĠC orm +ENCH MARK +rch ive +ica ção +Ġm ata +Ġchild birth +ĠOption ally +En s +Ġx http +Ġel ucid +_Osc InitStruct +)) ):Ċ +Ġint uit +ĠDon ate +Ġcorrel ates +> Delete +Ġequ ipe +Ġb oca +Ġinfl atable +er ah +ĠDateTime Kind +Ġcal ves +\ Lib +Ġem lrt +ĠTr ilogy +ĠP anc +ĠD uis +ĠpelÃŃcul a +WAR DS +_DE TECT +-section al +dh cp +For Row +-de struct +ĠPres enter +/s lick +, on +ĠCit adel +logged in +_sub type +Ġsig ue +Ġc uring +ĠFire wall +Ġfluores cence +ĠItal ians +иÑĤ ÑģÑı +.get Style +In Seconds +j ie +-S mith +Ġx link +Ġsub missive +он ÑĤ +arbon ate +ĠF aul +_go als +ĠCommission ers +chart Instance +_POST FIELDS +Ġmed ial +Ġman os +Ġdel t +sv m +.Ap is +ep hy +Ġasym pt +Ġapp Delegate +Ġimpro bable +ck a +sim d +/ Error +. âĢĵ +ĠP TS +de er +Ġs ina +m agnitude +ID ADE +'] }' +Ġmay ores +ĉ comment +/ console +" @ +v olt +.s ell +ĠM acy +Ġmel od +Ġim ágenes +_ch g +Ġin out +ident e +) '),Ċ +d ni +.b lob +Ġtyp ography +Ġe erie +_O ID +pes an +aj an +Ġch opping +Ġbl uff +ad f +_b ases +.Form atter +Ġ\ % +ĠPage Info +Car rier +ĠCal ibration +com o +-b odied +Ġfinanc ier +ĠIN A +. ERR +Ġhood ie +ĠSan ity +gu arded +.opend aylight +ISM ATCH +High lights +ün k +ani em +anger ed +assign ments +Ġregistr ado +ĠU PPER +ampil kan +ash ire +ĠNik ola +ĠC FL +ĠH DC +Ġp oids +ĠIP s +Ġprevent ative +ips oid +if ix +.c amel +.g a +V olumes +- ste +Y ahoo +_s ibling +H ighest +opt group +Ġkvin na +âĢĿ ãĢĤĊĊ +ĠAppl iances +Ġ" >< +') ")Ċ +ht t +ĠIdent ified +Ġpenc ils +Ġmember Id +Ġappend String +.load Data +Ġmock Mvc +Ġj ub +ĠSl ut +ĠTai pei +st att +Pol it +Ġpart ager +Did Change +Incre ases +) }. +ĠB aba +_CL IP +[ unit +Ġк лÑİÑĩ +Ġalc uni +ĠL ola +Ġcl inging +@ PostMapping +(con cat +Ġss id +ĠFa uc +ok it +ĠRecord ed +á lez +($ ('< +.assertIs Not +Ġk ali +V olt +Ġwarm ly +Ġsca res +get ti +füh rt +_d oes +. EMAIL +im ations +Ġspring fox +ĠDec om +arc y +Ġgl itches +ĠM off +ĠV oll +.b etween +Ġcoord en +ĠPart icularly +GB P +Ġsem ble +East ern +_M SB +]) {čĊ +m organ +ĠE VAL +d ere +HO USE +mo ire +ist ique +_l stm +-com mit +yster ious +Ġtw ink +-th umbnails +en ÃŃ +:' ', +Ġblack out +ĠFlo ors +Ġso fas +Ġou i +lesh oot +ĠRa q +- abs +Ġk ra +M ining +sha ft +.set Columns +Cl azz +PRE TTY +.play list +éĸ ¢ +-Sah aran +M ING +ĉ bl +è® ® +j f +DO CKER +hope fully +( ignore +ĠUsers Controller +ĠMitar beiter +ĠL ES +Ham ilton +-m etadata +ĠK K +ikt ig +Ġwoll te +egr ator +] bool +, current +Ġvalue Type +Ġexcav ation +ol and +Ġv erv +/file path +Auth Provider +Ġpro crast +ĉ ULONG +_MEM BERS +Ġup lift +ĠAut onomous +Ġart works +ĠOut reach +Ġp ore +Home page +Dialog Title +ĠGener ating +PAR SE +Ġsem anas +Ġhuman o +JSGlobal Scope +Ġvol te +Ġb ella +(is instance +Ġpl c +\C atalog +Ġeste emed +éĽ · +(s uffix +Ġswe eps +ĉ ORDER +Ġdo ivent +ĠSw arm +ĠComp iled +get Page +AD R +.R ichTextBox +ĠN aming +ag ged +ĠG ANG +r asing +ode led +Ġg ala +ĠJS Name +dd f +Ġill ust +ĠLans ing +[ port +-de ath +Ġdin heiro +ĠE ighth +Ġb ian +st Ã¥ +Ġvers ión +ĠLinear Gradient +ĠHard ing +. *) +ec zy +$ header +Ġv Ã¥r +Un checked +Ġko je +ĠPal adin +() )), +G iving +() })Ċ +Ġd ips +F riendly +Ġport rays +Ġhel ium +Ġinsurg ency +_ex piry +ĠstringByAppending String +Ġa antal +s lope +m ast +.get Integer +Ġ################ ######## +_PIPE LINE +Ġdens ely +Ġmut ating +m idi +ĠSe it +ay ne +NOW LED +ĠDes mond +ĠF Name +ĠN airobi +\ Context +Ġcalc ular +-d en +Ġc ott +] ):čĊ +ĠRecommend ation +ĠRole x +Ġvalidation Result +.p at +Ġn Ãły +ĠRest Client +ĠG PI +ĠAshe ville +ĠO SP +ĠPER MISSION +ÐĶ аÑĤа +/ notification +K night +_W ord +ĠB ender +rank ing +Ġpart ida +_res ervation +Ì Ģ +Ġm Name +Ġget ch +Ġb orr +Ġdilig ent +Disc uss +æŃ£ åľ¨ +ape ake +ion ed +-N azi +.c um +ĠK ron +=$ ('# +/s ingle +Ġerot isch +ĠV ib +Ġrat ified +Ġconcert ed +ĠREG ARD +Ġdo br +.Driver Manager +' r +Port able +ĉs uite +Ġrel aciones +ĠD op +emplo i +DO B +Ġcr umbs +Ġx ls +_App lication +(': ', +Ġ---------------------------------------------------------------- --------Ċ +m se +Ġber k +ĠReturn Value +ĠBel ly +Ġcam ar +ĠPe ek +els ing +Ġnot ifies +ĠTr istan +ĠG AR +em me +ĠElev ated +_C SV +(ch alk +Ġtw enties +ĠSearch Result += search +ĠMix ing +ý t +Ġrecru iter +ĠIDE OGRAPH +ĠA go +( Operation +$ values +Ġworld ly +ĠRosen berg +ĠConfigure Services +>* Ċ +Ġsn ork +_op acity +ĠinitWith NibName +i ado +A AC +Ġ] ). +; z +_par agraph +Ġnos es +stand s +if r +_m E +I raq +.P redicate +ena ire +]] ];Ċ +Ġun idad +Ġretire es +_h ello +Ġmode le +ĠUIT ableViewController +f write +_num ero +_vis ited +Ġrece be +( Notification +Fant astic +_sub menu +ĠP EM +ĠCup ertino +approx imately +class ed +.Read String +Ġdomic ile +_P W +Ġball park +ĠK ale +con tra +_f avorite +/ of +Qu ite +ĠOT A +Ġacceler ometer +did n +| ^ +ĠRohing ya +ivic rm +ann abin +обÑĭ ÑĤи +or ado +') + +Ha unted +, ID +( UIAlertAction +ur v +_b el +ĠMex icans +/ terms +ĠPaint er +Input Label +ĠV inci +ĠRos ie +\ uc +< Menu +Ġcool ant +(current User +_d ual +) "},Ċ +& p +Ġconver ged +Ġrestr ain +ĠYugosl avia += target +Ġimp uls +ds a +Search Tree +Ġh box +ĠImp ress +§ Ãĥ +get FullYear +(d a +ĠY YS +.al ignment +.Get Text +.token ize +ĠOlymp us +Ġmur ky +ore station +Ġdiss atisfaction +ĉT Array +_ kses +.Add Singleton +ĠStart Time +Ġfan atic +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĉ +Ġentity Type +. override +Ġ ------------- +ĠDat agram +f out +(with Id +Ġ# __ +Ł èĥ½ +ek yll +.f riends +ame leon +Ġz ach +.simple Button +ret orno +Ġkon k +/s mall +ĠQuick ly +un read +Don ate +Detail View +Ġdu a +Ġpenetr ated +OM UX +Ġn ir +_p data +"], [" +Ġlow es +Ġdop ing +Ġas ymmetric +Ġneed less +our cem +Ġup ro +ĠGu zzle +af b +Ġsext reffen +-c ollar +Ġcol ossal +Mon key +n ish +Ġhandle Message +Incre ased +* dx +ĠChatt anooga +f org +ĠOr den +Ġsh ri +ĠV and +Ġ" @" +Image Sharp +ĠWild cats +pon ible +.sc enes +Ġpaint ers +ĠPf izer +ĠZ ah +To Local +ĠFl am +Ġé taient +)) ^ +ĠSand box +ĠTR ADE +Ġchrom ium +Ġac claim +Ġpac man +´ t +) reader +M ari +.Dispatch er +.A DMIN +ĠRem ed +Sw eden +Ġoverl ays +. er +Ġp ang +Ġclean ly +aven port +Toy ota +patch es +Ġv tx +ĠE is +cl ado +ĠR itch +RO LS +Ġh ade +Ġconspic uous +Ġdo cks +(j q +ĠPrem iership +ĠBe z +ĠâĦ ĸ +ĠÑĥ Ñģл +_tot als +Ġprov a +ĠC ue +Ġsa úde +ĠGame Controller +IM IZE +, port +ãĢĤ ( +.C decl +Instant iationException +Ġcoll age +ĠIO C +Ġb ais +Ġon Finish +-st ars +set Size +Ġmog ul +Ġdis illusion +Ġche vy +(S chedulers +( IR +_loc s +Ġcann ons +Ġcancell ing +/b us +Ġbuf io +ĠY ours +ĠPik achu +Ġter me +r Ã¥ +f ahren +Ġowner Id +Ġoblig atory +Ġcul p +Ġacid ity +-m ult +ĠBam boo +Ġ' "> +_g s +Ġcomp il +n ard +-ex c +Ġrh yme +Ġbut to +s ays +ant asy +ë ¸ +Ġcitt Ãł +Ġche g +Time String +Ġpos itivity +ĠD abei +Ġw ang +Ġes cre +" c +ĉv ideo +ĠRank ed +.str ings +>> >( +Ġин ÑĤеÑĢ +Ġrest a +[: ,: +Ġrend re +Ġdes er +J os +Ġdis ruptions +Ġоп еÑĢ +s ampling +sup press +Ġcontainer View +ĠSeam less +Ġair y +Ġon load +.Window Manager +ĠPL A +br aco +.set PositiveButton +Ġp du +Ġg si +ĠC li +_gr adients +Ñı д +ĠWh isper +c stdint +Ġl äng +Ġform ulations +én om +ourn emouth +[$ _ +Ġordin arily +.set Username +Ġfacult ies +MIT TED +/ values +Ġwe ir +ĠA pt +M Z +ĉc f +uck en +ĉĉĉĉĉĉĉĉ ĉĉĉĉĉĉĉĉĉĉĉĉ +def ense +[i Var +ĠBusiness Exception +Select ors +(co ordinates +ĠRes ets +ĠDr inks +ole ans +(st ypy +_IO C +.x xx +ĠSl ater +ĠBel ize +Ġ/ ************************************************************************ +add in +_ep isodes +Ġis chem +legal ArgumentException +D anny +Ġp ared +.code haus +ĠAss y +ĉ Rect +â ŀ +.list a +Ġв аÑĪ +Ġv ets +HW ND +ison er +Ġx o +Ġor ally +ĠSt mt +.r nn +ĠD PI +ĠStr ikes +.setViewport View +Ġèĩª åĬ¨çĶŁæĪIJ +Y ELLOW +GL enum +part ners +ĠImp licit +Ġtak o +âĢĻ elle +Ġerm ög +total Count +G il +ĉ work +Ġpr atic +in ati +ab ies +ĠSk inner +Ġspir ited +Ġpancre atic +Ġh df +' em +Ġpsych osis +olic it +Ġ" {" +_at ual +Ġé lect +TE AM +Ġd ak +ĠSW AT +.Fragment Manager +Ġprovision ing +l ifetime +_EXTENSION S +ĠC ASCADE +Ġ! [ +(K P +Ġv em +ĠInterr acial +'] },Ċ +sp acer +_k v +W arehouse +R DD +_f sm +.Stretch Image +, Yes +ĠRefuge e +ĠBr inging +Ġv álido +.inter section +Ġsp ooky +_port al +Ġmo th +ĠZ odiac +ĠSOC IAL +M imeType +'] }} +_Bl ue +Ġbot anical +Ġfr ags +Ġfamil ial +- du +Ġse izing +(block s +.r d +.check NotNull +Ġmis er +Ġmax x +ĠK nee +View Item +Inner HTML +D anger +(( __ +Ġprz ypad +create Url +** , +ĠDecor ating +ATEG Y +?> / +.Design er +hex digest +ĠEvery where +all eries +.TEXT URE +.Block s +z ell +Ġpre ço +S uddenly +input Email +(s ync +.b d +gold en +> '); +ĠDick inson +>> (Ċ +ĠQUE UE +Ġget Column +ĠS AND +.p iece +lic er +Fl utter +Ġget Version +Ġresource Id +og l +ÅĤ aw +.Br anch +ĉ web +Ġfr amerate +PP P +Ġfr ay +C NT +Ġinformat ie +'] čĊčĊ +ne as +Header Code +Ġæ ¸ +Ġtr g +raw types +H onda +Ġmark eter +Ġrequest Data +ĠP g +ĉ not +Ġpage Info +Ġakt uellen +ãģķ ãĤĵ +ĠA MS +push ViewController +ĉ AL +Ġv ests +produ ce +-m ême +ĠRah man +F unny +E Z +_ Valid +Ġsquad ron +Ġl ash +Ġ irm +ias co +ĠPar an +Ġpet ites +ĠDec ay +Ġun initialized +priv ileged +Ġm bedtls +å¤ĩ 注 +Ġ^ . +Ġec static +D etroit +Ġpart en +Ġsou venir +.get Login +моÑĤ ÑĢ +en ção +ĠmÃŃn imo +ĠAccess ed +ri ó +M ic +ĠV ocal +.Set String +Ġmens ajes +åĢ į +Ġattr avers +ĠA ph +Ġ' );čĊ +ünd e +Ġench anted +ĠRoot State +ĠCLOSE D +ĉĉĉĉĉĉĉĉ čĊ +Ġcal iente +or ris +Ġphysic ists +h wnd +_v i +Ġráp ido +Ġcapital ized +ed By +Ġmach ining +Ġhub by +ĠSt acy +.B us +dr ink +H ur +Ġprop ia +Unit Test +Ġmiscon ception +__ ));Ċ +/d c +ĠMay weather +_m C +.create From +ĠQ Painter +rops ych +inn itus +ay as +Ġg eg +(d w +Ġus ado +Ġtrick le +Ġann ihil +ĠP asta +Ġ++ Ċ +(Expected Conditions +.post Value +ic ap +ĠDon etsk +_s oup +-p ublish +ĠP b +ment ions +AC CEPT +.P ull +,âĢĻ âĢĻ +Ġret arded +_AT OM +ĠTermin ator +-c ourt +ĠCLLocation Coordinate +Ġrever ence +ĠS SC +ut ely +ĠW ON +ĠG SL +fre i +.get Longitude +Ġopen FileDialog +.B utter +- important +_M ANY +ĠG ong +âĢľ How +Ġg orge += msg +ĠEz ek +create Command +: checked +Ġinf ographic +.W EST +Dir s +Ġguard a +Ġbeet le +< small +- android +Ġcred itor +ĠM éd +Ġfinal ist +Ġab l +ne v +_inter action +ĠMonter ey +j ah +Ġcand ies +ĠQu incy +èª Ń +Ġbatch Size +ak it +Ġo be +(p ara +Ġexperiment ed +Ġcouncill ors +Ġcl ashed +s qu +-st rokes +ĠG K +ĠEx pires +Ġprosec utions +ĠCreat ures +Ġy ö +x lim +_IM P +Entry Point +ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ +.Default CellStyle +Ġbre ve +ĠBrit ann +Ġsweat y +Ġle th +Ġflash back +per manent +ĠJ DK +_D etails +E uro +p pt +Ġrich TextBox +/ board +Ġtr ance +.c ycle +'); ");Ċ +Ġtox in +_de init +Ġover arching +Ġconfig parser +ĠKaw asaki +.th umb +Ġplay a +ĠJose f ++ _ +Ġzero es +Ġa up +ĠH ari +comm itted +N it +.file Path +ĠDis abilities +man ufact +-al igned +.RE SET +Ġrust y +E y +Ġou sted +cos a +Struct ured +.get D +Ġs ábado +> Loading +_m A +.get Random +bl ings +Ġchees es +tt i +. âĢ¢ +ĠBurg ess +ender it +. ',čĊ +(" "+ +ac b +% p +index ed +_pred icate +nes ia +Ġb ied +ĠC IT +( Pos +_r adi +ä»· æł¼ +B iz +ĠAdoles cent +Ġvi ên +c ycl +_C ancel +Ġcon clusive +Ġappell ate +inform atics +S J +Ġelect ive +role Id +Fetch er +ĉ Command +(" (% +Ġf art +IL A +get Block +A USE +Ġд ан +ĠAr te +Ġnot ifying +Ġge le +.s ame +ĠReg el +ĠBa ÅŁ +.c reation +ĠV N +_comm unity +Ġuns ustainable +SE X +Ġgrid Size +res cia +avers able +(', ')[ +ĠPh elps +á»ķ i +ANCE LED +- IS +.run ners +ĠSt okes +.P rodu +Ġwh ipping +_ac quire +Ġinvestig ación +f ried +.copy With +ĠHard cover +- Se +áŀ¶ áŀ +inv itation +les ai +ĠD orm +ĠÑģпиÑģ ка +Ġconcaten ated +oph il +Ġthink er +/font awesome +ĠLe opard +Ġ"/ ");Ċ +Ġresidual s +ĠMic rowave +Ġconform e +th rop +Ġdis emb +ĠO MG +ĠDisc ipline +ĠAc robat +/re pository +df a +_M ED +buf io +Ġméth ode +_H OLD +ias i +_ legacy +) ččĊ +æ£ Ģ +Get ProcAddress +Ġy ay +ot ence +order id +-t w +Ġdear ly +In coming +/ il +Ġneu rop +uc z +); čččĊ +ĠInnov ative +Ġprof und +ig mat +Selection Mode +re levant +.G O +Ġbru ises +Ġs ach +ode f +Ġre imb +/d esktop +-s pot +und ance +Ent ropy +\ core +Ġsug er +ĠM vc +ĠGN OME +_ind x +ĠYY STYPE +ĠMat lab +ĠC IF +Ġ* )) +Ġproduct List +ĠAl right +ac emark +ÑĤи в +mod ification +int ernational +Ġhom ers +Ġdict s +ĠQ Font +.SQL ite +Ġtransplant ation +ĠMessageBox Button +ĠEl ves +'] ])Ċ +(Q Icon +Ġcin emas +CO ORD +- China +Ġkh ẩu +æĪij çļĦ +Ġskull s +Ġpain staking +f ce +.XR Label +Ġspec ifier +Ġpref erring +/ activity +( Photo +á lt +.l ot +' '. +ann once +.google code +-p df +ĠP oke +_A CL +Ġend owed +dis cover +.om g +Ġwood land +.M agic +Ġvol ont +Not Allowed +Ġch ave +BM W +',' =', +ĠS IX +æĪij 们 +Ġkos her +Ġaspir ation +int l +_ref ptr +'+ Ċ +ment or +.cl ub +Window State +.A RR +Ġz za +Ġmessage Type +.e qu +Th or +Ġin just +Ġg ums +Ġborder Side +//// / +ĠTrans mit +Ġbuf size +Ġh ak +Ġell as +R ANDOM +ĉm c +Ġpe a +ek o +document o +Ġhyster ia +Ġaren as +Ġgun men +Ġm ike +Ġimp unity +atis ation +_Z ero +_COMP ANY +ĠG ors +Ġuse Class +( redis +ĠRUN NING +ĠB air +vel te +Ġ',' . +аÑĤÑĮ ÑģÑı +ö st +encode URIComponent +_re strict +Ġdec als +ĠPed ido +Ġalter cation +Dis plays +ĠApp licants +C US +Text area +ĠAng ola +.f uture +ĠUS HORT +Ġsuppress ing +Ġset zen +AP olynomial +Ġto ch +Ġhall mark +Ġ$ $$ +ĠCHAR SET +.r pm +ĠD ich +---------------- ---- +_p arm +è¿ ĺ +acc iones +h ait +WAR DED +_r outing +ĠN OM +Ġen clave +ĠLot to +ĉf r +complex Content +ĠBall ard +k ube +/w in +.getColumn Model +_RE PLACE +Header Value +Ġest udiantes +Ġap is +Ġb pm +ĠType Name +And Get +rit a +Pl ans +> Note +Ġfet isch +Ġton ed +_g oto +ons ense +Ġm olds +Ġinfiltr ation +ĠGuerr ero +ub bo +ck i +($ (". +_ activities +(ch anges +Ġof App +ĠKe pler +ĠD emp +ĠCont inent +.T icks +ĠUn signed +ĠJah res +Ġfresh men +ĠArch ived +ĠкоÑĤоÑĢ Ñĭй +Ġ' :: +T utorial +C c +Ġtable LayoutPanel +from Json +.level s +_trans ient +Ġendors ing +ĠD IC +la uf +Ġsh red +_E MIT +ific antly +AL A +/ proto +Ġnarrow ing +U tc +Fact ors +Ġsent ient +æŀ IJ +lix ir +ĠC ROSS +met eor +Ġgro in +Ġm db +ĠRot terdam +Ġcom ida +ĠOp Code +ĠDefault Value +Permissions Result +Ġheter ogeneous +Ġm oot +Ġde ceived +-in dependent +ĠObject OutputStream +Ġover power +.d up +Ġl db +Ġdomest ically +Ġbest ellen +Ġlo v +ĠContract ors +Tri angles +Ġfod der +Ġfilm es +ä¼ ģ +Ġrev olver +Startup Script +/ validation +ĠResource Type +i ÅŁ +ĠL az +f ef +Ġlst m +{ * +. attachment +.h its +ew ith +DO G +Al abama +Ġmedium s +.m Context +-c ols +åı ĭ +.not ice +Ġat tn +ĠP acking +ĠL n +_COM PLEX +/ Users +.sav etxt +ĠR ounds +?,?, ?,?, +Ġing l +ĠR OC +_f emale +ĠSt ard +]] ; +Ġwrest lers +Ġtorrent s +Ġsin h + ĊĊ +ë³ µ +s ense +how ever +.Ph ysics +Inf rastructure +ĠSac r +F el +ĠD ISTRIBUT +é ments +ĠValid ates +################################################ ############ +Ġ| / +Ġes l +Ġré seau +ĠB ip +BY TES +_W ATER +Turn ing +EL S +Ġj uxtap +Ġlesb ische +ý ch +( Unknown +Ne o +@ JsonProperty +Ġal umnos +ĠRaq qa +ime i +.get Bounds +.Mouse EventHandler +#### ### +Generic Type +/c ms +Ġturn o +Ġм ин +Ġfolk lore +ĠE vo +Ġconduct ivity +Ġle ben +Ġgear box +-v s +ĠÏ Ĩ +Ġdrink ers +Ġcon exao +ĠTe eth +Ġget Arguments +ĠR AT +ent ious +E duc ++ W +ĠInstitution al +ĠB ord +is Equal +(p wd +Ġign ited +ĠR ousse +Ġimpact ful +ĠM alk +Ġg eral +ĠP ivot +Ġa zt +Ġcsv file +ĠR ope +ĠSOL UTION +ĠArbit rary +Ġlet to +.Mouse Adapter +Ġ} }} +ĠSail or +der a +Put ting +Ġconcentr ates +Ġauth Domain +âĢĿ çļĦ +-f inals +, strlen +Mu on +ĠOrd inary +fire fox +ĠLa TeX +ĠH und +engine ering +/ blue +ed TextBox +(" "); +ĠC DDL +ke pt +ĠGet String +K ir +() =' +ĠO CD +ant ium +$ menu +ĠAppalach ian +Secret ary +ë¥ ĺ +ี ย +Sem antic +Ġ* [ +est one +ung kin +Max Y +-t one +"} ;čĊ +_P art +< Member +tr am +Ġtrans istor +Ġ---------------------------------------------------------------- ----------Ċ +ĠDes de +Ġright ful +ĠCorn el +æ ij +.H OUR +Ġsidel ined +ref errer +m aze +Ġhol ster +Ġcripp led +ĠDate Formatter +oph age +_m D +Ġdes elect +ra ud +ĠPK K +row Data +Ġlock smith +.res ponses +(product Id +_ST MT +Key Type +.Th en +z ee +Ġcr t +ĠGrand ma +@ Resource +Ġbit wise +-c mpr +ãĢĤ www +zeit ig +& display +Cart Item +- No +Ġnum éro +Ġm aur +Ġinst ancia +ĉd t +_n pc +Ġskate board +âĢľ All +ĠCrow d +Ġä n +Ġb raz +ca e +yn et +/p m +/s creen +OPT ARG +ĠV Box +Ġle opard +_g reater +c pt +< dd +Ġmechan ically +osp els +) f +.l wjgl +.get Port +ĠP REF +.Add Transient +pp ard +Ġí ļĮ +Ether net +Ġsal ine +(level s +Ġservice Provider +.A ngle +alt itude +illa ume +Ġs cape +_CAL C +_ quest +ĠDiss ertation +ĠE DM +-C ds +Ġhon orary +st ops +Ġsub dir +ĠV H +ĠChe at +Ġright fully +Q E +.Write Byte +fig ures +enn ie +( DBG +Ġvoks ne +Ġexp ended +UN ICATION +il inx +ĠRec ap +_ verts +Ġtra umat +Ġget Player +Ġverb ess +Ġcultiv ating +Ġiniti ator +Th ông +find First +_per ms +Ġbu c +Ġ""" čĊčĊ +T YPES +object Manager +(Configuration Manager +Ġtim id +Ġsnap chat +Ġcon seg +ĉd istance +_right s +_D es +ĠF lesh +- ver +Ġa fl +fra uen +Ġblas ph +ĠQual ität +ma f +Monitor ing +.D iff +Ġshore line +Ġresponse Body +mem set +< decimal +Smarty HeaderCode +Ġin sets +ĠBinary Tree +amed a +Ġn ihil +ĠN ay +ym ology +ĠW G +Ġt api +ĠInst alled +m aintenance +)} "Ċ +ĠX O +-per iod +s ar +Ġning una +ORM AT +.set PrototypeOf +ĠK b +ĠHen rik +ét ique +ĠLah ore +ĉ Address +Ġmel ts +N y +_adv ance +Ġveloc idad +Ġalum no +Ġsanit izer +Ġph ishing +ĠCom et +Ġch iar +ĉs pec +trim med +(state arr +on nen +Re venue +L ens +Ġcha ired +ĠAss umes +Tr ash +_un set +\ Bridge +Point Size +ĠPol ic +Ġsex uales +ĉd fs +ĠWide String +Ġaccru ed +Y W +_S CHEDULE +Ġk ite +Ġparach ute +[ table +Ġactive ClassName +.Qu ad +Israel i +ĠÅ ĵ +Ġho og +Ġch á»ī +ew ear +Ġtire lessly +set Error +.get Amount +.set Items +ĠM anson +ĠBay esian +_F lag +AC HER +/ original +Ġimm ac +ĠLos ing +' >ĊĊ +L ic +ĠMir age +ĠAssembly FileVersion +Te V +ĠValue EventListener +-s olving +Th o +rou lette +_W P +Ġunint errupted +Ġfield Type +.T yped +Ġam our +Ġmock ery +(v ol +ĠSub committee +ĠR uf +ero x +:UIButtonType Custom +ĠBl ur +Ġwy kon +nc es +ASH BOARD +!! ");Ċ +Ġmurder ers +.d aily +ĠDI AG +j ing +Ġdol phin +Ġl òng +Ġb ö +ĠV ocabulary +.St Object +') "> +Ġz un +Ġscrim mage +tr éal +ĠL ig +[ vi +C ole +Ġfrost ing +.Pl ayers +- translate +Fe els +=\" / +.Butter Knife +Ġ?> ;Ċ +Ġav i +inn ie +.F ailure +Ġsp indle +Configuration Exception +_h op +Ġpos ição +ĠA wait +UIImage PickerController +ĉ day +Ġgen om +C ab +ĠÑĢ езÑĥлÑĮÑĤаÑĤ +OR IGINAL +Ġejac ulation +(t cp +SE COND +Ġton ic +ĠList Box +Ġ ĉĉĊ +() >Ċ +Ġqu atre +ượ ng +with Errors +.M aybe +, âĢ¦ +token Id +_UN DEF +Ġfresh ness +ĠAmend ments +.map box +.C V +(b log +_get time +. quest +s parse +Ġres ale +Ġenthusi astically +ĠProstit utas +W a +C argo +.Parcel able +SENS OR +ĠRy u +La ughs +_N ative +/ pg +yst s +Ġphot oc +ç® Ģ +ado pt +.spec ies +conc iliation +Adjust ed +.Firebase Auth +ut tle +ord ination +Ġm unch +ĠSt ake +.p ing +ank er +(QString Literal +Ġsub script +ĠĠ ĉĊ +ĠM CC +_C md +se xy +i ou +ĠM ANY +Ġn anny +TR AIN +Ġflour ishing +ĠW atches +ĠQ Map +ĠF erm +Ġwas m +ĠA bed +_ UD +ĠGlass es ++ v +Att end +.Ch ain +Ġdec ency +ĠSupplement ary +h unter +-t xt +Ġ" }";Ċ +.set WindowTitle +(" +Ġmasc ara +( Profile +åĬŁ èĥ½ +imit é +Ġwild fires +- ROM +.is On +(group Id +Re pair +accum ulate +Ġ< ", +Ġhand written +Ġach eter +ĠM GM +ĠIr ma +->{ _ +ge e +cr iminal +Ġèĭ¥ è¦ģ +Ġmoment arily +") != +_l it +Ġexpires In +." ). +éķ¿ 度 +Ġfr ække +vl c +Ġor bs +), $ +Ġvent ured +/ >\ +char m +N uitka +eld ig +aton in +W itness +-l at +Ġset Hidden +Ġrelic s +Ġcons ulate +. IGNORE +" After +Ġset Address +Ġbeste ht +Ġ'' )ĊĊ +.x axis +Ġser ão +Ġmis led +_UN IFORM +ĠV IA +inc r +Ġzen ith +Ġvis cosity +Ġthin ly +.get SharedPreferences +.Error Code +"), " +ĠMillion en +Ġ/> )Ċ +Scroll Indicator +-se eking +ĠPOLIT ICO +as ca +_r l +N avig +(full file +Ġsol itude +Ġju ven +Ġhaul ing +ĠMac ros +ĠG ry +Ġexerc itation +ĠATT ACK +Tick Count +Ġr ites +Ġdo e +Particle System +Ġsl u +Window Text +ĠClass Name +Ġsl ander +ĉ Port +j ong +? a +.D ial +âĢĶ at +$obj PHPExcel +Ġso ar +EN N +appe ared +Ġquot id +em achine +Ġn ip +Ġmicro time +ĠAl ma +; ! +---------------------------------------------------------------- -------------------------------- +ĠPass age +Ġdump sters +ĠEx clude +Ġsuggest ive +ĠCircularProgress Indicator +_cl r +Array Type +ILL A +Elapsed Time +Dr iven +Ġresource Name +ĠG arrison +ser ir +-a head +Ġp innacle +ĠEs presso +S parse +Ġass ays +ĠGirl friend +im id +]=' \ +ONGL ONG +Ġportray ing +L ane +Ġb úsqueda +Ġrein forcements +ĠSpread sheet +ĠArray Collection +, arr +light box +ic ana +< " +build ers +K id +ĠMat SnackBar +EX PR +od cast +ĠFound ations +Ġind s +=' ${ +F izz +-function al +(work space +Ġstem med +_p atches +ĠJar vis +READ ING +Ġdisrespect ful +ĠQ Dom +Ġ$ {Ċ +est atus +Re ached +! .ĊĊ +IL T +ĠN DEBUG +ĠCour age +birth date +ĠT ing +Ġutil izado +án chez +Out door +Ġhand guns +Ref Count +É Ļ +rom o +Ġt ts +.S he +ĠP ane +ãĢij, ãĢIJ +ĠIO CTL +/ black +ins cription +Ġbi opsy +ĠTime Interval +.Test Check +ĠGUI Style +ĠCap ability +ĠBeit rag +don nees +T reatment +.back up +Ġsign ings +ĠB oca +dr m +.M AIN +Ġgo ede +ĠMark up +G REE +ĠBase Service +.C reator +Ġj ails +ĠK ahn +Ip Address +ACH I +Ġinhib ited +Ġ@ $_ +ĠAss ass +Ġenvi ado +Hero es +ÐŁ еÑĢ +ĠM aven +.l s +Ġ ive +| RF +Ġresize Mode +Ġrum pe +_attach ments +T U +Ġtact ile +Attempt ing +Ġro bin +y aw +Ġmerc enaries +ĠHab itat +end date +Ġo xy +ĉR andom +oh on +Is Null +ĠValidation Result +ãĥ ļ +um bed +pp v +Ġar p +ich ick +_r nn +ĠT FT +Tex Image +" On +ĠSam pler +top l +Ġj ane +y ling +ĠUN ICODE +Tab Index +< {Ċ +s uspend +uv ian +, application +ол иÑĩеÑģÑĤво +y at +ez ier +ĠCH UNK +ĠAd ler +/ Add +ĠKey Value +Ġspos ób +Sam pling +ch ers +_AM D +R u +.Must Compile +N ation +Ass oc +Man aging +ĠEng l +_G B +Ġsucc inct +Ġdis liked +ĠI ke +Bullet in +_ARCH IVE +Prop osal +Ġjog ging +.C REATED +Ġch ol +è£ ħ +Į ¨ +-p ush +Ġreserv a +core v +è tre +TH R +Ġincompet ence +Ġchar isma +æĦ Ł +Ġ" == +BT N +ĠLoc ator +iv et +('. ')Ċ +Ġfor IndexPath +ô me +Ġcapac it +w aters +ĠWR ONG +ho a +ĠM IPS +Ġem iss +ĠJacqu eline +(c mp +Ġe ens +Le o +.tim ing +CLUS ION +Ġ(" - +åĵ Ī +.k ode +ĠUnd ert +Ġbew ild +ĠEss en +.h d +Ġren egot +Ġm ower +Ġl sp +Ġpen chant +Ġman oe +Ġag li +Ġrec al +ĠOPER ATION +(^ )( +ĠÎ ½ +ĠSc oped +Ġ@ "Ċ += label +[ loc +Int l +ĠN z +table t +.Column Name +Ġscreen Size +DB us +co oked +- registration +âĢľ One +-n on +ĠwiÄĻ c +Ġcost a +.add Tab +. conditions +ĠH ess +MEM ORY +ĠAval anche +() }}Ċ +Ġtri plet +Ġl abyrinth +ĠNode List +ĠNY T +Ġy eni +d ff +.Html Controls +AV IS +/ Math +Ġmem cmp +Ø§Ø ¡ +оÑģ ÑĮ +c rap +(p ages +Ġl xml +ĠQ DateTime +_t cb +Ġopen id +Ġsyn aptic +ĠMD MA +(s lug +igm atic +en or +Ġcr amped +G OP +Ń IJ +.is File +ĠD ifferential +Ġ=" ";Ċ +ĉĉĉ ĠĠĠĠĉ +ĠC ooke +ĉU FUNCTION +Ġpersever ance +Relative Layout +IMPORT ANT +Ġex on +Ġо н +ib ase +(C ONT +n ovation +ä½ ķ +[ sub +Admin Controller +HTTP Header +cre ar +ĠN IR +ĠDrop DownList +Ġval ide +Ġde hydration +. '] +(W IN +Ġ... \ +Ġphotos hop +ĉ Init +_c ou +Ġtime Zone +dar win +rom atic +Navigation ItemSelectedListener +br ates +] --;Ċ +Ġtraged ies +ĠPed iatrics +SM ART +-A PI +ĠMessage Lookup +ĉ vo +Ġprejud ices +Ġm A +U ps +ĠMISS ING +ĉ ad +C ream +ĠT b +ĠMon a +_ ghost +ĉt ypes +Em b +ĠDocument ary +');ĊĊ ĊĊ +Ġl up +_ Reference +ĠB ATCH +Ġintertw ined +< Cell +ĠCab r +n ation +Ġis Connected +.remove Listener +Ġcon g +_t i +ĠSil icone +Ġê²° ê³¼ +ĠW AN +ĠG ibraltar +/ response +ĉp erson +ch ants +V IP +em ergency +Pixel Format +- Am +Ġsouth western +_pl l +if ers +_ON CE +ĠF ayette +.nc bi +_P anel +.Q ual +Ġpol ys +Ġcreate StackNavigator +� t +Ġlay offs +ĠBl anco +Fe at +ĠV imeo +_ch i +_l ifetime +POINT S +, private +Ġunb earable +print ing +Ġc gi +.B ACK +Ġintern s +ĠNew ly +inf eld +( IB +ĠK ata +ĠDef endants +Th r +é¢ Ħ +_V F +FFFF FFFF +Ġdavid jl +Ġbitter ly +S uggestions +.set Cancelable +FIN AL +ason s +_rw lock +_WRAP PER +Ġhapp iest +(row Index +ós ito +TOT YPE +Autom ation +Log File +Ġcons olation +ãĥ Ģ +Ġt êm +Ġpr er +rg yz +ĠG eg +ĉd to +.default Value +ĠK ami +ĠA SE +optim ized +Ġíı ¬ +Ġorigin ates +err Msg +Ġespa ço +(S YS +ĠMc B +d ance +_det ected +Ġfr ü +ĉĉ ĠĠĠĠĉĉ +< Date +(com b +ĠDec ide +\ Field +ĠProp osed +R ib +Ġdis likes +ĠW ien +ĉ Document +Ġtr af +Ġst oria +ĠT ells +') == +C ri +( VALUE +ĠBurn ett +, void +Ġdan h +Ġc cp +Block chain +:"- "`Ċ +IC lient +IS ODE +Iss uer +) }čĊ +, but +ĠU ph +( Sub +Ġtélé phone +ĠonData Change +Ġmarsh aller +-an alytics +, content +Ġdeb acle +_Value Changed +Ġfa una +Ġ# => +Ġf oyer +'util isation +ĠMü ller +ĠFet ish +Ġdefault Manager +Ġback track +B ah +Exp licit +_A SCII +Ġm Activity +(M sg +Ġê² Į +ĠTER MS +ĠAng ie +HS V +ĠMos que +.N ames +íĬ ¼ +rest e +_p arms +Ġgap ing +Ġcro pping +Data Frame +Ġrespons iveness +_ undo +_tr an +. terminate +Ġitalian e +Ġwalk through +Ġattract iveness +д е +_ST S +_ learn +Ġchocol ates +ier archical +-th inking +Ġ ))) +ish ments +.Log f +ĠTM Z +ĠCan ary +fo il +ĠVacc ine +.v x +ĠSur round +Inter mediate +Ġi ov +v ais +'; ";Ċ +ï½ŀ ĊĊ +éĢģ æĸĻ +âĢ¦ it +Se ats +Cl ar +W ars +ĠHutch inson +ĠHas an +! ')ĊĊ +ĠRich ie +che iden +($ (' +Y ork +Ġl ids +Ġal phanumeric +ĠG lock +.sh apes +Ġspark ing +_ epsilon +uplic ated +.dir ty +]) == +ĠìľĦ ì¹ĺ +Ġsc n +Ġ/ **************************************************************** +_PRE VIEW +_H C +ield ing +f gets +ĠAdd ison +Ġproduct Service +- figure +(ret val +z ano +Ġaut ob +ĉs d +_n umer +ĠSet LastError +ĠF ior +ific ance +Unt itled +Ġin field +Ġ{} ));Ċ +Ġsp ac +Ġro okies +(des cribing +ng en +ி à® +.r df +.M utex +Ġkne eling +ĠQ E +set Max +Read Stream +Ġvent as +s ut +cm peq +.WriteAll Text +ĠEx perienced +$ __ +Ġka um +ĠL IS +Ġdocument os +_HE ALTH +icont ains +Ġart isans +OWN ER +Ġblink ed +get Display +Ġto en +Ġrow Num +Ġav ril +Ġinv is +ĠK ear +toBe InTheDocument +ap ur +Ġr acked +ĠMc Master +_ATTR IB +H az +Ġfact ura +/ ts +ĠÑĢаз меÑĢ +Ġz f +Ġshort fall +.f asta +ĠCONST ANT +.man aged +g ems +Shared Pointer +Ġblur ry +b rightness +( components +Ġ... "ĊĊ +SE LL +ĠIllustr ator +.get Channel +Ġtrou vé +yst ers +Ġvo is +ĠLind en +Ġem ojis +Ġb rawl +ĠMS R +ĠE lo +ĠCroat ian +Popup Menu +L ewis +.J WT +Ġaston ished +B ush +(item Id +Ġdet achment +ĠEnc ore +å° Ķ +Ġre kl +Ġcr am +)$ / +.get Host +_re commend +- HT +_cal ibration +Auth enticate +.firebase app +UN IX +ĉC amera +ĠHE AP +I deal +. office +Ġgoof y +(S ymbol +Ġjou er +_part itions +Ġrapid ement +ĠGN UNET +id User +Ġsuperv ise +( Contact +AW N +ãģ ĺ +Ġna am +Ġa ust +åľ¨ 线 +_soft max +Allow Anonymous +amm able +RO UTE +* D +Ġad en +ĠCrist ina +ĠCrist iano +Ġblood stream +sub class +_person a +CH ILD +-k now +Ġnavigation Options +ĠZuk unft +ĠPix ar +Ty ler +Ġunder world +Ġsincer ity +Ġdispens er +Ġk ter +idd ers +.add Node +- checked +Ġke yst +ĠW TO +.sign als +Ġadvent urer +ĠP ang +\ R += pos +Ġdispens aries +ĠClo set +("{ \" +ide on +Ġnécess aire +() "Ċ +_RECE IVED +Ġrésult ats +Ġmod en +ĠIceland ic +; d +. allowed +(new User +Ġmerc iless +.Wait For +Ġday care +ĠCon veyor +ç ĸ +ð ¬ +ç ĥ +ç Ĺ +ç ł +è Ħ +é ² +å ¦ +çĿ Ģ +å¾ Ī +é ħ +ç ĭ +é ª +æ Ĥ +é ¥ +è ħ +æĥ ³ +å ¨ +é ¹ +ç Ĥ +å Ĵ +ç Į +è´ ¨ +æ ¢ +æ° Ķ +ð « +æķ Ļ +ç Ł +å Ħ +åıij å±ķ +åĪ Ľ +è ij +æ ħ +å ŀ +åģ ļ +æĪ ĺ +æ IJ +å¼ º +æ· ± +åĩ ł +ç ¿ +å © +è ŀ +å§ Ķ +åIJ Ħ +è İ +é ¸ +é º +åı Ĺ +èģ Į +å ĺ +æ ½ +é£ İ +èIJ ¥ +åħ ļ +è ľ +éĤ £ +é¢ Ĩ +ç ij +é ³ +æľ ¯ +ä» Ģ +æĪ ¿ +ç² ¾ +å ª +é Ĩ +å¤ ª +èĤ ¡ +è Ľ +åħ ī +æŀ ģ +åĬ ŀ +è ĵ +ç ĺ +å ´ +å Ĺ +èĬ ± +çł Ķ +å¿ « +å¸ Ī +è¶ Ĭ +è§ Ĥ +æ ¤ +æ ¦ +ç ŀ +èĤ ² +çĪ ± +çĻ ½ +ä¸ ĸ +ä»Ģ ä¹Ī +çľ ¼ +å ³ +è Ĵ +æ ĵ +è¢ « +å¹ ² +çĹ ħ +å£ « +ç Ĵ +è ¸ +æ ¾ +å·¥ ä½ľ +è® © +çĥ Ń +è¾ ĥ +åĦ ¿ +åĬ © +ç§ ¯ +ç ³ +ç ĵ +ç £ +å Ĥ +è ¹ +è ļ +å· ± +çĻ ¾ +åĬ ¿ +èµ Ľ +æ ¨ +æ ¿ +è ĸ +æĿ ij +å¸ ¦ +å¢ ĥ +æĬ ¤ +é Ń +å « +èĩª å·± +æµ İ +ä½ İ +åĮ » +éĺ ² +åĨ ľ +è Ĩ +ç Ĩ +é « +åĨ Ľ +æĪ ı +åį ĩ +æĸ ¯ +ä½ ı +èIJ ½ +åħ » +èĩ ´ +ç Ĭ +ç ĩ +ç ħ +è Ķ +ä¼ģ ä¸ļ +åĽ ¢ +æī į +æł ¡ +åĩ Ĩ +å¥ ĩ +åī ¯ +é ¼ +æ¼ Ķ +é© ¬ +èµ ° +ç¥ ŀ +åħ ĭ +æľ Ľ +æ² ¹ +è¾ ¹ +åį ĥ +å¾ Ģ +åĪ ĩ +æ © +ç ¶ +å Ļ +éĻ ħ +çī Į +社 ä¼ļ +游 æĪı +æĸ ½ +ç ħ§ +æİ § +æ» ¡ +è¯ Ĩ +éĩį è¦ģ +è¶ ³ +çķ Ļ +ç» Ĩ +åį ı +éĢ Ĥ +æ ĩ +æ § +é Ħ +è Ŀ +å¸Ĥ åľº +ç»ı æµİ +ä¹ ł +æĸĩ åĮĸ +éļ ¾ +ä¹ IJ +åĨ ³ +æ¬ ¢ +è§ ī +åĽ Ń +åħ ´ +åħ ħ +ä¸ ¾ +æī ¹ +è ķ +æĬ Ĭ +æĬĢ æľ¯ +ç© ¶ +第 ä¸Ģ +ä¾ ¿ +åĵ į +çİ © +åĿ ļ +èŀ į +åį Ĭ +åĸ ľ +å± Ĥ +ç¦ » +ä» ħ +é Ł +åij ³ +å¿ µ +åŃ £ +ç´ § +ä¹ ħ +é ¤ +é ŀ +è ¤ +åĢ Ļ +åĨ µ +ç Ł³ +åģ ¥ +æĢ İ +å® Ŀ +è¡ Ģ +åŁ Ł +æĹ © +çŁ¥ éģĵ +è´ Ł +åį ļ +å· ´ +äº ² +å± ŀ +ä¸ ¥ +äº ī +å¯ Ł +è º +ç ° +建 设 +产 ä¸ļ +åIJ ĥ +åŃ © +æĹ ħ +æł ¹ +æĿ IJ +ä¼ Ĺ +éļ ı +å® ĺ +åº ķ +å½ © +å¯ Į +æ¸ © +åį « +åī § +çĽ Ĭ +æĬ Ĺ +è´ ¢ +çº ª +æ Ĩ +çĶŁ æ´» +çº ¢ +çĶŁ 产 +è¿ ľ +éĴ ± +åĶ ® +ç¾ ¤ +çı Ń +æ¥ ¼ +éĩ ĩ +èī º +å± ħ +åģ ĩ +è° Ī +æĻ ļ +é ¬ +èĪ ª +å® ³ +è Ĺ +ç į +å µ +çİ ĭ +åº · +è İ· +ç» Ń +äº ļ +é£ Ł +åİ ĭ +æĭ Ľ +èĮ ĥ +è® ¸ +åĽ ´ +é ½ +éĻ į +çº ³ +åĵ ª +æķĻ èĤ² +å·² ç»ı +å¾ · +æŀ Ĺ +å®ī åħ¨ +é¾ Ļ +大 家 +éĿ Ĵ +åº ľ +æ² ³ +åı ¤ +èį ¯ +åĿ ĩ +æĻ º +ä¹ ¡ +çķ ¥ +åĨ · +ç¦ ı +å® ¤ +ç» ´ +æī ¿ +å± Ĭ +è¯ ī +åĪ » +è Ł +æ ª +å°± æĺ¯ +è¿Ļ 个 +ä¸Ń å¿ĥ +ä¸ĸ çķĮ +åŁİ å¸Ĥ +éĿŀ 常 +åĪ Ĵ +åı Į +æĢİ ä¹Ī +åĪ° äºĨ +æľ ĥ +åı ² +ä¾ Ĩ +å¾ ĭ +å¥ ĸ +ç» Ī +åª Ĵ +å® ģ +è¯ ¾ +èģĮ ä¸ļ +åħ į +æµ ĭ +æĢ ¥ +æķ ij +çĭ ¬ +èŃ ¦ +é¤ IJ +æĦ ¿ +è´ « +çĸ ij +å ļ +å¥ ¹ +åı Ī +åĽł 为 +ä¸į æĺ¯ +å¤ Ł +æĸ¹ éĿ¢ +éķ ĩ +äº Ĵ +éħ Ĵ +è® ² +çĸ Ĺ +æĺ ¥ +æ¹ ĸ +å¤ ľ +è´£ ä»» +人 æ°ij +åħ ° +çŁ Ń +æķ ħ +åĩ ı +æĻ ® +äº ® +ä¾ Ŀ +åį ° +éĿ Ļ +åĢ ĭ +å¾ ģ +åIJ ¸ +ç¼ º +æĶ » +åĩ Ģ +åħ ¸ +åĽ º +è® ¿ +ç ¹ +ç Ģ +æıIJ ä¾Ľ +ç» ĩ +å¾Ī å¤ļ +çłĶ 究 +è· Ł +主 è¦ģ +æĥħ åĨµ +çŃ ĸ +æŃ » +大 åѦ +æĶ¿ åºľ +å½± åĵį +ä¹ ° +åħ Ń +éĻ © +åħ « +æŁ IJ +è´¨ éĩı +åį ł +å· ® +æĽ´ å¤ļ +æľ ĭ +éĿ © +å® £ +çł ´ +è½ » +åº § +æĺ ¾ +ç¨ ³ +è´ µ +èĥ Į +èī ¯ +çĸ « +æ¯ Ĵ +ä¹ İ +åĢ Ł +è¿ · +çŃ Ķ +æ¿ Ģ +åij ¼ +äºĨ ä¸Ģ +è¶ £ +ä¼ ´ +ä¼ Ļ +è ¼ +ð¬ Ń +åĽ½ 家 +æ´» åĬ¨ +çİ° åľ¨ +ç§ij æĬĢ +åį ¡ +ä¸į åIJĮ +个 人 +è®° èĢħ +ä¸į æĸŃ +éĹ » +ä¹ Ŀ +èij Ĺ +ç» ¼ +ä¸ ĥ +æł ij +æľĭ åıĭ +åį ĸ +ä¼ ¤ +æ² Ļ +åĸ Ħ +å¥ Ĺ +è½ ® +ç© ¿ +è¡ ¥ +ä¸Ģ å®ļ +çª ģ +çĿ £ +è¿ ½ +å¨ ģ +åı ¦ +åĽ ° +æŀ ¶ +ç» Ŀ +æķ £ +æİ ¢ +æ´ Ĺ +ä¸ ´ +ä¼ ¼ +è´ ¸ +ä¸ ° +æĺ¯ ä¸Ģ +ç« ŀ +è¿ İ +èģ ļ +è « +æį Ł +æī § +é© ¾ +è¿ Ŀ +è ¥ +è ł +ä»ĸ 们 +æŶ åĢĻ +å® ĥ +人 åijĺ +è¿Ļ æł· +å·¥ ç¨ĭ +åĪĽ æĸ° +åŃ© åŃIJ +å¸ Į +éĥ¨ åĪĨ +éĵ ¶ +代 表 +é¦ Ļ +å¸ ® +æݨ è¿Ľ +çĽ ĺ +积 æŀģ +éĥ¨ éŨ +åŁ ¹ +æŃ ¦ +ä¸į ä¼ļ +çŃ ij +éĢ Ļ +çİ© 家 +æĭ ¿ +åİ Ĥ +æ¯ Ľ +çģ µ +æŃ Į +ç »¿ +å¦ Ī +çĽ Ľ +é¦ Ĩ +é¡ º +èĦ ¸ +å° ¼ +ä¸ ½ +å¥ ¥ +éģ ĩ +è¯ į +å° ģ +ä¸ Ŀ +好 çļĦ +æĭ ħ +èĦ ± +æģ ¶ +åİ ļ +åĬ ³ +çĽ Ł +æĬ ĺ +åı ¥ +æĢ Ģ +æŁ ĵ +书 è®° +åĨ ł +é² ľ +æ ¦Ĥ +éļ IJ +å¹ ħ +èµ ŀ +å¹ ķ +æ¥ Ń +éģ Ĺ +åĪ ¤ +è ĺ +å ¶ +æĬķ èµĦ +è¡Į ä¸ļ +äº ij +çݯ å¢ĥ +åѦ çĶŁ +åIJĪ ä½ľ +åģ¥ 康 +é£ ŀ +ä¸Ģ æŃ¥ +ä¸Ģ 缴 +åıij çĶŁ +éĺ ¿ +é¢Ĩ 导 +åĸľ 欢 +åºĶ 该 +çĤ º +è® Ń +æĿ Ģ +æ¸ ¯ +交 éĢļ +éĺ ¶ +éĴ ¢ +ä» ¤ +å° ½ +æ¯ į +è¡ £ +ç² ī +é¡ ¶ +ä¹Ł ä¸į +æĬ ĵ +èĭ ¦ +å¹ ¸ +ç¤ ¼ +第 ä¸ī +大 çļĦ +éģ İ +çĥ Ł +éģ ¿ +ä» į +åº Ĩ +æĢ ķ +è° ¢ +çĽ ĸ +å° Ħ +éľ ² +æĸ Ĺ +ç Ĭ¶ +åŃ ¸ +æ¯ ķ +å· ¨ +çŁ ¿ +çļ ĩ +å¸ Ń +çĹ ĩ +æī ¬ +å» ¶ +ä¾ § +æ· ¡ +çļĦ ä¸Ģ +ç¶ ² +æ´ ģ +ç ¸ +è§ Ī +çŃ ¹ +ç§ ĺ +è¯ Ĭ +çı ¾ +èª ī +æ¯ « +ð ¨ +åį ´ +æĪIJ 为 +èĥ½ åĬĽ +é» Ħ +æĹħ 游 +èĪ ¬ +æ¯Ķ è¾ĥ +èµ· æĿ¥ +äºĨ 解 +èĩª çĦ¶ +ä¸Ģ 次 +åŁº æľ¬ +æĽ ¾ +综 åIJĪ +èı ľ +è§ī å¾Ĺ +第 äºĮ +è· ij +æ³ ¢ +åĢ Ĵ +ç¡ Ģ +åħ µ +èį ī +çĶ ³ +çĶ ° +æĤ £ +è§Ħ å®ļ +èĥ ľ +èµĦ 产 +æ¢ ¦ +æľ Ŀ +è¿Ļ éĩĮ +å¤ « +æĮ ¥ +ä½ Ľ +å® Ī +éĽ ¶ +æĸ ¼ +ç¯ ĩ +å² Ľ +åĵ ¥ +éŃ Ķ +ä¸į åĪ° +æī ĺ +åº Ĭ +æ¬ § +èį £ +æ± ĩ +æī © +åģ ı +å¢ Ļ +è® ¯ +å© ļ +æĥ ł +æ´ ĭ +å® ľ +æ¶ ¦ +æħ ¢ +éĢ ı +å® ½ +é¡ ¾ +ç´ ¯ +æ± ¡ +çĪ Ĩ +ç§ Ł +æĥ Ĭ +æ¶ ¨ +é¥ ° +éĺ µ +é¥ ® +æļ ĸ +åº Ł +æĹ Ĺ +éļ Ķ +ç¶ ĵ +åĭ Ļ +å¯ ¦ +éĢ Ķ +æī « +çĥ Ī +éĽ » +åĪ ij +éĹ ľ +éĹ ª +å¥ ĭ +å Ĥ¨ +ç¼ © +ä¾ µ +å ¬ +𬠶 +åĽ½ éĻħ +ç»Ħ ç»ĩ +ä¸ĵ ä¸ļ +åıij çİ° +å¸Į æľĽ +ç»ı èIJ¥ +åı « +æĿ¥ 说 +éļ ľ +ä»» ä½ķ +交 æĺĵ +éĩį çĤ¹ +çļ ® +ç» į +æ´ ¾ +ç§ij åѦ +åºĶ çĶ¨ +建 çŃij +èĤ ī +æĶ¹ éĿ© +åŁº ç¡Ģ +æ± ī +åĩº æĿ¥ +è¿Ļ ä¹Ī +åĪ ļ +åĿ IJ +ä¸į ä»ħ +ä¼ļ è®® +éĿ ł +åªĴ ä½ĵ +æ° ¸ +åĨ ² +èĭ ı +å¤ ® +çĪ ¶ +åł Ĥ +å®ŀ éĻħ +è¡ Ĺ +ç« ¥ +éĺ ħ +äºĭ æĥħ +åİŁ åĽł +éħ ¸ +以 æĿ¥ +å¨ ± +å® « +åĿ Ĺ +ç» © +éĩ İ +ä¸į å¾Ĺ +ä¼ł å¥ĩ +ç¡ ¬ +åİ ħ +æĹ ¢ +ç» ĥ +èĦ ij +å¼ ± +æİ Į +è´ ´ +æĮ Ĥ +åħ³ éĶ® +å° ļ +é¥ Ń +åº Ħ +çĻ ¼ +åľ ĭ +æİ Ī +个 æľĪ +äº Ī +å¸ ģ +è· Ŀ +æ² ī +ç« Ł +åĨ ¬ +æĬ ½ +éĨ Ĵ +å¼ Ł +è§ ¦ +èģ ĺ +è± Ĩ +æļ ´ +åijĬ è¯ī +è± ª +èµ ¢ +è· ¨ +è³ ĩ +çĪ ¸ +æĬ ± +æµ ª +éº » +ä» ª +è¡ ¡ +å¥ ¶ +çģ ¾ +èµ ¶ +èĤ ¥ +å§ IJ +åĢ º +éľ ĩ +è® ¢ +æ¬ Ĭ +ç · +å» ī +ä¿ Ĺ +å¿ ĺ +å¦ ĩ +ç¼ ĵ +åŃ ķ +æ¼ « +è£ ģ +çĩ ĥ +é» ĺ +çī ¢ +çĪ · +æĬ µ +å® ¾ +æľī ä¸Ģ +è¿ ¹ +è¿ « +è² Į +æľī çļĦ +ð¬ ĺ +è¿ĺ æĺ¯ +æīĢ 以 +ä¹Ł æĺ¯ +è¿Ļ äºĽ +对 äºİ +åIJ § +缮 åīį +èĩªå·± çļĦ +èĥ½ å¤Ł +å¦Ĥ ä½ķ +æľº æŀĦ +åıª æĺ¯ +ç½ij ç«Ļ +åħ¨ éĿ¢ +为 äºĨ +å¼Ģ åıij +æĸ° éĹ» +éĩij èŀį +ç» § +客 æĪ· +ä¸Ģ èµ· +èĮ ¶ +åħ³ 注 +æ°´ å¹³ +åİĨ åı² +å¢ŀ éķ¿ +é ± +åŁº éĩij +åº Ń +åı ¶ +ä¿ ĥ +éĽ ¨ +æ¶Ī è´¹ +èĪ ¹ +çŁ¥ è¯Ĩ +æĪĺ çķ¥ +ç»ı éªĮ +å³ ° +æĽ ² +èĦ ļ +åĨ ° +å¤ ı +å½ Ĵ +ç¬ Ķ +èĻ ij +çĶ ² +åľ Ī +è¯ Ĺ +é½ IJ +容 æĺĵ +çłĶ åıij +éª ¨ +çº ¸ +è· µ +æĹ § +çķ ¶ +åĪ ¸ +è´ · +åı ¬ +ç§ ĭ +æ¶ ² +è¡Į æĶ¿ +çĮ ® +èĤ ¤ +éĢ IJ +è¶Ĭ æĿ¥ +è¶ĬæĿ¥ è¶Ĭ +æĦı è§ģ +èĪ ŀ +åī Ĥ +æ¶ ī +ç¨ĭ 度 +åħ¬ åħ± +æ¢ ° +æľ « +çº ¯ +åĶ ± +æ´ ² +æĬ ¢ +æ¤ į +å¿ Ļ +ä¼ ° +å¼ ¹ +æ³ ī +æľĢ 大 +è¶ ĭ +å· § +ç¦ ģ +æī ¶ +åį ± +çı ł +çĨ Ł +æĭ ľ +主 ä¹ī +æĿ Ĥ +éĻ Ħ +éģ į +æIJ Ń +æĮ ¯ +å¤ļ å¹´ +æķ ¬ +æij Ħ +çº · +å¼ ĥ +æ¹ ¿ +å¨ ĺ +æ¡ £ +é© ¶ +æľ Ĺ +æ® ĸ +æ¦ ľ +åĵ ¡ +ä¸Ģ ä½ĵ +æŁ¥ çľĭ +ç¹ ģ +æµ ĵ +åħ¬ å®ī +æ½ ľ +è´ ¯ +éª Ĺ +æ IJľ +å· ¡ +è ¬ +é Ĭ +å§Ķ ä¼ļ +æĤ ł +åī © +æı Ń +åŃ£ 度 +ð «ĺ +𬠬 +ä ´ +ð ª +ä½Ĩ æĺ¯ +éĥ½ æĺ¯ +å¹³ åı° +åѦ ä¹ł +åĵģ çīĮ +ä¸ Ķ +è¿Ļ ç§į +æĶ¿ çŃĸ +æĭ ¬ +认 为 +ä¸Ģ èĪ¬ +æłĩ åĩĨ +æĶ¯ æĮģ +模 å¼ı +åħ³ ç³» +çļĦ æĺ¯ +è¿Ļ ä¸Ģ +ä¸į è¦ģ +çĶ ļ +ç²¾ ç¥ŀ +æĭ ¥ +åĪ© çĶ¨ +ä¿Ŀ æĬ¤ +ä½ľ çĶ¨ +èĭ ¥ +åĽ½ åĨħ +ä»ĭ ç»į +ä¸Ģ ä¸ĭ +å·¥ ä¸ļ +缮 æłĩ +æľĢ åIJİ +ä»· åĢ¼ +å° į +éĵ ģ +è° ģ +ç»ĵ æŀĦ +éĽ ª +æĻº èĥ½ +ä¼ł 绣 +ä½ĵ èĤ² +çĶŁ æĢģ +æĭ į +æİ ª +åĨľ ä¸ļ +çī¹ èī² +è§Ħ 模 +æŶ 代 +è¿ĩ ç¨ĭ +éĴ Ī +æĿ ¾ +åĶ IJ +åĮ» çĸĹ +çģ ¯ +åĪ¶ éĢł +æł¸ å¿ĥ +ä¸į åı¯ +ç³» åĪĹ +åIJ ī +åľ £ +åĢ ij +ä½ ³ +æĿ¥ çľĭ +æ¯Ķ èµĽ +ä¸ĭ æĿ¥ +åĩº äºĨ +å¹² éĥ¨ +å¾® ä¿¡ +å½ĵ åľ° +åį · +åį« çĶŁ +ä¼ Ł +çĸ« æĥħ +è° · +åĩł 个 +éĺ ´ +çĶŁ çī© +å° ¤ +ä¼ Ĭ +èĤ ¯ +éĿ¢ 积 +åĪĽ éĢł +æı ¡ +åľ Ĩ +æĻ ĵ +æĪIJ äºĨ +åĩ ¡ +çĸ ¾ +ç«ŀ äºī +è® ¨ +主 é¢ĺ +é² ģ +è¿ ª +ä¿ Ħ +æĢ ª +ä¸ ¦ +èĻ ļ +æ½ ® +çĥ § +èĢ ³ +æ± ł +éĢĤ åIJĪ +æł¹ æľ¬ +åĬł 缣 +çĶµ è§Ĩ +æ· · +ç¼ ĺ +çª Ĺ +çĬ ¯ +æĥ ¯ +æĦı ä¹ī +åĬŀ æ³ķ +ä¼ ij +æ» ij +åĭ ĩ +æķ ¢ +å¯ » +è¦ Ĩ +éĢ ĥ +ç»ı çIJĨ +åĿ ı +æ³ ½ +ä¹ ĺ +åĪ º +å± ı +é¡ ¿ +äº ¡ +éĤ Ģ +åħ ¼ +åĭ ¤ +æ® ĭ +æĺ ł +æ¯ķ ä¸ļ +æĪ ª +è· Į +å£ ģ +åı¦ ä¸Ģ +羣 å®ŀ +ç£ ¨ +è¯ ļ +å¿ħ è¦ģ +æģ ĭ +æĩ Ĥ +å¾ Ĵ +è° ĵ +æķ ı +æ Ļ¨ +èĥ ¸ +æĭ ¼ +å¦ Ļ +è¯ ¸ +èģ Ĭ +æĤ ī +éº ¼ +åĩ Ń +èĪ Ĵ +æ¶ Ĥ +è¿ ģ +æ² ¿ +å¡ ij +æĽ ¿ +æ¾ ³ +å¿ į +èĢ Ĺ +éľ ¸ +åĩł å¹´ +åĪ Ĭ +èĦ ī +èħ IJ +æ¡ Į +çº ł +æ» ļ +æĤ ² +åĨ Ĵ +å¦ ¹ +çķ ħ +çº µ +æij ĩ +å¤ º +è·¯ ä¸Ĭ +å¿ ½ +èĸ ª +æģ IJ +æĦı æĢĿ +å« Į +æı ´ +æ° § +èĢ Ģ +éĺ » +è½ ¨ +å¹ » +æį ķ +åĿ ¦ +åĵĪ åĵĪ +çĭ IJ +æ» ¨ +è² » +è¿ Ł +人 éĥ½ +ç» ĺ +åı ¹ +çµ IJ +æī ° +æ» ĭ +å¥ ij +åĭ Ł +ç¢ º +ð ¦ +éĽĨ åĽ¢ +æĿ İ +å¼Ģ å±ķ +æıIJ åįĩ +åħ¨ åĽ½ +æ±½ 车 +åѦ æł¡ +æł¹ æį® +è¿Ļ æĺ¯ +åĩº çİ° +éĻ Ī +ç½ Ĺ +èİ· å¾Ĺ +åĪ ĺ +éĶĢ åĶ® +æľª æĿ¥ +éľĢ æ±Ĥ +å®ŀ æĸ½ +åĿļ æĮģ +åħ¨ çIJĥ +éĵ¶ è¡Į +æݧ åĪ¶ +é¡ » +åľ° åĮº +æīĵ éĢł +çļĦ è¯Ŀ +帮 åĬ© +ä½ĵ ç³» +è¾¾ åĪ° +è§Ħ åĪĴ +åŁ¹ è®Ń +两 个 +æĬ¥ åijĬ +åľ° æĸ¹ +å®Į åħ¨ +æİ ī +ç»ĵ åIJĪ +宣 ä¼ł +æ³ķ å¾ĭ +èīº æľ¯ +çĶµ å½± +èª ª +ä¸Ģ çĤ¹ +è¶ħ è¿ĩ +çĶµ åŃIJ +æĢĿ æĥ³ +æķĻ åѦ +éĺ¶ æ®µ +åķĨ ä¸ļ +çī© æµģ +åĪĽ ä¸ļ +æĸ¹ æ¡Ī +çİ° 代 +æ¡ ¥ +èIJ½ å®ŀ +带 æĿ¥ +产 çĶŁ +ç§ Ģ +æ³ ° +ä¹ ± +åħ· ä½ĵ +åĸ Ŀ +èĵ Ŀ +å® Ĺ +åįĩ 级 +æ·± åħ¥ +ä¿Ŀ éĻ© +ç®Ģ åįķ +çĹ Ľ +稳 å®ļ +è¾ Ĩ +å±ŀ äºİ +å· Ŀ +ä¸į å°ij +åĴ ¨ +举 西 +å½¢ å¼ı +娱 ä¹IJ +æŃ£ 常 +é¸ ¡ +åħħ åĪĨ +å®ŀ è·µ +éĩĮ éĿ¢ +è· ³ +èĻ İ +æĪIJ éķ¿ +æļ Ĺ +çĿ ¡ +ç½ ª +çIJĨ 念 +æĮ ij +èµĦ æľ¬ +å¤ļ å°ij +ä¸ĭ éĿ¢ +å¸ Ŀ +åħ¬ å¼Ģ +æ¸ IJ +éķ · +å± ĭ +欢 è¿İ +å¿ĥ çIJĨ +çĤ İ +æ¹ ¾ +è® ĵ +éĤ Ħ +ç³ ĸ +ä¹ Į +åĬ ± +çī Ļ +èħ ¿ +å² Ĺ +ä¼ į +æĪIJ åijĺ +åŃ Ķ +å°ı ç¼ĸ +èij £ +æ³ ¡ +åħĪ è¿Ľ +åħ § +åĺ ´ +è´ Ŀ +è » +æIJ ŀ +æ³ Ľ +é¸ Ł +ç½ ² +èĽ ĭ +主 ä»» +缮 çļĦ +ä¹ ı +æ´ ¥ +æĪ ´ +严 æł¼ +çħ ¤ +çĮ « +åĶ ¯ +å° Ĭ +çĶ ľ +åŀ ĥ +åľ ¾ +æĭ Ł +çĦ ¦ +é« Ķ +å® ı +æ© Ł +é© » +æĹ ģ +å½ » +éĥ½ ä¸į +æij © +ä» ĵ +ä¹ ³ +å² ¸ +è° ĭ +大 å¤ļ +çģ Ń +èħ ¾ +æŁ ľ +èĪ į +åħļ çļĦ +å° ĺ +åįģ å¹´ +æĭ Ĵ +è£ ¡ +æŁ Ķ +å¹ ¼ +éĶ ģ +ä¸ĵ 项 +æī İ +驾 驶 +ç¢ İ +è¢ ĭ +éĶ ĭ +å£ ® +å° ĸ +çĶµ æ±ł +è¿ Ķ +æ¼ ı +å¾ ª +èı Į +èĥ ĥ +è¾ ħ +éĢ Ĵ +èĥ İ +éĻ ª +å¯ ¿ +å¥ Ķ +çĮ Ľ +çº ¹ +çŁ¥ åIJį +å¿ Ĩ +æ¡ ĥ +æ£ ĭ +éĢ Ĩ +çĤ ¼ +ç± į +çī § +æł· çļĦ +è¾ Ľ +åł Ĩ +å®ŀ åľ¨ +ä¼ ı +å® ¿ +èµ ı +è£ Ĥ +åįĬ å¹´ +åĢ ¾ +满 æĦı +æ¢ ¯ +æĦı åij³ +åŃ ¤ +ç¥ Ŀ +æĻ ¶ +èµ Ķ +åģ ¿ +èĦ Ĥ +ç½ ļ +ç¢ į +æ² ĥ +æ ĵį +å´ ĩ +æļ Ĥ +è· ĥ +æIJ ¬ +å© Ĩ +é ī +éī ´ +åħ´ 趣 +èIJ¥ ä¸ļ +è® Ĭ +èĦ ı +è¾ Ī +å·ŀ å¸Ĥ +è´« åĽ° +ç© · +ä¸Ń å°ı +æ¼ Ĥ +çĻ Į +èľ ľ +ä¼Ļ ä¼´ +çī µ +æĤ Ł +éĻ · +èµĽ åŃ£ +æ¨ £ +åģ ¶ +æĺ Ĩ +è¢ Ń +æį IJ +èī ° +æ Ĥ¬ +çĶ ¢ +èij ¡ +çĽ Ĺ +å© ´ +å° İ +çº ½ +åĢ ¡ +æī ® +è¨ Ń +æĬ ij +ç¡ ķ +è¾ ĸ +éĥ ģ +è¾ © +éĤ » +çİ° åĩº +è¦ ı +å½ ¹ +éĺ Ķ +åī µ +è¯ ± +æĥ ij +æ· Ģ +é¢ Ī +ä¾ ¦ +æģ ° +æ£Ģ å¯Ł +éĨ « +çĦ¶ æĺ¯ +åĭ ĥ +èĮ « +ä ĵ +𠬸 +ä½ľ 为 +çļĦ 人 +éĤ£ ä¹Ī +ç¾İ åĽ½ +è¿ĺ æľī +æıIJ é«ĺ +èĻ ½ +åħ· æľī +åĮħ æĭ¬ +æĪĸ èĢħ +ä¸į è¿ĩ +ä¸Ĭ æµ· +åĮ» éĻ¢ +èµĦ éĩij +çĶļ èĩ³ +åĪ¶ 度 +解 åĨ³ +èģĶ ç½ij +继 ç»Ń +建 ç«ĭ +è¿Ľ ä¸ĢæŃ¥ +æĿIJ æĸĻ +ä»Ĭ 天 +å¿ħ é¡» +åIJĦ ç§į +çİ° åľº +ä»ĸ çļĦ +å¢ŀ åĬł +é¢Ĩ åŁŁ +åıĤ ä¸İ +æĮģ ç»Ń +ä¹ĭ ä¸Ģ +çī¹ åĪ« +é± ¼ +åħ± åIJĮ +åĬ ª +çİ ī +人 们 +åħĪ çĶŁ +ä¼ĺ åĬ¿ +ä¿Ŀ æĮģ +ä½ľ åĵģ +çī Ľ +æĪIJ æľ¬ +æĶ¶ åħ¥ +åıĬ æŶ +è´Ł è´£ +æİ¥ åıĹ +èį IJ +åıª è¦ģ +羣 çļĦ +导 èĩ´ +æľº åĪ¶ +è¡Į åĬ¨ +æĸ° çļĦ +å®Į åĸĦ +为 ä»Ģä¹Ī +ä¸Ń 央 +æĪIJ ç«ĭ +æĦŁ è§ī +åıĺ åĮĸ +åıĹ åĪ° +并 ä¸į +åŃ Ļ +æĸ½ å·¥ +æĺİ æĺ¾ +è¿ĩ åİ» +åıij æĮ¥ +羣 æŃ£ +åŁº åľ° +æĺİ ç¡® +èĥ ¡ +许 å¤ļ +ä¸Ģ å¹´ +æĸ¹ åIJij +æģ © +缸 ä¿¡ +åľ ³ +详 ç»Ĩ +äºĭ ä¸ļ +çĶŁ åij½ +åĴ¨ 询 +æĸĩ æĺİ +çij ŀ +绿 èī² +èİ « +æĦı è¯Ĩ +æĬķ åħ¥ +åĬł å¿« +æ¢ ħ +ç¿ » +å¼Ģ æĶ¾ +æĻ® éĢļ +åįı ä¼ļ +æĪIJ 绩 +ä» Ļ +å¯ Ĵ +è¯ģ åĪ¸ +认 è¯Ĩ +ä¸ ¹ +大 éĩı +è¿ ħ +åģļ åĪ° +设 æĸ½ +è´¸ æĺĵ +èĥ½ æºIJ +æŶ æľŁ +ä¸Ģ 天 +æ²» çIJĨ +åĺ ī +å® ĩ +丰 å¯Į +举 è¡Į +æĪIJ æŀľ +èĤ¯ å®ļ +çĭ Ĺ +åĬ¨ åĬĽ +æ£ ® +åĩł ä¹İ +åĽł ç´ł +æ°ij æĹı +æ´ ŀ +ç½ij åıĭ +åIJĪ çIJĨ +广 大 +æ® Ĭ +æ´ Ľ +æĿ ¯ +èĴ Ļ +çĶ¨ äºİ +èŀį èµĦ +ç¥ ĸ +æľº 械 +举 åĬŀ +èĩª åĬ¨ +åĬŀ åħ¬ +é» ŀ +éĽ Ħ +åĢ¼ å¾Ĺ +çĮ ª +以 为 +æĺ Į +è·Ŀ 离 +åIJ¸ å¼ķ +ç» ķ +éļ Ĩ +计 ç®Ĺ +éĺŁ ä¼į +大 ä¼ļ +å¼ķ èµ· +çī¹ çĤ¹ +èĥ ¶ +å¹´ è½» +æľ¬ 身 +æľº åħ³ +å®ĺ æĸ¹ +éĥ ij +æµ Ļ +è§Ĵ èī² +èij£ äºĭ +为 主 +æĹł 论 +ä¹ł æĥ¯ +æ¥ ļ +æĭ ĵ +绣 计 +åħ Ħ +广 æ³Ľ +åį Ģ +污 æŁĵ +è« ĭ +èĬĤ 缮 +ä¼ ¦ +è¦Ĩ çĽĸ +èĢ IJ +æī¶ è´« +ç»ı åİĨ +éĩįè¦ģ çļĦ +èĤ¡ 举 +æĭĽ èģĺ +åĽĽ 个 +æĩ ī +èĥ ŀ +æij Ĩ +é«ĺ éĢŁ +éº ¦ +åİŁ åĪĻ +èİ ± +æĽ´ 好 +éķ ľ +åĩ Į +åŀĥ åľ¾ +éĢ ² +çģ ° +éĵ º +äºĭ æķħ +çĶ ĺ +空 æ°Ķ +é¾ Ħ +èı ² +çĵ ¶ +æĺ ¨ +æĹ¥ æĬ¥ +æµ ® +åľ° åĽ¾ +åij Ī +大 åĬĽ +ç» ª +å¸ ħ +æľį åĭĻ +ä¸į éĶĻ +乡 æĿij +å± ¥ +å¹³ æĸ¹ +éĹ ² +æī £ +ç´ł è´¨ +èµ ´ +éģ Ń +èIJ ¨ +èĩª 主 +éĩij å±ŀ +èī¯ 好 +两 å¹´ +æ³ ¥ +é¢ ľ +ç²¾ 彩 +ä¸Ń åįİ +æĻ ĭ +ä¹ł è¿ij +ä¹łè¿ij å¹³ +æĪĺ 士 +åģļ çļĦ +éª ij +æ» ´ +çĵ ľ +çīĪ æĿĥ +èĤ ł +æľĥ åĵ¡ +çı į +ç¨ ® +ä »¿ +çī© ä¸ļ +åĢĭ 人 +å¦ » +ä¼ ¸ +æ± Ĺ +æĹ º +çIJĨ æĥ³ +æij ¸ +è¿Ŀ æ³ķ +å®Į æķ´ +åİ ¦ +è¸ ı +æĸ ij +æ¡ Ĥ +ä½ĵ åĪ¶ +å¸ « +æĿ Ĩ +æ® ¿ +æ¯ ģ +é¦ Ī +è§Ĵ 度 +æ¬ £ +çĥ ¦ +èĤ º +éĩĩ 访 +æij ĺ +æĮ ¡ +æ· ĺ +åħ» èĢģ +çĤ ¸ +è¿ Ī +åİ ī +åĿ Ĭ +è¾ £ +åĩ Ŀ +æ³ ª +çĸ ı +æİ ĺ +åĥı æĺ¯ +éĽ ķ +ç¼ Ŀ +èį · +æį · +åł ¡ +åı¥ è¯Ŀ +çĸ ¼ +æł ı +éģ µ +ç¢ ³ +å·¥ åķĨ +æIJ º +åĪ ¥ +ä¹ Ļ +æĹ ĭ +æĥ ľ +ä¸Ģ 大 +å±Ĥ 次 +èµ ĸ +æĬ ¬ +æ¨ Ĥ +è¯ ŀ +åħ Ĵ +ç¯ ® +èĤ ĥ +å§ ¿ +æĬ ļ +çĵ · +çĶµ åĬ¨ +æĸ° åĨł +æ¶ µ +ç¢ ij +æ· ® +æĹ ¨ +è¸ ª +æ¸ Ķ +æĦ Ī +åı Ķ +åįĹ çľģ +ç¾ © +å§Ķ 书记 +è² ¸ +æ¶ Į +è« ĸ +èIJ Ħ +æı ı +å¿ § +è¾ ¦ +å¦ Ĩ +æī Ń +åij µ +éģ ¥ +è¨ ± +ä» ĩ +åįģ ä¸ī +åī ² +èª į +èĪ ° +é¢ ĩ +é¥ ± +çĭ ł +é«ĺ çļĦ +çµ ± +æħ İ +é¢ ģ +åIJĪ éĢĤ +æµ ´ +èµ ĭ +æĬ ¼ +å¦ ¥ +éĻ¢ éķ¿ +èĢ ķ +è¾ ¨ +æħ ° +åįģ åĽĽ +æľ µ +èĵ Ħ +æŀ ¢ +å» · +æĤ Ħ +æ¶ ¯ +çŁ © +åŃIJ éĩĮ +çĬ ¹ +å±Ģ éķ¿ +é IJ +å¥ ł +ä¼ļ éķ¿ +æĵ ļ +ä¸į åıĬ +åįģ ä¹Ŀ +æ¬ º +èº º +éĺ IJ +çº Į +è¨ » +åĨ Ĭ +èŃ ĺ +é«ĺ çŃī +èħ º +å¤ ķ +ç» ij +åĶ ¤ +èķ ´ +çķ ľ +æħ ĭ +åı Ļ +åı ĥ +å³ ¡ +人 大 +éħ ¿ +éģ © +å¥ ¢ +åı£ æ°Ķ +éĮ Ħ +é ı +åĭ ĺ +è´ ¿ +éļ ª +é ĭ +éļ ¶ +ð ¥ +𬠣 +ð £ +ð« į +𬠳 +ð« ĵ +ð« Ħ +ð« Ł +𨠱 +ä Ĺ +以 åıĬ +æľī éĻIJ +åij ¢ +åIJ Ĺ +çľĭ åĪ° +计 åĪĴ +è¿Ľ åħ¥ +缴 æİ¥ +åĪĨ æŀIJ +åıª æľī +设 å¤ĩ +åħ¶ å®ŀ +åĬł 强 +ä¸Ń çļĦ +ä¿Ŀ éļľ +èĢģ å¸Ī +人 æīį +å¾Ĺ åĪ° +é£İ éĻ© +ä¸Ģ ç§į +空 éĹ´ +æĪij åĽ½ +ä¹ĭ åīį +ä¸ĵ 家 +æĿ ¨ +æĹ¥ æľ¬ +群 ä¼Ĺ +åıĤ åĬł +æķĪ æŀľ +æľī åħ³ +家 åºŃ +åĮº åŁŁ +åĬª åĬĽ +éļı çĿĢ +æĹł æ³ķ +交 æµģ +è¡Į 为 +æ£Ģ æŁ¥ +æľŁ éĹ´ +å¦Ĥ æѤ +èĤ¡ 份 +å½ĵ æŶ +è£ħ å¤ĩ +åĩĨ å¤ĩ +éħĴ åºĹ +è¿IJ åĬ¨ +æıIJ åĩº +å·¦ åı³ +æݪ æĸ½ +é£Ł åĵģ +æ¶Īè´¹ èĢħ +åѦ éĻ¢ +æĮĩ 导 +è¿IJ èIJ¥ +éĩį 大 +åĨľ æĿij +éĢł æĪIJ +æĶ¿ æ²» +éĴĪ 对 +æŃ£ å¼ı +åıĸ å¾Ĺ +éĤ£ 个 +éĽĨ ä¸Ń +åıª èĥ½ +å¿« éĢŁ +身 ä½ĵ +åħļ åijĺ +èģĶ åIJĪ +åĬĽ éĩı +éĥ½ æľī +æ ħ§ +å¡ Ķ +åĪ« 人 +表 çİ° +æķħ äºĭ +ä¸Ģ åĪĩ +å° ĩ +èµĦ æĸĻ +åŁ¹ åħ» +éĺħ 读 +æľī 人 +èIJ¥ éĶĢ +çĽij çĿ£ +çݯ ä¿Ŀ +èĢĥ èĻij +æ·± åľ³ +严 éĩį +èĮĥ åĽ´ +å§Ķ åijĺ +çĽij 管 +ä¸ī 个 +è£ħ ä¿® +åħ¬ éĩĮ +åĪĨ åĪ« +çIJĨ 解 +éŁ © +åĬł å·¥ +认 羣 +ä¸į 好 +åİ» å¹´ +éĻį ä½İ +æľº ä¼ļ +åįı è®® +符 åIJĪ +å¢ŀ 强 +æĬĢ èĥ½ +é¦ĸ åħĪ +ç§ ¦ +ä¸ ģ +å° ¾ +æľī äºĨ +åľ° 产 +æ¸ ł +æĸ¹ 便 +移 åĬ¨ +éĢŁ 度 +å°¤ åħ¶ +éĢļ çŁ¥ +åĿ Ľ +éģ¿ åħį +æģ ¢ +è´ ¡ +èģĮ å·¥ +å®ŀ åĬĽ +æĺ¯ä¸Ģ ç§į +åIJ¯ åĬ¨ +çĸ¾ çĹħ +æĿ¥ äºĨ +缸 对 +çİ° å®ŀ +èŀį åIJĪ +åIJĮ æł· +åħ¬ åijĬ +çī¹ æ®Ĭ +ç´ « +ä¸ĭ åİ» +ä¼ł æĴŃ +æľĢ 好 +ä¼ĺ è´¨ +æ² Ĵ +æĮ º +æĹ ¦ +è¯ º +ä¸Ģ åIJį +éģĵ è·¯ +示 èĮĥ +è¿ĩ æĿ¥ +åIJĮ åѦ +é¼ ĵ +æĿ Ń +æľ¬ 次 +åIJĮ æĦı +ä¸ĸ 纪 +ç¾ Ĭ +æ¬ ² +å·¥ èīº +çĵ ¦ +人 士 +æľī æīĢ +ä»İ äºĭ +æľī å¾Īå¤ļ +ä¸į äºĨ +å²Ĺ ä½į +åıĺ å¾Ĺ +åĬ³ åĬ¨ +å¤Ħ äºİ +å¹³ åĿĩ +å½¢ 象 +å¡ ŀ +åħ± 享 +çĿ Ľ +åĪ© 润 +æŃ£ æĺ¯ +å¾Ģ å¾Ģ +缸 æ¯Ķ +æ¨ ª +åĪ · +æµĻ æ±Ł +大 éĥ¨åĪĨ +å¤ļ 个 +æĤ¨ çļĦ +çĶµ åķĨ +å¾® åįļ +å§ĭ ç»Ī +çĬ¯ 罪 +æĺ¯ åľ¨ +ç»Ħ åIJĪ +åİŁ æĿ¥ +æ¸ħ æ¥ļ +åIJĦ åľ° +æĦŁ åıĹ +å½ĵ ä¸Ń +è¶ĭ åĬ¿ +æĻ¯ åĮº +羣 æĺ¯ +ä¾Ľ åºĶ +转 åŀĭ +çĭ Ĥ +èĨ ľ +èĭ Ĺ +å¿ ł +å¾Ī 大 +èĤ¡ æĿĥ +ç¾İ åħĥ +æİĴ åIJį +åĬ¨ çī© +éĶ ħ +å¢ ¨ +主 å¸Ń +å¾Ī 好 +ç»Ŀ 对 +æĿ ľ +转 è½½ +çĴ ĥ +æĿij æ°ij +åIJ ¨ +åĽŃ åĮº +é«ĺ 度 +çī© è´¨ +è¾ ī +æĹ¥ 常 +æı Ĵ +ä¸ī å¹´ +ä½ĵ çİ° +æīį æĺ¯ +代 çIJĨ +ä¸į 管 +æģ Ĵ +åľ° ä½į +ç² ® +èĸ Ħ +æĺİ çĻ½ +ä¸Ģ èĩ´ +æĽ ¼ +åĵ Ń +åĩ ¤ +åĬ ² +æķ Į +æĪĺ æĸĹ +主 ä½ĵ +åħ¬ å¸ĥ +åıĤ èĢĥ +èĪª 空 +å¯ º +åѦ ä¼ļ +åıį æĺł +ç¾İ 丽 +太 éĺ³ +建 æĪIJ +æħ¢ æħ¢ +åIJĦ 个 +éĤ ¦ +ç»Ħ æĪIJ +ä¸ī 大 +éĶ ¦ +大å¤ļ æķ° +æ¦Ĥ 念 +éŃ Ĥ +åħ¬ çĽĬ +èį Ĵ +身 份 +æ·± åĪ» +åħ © +ç»ı åħ¸ +åIJĦ 项 +èĻ ķ +è¿Ľ æŃ¥ +åįģ äºĮ +æī§ æ³ķ +æĥ³ åĪ° +æĦŁ æŁĵ +åķĨ åĬ¡ +å°ı ç»Ħ +èĶ ¬ +çıŃ åŃIJ +åIJĮ å¿Ĺ +éĿ¢ 临 +çĤ Ĵ +å¤ļ ç§į +è§Ĥ çĤ¹ +åĵª éĩĮ +å° Ŀ +å§ Ĩ +èħ ¹ +åŁİ åĮº +太 å¤ļ +çĹħ æ¯Ĵ +åľ¨ äºİ +æīĢ è°ĵ +æĻ ° +æŀ Ŀ +æĭ ĸ +å® ħ +æķ´ æ²» +ä½ı æĪ¿ +åģ · +çĨ Ĭ +èµ ģ +æ° Ľ +æł¼ å±Ģ +åŁºç¡Ģ ä¸Ĭ +èĥ Ĩ +åħ ½ +鼶 åĶ® +åĿ ¡ +女 åŃ© +æĴ ŀ +åħ¨ åĬĽ +åĴ ĸ +èĤ © +çľ ī +èĩ³ äºİ +åħļ ç»Ħ +ä¸Ģ 件 +æĭ Ĩ +äºĭ å®ŀ +åĤ ³ +æ¹ ĺ +網 ç«Ļ +循 çݯ +åIJĮ æ¯Ķ +æĭ Ķ +åĮ» èį¯ +åħ» æ®ĸ +åĽº å®ļ +å®ŀéĻħ ä¸Ĭ +è®° å¾Ĺ +åĪ© äºİ +æĤ ¦ +æĭ ³ +èĤ Ŀ +æķĪ çĽĬ +è© ² +æ°ij 主 +çĹĩ çĬ¶ +é¢ ¨ +å¹¼ åĦ¿ +å§ ij +æĪ Ĵ +ä¸ĭ çļĦ +æ¸ ¡ +å¹´ åºķ +è®° å¿Ĩ +åIJ IJ +大 å¹ħ +å¾ ½ +åħ¬ ä¼Ĺ +ä¿¡ å¿ĥ +çİ Ľ +ä¼ļ ä¸Ĭ +ä¹ Ķ +æijĦ å½± +æ£ĭ çīĮ +éĻ ķ +åºĶ æĢ¥ +æĶ¶ è´¹ +æݧ èĤ¡ +仪 å¼ı +çŀ ¬ +æīĢ åľ¨ +ç¢ ° +å§ ĵ +é¡ Į +æĶ¯ éĥ¨ +使 åij½ +çĤ ī +å¯ Ħ +ç¿ ¼ +åľ° ä¸ĭ +è¾ ŀ +ä¿ ± +主 æĮģ +è´§ å¸ģ +æģ ¨ +èĤ Į +çĽ Ī +éĶ » +å¿Ĺ æĦ¿ +ç±» ä¼¼ +æĮ ĸ +éĢ » +ç¸ ½ +纪 念 +åķ ¥ +å¼ ¯ +åIJį åŃĹ +åģ¥ 身 +çļĦ å¿ĥ +é© ± +èĥĮ åIJİ +æ³ķ å¸Ī +ç² Ĵ +èĥ½ éĩı +è¾ ° +èī ³ +å½ ¼ +段 æŶéĹ´ +åIJĪ æ³ķ +æĵ ¦ +ç¾ ½ +åİ ¨ +æĪij 说 +äºĭ åĬ¡ +åĩł 天 +åħ ģ +ç¼ ´ +åį ĵ +两 ç§į +çĭ¬ çī¹ +å¸ ¶ +éĴ » +æĥ © +é¢Ĩ åħĪ +足 å¤Ł +å£ ³ +æĦıåij³ çĿĢ +åĪĨ å¸ĥ +ä¹ ĥ +éģ ĭ +ä½ © +è° ± +çģ £ +èį ¡ +è´¯ å½» +å¹ ¾ +ç£ ģ +åħ¸ åŀĭ +åī ĩ +åĨ » +æ¬ ł +ä¸į ä¹ħ +æµ ¦ +éŃ ħ +å¼Ģ äºĨ +使çĶ¨ èĢħ +è¿Ļ 款 +å° Ī +èĦ± è´« +æĶ» åĿļ +ç®Ĺ æĺ¯ +ç¨ Ģ +æĹł 人 +åł µ +å¥ ı +éĥ½ å¸Ĥ +åı¯ è§ģ +ä¸į åĩº +æ ·» +äº ı +ç¾İ 好 +èĥ ĸ +éŁ µ +æłĩ å¿Ĺ +èĬĤ èĥ½ +æĬ « +å° º +å¯ ¸ +ä¸Ģ 代 +é¢ Ĺ +èĢ ¶ +èĴ ¸ +åĸ ® +æ »¿ +çĮ ľ +æµ Ĩ +åŁ ĥ +åįĥ ä¸ĩ +èµ Į +èģ ² +ä½ľ é£İ +è³ ª +å¯ ¨ +å¹´ 人 +åį° 象 +æ¡ ¶ +æĴ ¤ +åįģ äºĶ +æ¯ ħ +æ² ª +åĽ½ æľī +大éĩı çļĦ +å¾ ¡ +å¯ ĵ +è¦ ĸ +æ¼Ĥ 亮 +çľ ł +ç ĤŃ +é» İ +èĻ ¹ +åĪ© äºļ +èŃ ī +æµ ı +åįģ åħ« +ä¸ ¢ +è¾ ½ +æľīä¸Ģ äºĽ +æħ Ī +åģľ 车 +å® ł +解 æĶ¾ +æľī å¤ļ +éĤ Ĭ +常 è§ģ +æĬ ¹ +çº ¤ +è¦ ª +æ¡ Ĩ +èİ ŀ +æ°§ åĮĸ +è¿Ļ 件 +åĩ ° +æŁ ´ +åıij çĶµ +é¼ ł +转 åĮĸ +å¨ ĥ +æĮ ¤ +ç½ © +å¯Ĩ åĪĩ +æĪij ä¸į +é«ĺ æĸ° +ä¸Ģ ç¯ĩ +è¿Ľ ç¨ĭ +è¡ ° +è¿ĺ ä¸į +ç ħĮ +æĸ° åįİ +èĤ ¿ +æ» © +ä¸Ģ æµģ +è¯ Ī +å®ŀ ä½ĵ +å¤ĸ åĽ½ +èº ² +èµ ł +è¦ º +æ¢ Ŀ +ä¸į è§ģ +è¨ Ĭ +åĮ ¹ +åį µ +çĩ ¥ +æħ ķ +é½ ¿ +å® ´ +é¥ ¼ +èij¡ èIJĦ +å°ı å¿ĥ +æģ ¼ +éĻ Į +æĺ Ĥ +åĥ ¹ +èĬ Ŀ +æ¯ı 个人 +åīį æıIJ +ä½ĵ ä¼ļ +æ¨ Ļ +æIJľ çĭIJ +对 åħ¶ +ä¸ § +èľ Ĥ +æµ ¸ +èª ¿ +åĿ ª +é¢ ĸ +åIJį 为 +ç¬ ¼ +èĪ Į +æľ¬ 书 +èģ ¯ +çº º +ç®Ģ 缴 +éĽ ¢ +ç¾İ çļĦ +éļ ¨ +é«ĺ å³° +è¿Ļ 家 +å Ĥ¬ +å° ¸ +ç¡ķ 士 +èŃ · +è° ¨ +æĺ ı +æĶ¿ åįı +è¡ Ķ +ç¿ Ĵ +åľ Ĵ +åĽ½ æ°ij +主 è§Ĵ +è£ ķ +ä¼ ª +åº ŀ +æ°ij èIJ¥ +æĥ § +ç§ĺ 书 +çĹ ķ +çĻ¾ åĪĨ +æº ¶ +æĹł çĸij +çļĦ çľ¼ +æĵ İ +ä¼Ł 大 +å½ ° +åħ¬å®ī å±Ģ +ç³ ķ +å¼ ¥ +åĤ Ļ +ä¹ ¾ +毫 ä¸į +注 æĺİ +åī¯ æĢ» +æĦ ī +æķ ¦ +é¦ ¨ +æĶ Ģ +éĢ Ŀ +åı¯ éĿł +å¤ ¸ +åľ ĺ +éĿ¢ ä¸Ĭ +æĬ ĸ +èĦ Ĩ +é© ° +ä¼ IJ +å¦ ¨ +å®ļ äºĨ +ç³ Ĭ +æŃ ¡ +éĥ¨ éķ¿ +ç§ ī +èĪ Ĩ +åĪij äºĭ +åIJ µ +æ¤ Ĵ +è¡ ĵ +è± « +èı © +åŃ µ +é¥ ² +å°± 好 +åł ª +ä¸ī è§Ĵ +åľº æ¯ĶèµĽ +ä¸į åģľ +æĵ ħ +åħ¨ æĸĩ +æ³ ģ +åѦ ä½į +æ± ° +éł ĺ +åı ł +éļ Ľ +å¸ IJ +çľĭ åĩº +åĮ ł +å±Ģ éĿ¢ +æ³ Į +è° Ĭ +åIJĮ æľŁ +æĬķ æłĩ +å¥ ´ +æĿ¥çľĭ çľĭ +èĦ ¾ +èŀ º +æŃ ī +çĽ ¯ +ç¨İ åĬ¡ +å» Ĭ +æİ © +æħ ¨ +çĽ ¼ +èĬ Ĵ +è® Ģ +æĮ £ +èĮ ħ +æĸ ¥ +æ¤ ħ +åĪ° æĿ¥ +èijĹ ä½ľ +çĭ ± +äºĮ æīĭ +ä»İ æĿ¥ +çĸ ² +åºĬ ä¸Ĭ +æĸ° 浪 +æ³ Ħ +å¢ŀ åĢ¼ +ä¸ Ľ +æļ ij +ä»İ ä¸ļ +æ· ĭ +å¤ļ æł· +æľ ´ +份 é¢Ŀ +æŀ £ +西 çľģ +æľ¬ è´¨ +æ·± æ·± +èī ĩ +ç» µ +产 åĢ¼ +æ¼ ł +èħ » +çŃ Ľ +åİ Į +æģ Ń +å«Į çĸij +æĪ ¶ +æ» ŀ +èĨ Ģ +åĬ £ +座 è°Ī +常 æĢģ +çļĦ æĥħ +è¦ ½ +å¯ Ĥ +åĮ Ĩ +èĩ º +é¡ ¯ +çķ ı +éģ £ +åį ľ +çŃī å¥ĸ +è² ¬ +æº ¯ +é İ +çĤ¹ 头 +èĵ ¬ +æ± º +éħ ¬ +éģ Ĭ +è³ ¼ +註 åĨĬ +æľ¬ æĬ¥ +çµ ķ +æ´» æĢ§ +åħ ij +éĮ ¯ +åĨ ¶ +åĸ » +æº ĸ +èĤ ¢ +æº ĥ +æĹ ¬ +åī Ĭ +çIJĨ äºĭ +å± ł +æ² § +èļ Ģ +鼻 åŃIJ +为 æŃ¢ +常 å§Ķ +çµ Ĥ +éĬ · +çĭ Ģ +ä¾ £ +èĥ Ģ +èŃ ° +çĶ¨ 车 +åĻ ª +æŃ · +åį Ķ +åĪ ¹ +ç«Ł æĺ¯ +é© Ĺ +èIJ Ŀ +çĻ « +çĹ « +æŃ § +å¼ Ĭ +åª ½ +çı Ĭ +è¡ · +éľ ī +åŁº çĿ£ +éļ ± +æ° ¨ +ç» ¸ +å°¼ æĸ¯ +çĥ ĺ +æľŁ åĨħ +è° ħ +éĽ ĩ +éļ Ļ +å ĸī +åī ¥ +çĹ ĺ +æĮ ½ +çĵ £ +æ¹ Ľ +æ¨ ± +æ¾ İ +æ¹ ĥ +åĨ¬ 奥 +æ£ µ +å® ° +åŀ Ĵ +æ§ ĭ +ä¾ Ī +èĮ Ħ +åĺ ¿ +èı ĩ +ç ĻĤ +åĬ ĥ +é į +èĶ ½ +çŀ Ń +æķ ŀ +ä¹ ĸ +éŁ § +è¾ ľ +æĩ Ī +ä½ £ +çŀ » +åŁ Ķ +èĪ ħ +å®ŀ äºĭ +é ¨ +å§ ¥ +çµ ¡ +åĺ » +çķ ¢ +æ²ĥ å°Ķ +è¿ Ħ +èĤ ĩ +æħ ij +ã § +ä ı +ð ł +ð¬ ĩ +ð« Ń +ð« IJ +ã ³ +© ½ +ð« ł +ã Ľ +ð¬ į +é ¿ +ð¬ Ĵ +ã Ļ +𬠤 +ð ¬´ +ð« ĸ +ð ¤ +ã ¬ +ä ² +ð« Ķ +ð« ļ +è¦ģ æ±Ĥ +ä¸Ģ äºĽ +å®ŀ çİ° +èĢĮ ä¸Ķ +åĽł æѤ +çĶ± äºİ +åħ³ äºİ +çĦ¶ åIJİ +æݨ åĬ¨ +ä¸Ģ æł· +æĮī çħ§ +è¿Ļæł· çļĦ +å½¢ æĪIJ +æľī äºĽ +æĽ´ åĬł +ç»ı è¿ĩ +建 è®® +æ²» çĸĹ +ä½ł 们 +æīį èĥ½ +ä¿ĥ è¿Ľ +åijĺ å·¥ +ä½ĵ éªĮ +èĪ ĩ +åģļ 好 +ä¿Ŀ è¯ģ +æķ´ 个 +æĺ¯ ä¸Ģ个 +éĩĩ çĶ¨ +çIJĨ 论 +æ¯Ķ å¦Ĥ +ä¸Ĭ çļĦ +æݨ èįIJ +çĶ³ 请 +天 空 +éĥ¨ èIJ½ +åįģ åĪĨ +æĿ¥ èĩª +ä¹ĭ éĹ´ +è°ĥ æķ´ +æ¯ı 天 +è°ĥ æŁ¥ +æĤ£ èĢħ +è¿ĩç¨ĭ ä¸Ń +é¦Ļ 港 +广 åijĬ +éĿ¢ 对 +满 足 +éķ¿ æľŁ +è§Ħ èĮĥ +æķ´ ä½ĵ +æĶ¹ åıĺ +æĻº æħ§ +å¦Ī å¦Ī +å¦Ĥ ä»Ĭ +åIJĪ åIJĮ +éĥ½ ä¼ļ +åĦ¿ ç«¥ +åĩı å°ij +éŁ³ ä¹IJ +ç»ı 常 +ä¸Ĭ å¸Ĥ +ä¼ĺ ç§Ģ +çļĦ éĩįè¦ģ +ä¸Ģ æĿ¡ +æµ· å¤ĸ +åı¦ å¤ĸ +ä¸Ģ 家 +åİĭ åĬĽ +大 åŀĭ +çľĭ çĿĢ +åĪ Ģ +幸 ç¦ı +æݨ 广 +åIJ Ľ +å¾ IJ +æī¾ åĪ° +äºİ æĺ¯ +èĩª 身 +ä¸Ģ ä½į +åľŁ åľ° +åĬł åħ¥ +æİ¢ ç´¢ +æ¢ ģ +主 åĬ¨ +å°± ä¸ļ +女 æĢ§ +çªģ çł´ +ä¸įåIJĮ çļĦ +è¿IJ è¾ĵ +èĩª çĶ± +å±ħ æ°ij +æѤ 次 +çļĦ æŶéĹ´ +家 éķ¿ +ä¸Ģ个 人 +æ£Ģ æµĭ +åĨħ éĥ¨ +广 å·ŀ +缴 æĴŃ +ä»İ èĢĮ +è´· 款 +åı¬ å¼Ģ +æĶ¹ éĢł +人 çĶŁ +å±ķ 示 +æ¯ı å¹´ +女 人 +çļĦ æĸ¹å¼ı +æķĪ çİĩ +å±± 举 +æ¸ł éģĵ +ä¼¼ ä¹İ +æ¡Ī 件 +åĪ© çĽĬ +çľĭ çľĭ +å¿ĥ éĩĮ +ç»´ æĬ¤ +å®Ŀ å®Ŀ +ç½ij ä¸Ĭ +论 åĿĽ +å°± åı¯ä»¥ +ä¸į 足 +æģ¢ å¤į +å¸ĥ å±Ģ +è´¡ çĮ® +ä¸ĭ éĻį +æİĮ æı¡ +çļ® èĤ¤ +å·¥ åħ· +éĩį åºĨ +åĵģ è´¨ +æݨ åĩº +çĶ· 人 +æī¿ æĭħ +çªģ åĩº +èĢĮ è¨Ģ +æ² Ł +åįı è°ĥ +æĺ¯ ä»Ģä¹Ī +æ± ¤ +æĴ ij +çĭ¬ ç«ĭ +çݯ èĬĤ +æī© 大 +æ´ ª +æĿ ° +çĽ IJ +ä» ģ +æ¶ī åıĬ +èĢģ 人 +åį³ 使 +åįĹ äº¬ +éħį åIJĪ +é¬ ¼ +çĪ¶ 亲 +ç½Ĺ æĸ¯ +å°ı åĮº +æķĻ æİĪ +åĨ³ çŃĸ +é¢Ħ 计 +æľ¬ 人 +ä¼ ¯ +ç« ¹ +åĪ° åºķ +å¸Ĥ æ°ij +åĩº åı£ +éĩĩ è´Ń +æĢ» ç»ĵ +æѦ æ±ī +åĬł 大 +广 举 +æµģ ç¨ĭ +人 åı£ +å¦Ĥæŀľ ä½ł +åĩº åİ» +åĩ ī +åĨľ æ°ij +çİ° 象 +åĬĽ 度 +ç»Ļ äºĪ +åħļ å§Ķ +è¯Ń è¨Ģ +线 ä¸Ĭ +æĢİ æł· +åĦ¿ åŃIJ +ç¡® å®ŀ +ä¹ĭ å¤ĸ +éĥ½ åľ¨ +èī ¾ +çļĦ æĥħåĨµ +éĩĮ çļĦ +åĽ´ ç»ķ +æĽ´å¤ļ çļĦ +ä¾Ŀ æ³ķ +åħ¬ åĽŃ +家 éĩĮ +æ¯į 亲 +ä¸į åĨį +èĭ ¹ +æ³ķ éĻ¢ +飩 åĽ½ +缸 å½ĵ +ä¸į çŁ¥ +è¯Ħ ä¼° +ä¸į çĶ¨ +顺 åĪ© +éĩį è§Ĩ +è´¢ åĬ¡ +ä»ĸ åĢij +åıij è¡Į +ä¸ĵ éŨ +åħ· å¤ĩ +并 ä¸įæĺ¯ +足 çIJĥ +é ŀĭ +åıij 表 +æ°¸ è¿ľ +èIJ¥ åħ» +éħį å¥Ĺ +æķ´ åIJĪ +è´ º +åĽŀ çŃĶ +æĶ¶ çĽĬ +ä¹Ł 许 +è» Ĭ +æİ¥ 触 +æĶ» åĩ» +åĽĽ å·Ŀ +æĢ§ èĥ½ +åĽŀ åĪ° +èħ ° +ä¹Ł 没æľī +å¼ Ħ +设 ç«ĭ +éĺ² æݧ +æĬĢ å·§ +éĢļ 常 +è´¢ æĶ¿ +éĥ¨ ç½² +åľº æĻ¯ +æ±Ł èĭı +表 è¾¾ +åĸ · +女 åĦ¿ +èĪ ¶ +çµ ¦ +ä¼ļ åijĺ +æĪĸ 许 +äº © +举 æĸ¹ +天 æ´¥ +è¿ij å¹´ +çľĭ æĿ¥ +æ¯Ķ ä¾ĭ +å² © +éĵ ľ +çİ » +å®ŀ éªĮ +æĢĿ ç»´ +æĭħ å¿ĥ +æ² Ī +身 è¾¹ +æ·± åĮĸ +ç²¾ åĩĨ +ç§ģ æľį +æ¶Ī éĺ² +åİ» äºĨ +ç»Ĩ èĥŀ +çIJĥ éĺŁ +æĺİ æĺŁ +é£Ł çī© +å¾Ī å¿« +让 ä½ł +ä¿¡ çĶ¨ +åĶ¯ ä¸Ģ +åħ¶ å®ĥ +çŃī æĸ¹éĿ¢ +å¾ĭ å¸Ī +æŃ» 亡 +æ Ł³ +ä¸Ģ æī¹ +ä¸Ĭ 涨 +æľº åľº +å½¢ åĬ¿ +æĦ¿ æĦı +éĽĨ ä½ĵ +æĸ° åŀĭ +æįŁ 失 +æĽ ¸ +ä¸ĭ åįĪ +æ¯ı 次 +æĪIJ å°± +åħ¬ è·¯ +èĻ « +åĴ ± +西 å®ī +æľĢ ä½³ +ç§ij çłĶ +å¤į æĿĤ +æľº åĻ¨ +çĪ± æĥħ +çħ§ çīĩ +å¹´ é¾Ħ +è³ĩ æĸĻ +ç² Ĺ +åĩĨ ç¡® +åĬł ä¸Ĭ +åĩº çīĪ +è° IJ +家 å±ħ +èĥĮ æĻ¯ +ä¸Ģ 线 +äºĭ 项 +åĬ¨ ä½ľ +ç¥ ¥ +æĢ» ä½ĵ +æĪ¿ åŃIJ +ä¹Ł å°±æĺ¯ +大 æ¦Ĥ +é«ĺ æķĪ +åIJ ¹ +æİ ĪæĿĥ +éĻĦ è¿ij +æ¡Ī ä¾ĭ +éĹ ¹ +çĪ¸ çĪ¸ +彩 票 +æĢ Ĵ +举 æĬ¥ +æĻ® éģį +çķĻ ä¸ĭ +è¡£ æľį +æĹłè®º æĺ¯ +åħħ 满 +æ·± 度 +æ¡ ij +æĪª èĩ³ +带æĿ¥ çļĦ +éĻ µ +æĦŁ æĥħ +èµ ļ +åĵª äºĽ +æķ´ æĶ¹ +æĪIJ çĨŁ +å¨ ľ +é¼ » +çŁ Ľ +çĽ ¾ +好 好 +第 åĽĽ +åĨł åĨĽ +è´¢ å¯Į +æľĢ 好çļĦ +车 åŀĭ +éĸ Ģ +åį³ å°Ĩ +åĪĨ 为 +éĿĴ å²Ľ +纷 纷 +ä»Ĭ æĹ¥ +å¹³ è¡¡ +å¹³æĸ¹ ç±³ +éĤ£ ç§į +åĩº çĶŁ +éĿĴ æĺ¥ +人 群 +人 å·¥ +ä¹ĭ ä¸ĭ +æ¹ĸ åĮĹ +åľ¨ æѤ +åįļ 士 +æŶ åĪ» +æ²³ åĮĹ +æĶ¾ å¼ĥ +éĢļ éģĵ +森 æŀĹ +çĸ Ĩ +æķ ¸ +èĬ ³ +æīĵ åĩ» +æĽ ¹ +åĮĸ åѦ +æĥ³ 象 +ä¸ĩ 人 +è´¢ ç»ı +åħĥ ç´ł +ä¼ļ 计 +åħ¨ ä½ĵ +æĦ Ľ +é«ĺ ä¸Ń +æľº éģĩ +声 éŁ³ +æĹħ è¡Į +æµ © +æŁ ± +å°ij å¹´ +åĽ½ å¤ĸ +èijĹ åIJį +çĶŁ åŃĺ +å§ ľ +带 é¢Ĩ +é¢ľ èī² +ä¸Ĭ ä¸ĭ +产ä¸ļ éĵ¾ +æĽ´ 好çļĦ +å² Ń +ä¼ĺ æĥł +便 æĺ¯ +åħ§ 容 +ä¸Ģ åıª +çIJ ´ +梦 æĥ³ +秣 èµģ +å¼Ģ åIJ¯ +è´Ń çī© +åĮħ åIJ« +åĪ© çİĩ +èµ· äºĨ +æľī åĬĽ +éĤ£ éĩĮ +审 æī¹ +对 æīĭ +çİ° éĩij +天 çĦ¶ +çĽ Ĵ +çĪ ½ +å¿ħ çĦ¶ +åĮĸ å·¥ +ä¸ĵ åĪ© +åķ ¡ +å¼Ģ å¿ĥ +人 ä½ĵ +éģĵ 士 +æĢģ 度 +空 è°ĥ +æĭĽ åķĨ +å§ » +第 äºĶ +æ£ Ĵ +ä¸Ģ ç³»åĪĹ +åį± æľº +转 åıĺ +åľº æīĢ +é¸ £ +æĪ¿ éĹ´ +éĢ ¼ +è¯ķ çĤ¹ +对 å¤ĸ +åĩº åı° +åľ¨ è¿Ļ +åİĤ 家 +å·¨ 大 +ç®Ģ ä»ĭ +çľĭ äºĨ +åħļ 建 +æĮĩ æĮ¥ +çŁ³ æ²¹ +ä¸į åı¯èĥ½ +èİ ² +ä¸į 太 +åĪĽ æĦı +第 ä¸Ģ个 +è´µ å·ŀ +è¿ĩ äºĨ +æľ¬ æĿ¥ +éģĵ å¾· +çŃĶ æ¡Ī +éĻ ¶ +ä¸Ģ è·¯ +èĤ ĸ +æ¸ħ æ´ģ +æľī æľº +åIJį åįķ +æĿ ± +åij¼ åIJ¸ +ä¸ Ī +ç¦ı 建 +è¯ķ éªĮ +å¼ķ åıij +ä¹Ł 没 +ä¸į ä½ı +çĨŁ æĤī +èIJ ¬ +ä¸į èī¯ +çł ĸ +èĩ´ åĬĽ +çŃ¾ 订 +åIJ Ĭ +ä¾ ¯ +çĺ ¦ +å§ij å¨ĺ +æĸ ¤ +妻 åŃIJ +æĺ¥ èĬĤ +çĪ ¬ +æĽ Ŀ +çĥŃ æĥħ +éķ¿ æ²Ļ +èIJ¥ éĢł +éħ · +éĵ Ŀ +åŁºæľ¬ ä¸Ĭ +åij¨ åĽ´ +ä»Ģ 麼 +认 åı¯ +åĪĨ åŃIJ +ä¸Ģ æĸ¹éĿ¢ +è½ ´ +å¼ · +马 ä¸Ĭ +éĽ ¾ +èĩ £ +å° ¿ +çĶŁ æĦı +å®ī å¾½ +ç¥ŀ ç»ı +åĩº å¸Ń +èį¯ åĵģ +çIJĨ çĶ± +åįı åIJĮ +æµģ åĬ¨ +åıij åĬ¨ +åĿļ å®ļ +表 æĺİ +åIJİ éĿ¢ +ä¹ī åĬ¡ +å¦ ĸ +æľī åı¯èĥ½ +å¹´è½» 人 +大 éĻĨ +å² ³ +ä¸į èµ· +çŀ¬ éĹ´ +ä¸įå¾Ĺ ä¸į +çŃ¾ 约 +åIJĪ æł¼ +åħļ æĶ¯éĥ¨ +æµİ åįĹ +便 åĪ© +éļı æŶ +å¥ ī +称 为 +产 æĿĥ +åIJ ķ +çĽ Ĩ +课 åłĤ +ç· ļ +æ£ ī +线 ä¸ĭ +èĩª è¡Į +举 æݪ +åݦ éŨ +èĩª ä¿¡ +å½± è§Ĩ +ä» Ķ +çĶŁæ´» ä¸Ń +æĿĥ çĽĬ +çĻ½ èī² +å°± ä¸į +è¿Ľ å±ķ +æ¯ı æĹ¥ +ä¾Ľ ç»Ļ +æĿĥ åĪ© +æĹł æķ° +çIJĨ è´¢ +ä¾Ŀ æŧ +ä¸Ĭ åįĪ +è¯Ĩ åĪ« +çĽĪ åĪ© +çł Ĥ +许 åı¯ +åIJĮ äºĭ +åĺ Ľ +éģ ¸ +çĿĢ åĬĽ +éŨ åı£ +ä¸į å¤ļ +åħ¶ 次 +ç¢ § +çī© çIJĨ +åĨħ å¿ĥ +çĻ¾ å§ĵ +æĢ» 绣 +å¹² åĩĢ +积 ç´¯ +åıį é¦Ī +æłij ç«ĭ +社 交 +ç§ © +åįģ ä¸Ģ +éĤ ĵ +驱 åĬ¨ +å±ķ è§Ī +èĪĴ éĢĤ +åŁº åĽł +å·® å¼Ĥ +转 让 +å°ı å§IJ +æł· åŃIJ +ç¿ Ķ +é«ĺ åħ´ +å½±åĵį åĬĽ +æīĭ ç»Ń +缸 åIJĮ +缸 åºĶ +æĻ Ĵ +è§ Ģ +å¸Ĥ å§Ķ +èĬ ¯ +å±ķ çİ° +åľ° çIJĥ +éĤ ª +ä¸Ģå®ļ çļĦ +åħģ 许 +ä¿¡ ä»» +æī ij +éĻ¢ æł¡ +ç®Ģ 称 +åģļ æ³ķ +ä¹ĭ è·¯ +æĹĹ ä¸ĭ +èħ Ķ +æ¶Ī 失 +ä¸ĸçķĮ ä¸Ĭ +åŁİ 乡 +èĪŀ åı° +å¾Ī 大çļĦ +绣 çŃ¹ +åħ¬ å¹³ +èĤ ¾ +çļĦ 好 +æ± ģ +çľ¼ åīį +éĽ £ +å¹ ½ +åħ± 产 +主 åĬŀ +å¤Ħ ç½ļ +åº Ļ +éģĵ çIJĨ +å¼ µ +æİ¥ çĿĢ +çĮ İ +çģ Į +çĶ± æѤ +人 åĬĽ +æµģ è¡Į +ä¾ ł +åı¯ä»¥ 说 +èĴ ĭ +å½¢ æĢģ +æĹ¥ åŃIJ +æ¼ Ĩ +çķĻ åѦ +缸 éĹľ +æľĢ å¤ļ +åĩŃ åĢŁ +åħ¬ 交 +æĮĸ æİĺ +æĿĤ å¿Ĺ +主 人 +éļľ ç¢į +æł¡ éķ¿ +æĸ¹ ä½į +ä¸Ĭ çıŃ +å¤ļ åħĥ +è ĥģ +éŃħ åĬĽ +èĮ Ĥ +åħħ çĶµ +强 大 +çĥ ¤ +å¥ĭ æĸĹ +å®ŀ çĶ¨ +éĺ ģ +ç»Ļ äºĨ +æľ¬ ç§ij +æł ĭ +æĭ ¨ +æķĻ ç»ĥ +éĥ½ çŁ¥éģĵ +æ¯ķä¸ļ çĶŁ +ç¢ Ĺ +åŀ Ĥ +è® ¼ +å®ģ æ³¢ +åѦ èĢħ +è°¢ è°¢ +åŁİ éķĩ +æĢİä¹Ī åĬŀ +éģ Ķ +æĪIJ 交 +æ½ľ åĬĽ +åį § +æĸ° å¼Ģ +éħį å¤ĩ +主 åĬĽ +åij³ éģĵ +çĥ Ĥ +é£ŀ è¡Į +å« ģ +大 大 +ç»Ļ 大家 +å¤ĸ éĿ¢ +éĨ ī +åıij è¨Ģ +æĹ© é¤IJ +åIJĦ èĩª +å® Ļ +èį£ èªī +æĬ« éľ² +é¡ ŀ +åĨħ çļĦ +èĤ ª +è¾ IJ +æ³ µ +æĬ Ľ +æĺŁ æľŁ +ä¸Ģ 带 +çĶŁ ç´ł +ç»ı éĶĢ +åĩ ¶ +åľ° ä¸Ĭ +åij½ è¿IJ +åĵ ² +ä¸Ĭ åİ» +æĸĩ çī© +è¯ ij +æĮ¯ åħ´ +éķ¿ æŶéĹ´ +ç¥ Ń +åIJĪ èĤ¥ +è¿Ŀ è§Ħ +èģ ª +ä½İ äºİ +éĢĤ å½ĵ +æľī åºı +æľ¬ ç½ij +çķĻ è¨Ģ +æĥ³ æ³ķ +çŃ¾ ç½² +å§ ļ +æĢ§ æł¼ +èĴĻ åı¤ +æŁ ı +åŀ « +åѦ åİĨ +ä»ħ ä»ħ +讲 è¯Ŀ +éĶ IJ +æĢ ĸ +åī ª +èĭ į +åIJ ĵ +强 çĥĪ +åģ¥ åħ¨ +çĸ ¯ +åı¤ 代 +å¥ Ī +ä¸į çĦ¶ +乡 éķĩ +æľĭåıĭ 们 +åĤ ħ +èģ ½ +个 æĢ§ +æ³ķ è§Ħ +å°ı éķĩ +çĶ» éĿ¢ +第 åħŃ +網 è·¯ +åīį æĻ¯ +åIJ¬ 说 +ä¼ł åªĴ +æĿ¡ ä¾ĭ +åĪ« çļĦ +ä¸į æĩĤ +顾 éĹ® +强 度 +éĺ¿ éĩĮ +èµ° åĬ¿ +å¸ ½ +çļĦ ç¡® +åĮº åĪ« +éĮ ¢ +主 管 +ä¸Ģ çľĭ +æĸ ľ +åŃĺåľ¨ çļĦ +ä» ² +åį± 害 +éĵ Ń +游æĪı ä¸Ń +éħ ± +é¾Ļ 头 +人 å¿ĥ +éĢĢ ä¼ij +æµı è§Ī +åĬ « +éĺ² æ²» +ç® Ń +å± Ī +è¾½ å®ģ +å£ ¤ +è¿İ æĿ¥ +éŀ į +çĶ¨ æĿ¥ +大 åľ° +ä» ° +éĢļ 讯 +å¼Ģ å·¥ +è£ ¤ +å¦Ĥ åIJĮ +éª ¤ +éĺŁ åijĺ +è½ © +ç¾İ æľ¯ +èĻ Ł +åIJĮ ä¸Ģ +åľ ĸ +书 æ³ķ +æīĵ åį° +åIJ« æľī +éĽĨ æĪIJ +éĹ · +å¸Ĥåľº ä¸Ĭ +æĹģ è¾¹ +åľ° æĿ¿ +产çĶŁ çļĦ +ç² ¤ +éĩį ç»Ħ +è¡Ģ 液 +çŃ ĭ +åĬŀ äºĭ +常è§ģ çļĦ +ä¸Ĭ åįĬå¹´ +å±ı å¹ķ +åIJī æŀĹ +å· © +åĸľ çĪ± +ç¿ ł +ä¸ī ç§į +æ¡Ĩ æŀ¶ +举 èİŀ +çĶĺ èĤĥ +èĬ ¬ +åĽ¾ 书 +åĩ¤ åĩ° +æ°Ķ åĢĻ +å° ´ +å° ¬ +两 天 +è¾ħ 导 +åĢŁ 款 +æĹ¥ èµ· +æ´ Ĵ +ä¸Ģ 度 +è¹ Ī +æ½ Ń +æī ĩ +çĻ ľ +æĸ° åħ´ +åĤ ² +诸 å¤ļ +è´ ª +éĻ· åħ¥ +èĪ Ł +èĤº çĤİ +ä¸Ģ æł·çļĦ +åİ ĺ +åľ° çIJĨ +æĬķ 注 +éļ Ĭ +åħī ä¼ı +ä¿Ŀ åģ¥ +åħ Ķ +åħ¬ åĬ¡ +æīĵ çł´ +çĶ· åŃ© +åĬ³ åĬ¡ +ä½ł ä¼ļ +çĶ¨ åľ° +æº ¢ +åıij è¾¾ +èĤ ļ +è¿ĩ äºİ +èĩ Ĥ +éĢĻ 樣 +è½» è½» +ä¸Ń åħ± +åIJĦ åĽ½ +åĶ ĩ +å®ŀ ä¹ł +èĻ ¾ +æ§ ½ +ä¸į ä¸Ĭ +åħį çĸ« +åįł æį® +å·¥ ä¼ļ +åĽ Ĭ +èĪª 天 +åı¯ çĪ± +æĸĹ äºī +çĺ ¤ +å¦Ĥ æľī +éĽ ĸ +对 æĪij +åĩº 秣 +好 çľĭ +太 大 +æ°´ åĪ© +åĬ¿ åĬĽ +åħ¨ æ°ij +ç½ ¢ +èµ¢ å¾Ĺ +çĶµ ä¿¡ +车 éĹ´ +æĻĤ åĢĻ +å°ij æķ° +éĵ ¸ +åħ³ èģĶ +ä¸įä»ħ ä»ħ +为 æĤ¨ +åĴ ¸ +æľº åĬ¨ +è£ Ļ +åĵį åºĶ +éģ ł +è² · +ç© ´ +å¢ ħ +éĶ ¡ +çµ Ħ +çģ« 车 +è³ĩ è¨Ĭ +åĨ³ èµĽ +污 æ°´ +èª ŀ +å´ Ľ +ç´§ å¯Ĩ +缺 å°ij +å¤ļ 人 +æĢ» 书记 +éĶ Ī +èij Ľ +å¿ĺ è®° +éĻĮ çĶŁ +éķ¿ 大 +åħĪè¿Ľ çļĦ +ç¡ ħ +åıij æĺİ +å©´ åĦ¿ +æīİ å®ŀ +èĽĭ çĻ½ +ä¸Ģ çĻ¾ +缮 åħī +æ ħĮ +åĬł æ²¹ +åIJ ŀ +ä¸Ģ 群 +ä¸Ń ä»ĭ +å¸ ĸ +å¿ Į +èģĮ èĥ½ +广 æĴŃ +çĽij å¯Ł +ç§ĺ å¯Ĩ +çĭ ® +è¿Ļ æĿ¡ +éĢ ¢ +æĢ ¨ +åįģ åħŃ +è© ¦ +说 åĪ° +åĩĿ èģļ +æĮĩ 示 +æ° ¢ +å¼ ĺ +éĺ Ģ +æĸ © +éł ħ +ä¸Ģ å¼Ģå§ĭ +æİĴ è¡Į +åľ¨ æĪij +纪 å½ķ +æĬ Ħ +æł ª +说 æ³ķ +ä¸Ń èį¯ +好 å¤ļ +åıª ä¸įè¿ĩ +çķĻ åľ¨ +个 å°ıæŶ +认 çŁ¥ +çķ « +è§ģ è¿ĩ +å°ı å¾® +ä½Ľ å±± +çľ ¾ +讲 è¿° +æ¢ ³ +称 åı· +æĹ¥ æĻļ +è¢ ĸ +åķ ¤ +æľª ç»ı +æľĢ æĹ© +æī® æ¼Ķ +è¡Ģ 管 +çº ± +æĥħ èĬĤ +第 ä¸ĥ +æį § +ä» Ĺ +æ¿Ģ çĥĪ +æĹł 线 +ä¸į 容æĺĵ +å¼Ģ å¹ķ +æĸ° çĶŁ +ä¸ĵ 注 +èij ± +åįĹ æµ· +çĩ Ł +èµ· ä¾Ĩ +æ´¾ åĩº +åĦ Ĵ +ä¾ ¨ +è¼ ĥ +åįļ è§Ī +éĢ ¾ +åĮ Ģ +ç»ıæµİ åѦ +æ¸ Ĺ +ä¿Ŀ èŃ· +çī º +çī ² +çİ « +çij ° +æľĢåIJİ ä¸Ģ +æĶ¿ åĬ¡ +æ§ Ľ +èĻķ çIJĨ +éļIJ æĤ£ +æī¿ åĮħ +æ¥ µ +æ¡ © +çĽ ² +导 åIJij +èĩ´ å¯Į +ç¼ Ĩ +æģĭ çĪ± +ä¸į åĬ¨ +ç»Ļ 人 +å· ¢ +表 æĥħ +举 åįĹ +åĨħ å¤ĸ +è¾Ī åŃIJ +åı ī +åįļ ä¼ļ +åĬŁ æķĪ +æ¸ ´ +å± ¬ +æİĴ éĻ¤ +éĢ Ľ +ä¸Ģ ä¼ļ +ä¸į å¼Ģ +å¼Ģ å¥ĸ +é»ij é¾Ļ +é»ijé¾Ļ æ±Ł +å¿« ä¸ī +度 åģĩ +åĿ ¤ +éĤ® 件 +æĩ Ĵ +ä¾Ľ çĶµ +å» £ +好 è¯Ħ +ç§ĺ书 éķ¿ +æĪĺ åľº +好 å¥ĩ +ä¾µ æĿĥ +æĨ ¾ +æľĢ åĪĿ +æī¹ åıij +åİ ķ +è¼ ķ +æŀ ¯ +ä¸ļ åĨħ +è´Ń æĪ¿ +ä¸į åľ¨ +纪 å§Ķ +æīĢ éľĢ +å¸Ĥ éķ¿ +è³ ½ +å¼ķ æĵİ +çģµ éŃĤ +éĬ Ģ +æ» ¤ +çĿ IJ +å¤ļ 项 +åĽŀ 头 +èī ĺ +å¤į å·¥ +éĥ¨ 件 +ç´§ ç´§ +æŁIJ ç§į +使 åħ¶ +æĸ° 人 +æŀ ļ +æ³ķ å®ļ +å·´ å·´ +涵 çĽĸ +ç¨ » +æĭ ¾ +æĻ ķ +è½ ¿ +éĢļ è¡Į +åĵ Ģ +æ³ Ĭ +温 馨 +éĽĨ èģļ +çĨ Ļ +åĩ ij +åįģ ä¸ĥ +æ°Ķ æģ¯ +æıIJä¾Ľ çļĦ +æ³ ³ +奥 è¿IJ +çģ¾ 害 +åĩĢ åĮĸ +è·¨ è¶Ĭ +åĵª æĢķ +éŁ ¿ +å¢ŀ æ·» +çĦ Ĭ +æ®ĭ çĸ¾ +ç¢ Į +æĤ Ķ +è§ģ è¯ģ +è¾ĸ åĮº +å¿ĥ èĦı +éļ § +åį ¸ +åı¯èĥ½ æĢ§ +æľī 趣 +åī¯ 书记 +åĮĸ å¦Ĩ +ä¿ Ĥ +æ£ ļ +éĨ ĩ +带 头 +éł Ī +追 究 +æij Ķ +è¿Ļ éĥ¨ +ä¸į 论 +ç¥ ¸ +å ³» +éģ ķ +çĶŁ èĤ² +å¤ ł +å¤ĸ 交 +è¯Ħ 为 +ä»İ å°ı +å°ı å°ı +é ¥¿ +æĴ ¼ +è·¨ å¢ĥ +被 åijĬ +åįĹ å®ģ +身 å¿ĥ +åĨį çĶŁ +æīĢ 说 +æŶéĹ´ åĨħ +åĪĹ åħ¥ +éĿĴ æµ· +çĪ± 好 +çª Ħ +èĪ Ī +è¿ĩ 渡 +æ¿ Ł +éĽ Ģ +审 è®® +åĽ½ èµĦ +æŃ¥ ä¼IJ +轨 éģĵ +ä¿¡ 念 +ä¸ī åĪĨ +çĨ ¬ +åѵ åĮĸ +ç¼ ł +éĥ Ĭ +èĪĴ æľį +纪 æ£Ģ +ä¸Ģä¸ĭ åŃIJ +鼻 話 +è² ł +éĴ ¥ +åĮ Ļ +çĹ ´ +è¶ ģ +ç» £ +çĪ µ +è½ ° +éª Ħ +å§ ¨ +æĭ ĺ +çĮ ´ +è® ¶ +è¿Ļ 座 +çį ¨ +æ·ĺ æ±° +çĹħ ä¾ĭ +æ²Ļ åıij +è§Ĩ 为 +头 æĿ¡ +å¿ħè¦ģ çļĦ +åı¯ è°ĵ +è¯Ŀ 说 +ç¯ Ħ +æĹ© çĤ¹ +æŀ¢ 纽 +ç¾ ¡ +çĪ± åĽ½ +çªģ åıij +éĢ Ĭ +æ½ į +èį£ èĢĢ +èŁ ¹ +æ¦Ĥ çİĩ +å¾Ī ä¹ħ +æĥ ķ +è¨ ´ +åľĨ 满 +çļ ± +åĪĨ æ³Į +åħħ 足 +çľĭ æ³ķ +è¾ Ł +æĭ ¦ +æĭ © +对 åºĶ +为 æł¸å¿ĥ +èħ Ĭ +å¤ļ ä¹Ī +æµ ij +å®ı è§Ĥ +èĦ ĸ +åIJĪ èµĦ +çĶŁ 涯 +å®ŀ è´¨ +ä¼ĺ çĤ¹ +çĶ¨ æ°´ +寿 åij½ +æ² « +åIJ ģ +è© ¹ +åĽ½ éĺ² +å´ © +åĿ İ +èĨ ı +ä¸Ģ è½® +éģĹ äº§ +æ¹¾ åĮº +ç» İ +åįķ 纯 +æ¾ Ħ +åīį åĪĹ +身 å½± +é»ĺ é»ĺ +æį ī +çĴ ° +èı Ĭ +æĢ ľ +åħĭ æĢĿ +æĢ» å±Ģ +çĩĥ æĸĻ +ä¸ļ æĢģ +åIJĦ æł· +åĴ ½ +åĩº èī² +åĪĿ å¿ĥ +åı Ľ +çłĶ 讨 +è¡ « +åİĨ ç¨ĭ +ç¦ ½ +è¶³å¤Ł çļĦ +èį Ĩ +çľĭ å¾ħ +è´ © +åĨ³ å¿ĥ +è£ ¹ +å¸Ī èĮĥ +åŀ Ħ +æĿ ł +åĩ ¸ +çĬ¹ 豫 +çĥŃ è¡Ģ +åIJĪ ä¼Ļ +éħ µ +èIJ½ åľ¨ +åįł åľ° +è¡ ¬ +èĵ ī +æĦ ¤ +æ¸ Ĭ +åĪĨ æķ° +ç¬ij çĿĢ +太 å¹³ +çĤ « +æݨ ä»ĭ +æĸ¯ åĿ¦ +å½¢ 容 +æĵ Ĭ +æĦŁ åħ´è¶£ +åĨĽ 人 +åĩĮ æĻ¨ +对 çħ§ +åıij çĹħ +å· ¾ +èĪ ī +æª ¢ +ç¬ij äºĨ +ç¡® è¯Ĭ +è´Ł åĢº +壮 大 +æĪ ļ +äºĴ èģĶ +èª ² +èħ ¦ +æĹ ± +åıĹ æ¬¢è¿İ +åį ī +éĻ¢ 士 +æ© ¡ +ä¸Ģ 对 +è¾ ± +æ² Ĥ +åı² ä¸Ĭ +æIJ ı +å´ ĸ +代 è°¢ +ç£ · +é¡ ĺ +æµ ĩ +常 çĶ¨ +åį ij +åĩº åĽ½ +è¯ ł +稳 æŃ¥ +ç»ı 纪 +å¤ļ å¤ļ +æīĢ å¾Ĺ +为 主é¢ĺ +ä¸Ģ åĪĨ +æł ½ +é¡ § +çº ² +åĥ ħ +å£ ĵ +åĦ ª +ç¿ ° +æİ Ģ +人 为 +åª ³ +æ´ ½ +èĿ ¶ +å¤į åħ´ +ä¼ļ å½±åĵį +åIJĦ çķĮ +éĤ£ ä¸Ģ +é¢ ¤ +çĢ ı +çĢı 覽 +å¯ ŀ +åı¯ æĢķ +åį³ æŶ +çķ ´ +ä¸ĭ åįĬå¹´ +ç¬Ķ è®° +éĻĦ åĬł +çĥŃ æ°´ +å¥ ¸ +ç£ ħ +æĿ ī +æ¸ħ åįİ +éĸ ± +ç° ¡ +å¤Ħ å¤Ħ +åIJĪ éĩij +æ²³ æµģ +ç´ ° +è´Ł éĿ¢ +çļĦ 羣å®ŀ +åĻ¨ 械 +èĴ IJ +西 äºļ +å· ħ +ç² ¹ +åİŁ æĸĩ +æŀ ķ +è¡Ģ åİĭ +åļ ´ +å¸ ĺ +åĨ Ģ +æĮ « +çĶµ è·¯ +å°ı ä¼Ļä¼´ +èĿ ´ +æľĢ å¿« +æĭ Į +å® ª +æĸ · +ç¿ ħ +åĴ ³ +åĹ ½ +ç¾ ŀ +躺 åľ¨ +èµĽ 车 +æ² IJ +éĻIJ 度 +为 ä¸Ģä½ĵ +èĴ ľ +å¹ « +æIJ ħ +åĭ ĭ +åī ĸ +纳 ç¨İ +éķ¿ æķĪ +ç½ ķ +åī¯ æľ¬ +ç© į +éĴ © +ç¹ ¼ +åĽ½ åľŁ +è¼ ī +ä¸į å¿ĺ +èѦ 示 +çģ ¿ +å¿ĥ å¾Ĺ +æĦ ļ +忽 çķ¥ +åĽŀ äºĭ +åįł æľī +æ· Ħ +çī ¡ +çĽij äºĭ +ç¿ ¡ +éĴĪ对 æĢ§ +çª ĥ +è£ ½ +èĨ Ŀ +ç³ Ł +港 æ¾³ +太 太 +æ¾ ¡ +ç»Ĩ åĮĸ +åĶ® åIJİ +å®ŀåľ¨ æĺ¯ +ç« £ +çį ² +åĢ¾ åIJij +å¼ķ çĶ¨ +é¹ ħ +ç¬ij 容 +ä¹IJ 趣 +æ°ij æĶ¿ +éŨ æĪ· +å± ģ +è¿· 失 +éĶ Į +å°ı 康 +åĭ ī +æ³ ¼ +ä¾ĭ åŃIJ +ä¸ī ä½į +å» ł +èĶ ĵ +广 éĺĶ +èĢ į +èĢģ èĻİ +åĭŁ éĽĨ +èĦļ æŃ¥ +æĭ ¯ +åŃĹ åı· +çĦ ° +é¢ ł +èļ Ĥ +èļ ģ +é£ ¯ +人 æĢ§ +æĴ ° +åİ ¢ +å±Ģ éĻIJ +æľª æĪIJ +åĵª åĦ¿ +大 åıij +ä¸į å®ļ +å¾ģ æ±Ĥ +éĥ µ +åĢº æĿĥ +çĪ± ä½ł +èº ģ +ä»ħ ä¾Ľ +è¿ľ å¤Ħ +éĨ Ľ +åĥ µ +积æŀģ æĢ§ +æİ ¡ +åīį ä¸ī +äºİ ä¸Ģä½ĵ +çŀ Ħ +çĿ ģ +æ² ¸ +åħ± èµ¢ +éĢĢ å½¹ +è´Ŀ å°Ķ +æİ ı +æĪ ² +è¡ į +éĶ Ĥ +ä¸ĩ ä½Ļ +ç§ij åĪĽ +æ¼Ķ åĶ± +欧 åħĥ +æ·¡ æ·¡ +éĿĴ å±± +èĹ Ŀ +ç» ½ +令 çīĮ +éĽĨ 群 +ä½ľ çī© +çĢ ij +å¤ ¯ +ç½ij 游 +åħ« 大 +éª ļ +èª ĵ +ä¼ļ å±ķ +åħļ åı² +æ£Ģå¯Ł éĻ¢ +åĸ ĺ +éĺ ± +èĢĮ åĩº +éĢļ 车 +éĴ ĵ +æĥħ 人 +æ¸ Ľ +ä¸Ń ç§ĭ +çĪ Ń +åıª åī© +æĺ Ķ +éĩİ çĶŁ +ç¡ « +èIJĿ åįľ +æĬµ æĬĹ +çĻ« çĹ« +éĻ Ģ +èĶ ļ +å¸ ľ +满 满 +èı ± +éļĨ éĩį +æĺŁ çº§ +æ½ ĩ +åħ¬ åħĥ +è° £ +æ¯Ķ äºļ +æ¡Į åŃIJ +èµ £ +è² ¼ +æĦ¿ æľĽ +é¡ ½ +æ´¾ éģ£ +ç¥ Ľ +åª ļ +éĺ ľ +èij « +èĬ ¦ +æ³ » +å¡ Į +çĭ Ń +å»ī æĶ¿ +å¥ij æľº +æĹĹ èĪ° +æĥ « +严 åİī +åıĭ æĥħ +å¦ Ĭ +å¨ ł +åĵª 家 +èĨ ¨ +è¶ Ł +æĮ ª +èĻ IJ +é łģ +çŀ © +éº Ł +ç¨ £ +èģĶ éĢļ +åı ® +çİĭ èĢħ +ä¸į ç¡®å®ļ +ç ijľ +è° İ +çī¢ è®° +ç¢ ¼ +æĬ¤ èĤ¤ +é¡ · +çĦ ķ +åģļ 强 +éļ± ç§ģ +éļ±ç§ģ æ¬Ĭ +åıĹ å®³ +ä¸į çĶ± +çĥ ¹ +é¥ ª +é© ³ +ä¼ ½ +ä¸Ŀ 绸 +è¥ Ħ +åįģ ä½Ļ +éº Ĺ +æ¬Ĭ åĪ© +èģ ŀ +åı¤ èĢģ +éģ ı +åIJĦ å¼ı +å°± è¡Į +åħ¥ å¢ĥ +ç ĥģ +èľ ĺ +èĽ Ľ +çº ¬ +çŁ « +è» Ł +æ´Ĺ è¡£ +æĦ § +é¢Ħ æ¡Ī +éľ Ĩ +æ·± åİļ +éĺ¿ æĭī +åĨĻ åŃĹ +åį ¦ +éķ Ģ +模 æł· +åĤ į +æIJ į +èĸ ¯ +åł ħ +åħ¬ 积 +è¨ İ +ä¼ł æŁĵ +æ¯ ¯ +çIJĨ å·¥ +åĨ· éĵ¾ +ç«ĭ æĸ¹ +æ¢ Ń +åľ£ è¯ŀ +综 èīº +çİ© ç¬ij +æĥ³ ä¸įåĪ° +æijĩ 头 +æ· ¹ +åģĩ æĹ¥ +åĢ ĺ +èĢ ½ +èİ ĵ +åŁ · +èĩª è´¸ +åįĬ 天 +æª Ķ +æ¾İ æ¹ĥ +éķ ij +ä¸ « +éĩĮ ç¨ĭ +å¼Ģ èįĴ +èı ı +å®Ŀ è´µ +èŃ ¬ +åķ Ł +æŁ ł +æª ¬ +é© Ń +æ± Ľ +çĨĬ çĮ« +èķ ī +éļı ä¹ĭ +å± ij +è¾ĥ 强 +èĥ ³ +èĨ Ĭ +éĿĻ éĿĻ +åĴ ª +æĭĽ åij¼ +代 è¨Ģ +ä¿¡ ç®± +è£ħ éħį +æĤ į +åįķ 车 +èIJ İ +å¤ļ 彩 +éĻ ¸ +ä»İ 严 +æ© Ħ +æ¦ Ħ +éĢ ® +éĩĮ æĸ¯ +姿 æĢģ +太 æŀģ +éĩ Ŀ +æº ī +è¿ Ń +ç§ ¸ +ç§ Ĩ +å·¥ å§Ķ +æ± ķ +èģ Ĩ +ä½ ¬ +ç¼ ħ +çĶ ¸ +åī¯ å±Ģéķ¿ +éĹ º +èª ¤ +è¤ IJ +ä¸į éĻIJ +èħ ķ +åij ķ +çŁ ¶ +åĨľ 家 +管 å§Ķä¼ļ +é¥ º +èĬ ľ +æ¾ Ī +è© ¢ +å¨ģ å°¼æĸ¯ +ä½ķ åĨµ +å°ı ä¼Ļ +奢 ä¾Ī +è¿Ļ ç¯ĩ +è¯ µ +ç«ł ç¨ĭ +ç´ Ģ +éIJ ĺ +éĤ ¢ +ç³ Ļ +ç¼ Ģ +ä¹ Ĵ +ä¹ ĵ +çī¢ åĽº +åĿ ŀ +å¼ Ī +ä¾ĭ å¤ĸ +å» ³ +è§Ħ ç«ł +èĬ Ļ +ç¯ · +èº ¯ +æł Ī +åĿļ å®ŀ +åŁº 建 +çĿĢ çľ¼ +ç· ´ +èij © +ç¼ ļ +æ¦ Ĩ +主 åĭķ +ç¥ Ģ +äºĴ éĢļ +å°¤ 为 +å® Ľ +éª ¼ +æ± ² +ä¾ ĥ +æĤł ä¹ħ +æij § +æĭ ĩ +é« ĵ +éº Ĵ +éĻ Ľ +æŀ ¸ +æĿ ŀ +è´ ¬ +å°ı é¾Ļ +åĵ ® +èĵ¬ åĭĥ +åĮ Ī +çķľ çī§ +å¨ © +个 å¤ļ +æ² ¥ +æĺ § +çĦ ļ +æĬij éĥģ +çĸ ¡ +èĺ ij +éģİ ç¨ĭ +æ© ± +éĿ ĵ +大 çIJĨ +é« ¦ +åĪĨ 辨 +æ¸ ¤ +çĸ ¤ +åĬ¨ èĥ½ +å¼ł 家 +ä¸ĩ åįĥ +æ» ¥ +é¥ ¥ +åºŁ å¼ĥ +å¸ ³ +æ¼ ³ +è± IJ +ä» ij +å« ī +å¦ Ĵ +çŀ Ĵ +è¡ ħ +çĭ ¸ +å¾ģ ç¨ĭ +éĤ ¯ +éĥ ¸ +ç¥ Ī +ç¥ · +è¶ ´ +ç»ĵæŀĦ æĢ§ +è§Ĩ åIJ¬ +è¬ Ŀ +çĴ Ģ +çĴ ¨ +åĩº å¤Ħ +è¯ Ģ +å¾ ĺ +å¾ Ĭ +çľ ¨ +åĸ ĩ +åı Ń +åĺ ² +çķ ¸ +å¹² äºĭ +æļ § +æ² Ľ +åĦ Ħ +å» ĵ +åİ¿ éķ¿ +èĥ ļ +çIJ ¢ +çŃ · +éĩ ĭ +ä¾ ® +åIJ © +åĴ IJ +åĮ ¿ +æĬ¬ èµ· +æ³ £ +æ¶ ¤ +éº ½ +æĽ Ļ +åī¯ éĻ¢éķ¿ +åħļ åĴĮ +æķ£ åıij +润 æ»ij +åĵ º +æĥ ¬ +漫 éķ¿ +ä¸į æĩĪ +åŁ ł +åĹ ĵ +èĢģ çĪ· +è® ½ +æĪĺ ç»ĦåIJĪ +æ£ ł +åħ¨ åŁŁ +èł ¢ +è¯ ¡ +åīį çŀ» +æķ Ľ +ä¸Ģ å°ģ +å¹ Ĥ +èİ Ĩ +è¯Ŀ è¯Ń +ç»Ĩ åĪĻ +å± ¿ +åµ Į +éĢ į +åĺ ± +æ¸ ² +çĥ ¯ +çĿ ¹ +é¦ Ĵ +èħ ¥ +æĬĹ åĩ» +çĿ « +èį Ķ +éļ İ +æ³ī æ°´ +è¬ Ĥ +ç Ĥ¬ +åĩı æİĴ +è¸ Ĭ +è ·» +æ· Į +éľ ¾ +å¥ĩ 纳 +å¯ Ŀ +æ¤ İ +æŁ ¬ +æĸ¯ åŁº +åħ¬ ç«ĭ +è¨ ĵ +é£ Ļ +é© ¿ +åĤ µ +èĽ Ļ +ç¯ĩ ç«ł +åĪĨ æĶ¯ +ä¸Ĭ å¹´ +çŃ Ŀ +ç¼ ¤ +èĢģ æŧ +åĻ ¬ +æľ ¦ +èĥ § +æ¶Ī è²» +æĵ Ķ +æ¦ ´ +æ¿ Ĵ +ç³ ¯ +æ³ ¸ +æį Ĩ +ç» ļ +èµ İ +çIJ IJ +èµ Ĥ +æħ ® +æ² Į +çĦ Ļ +æĴŃ æĬ¥ +æ· ĩ +åĪĩ åħ¥ +çij ķ +çĸ µ +éģ ´ +ç¨ ļ +ç© © +èŀ ĥ +æ£ ķ +æĨ § +æĨ ¬ +ä¼ º +æ¯ Ĺ +æį į +æĬ ī +ç´ Ĭ +å¼ Ľ +æĭ Ń +æĹı èĩªæ²» +åĿ · +ç« ¶ +è© ³ +è¿Ħ ä»Ĭ +è° ´ +çŀŃ è§£ +æŁ ¿ +é¢ Ĭ +ç° § +çĥŁ èĬ± +ä¾ ¥ +çĿ ¦ +éħ Ŀ +æ° ĵ +çIJ ī +å§ Ĭ +æ² ® +æħ · +èľ ķ +çij ļ +éĩĩ çŁ¿ +åł ° +åºķ èķ´ +èĨ ³ +è¾ ķ +éŁ Ń +åĴ Ļ +ç² ½ +åī Ķ +æ² ¦ +èĤ ´ +éķ ¶ +æĺ ¼ +è¾ Ĺ +å© ª +åĮ ® +æĸ ĵ +æ± ¶ +éĥ ´ +éł » +çª Ĵ +è¢ ± +åĽ ± +èĢ ĺ +è ļĮ +çĭ Ļ +çĹ ¹ +ç¥ ī +æı ® +æ· Ĩ +ç£ ĭ +éĺ ª +æ « +ã ¸ +Ļ ¶ +ã ij +𣠲 +ä ¢ +ã Ń +𬠨 +ð¬ Ģ +𬠮 +𬠯 +ð¬ ľ +𪠨 +ð« Ĺ +ð¬ Ĭ +𬠱 +ð¬ Ł +ä İ +ð ¡ +ä ĥ +ã ł +ð © +ð© ¾ +𬠺 +ð¬ Ļ +ãĢ Ķ +ãĢ ķ +çļĦ æŶåĢĻ +æľīéĻIJ åħ¬åı¸ +ä¹ĭ åIJİ +ä¸ļ åĬ¡ +åķ Ĭ +èĻ½ çĦ¶ +æĭ¥ æľī +äºĴ èģĶç½ij +éĤ£ äºĽ +ä½ł çļĦ +åĨ³ å®ļ +éĻ¤ äºĨ +åĽ¢ éĺŁ +åı¯ æĺ¯ +以 åIJİ +社 åĮº +çļĦ éĹ®é¢ĺ +并 ä¸Ķ +æķĻ å¸Ī +å°± ä¼ļ +天空 éĥ¨èIJ½ +æľĢ ç»Ī +å½ĵ çĦ¶ +ä¹Ł æľī +ç¡® ä¿Ŀ +æĥ³ è¦ģ +è´Ń ä¹° +人 çļĦ +åIJ ´ +çļĦ åıijå±ķ +ä¸į çŁ¥éģĵ +软 件 +æĪij们 çļĦ +çĪ¶ æ¯į +åī ij +èĢĮ æĺ¯ +å®ī æİĴ +åIJİ æĿ¥ +çļĦ åľ°æĸ¹ +èµ µ +èĢĥ è¯ķ +çªģ çĦ¶ +ä¸Ģå®ļ è¦ģ +åĪ¶ ä½ľ +è¯Ħ ä»· +åħį è´¹ +è´¹ çĶ¨ +绣 ä¸Ģ +çĦ¶ èĢĮ +è¿Ļ 次 +éĿĴ å¹´ +人 ç±» +äº ¦ +让 人 +è´Łè´£ 人 +éĩĩ åıĸ +çļĦ äºĭæĥħ +ä¹Ł ä¼ļ +车 è¾Ĩ +æĽ´ æĺ¯ +强 åĮĸ +æĪij åĢij +以 åīį +ä¼ĺ åĮĸ +å§Ķåijĺ ä¼ļ +åĽ° éļ¾ +å¹´ 度 +ä½į äºİ +æĮĩ åĩº +åĨį 次 +åĬŀ çIJĨ +æ¯ı 个 +对 æĸ¹ +è¿Ľè¡Į äºĨ +æľĢ é«ĺ +课 ç¨ĭ +身 ä¸Ĭ +æĽ¾ ç»ı +åĮ» çĶŁ +å®ī è£ħ +æľ ± +è¿IJ è¡Į +åıĮ æĸ¹ +æľĢ 大çļĦ +æŀĦ 建 +è¿ŀ ç»Ń +çļĦ å°ı +她 çļĦ +çŃī çŃī +æĶ¹ åĸĦ +åIJĦ ç±» +éģĩ åĪ° +æľī çĿĢ +人 çī© +æĢ» æĺ¯ +è¿ħ éĢŁ +åĪ¶ å®ļ +å®ĥ 们 +å®ĺ ç½ij +è¿ĺ è¦ģ +ç»Ī äºİ +æĪ¿ åľ°äº§ +è¯ģ æĺİ +èĤ¡ 票 +åºĶ å½ĵ +èĭ± åĽ½ +è¿IJ çĶ¨ +æľĢ æĸ° +享 åıĹ +让 æĪij +æĻļ ä¸Ĭ +å¾ ŀ +å°ı 说 +å°¤åħ¶ æĺ¯ +è®Ń ç»ĥ +åħ¨ å¸Ĥ +æĮij æĪĺ +æľī çĤ¹ +带 çĿĢ +çļĦ ä¸ľè¥¿ +é£İ æł¼ +é»Ħ éĩij +å¼ķ 导 +æѤ å¤ĸ +æľĢ è¿ij +追 æ±Ĥ +强 è°ĥ +ä¹Ł åı¯ä»¥ +æĦŁ åĪ° +èĩª æĪij +çī¹åĪ« æĺ¯ +æĪIJ éĥ½ +éĢIJ æ¸IJ +å¿« ä¹IJ +ä¹ĭ ä¸Ń +æĬķèµĦ èĢħ +ä»ĸ们 çļĦ +æ° ı +å·¥ä½ľ 人åijĺ +äºĨ ä¸Ģ个 +åķ ¦ +ä¸Ģ åĢĭ +åŁº å±Ĥ +æ²Ł éĢļ +第ä¸Ģ 次 +并 没æľī +çļĦ å·¥ä½ľ +åľ¨ è¿ĻéĩĮ +æŀ ª +æĶ¯ æĴij +æŶ å°ļ +æĿ¥ åĪ° +æĶ¶ è´Ń +éĿ© åij½ +æĺ¯ ä¸įæĺ¯ +讨 论 +ä¸ļ 绩 +å°± èĥ½ +ç«ĭ åį³ +è¡Ĺ éģĵ +åľ¨ ä¸Ģèµ· +æľĪ 份 +é«ĺ 端 +å¾Ī éļ¾ +ä¿Ħ ç½Ĺæĸ¯ +æīĭ 段 +åģļ åĩº +ä¼Ĺ å¤ļ +å®ŀ è¡Į +æīĵ å¼Ģ +游 客 +ä¾Ŀ çĦ¶ +å°± åĥı +离 å¼Ģ +说 éģĵ +æĸ° èĥ½æºIJ +æº ª +äº ķ +令 人 +ä¸Ģ åľº +æĪij æĥ³ +两 人 +èĩ³ å°ij +çļĦ çĶŁæ´» +æĺ¯ ä¸ª +èĭ± è¯Ń +æ²Ĵ æľī +æĢĿ èĢĥ +éĻIJ åĪ¶ +åı° æ¹¾ +ä¸Ģ æŦ +çļĦ ä¸Ģ个 +é«ĺ 级 +åĬŀåħ¬ 室 +å¾· åĽ½ +æĪij å°± +å®ļ ä½į +éĢĤ åºĶ +æĮĩ æłĩ +åħ¨ çľģ +ä¸Ĭ è¿° +å®ĥ çļĦ +åĽŀ 家 +欧 æ´² +éĵģ è·¯ +é¼ĵ åĬ± +çļĦ å½±åĵį +é«ĺ æł¡ +天 ä¸ĭ +é«ĺ è´¨éĩı +æĿŃ å·ŀ +èµĦ 讯 +æĶ¾ åľ¨ +æľī ä¸Ģ个 +å°± è¦ģ +ä¸Ĭ éĿ¢ +解 éĩĬ +éĢIJ æŃ¥ +å°½ 管 +æľī ä»Ģä¹Ī +çļĦ äºĭ +çĻ» è®° +人æ°ij å¸ģ +è§Ĥ ä¼Ĺ +è§Ĥ å¯Ł +çĶµ èĦij +çļĦ åIJĮæŶ +ä½ľ ä¸ļ +宣 å¸ĥ +çļĦ ä½ľçĶ¨ +åĽŀ æĿ¥ +éļ¾ 以 +æīĢæľī çļĦ +å°ı åѦ +æıIJ åīį +æ¤į çī© +åĩ ¯ +ä¸Ĭ äºĨ +å°± åľ¨ +åħĪ åIJİ +æīĭ æľ¯ +éĥ Ń +éĿ¢ åīį +æ¯ķ ç«Ł +äºĮ æĺ¯ +红 èī² +éĺ³ åħī +èĭ¹ æŀľ +å¾Īå¤ļ 人 +ç»Ļ æĪij +åĵ ¦ +çľ¼ çĿĽ +éł Ń +ä¸Ģ æĺ¯ +åıijå±ķ çļĦ +åıį åºĶ +æĪ¿ å±ĭ +æľŁ å¾ħ +ç§į æ¤į +æĸĩ åѦ +åį³ åı¯ +é¦ĸ 次 +èĭ± éĽĦ +å¤ļ 次 +åĮħ è£ħ +æ²³ åįĹ +ä¹ĭéĹ´ çļĦ +ä»į çĦ¶ +åIJ¬ åĪ° +èij£äºĭ éķ¿ +è§Ħ åĪĻ +ä¸Ģ 份 +大 ä¼Ĺ +使 å¾Ĺ +è¿Ľ åı£ +ä¸Ģ çīĩ +æĢ§ çļĦ +çļĦ 大 +æĪij æĺ¯ +äºĴ åĬ¨ +æ° £ +çļ Ĩ +åħ¬åı¸ çļĦ +ä¸Ģ è¾¹ +åıĬ åħ¶ +èī¯ 好çļĦ +æĭĵ å±ķ +å½ĵ å¹´ +广 åľº +åģļ äºĨ +åŁº äºİ +æıIJ éĨĴ +åħĦ å¼Ł +èĢģ æĿ¿ +è¿ij æĹ¥ +çĬ¶ åĨµ +注 éĩį +åĪļ åĪļ +è°ĥ çłĶ +å¿ĥ ä¸Ń +æĬĬ æı¡ +éļı åIJİ +ä¸į å¤Ł +åĪĽ ä½ľ +ç«Ļ åľ¨ +缸 äºĴ +çĸ«æĥħ éĺ²æݧ +å¹´ 代 +带 åĬ¨ +伤 害 +ç«Ł çĦ¶ +å¼ķ è¿Ľ +ç´¯ 计 +让 æĪij们 +åĽŀ æĶ¶ +æĬ¥ åIJį +åĬ© åĬĽ +èģĶ 缣 +çŃĸ çķ¥ +åij¨ è¾¹ +åĭ Ĵ +è¿ĺ åľ¨ +æµģ éĩı +寻 æī¾ +çĶµ åĬĽ +èĪ¹ èĪ¶ +è¿ĺ èĥ½ +æĭħ ä»» +çļĦæĥħåĨµ ä¸ĭ +çļĦ åİŁåĽł +缺 ä¹ı +çIJĥ åijĺ +å²ģ çļĦ +çĶ· åŃIJ +å·¥ èµĦ +è¿ijå¹´ æĿ¥ +åij Ģ +æıIJä¾Ľ äºĨ +她 们 +家 åħ· +çĩ ķ +è½» æĿ¾ +æł¡ åĽŃ +èĢĥ æł¸ +åį± éĻ© +åħļ ç»Ħç»ĩ +æĢ» ç»ıçIJĨ +çļĦ æĸ° +çİ» çĴĥ +è¿Ļ ä½į +对 æѤ +家 人 +çļĦ è¦ģæ±Ĥ +温 度 +æĮĩ æķ° +缴 åĪ° +æѤ æŶ +æ¹ĸ åįĹ +éĥ½ è¦ģ +ä½ľ åĩº +åIJĦ ä½į +èĢĥ çĶŁ +ä¾Ŀ æį® +说 è¯Ŀ +æĪij ä¹Ł +å·¥ åİĤ +åıĺ æĪIJ +ä»ĸ 人 +æĪij è§īå¾Ĺ +åIJĦ 级 +ä¼łå¥ĩ ç§ģæľį +ä¸Ĭ åįĩ +好 åĥı +åĬł éĢŁ +äºĮ åįģ +è¢ ģ +è£ħ 饰 +éĥ½ èĥ½ +ä¸Ģ å¼ł +åĬ¨ æĢģ +å¹´ çļĦ +è¿Ļ å°±æĺ¯ +ä¹Ł è¦ģ +èµĦ æł¼ +æĪĺ äºī +æĦŁ è°¢ +åŁ¹ èĤ² +天 æ°Ķ +女 士 +åı¯èĥ½ ä¼ļ +çļĦ 产åĵģ +ä¹Ł å°± +主è¦ģ æĺ¯ +åĪº æ¿Ģ +ç»Ļ ä½ł +大 æķ°æį® +åĮ» åѦ +åĪ ¤æĸŃ +ä»ĸ 说 +表 æ¼Ķ +äºļ æ´² +ä¸ĵ é¢ĺ +ç«ŀäºī åĬĽ +éĤ£ æł· +å±ķ å¼Ģ +å¹³ æŶ +æİ¥ ä¸ĭæĿ¥ +æī¿ 诺 +æ³ķ åĽ½ +åħ³ å¿ĥ +ä¼ļ æľī +éĤĢ 请 +é¢Ħ éĺ² +对 æİ¥ +好 äºĨ +åĴ± 们 +çļĦ æĦŁè§ī +æĢĿ è·¯ +éĥ½ 没æľī +çļĦ æĸ¹æ³ķ +女 åŃIJ +åı¸ æ³ķ +è¿ĺ ä¼ļ +è¶ĬæĿ¥è¶Ĭ å¤ļ +åĽł çĤº +æµ· åįĹ +人 æķ° +å°Ĩ ä¼ļ +ä¸ļ 主 +é¤IJ 饮 +å±ħ ä½ı +åıij åĩº +è¿ij æľŁ +å¼ķ é¢Ĩ +æľºåĻ¨ 人 +åĩºæĿ¥ çļĦ +çľĭ è§ģ +ä¿ Ĭ +让 ä»ĸ +ä¸į æĥ³ +å·¥ä½ľ çļĦ +è¡¥ åħħ +æµ ħ +çī¹ å¾ģ +ä¸Ĭå¸Ĥ åħ¬åı¸ +ç¾İ é£Ł +广 西 +æ¯ı ä¸Ģ个 +èIJ½ åľ° +åĵģ ç§į +åĴĮ è°IJ +å½» åºķ +é«ĺ èĢĥ +æĺ¨ å¤© +åīį å¾Ģ +çĽij æµĭ +çĻ¾ 度 +åľ¨ ä¸ŃåĽ½ +çļĦ éľĢæ±Ĥ +亿 ç¾İåħĥ +åѦ æľ¯ +æĶ¶ åĪ° +æĿ¿ åĿĹ +ä¸Ģ 段 +æŀĦ æĪIJ +ä¼ģä¸ļ çļĦ +表 éĿ¢ +æķ´ çIJĨ +ç»ĵ å©ļ +人 家 +åģľ æŃ¢ +åѦ ç§ij +æĺ¾ å¾Ĺ +ä¼ij æģ¯ +é¢Ħ æľŁ +æĪĸ æĺ¯ +çļĦ 主è¦ģ +åºĶ 对 +èµ° äºĨ +ä¸Ń éĹ´ +èµ° è¿Ľ +åijĪ çİ° +æIJŃ éħį +é¹ ı +æĺ¯ åĽłä¸º +æĥħ 绪 +å®ļ æľŁ +社ä¼ļ 主ä¹ī +çŃī 级 +磼 çĽ¾ +é£ŀ æľº +èĩ³ ä»Ĭ +æĶ¶ éĽĨ +çļĦ æķħäºĭ +åĪĩ å®ŀ +å®ŀçİ° äºĨ +å½¢ æĪIJäºĨ +åįĹ æĸ¹ +ä¸Ń åѦ +æµ· æ´ĭ +åIJ¦ åĪĻ +æĭį æijĦ +大åѦ çĶŁ +åĩºçİ° äºĨ +æĦı å¤ĸ +ä¹Ł èĥ½ +çļĦ èĥ½åĬĽ +åĿIJ åľ¨ +åĪĻ æĺ¯ +èĢĥ å¯Ł +å°Ĭ éĩį +éĺ² æŃ¢ +ç´§ å¼ł +读 书 +åĩº è¡Į +å°± æľī +å±¥ è¡Į +çݰ代 åĮĸ +åĽ½ åĬ¡ +åĽ½åĬ¡ éĻ¢ +ç»´ ä¿® +åİŁ åĪĽ +æĺ¯ æĮĩ +ä¼ij éĹ² +çĤ ® +æĸ° æĹ¶ä»£ +éĢĻ åĢĭ +ä¸į æķ¢ +å®Į ç¾İ +ç»Ĩ èĬĤ +éŃ ı +èĶ¬ èıľ +é¢Ĩ导 çıŃåŃIJ +è¶ħ 级 +è¡Į æĥħ +人工 æĻºèĥ½ +åį° 度 +åŁºç¡Ģ 设æĸ½ +åıĪ æĺ¯ +èį¯ çī© +åIJ¸ æĶ¶ +åį´ æĺ¯ +éĥ İ +å¥ĸ åĬ± +çļĦ æľĭåıĭ +ä¿Ŀ çķĻ +è§Ħ å¾ĭ +æĸ° çĸĨ +è¿ĺ åı¯ä»¥ +æİ¥ è¿ij +æѤ åīį +æī¹ åĩĨ +æĢİä¹Ī æł· +çļĦ ä½įç½® +ä¸Ģ åĿĹ +æĭĴ ç»Ŀ +顾 客 +ä¹Ł åľ¨ +ä¸Ģ çĶŁ +éĥ¨ éĺŁ +å¹´ åīį +æĸ¹éĿ¢ çļĦ +å°Ŀ è¯ķ +羣æŃ£ çļĦ +ç¦ģ æŃ¢ +è¿ĺ 没æľī +æ°ij çĶŁ +èµ° åIJij +èĦ¸ ä¸Ĭ +å½ĵ 天 +éĽĨåĽ¢ åħ¬åı¸ +çļĦä¸Ģ ç§į +西 æĸ¹ +åĽŀ åºĶ +ä¸Ģ 声 +常 常 +æıIJ åĪ° +èħ¾ 讯 +æľį è£ħ +为 ä½ķ +äºij åįĹ +å°± ç®Ĺ +ä¼ł æī¿ +åıį èĢĮ +ä¸ĩ åIJ¨ +è´¢ 产 +å¦Ĥ ä¸ĭ +æĹ¥ åīį +åİŁ æľ¬ +æľĢ éĩįè¦ģçļĦ +认 è¯ģ +ä¸Ģ éģĵ +ä¿¡æģ¯ åĮĸ +å¾Ĺ åĪ°äºĨ +éĢ² è¡Į +æĪij è¦ģ +éĢļ ä¿¡ +室 åĨħ +èµļ éĴ± +æĶ¶ èĹı +解åĨ³ æĸ¹æ¡Ī +æĪ¿ 产 +çĭ ¼ +æ´» åĬĽ +ç»ıæµİ åıijå±ķ +çŃī å¾ħ +ä¹Ł å¾Ī +åĿ ij +å¾Ī 好çļĦ +éļ¾ 度 +ä¸į å¦Ĥ +人æ°ij æĶ¿åºľ +åĩº åıij +åīį æľŁ +æ¼Ķ åijĺ +女 çĶŁ +èģļ çĦ¦ +审 计 +é¢Ħ æµĭ +ä¾Ŀ æīĺ +äºĶ å¹´ +è¡¥ è´´ +æ¸ħ æĻ° +éª Ĥ +çľĭ èµ·æĿ¥ +çļĦ åŃ©åŃIJ +é¢ij éģĵ +ä½ı å®ħ +éĿ¢ åIJij +æľĢ ä½İ +æĹ¢ çĦ¶ +ä¸Ģ å¥Ĺ +æķ° åѦ +群 ä½ĵ +åĮĹ京 å¸Ĥ +å±ħ çĦ¶ +æ°Ľ åĽ´ +éĢĶ å¾Ħ +çļĦ åŁºç¡Ģä¸Ĭ +èģĮ è´£ +åı¯èĥ½ æĺ¯ +åĨĽ äºĭ +æĪIJ æķĪ +åŃ©åŃIJ 们 +计ç®Ĺ æľº +èµ ¤ +产ä¸ļ åıijå±ķ +å·¨ 大çļĦ +å·¥ 人 +çĶŁ éķ¿ +éĥ½ åı¯ä»¥ +çļĦ æľºä¼ļ +èµĦ è´¨ +çĹĽ èĭ¦ +ç²ī ä¸Ŀ +å¢ ĵ +å¹³ å®ī +管 éģĵ +è·Ł çĿĢ +饮 é£Ł +åķĨ 家 +å¤ļ 家 +åı¸ æľº +åºĶ该 æĺ¯ +éĢı éľ² +认 å®ļ +è¡Įä¸ļ çļĦ +çļĦ ä¼ģä¸ļ +æ¯ı ä¸Ģ +èĮĥåĽ´ åĨħ +è¾ĥ 大 +è´ ¤ +大 èµĽ +å¤ļ äºĨ +é¸ ¿ +临 åºĬ +åľ¨ è¿Ļ个 +çļĦ åĨħ容 +éĶĢ éĩı +å¾Ī å°ij +åŃ Ł +ç»´ æĮģ +åĴĸ åķ¡ +æľ¬ åľ° +èī² 彩 +并 éĿŀ +èĢĮ å·² +温 æļĸ +èIJ § +æĬĵ ä½ı +èĢĮ ä¸įæĺ¯ +åĸ Ĭ +çļĦ åħ³ç³» +çī© åĵģ +éĤ£ æĺ¯ +åĨľ 产åĵģ +è¿Ļ æŶ +å©ļ 姻 +æ°´ æŀľ +æĶ¶ èİ· +ä»ĺ åĩº +客æĪ· 端 +æ¼Ķ åĩº +åħ¨ æĸ° +è¿Ļ ä¹Łæĺ¯ +æĺ¯ çĶ± +è§Ĥ 念 +æľī 个 +éĢł åŀĭ +èĥľ åĪ© +ä¸ī æĺ¯ +è¶ħ å¸Ĥ +åħļ建 å·¥ä½ľ +æĶ¾ å¿ĥ +线 è·¯ +æĭĽ çĶŁ +åIJĥ é¥Ń +è½ ī +å°½ éĩı +è§ģ åĪ° +åIJĮæ¯Ķ å¢ŀéķ¿ +åįİ ä¸º +æĪij å¸Ĥ +æıIJ åĩºäºĨ +æ°ij èѦ +åįļ çī© +åįļçī© é¦Ĩ +è¯ļ ä¿¡ +åīį éĿ¢ +å±± 西 +è¾ħ åĬ© +转 移 +æĽ´ 为 +丰å¯Į çļĦ +åį ¢ +å¿« éĢĴ +æĺ¾ èijĹ +çī© èµĦ +åĪ° è¾¾ +æľī åĪ©äºİ +åij Ĩ +åŃ©åŃIJ çļĦ +ä¸į ä½Ĩ +çłĶ究 éĻ¢ +çĶ³ æĬ¥ +æļ ¨ +æ°ij éĹ´ +åį » +çļĦ å£°éŁ³ +å¸Ĥåľº çļĦ +ä¸Ģ åı¥ +çľģ 级 +æĿ¥ çļĦ +åĵª 个 +æīį ä¼ļ +åĪĨ éħį +èĶ ¡ +ä»ĸ åľ¨ +åħ± æľī +å¡ ĺ +èĴ Ĥ +éľ į +åıĤ è§Ĥ +ä¸Ī 夫 +ä¾Ŀ éĿł +æľī æŶ +äºĨ å¾Īå¤ļ +ä¸ĸçķĮ æĿ¯ +家 æĹı +ä¸į éľĢè¦ģ +大 å¸Ī +èŀį åħ¥ +éĿŀ æ³ķ +çĹħ 人 +åIJİ æľŁ +大家 éĥ½ +ç½ij åĿĢ +åİŁ æĸĻ +便 å®ľ +æ¶ Ľ +仿 ä½Ľ +å·® è·Ŀ +åı¦ä¸Ģ æĸ¹éĿ¢ +产åĵģ çļĦ +èµ « +æĥħåĨµ ä¸ĭ +éĴ¢ éĵģ +æľ¬ ç«Ļ +纳 åħ¥ +å·² æľī +æľī 没æľī +ä¼° 计 +é£ ĺ +æľŁ è´§ +åĢĭ人 è³ĩæĸĻ +ä¸ĵä¸ļ çļĦ +çĪĨ åıij +èĩ´åĬĽ äºİ +çİ°åľ¨ çļĦ +æľī åĵªäºĽ +çł´ åĿı +æķ°åŃĹ åĮĸ +åľ° éĿ¢ +é»ij èī² +å¹¼åĦ¿ åĽŃ +çļĦ ç²¾ç¥ŀ +äº Ń +导 æ¼Ķ +çİ° æľī +æѦ åĻ¨ +èĭı å·ŀ +çİ Ħ +æ±Ł 西 +延 伸 +论 æĸĩ +è¾ĥ 为 +çİ© æ³ķ +é¼ İ +åIJĮ æŃ¥ +éĩĬ æĶ¾ +æĽĿ åħī +åĿļ åĨ³ +å§Ķ æīĺ +å°Ĩ åľ¨ +äºĪ 以 +ä½ľ æĸĩ +èĢĮ åľ¨ +ä¼ĺ åħĪ +åĽŀ åİ» +ä¿® å¤į +åĽ½åĨħ å¤ĸ +çŃĸ åĪĴ +åıij æĶ¾ +å¿ĥ æĥħ +çļĦ åİĨåı² +éĿ¢ è¯ķ +举 åĮĹ +ä¿¡ åı· +ç²® é£Ł +è¯ģ 书 +æŁIJ äºĽ +è¿IJ ä½ľ +åĨ² åĩ» +çĥŃ çĤ¹ +æŶ æŶ +æŶæŶ 彩 +åľ° çĤ¹ +ä¸Ģä½ĵ åĮĸ +éļ¾ é¢ĺ +æĽ ° +ç«ĭ åĪ» +æĺ¯ éĿŀ常 +åħ± åĴĮ +åħ±åĴĮ åĽ½ +æ¿Ģ åĬ± +æľīæķĪ çļĦ +å¤Ħ ç½® +该 åħ¬åı¸ +æ£Ģ éªĮ +èѦ æĸ¹ +è´ ¾ +äºĨä¸Ģ ä¸ĭ +ä»Ĭ åIJİ +çħ ® +çĶ¨ åĵģ +读 èĢħ +æĪij åľ¨ +åĽŀ å¤į +ä¸Ģ 座 +è¿ĺ 没 +å®ļ åĪ¶ +没 æĥ³åĪ° +å¤ ¹ +ä¼ł éĢĴ +ä¸Ģ 款 +强 大çļĦ +çļĦ è¡Į为 +å¤ı 天 +åıijåĬ¨ æľº +é¢ĨåŁŁ çļĦ +å®ŀéªĮ 室 +ä¸Ģ æĬĬ +æĺ¯ ä¸ºäºĨ +éĻķ 西 +æĭħ ä¿Ŀ +è¾¾ æĪIJ +è¦ģ æĺ¯ +æĺİ å¤© +ç»Ļ ä»ĸ +建ç«ĭ äºĨ +ä¸į è¡Į +ä¸Ń æĸĩ +åľ° 说 +åIJİ çļĦ +çĽij æݧ +éĢ ¸ +æĢ» éĥ¨ +æľ¬ æĸĩ +é¹ ¿ +æĻ¯ è§Ĥ +çļĦ 缮æłĩ +èĽ ĩ +åĨ ¯ +ä¸Ń åĮ» +æķĪ åºĶ +产 éĩı +åŃ Ŀ +è´¦ æĪ· +è¿Ŀ åıį +èij£äºĭ ä¼ļ +京 举 +责任 ç¼ĸè¾ij +åķı é¡Į +çĪ± å¿ĥ +èѦ å¯Ł +é¤IJ åİħ +å¸Ĥ æĶ¿åºľ +天 天 +æĸ° é²ľ +éĥij å·ŀ +è¶ħ è¶Ĭ +å½ Ń +çŁ¥è¯Ĩ 产æĿĥ +åĽŀ å¿Ĩ +è·¯ 线 +å»ī æ´ģ +éĿĴ å°ijå¹´ +åıĸå¾Ĺ äºĨ +çľĭ åĪ°äºĨ +é¦ ¬ +ç²¾ åĵģ +åľ° éĵģ +æĮģ æľī +ä¸ĭ äºĨ +æľī æŶåĢĻ +ä¸Ģ 人 +æĴ Ĵ +ä»Ķ ç»Ĩ +èĢģ åħ¬ +äºĭå®ŀ ä¸Ĭ +èģĶ èµĽ +ä¾ĽåºĶ éĵ¾ +é¢Ħ ç®Ĺ +åĪ¶éĢł ä¸ļ +å®īåħ¨ çĶŁäº§ +俱 ä¹IJ +俱ä¹IJ éĥ¨ +çļĦ æł¸å¿ĥ +æīĵ ç®Ĺ +å½± çīĩ +æIJŃ å»º +ä¹Ł ä¸įä¼ļ +æĭħ å½ĵ +å±Ĥ éĿ¢ +åѦ åijĺ +临 æŶ +缸 ç»ĵåIJĪ +对 æ¯Ķ +ä»ĸ æĺ¯ +æĸ° åĮº +è¿Ľ åİ» +çĻ¾ å¹´ +ä¿ © +å°½ å¿« +çĶµåŃIJ åķĨåĬ¡ +æĽ´ æľī +æ¸ħ çIJĨ +åı¦ ä¸Ģ个 +åĤ » +ä»Ģä¹Ī æł·çļĦ +æĺ¯ æľĢ +åij¨ å¹´ +å¾Ī 容æĺĵ +åĽ¢ ç»ĵ +ç´ Ħ +æĹ© å·² +çļĦ åıĺåĮĸ +éľ ŀ +æĹ¥ ä¸ĬåįĪ +失 åİ» +ä¸Ń åľĭ +çļĦä¸Ģ äºĽ +å°ı åŃ© +ä¸ĭ è·Į +éĶ» çĤ¼ +é ij +éij « +å¿ĹæĦ¿ èĢħ +èĤ¡ å¸Ĥ +èµĽ äºĭ +许åı¯ è¯ģ +åı¯ æĮģç»Ń +åijĬè¯ī è®°èĢħ +éĢ» è¾ij +å¼ķ åħ¥ +çļĦ è¿ĩç¨ĭä¸Ń +è§Ĩ è§ī +èĩªæ²» åĮº +è¯ģ æį® +è£ħ ç½® +第ä¸ī æĸ¹ +å¹´ æĿ¥ +å¹¿ä¸ľ çľģ +带æĿ¥ äºĨ +éķ¿ æ±Ł +访 éĹ® +å·® ä¸įå¤ļ +æĺ¯ æĪij +éģŃ éģĩ +æĬĵ 好 +é«ĺ è¾¾ +并 åľ¨ +èĩª è§ī +ä¾ĽåºĶ åķĨ +æĥħ æĦŁ +ä½ı äºĨ +çļĦ èģĮä¸ļ +çļĩ å¸Ŀ +西 éĥ¨ +åĴĮ å¹³ +çļĦ åĬĽéĩı +æ± ª +åħħåĪĨ åıijæĮ¥ +æĬķ è¯ī +èµ· åĪ° +äºĴ 缸 +æ¾³ éŨ +æİ¥ åĪ° +æ°´ æ³¥ +模 åŀĭ +ä¸Ģ åįĬ +秩 åºı +æĪij们 åľ¨ +æī¿ 认 +ä¸Ģ éĥ¨åĪĨ +åįł æ¯Ķ +å¦ĩ 女 +ç² ĺ +äºĨ解 åĪ° +ä¸Ģå®ļ ä¼ļ +åIJĦ 大 +èµ° åĩº +为 大家 +é«ĺ éĵģ +åı¯ä»¥ åľ¨ +ä½Ĩ åľ¨ +çĶŁæĢģ çݯå¢ĥ +èı ¯ +çļĦ ä»·æł¼ +麻 çĥ¦ +æ¿Ģ åıij +éĤ£ å°± +çļĦ æł·åŃIJ +为 æѤ +天 åľ° +çļĦ 缮çļĦ +åĢº åĪ¸ +å·² ç¶ĵ +åĽĽ 大 +åIJĮæŶ ä¹Ł +å½¼ æѤ +æĭ¿ åĪ° +åIJ« éĩı +åįģ 大 +éļ¾ éģĵ +å¼ Ĺ +ä¸Ģ 段æŶéĹ´ +çħ§ 顾 +æķ°æį® æĺ¾ç¤º +æĪIJ为 äºĨ +èµ° åĪ° +æľ¬ åħ¬åı¸ +ç»Ī 端 +ä¹Ł ä¸įæĺ¯ +头 åıij +大 约 +é£İ æĻ¯ +æ¶Ī èĢĹ +审 æŁ¥ +äºī åıĸ +æ³ķ æ²» +äºĭ çī© +ç¼ĵ 解 +æĥ ¨ +缸åºĶ çļĦ +çļĦ æķĪæŀľ +åıį å¤į +åıijçĶŁ äºĨ +éĢĻ äºĽ +ç»ĥ ä¹ł +åݨ æĪ¿ +å¼Ģ æĭĵ +欣 èµı +夫 妻 +ä¸į ä¸Ģæł· +产 èĥ½ +èĬ¯ çīĩ +è¦ģ ç´ł +åıį 对 +çİĩ åħĪ +è´§ çī© +æĹ¥ çĶµ +ä½ľ 家 +æĶ¹ è¿Ľ +æĪIJ åĪĨ +åĽł èĢĮ +åĩı èĤ¥ +æ½ ĺ +å±±ä¸ľ çľģ +åĬ Ŀ +åŁ ĭ +æѦ è£ħ +æ±ĩ æĬ¥ +ä¸Ģ个 æľĪ +çĥŃ éŨ +大 éģĵ +æ´» åĭķ +éĥ½ å¾Ī +çĶµ 梯 +ç´§ æĢ¥ +åĢº åĬ¡ +客 æľį +ä¸Ģ éĥ¨ +ä½ł æĺ¯ +çİ° çĬ¶ +æŃ£ç¡® çļĦ +ä¹ĭ å¤Ħ +ç¼ĸ åĪ¶ +ä½ł åı¯ä»¥ +çŃī åľ° +èİ ī +对 è¯Ŀ +æ·ĺ å®Ŀ +è°ĥ èĬĤ +æİĴ æĶ¾ +åºĵ åŃĺ +ç´ ļ +çļĦ ä¼ĺåĬ¿ +æĿĥ å¨ģ +以ä¸ĭ ç®Ģ称 +ä¸Ģ 项 +èģļ éĽĨ +ä¼łç»Ł çļĦ +æ·· åIJĪ +è¿Ļä¸Ģ çĤ¹ +ä¸Ģ çľ¼ +æĹł éĻIJ +èİ·å¾Ĺ äºĨ +éĢī æīĭ +åĪ¶ åĵģ +åįı ä½ľ +çĭ¬çī¹ çļĦ +ä¸Ģ 级 +è¿Ļ个 éĹ®é¢ĺ +æĸ Į +æĺ¯ æĪij们 +æķĮ 人 +æ¸ħ æ´Ĺ +ä¸Ģ缴 åľ¨ +å°ı ç±³ +çļĦ è¿ĩç¨ĭ +åľ¨ åĮĹ京 +ä¸Ģ æĶ¯ +æĹ© ä¸Ĭ +æĸĩ èīº +ç¦ı åĪ© +é£Ł çĶ¨ +æĦŁ åĬ¨ +åħ¨ ç¨ĭ +æĶ¯ åĩº +æĸ° 建 +å¸ ķ +æĺ¾ çĦ¶ +羣 çļĦæĺ¯ +æĸ°éĹ» ç½ij +èĥ½ åIJ¦ +åįı åĬ© +亲 èĩª +å¾Ī æľī +çĻ¼ å±ķ +æĦı 大 +æĦı大 åĪ© +çĶµ ç½ij +æĹ¥ çĽĬ +çĨ ± +èĤĮ èĤ¤ +çĶ· æĢ§ +ç»Ħ 建 +çŃī éĹ®é¢ĺ +æ¶Ī éĻ¤ +æĬ¤ çIJĨ +å¡ij æĸĻ +ä¹Į åħĭ +ä¹Įåħĭ åħ° +åķĨ æłĩ +çIJ ³ +æĸ° æīĭ +çļĦ çī¹çĤ¹ +åĴ ¬ +å½ĵ ä¸ĭ +设计 å¸Ī +èµĶ åģ¿ +第 åįģ +æĻºèĥ½ åĮĸ +å¼Ģåıij åĮº +åı¯ä»¥ éĢļè¿ĩ +åħ±äº§ åħļ +åİī 害 +çģµ æ´» +æŶ åħī +éĥ¨ ä½į +人 æĸĩ +è¿Ľ æĿ¥ +ä¹ĭ æīĢ以 +ä¸ī åįģ +çļĦ åѦçĶŁ +éĺ² æĬ¤ +åĽ½ 产 +æ·±åľ³ å¸Ĥ +éĤ£ å°±æĺ¯ +åĪ° ä½į +çī¹ æľĹ +çī¹æľĹ æĻ® +å®ŀ æŶ +åı° çģ£ +èĢĮ ä¸į +æĮĩ å®ļ +åĿ Ŀ +èħIJ è´¥ +çī¹ å®ļ +å¢ŀ éĢŁ +æłĩ çŃ¾ +æĪ¿ ä»· +æĦ ģ +贯彻 èIJ½å®ŀ +æĢ§ è´¨ +çłĶ究 çĶŁ +ç¾İ 容 +æī¹ è¯Ħ +究 ç«Ł +人åĬĽ èµĦæºIJ +éĸĭ å§ĭ +åĽŀ å½Ĵ +èIJ¥ åķĨ +èIJ¥åķĨ çݯå¢ĥ +ä¸ŃåĽ½ 人 +çļĦ åŁºæľ¬ +è¯Ŀ é¢ĺ +æłĩåĩĨ åĮĸ +西 èĹı +åĭ ¾ +çļĦ 设计 +ç®Ģåįķ çļĦ +å¤į åĪ¶ +æ¸IJ æ¸IJ +以 å¤ĸ +èģĶ åĬ¨ +两 次 +æĢ§ åĴĮ +æĽ´ 大 +çļĦ åIJįåŃĹ +éŁ ¦ +ä½ł è¦ģ +å¢ĥ å¤ĸ +æĹ© æľŁ +åĪĿ æŃ¥ +è´¦ åı· +害 æĢķ +æĺ¨ æĹ¥ +åĪļ æīį +ç¥ŀ ç§ĺ +ç²¾ å¿ĥ +æµģ éĢļ +åħ¨ æĸ¹ä½į +以 å¾Ģ +ä¹Ł å°Ĩ +æĺ¯ ä¸ŃåĽ½ +åĽ½å®¶ 级 +å°Ĩ åĨĽ +æij Ĭ +æľĢ 为 +第ä¸Ģ æŶéĹ´ +æ¶Ī æ¯Ĵ +å°Ĩ äºİ +å¨ģ èĥģ +èĭ± æĸĩ +æīĭ ä¸Ń +çIJĥ è¿· +è§Ĥ çľĭ +离 å©ļ +æľ¬ åľŁ +åĪĨ æķ£ +æĻ ´ +è¦ģ 注æĦı +浪 è´¹ +管 æݧ +åĩº åĶ® +æĢ» è£ģ +ä¸Ģ éĺµ +å¨ ĩ +äºĶ 个 +å½ĵ åĪĿ +çºł 纷 +ä¸ĵ çĶ¨ +å¤ĩ æ¡Ī +åĪĿ æľŁ +å®ĥ æĺ¯ +åĮº åĿĹ +åĮºåĿĹ éĵ¾ +大 è¿ŀ +è¿Ļ ç±» +åıĺ æĪIJäºĨ +éĤĦ æĺ¯ +åįļ 客 +çı¾ åľ¨ +ä¸Ģ æĸ¹ +å®ĮæĪIJ äºĨ +è¿Ļ个 æŶåĢĻ +åħ¨ å¹´ +ä¸Ĭ 线 +ç½ IJ +ç«ŀ èµĽ +åĩºçīĪ 社 +åĵ¥ åĵ¥ +å¯ « +å¾Ĺ 以 +èĬ± åĽŃ +äºĨ èµ·æĿ¥ +èĦ±è´« æĶ»åĿļ +çļĦ åİŁåĪĻ +讲 解 +æ¶Ī åĮĸ +æįŁ 害 +æļĤ æŶ +å¾Ĺ çŁ¥ +éĢĤ çĶ¨ +éŨ åºĹ +解 读 +æĻ® åıĬ +人æ°ij æ³ķéĻ¢ +åī¯ 主任 +å¿ĥ çģµ +è¯Ĭ æĸŃ +ç¾İ 女 +æŁ ¯ +å¹´ 以æĿ¥ +æ´» è·ĥ +åĢŁ åĬ© +åħ± 建 +è¯ī 讼 +æĶ¾ æĿ¾ +çªĹ åı£ +ä¼ģ æ¥Ń +åĬł æĭ¿ +åĬłæĭ¿ 大 +ä¹° äºĨ +主 æµģ +æĩĤ å¾Ĺ +å°Ĩ åħ¶ +éĢı æĺİ +å·¥ä½ľ ä¸Ń +èĤ¡ ä»· +æ¡£ æ¡Ī +没æľī ä»»ä½ķ +åijĬ çŁ¥ +å¹´ åĪĿ +æĹ¥ ä¸ĭåįĪ +åİĤ åķĨ +èĬĤ å¥ı +主 导 +è£ Ŀ +åħ³éĶ® è¯į +èģĬ 天 +åĨĻ ä½ľ +æĶ¹éĿ© å¼ĢæĶ¾ +æľī æľĽ +éĢļ æĬ¥ +èIJ Į +æĢ» é¢Ŀ +çŁŃ æľŁ +ä¸Ģ çķª +çĶŁæ´» çļĦ +åĮĸ çļĦ +æĺ¥ å¤© +è¿Ļ åľº +æĸ°å¼Ģ ä¼łå¥ĩ +æĺ¯ è¦ģ +å°ļ æľª +åıĺ æĽ´ +ä¸Ģ åij¨ +客 è§Ĥ +æĹ¥ èĩ³ +é¹ ° +çİ ² +å°Ĩ æĿ¥ +客 人 +åıĺ éĿ© +说 äºĨ +åİŁ çIJĨ +èģĮ åĬ¡ +åıĪ æľī +ä¸Ģ åı¥è¯Ŀ +æĦŁ åıĹåĪ° +ç¬Ķ èĢħ +移 æ°ij +西 åįĹ +ä¹ĥ èĩ³ +æŃ£ è§Ħ +åĪĿ ä¸Ń +çĬ ¬ +å½ĵ äºĭ +å½ĵäºĭ 人 +æĪij们 è¦ģ +åħ¥ åı£ +éĤ£ æŶ +æľīéĻIJ 责任 +å°ij 女 +è¿Ļä¹Ī å¤ļ +åĪĨ åħ¬åı¸ +å®ĩ å®Ļ +çļĦ éĢīæĭ© +å§IJ å§IJ +åıij èµ· +è» į +æĽ´å¥½ åľ° +éĻĨ ç»Ń +æľ¬ æľįåĭĻ +å« © +赶 ç´§ +èĦĤ èĤª +第äºĮ 天 +æĪij ä¼ļ +两 ä½į +æķ ² +åħ¬å®ī æľºåħ³ +ç§ijæĬĢ åĪĽæĸ° +å°º 寸 +è¾IJ å°Ħ +å®Ĺ æķĻ +转 æį¢ +åĩº çİ°åľ¨ +ä¸Ģ é¢Ĺ +æľŁ éĻIJ +åIJĮåѦ 们 +åĮĹ æĸ¹ +ä½ł å°± +ä¸Ģ带 ä¸Ģè·¯ +èĢģ å©Ĩ +游æĪı çݩ家 +çļĦ ç»ĵæŀľ +è¡¥ åģ¿ +å¤ĸ è´¸ +对 å¾ħ +ç»´ çĶŁç´ł +ç»ıéĶĢ åķĨ +è¿ĺ å°Ĩ +åŃIJ 女 +æĽ´ é«ĺ +ä¸į 大 +éī´ å®ļ +让 ä»ĸ们 +æīĢè°ĵ çļĦ +æŃ» äºĨ +帮 æī¶ +åĵ² åѦ +以ä¸Ĭ çļĦ +çļĦ åħ³éĶ® +æĹ© å°± +æĬ¥ ä»· +éģµ å®Ī +æī© å¼ł +æĺ¯ å¾Ī +å¼Ģ éĢļ +æĸ° åĬł +æĸ°åĬł åĿ¡ +ç¿» è¯ij +询 éĹ® +é¸ Ń +ä½ĵ åĨħ +两 个人 +çĪ ¹ +éľ ľ +乡æĿij æĮ¯åħ´ +çĿ¡ è§ī +å®ĺ åijĺ +åĪĽ å§ĭ +åĪĽå§ĭ 人 +ä¼Ĺ 人 +åį³ 便 +çĸ« èĭĹ +ä¼ģä¸ļ 家 +æ¸ £ +ç²¾ åĬĽ +å¤ĸ éĥ¨ +èģª æĺİ +è¿Ļ ä¹Ł +å½ķ åıĸ +åĨ² çªģ +åħ¨ 身 +åŃ£ èĬĤ +忽 çĦ¶ +çļĦ æĢģ度 +åĤ¨ å¤ĩ +ä¿Ŀ åħ» +çļĦ æĥ³æ³ķ +ä¸Ĭæµ· å¸Ĥ +æIJº æīĭ +çļĦ ä¿¡æģ¯ +åķĨ åľº +çļĦ æĢĿæĥ³ +æĿĥ åĬĽ +毫 æĹł +æĢĢ åŃķ +硬 件 +åĨħ èĴĻåı¤ +æİ¢ 讨 +åħ» çĶŁ +çļĦ 表çİ° +空 ä¸Ń +æģIJ æĢĸ +å¾Ī é«ĺ +ç»ıæµİ 社ä¼ļ +ä¸Ĭ æĿ¥ +延 ç»Ń +éĩį å¤į +éĺ² èĮĥ +çļĦ å½¢å¼ı +æľĪ åºķ +èĢģ 年人 +绿 åĮĸ +å±± åĮº +æĭ¿ åĩº +æĹħ 客 +æĽ´ æį¢ +åħ¬ 主 +èĬĤ 约 +åħ¨ åİ¿ +åĽŀ æĬ¥ +çIJĨ æĢ§ +çĸ¯ çĭĤ +æ¶ī å«Į +åī§ æĥħ +åĨ¬ åŃ£ +åIJİ ç»Ń +è¿Ļæĺ¯ ä¸Ģ个 +æ¼Ķ 讲 +ä¸Ģ å±Ĥ +æľīåħ³ éĥ¨éŨ +æĹł å¥Ī +ç§į ç±» +缸åħ³ çļĦ +æĪĸèĢħ æĺ¯ +æī¶ æĮģ +å¤ļ æķ° +çļĦ ä½ľåĵģ +ä¸ĭ ä¸ĢæŃ¥ +å¸Ī åĤħ +é«ĺéĢŁ åħ¬è·¯ +好 åıĭ +ä¼ĺç§Ģ çļĦ +è¿Ľ äºĨ +æģIJ æĢķ +äºĨ åIJ§ +大 è§Ħ模 +çļĦ ä¸ĸçķĮ +æĢĢ çĸij +å· · +åħ´ å¥ĭ +æĪ ° +æĿij éĩĮ +æľĭåıĭ åľĪ +åĨ¬ 天 +ä¸Ńåįİ äººæ°ij +åįı åķĨ +è¯Ħ éĢī +æĹ Ń +å¢ŀåĬł äºĨ +åıĹ ä¼¤ +ä¸Ģ èĤ¡ +便 æį· +ä¸ ij +é¹ ¤ +å¤ĸ è§Ĥ +å·¥ç¨ĭ å¸Ī +åĴĮ åħ¶ä»ĸ +è¿Ļ å°± +ä¸Ńå°ı ä¼ģä¸ļ +西 åĮĹ +åĽ½æľī ä¼ģä¸ļ +èĭ¥ æĺ¯ +åı¯ æĥľ +çĶŁ æĹ¥ +åĩ ½ +ä¹° åįĸ +ç¥Ŀ ç¦ı +人æ°ij 群ä¼Ĺ +åħī æĺİ +åħ¬ å¯ĵ +æĺ¯ è°ģ +æĪij çŁ¥éģĵ +è¯Ń æĸĩ +æķı æĦŁ +ä¸įéĶĻ çļĦ +æĿ¥ 讲 +æ³¢ åĬ¨ +çļĦ 第ä¸Ģ +åľ° éľĩ +åľ¨ åħ¨åĽ½ +骨 å¹² +å®ī ç½® +家 çĶµ +ä¸İ æѤ +ä¸İæѤ åIJĮæŶ +åıĹ çģ¾ +çĥŃ çº¿ +çļĦ æĬĢæľ¯ +æµĭ éĩı +ä¾Ŀ èµĸ +ä¸ŃåĽ½ çļĦ +çī¹ æĢ§ +è¾ĥ é«ĺ +è¸ © +ä¼ļ åľ¨ +建 éĢł +导 èĪª +æĥ³ èµ· +åħ¨ ä¸ĸçķĮ +建 æĿIJ +ç¯ Ģ +çļĦ åŁºç¡Ģ +èĩªåĬ¨ åĮĸ +åīį åIJİ +çĿ¡ çľł +æݨ è¡Į +æį® äºĨ解 +ä»Ģä¹Ī æŶåĢĻ +ä¸į åĸľæ¬¢ +çħ¤ çĤŃ +éĤ£ä¹Ī å¤ļ +å¸Ĥåľº åĮĸ +ä¸į管 æĺ¯ +ç«ĭ åľº +éĥ½ 没 +课 é¢ĺ +æĪij们 å°Ĩ +è¿ĩ çļĦ +åĨį åĬłä¸Ĭ +çĪ ¾ +身 æĿIJ +çĶ· 女 +è¿ľ è¿ľ +çĶ· çĶŁ +èĩªèº« çļĦ +è´Ł æĭħ +çĻ¾ ä¸ĩ +西 çıŃ +西çıŃ çīĻ +åĩĢ åĪ©æ¶¦ +æ¾³ 大 +澳大 åĪ©äºļ +ä¸į åİ» +æī¿ åıĹ +楼 çĽĺ +å¢ĥ åĨħ +æ·· åĩĿ +æ··åĩĿ åľŁ +æĢĿæĥ³ æĶ¿æ²» +å¸Ĥ åĮº +æĭĽ æłĩ +åĽ¢ ä½ĵ +è¿Ľ 度 +åĨĽ éĺŁ +åıį å¼¹ +äºĨä¸Ģ äºĽ +æİ¥ å¾ħ +çļĦ åŃ¦ä¹ł +éħį éĢģ +é£Łåĵģ å®īåħ¨ +æĽ¿ 代 +æĺ¯ ä»¥ +éĢļ çĶ¨ +çłĶ究 æīĢ +ç¦ ħ +æī Ķ +éļĶ 离 +ä¸ĩ å¹³æĸ¹ç±³ +çļĦ è§Ħå®ļ +ç»Ļ æĪij们 +æ¿Ģ åħī +ä¼ļ åĩºçİ° +çŁŃ ä¿¡ +ç©¿ çĿĢ +æ²Ī éĺ³ +æķĻ æĿIJ +éĺ² çĸ« +ä¼ĺ èī¯ +约 å®ļ +æĪij çľģ +åħ¬ æ°ij +éģ¸ æĵ +éģ¸æĵ ĩ +å·² æĪIJ为 +ä¸į å¿ħ +ç¥ĸ åĽ½ +并 æľª +åľŁ 壤 +å¾® ç¬ij +äºĭä¸ļ åįķä½į +çļĦ 游æĪı +åħ¬ 示 +åIJĪçIJĨ çļĦ +çª Ŀ +æ°Ķ 象 +家 ä¸Ń +亮 缸 +åį« æĺŁ +è®° è½½ +è§Ĩ éĩİ +åľ°åĮº çļĦ +ä½Ĩ ä»ĸ +èĤĮ èĤī +äºı æįŁ +åĬŀ åѦ +ä¸Ģ è¡Į +è¯ŀ çĶŁ +åıijå¸ĥ çļĦ +çļĦ æľįåĬ¡ +çļĦ çłĶ究 +åij¨ æľ« +产ä¸ļ åĽŃ +é«ĺ 温 +æĪIJåĬŁ çļĦ +æŃ¥ 骤 +åŃĺ åĤ¨ +åŃIJ åħ¬åı¸ +让 她 +ä¸Ń æľī +åĺī 宾 +å¦ ® +æĺİ å¹´ +äºĨ åIJĹ +äºī è®® +æĪ Ī +ä¸Ģ æľ¬ +ç¾İ丽 çļĦ +ä½ł 说 +大 人 +æĶ» çķ¥ +ä¸į æľĥ +å¾ħ éģĩ +ä¸Ģ è¾Ĩ +çīĪæĿĥ æīĢæľī +æ°ij ä¼Ĺ +åĬŁ 夫 +å±ķ ä¼ļ +大 èĦij +æ¯ı æľĪ +å°ı 麦 +æµĻæ±Ł çľģ +çļĦ æīĢæľī +ä¸ĭ æ»ij +èĵĿ èī² +è¦ģ æĥ³ +åѦçĶŁ çļĦ +å½ĵ ä½ł +ä½ľ æĪĺ +家 乡 +å¤ļ åIJį +é«ĺ äºİ +åĿļ 强 +è¿ŀ éĶģ +åIJİ æŀľ +人 äºĭ +ç´ ħ +æ¿Ģ åĬ¨ +è¿Ľ æĶ» +ç© Ĩ +ä¸ ĺ +让 èĩªå·± +以 æѤ +夫 人 +å¼Ģ 设 +æ°Ķ è´¨ +鸡 èĽĭ +çĦ¡ æ³ķ +åIJĥ äºĨ +åĪĨåĪ« 为 +èģĶåIJĪ åĽ½ +å½ĵ 代 +å¦Ĥæŀľ æĺ¯ +è¿ľ ç¨ĭ +åĸ Ĥ +è®° ä½ı +æ¸ħ åįķ +åIJĪä½ľ ä¼Ļä¼´ +åİ» åģļ +æķħ éļľ +模 æĭŁ +å¸Ī çĶŁ +åīį æĿ¥ +çĶµè§Ĩ åī§ +çĥŃ çĪ± +éľ² åĩº +é«ĺ å±Ĥ +çĶµ åĻ¨ +纪 å¾ĭ +å¼Ģåıij åķĨ +éķ¿ å®ī +è½½ ä½ĵ +çļĦ å°±æĺ¯ +被 人 +åıĹ çIJĨ +篮 çIJĥ +èİ İ +交 ç»Ļ +æľªæĿ¥ çļĦ +两 大 +åIJķ å¸ĥ +çŃī 人 +çļĦ æĹ¥åŃIJ +åIJĪä½ľ 社 +æĮij éĢī +åŃĺ 款 +ç³»ç»Ł çļĦ +æĬĬ å®ĥ +没æľī ä»Ģä¹Ī +ä»İ æѤ +ä¸Ń åįĪ +çĸ¼ çĹĽ +å·© åĽº +浪 漫 +缸åħ³ éĥ¨éŨ +éķ¿ åŁİ +纤 ç»´ +ä¸Ĭ éŨ +çĪĨ çĤ¸ +èµ· çĤ¹ +çļĦ éĢļçŁ¥ +èĢĮ æĿ¥ +çļĦ èĢģ +æīĭ éĩĮ +è¯Ń éŁ³ +è¾Ľ èĭ¦ +æ±Łèĭı çľģ +çĶ¨ äºĨ +身份 è¯ģ +æľī åĬ© +æľīåĬ© äºİ +çī© èģĶç½ij +åĩº éŨ +å¼Ł åŃIJ +æĥ ¹ +è¿Ļ件 äºĭ +æĪij们 åı¯ä»¥ +çļĦ çĶŁåij½ +æľīä¸Ģ ç§į +åºĹ éĵº +åıĮ æīĭ +çļĦ æ¶Īæģ¯ +èĢIJ å¿ĥ +å°´ å°¬ +éĤ£ 天 +é¦ĸ æī¹ +æĺ¯ä¸Ģ 家 +人 æ°Ķ +åıį æŃ£ +æĪij åĴĮ +å®ł çī© +ä¸į 对 +寻 æ±Ĥ +缸 ä¼¼ +åľ¨ ç¾İåĽ½ +åı« åģļ +åĹ İ +ç«ĭ 足 +çĶ¨ éĢĶ +åħ Ĩ +大 æ°Ķ +åIJij ä¸Ĭ +ä»ĸ å°± +é¡¹çĽ® 建设 +èĭ¥ å¹² +æĺ¯ æľī +æ¿Ģ æĥħ +çļĦ æĦıä¹ī +æĺ Ń +严éĩį çļĦ +å¯Ĩ éĽĨ +èĪŀ è¹Ī +èį£ èİ· +èİ· æĤī +æ±Ł åįĹ +åģĩ å¦Ĥ +æĪ· å¤ĸ +线 ç´¢ +ç§ģ 人 +转åŀĭ åįĩ级 +çļĦ ä»·åĢ¼ +åįķ çĭ¬ +èĢģ çĻ¾å§ĵ +å°į æĸ¼ +åĽ½éĻħ åĮĸ +ä¼° åĢ¼ +æľįåĬ¡ ä¸ļ +èĩ Ń +æİī äºĨ +解åĨ³ äºĨ +ä¹Ł ä¸įèĥ½ +åħ ¹ +æĸ¯ çī¹ +æķħ æĦı +è¿ĩ 度 +èĬĤ æĹ¥ +çĻ½ çĻľ +çĻ½çĻľ é£İ +继 æī¿ +äºĨ ä¸įå°ij +äºĮ 人 +è§ģ éĿ¢ +æĥ³ æĥ³ +å¤į åIJĪ +康 å¤į +åİ¿ åŁİ +åľ¨ åĽ½åĨħ +åľº åľ° +éĻ¶ çĵ· +è¿Ļ 项 +çľ¼ ä¸Ń +çł ¸ +æĦŁè§ī åĪ° +æŀľ çĦ¶ +æĶ¾ åħ¥ +约 æĿŁ +æİĴ æŁ¥ +车 主 +çļĦ æĦıæĢĿ +æĸ° åŁİ +æĥ³ çĿĢ +éģ Ĥ +èĮ¶ åı¶ +ä¹° æĪ¿ +åĨľ æĪ· +é«ĺ æīĭ +çİī ç±³ +æĸ°åĨł èĤºçĤİ +çħ§ æĺİ +æĮĩ åįĹ +è¸ ¢ +æķij æı´ +æĻ¯ çĤ¹ +ç¨İ æĶ¶ +çļĦ æīĭ +æŃ£ 好 +è¦ģ æĬĬ +éļı æĦı +åħ¶å®ŀ æĺ¯ +ç»Ļ èĩªå·± +è°Ī åĪ¤ +æ¯ı天 éĥ½ +æĢģ åĬ¿ +é¢Ħ 约 +åİĨåı² ä¸Ĭ +å®Ŀ è´Ŀ +åīį è¿Ľ +ä¹Łå°±æĺ¯ è¯´ +çļĦ æĦıè§ģ +åı£ 罩 +åİĺ ç±³ +èĬ± è´¹ +ä½ĵèĤ² æĬķ注 +åħ¬ä¼Ĺ åı· +èijĹåIJį çļĦ +å¼Ģ æĪ· +æĭį åįĸ +å²ģ æľĪ +åĨħ 涵 +å®Įæķ´ çļĦ +é«ĺ åİĭ +åħ¬åĬ¡ åijĺ +使çĶ¨ çļĦ +çĶŁäº§ 线 +妹 妹 +èµ° 访 +æĺ¯ åı¯ä»¥ +åľ¨ 家 +æļ´ åĬĽ +æ³° åĽ½ +è´¨ çĸij +ä¸į éģİ +天çĦ¶ æ°Ķ +缺 çĤ¹ +å°ı åŀĭ +ä¸įä»ħ æĺ¯ +é»ij æļĹ +æ¢ ¨ +æĸĩ æĹħ +è¦ģ æľī +ä¸Ń å±± +çļĦ æķ°æį® +å¾Ĺ å¾Ī +以 便 +对 ä»ĸ +åĬł 以 +çĻ¼ çı¾ +设 å®ļ +èĤļ åŃIJ +éĿ ĸ +å¥ī çĮ® +ä¸į åıĺ +åı£ ç¢ij +åľ¨ åĵªéĩĮ +ä½ IJ +è¿Ļ 两个 +çļĦ æĸ¹åIJij +æŀ « +äºĮ 次 +çīĩ åĮº +éł IJ +ç£ Ĭ +æĭ¿ çĿĢ +å·²ç»ı æĪIJ为 +ä¹ĭ ä¸Ĭ +å®Ĺ æŨ +奶 奶 +é«ĺæĸ° åĮº +社 æľĥ +è·Ł 踪 +æľįåĬ¡ ä¸Ńå¿ĥ +æī ¯ +æīĭ æĮĩ +礼 çī© +宿 èĪį +çĶ¨ å¿ĥ +æıIJé«ĺ äºĨ +亮 çĤ¹ +ä¸į æĦ¿æĦı +æĴŃ æĶ¾ +å¤ļå°ij éĴ± +没 ä»Ģä¹Ī +æķ° åįģ +æĢ» çĽij +çļĦ åŁİå¸Ĥ +æī¾ åĪ°äºĨ +åĨħ åľ° +åĪ° çİ°åľ¨ +æĪĺæĸĹ åĬĽ +åİŁ å§ĭ +åĥ § +åĢĴ æĺ¯ +æľĢ åħ· +è´«åĽ° æĪ· +éĢģ åĪ° +级 åĪ« +åĩº èµĦ +æĪª æŃ¢ +ç§į åŃIJ +èĥ½ ä¸įèĥ½ +幸 è¿IJ +èĸ ĩ +项 éĵ¾ +æĮĤ çīĮ +ä¸Ģ 樣 +ä¹ĺ 客 +èIJ½ åIJİ +ä½Ĩ æĪij +æĹ© åľ¨ +åĬ¨ 漫 +å¹³ çŃī +对 ä½ł +ä¸į æĢķ +å¤ĸ çķĮ +å¤ļå¹´ æĿ¥ +é¦ĸ 个 +æ²³ åįĹçľģ +æĪĸ åħ¶ä»ĸ +éķľ 头 +åįĹ æĺĮ +ä¸Ģ éĿ¢ +éĢłæĪIJ çļĦ +å´ Ķ +çŃ Ĵ +æķĻèĤ² éĥ¨ +åľ° åŁŁ +æĺĨ æĺİ +å·´ é»İ +æīĭ 游 +ä¸Ģ æŶ +çł į +顶 级 +åħ± 计 +åİŁ æ²¹ +è¾ī çħĮ +说 æĺ¯ +æĸ°åįİ ç¤¾ +ç»ıåİĨ äºĨ +ä¸į æŃ¢ +è¦ģ ä¹Ī +èĢħ çļĦ +æĢ» æĬķèµĦ +è¡Į 驶 +ä¸Ĭ å¸Ŀ +å¹´ 纪 +çIJ ¼ +ä¼ł 说 +ç²¾ èĭ± +æĸ¹ éĴĪ +æ±Ł æ¹ĸ +æĪIJ çĤº +æĢ» éĩı +æĬķ æĶ¾ +åĬ¨ çĶ» +èĹ ¤ +çĶµ æºIJ +éĴ Ļ +åIJĮ è¡Į +æĻ®éĢļ çļĦ +åĽ¾ä¹¦ é¦Ĩ +è¯Ī éªĹ +æħĪ åĸĦ +è¿Ļ 份 +主æĮģ 人 +å°± è¿Ļæł· +èĢĮ æĪIJ +èĩªè¡Į 车 +ä¸ŃåĽ½ çī¹èī² +èĤ¿ çĺ¤ +åIJ ¾ +å¼Ł å¼Ł +åıĹ çĽĬ +éĢīæĭ© äºĨ +æĺİæĺ¾ çļĦ +æĬ¥ èĢĥ +ç¬ij éģĵ +éĽĸ çĦ¶ +温 å·ŀ +éĿŀ æ´² +ç§į ç§į +åıĤåĬł äºĨ +è´§ è¿IJ +éļı 便 +å°± 没æľī +ç¸ £ +央 è§Ĩ +ç©¿ è¶Ĭ +çļĦ çݰ象 +åĩł 次 +çļĦ é£İéĻ© +æŃĮ æĽ² +æľ¬ å±Ĭ +å¹´ åĨħ +ä¸į è¶ħè¿ĩ +è¿ĩ å¤ļ +å¿ħé¡» è¦ģ +ç»ĵ 论 +åĢŁ éī´ +ç¥ŀ å¥ĩ +æľŁ æľĽ +ä¸ĵ 享 +éĿŀ常 éĩįè¦ģ +æĦıè¯Ĩ åĪ° +åIJĪ 并 +æĬĬ èĩªå·± +å¥Ĺ è£ħ +éŃĶ æ³ķ +å¤ı åŃ£ +ä¸į åĥı +å¢ĥ çķĮ +æĥĬ åĸľ +æľīä¸Ģ 天 +çĦ¦ çĤ¹ +æĪij 认为 +åħ° å·ŀ +çĶµ æ°Ķ +èģĶç³» æĪij们 +ç§ij æĻ® +她 说 +çļĦ æĸĩç«ł +å¥ĩ æĢª +åıĭ 好 +饮 æĸĻ +çļĦ æĶ¯æĮģ +çŃĶ åºĶ +éĩį éĩı +çij ¶ +åĩı è½» +ç§ijåѦ 家 +å·´ 西 +éĩijèŀį æľºæŀĦ +åħļ å§Ķ书记 +貸 款 +ç²¾ èĩ´ +ä»İ æľª +åį° åĪ· +åĽŀ 顾 +é¦ĸ éĥ½ +åıij èĤ² +éĹ® éģĵ +è¾¾ åĪ°äºĨ +å¿į ä¸įä½ı +æīį æľī +æįIJ èµł +ä½Ľ æķĻ +ä¸į æ¸ħ +éĺŁ éķ¿ +缸 åıį +æĬ¥ èѦ +大 åħ¨ +欧 缣 +帮 å¿Ļ +çļĦ æĻĤåĢĻ +缮 å½ķ +足 以 +èī° éļ¾ +ä»ĸ ä¹Ł +å·¥ ä½ľèĢħ +头 èĦij +缺 éĻ· +æĪIJç«ĭ äºĨ +å°± å¼Ģå§ĭ +认 åIJĮ +é»Ħ èī² +çĹħ æĥħ +覺 å¾Ĺ +è¿Ļ 两 +ä¿¡ ä»° +åľĭ 家 +ä¸įä»ħä»ħ æĺ¯ +çĭ¬ 家 +èĪ¬ çļĦ +æĿIJ è´¨ +æµ· ä¸Ĭ +çĤº äºĨ +æľºåĬ¨ 车 +缸å½ĵ äºİ +å¤ļåħĥ åĮĸ +æĽ´ 大çļĦ +èĽ ® +åģĩ æľŁ +å¼ı çļĦ +交éĢļ è¿IJè¾ĵ +çľģ å§Ķ +ä¸į ç®Ĺ +æĶ¾ ä¸ĭ +éĹ ¯ +人 åľ¨ +港 åı£ +æŨ åľ¨ +åij½ 令 +æŁIJ 个 +å¹³ 稳 +åıª 好 +人 人 +äº ŀ +äºĮ ç»´ +äºĮç»´ çłģ +æŀģ 为 +åĪ« å¢ħ +åħ¶ ä½Ļ +大 äºĭ +主管 éĥ¨éŨ +æĹł éĶ¡ +éĹ µ +éģŃ åĪ° +说 è¿ĩ +为 ä½ł +解 çŃĶ +éªĮ æĶ¶ +çļĦ ç»ıéªĮ +åĮ¹ éħį +çģ« ç®Ń +豪 åįİ +æŁIJ æŁIJ +çļĦ æĹ¶ä»£ +书 éĿ¢ +æģĴ 大 +延 éķ¿ +ä¸Ģ åIJĮ +æľª èĥ½ +交 æį¢ +çĶ¢ åĵģ +çŃī åĪ° +åĪĨ 离 +æīĵ çĶµè¯Ŀ +å¹² çĩ¥ +è¾ĥ å¤ļ +å¤ļå¹´ çļĦ +èĥĮæĻ¯ ä¸ĭ +为 ä¾ĭ +æijĺ è¦ģ +å´Ľ èµ· +æѤ åĪ» +æľī æľºä¼ļ +æĿ¡ 款 +é¢Ĩ导 å°ıç»Ħ +çļĦ 身ä½ĵ +åįķ ä¸Ģ +央 è¡Į +ä¸įæĸŃ æıIJé«ĺ +ä»·åĢ¼ è§Ĥ +èĬ ½ +èIJ į +æ³ķå¾ĭ æ³ķè§Ħ +ä¸į éĶĪ +ä¸įéĶĪ éĴ¢ +åĩº äºİ +èĻļ æĭŁ +æį® æĤī +çĥ¦ æģ¼ +åħ¨ æĸ°çļĦ +æī« æıı +çĻ» éĻĨ +èīºæľ¯ 家 +çļĦ é£Łçī© +çļĦ åŃĺåľ¨ +客 åİħ +æĪij们 å°± +æŁ¥çľĭ æĽ´å¤ļ +è¯Ħ 审 +å¸Ĥ åł´ +è¬ Ľ +å·¨ 头 +ä¸ŃåĽ½ ç»ıæµİ +äºĨ èĩªå·±çļĦ +åĨ³ è®® +çĽijçĿ£ 管çIJĨ +æĬķ 票 +åĨį 度 +è¡Į çĤº +注 åħ¥ +ä½ľä¸º ä¸Ģ个 +æ¯ı个人 éĥ½ +åįķ åħĥ +è¦ģ çŁ¥éģĵ +被 称为 +ä¹ĭ éĻħ +解 éĻ¤ +ä¸ ¸ +æº « +ä¸ī æĺŁ +é²ľ æĺİ +ä¹Ł éĥ½ +æŶ æľº +åĩº æīĭ +æĥħ å½¢ +åķĨ è´¸ +éĢī 举 +对 èĩªå·± +çĶŁ åĬ¨ +åħĭ æľį +个 ä½ĵ +èĭ ij +ç¨ ± +大 åݦ +æĺ¯ å¯¹ +åĪ© æģ¯ +è¿IJåĬ¨ åijĺ +åĮĸ 解 +åīį 沿 +æĦŁ æģ© +æĢ» ä¹ĭ +é«ĺæĸ° æĬĢæľ¯ +åĿĩ 为 +åħ¨ åĮº +æ°Ķ æ°Ľ +åı¯ä»¥è¯´ æĺ¯ +ä½ı 宿 +åħļåijĺ å¹²éĥ¨ +åĹ ¯ +è·µ è¡Į +çļĦ ä¸ĵä¸ļ +èĢĥ éªĮ +èķ ¾ +åħ¬ åŃIJ +çļĦ çĬ¶æĢģ +æ½® æµģ +ä¿¡ æīĺ +è´ ¼ +åIJĦ æĸ¹ +æķij åĬ© +éĿŀ常 çļĦ +æ¡¥ æ¢ģ +åħ¬ æĸ¤ +ä¼¼ çļĦ +çľĭ 好 +å±Ģ éĥ¨ +å®ī éĿĻ +éħį 件 +常 è§Ħ +å¼Ģ 车 +第äºĮ 次 +ä¸Ĭ 级 +åıĤ èµĽ +家 å±ŀ +强 åĬ¿ +åľ¨ ä»ĸ +åIJij åīį +ä¹ĭ åľ° +éĥ ¡ +è¡Į ç¨ĭ +èѦ åijĬ +è§Ħå®ļ çļĦ +åķĨ åŁİ +äºĶ 大 +æķĻ 室 +åįģ 足 +æīĢ以 åľ¨ +å°Ĩ 继ç»Ń +çŃī æĸ¹å¼ı +家 ä¼ģä¸ļ +交 ä»ĺ +çĤ¹ è¯Ħ +ç»ĵ ç®Ĺ +ä¹Ł åı¯ +å¤ĸ æ±ĩ +è¿Ļç§į æĥħåĨµ +æİĪ äºĪ +å¸ĥ ç½® +æĪIJç«ĭ äºİ +é¢Ħ èѦ +管çIJĨ 人åijĺ +å©ļ 礼 +ç»ĵæĿŁ åIJİ +åħ¥ éĢī +æĹł æ¯Ķ +åĴĮ åıijå±ķ +çĻ½ éħĴ +çİ© åħ· +ä¸ĩ ç¾İåħĥ +çļĦ æĪIJ绩 +æĭį çħ§ +èĢĥèĻij åĪ° +ä¼ģä¸ļ åıijå±ķ +äºĨ 个 +çĶŁ æ°Ķ +çļĦ 女人 +äºĶ åįģ +çĪ· çĪ· +纽 约 +éĥ½ 被 +ä¸Ĭ 课 +çĽ ¡ +ä¼łç»Ł æĸĩåĮĸ +æ½ľ åľ¨ +åıij å°Ħ +ä¸Ģ 身 +éĺ² å®Ī +åĪ ® +é¢ĺ 缮 +åľ¨ åĨħçļĦ +ç¾İ 好çļĦ +è¿ĻéĩĮ çļĦ +ä¸Ģ ä¸Ŀ +人 åĿĩ +åĢ¡ 导 +身 åIJİ +æī© å±ķ +大 éŨ +å°± 被 +该 é¡¹çĽ® +æŀ¶ æŀĦ +ä¸Ģ åı£ +ä¿¡æģ¯ æĬĢæľ¯ +å¼Ģ ä¸ļ +æĶ¶ åıĸ +ç½ij 页 +æĶ¯ æı´ +å°ģ éĹŃ +å¡ij éĢł +大 èĥĨ +å¿«éĢŁ åıijå±ķ +çľĭ ä¼¼ +æ¸ Ŀ +è¿Ļæł· ä¸Ģ个 +模 åĿĹ +注æĦı åĪ° +çł´ 解 +èĩª ä»İ +åijµ åijµ +ä¹ĭ å¾Į +ä¹ĭ æĹħ +è·Ł æĪij +æ³ķ 人 +æİĴè¡Į æ¦ľ +åĿļ å®Ī +好 å¤Ħ +çŁ³ 头 +并 å°Ĩ +èĪ ± +æŃ ĩ +两 岸 +å¤ļ ä¹ħ +象 å¾ģ +个æĢ§ åĮĸ +çļĦ è§Ĵ度 +å¸ Ĩ +ç¦ı å·ŀ +æŁ¥ å¤Ħ +两 åĽ½ +åIJ¸å¼ķ äºĨ +é¦ĸ å¸Ń +大 åĵ¥ +é¤ Ĭ +涨 å¹ħ +éĢī çĶ¨ +許 å¤ļ +èIJ½ æĪ· +åĵĪ å°Ķ +åĵĪå°Ķ 滨 +åģļ ä»Ģä¹Ī +以 åħį +é¾ į +æĹł éľĢ +åĪ°åºķ æĺ¯ +æĢ ¡ +åijĬè¯ī ä½ł +éĺ² æ°´ +è¿Ļ æŶåĢĻ +欢 ä¹IJ +转 åIJij +è¿Ļ个 åľ°åĽ¾ +åħ¥ é©» +èįī åİŁ +æĹ¶ä»£ çļĦ +åıĺ åĬ¨ +åĬłå¼º 对 +åģ¶ å°Ķ +å®Ī æĬ¤ +æ°Ķ 温 +人 éĹ´ +æľĿ é²ľ +ç»ı è´¹ +åĽŃ æŀĹ +å·¥ åľ° +è§Ħ æł¼ +åĩł åįģ +è¯ķ åĽ¾ +å¦ ĥ +éĤ£ æŶåĢĻ +å¼ĺ æī¬ +ä¸ļ çķĮ +çļĦ éĢŁåº¦ +ä¼ļ ä¸įä¼ļ +èIJ¥ æĶ¶ +å°ıå¾® ä¼ģä¸ļ +çľĭ è¿ĩ +æĬĬ ä»ĸ +éģµ 循 +è¿Ļ è¾¹ +没æľī 人 +å£ ¶ +æ¹ĸ åįĹçľģ +æŀģ åħ¶ +çļĦ人 çĶŁ +ä»ĸ è¿ĺ +转åĮĸ 为 +èµ° è¿ĩ +æĬ± çĿĢ +çīĽ 奶 +ä¸ĩ 亩 +å¿ĥ æĢģ +æĹ¥å¸¸ çĶŁæ´» +ä½ĵ æ£Ģ +æĻ ĥ +çŃī é¢ĨåŁŁ +æĩī 該 +åı¯ä»¥ çľĭåĪ° +æī¾ ä¸įåĪ° +èĢģ å¹´ +æĬĬ æĪij +积 åĪĨ +梳 çIJĨ +ç» ³ +çļĦ æĶ¿æ²» +å¸Ŀ åĽ½ +éĻª ä¼´ +æ´Ľ éĺ³ +åħ¬ æŃ£ +å¼Ģ åı£ +çī¹èī² çļĦ +åĽ° å¢ĥ +ä¸Ĭ æľī +ç«ĭ ä½ĵ +æīĵ å·¥ +åķ¤ éħĴ +åľ¨ éĤ£éĩĮ +éĤ£ è¾¹ +个 åĪ« +ä¸Ģå®ļ æĺ¯ +çļĦéĩįè¦ģ æĢ§ +主 å¼ł +åĴĮ æľįåĬ¡ +ä¸Ĭ ç½ij +è¡¥ åĬ© +åıª éľĢ +å¼ ¦ +éģ ® +åĬĽ äºī +度 è¿ĩ +èij ¬ +é¡¿ æŶ +éĦ ī +纺 ç»ĩ +åľ° åĿĹ +ä¿¡çĶ¨ åį¡ +ç½ļ 款 +åijĬè¯ī æĪij +éĽ Ļ +书 çĶ» +è¨Ń è¨Ī +æĢ» ä¼ļ +åĪ¤ åĨ³ +ä¿¡ èªī +个 èĤ¡ +å¹³ 常 +æĢİ éº¼ +ä½ĵ çİ°åľ¨ +é»Ħ æ²³ +åĽĽå·Ŀ çľģ +羣 缸 +åIJĦ项 å·¥ä½ľ +åĬ¨ åijĺ +å³° ä¼ļ +ä¸Ģ æľŁ +æľī ä¸Ģå®ļçļĦ +é«ĺ度 éĩįè§Ĩ +ç¹ģ èį£ +åıijçİ° äºĨ +ç½ij 红 +æīĭ æ³ķ +家 åĽŃ +仪 åĻ¨ +è¾ĥ ä½İ +çļĦ å®īåħ¨ +æ¡ IJ +ä»ĺ 款 +æĬij åĪ¶ +åįĵ è¶Ĭ +æŃ£ éĿ¢ +åĵ ij +强 åĪ¶ +ä»Ĭ天 çļĦ +æĪĺ èĥľ +楼 å¸Ĥ +æĭ¿ ä¸ĭ +é¢ľ åĢ¼ +举 éĥ¨ +çłĶ åĪ¶ +çļĦ æĪĺçķ¥ +åľ¨ ä¸Ģ个 +ä¸ī 人 +å®Į äºĨ +æĸ° æĬĢæľ¯ +ç»ıæµİ æķĪçĽĬ +å¯Į æľī +æ¾³ æ´² +åĬ© çIJĨ +é¢Ĩ åıĸ +è° Ń +çĩĥ çĥ§ +ç´ł åħ» +éĤĦ æľī +è¿Ľ èĢĮ +ä»Ģä¹Ī æĺ¯ +çłĶ究 ä¸Ńå¿ĥ +éĢĤ çĶ¨äºİ +æİ¥ æĶ¶ +失 æľĽ +äºĮ 级 +éĹ´ çļĦ +åİŁ æłĩé¢ĺ +èªį çĤº +æį ¡ +对 çĿĢ +对 éĿ¢ +ä¸Ń åİŁ +éĵ ĥ +çĶŁäº§ çļĦ +åıijå¸ĥ ä¼ļ +士 åħµ +è¿Ļ åı¥è¯Ŀ +ç¼´ 纳 +ä¸Ģ个 个 +åѸ çĶŁ +çĸij éĹ® +交 èѦ +示èĮĥ åĮº +天 使 +åľ¨ ä¸Ĭæµ· +åIJĮ æĻĤ +è½» æĺĵ +åĶ¯ä¸Ģ çļĦ +çĥŃ éĹ¹ +ä¹IJ è§Ĥ +çļĦ 身份 +åĸĦ äºİ +大 åİħ +èĤ¯å®ļ æĺ¯ +éĺ² çģ« +å¤ĸ åĩº +æį® 说 +é¡¹çĽ® çļĦ +ä¸Ģ åı° +èĻļ åģĩ +ä¸Ģ ç¬Ķ +ç«ĭ æ³ķ +严 èĤĥ +æī¿ åĬŀ +åįģ åĩł +çļĦ 空éĹ´ +æľ¬ ç½ijç«Ļ +åģļ å¾Ĺ +ä¿Ŀ 温 +æľĪ åĪĿ +åľ¨ ç½ijä¸Ĭ +åIJĦ æĸ¹éĿ¢ +ä¸ī 天 +交æĺĵ æīĢ +解 æŀIJ +åħļ ä¸Ń央 +è¿Ľ åĩºåı£ +åĴĮ 社ä¼ļ +次 æķ° +ä¹ĭ 家 +ç»´ 度 +æ´¾åĩº æīĢ +产çĶŁ äºĨ +带 æľī +å¾Ī 强 +æľīäºĽ 人 +å¹´ åIJİ +äºĨ 许å¤ļ +å¯Ĩ 度 +åѦ æľŁ +çıł æµ· +æľĢå¤ļ çļĦ +è¾¹ ç¼ĺ +容 éĩı +第äºĮ 个 +ä¸Ģ缴 æĺ¯ +ä¸į ç¦ģ +æŃ ² +ä»ĭç»į äºĨ +ä¼ĺ éĽħ +æ¯Ķ è¼ĥ +èģĮ ä½į +温 æŁĶ +æľī éĴ± +æľĢ é«ĺçļĦ +åįļè§Ī ä¼ļ +ä¸į æĪIJ +éĶĻ äºĨ +è¯ģ çĽij +è¯ģçĽij ä¼ļ +æĪIJ 人 +åĿĩ åĮĢ +æľī åĪ© +è¶Ĭ åįĹ +æīĵ äºĨ +好 åIJĥ +ç³» çµ± +è·Ł éļı +çļĦ åľ°ä½į +æŃ£ å¦Ĥ +ç¨į å¾® +åį° åıij +åĪĽ ç«ĭ +é£İ åħī +å°Ĩ æĪIJ为 +ä¸į é«ĺ +é¢ij ç¹ģ +设 æľī +ä¼ ŀ +æĭĨ éĻ¤ +å½± åĥı +æ¸Ĺ éĢı +å¹´ å¼Ģå§ĭ +ç½ij æĺĵ +è¦ģ åģļ +çĶµåĬ¨ 车 +羣 å¿ĥ +æµ· åĨĽ +ä¼ł æĿ¥ +å·® åĪ« +è°¨ æħİ +çĥŁ åı° +åįĥ å¹´ +è¯ģ å®ŀ +çIJ ª +çļĦ åħ·ä½ĵ +åĪ° å¤Ħ +ä¸į å®ľ +èľ Ģ +èĥ½åĬĽ åĴĮ +çīº çī² +çļĦ éĴ± +大 éĺŁ +é¦ĸ è¦ģ +ä¸į æĦ¿ +çİ« çij° +人æ°ij ç½ij +è¿ĺæĺ¯ è¦ģ +åĽĽ å¹´ +æįŁ 伤 +çļĦ åģļæ³ķ +éĿ Ī +è¡Ķ æİ¥ +åIJĪ æĪIJ +没 人 +éŨ æ§Ľ +ä¿¡ è´· +çļĦ 缸åħ³ +举 é£İ +社 ä¿Ŀ +ä¸ĭ 游 +åĿĹ éĴ± +è¿ĩ åIJİ +çļĦ åºĶçĶ¨ +é¥ ¶ +é¢ģ åıij +ä¸Ģ å¤Ħ +åįİ å¤ı +为 ä¼ģä¸ļ +åıª ä¼ļ +ä¾µ 害 +çļĦ åĬŁèĥ½ +åѸ ç¿Ĵ +ä¸Ńåįİ æ°ijæĹı +åıijå¸ĥ äºĨ +è¿İ æİ¥ +æĪij èĩªå·± +è¿ĺ éľĢè¦ģ +太éĺ³ èĥ½ +åİ» ä¸ĸ +æĺ¯ ä½ł +åIJĪ åĬĽ +ç»ĺ çĶ» +åı° åĮĹ +çĿ£ ä¿ĥ +åĮĹ éĥ¨ +æľī å¤ļå°ij +å¾Ī éĩįè¦ģ +åĪĴ åĪĨ +åı· 线 +æĶ¾ 大 +ä¼ļ 被 +èİ· å¥ĸ +ä¹ĭ åĨħ +失 åİ»äºĨ +çݩ家 们 +éĩĩ éĽĨ +å£ ¹ +家 ä¼Ļ +çĻ½ 天 +åĽłä¸º ä»ĸ +社ä¼ļ æ²»çIJĨ +å¼Ģ åĪĽ +çĶµ ç¼Ĩ +æĸ° ä¸Ģ代 +并 è´Ń +å°± å·²ç»ı +çļĦ 社ä¼ļ +éĻ¤ éĿŀ +åı¯ä»¥ çĶ¨ +å© ī +æ¯Ķè¾ĥ 好 +å®ŀ ä¸ļ +åĪĽ åĬŀ +æıIJ èµ· +é» ĥ +ä½ı åľ¨ +å¸Ĥ æĶ¿ +éĿ¢ä¸´ çļĦ +èĥ½ åľ¨ +çŁŃ çŁŃ +羣 人 +æĺİ æĺİ +èµĦ åĬ© +çļĦ ä¸įåIJĮ +å°ı æľĭåıĭ +é¢ĺ æĿIJ +ç¾İ åij³ +æĺŁ åº§ +ä¸į ä¸Ģæł·çļĦ +çľĭ ä¸Ĭåİ» +ä¸Ģ æł¹ +广 å·ŀå¸Ĥ +åıijçĶŁ çļĦ +é«ĺ ç§ijæĬĢ +ä¸Ģ è¾ĪåŃIJ +交 åıī +ä½ĵç³» 建设 +åĽłä¸º æĪij +çıį æĥľ +ä¸Ĭ åѦ +æĪĺ æľ¯ +æѤ ç±» +交 å¾Ģ +æĮī æij© +人们 çļĦ +åħ¶ 實 +åİŁ æĿIJæĸĻ +渴 æľĽ +缸 å¤Ħ +å¾® å¾® +æ® · +ä¹ĺ åĿIJ +å¼Ģå±ķ äºĨ +é«ĺ åĵģè´¨ +æĹłäºº æľº +ä¸įæĺ¯ å¾Ī +çļĦ æĬķèµĦ +èĬĤ çľģ +èĩ ī +ç²¾ éĢī +çļĦ æłĩåĩĨ +åįĹ éĥ¨ +认è¯Ĩ åĪ° +å¹³ éĿĻ +èĹ ¥ +æī« é»ij +æī«é»ij éĻ¤ +æī«é»ijéĻ¤ æģ¶ +éĢĻ 種 +建çŃij éĿ¢ç§¯ +ç¡® ç«ĭ +管çIJĨ åĬŀæ³ķ +æĦı å¿Ĺ +ä¸ ¨ +让 åŃ©åŃIJ +æķij çģ¾ +å½ĵ ä»Ĭ +çģ« çģ¾ +åIJĦ éĥ¨éŨ +ä¾µ çĬ¯ +æ¯ı åij¨ +æı ½ +ä¸Ģ次 æĢ§ +åħ¶ä»ĸ 人 +éĶĻ è¿ĩ +ä¸İ åħ¶ +åĭĩ æ°Ķ +çĩĥ æ°Ķ +é¦ĸ å±Ĭ +æľį 饰 +ç² ¥ +å®Į æ¯ķ +å°± æĬĬ +åĬŀäºĭ å¤Ħ +ä¸Ģä¼ļ åĦ¿ +离 ä¸įå¼Ģ +å¦Ĥæŀľ æĤ¨ +ä»ĵ åºĵ +导 å¸Ī +åIJĪéĢĤ çļĦ +毫 ç±³ +å®īåħ¨ æĢ§ +ä¾Ŀ çħ§ +产ä¸ļ åĮĸ +ä½ł çľĭ +羣çļĦ å¾Ī +åѤ çĭ¬ +éĺ² å¾¡ +å¾Ī ç®Ģåįķ +é£İ æ°´ +ä½Ĩ ä¹Ł +æݨ åĩºäºĨ +æ°ijèIJ¥ ä¼ģä¸ļ +çłģ 头 +å¤įæĿĤ çļĦ +ç»ĦæĪIJ éĥ¨åĪĨ +åħħ满 äºĨ +è¿ij åĩłå¹´ +çľģ æĶ¿åºľ +æľī å¿ħè¦ģ +éĻ ³ +ä¹ĭ ç±» +ä¹ĭç±» çļĦ +æĢ§ ä»· +æĢ§ä»· æ¯Ķ +åķĨ åºĹ +å¸Ĥ åĢ¼ +人æīį åŁ¹åħ» +æ·± åıĹ +管çIJĨ å±Ģ +æģIJ æĥ§ +ä»ħ æľī +æĬµ è¾¾ +æµ· åħ³ +èµĭ äºĪ +äºĭ åĦ¿ +ä»· éĴ± +æīĭ ä¸Ĭ +èĩª å¾ĭ +åħ³ çĪ± +享 æľī +éģĹ æĨ¾ +å¾Īå¿« å°± +æĽ´ å¿« +æłĩ è¯Ĩ +åºĨ ç¥Ŀ +ä¹Ł 好 +ä¸į æĺĵ +æĪij å¾Ī +æĶ¹éĿ© åıijå±ķ +å¤ĸ åľ° +æĬµ æĬ¼ +è¯Ĺ 人 +åİķ æīĢ +æĸ° åªĴä½ĵ +èĸ Ľ +è°Ī è¯Ŀ +ä¸Ģå®ļ ç¨ĭ度 +èµ° åľ¨ +æľĢ 强 +åĬŁ çİĩ +åħ± è¯Ĩ +大 æ¡¥ +ä¸ĭ æĸ¹ +å¤ĸ èµĦ +ç¢ ± +å·¡ è§Ĩ +æ¹ĸåĮĹ çľģ +个 çĻ¾åĪĨ +个çĻ¾åĪĨ çĤ¹ +çļĦ 责任 +çļĦ åĵģçīĮ +åĬ© æݨ +åĪĽéĢł äºĨ +ä»» èģĮ +å¿« æį· +æĿij åºĦ +åİ» çľĭ +æīį èĥ½å¤Ł +å± ¤ +æĪij 家 +æĺ¯ä¸Ģ 款 +ç¾ ħ +åĨ° éĽª +æŀģ 大 +çģ¯ åħī +éĨ ĭ +ä¸İ åħ¶ä»ĸ +æıIJåĩº çļĦ +éĿł è¿ij +è°ĥ åĬ¨ +å°½ åı¯èĥ½ +åıij åĬĽ +ç»Ļ 她 +éĢĤ éĩı +è·¨ åĽ½ +åħĪ è¡Į +æĸ° æĿIJæĸĻ +ä½ľ äºĨ +满 äºĨ +ä¸į 满 +çļĦçľ¼ çĿĽ +çľĭ å¾Ĺ +è¿Ļ ä¸Ģ次 +é½IJ åħ¨ +çļĦä¸Ģ éĥ¨åĪĨ +ä¸ Ļ +æ¸ħ æĸ° +說 æĺİ +身边 çļĦ +æīĢæľī 人 +å½° æĺ¾ +è± ¹ +åį ¿ +è¿IJ 转 +æĮĩ å¼ķ +å¸Ĥ åħ¬å®īå±Ģ +åıĤ å±ķ +ä¹ĭ æŶ +éĩijèŀį æľįåĬ¡ +èµĦæľ¬ å¸Ĥåľº +èĥ½ 让 +å¿ĺ äºĨ +天 åłĤ +æ¯Ķå¦Ĥ 说 +éĬĢ è¡Į +èĽĭ ç³ķ +çĶ © +æł¸ å®ŀ +æĻ® 京 +ä¼ĺ ç¾İ +åı£ èħĶ +漫 çĶ» +çľ¼ éĩĮ +äºĨ ä¸ĭæĿ¥ +æĪij们 ä¹Ł +ä¾ į +为 ä¸Ńå¿ĥ +å¥ĩ 迹 +éĿĴ çĿIJ +æĪªèĩ³ 缮åīį +åĩº ä¾Ĩ +æĢ» åħ¬åı¸ +å¼¥ è¡¥ +ç®Ĺ æ³ķ +å·¥ä½ľ 室 +æīĢ以 æĪij +æ°´ åĪĨ +æīĢ å±ŀ +ä¸į 说 +ä½Ĩæĺ¯ åľ¨ +è¦ģ åİ» +åĪĽä¸ļ èĢħ +ä¸į æ¸ħæ¥ļ +åĽĽ åij¨ +æĺ¯ ä»İ +çļĦ æł¹æľ¬ +çģ ¶ +æ¯Ľ æ³½ +æ¯Ľæ³½ 举 +æµ· åı£ +åĽĽ åįģ +ä¹Ł 被 +èģ · +ä¸Ģ æīĭ +绩 æķĪ +çļĦ çĶ·äºº +书 ç±į +ä¸Ģ èĦ¸ +大 äºİ +鼶 éĥ¨ä»¶ +åħ³ æĢĢ +å¹³ ç±³ +æļ´ éľ² +å¾Ĺ å¤ļ +ä¸ī 级 +æľ¬ åij¨ +两 èĢħ +对 ä¸ŃåĽ½ +åıª è§ģ +欧 ç¾İ +å¦Ĥæŀľ æľī +å·²ç»ı æĺ¯ +çľĭ å®Į +çģ« éĶħ +èµ IJ +ä¸Ģ éģį +æĦŁ åĨĴ +ç»ĵ å±Ģ +ä»ĵ åĤ¨ +å®ŀ åľ° +åī¯æĢ» ç»ıçIJĨ +ä¹Łä¸į çŁ¥éģĵ +碰 åĪ° +åIJĪ 计 +客æĪ· çļĦ +ç½Ĺ 马 +æĦī å¿« +é£ Ľ +çĥŃ çĥĪ +伦 æķ¦ +åĮ» ä¿Ŀ +éĺ¿éĩĮ å·´å·´ +åĨį 说 +为 åŁºç¡Ģ +çĶŁäº§ ç»ıèIJ¥ +è¿ĻäºĽ 人 +åĪĹ è½¦ +æ²³åĮĹ çľģ +è¿Ļ 段 +æ´»åĬ¨ ä¸Ń +å© · +çĶŁ çIJĨ +ä¸ŃåĽ½ 人æ°ij +éĦ Ĥ +åIJ¬ åıĸ +å¤į ä¹ł +æľī çĽĬ +æĶ¶ æĭ¾ +å¾Ī åı¯èĥ½ +ç½ij绾 游æĪı +们 çļĦ +èµĭ èĥ½ +éļ¾ å¾Ĺ +åĪĨ æīĭ +羣 è¯ļ +åħ¬åı¸ åľ¨ +åĿĩ è¡¡ +åı£ åij³ +çīµ 头 +ä¸ĢèĪ¬ çļĦ +轿 车 +çŃī äºİ +æ²ī é»ĺ +æĪij éĥ½ +å°ı ç¨ĭåºı +ä¸Ģ åī¯ +æī¿ è½½ +åľ° è´¨ +çķĮ éĿ¢ +çĶµ æľº +çĦ¦ èĻij +éĶĢåĶ® é¢Ŀ +æĸ° 车 +ä¸Ĭ 游 +主 æ¼Ķ +éļIJ ç§ģ +åıijå±ķ æĪĺçķ¥ +çļĦ åĬªåĬĽ +å¼Ģ åħ³ +解åĨ³ éĹ®é¢ĺ +çĿ£ 导 +对 æĬĹ +å¾Īå¤ļ 人éĥ½ +æĹł æķĪ +产åĵģ è´¨éĩı +å®ī å¿ĥ +åįİ äºº +ä¸į 符åIJĪ +èĩª 家 +éĺµ å®¹ +çļĦ åIJĦç§į +çļĦ çIJĨ念 +çļĦ æĸĩåĮĸ +为 èĩªå·± +å±± æ°´ +游 æ³³ +éľĩ èį¡ +çĶŁæ´» æĸ¹å¼ı +è¿ľ 离 +çŁ³ åĮĸ +æѤ äºĭ +æĺ¯ çľŁçļĦ +çļĦ æ¯Ķä¾ĭ +çĶ¨ çĶµ +奥è¿IJ ä¼ļ +ä¿Ŀ å®ī +èĽĭçĻ½ è´¨ +çļĦ å¿ĥçIJĨ +å· « +åı· çłģ +æ°Ķ ä½ĵ +åıij æĶ¹ +åıijæĶ¹ å§Ķ +åĮ» å¸Ī +æ¶Ĥ æĸĻ +æĺ Ĭ +å¸Ĥ 级 +ä¸ĸçķĮ çļĦ +åĪĨåĪ« æĺ¯ +çł´ 产 +ä¸Ģ æĿ¯ +æĭī å¼Ģ +å¹³ åĩ¡ +çļĦ åıijçĶŁ +åĬ¨ æīĭ +ä¸Ģ缴 以æĿ¥ +æīĭ å·¥ +éĩĮéĿ¢ çļĦ +æĹł åħ³ +ä»ĭ åħ¥ +èµ° ä¸Ĭ +å°±æĺ¯ è¦ģ +å¹´ éĹ´ +åĩº çı¾ +å½± éŁ¿ +å¹ħ 度 +éĽ ģ +éģĵ åħ· +缮çļĦ åľ° +åIJİ èĢħ +ä¸Ĭ æ¼Ķ +äºĨ åĩł +æ®ĭçĸ¾ 人 +å¿Ļ ç¢Į +æĺ¯åIJ¦ æľī +并 对 +ä¼ļ 导èĩ´ +æ°´ åºĵ +ç»Ĩ èĩ´ +åIJİ æĤĶ +å¿ĥ æĢĿ +åģļ äºĭ +åİĤ æĪ¿ +çĿ ¿ +è¿IJèIJ¥ åķĨ +头 éĥ¨ +çļĦ è§Ĵèī² +æĺ¯ ä»ĸ +æĹ¢ æľī +å°ıæŶ åĢĻ +强 åĬ² +主 æĴŃ +åħ¨åĽ½ åIJĦåľ° +æį ı +æįŁ åĿı +åķĨ ä¼ļ +ä¿Ŀ ç½Ĺ +çľģ å¸Ĥ +éļ§ éģĵ +æľī ä¸įå°ij +è¦ģ åľ¨ +建设 é¡¹çĽ® +ç³ĸ å°¿ +ç³ĸå°¿ çĹħ +æĿ¡ä»¶ ä¸ĭ +ä¼ĺè´¨ çļĦ +é¦ĸ åıij +å½ĵæŶ çļĦ +丰 çĶ° +大 çĽĺ +缸 继 +å®ģ å¤ı +åħ¥ ä½ı +æĪij è¿ĺ +åħĭ æĸ¯ +å®ļ ä»· +å¹³æĸ¹ åħ¬éĩĮ +çļĦ çŁ¥è¯Ĩ +æĪij们 ä¼ļ +åħĥ å®Ŀ +ä½ĵ éĩį +è³ £ +对 æĪij们 +çŁ³ 家 +çŁ³å®¶ åºĦ +ç²¾ åįİ +å½¢ çĬ¶ +åıĹ åĪ°äºĨ +ä¿® 订 +ç¾İ åľĭ +é«ĺ æ¸ħ +çľ¼ éķľ +è§īå¾Ĺ èĩªå·± +带 ç»Ļ +åĶ® ä»· +éŨ 票 +åŃķ å¦ĩ +çĶµè§Ĩ åı° +åıij ä½ľ +çļĦ åij³éģĵ +éķ¿ è¿ľ +åħ¬åħ± æľįåĬ¡ +æŃ£å¸¸ çļĦ +æľī è¿ĩ +é£İ æĥħ +æ¯Ķ éĩį +åIJ » +管çIJĨ å·¥ä½ľ +综åIJĪ æĢ§ +å·² 被 +说 èµ· +æİĴ æ°´ +ä¸įæĸŃ åľ° +æĥħ æĢĢ +è¾ĵ éĢģ +è¿ĩ æķı +çļĦ åı¯èĥ½æĢ§ +æľį çĶ¨ +æľī 许å¤ļ +å§Ķ åī¯ä¹¦è®° +åĮĸå¦Ĩ åĵģ +æļĤ åģľ +æĬķèµĦ 人 +çıŃ çº§ +说 çĿĢ +åįĹ åĮĹ +åĪĨ è¡Į +çıł å®Ŀ +å¯ ¶ +å¢ŀ å¤ļ +被 åĬ¨ +çī¹æ®Ĭ çļĦ +éĹľ ä¿Ĥ +çļĦ èĦ¸ +æĥ Ł +ä¸į ä¸Ģå®ļ +ç¶ Ń +çģ« çĪĨ +秣 éĩij +çŀ § +éĩį 建 +è· ª +ä¸Ģ 種 +çļĦ åIJĪä½ľ +å®ī æħ° +ä»į æĺ¯ +ä¸ĵä¸ļ åĮĸ +è°ĥ 解 +ä¸į 妨 +éĢĻ æĺ¯ +å¿ħ éłĪ +ä¼Ĭ æľĹ +å¾Ĺ äºĨ +æľįåĬ¡ å¹³åı° +å§ ¬ +åħĪ éĶĭ +çİĭ åŃIJ +çļĦä¸Ģ åĪĩ +æĢ» çIJĨ +åĵ ¼ +çª ij +çļĦå¿ĥ æĥħ +çļĦ éĩį大 +çij Ł +ä¸Ģ ç¬ij +åıijå±ķ ä¸Ń +åģ¥åº· åıijå±ķ +åĵģçīĮ çļĦ +ç¦ ® +ä½Ļ 人 +ä»Ĭå¹´ 以æĿ¥ +æķ° çłģ +çŃ¾ è¯ģ +åİ» æī¾ +åŁºéĩij ä¼ļ +æĬ± æĢ¨ +æŃ£ å½ĵ +çıŃåŃIJ æĪIJåijĺ +ä¸į åIJĪæł¼ +åĪ¶ å®ļäºĨ +ç¼ĵ æħ¢ +åĪ¶ 约 +æłı 缮 +å¸Ĥåľº ç»ıæµİ +ç»ĦæĪIJ çļĦ +严 å³» +æĹ¥ 讯 +ä¸ĢçĤ¹ çĤ¹ +æĺ¯ æĢİä¹Ī +çļĦ çħ§çīĩ +éĺ» æŃ¢ +模 ç³Ĭ +ç¼ ¸ +éģķ åıį +æIJ¬ è¿ģ +éĩij éĴ± +å½ ¬ +ä¸į å®ī +æĪĺçķ¥ åIJĪä½ľ +å¡« åĨĻ +讲 究 +åħħåĪĨ åĪ©çĶ¨ +èĥ½ å¤ł +èij¡èIJĦ éħĴ +éĩĩçĶ¨ äºĨ +åľ¨ ä»Ĭå¹´ +ä¸Ńå°ı åѦ +åľ¨ æĦı +çļĦ åİĭåĬĽ +ä¸į 幸 +åĪ¶ èį¯ +åı¯ä»¥ 让 +被 è¯Ħ为 +ç»Ĩ èıĮ +æĪı åī§ +åįĬ 导 +åįĬ导 ä½ĵ +è§Ĩ è§Ĵ +åĸľ æŃ¡ +å¾ģ æĶ¶ +è°ĭ åĪĴ +æŀģ 大çļĦ +çĤ¹ èµŀ +è®°èĢħ ä»İ +两 åIJį +èĩª åĬ© +èµ· æŃ¥ +æĬ¤ 士 +å®Ŀ 马 +太 åŃIJ +å°ıå°ı çļĦ +温 æ³ī +åĩºç§Ł 车 +秣 æĪ¿ +两 家 +éľĩ æĴ¼ +ç§ī æī¿ +ä¸Ģ件 äºĭ +çĥĪ 士 +å®ĺ åħµ +转 身 +ä¹IJ åĽŃ +çĻĮ çĹĩ +模 èĮĥ +æĦ £ +è¿ĩåİ» çļĦ +代 ä»· +çļĦ æ¦Ĥ念 +åĩł çĻ¾ +è´µ éĺ³ +æĭħ 忧 +éĢĤ å®ľ +çݯå¢ĥ ä¿ĿæĬ¤ +çĥ « +ä½ł æĥ³ +æѤ åIJİ +ä½ł ä¹Ł +çį İ +éĻ¤ æѤ +éĻ¤æѤ ä¹ĭå¤ĸ +è°ĥ 度 +ç§ij 缮 +æīĢ说 çļĦ +åĬ ĩ +忽 è§Ĩ +ä¸ī 次 +ä¸Ģ æĹ¥ +åŀĤ 缴 +ç«ŀ æĬĢ +éĿ¢ åĮħ +大 æĪĺ +æIJº 带 +å¦Ĥæŀľ 没æľī +åħ» æĪIJ +åĩº è¡Ģ +çĪ±å¥½ èĢħ +æīĵ éĢļ +èµ· è¯ī +åijĪ çİ°åĩº +æŃĮ æīĭ +åľ¨ å¤ĸ +é¢Ĩ导 å¹²éĥ¨ +åĨ ¥ +èĪĨ 论 +æıIJ åıĸ +éĺ¿ å°Ķ +æľĽ çĿĢ +ä¸ī äºļ +è² ¡ +åĪ ·æĸ° +æĻļ æĬ¥ +è¿ĺæľī ä¸Ģ个 +åĨ° ç®± +ç½ij çĤ¹ +åĩº åħ· +强çĥĪ çļĦ +æĪij çĽ¸ä¿¡ +å¸ĮæľĽ èĥ½ +çīĻ 齿 +äºĭ å®ľ +ä¸ļåĨħ 人士 +代 æĽ¿ +åıĺ å½¢ +éĽ ² +è°ĥ æݧ +åĪĽæĸ° åĪĽä¸ļ +æĭĨ è¿ģ +æł¸ æŁ¥ +éĢ Ĺ +åħ¥ åѦ +æĦı åIJij +æı Ľ +ä¸ĭ 次 +ä¼ł è¾ĵ +ä»ĸ们 åľ¨ +èĢĮä¸Ķ è¿ĺ +æĹ¥ åľ¨ +æķĻ è®Ń +æ´» çĿĢ +çļĦ æľīæķĪ +å¤įå·¥ å¤į +å¤įå·¥å¤į 产 +æĺ¯ä¸Ģ 件 +çŃī çĿĢ +å¾ © +åĭĩ æķ¢ +éģŃ åıĹ +å¥Ķ é©° +讲 座 +说 å®Į +ç»Ļ åĩº +è° ¦ +è¯Ĭ çĸĹ +çĽ² 缮 +客 è¿IJ +å°± è¿ŀ +å¼Ģ åħĥ +å¼Ģåħĥ æ£ĭçīĮ +ä¸įæĸŃ æıIJåįĩ +çĶ¨æĪ· çļĦ +æĴ ķ +ä¾Ľ æ°´ +ç¶ĵ æ¿Ł +ä¸Ń åĮ»èį¯ +èģĶ æĥ³ +åħ¬äº¤ 车 +èĪª çıŃ +æĬĢ è¡ĵ +å¼ķèµ· çļĦ +å° ¹ +èµĦ æ·± +åĽ½èµĦ å§Ķ +èĺ Ń +é¼» åŃIJ +éĹ ½ +æİĴ éĺŁ +è§Ĥ åħī +éģĹ åĿĢ +举 京 +é¥Ń åºĹ +ä¸įæĸŃ çļĦ +å°±æĺ¯ ä¸Ģ个 +éķ¿ ä¹ħ +çļĦ è§ĤçĤ¹ +å¨ ¶ +æĪij çİ°åľ¨ +çķ ° +å¾Ĺ åĩº +å¿ħ å®ļ +ä¸į åıĹ +åıª éľĢè¦ģ +åĽ° æī° +ç§ijåѦ æĬĢæľ¯ +çīĽ èĤī +è¾ĥ é«ĺçļĦ +è·ij æŃ¥ +æ² ¾ +èı© èIJ¨ +æľĢ å¾Į +ä¿Ŀ å¯Ĩ +æ²» å®ī +éĤ ± +常 è¯Ĩ +èĦ¸ èī² +åĮĹ å¤§ +æ±ĩ èģļ +æijĨ èĦ± +é¾Ļ头 ä¼ģä¸ļ +女 åıĭ +çŃī å·¥ä½ľ +ä¸Ń ç¾İ +èģĮ åľº +èĦij è¢ĭ +åĨĻ çļĦ +饲 æĸĻ +åĬ³ åĬ¨åĬĽ +å± ¯ +æĮģ èĤ¡ +åĽ¾ åĥı +è¿ĩåİ» äºĨ +è² ¨ +è¾ ² +éĹ® æĪij +è·Ł ä½ł +çĶŁ æŃ» +审 ç¾İ +é¢Ĺ ç²Ĵ +ä¸Ń æĸ¹ +åĬł çĥŃ +æĹħè¡Į 社 +çĻ¼ çĶŁ +ä¸į åłª +åĤ · +æ¥ ł +åĬŀ æ¡Ī +æŁ Ħ +æĹ¢ æĺ¯ +å¤Ħ åĪĨ +羣å®ŀ çļĦ +æĬ¥ 纸 +å¸Ī çĪ¶ +å®īå¾½ çľģ +åī¯ 主å¸Ń +ä¹ĭ éģĵ +导 å¼¹ +åŃ¦æł¡ çļĦ +åŁİå¸Ĥ çļĦ +è°Ī åĪ° +æ¢ Ĺ +å¹³ éĿ¢ +说 ä»Ģä¹Ī +é¢ij çİĩ +éķ¿ ä¸īè§Ĵ +çļĦ åĪ©çĽĬ +é» ¨ +è±Ĩ èħIJ +å®ŀéĻħ æĥħåĨµ +æŀĹ ä¸ļ +纪æ£Ģ çĽijå¯Ł +ä½ı éĻ¢ +çļĦ æķ´ä½ĵ +åīį è¡Į +æĮ ¨ +çħ¤ çŁ¿ +åī¯æĢ» è£ģ +å°ı åIJĥ +æŀģ 端 +å©Ĩ å©Ĩ +çİ° è´§ +è¯Ĺ æŃĮ +éĴ¥ åĮĻ +缩 çŁŃ +ä½Ĩ è¿Ļ +æĸ° åĵģ +è¿Ļ 对 +çŁ¥åIJį 度 +å¿ĹæĦ¿ æľįåĬ¡ +大 å±Ģ +è¡¡ éĩı +ä½ĵçİ° äºĨ +æ¡ĥ èĬ± +åIJ¸å¼ķ åĬĽ +åł ¤ +æĵħ éķ¿ +åĴ Ĵ +缸 æľº +ä¸Ģ ç«Ļ +ä¸Ģç«Ļ å¼ı +æľĢ ç¾İ +æ°¸ ä¹ħ +çļĦ éĥ¨åĪĨ +åĪĨ å·¥ +å·¥ç¨ĭ 建设 +æIJŃ è½½ +æ°´ ä¸Ń +èĮ ¨ +çļĦ æĵįä½ľ +绣 æ²» +çķħ éĢļ +åħļçļĦ åįģ +è¼ ¸ +æ¸ ¬ +ç¾İ è§Ĥ +ä¸į åĪ© +åıį æĢĿ +éªĦ åĤ² +æłĩ çļĦ +æĿĢ 人 +éĺ¿ å§¨ +é£Ł æĿIJ +åIJĥ çļĦ +åIJİ åĨį +çŁ £ +两 侧 +æ¸ħ æ°´ +è¿Ľ çIJĥ +å¼Ģå§ĭ äºĨ +åIJ¬ äºĨ +çĦĬ æİ¥ +çŁ ® +å¨ Ł +为 人 +éĢģ ç»Ļ +åĨĴ éĻ© +æķ · +ç»Ī æŃ¢ +æīį çŁ¥éģĵ +è¿IJ æ°Ķ +éĢļ é£İ +æĥĬ 讶 +ç§ijåѦ éĻ¢ +æıIJ éĹ® +太 åİŁ +缸åIJĮ çļĦ +ä» ķ +èģ ĸ +æĥħ æ³ģ +é¢Ĩ导 人 +åĩºæĿ¥ äºĨ +沿 线 +éĻ ½ +æĦŁ 覺 +ä»į åľ¨ +æ© Ļ +约 为 +åĸĿ éħĴ +çĶ¨ èį¯ +ä¸ĭ ä¸Ģ +æ³ķ å®ĺ +顺 åºı +åģļ ä¸Ģ个 +åĭ ¢ +æŃ ª +çĶµ ç«ŀ +ä¼´ éļıçĿĢ +ä¹ĭ åĬĽ +ä¹ĭ 人 +äºij 计ç®Ĺ +åĪ«äºº çļĦ +ç§ijåѦ åıijå±ķ +第 åħ« +å¹² æī° +女 ç¥ŀ +è¿Ļæł· åģļ +å¤Ħ åľ¨ +æ°´ è´¨ +éķ¿ æĺ¥ +å¸Ĥåľº éľĢæ±Ĥ +ç»´ æĿĥ +èĢ³ æľµ +æĸĩåĮĸ çļĦ +奶 ç²ī +ä¼ł è¾¾ +æīĭæľº çīĪ +æĽ¾ åľ¨ +äºĮ æľŁ +åİŁåĽł æĺ¯ +æºIJ 头 +åıĪ èĥ½ +è£ ¸ +æĬĢæľ¯ åĪĽæĸ° +æĸĩåĮĸ æĹħ游 +åıij 票 +å¹´ 级 +ä½ł ä¸į +ä¹ĭ å¿ĥ +æķ° çĻ¾ +åIJij å¾Ģ +èĢģ 家 +åľĭ éļĽ +çļĦ é«ĺ度 +æľĿ éĺ³ +æ¸ħ éĻ¤ +èĩª æľī +书 ä¸Ń +游æĪı è£ħå¤ĩ +ä¸ĩ å¤ļ +驾驶 åijĺ +ä½ł çŁ¥éģĵ +åĽ½ åºĨ +é£Ł åłĤ +æİ¥ åı£ +æĢ» æķ° +åħ¶ä»ĸ çļĦ +çĶŁåij½ çļĦ +ä½ł åľ¨ +çļĦ 缮åħī +è¿Ļ æĸ¹éĿ¢ +éĥ½ 说 +çĸĹ æ³ķ +åĭĩ 士 +åľ¨ åħ¨çIJĥ +ä¿ĿéĻ© åħ¬åı¸ +çĿ£ æŁ¥ +åĸĦ èī¯ +表 å½° +è¹ ² +è·¯ 段 +æľĥåĵ¡ è¦ı +æľĥåĵ¡è¦ı ç¯Ħ +æĪ· åŀĭ +ä¿ĥ 使 +ä¿® 建 +é«ĺ æ°´å¹³ +åģļ åĩºäºĨ +主 åľº +è¡Į èµ° +空 çĻ½ +æľī人 说 +è¿Ļ个 ä¸ĸçķĮ +åIJį ä¹ī +å®Į ç¾İçļĦ +羡 æħķ +åıĬ åħ¶ä»ĸ +åı¯ çĶ¨ +æĭ IJ +è¾ĥ 大çļĦ +æĬĢæľ¯ åĴĮ +å°¼ äºļ +çĻ¾ è´§ +æı ī +éĢī è´Ń +éĺŁ åıĭ +ä¼ł æĦŁ +ä¼łæĦŁ åĻ¨ +åıªè¦ģ ä½ł +为ä»Ģä¹Ī è¦ģ +ä¸ĵ注 äºİ +ä½Ļ é¢Ŀ +åħ¸åŀĭ çļĦ +缮åīį å·² +欲 æľĽ +èģĶ 绾 +æµģ ä¼ł +çļĦ 家åºŃ +åı· åı¬ +çıį è´µ +ä¼Ł 大çļĦ +éī´ äºİ +è·Ł ä»ĸ +产 çī© +ä¸į å·² +è¿Ŀæ³ķ è¡Į为 +头 ä¸Ĭ +åĪĨ 解 +åı¯ä»¥ çľĭåĩº +æł¡ åĮº +åŃĹ ä½ĵ +ä¿® çĤ¼ +çĶļèĩ³ æĺ¯ +微信 åħ¬ä¼Ĺ +åıĸ 代 +èIJ¥ä¸ļ æĶ¶åħ¥ +æ½į åĿĬ +ä½ł èĥ½ +社ä¼ļ ä¿Ŀéļľ +æ¯ĶèµĽ ä¸Ń +污水 å¤ĦçIJĨ +夫 å¦ĩ +ä¸Ģ å¹ħ +沿 æµ· +åı£ æĦŁ +ä½Ĩ åį´ +å½ĵ æĹ¥ +çļĦ æľĢ大 +æ¯ı ä¸Ģä½į +没 äºĭ +çī¹ åĪ¥ +å¼Ģ åѦ +è·¯ éĿ¢ +å¿ĥçIJĨ åѦ +æĶ¾ ç½® +éĩįåºĨ å¸Ĥ +ä½ł èĩªå·± +æ¶Īè´¹èĢħ çļĦ +ä¸Ģ æ³¢ +èѦ æĥķ +åį§ 室 +注 å°Ħ +é£İ 鼨 +沿 çĿĢ +åijĬ 訴 +表 çİ°åĩº +åĽĽ æĺ¯ +åı¤ åħ¸ +æĽ´ éĩįè¦ģçļĦ +好 äºĭ +çľ¼ 泪 +æ¨ ĵ +审 åĪ¤ +碰 æĴŀ +车 ç«Ļ +è¿Ľåħ¥ äºĨ +éĽĨ åIJĪ +æł¼ å¤ĸ +宾 é¦Ĩ +æĶ¯ä»ĺ å®Ŀ +她 æĺ¯ +æĺ¯ å¦Ĥä½ķ +人 次 +çļĦ æĪIJåĬŁ +æĹł åĬĽ +æµ· æĭĶ +æĺ¥ åŃ£ +éĥ½ ä¸įä¼ļ +çŃī å¤ļç§į +ä¸Ģ个 å°ı +åģľè½¦ åľº +让 æĽ´å¤ļ +è¿Ļ çĤ¹ +æĪIJ åĵģ +éĴ ī +éģĩ è§ģ +çıŃ ä¸»ä»» +æĦı æĦ¿ +çļĦ åIJĮåѦ +游 è§Ī +åİĭ 缩 +åľ¨ ä¼łå¥ĩ +å¼¹ æĢ§ +æĹ¥ åĨħ +ç¦ı建 çľģ +è§Ĵ èIJ½ +åĪĨ å¼Ģ +ä¼ļ 让 +å¤ĸ åĽ´ +çĨŁæĤī çļĦ +çĨ Ķ +ä¸ĩ è¾Ĩ +å¤ľ éĹ´ +车 身 +ä¸Ń æľŁ +å®ĮåĸĦ çļĦ +åĵģ ç±» +åıĭ è°Ĭ +éĢīæĭ Ķ +éªij 士 +å½ ¦ +çļĦ çľĭæ³ķ +åĽ½ çİĭ +è¾£ æ¤Ĵ +åıijå¸ĥ æŶéĹ´ +åı¤ åŁİ +éļı æľº +ç« ĸ +å¼Ģ è¾Ł +ä¼Ĺ çĶŁ +没 åĬŀæ³ķ +åįĥ éĩĮ +æĿ¥æºIJ äºİ +çļĦ æĿĥåĪ© +æ¯Ķ åĪĨ +满æĦı çļĦ +ä¿® è¡Į +åĿ ł +大 æµ· +èİ ¹ +åĩº 身 +è« ĩ +åħ³ èĬĤ +åIJį 人 +éľĢè¦ģ 注æĦı +æĹ© æĻ¨ +å¤ĸ åįĸ +åıĪ è¦ģ +æ¶ī æ¡Ī +çĶ³è¯· 人 +éĻĦè¿ij çļĦ +åĬłå¿« æİ¨è¿Ľ +æĸ° å¹´ +大 è¡Ĺ +ä¸Ģ é»ŀ +èĭı å®ģ +æĤĦ æĤĦ +èĦ¾ æ°Ķ +å¸Į èħĬ +éļı åį³ +æķ¢ äºİ +å®ŀè·µ ä¸Ń +æĺ¯ æ²¡æľī +æľī趣 çļĦ +æĿ¥èĩª äºİ +è£ģ åĪ¤ +女 åŃ©åŃIJ +èĩ³ åħ³ +èĩ³åħ³ éĩįè¦ģ +æĻº åĬĽ +èµ° åĩºåİ» +çŁŃ æĿ¿ +大 åĽ½ +çļĦ 认è¯Ĩ +å¹´ å¤ľ +åĨį åĪ° +åIJĮ æł·çļĦ +å¯Ĩ å°ģ +å¤ĸ交 éĥ¨ +çĶŁ æķĪ +æĤ¨ åı¯ä»¥ +ä½ł åĢij +è¿ĩ å¹´ +å¼ ĵ +è¡Į æĿİ +æ¯Ķ èµ· +身 é«ĺ +è¿Ļ个 人 +ä¸Ń å¤ĸ +éģĵ æŃī +çĽ¯ çĿĢ +亲 åŃIJ +éĹ ¸ +çĻ½ äºij +èĦĸ åŃIJ +ä¸ĢåĪĩ éĥ½ +æ· ij +è° ľ +åģ¶ çĦ¶ +éĿł è°± +é«ĺ 管 +ä¸ĭ åıij +æĶ¾ åĪ° +ç±» åĪ« +ä¸ĭ åĪĹ +æ·· ä¹± +åIJĪæ³ķ æĿĥçĽĬ +çݯ çIJĥ +æľīæķĪ åľ° +åķĨ æĪ· +æ¹ĸ 人 +æµ· 岸 +æĬķ 产 +两 个æľĪ +éĥ½ éĿŀ常 +å¢ŀ强 äºĨ +æĿ¥ åĪ°äºĨ +åī© ä½Ļ +æĤ¨çļĦ åŃ©åŃIJ +æµģ æ°´ +æŃ£ ä¹ī +天 çĮ« +åģļ è¿ĩ +ä½ķ æŶ +æĪij åİ» +çľģ 份 +å¥ĸ éĩij +该 å¦Ĥä½ķ +ä¸ĭ çıŃ +åģ¶ åĥı +æijĨ æĶ¾ +æĸ° 模å¼ı +æĬķ è³ĩ +è·¯ åı£ +åĨľæ°ij å·¥ +大 åѸ +件 äºĭ +æł¹æľ¬ ä¸į +æµĵ 度 +æµĵ åİļ +è½® èĥİ +æĪ¿ ä¼ģ +éĿŀ常 好 +ä»İ ä¸Ń +人 æł¼ +ç¿ ģ +æŶéĹ´ åĴĮ +è¿Ļ ä¸įæĺ¯ +åĪ¸ åķĨ +æĥĬ 人 +åĻ¨ å®ĺ +åĩĨ åĪĻ +æĥħ æĻ¯ +æĽ´ é«ĺçļĦ +åѦ 家 +泡 沫 +åľ°æĸ¹ æĶ¿åºľ +å°± çŁ¥éģĵ +åij¼ åIJģ +ç»ı è´¸ +èĬ± éĴ± +æľī ä¸Ģ次 +æĦŁ æħ¨ +ä¸Ģ åįĥ +å¤ľ æĻļ +詹 å§Ĩ +詹å§Ĩ æĸ¯ +è¦ģ éĹ» +ç» Ĵ +æºIJ äºİ +çļĦ è´¨éĩı +注æĦı äºĭ项 +æħ¢ æĢ§ +稳å®ļ çļĦ +建设 åĴĮ +æĻ¯ 象 +éĩı åĮĸ +çļĦ 話 +è¯Ħ 级 +æº ľ +红 åĮħ +éĢļ éģİ +社ä¼ļ 责任 +æĸ° 产åĵģ +åĨ· éĿĻ +çľĭ ä¸įåĪ° +èģĶ éĤ¦ +éŃ Ħ +çļĦ åīįæıIJ +çļĦåīįæıIJ ä¸ĭ +è¾ĥ 好 +çļĦ æĦŁæĥħ +客æĪ· æıIJä¾Ľ +çĭ¬ èĩª +å¢ŀ æĶ¶ +æĸĩ çĮ® +æĭ¼ åij½ +管çIJĨ åĴĮ +æµģåĬ¨ æĢ§ +åħ¨ 家 +ä¸Ĭ æĸ¹ +æݨåĩº çļĦ +ä¸ī åĽ½ +ä¸Ģ个 æĺ¯ +æĸ° ä¸Ģè½® +æĸĩåĮĸ éģĹ产 +æ® º +大 æ¹¾åĮº +éĥ½ éľĢè¦ģ +çļĦ å®ŀéĻħ +ç· Ĭ +大 å¥ĸ +åħī èĬĴ +便 äºİ +çļĦ 表æĥħ +æ¼Ķ ç»İ +红 åĨĽ +å½ĵ æĪij +æ²» æĦĪ +é¢Ŀ 度 +éĿ ľ +ä»»ä½ķ 人 +è¡Ĺ 头 +çī¹ æĸ¯ +çī¹æĸ¯ æĭī +åĮ»çĸĹ æľºæŀĦ +ç»Ļ åŃ©åŃIJ +è§Ħ 磩 +è£ ľ +çļĦ 身影 +ä¸ĵ æłı +æĿ¥ 临 +ç«¥ å¹´ +å¤į èĭı +è¨ Ĥ +åŀĭ åı· +åĽ¾ æ¡Ī +ç®Ģ åİĨ +æĭ ± +èį· åħ° +ä»» æĦı +æī¿ æİ¥ +è¿Ļ æīį +客 车 +æľĿ çĿĢ +éłħ 缮 +åı° é£İ +çļĦ æĪ¿åŃIJ +éª ı +æĿ± 西 +éģĹ ä¼ł +è¶Ĭ å¤ļ +äºĨ ä»ĸçļĦ +ä¸Ĭ åij¨ +管çIJĨ åĪ¶åº¦ +失 ä¸ļ +çĶ· åıĭ +æİ¥ ç§į +å¨ģ åIJį +çĴ° å¢ĥ +åıijçĶŁ åľ¨ +个 åĽ½å®¶ +åĪĽæĸ° åıijå±ķ +æĶ¹åıĺ äºĨ +åģ¥åº· çļĦ +åĢ¼å¾Ĺ ä¸Ģ +åĢ¼å¾Ĺä¸Ģ æıIJ +åĽ¢ ä¼Ļ +åģĩ 设 +åı° ä¸Ĭ +è§ĦèĮĥ åĮĸ +éĻª åIJĮ +座 æ¤ħ +åı¯ æĢľ +åħĭæĢĿ 主ä¹ī +æ³ķå¾ĭ 责任 +ä¸Ģ é¡¿ +æĬ¬ 头 +为 éĩįçĤ¹ +è¿ľ æ´ĭ +éĢı è¿ĩ +åħ¨çIJĥ åĮĸ +趣 åij³ +票 æĪ¿ +æ¯ı 人 +åIJĦç§į åIJĦæł· +äºĨ åĩºæĿ¥ +ç»Ŀ对 æĺ¯ +ä¸ĭ å±ŀ +ä¸Ģ åıĮ +è¿Ļ åĿĹ +æĬĹ çĸ« +è¦ģ çĤ¹ +å½¢æĪIJ çļĦ +æĪij çľĭ +ä¸ĩ éĩĮ +èĢĥ çłĶ +为 åħ¶ +æ°ij 宿 +å¤ļ ä½į +大 èĩ´ +ä»ĺ è´¹ +åħ¥ æīĭ +å±ħ 家 +æīĢåľ¨ åľ° +人 身 +è¿ĩ å¾Ĺ +è¯ķ è¯ķ +访 è°Ī +åĬł éĩį +å°± ä¸įä¼ļ +çĶŁäº§ ä¼ģä¸ļ +åĽŀ åĽ½ +åºķ 线 +赶 åĪ° +æĶ¯ éĺŁ +æĪij们 éĥ½ +éĤ® æĶ¿ +缴 èĩ³ +éĴ¢ çIJ´ +åħ ľ +çłĶ讨 ä¼ļ +æľĪ 亮 +åĿļæĮģ 以 +åħ¬å®ī éĥ¨ +éĴ¢ 管 +å°ı çĻ½ +ç½® ä¸ļ +èģ ĭ +书 åĨĻ +æĿ ı +éħį æĸ¹ +èĢĮ åıĪ +çijŀ 士 +çķĮ çļĦ +èĢģ 大 +æĪIJçĨŁ çļĦ +å¹² ä»Ģä¹Ī +ä¸ĵ项 æĸĹäºī +çŃī å¤ļ个 +èĦ± 离 +ä¸ī 个æľĪ +çłĶ究 åijĺ +æĹĭ 转 +æŀģ èĩ´ +åħį è´£ +åħįè´£ 声æĺİ +å¾Īå¤ļ çݩ家 +车 ä¸Ĭ +交 äºĴ +å·² æĺ¯ +ä¸Ģ å°ı +çļĦ éĩįçĤ¹ +èĬ± äºĨ +ä¸į æĺİ +æľīåħ³ è§Ħå®ļ +çĬ¹ å¦Ĥ +çľ ¸ +å¯ ¡ +çļĦ è¡£æľį +åĮħ 裹 +身 åŃIJ +å¸ĪèĮĥ 大åѦ +äºĭ åħĪ +线 æĿ¡ +æ³ķ åĪ¶ +åħ» æĬ¤ +稳å®ļ æĢ§ +éĤ µ +åŀĦ æĸŃ +é¡ į +èĢĥ åı¤ +æĿł æĿĨ +èĭı èģĶ +æ°´ çĶµ +åħ·ä½ĵ çļĦ +æ¿Ģ æ´» +æĪij æł¡ +åĪļ å¼Ģå§ĭ +åĩ¸ æĺ¾ +ç¦ ¾ +åħ¼ èģĮ +éĢı éģİ +åľ¨ 游æĪıä¸Ń +社ä¼ļ åıijå±ķ +好 çİ© +å¹» æĥ³ +ä¸į 代表 +注æĦı åĬĽ +æ£ į +çĶ¨ æīĭ +ç¾İ 人 +许å¤ļ 人 +å¾Ī æĺ¯ +çļĦ çłĶåıij +æīĵ åĩº +åIJĪä¼Ļ 人 +ä¸Ģ å¤ľ +ç¼ĵ ç¼ĵ +ä¿® æŃ£ +æĦŁ çŁ¥ +ç»Ī 身 +æ¿Ģ ç´ł +çݯå¢ĥ ä¸ĭ +次 ä¼ļè®® +ç»ıæµİ å¢ŀéķ¿ +æī Ľ +åıij éħµ +åĪĨæŀIJ å¸Ī +åľ¨ æľªæĿ¥ +主è¦ģ æľī +ä¸Ģ åŃ£åº¦ +çļĦ 说æ³ķ +ä»İæĿ¥ 没æľī +è´§ 车 +缩 å°ı +太 è¿ĩ +æķĪ åĬĽ +ä¸į ä¸ĭ +æĬķ 稿 +èį¯ ä¸ļ +ç»Ħ éķ¿ +ç«Ļ çĤ¹ +å¾Ī åĸľæ¬¢ +éIJ µ +åĬ¿ 头 +æ¼ı æ´ŀ +æĦ¤ æĢĴ +åħħ å®ŀ +åĪĽä¸ļ æĿ¿ +çĪ ª +æľª å¿ħ +åºķ éĥ¨ +å¾Ĺ åĪĨ +人æ°ij åĮ»éĻ¢ +äºĮæīĭ æĪ¿ +å·²ç»ı 被 +大 楼 +æĸ° æĪ¿ +辦 æ³ķ +çĶ¨ åĬĽ +æĭĵ 宽 +åĨħ åľ¨ +æĴŃ åĩº +饰 æ¼Ķ +ä¹Ł 让 +ä½ľ çĤº +çī©ä¸ļ 管çIJĨ +åį´ ä¸į +为 ä¸ŃåĽ½ +å±Ģ åĬ¿ +ä¸į èĤ¯ +æľĢ æĸ°çļĦ +åı¯ä»¥ éĢīæĭ© +æĺ¾ çİ° +å°± ç®Ĺæĺ¯ +åľ¨ æł¡ +é¾ Ł +两 æĿ¡ +çļĦ å®ŀåĬĽ +è¶Ĭ 好 +她 åľ¨ +å¿ł è¯ļ +ä¹Ł éľĢè¦ģ +游æĪı æĵįä½ľ +è¶ħ åĩº +å¦Ĥæŀľ ä¸į +æīĢåľ¨ çļĦ +ä½ł è¿ĺ +以 åĨħ +æľī ä¸Ģå®ļ +åı¯ è¾¾ +è·ij åĪ° +åī Ľ +建ç«ĭ åģ¥åħ¨ +æķ´ 车 +åīį æĸ¹ +éĹ´ æİ¥ +çŃ¹ å¤ĩ +çĸ² åĬ³ +离 å¼ĢäºĨ +æ± Ŀ +éĿ¢ éĥ¨ +ä¹ĭåīį çļĦ +åıĺ 为 +å¦Ĥæŀľ 说 +对 ä»ĺ +åĿĩ åı¯ +被åijĬ 人 +ç²¾ ç¾İ +èģļ ä¼ļ +çĿĢ æĢ¥ +è°· æŃĮ +ä¸Ģ åı· +红 åĪ© +ä¼łå¥ĩ 游æĪı +å» ĸ +è´ ŀ +ä¹° åĪ° +éŃ ļ +ä½ĵ è´¨ +å°ij äºĨ +æ³ī å·ŀ +åIJ Ł +ç»Ŀ ä¸į +é»ij æģ¶ +é»ijæģ¶ åĬ¿åĬĽ +ä¸Ĭ æĺł +çļĦè¯Ŀ é¢ĺ +ä¸ĩ人 次 +ä¸ĸ éĹ´ +çĶ¨ å·¥ +è´¯ ç©¿ +å®Ŀ çŁ³ +ä½ł 好 +åĪĩ åī² +强 åĽ½ +åĽŀ èIJ½ +æ°´ æĻ¶ +模 仿 +æ´ª æ°´ +éĢĻ 麼 +åįģä¸ī äºĶ +ä½ ij +éĻ Ħ件 +çļĦ å¢ŀéķ¿ +éĻĦ å±ŀ +çİ° å·² +帮 ä½ł +éĩij çīĮ +é«ĺ åİŁ +åľ¨ 家éĩĮ +éĺ² èħIJ +ç¡®å®ŀ æĺ¯ +宣 讲 +天 æīį +ç»ıèIJ¥ 管çIJĨ +éĶħ çĤī +åIJĪ ä¸Ģ +è§Ĥ èµı +éķ¿ è¾¾ +主ä¹ī æĢĿæĥ³ +éĤ£ 麼 +é£İ äºij +为主 çļĦ +æļij åģĩ +æĮģ ä¹ħ +å¼Ĥ åľ° +å¼Ģ éŨ +模 æĿ¿ +æī¹ 次 +ä¸į 便 +天 çĶŁ +åĩł 个æľĪ +ä¸ĵ ç§ij +åı¦ æľī +åħ¬å¸ĥ çļĦ +æĩ · +åľº åIJĪ +çļĦå¿ĥ æĢģ +è¿ĺ 好 +å®ŀ æĪĺ +èĢģå¸Ī çļĦ +åħ© åĢĭ +åı¯ åľ¨ +éĤ£ ä½į +å¥ł å®ļäºĨ +ä¿ĥ éĶĢ +æı´ åĬ© +ä¸ĩ çī© +æĥħ æĬ¥ +é¦ĸåħĪ è¦ģ +æĸĩåĮĸ åĴĮ +éĥ½ å·²ç»ı +ä¸Ĭ ä¸ĸ纪 +åĨľ åľº +大 æī¹ +æĺİçĻ½ äºĨ +çļĦ æĪIJéķ¿ +çļĦ æ¯ĶèµĽ +失 误 +åģļ æĪIJ +ä»Ĭ天 å°ıç¼ĸ +é¢Ĩ è¢ĸ +æıIJåįĩ äºĨ +å¾IJ å·ŀ +ä»į æľī +è¿ĩ 滤 +å¹½ é»ĺ +çĥŃ éĩı +ä¸Ģ é¦ĸ +æ¼Ĥ亮 çļĦ +åĩł ç§į +åĢ¡ è®® +å°±åı¯ä»¥ äºĨ +æİĴ åĪĹ +éĩį éĩį +ä¼ģä¸ļ åĴĮ +ä¸ĵ å±ŀ +çħ İ +亲 æĪļ +çĻ¾åĪĨ ä¹ĭ +稿 件 +è¿ĺ å¾Ĺ +人 åĵ¡ +äºī 夺 +æĽ´ 容æĺĵ +大 èĩªçĦ¶ +鼻 èħ¦ +太 空 +åľ° å¤Ħ +å¤ ¢ +ä»ĸ 对 +å¿ħ å°Ĩ +ä¸į å½ĵ +严 è°¨ +åĩº åľº +å·²ç»ı æľī +é¢Ĩ åĨĽ +é«ĺ æ¡£ +ä¸Ģ æīĢ +æł Ĺ +让 åѦçĶŁ +æĽ¹ æĵį +æŁIJ ä¸Ģ +伸 åĩº +èĬ± åįī +æ¸ħ éĨĴ +èģĶç³» æĸ¹å¼ı +åĪĨ å±Ģ +èħ ³ +æ©¡ èĥ¶ +éķ¿ å¾Ĺ +绿 åľ° +è¢ į +çļĦ èīºæľ¯ +女 æľĭåıĭ +ä¸Ń è¶ħ +离 åŃIJ +å¤ļæł· åĮĸ +éĺ³ åı° +ä½İ 碳 +ä¸Ģ ç±» +çŃīæĸ¹éĿ¢ çļĦ +å¾Ĺ 好 +模 åħ· +ä¸ĩ 亿 +çķĻ æĦı +临 æ²Ĥ +å°ij éĩı +çľĭ åIJij +ç»ıèIJ¥ èĢħ +çķĻä¸ĭ äºĨ +åĿı äºĨ +åijĬ åĪ« +羣 çIJĨ +ç¼´ è´¹ +æĬĬ ä½ł +çļĦ ä»»åĬ¡ +æĪij 对 +ä¹° åħ¥ +çĻ» ä¸Ĭ +æľī 两个 +ä¸Ģ 头 +æĵį æݧ +åħ¨ è¦ĨçĽĸ +çĿĢ æīĭ +å¢Ļ éĿ¢ +å¤ļ æĸ¹ +åı¯çĪ± çļĦ +ä¹Ł åı¯èĥ½ +æľĢ æľī +è¿ĻäºĽ éĥ½æĺ¯ +æĥ ¡ +å® ® +å¾Ī å°ı +éĹ®é¢ĺ æĺ¯ +åĿĩ æľī +å¾ģ éĽĨ +说 åĩº +æľī æĦı +é¢ Ĥ +æī¬ å·ŀ +åķĨä¸ļ 模å¼ı +çĶŁ èĤĸ +æįIJ 款 +å² Ĥ +ç¾İ æĻ¯ +è¿ĺ 羣 +æĭ¥ æĬ± +身ä½ĵ åģ¥åº· +æ·± å¤Ħ +çľ¼ ç¥ŀ +çļĦ 形象 +ä¼ĺ è¶Ĭ +å½ĵ æĪIJ +åĮº åĪĨ +åİ» éĻ¤ +注 å®ļ +å§IJ 妹 +åĮº åĨħ +é© ļ +æļĹ ç¤º +æĺİ äº® +æħ° éĹ® +å¸Ĥåľº 份é¢Ŀ +çĮª èĤī +çļĦ èµĦéĩij +åİĨ ç»ı +å§ĭç»Ī åĿļæĮģ +çĶŁ æľº +ä¸į 顾 +éĩij åĪļ +大 声 +éĻķ 西çľģ +é² į +åĨľä¸ļ åĨľæĿij +æľī 害 +éŨ è¯Ĭ +æ¯ı ä¸Ģ次 +çļĦ åĽłç´ł +é¢Ŀ å¤ĸ +åİ¿ 级 +çļĩ åIJİ +åĽ½ ä¼ģ +é¦ĸ éĢī +ç¼ĸ åĨĻ +æĭ¿ èµ· +åģ· åģ· +ä¸İ ä¸ŃåĽ½ +åįĸ 家 +ç»Ļ ä»ĸ们 +ç¥ŀ è¯Ŀ +åѸ æł¡ +æĪij ä¸Ģ缴 +çŁ¥éģĵ äºĨ +åį Ĵ +åĴĮ åľ°åĮº +ä»Ģä¹Ī éĥ½ +çĶ» 家 +æľ¬ çĿĢ +ä½Ļ åIJį +审 çIJĨ +ä¸Ģ åIJij +åıijå±ķ è¶ĭåĬ¿ +åĮº éĹ´ +注åĨĮ èµĦæľ¬ +çIJ ¦ +ä¸į åı¯ä»¥ +çļĦ åĦ¿åŃIJ +åĢ¼ çıŃ +ä¸¥æł¼ çļĦ +å®ŀä½ĵ ç»ıæµİ +æľī æĿĥ +æĪij åıĪ +éĵ¶ æ²³ +ç«ĭ 马 +æĿĢ äºĨ +åĮħ 容 +管 家 +身 é«Ķ +éĵ ħ +å°ı åŃIJ +管çIJĨ ç³»ç»Ł +æľīçļĦ 人 +é£İ çĶµ +æĻºèĥ½ åĪ¶éĢł +ç²¾ ç¡® +æĭĽåķĨ å¼ķ +æĭĽåķĨå¼ķ èµĦ +äºĮæīĭ 车 +åİ¿ å§Ķ +èīº 人 +å¥ ķ +è¿İ æĿ¥äºĨ +ç»ĵæĿŁ äºĨ +çļĦ ä¼łç»Ł +æĭ¼ æIJı +奥 迪 +çĸij æĥij +ä¹ĭ æĹ¥èµ· +æłĩå¿Ĺ çĿĢ +åľ° åįĢ +è¯ł éĩĬ +åĪ° æľŁ +åħ¨ éĥ½ +çŁŃ æļĤ +æĺ¯ æĪijåĽ½ +æĪij å·²ç»ı +æ»´ æ»´ +天 èµĭ +对 她 +åį«çĶŁ éĹ´ +çĶŁäº§ åŁºåľ° +æĹ¥ è®° +çļĦ æķĻåѦ +åĵ ĩ +æ°ij äºĭ +è¿ĺ åİŁ +æīĭ ä¸ŃçļĦ +çļĦ èī¯å¥½ +æ· « +ä¸Ńåħ± ä¸Ń央 +åĪ ĥ +åĵ Ħ +åľ¨ ä»ĸçļĦ +å°Ī æ¥Ń +åľº éĿ¢ +éĤ» å±ħ +çĹ Ĵ +å¦ Ħ +å¤ĸ ç§ij +ä¸į éĢĤ +举åĬŀ çļĦ +é Ĥ¹ +åħļçļĦ 建设 +çĻ¼ 表 +è·¨ çķĮ +æ²ī æ·Ģ +大 çīĩ +è¶Ĭ é«ĺ +å°Ĩ æĺ¯ +è§ī éĨĴ +åĤ¨ åŃĺ +å¢ŀ 大 +ä¸į 让 +æķ´ å½¢ +å¹³åı° ä¸Ĭ +åĩł ä½į +è¯ī æ±Ĥ +好 ä¸į好 +åľ į +æĸĩ æľ¬ +éĢ² åħ¥ +ç´ į +æł¹ æĵļ +èįī æ¡Ī +åħŃ ä¸ª +åĭ ¿ +åĪ¶ æĪIJ +饮 æ°´ +æ°¸ æģĴ +èĩª æĿĢ +åı¸ 马 +éļ¾ çĤ¹ +为 æĪij们 +å¼ § +åī© ä¸ĭçļĦ +åĩĨå¤ĩ 好 +çļĦ æľĢä½³ +èģĶåIJĪ ä¼ļ +æĤ£èĢħ çļĦ +æĪijä¸į çŁ¥éģĵ +ä¸ĭ ä¸Ģ个 +åıijå±ķ æĸ¹åIJij +ç¬ ¨ +æīĢ以 æĪij们 +åĨĻ äºĨ +éĢł æĪIJäºĨ +æ²Ļ æ¼ł +çŃĽ éĢī +çģ¾ åĮº +ä¸Ĭ çľĭ +éħ ¶ +æ»ļ åĬ¨ +éļ¾ åħį +åIJī åĪ© +ä¸Ģ ä¸Ģ +ç²¾ å¯Ĩ +伸 æīĭ +礼 仪 +åħ¨ æĺ¯ +è¶Ĭ 大 +ä¸Ń æłĩ +åıĸ åĨ³ +åıĸåĨ³ äºİ +éĢĶ ä¸Ń +讨 åİĮ +æīĭ åĨĮ +第 ä¹Ŀ +åŃĶ åŃIJ +çĦ¶ å¾Į +ä¸Ģ åħ± +æµ· æĬ¥ +款 å¼ı +æķ´ 天 +è¾¹ çķĮ +è·¯ è¾¹ +æĻĭ 级 +åIJIJ 槽 +çļĦ åħ³æ³¨ +æĪij 没æľī +å°±æĺ¯ åľ¨ +缮 çļĦæĺ¯ +åį³ä½¿ æĺ¯ +顶 å°ĸ +å·²ç»ı åľ¨ +å®īåħ¨ éļIJæĤ£ +æłĩ æĿĨ +åįĹ éĢļ +ä¼ļ 对 +座 ä½į +èµ¢å¾Ĺ äºĨ +åİŁæĿ¥ çļĦ +身 为 +书 åºĹ +è¢Ń åĩ» +ä»Ĭ æĻļ +以 èī² +以èī² åĪĹ +æĬĸ éŁ³ +åį´ 没æľī +丧 失 +çļĦ å±ĢéĿ¢ +åįģåĽĽ äºĶ +çŃī 缸åħ³ +æ±ĩ æĢ» +å¤ĸ 表 +为 æ°ij +éľĩ æĥĬ +å¥Ĺ è·¯ +çĬ¯ç½ª å«Įçĸij +å°Ĩ 以 +çİĩ é¢Ĩ +éħĴ åIJ§ +è¡Įä¸ļ åıijå±ķ +å¹´ èĩ³ +åĻ¨ æĿIJ +åĴĮ æĬĢæľ¯ +æľĢ å°ı +è¿Ļä¸Ģ åĪĩ +èģĮ 称 +å½ĵ ä½ľ +æİĢ èµ· +åĴ ĭ +ä¸Ń éĥ¨ +æīĭ èĩĤ +ç½¢ äºĨ +媳 å¦ĩ +æ´½ è°Ī +æĹ¶ä»£ ä¸ŃåĽ½ +人çĶŁ çļĦ +æŀģ éĻIJ +ç¦ Ħ +åĮº æĶ¿åºľ +æľ¬ éĴ± +礼 åĵģ +çļĦ éĤ£ä¸ª +侦 æŁ¥ +太å¤ļ çļĦ +å®ŀæĸ½ æĸ¹æ¡Ī +é«ĺ æłĩåĩĨ +æĮĩæĮ¥ éĥ¨ +åĢ¾ æĸľ +çī¹èī² 社ä¼ļ +çµIJ æŀľ +éĴ» çŁ³ +移 æ¤į +çī¹ ç§į +èĩª æĦ¿ +æĭľ çĻ» +åįķ 身 +åį´ åıĪ +åĪ¥ 人 +åIJĪ è§Ħ +æľº çĶµ +çī¹ æĦı +å½ĵåīį ä½įç½® +ä¹° 家 +åIJĪ 约 +èĤ© èĨĢ +为 åĩĨ +家 è£ħ +çļĦ çĥŃæĥħ +éĿŀ éģĹ +çļĦ éŃħåĬĽ +åİŁ åijĬ +社ä¼ļ åIJĦçķĮ +ä¹° çļĦ +å¤ļ åIJĥ +éĽķ å¡ij +èµ· ä¹ī +åĬł åī§ +éĤ£ä¸Ģ åĪ» +å°Ĩ è¿Ľä¸ĢæŃ¥ +æ¡Ĥ æŀĹ +æĽ´ 强 +对 ä¼ģä¸ļ +æĹł æĦı +ä¹łè¿ijå¹³ æĸ° +æµģ 失 +å¾® 软 +缸 对äºİ +座è°Ī ä¼ļ +主 èIJ¥ä¸ļ +主èIJ¥ä¸ļ åĬ¡ +ç§ģ åĭŁ +å±ķ示 äºĨ +常æĢģ åĮĸ +è² ´ +符 åı· +å¹´è½» çļĦ +å°± éľĢè¦ģ +ä¹Ł æĽ¾ +çļĦæĥħ 绪 +è¾¾ æłĩ +èĩ ¨ +ä½į å±ħ +ä»ħ 为 +é¦ĸ 家 +éĺ´ éĺ³ +ä¸įåĨį æĺ¯ +åĽłä¸º å®ĥ +ä¼ģä¸ļ åľ¨ +çĺ ¾ +åIJ¬ è§ģ +åİŁ æľī +åĪ¶ è£ģ +å¯Ĥ å¯ŀ +éĢļè¿ĩ 对 +æ»ij éĽª +è¿Ļ å¼ł +çļĦ çIJĨ解 +æĸ° ä¸ŃåĽ½ +è¿Ļ åĦ¿ +ä½İ ä»· +æĥ³ è¿ĩ +çļĦ ä¿¡å¿ĥ +建çŃij çī© +çļĦ é¢ľèī² +ä¸į åºĶ该 +æĹłçĸij æĺ¯ +å¼ķèµ· äºĨ +åħ¨ åijĺ +æĿ° åĩº +è¿Ļæĺ¯ æĪij +èª ° +èĺ ĩ +éĺµ åľ° +åħħ åĢ¼ +çŁ¿ ä¸ļ +çĿĢ ä»ĸ +ä¿¡ 访 +ä¸ĩ è¾¾ +æij© æĵ¦ +å¼Ģ 端 +èı² å¾ĭ +èı²å¾ĭ 宾 +车 åŃIJ +æľ¬èº« çļĦ +çģ«è½¦ ç«Ļ +常 å·ŀ +为 代表 +为代表 çļĦ +广 çĶµ +亲 人 +åı³ æīĭ +éĽĨ è£ħ +éĽĨè£ħ ç®± +çļĦ åį°è±¡ +æ©Ł æľĥ +åĮĨ åĮĨ +åħī çĶµ +大 æĸ¹ +è¿ĺ æľª +åĪ© 好 +ç»Ŀ 大å¤ļæķ° +åľ¨ è¿Ļç§į +ä¸Ģ ç»Ħ +æĸ° èĤ¡ +转 åıij +æ³ķ åºŃ +æĹł æīĢ +éģĵ è·¯ä¸Ĭ +çŁ¿ å±± +èij ī +æĶ¶ åĽŀ +称 ä¹ĭ +称ä¹ĭ 为 +æıŃ éľ² +åı£ 岸 +åIJ ¼ +å¿ĥ æĥ³ +çļĦ 梦æĥ³ +éĽ ¯ +ä¹ĭ åĪĿ +å¥ĸ 项 +订 éĺħ +èĵĿ 天 +åĿ¦ åħĭ +ç«ĭ æ¡Ī +èģĶ æīĭ +ä½Ĩæĺ¯ æĪij +帮 æĪij +ä»ħ 代表 +说 æĪij +çļĦ è¶ĭåĬ¿ +æ¯Ķè¾ĥ 大 +èµ° å»Ĭ +éĩįçĤ¹ é¡¹çĽ® +èµĮ åľº +åIJį çīĩ +æĦŁ åı¹ +åľ¨ åľ°ä¸Ĭ +åıij çĥŃ +èĮĥ çķ´ +çļĦ éģĵè·¯ +éĩij èī² +ä»ĸ åıĪ +ä¼ļ 产çĶŁ +æ°ij åĽ½ +å®ĺæĸ¹ ç½ijç«Ļ +æĶ¶çĽĬ çİĩ +çļĦ åĪ°æĿ¥ +çļĦ åĬŀæ³ķ +æĶ¹ åĪ¶ +ä¸ĩ ç§ij +ä¸į äºĪ +è¿ĻäºĽ éĹ®é¢ĺ +çĪ± ä¸Ĭ +çIJĥ åľº +è´£ 令 +æİĪ 课 +åľ¨ é¦Ļ港 +ç»Ĩ èħ» +å¤ļ ä¸ĩ +åIJĮ å¹´ +大 使 +æĸ ĭ +ä¹Ł 为 +æĥł å·ŀ +åIJī 祥 +çĶ° åĽŃ +åĽ½å®¶ éĺŁ +éĩį çĶŁ +åľ¨ åħ¶ +é¦Ļ åij³ +è´Ł èį· +亲 åĪĩ +èĩª 豪 +没 éĶĻ +åĽłä¸º åľ¨ +æĺŁ æĺŁ +éĤ ij +è¿ĺæľī å¾Īå¤ļ +æij© æīĺ +æij©æīĺ 车 +æŃ¥ è¡Į +管çIJĨ ä½ĵç³» +èĦļ ä¸ĭ +éģİ åİ» +æ±ī è¯Ń +对 ä¸įèµ· +çļĦ ç»ıåİĨ +åıĬ 缸åħ³ +ä¸įå°ij 人 +éĩį ç£ħ +åĬ³åĬ¨ èĢħ +大åĬĽ åıijå±ķ +æĢİä¹Ī åģļ +çĭĹ çĭĹ +举åįĹ äºļ +åĭĩ äºİ +åħ¬ éĸĭ +çĵ· çłĸ +åıĤ çħ§ +广æĴŃ çĶµè§Ĩ +举 åĬ¨ +æ±Ł 西çľģ +æķĪ èĥ½ +åĶ¯ æľī +éĿ¢ è²Į +èĩªåĬ¨ 驾驶 +æ¦ľ åįķ +å½ĵ æĪij们 +仲 è£ģ +æľ¨ æĿIJ +ç±³ åħ° +çĻ½ éĵ¶ +çļĦ 人éĥ½ +å°± åĥıæĺ¯ +æŃ¥ åħ¥ +åįł çĶ¨ +åĩ» è´¥ +让 大家 +ä¼ļ è®©ä½ł +åİ¿ æĶ¿åºľ +è¦ģ çĶ¨ +çŃī å½¢å¼ı +åįĩ é«ĺ +责任 æĦŁ +å¤ĩ çĶ¨ +ä»ĸ 认为 +æ¸ħåįİ å¤§åѦ +ä»ĸ èĩªå·± +éĸ± è®Ģ +太平 æ´ĭ +éĶģ å®ļ +çŃ Ĩ +è¿Ļ çīĩ +æī§ æĶ¿ +è¿ĶåĽŀ æIJľçĭIJ +å°± æѤ +éģĩ åĪ°äºĨ +å¼Ģå¹ķ å¼ı +管çIJĨ éĥ¨éŨ +姿 åĬ¿ +设 æĥ³ +åĽĽ åŃ£ +æĬĢæľ¯ 人åijĺ +å·® çĤ¹ +è¾ŀ èģĮ +èĢģ 師 +çļĦ æĦŁåıĹ +ä¹Ł éĿŀ常 +å¹´ ä¸ĬåįĬå¹´ +æĢª çī© +èĮĥ æĸĩ +æĪĺ å½¹ +åIJ« ä¹ī +åħ¨ è¿ĩç¨ĭ +èĢĮ éĿŀ +éĢļ讯 åijĺ +è¿Ļæł· æīįèĥ½ +æľº ç»Ħ +è£ ı +çķ¶ çĦ¶ +èµĮ åįļ +åIJĦ æľī +å·¥ä½ľ æľºåĪ¶ +äºĭ åIJİ +åī§ éĻ¢ +å±Ĭ æŶ +åĺ´ éĩĮ +主 线 +ä¸Ģ åľĪ +主è¦ģ åİŁåĽł +å°¸ ä½ĵ +åĮ»çĸĹ åĻ¨æ¢° +ä½ł æĢİä¹Ī +ä½Ĩ çĶ±äºİ +æŶ 空 +çĶ· æľĭåıĭ +çĶľ èľľ +é«ĺ åľ° +æĻ ĸ +èĴIJ éĽĨ +åĩĿèģļ åĬĽ +å¤ĩ åıĹ +æĸĩ åĪĽ +马 æĿ¥ +马æĿ¥ 西äºļ +æŁ´ æ²¹ +使 人 +æķĻ ä¼ļ +ç§ĭ 天 +æĺİ çıł +åħŃ åįģ +çݯå¢ĥ ä¸Ń +æ¸ħ æĻ¨ +积æŀģ åıĤä¸İ +å·ħ å³° +为 æľŁ +çŃ¾ åŃĹ +æĦŁ æ¿Ģ +ç§ĭ åŃ£ +æĿij åŃIJ +æ¢ħ 西 +æļ´ 鼨 +çĶŁæ´» åľ¨ +çªĹ æĪ· +æģ¶ åĬ£ +纯 ç²¹ +åľ¨ æİ¥åıĹ +没 èĥ½ +è¡Į 人 +åĭ º +æĭ¨ æīĵ +ä½ľ åĩºäºĨ +çļĦ 主é¢ĺ +æľª ä¾Ĩ +ä¸Ń æľĢ +æ¾ ľ +é«ĺ è¡Ģåİĭ +åħ´ èµ· +æŃ£ èĥ½éĩı +åŁ¹è®Ń çıŃ +æİ¥ åħ¥ +çĦ¶åIJİ åĨį +åѦçĶŁ 们 +é¢ĨåħĪ çļĦ +çģ« çĥŃ +ä¸ĵ èģĮ +æĪĸèĢħ 说 +建 è¨Ń +é» ı +对 åħ¬åı¸ +çī¹ æľīçļĦ +åħī èį£ +å½ĵ åľº +éĿ¢ åŃIJ +èµĦ产 管çIJĨ +æĹ¶æľŁ çļĦ +çŀ İ +åįİ ä¸ľ +åıĪ ä¸Ģ次 +èĥİ åĦ¿ +å®ļ çĤ¹ +头 çĹĽ +液 ä½ĵ +æĺ¯ä¸Ģ ä½į +帽 åŃIJ +å¹´ èµ· +ä¸į ä½İäºİ +è¾ĥ å°ij +éĿ¢ä¸´ çĿĢ +å±Ĥ å±Ĥ +èĿ´ èĿ¶ +èī° èĭ¦ +éĺ¿ æł¹ +éĺ¿æł¹ å»· +æ¦Ĥ æĭ¬ +请 éĹ® +èµ· åºĬ +å±Ģ å±Ģéķ¿ +稳 åģ¥ +å¦Ĥæŀľ æĪij们 +éħĴ ç²¾ +æĪ· åı£ +æĦŁ æĤŁ +æĪij们 éľĢè¦ģ +æĬĢ èīº +èĩª åªĴä½ĵ +è¿Ľ åĮĸ +æ¿ĢçĥĪ çļĦ +ä½ĵ 温 +èļ ķ +èĩ´ è¾ŀ +宪 æ³ķ +ä¸Ģ çŃīå¥ĸ +çĵ¶ é¢Ī +æĥł æ°ij +èµ° è·¯ +çİ° ä»» +åķĨ éĩı +ä¸ĭ 车 +åĪ ł +責 ä»» +èŀįåIJĪ åıijå±ķ +ç´ł æĿIJ +æ²¹ ä»· +åģļ 人 +çŀ ª +æĶ¹éĿ© åĪĽæĸ° +çļĦ åĮºåĪ« +è·¨å¢ĥ çĶµåķĨ +æ¶īåıĬ åĪ° +æīĺ 管 +æĪij è¿ĺæĺ¯ +åĿIJ æłĩ +ç½ij 讯 +å½ĵåľ° çļĦ +追 溯 +åľŁ èĢ³ +åľŁèĢ³ åħ¶ +åºķ ä¸ĭ +åĩł åįģå¹´ +ç©¿ è¿ĩ +çĶŁæĢģ æĸĩæĺİ +æݨ èĸ +æݨèĸ ¦ +éł Ĩ +åĴ³ åĹ½ +åĪĨ æĪIJ +çĹķ 迹 +æĪ· ç±į +éĥ½ ä¸įèĥ½ +æĻļ ä¼ļ +åĢ © +ä½ĵ åĬĽ +è¿Ļ个 èģĮä¸ļ +æĹł å½¢ +åıª æĥ³ +è¿Ľ åıĸ +æĿĢ æŃ» +èĦ Ĭ +äºij åįĹçľģ +æľª çŁ¥ +ç¾İ èģĶ +ç¾İèģĶ åĤ¨ +å¤ĸ å½¢ +诱 æĥij +çĽ £ +è¡Į 使 +åłĨ 积 +çĨŁ ç»ĥ +éĺIJ è¿° +æľĢ大 éĻIJ度 +å·¡ æŁ¥ +夺 åĨł +ä¼ģä¸ļ æĸĩåĮĸ +çĭ® åŃIJ +ä¿Ŀ å®Ī +ä¸ºæł¸å¿ĥ çļĦ +æī© æķ£ +åĪ¶éĢł åķĨ +æŁĶ 软 +为ä¸Ģä½ĵ çļĦ +游 çİ© +çĶŁ çĹħ +幫 åĬ© +åĶ± æŃĮ +æīį åı¯ä»¥ +宽 æĿ¾ +è¦ģ æ¯Ķ +æĺ¯ æĢİæł· +çģ° èī² +çİĭ åĽ½ +æIJħ æĭĮ +计 éĩı +åij¨åĽ´ çļĦ +æĻºèĥ½ æīĭæľº +常 åĬ¡ +常åĬ¡ åī¯ +é© ´ +å°Ĩ è¿ij +寻 常 +ä¸ŃåĽ½ å¸Ĥåľº +容 åĻ¨ +å±± ä¸Ĭ +èĥĮåIJİ çļĦ +亲 å¯Ĩ +æīĢ以 说 +éİ ® +çļĦ çIJĨçĶ± +大 åŁİå¸Ĥ +常 å¹´ +æĹħ游 ä¸ļ +å°±æĺ¯ è¿Ļæł· +åĨį æĿ¥ +é«ĺ ä½į +åĨħ 饰 +æŀĦ éĢł +ä¸Ģ èµ·æĿ¥ +çĶ³ è«ĭ +å·²ç»ı å¼Ģå§ĭ +çļĦ åĬ¨ä½ľ +被 è¿« +éģį å¸ĥ +åīĸ æŀIJ +å°ı äºĭ +å¿ĥ ä¸ŃçļĦ +ä½ĵåĪ¶ æĶ¹éĿ© +çļĩ 家 +æķĻ åłĤ +åIJĥ å®Į +åĽ½æ°ij åħļ +æĺİç¡® äºĨ +åıijå±ķ è§ĦåĪĴ +第ä¸Ģ æŃ¥ +å¾Ĺ èµ· +åľ¨ åĵª +çļĦ è·¯ä¸Ĭ +é» Ķ +çķ¶ æĻĤ +大åĬĽ æĶ¯æĮģ +åıĮ éĩį +çŁ¥éģĵ èĩªå·± +åIJĪä½ľ åįıè®® +æ°Ķ åĬ¿ +éķ¿æķĪ æľºåĪ¶ +ç½ķ è§ģ +åĽŀ æĿ¥äºĨ +ä»ĸ ä¼ļ +ä¸Ń æĸ° +ä¸Ńæĸ° ç½ij +çļĦ åķĨåĵģ +èµł éĢģ +決 å®ļ +å¸Ĥåľº çĽij管 +çķĻ åѦçĶŁ +çĶµ åİĭ +äºļ 马 +äºļ马 éĢĬ +è¿ĺæĺ¯ æ¯Ķè¾ĥ +ä¿ĥè¿Ľ äºĨ +æµģ åħ¥ +æijĦ åĥı +æijĦåĥı 头 +æıIJ åıĬ +åıij æİĺ +æī¾ åĩº +æ¢Ŀ 件 +ç¹¼ çºĮ +æĪij åĸľæ¬¢ +å¥ İ +æ¦ľ æł· +å¼Ģ èĬ± +æ²ī éĩį +åŁº åĩĨ +ä»ħä»ħ æĺ¯ +轨éģĵ 交éĢļ +åĶIJ å±± +çŃī ä¸Ģç³»åĪĹ +ä¸įè¿ĩ æĺ¯ +åŃĺåľ¨ çĿĢ +èĬ± çĶŁ +å¤ · +ç»Ī 究 +ä¹Łæĺ¯ ä¸Ģ个 +åįģ åŃĹ +èĸª éħ¬ +伤 å¿ĥ +æĺ¥ ç§ĭ +åĨ· åį´ +ç²¾ çģµ +çļĦ åľ°åĽ¾ +æ¯Ķ çī¹ +æ¯Ķçī¹ å¸ģ +æĢ§ åĪ« +ä½Ļ ä¸ĩåħĥ +ä¸įå¿ĺ åĪĿå¿ĥ +å¿ĥ çĸ¼ +æĽ² 线 +é«ĺ ä½İ +è¦ı å®ļ +æĻ¯ èī² +è¦ģ 说 +åħ¬åı¸ å°Ĩ +液 åİĭ +è¿Ŀ 约 +åİļ 度 +åºŀ 大çļĦ +è¿ĺæĺ¯ å¾Ī +é¦ĸåħĪ æĺ¯ +çµ ² +åĬ¡ å®ŀ +並 ä¸Ķ +å¢ŀ è¿Ľ +ç»Ħç»ĩ å¼Ģå±ķ +èµ·æĿ¥ äºĨ +è¾ĥ å°ı +导 游 +两 åľ° +ç¿ ĺ +çģ¿ çĥĤ +é£İ éĩĩ +æĶ¯ 线 +æĶ¯çº¿ ä»»åĬ¡ +娱ä¹IJ åľĪ +天津 å¸Ĥ +åĮħ åĽ´ +æľ¬ èµĽåŃ£ +éĩįè¦ģ 讲è¯Ŀ +åıĮ åIJij +åįİ ä¸½ +éĶ ¤ +åĦ¿ 女 +åįĸ åĩº +ä¾Ĩ 說 +ä»ĭç»į ä¸Ģä¸ĭ +åIJ¦ 认 +åĭ Ŀ +æĻ®éĢļ 人 +çļĦ åĬ¨åĬĽ +涨 åģľ +åŁºéĩij 管çIJĨ +ä¸Ģ个 éĩįè¦ģ +è¿IJ æ²³ +çħ ŀ +è´¢æĶ¿ éĥ¨ +è¡Įä¸ļ åįıä¼ļ +éĥ½ å°Ĩ +è¨Ģ 论 +ä¸ĭ ä¾Ĩ +墨 西 +墨西 åĵ¥ +åĽłä¸º ä»ĸ们 +æĢİä¹Ī åĽŀäºĭ +åĬłå¤§ 对 +èĬ Ń +çīĮ åŃIJ +ä¼ļ 使 +妹 åŃIJ +ç«Ļ éķ¿ +å¿ħ å¤ĩ +æłij æľ¨ +æģ¶ æĦı +æ²³ éģĵ +å¯Į è£ķ +ç¹ģ åįİ +代表 åĽ¢ +æµij 身 +é¦ĸ ä½į +èĪªç©º åħ¬åı¸ +鼻 å½± +ä¸ĵ è¾ij +æ°´ æºIJ +ä¸Ń æ¯Ĵ +並 ä¸į +èĢĮ åİ» +é ĥĿ +äºİ æѤ +æĸĩåĮĸ 建设 +èĤ¯å®ļ ä¼ļ +å¸ĮæľĽ 大家 +æıı åĨĻ +ä½İ è°ĥ +æĸ°åħ´ 产ä¸ļ +æ·Ħ åįļ +æĶ¾ å¼Ģ +çļĦ æĢ§æł¼ +çĸ¾çĹħ çļĦ +æķ´ é¡¿ +线ä¸Ĭ 线ä¸ĭ +éĢī 项 +çļĦ 认åı¯ +æķ´ é½IJ +çĶļ ä¹Ī +çľģ åĨħ +åı¤ 人 +æ°ij ä¿Ĺ +çī¡ 丹 +éŨ çªĹ +éĤ£ æł·çļĦ +çĽijäºĭ ä¼ļ +ç¿¡ ç¿ł +ç¦ ¹ +åįĥä¸ĩ ä¸įè¦ģ +æĶ¶ 缩 +çļĦ æĸĩåŃĹ +åĴĮ å°ļ +æĮĩ 令 +åħ±äº§ åħļåijĺ +çļĦ çĪ¶äº² +å®Į å·¥ +åĬ¡ å·¥ +马 æĭī +马æĭī æĿ¾ +æµĭ è¯Ħ +å² ļ +ä¸į åģļ +ä¸ĥ å¹´ +åĿĩ ä»· +主 è§Ĥ +å¾Ī ä¸įéĶĻ +èĤ¡ä¸ľ 大ä¼ļ +äºĶ ä¸Ģ +é£İ åIJ¹ +å¼Ģ éĩĩ +è¿Ļä¹Ī 大 +èĥ½ çľĭåĪ° +èĢĥ è¯Ħ +åį³ 便æĺ¯ +çݰ代 åĨľä¸ļ +æ¯Ķè¾ĥ é«ĺ +è¦ģ çľĭ +没 äºĨ +解 決 +çݯ æ¯Ķ +åĨ² åĬ¨ +æ·± å¤ľ +åĩł åįĥ +ä¿ ı +ç½ij æ°ij +å°± 没 +ä»ĸ 表示 +éĩı åŃIJ +æĹ©é¤IJ åĬłçĽŁ +åįĬ å²Ľ +æIJŀ ç¬ij +ä¸Ĭ æĬ¥ +å¯ © +é¢Ħ 订 +èľĤ èľľ +æŁ¥ æī¾ +ä¼Ĺ æīĢ +ä¼ĹæīĢ åij¨ +ä¼ĹæīĢåij¨ çŁ¥ +æĹ© æĹ¥ +åıij æī¬ +åĴĮ 个人 +åĬłåħ¥ äºĨ +åĸ® ä½į +åĪĨ æĺİ +第ä¸Ģ æī¹ +ç¾İ åĨĽ +æĿĢ æīĭ +éŨ å¤ĸ +åķĨ åľĪ +ä¸Ģ åĪ» +çļĦçľ¼ ç¥ŀ +éľ Ħ +äºĽ ä»Ģä¹Ī +åĬł æ·± +æ¯ı ä½į +å¸Ĥ éĿ¢ä¸Ĭ +åıĶ åıĶ +çļĦ éĤ£ç§į +粤 港澳 +è´´ å¿ĥ +æĸĩåĮĸ 产ä¸ļ +红 æĹĹ +åĺī åħ´ +æĶ¶ çĽĺ +å®ĮæĪIJ åIJİ +ä¼ģä¸ļ 管çIJĨ +纵 横 +ä¸į ä¿¡ +æĪIJ éĥ½å¸Ĥ +æ´Ĺ 澡 +举è¡Į çļĦ +çĶ¢ çĶŁ +ç©¿ ä¸Ĭ +åĪļ 好 +åħī 线 +æīĵ æŀ¶ +è¿Ļ æľ¬ä¹¦ +åĶ®åIJİ æľįåĬ¡ +åĩł åĪĨ +ä¸Ĭ 次 +ä¸į åĪĨ +产 åIJİ +éģ¿ å¼Ģ +ç»Ī æŀģ +代表 大ä¼ļ +æ¼Ķ æĬĢ +åĽŀ è´Ń +åѦ è´¹ +éĺ» ç¢į +ä¸Ģ大 æī¹ +ç«£ å·¥ +åĨ³ å®ļäºĨ +ä½Ĩ å¦Ĥæŀľ +çĶµ æµģ +ä¸Ŀ 毫 +èĥ½å¤Ł åľ¨ +éĶĢåĶ® æĶ¶åħ¥ +åľ¨ åŃ¦æł¡ +æ°´ åĩĨ +è§Ĩ 线 +èĩª åľ¨ +åķĨä¸ļ éĵ¶è¡Į +为äºĨ 让 +çį² å¾Ĺ +çݩ家 æľĭåıĭ +éĿ¢ èĨľ +åĪĨ åī² +åī§ æľ¬ +ç« Ń +说 å¾Ĺ +æĥ³ çŁ¥éģĵ +çļĦ人 çī© +èĮħ åı° +åIJĮ ä¸Ģ个 +æķ°æį® ä¸Ńå¿ĥ +çĶ Ħ +åĸľ æĤ¦ +ä¸ĭæĿ¥ çļĦ +å®ļ åIJij +æŀģ åħ· +çļĦ åľŁåľ° +éĤ£ åĢĭ +æijĦ åħ¥ +äºĨ æĪijçļĦ +马 è·¯ +åħ¨ 社ä¼ļ +è®® æ¡Ī +å±ĭ åŃIJ +åIJį åı« +åĮ ª +åľ¨ å¤ĸéĿ¢ +åįİ åįĹ +åıij è´§ +å¯Ĵ åĨ· +é«ĺçŃī æķĻèĤ² +详ç»Ĩ çļĦ +个 é¡¹çĽ® +çĶŁäº§ åĬĽ +æŶ 常 +å°± æľĥ +ä¸ĩ èĤ¡ +éĻĮçĶŁ 人 +æıı ç»ĺ +å½ĵ çĦ¶æĺ¯ +æĭī åĬ¨ +éĵ¾ æĿ¡ +æī£ éĻ¤ +ä¸Ģ缴 éĥ½ +å°ı åŃ©åŃIJ +伤 åı£ +第äºĮ å±Ĭ +è´Ń ç½® +çļĩ 马 +æĹł èģĬ +表 åĨ³ +诸 å¦Ĥ +åĵį èµ· +é£İ æļ´ +ä¸Ģæµģ çļĦ +ç ·¨ +解æĶ¾ åĨĽ +室 å¤ĸ +å°± è¿Ļä¹Ī +å³ ¶ +æīĢæľī 人éĥ½ +æIJľç´¢ å¼ķæĵİ +çļĦ æĪIJæľ¬ +åħļ æĶ¿ +åıijè¡Į 人 +çļĦ äºĭå®ŀ +对 该 +åıĹ æįŁ +ä¿Ħ ä¹Į +é²ľ èĬ± +åĨľ èį¯ +æŀģ éĢŁ +æĢ¥ æĢ§ +两 ä¼ļ +ä¸ĢèĪ¬ æĿ¥è¯´ +æµ· é²ľ +åĨ Ī +çĶ¨ 人 +çĶ¨äºº åįķä½į +åĢ ª +åĦª æĥł +æł¹ æºIJ +åĽ¢ è´Ń +ç¾İ æ´² +ä¸ĭ è¡Į +å¹´ æľ« +èľ ¡ +è¯ģ 件 +åľ¨ æĪijåĽ½ +ä¸į åºĶ +æĮī æŶ +åłª 称 +åľº ä¸Ĭ +å¹²éĥ¨ èģĮå·¥ +æľī å¾Ī大çļĦ +æķ°åŃĹ ç»ıæµİ +æ¼Ķ ç»ĥ +æį® ç»Łè®¡ +å¾Ģ æĿ¥ +广åijĬ æľįåĬ¡ +çļĦ è·Ŀ离 +æŃ ¸ +è¨Ģ è¯Ń +被 èªī +被èªī 为 +åĭī 强 +å°Ĭ æķ¬ +ä¸ĩ 亿åħĥ +ä¸ŃåĽ½ åĽ½éĻħ +å¹² é¢Ħ +å¹´ 产 +èĢķ åľ° +èĮ İ +åį³ æĺ¯ +æĺ¨ æĻļ +æĪIJ为 ä¸Ģ个 +çºł æŃ£ +åij½ åIJį +é¢ģ å¸ĥ +çĮľ æµĭ +ä¿ĿèŃ· æĶ¿çŃĸ +æĭ ¢ +æ´» æ³¼ +çŃī éĥ¨éŨ +åѦ åĪ° +å¢ŀåĢ¼ ç¨İ +èĪª 线 +åĨ ¤ +åįģ åĩłå¹´ +æݧèĤ¡ èĤ¡ä¸ľ +ä¸Ģ éŨ +个 å·¥ä½ľ +ä¸ªå·¥ä½ľ æĹ¥ +æĸ° 西 +æĸ°è¥¿ åħ° +论 è¯ģ +ä» Ĩ +åı¦å¤ĸ ä¸Ģ个 +æĶ¹ ç¼ĸ +严 ç¦ģ +åĸľ 好 +个人 ä¿¡æģ¯ +满æĦı 度 +åĵ ¨ +å¸Ī èµĦ +æĶ¹ 为 +ç«ŀäºī 对æīĭ +åĩº çĤī +åķĨ 人 +大 æ£ļ +æĮĩ导 ä¸ĭ +å¦ĩ ç§ij +è¼ ª +æī ģ +åIJĮæŶ è¿ĺ +并 éĢļè¿ĩ +æĪĺ éĺŁ +èĶĵ 延 +ä¿ ŀ +éĢĤå½ĵ çļĦ +åīį è¾Ī +åĵģ åij³ +湿 åľ° +æĪIJ åŀĭ +ä¸į åıªæĺ¯ +æĥ© ç½ļ +åĩºåı° äºĨ +çİ© 游æĪı +æīį åıijçİ° +åºĶ èģĺ +å¤ĸ æĿ¥ +åįł é¢Ĩ +å±ķ æľĽ +å« Ĥ +港 èĤ¡ +æ¡Į ä¸Ĭ +æĶ¯ æŁ± +çļĦæĥħ å½¢ +广éĺĶ çļĦ +æĶ¯ è¡Į +å´© æºĥ +æľĪ ä¸Ń +æľĪä¸Ń æŬ +ç»į åħ´ +临 è¿ij +æĬ¤ æłı +æļ ® +åįķ èģĮä¸ļ +è¾¹ å¢ĥ +æĹ¥ çħ§ +ä¸Ģ åłĨ +缴 å¾Ħ +åħ±åIJĮ ä½ĵ +æĸ°åįİ ç½ij +æīĵ 好 +çĶµåĬ¨ 汽车 +ä¸į æĺİçĻ½ +éĢĻ 裡 +缼 大 +çİĭ æľĿ +åĨį ä¸Ģ次 +åĬŀåħ¬ åİħ +è´¨ æĬ¼ +åIJĪ åĩ» +人们 对 +鼶 é£Ł +éĥ½ä¸į çŁ¥éģĵ +çļĦ è¯Ńè¨Ģ +åĭŁéĽĨ èµĦéĩij +åĬ¨ èĦī +å½ ¤ +è¿Ļ åĩłå¹´ +çŁŃ è§Ĩé¢ij +太 é«ĺ +常 å§Ķä¼ļ +åĬł çıŃ +éĩį å¿ĥ +åªĴä½ĵ æĬ¥éģĵ +没 æ³ķ +éĹ» åIJį +çĥŃ åº¦ +å¹¿æ³Ľ çļĦ +åħŃ å¤§ +çī© ä½ĵ +ä¸į 该 +é¢ĺ 主 +精彩 çļĦ +为 è¿Ľä¸ĢæŃ¥ +èĻ ŀ +åĽº çĦ¶ +è´µå·ŀ çľģ +çºł ç»ĵ +代çIJĨ 人 +æ³ķå®ļ 代表 +åı¦ä¸Ģ ç§į +ä¸į åIJ« +æĭ¯ æķij +ä¼ļ ç»Ļ +è¯Ĺ è¯į +åIJĮ ç±» +å¾Ĺ ä¸įåĪ° +æĬĵ ç´§ +以 åħ¶ +åħ¥ åħļ +è¿ĺ åı¯ +æľŁ åĪĬ +å¾Īå¤ļ æŶåĢĻ +æĹ¥ åIJİ +åħ¬ 约 +ä¸Ģ 举 +æ¯Ķè¾ĥ å¤ļ +éĩij æ²Ļ +æį ŀ +æİĴ åĩº +æѦ æľ¯ +ä¸į æĸ· +ä¸Ń èĢĥ +ä¿¡ èµĸ +ä»İä¸ļ 人åijĺ +çģ« çĦ° +éĨĴ æĿ¥ +ä½İ 温 +éĢ¾ æľŁ +åĬ± å¿Ĺ +éħ ¥ +åı¯è°ĵ æĺ¯ +è¿Ļ æĦıåij³çĿĢ +é¢ł è¦Ĩ +åĮĹ京 大åѦ +ä¸ĵ 线 +åıĬ 以ä¸Ĭ +è¨ ª +èĢĮ åIJİ +çŁ¥ ä¹İ +ä¸Ģ对 ä¸Ģ +å¨ĥ å¨ĥ +çģ¾ éļ¾ +åħ¨ å±Ģ +æīĢå¾Ĺ ç¨İ +å®ŀ æĥł +èļĤ èļģ +ä¹Ł çŁ¥éģĵ +温 åĴĮ +èIJ½ ä¸ĭ +åŀĭ ä¼ģä¸ļ +åĨį ä¹Ł +ä¾Ľ çĥŃ +é«ĺ æ½® +çĢı覽 åĻ¨ +çļĦ 巨大 +åħĪ 天 +å¹´ ä¸ŃåĽ½ +类似 çļĦ +çIJĨäºĭ ä¼ļ +空 éĸĵ +çģµ æĦŁ +åĬĽ æ°Ķ +带 ä¸Ĭ +ä¸į好 æĦıæĢĿ +æľī ä½ķ +å·² åľ¨ +åıĸ åĩº +è¿Ŀæ³ķ çĬ¯ç½ª +åŃ¦ä¹ł 贯彻 +åľ° 带 +楼 梯 +çŃī æĥħåĨµ +ä»İ åīį +çļĦ ä¹łæĥ¯ +ç³Ł ç³ķ +å°± èĥ½å¤Ł +è© ķ +ä¸Ģ å¾ĭ +æĮ« æĬĺ +åİŁæĸĩ åľ°åĿĢ +å½ĵ å±Ģ +ä¸į éĢļ +æķ° åįĥ +éĺŁä¼į 建设 +æŶ èĬĤ +åģļ èµ· +çļĦ è®°å¿Ĩ +ç½ij绾 å®īåħ¨ +åĩ¡ æĺ¯ +æ° ¯ +éĽķ åĪ» +åŁĥ åıĬ +æĪij åı¯ä»¥ +çĽij çIJĨ +æĽ´ åħ· +åŁİ 管 +èĭ ¯ +åı¥ åŃIJ +èĭ¥ æľī +ä»İæĿ¥ ä¸į +缸åħ³ è´Łè´£ +å®īåħ¨ æĦŁ +æĽ´ è¦ģ +çļĦæĥħ æĦŁ +çī¢ çī¢ +è¾ĥ 好çļĦ +æ° ® +ç¬ij è¯Ŀ +车 å±ķ +ä¹ĭ ç¾İ +ç®Ģ 约 +ç±»åŀĭ çļĦ +èĢģ åĮĸ +çľĭ ä½ł +è¿ĩ åĪĨ +éŨ åīį +ä¸Ģ éĹ´ +æĥ³ åİ» +åª Ľ +åľŁ è±Ĩ +åıĪ 称 +ä¸Ń ä¿¡ +åŃĺ éĩı +马 äºij +èĩ´ 使 +åħĪ åīį +èĢģ åŃIJ +æīĵ æī® +æ¯ķä¸ļ äºİ +æ¯ķä¸ļ åIJİ +ç¾İ好 çĶŁæ´» +å·¥ä¸ļ ä¼ģä¸ļ +就好 äºĨ +èħIJ èļĢ +çıį çıł +åĪ° è¿ĻéĩĮ +æīĢéľĢ çļĦ +è¿Ļæĺ¯ åĽłä¸º +çIJĨæĥ³ çļĦ +å·®å¼Ĥ åĮĸ +é ® +é® ® +äºļ 太 +æĹł ç©· +æıIJ çİ° +ä¸ĵä¸ļ æĬĢæľ¯ +çĶ¢ æ¥Ń +åѦ åŃIJ +ç§ij å¹» +åįłåľ° éĿ¢ç§¯ +ä¸į åĩĨ +æľªæĪIJ 年人 +æĶ¶ å½ķ +è¿ĺ 款 +éĴ¢ çŃĭ +æ¼ ¢ +å¾Ĺ æĦı +综åIJĪ ä½ĵ +æŀģ é«ĺ +åįķ è¯į +é«ĺæķĪ çļĦ +骨 头 +æī§ çĿĢ +缼 ä¸ĸ +模 çī¹ +æĽ´ èĥ½ +ç»Ŀ æľĽ +对åºĶ çļĦ +æ¨ Ĭ +æĸ° ä¸ī +æĸ°ä¸ī æĿ¿ +æģ° æģ° +åIJį 家 +æł¸å¿ĥ æĬĢæľ¯ +个 å°ı +æĢİä¹Ī ä¼ļ +说 ä¸įå®ļ +西 çĵľ +åĵ İ +ç¢ Ł +å¿ħ ä¸įåı¯ +å¿ħä¸įåı¯ å°ij +ä¹ĭ éĸĵ +åĪĨ 管 +交éĢļ äºĭæķħ +å¼Ģ åĬŀ +å¾ģæ±Ĥ æĦıè§ģ +äº ¨ +鼻åŃIJ éĥµ +鼻åŃIJéĥµ 件 +ä¿¡æģ¯ æľįåĬ¡ +ä½ł è§īå¾Ĺ +缴 è§Ĥ +å·² å®ĮæĪIJ +åĪĨ ä¼ļ +åĽŀ åįĩ +éļ » +好 人 +äºĨ解 ä¸Ģä¸ĭ +åį« æµ´ +æľĢ çĪ± +åºŀ 大 +客 æĪ¿ +çijŀ åħ¸ +éĥ½ ä¸įæĺ¯ +é¤ ¨ +èĹ ī +çļĦ åIJĦ项 +为 缮æłĩ +çļĦ è®¤çŁ¥ +å½±åĵįåĬĽ çļĦ +夸 å¼ł +佩 æĪ´ +æ±ĩ çİĩ +çļĦ çĪ±æĥħ +æĺ¥ é£İ +æĺ¯ æĪijçļĦ +æ¨ ¹ +åįĬ å°ıæŶ +å±± åİ¿ +å±± 西çľģ +èĢĮ è¿Ļ +æĽ´å¤ļ ä¿¡æģ¯ +è¿ĺ æľīä¸ĢäºĽ +ç²¾ ç»ĨåĮĸ +ç¾İ åѦ +çĶ± æĸ¼ +ä»ħä¾Ľ åıĤèĢĥ +å¾Ī é«ĺçļĦ +åıł åĬł +è¿Ļä¹Ī 说 +å±ķ åĩº +åĽĽ å¤Ħ +ä¸ĩ 家 +æĭĽ åĭŁ +çļĦ 强大 +æĤ£ æľī +å°ı äºİ +ä¹Łè®¸ æĺ¯ +对 èĩªå·±çļĦ +èģĮä¸ļ æķĻèĤ² +æĿ¥ è¿Ľè¡Į +æ¡£ 次 +æīĵ èµ¢ +éĥ½æľī çĿĢ +åº ¸ +è¯Ń æ°Ķ +çĶ² éĨĽ +空 åĨĽ +车 åĨħ +åĽłä¸º ä½ł +å®ŀ æķĪ +æĥħ ä¾£ +åıijè¾¾ åĽ½å®¶ +éķľ åŃIJ +æ¯į å©´ +ä½Ĩæĺ¯ ä»ĸ +积æŀģ æİ¨è¿Ľ +大å¹ħ 度 +çļĦ 女åĦ¿ +é¤IJ æ¡Į +åIJ¬ å¾Ĺ +çļĦ 积æŀģæĢ§ +好 åIJ§ +æĹ¥ æ¶Īæģ¯ +æľī ä»»ä½ķ +æ¯Ĵ åĵģ +æĹ©çĤ¹ åĬłçĽŁ +第ä¸Ģ 天 +å°½ åĬĽ +æł ĸ +主 æīĵ +æĺ¯ä¸Ģ åIJį +çĪĨ æĸĻ +äºĭä¸ļ åıijå±ķ +å¾® åķĨ +äºİä¸Ģä½ĵ çļĦ +çĶŁ çĮª +èĩªçĦ¶ èµĦæºIJ +çŀĦ åĩĨ +è§Ħ模 åĮĸ +并 ä¸İ +èĤ¥ èĥĸ +家 çĶ¨ +大 çĪ· +é¢Ħ åijĬ +æĿ¥ åģļ +éĺ³ åİ¿ +æŀĦ çŃij +é¢ģ å¥ĸ +åİĨåı² æĸĩåĮĸ +æľįåĭĻ æĪĸ +æĢ» åĨ³èµĽ +åıij åŀĭ +æĪij 羣çļĦ +æĽ ¦ +åıĤ ä¼ļ +èĦĨ å¼± +åĩĨ åħ¥ +èħ¹ éĥ¨ +åı¸ 令 +æĤ² åī§ +天 ä¸Ĭ +åı£ ä¸Ń +ä¸ĩ 个 +åѦ ä¸ļ +æıIJ åĢ¡ +两 è¾¹ +大 èĤ¡ä¸ľ +åı¤ éķĩ +è¡Ģ ç³ĸ +çļĦ ç¨ĭ度 +æ£ī èĬ± +åIJİ åı° +å°± åĮ» +æķ´ æķ´ +èĴ ² +çĽĪåĪ© èĥ½åĬĽ +ç± ½ +èĦ « +çľĭ éĩį +家 éķ· +èģĺ çĶ¨ +èµĽ éģĵ +åīį èĢħ +建 èŃ° +å¾ĭå¸Ī äºĭåĬ¡ +èīºæľ¯ åĵģ +æľī èĩªå·±çļĦ +åIJ¦ å®ļ +社 åĽ¢ +åij¨ äºĶ +带 åĪ° +å·¥ä½ľ ä¼ļè®® +èĤ¡ æľ¬ +å¤ĸ åĮħ +家 åħ¬åı¸ +çĽij çĭ± +èĪ Ĭ +åIJį æł¡ +西 æ¹ĸ +è¶ħè¿ĩ äºĨ +åįĹ å±± +ç»Ħ 件 +åĢ¼å¾Ĺ 注æĦı +æĮ£ æīİ +äºĭ 迹 +ç¶ĵ çĩŁ +ç§ij 室 +好 åIJĹ +æ¤ħ åŃIJ +åľĪ åŃIJ +ä½Ĩ 她 +æµģ çķħ +åIJĦèĩª çļĦ +èģĮ åijĺ +è¡į çĶŁ +åħ¨ åľº +æĴ¤ éĶĢ +åį´ 被 +å®ģ éĿĻ +åīį æīĢ +åīįæīĢ æľª +åīįæīĢæľª æľī +主 ä¸ļ +åĮĹ ç¾İ +è¯Ħ å®ļ +åĵģ å°Ŀ +大家 éĥ½åľ¨ +主 å¸ħ +ç»Ĩ å¿ĥ +ä¿¡æģ¯ æĬ«éľ² +çļĦ ç«ŀäºī +éĢĻ樣 çļĦ +ç§ijåĪĽ æĿ¿ +éĩĩ æijĺ +票 æį® +éĢIJ å¹´ +èĭ± è¶ħ +è¡Įä¸ļ åĨħ +人 寿 +åIJİ åĭ¤ +å¦Ĥ æĦı +ç¬Ķ è¯ķ +æ·¡æ·¡ çļĦ +ä¸į èĪĴæľį +ä½ĵ 积 +ä¹Łä¸į è¦ģ +éĿ¢ æĸĻ +æł· æľ¬ +ç¥ ģ +æĮī è§Ħå®ļ +大æ¦Ĥ æĺ¯ +æĥħåĨµ è¿Ľè¡Į +åIJĦ åįķä½į +çļĦ ç¬ij容 +åĩºèī² çļĦ +代表 æĢ§ +çļĦ ç¾İ好 +éĴ ¦ +å¾® çĶŁçī© +è¶Ĭ æĺ¯ +æĸ¹ åı¯ +å¹² èĦĨ +éģĬ æĪ² +çļĦ åħ´è¶£ +éĹ® è´£ +åĽłä¸º æĪij们 +èĢĥ éĩı +çĶŁ çĶŁ +éĺ» åĬĽ +ä¸į åħģ许 +æıIJ è®® +åĩı æĮģ +åıªæĺ¯ ä¸Ģ个 +æĪij æĬĬ +åıijçİ° èĩªå·± +å¢ŀ å¹ħ +å¦ į +èĹĿ è¡ĵ +ä¸Ģ家 人 +åĪĨ 级 +çļĦ æķ°éĩı +è½® èŀįèµĦ +çŃī åĽłç´ł +大 夫 +èģĺ 请 +é£İ æľº +绽 æĶ¾ +ä»»ä½ķ ä¸Ģ个 +éł Ĥ +éĺ¶ çº§ +æĬĬ 她 +è¿Ľ åĨĽ +èĥ½ åģļåĪ° +åŁ¹è®Ń æľºæŀĦ +çī© æĸĻ +ç«¥ è¯Ŀ +æĮĩ导 æĦıè§ģ +éĺ ® +æ·±åħ¥ æİ¨è¿Ľ +主 æľº +æ¸Ķ ä¸ļ +ä¸į æľį +æµĵ éĥģ +è¡Ĺ ä¸Ĭ +ä¾Ŀ 次 +æŶ 段 +æ¢ µ +çļĦ åĸľçĪ± +å¾Ī éķ¿ +åĪĿ 级 +æŀľ æĸŃ +æĬ¢ æķij +é¼ĵ èĪŀ +ä¾Ľ éľĢ +æ·±åħ¥ å¼Ģå±ķ +产ä¸ļ éĽĨ群 +åĻª éŁ³ +åIJ¬ çĿĢ +æ·±åĪ» çļĦ +å¿į åıĹ +çĶµ ç£ģ +强 èĢħ +æ»ĭ åij³ +æĽ¼ èģĶ +åı¯ä»¥ 缴æİ¥ +大 ç±³ +æŃ· åı² +æĶ¿åĬ¡ æľįåĬ¡ +åħ¬ å¼ı +社 群 +éģĵ士 èģĮä¸ļ +ä¹ĭ æĥħ +æµ· æ°´ +æ¼Ķ å¥ı +åºĹ éĩĮ +迹 象 +åıijå±ķ çIJĨ念 +é«ĺ 空 +åij¨ åĪĬ +åĽŀ åĪ°äºĨ +ä¸į éĢĤåIJĪ +åłµ å¡ŀ +åĬ Ī +æ°´ ä¸Ĭ +çĢij å¸ĥ +纳ç¨İ 人 +çĩĥ æ²¹ +å·¥ç¨ĭ é¡¹çĽ® +峡 è°· +æľī éĴĪ对æĢ§ +åľĨ å½¢ +æľ¬ å¸Ĥ +è¿Ļ è¯Ŀ +管çIJĨ èĢħ +ç¡®è¯Ĭ çĹħä¾ĭ +æĬĬ æīĭ +彩 èī² +ä¸Ĭ åīį +夯 å®ŀ +ç¾Ĭ èĤī +å¾Ģ å¹´ +æĵħ èĩª +è¿· 人 +èĪª æ¯į +ç²¾ ç»Ĩ +åľ¨ æĪijçļĦ +åĪĽ æĬķ +麦 åħĭ +æľĪ ç»ı +åĮĹ æµ· +ä¹ĭ æĺŁ +åı¶ åŃIJ +å¸Ĥåľº ç«ŀäºī +è¿Ļ äºĭ +åıĥ èĪĩ +产 åľ° +åĶ ī +åķĨåĵģ æĪ¿ +èĪª è¿IJ +ä¼ĺ å¼Ĥ +ä»ĸ们 æĺ¯ +鼨 æ°´ +è¯į æ±ĩ +åĨľ çĶ° +欧 éĺ³ +çŁŃ 线 +管 ç½ij +æł¹ åŁº +åıªæľī ä¸Ģ个 +éŀĭ åŃIJ +å¸Ĥ å§Ķ书记 +åĪ» æĦı +è¡Į 车 +åıĪ 被 +åı¯éĿł æĢ§ +è´ ± +ä»» åij½ +åºĶ åľ¨ +å°± å¾Ĺ +æľįåĬ¡ ä½ĵç³» +æĶ¿ æĿĥ +åıijè¨Ģ 人 +è¿ĩ å¾Ģ +两 åıª +èĻ½ 说 +éĢģ ä¸Ĭ +ä»Ģä¹Ī äºĭ +æķ£ æĸĩ +æİĮ æݧ +èĸĦ å¼± +ä¸ĭéĿ¢ å°± +主è¦ģ åĨħ容 +å¾Ī éĩįè¦ģçļĦ +å°± 说 +çĻ½èī² çļĦ +éĤ£ä¸ª æŶåĢĻ +ç»ı纪 人 +çļĦ æ¯į亲 +ç¬Ķè®° æľ¬ +åºķ å±Ĥ +è¿ij 代 +解 说 +è²ł 責 +æľĢ大 åĮĸ +åķĨ éĵº +æł¡ åıĭ +æ² ģ +ä¸į åĩºæĿ¥ +éĻ· éĺ± +ç¨ ħ +åħ¬å¸ĥ äºĨ +åĩĢ åĢ¼ +çĽ¸å¯¹ è¾ĥ +ç¬ Ľ +æł¸ ç®Ĺ +åįİ ä¾¨ +æĢ¥ æķij +æĮº 好 +åħĴ ç«¥ +äºĮ èĥİ +åĩº èĩª +åĿ Ł +æīĭ ä¸ĭ +å± ¡ +åĪĽéĢł æĢ§ +ä¸¥æł¼ æĮīçħ§ +åĨį åİ» +举 缣 +人 æµģ +äºĨä¸Ģ 声 +å°ıæŶ åīį +è´µ æĹı +éľ ĸ +ä¹Łæĺ¯ éĿŀ常 +éĢ ± +çľĭäºĨ çľĭ +ç¹ģ æ®ĸ +èĩ³ æѤ +é¢Ħ å¤ĩ +å¾Ī æĺİæĺ¾ +æ¼Ķ èīº +åĿIJ çĿĢ +ä¿Ħ åĨĽ +åľ¨ è¿ĩåİ» +ä¹ĭ äºĭ +æĬĵ èİ· +åĿIJ ä¸ĭ +çĶ± ä¸ŃåĽ½ +ä¹Ł å¼Ģå§ĭ +çŃĶ å¤į +åŀĥåľ¾ åĪĨç±» +éĴĵ é±¼ +åIJĦ 種 +缸 éģĩ +ä¸įåģľ çļĦ +æī¹ éĩı +éĩįè¦ģ ä½ľçĶ¨ +å§Ķ å±Ī +åħŃ å¹´ +ä¸ĥ åįģ +ä¹ĭ æĪĺ +é£İéĻ© 管çIJĨ +éŁ³ æ¨Ĥ +è¡ĮæĶ¿ å¤Ħç½ļ +æľ¬ äºĭ +æĴ° åĨĻ +èģļ åIJĪ +éĢĤ æŶ +æIJ¬ 家 +ç¢İ çīĩ +缼 å®´ +ç®Ģ æ´ģ +åı¬ éĽĨ +ç®Ģ åĮĸ +åĮĹ京 æŶéĹ´ +第ä¸ī å±Ĭ +æĿ¥ åĽŀ +常çĶ¨ çļĦ +京 æ´¥ +京津 åĨĢ +梦 å¹» +è¯ķ è¡Į +æľº åºĬ +åĪ° æľĢåIJİ +åĬ© æīĭ +åĪĨ 彩 +åĩº åĵģ +åĪ¹ 车 +åIJ¯ åıij +侧 éĿ¢ +æ¯ı å½ĵ +缸åħ³ è§Ħå®ļ +ä¸ĸ 人 +è´Ń 车 +å¿ĥ 缮 +å¿ĥ缮 ä¸Ń +äºĶ éĩij +è¿ĺ è®°å¾Ĺ +ä¾Ŀ çĦ¶æĺ¯ +æıIJ æ¡Ī +çĶµåķĨ å¹³åı° +åģļ åĪ°äºĨ +æĿľ ç»Ŀ +å®ī åįĵ +ä¸ĸçķĮ åIJĦåľ° +åīį éĢĶ +æ´Ĺ åĩĢ +å¥ĭ åĬĽ +åŁİå¸Ĥ 建设 +å¤ļ åĬŁèĥ½ +ä¼ļ éĢłæĪIJ +åıijå¸ĥ ä¼ļä¸Ĭ +究 ç«Łæĺ¯ +åĪĨ 红 +çŁ¥ èŃĺ +éĿ¢ æĿ¿ +æĹł 声 +æĢ¥ éľĢ +失 çľł +çĪ¸ å¦Ī +äº Ĥ +åħ¨ æĻ¯ +ç»ıåħ¸ çļĦ +åī§ ä¸Ń +é¢Ĩ导 ä¸ĭ +åħļ åĨħ +åħ¥ ä¾µ +æĭī æĸ¯ +ä¸Ģ å¹ķ +åĬł ä¹ĭ +èĤ Ĩ +èĭ± æł¼ +èĭ±æł¼ åħ° +å·§ åħĭ +å·§åħĭ åĬĽ +ä¸Ģ å¿ĥ +èģ Ĥ +å¾Ģå¾Ģ æĺ¯ +管çIJĨ å±Ĥ +çĻ» åħ¥ +建ç«ĭ èµ· +建 åĽ½ +åŃIJ 宫 +åºĶ ä»ĺ +æİ¢ 究 +第ä¸Ģ ä½į +ä½Ļ 家 +çŃī æ´»åĬ¨ +æīĢ èĩ´ +è¾ĥ å¿« +æĺ¯ éĿŀ +æıIJ åIJį +äºĮ èĢħ +åıªåī© ä¸ĭ +åħ¶ä¸Ń åĮħæĭ¬ +ç¼ĸ ç¨ĭ +çł´ ç¢İ +ä¸Ń 举 +å·¥ä½ľ æĬ¥åijĬ +çŃ¾ åIJį +éħĴ ä¸ļ +çŁ¥ æĻĵ +çĥŃ å¿ĥ +éĿŀ åĩ¡ +èIJ¥ä¸ļ æī§ +èIJ¥ä¸ļæī§ çħ§ +人大 代表 +ä¸Ģ个 æĸ°çļĦ +å¨ģ æµ· +éĤ£ 人 +涨 ä»· +æ¶Ī çģŃ +éļ¾ å¿ĺ +ç¶ĵ é©Ĺ +åı£ è¢ĭ +ç³» æķ° +æĸĩ ä¸Ń +好 转 +æĸ° 鼶åĶ® +讲述 äºĨ +å¼Ģ çĽĺ +çķĻ ç»Ļ +æħ¢æħ¢ çļĦ +æĤ² 伤 +æľ¬ æľŁ +äºĨ å¤ļå°ij +è¿Ļ 让 +åIJĮ çŃī +æ¸ħ æĺİ +个 åŁİå¸Ĥ +æºĸ åĤĻ +åĩłä¹İ æĺ¯ +强 åĬĽ +ä¿ ¯ +æ°´ 稻 +åĽºå®ļ çļĦ +æł¸ åĩĨ +说 æľį +顯 示 +è¿Ļ å¥Ĺ +æĻºæħ§ åŁİå¸Ĥ +å±ĭ 顶 +ä¸į æĿ¥ +çĶŁ é²ľ +çŁ¥ æĥħ +æĬķ 身 +åijĬè¯ī æĪij们 +ä¸ī åĽĽ +ä¸ĩ ä¸Ģ +è¾Ĩ 车 +为 ä¹ĭ +åĪ° æŶåĢĻ +è¿Ļ æīįæĺ¯ +åIJį çīĮ +åºŁ æ°´ +åݻ年 åIJĮæľŁ +å¹´ éĻIJ +éģĭ åĭķ +åıĮ çľ¼ +è¦ģ ç´§ +对 çŃĸ +åľº é¦Ĩ +çĻ¾ ç§ij +è¶Ĭ éĩİ +å¯Į åIJ« +大å¤ļæķ° 人 +æľĢ å°ij +åı¬ åĶ¤ +åħ¸ èĮĥ +åĨľ æľº +æŃ£ æĸĩ +åºĶçĶ¨ äºİ +æ·± èĢķ +ä¿ Ń +ä»Ģä¹Ī ä¸ľè¥¿ +å¥Ĺ é¤IJ +å½ĵ éĢī +å·¦ æīĭ +è°ĥ çIJĨ +æĻļ é¤IJ +éļ¾ åħ³ +åĩŃ è¯ģ +çĪ± 人 +æĮĩ è´£ +è´£ ç¼ĸ +çļĦä¸Ģ 款 +éĵ ² +åįģ 个 +èĢ » +æľįåĬ¡ åķĨ +åľ° çĭ± +è¿ŀ å¿Ļ +åĽ° æĥij +çļ ĵ +ä¸į åIJĥ +çİ°åľ¨ å·²ç»ı +çĽĺ çĤ¹ +ä¸įåģľ åľ° +管çIJĨ 模å¼ı +è¿Ļ 段æŶéĹ´ +æ¤ ° +礼 åĮħ +æµģ 转 +æī« çłģ +éĽĨä¸Ń åľ¨ +æ±Ĥ åĬ© +åįĬ 个 +å¿«éĢŁ å¢ŀéķ¿ +å¾Ģ ä¸ĭ +è¯Ħ åĪĨ +å°± æĥ³ +åķĨåĬ¡ éĥ¨ +æľī éĹ®é¢ĺ +èİ· åĪ© +æ¯Ľ çĹħ +æĦŁ åºĶ +èī¯ æĢ§ +åĪĨ æѧ +åĨ ī +æĪij们 çİ°åľ¨ +è¦ģ åĬłå¼º +å·§ å¦Ļ +èŀº æĹĭ +åĪĩ æį¢ +çĭ Ħ +顺 çķħ +å°¤åħ¶ æĺ¯åľ¨ +èĬĿ 麻 +éļ¾ è¿ĩ +æĹĹ å¸ľ +å¤į åį° +å¤įåį° 件 +å¿ħ éľĢ +对å¤ĸ å¼ĢæĶ¾ +éļ¾ åıĹ +åİŁæĿ¥ æĺ¯ +ç®Ĺ äºĨ +é«ĺ å±± +离 èģĮ +çµĦ ç¹ +çµĦç¹ Ķ +å±ģ èĤ¡ +çĻ¾ 家 +éģĩ ä¸Ĭ +æĺĶ æĹ¥ +ä¸į 容 +çĽij管 éĥ¨éŨ +主 æĦı +æµģ åŁŁ +è·Į å¹ħ +èĩ³ ä¸Ĭ +åĪ« 说 +æĺ¯ æ¯Ķè¾ĥ +å®ıè§Ĥ ç»ıæµİ +å¸Ĥåľº 主ä½ĵ +污æŁĵ çī© +æķij æ²» +丰 æĶ¶ +åŃĺ æĶ¾ +åĩ Ħ +éĩij å±± +æį¢ äºĨ +ä¸ĵ 人 +éĹľ æĸ¼ +æĹ¢ è¦ģ +åĽ½ 足 +éļ ĭ +åıį åĩ» +èµ· 身 +åħĪ æĺ¯ +å¸ĮæľĽ èĥ½å¤Ł +åĪ¶ 订 +åºĹ éĿ¢ +åĸ Ģ +æķĻ ä½ł +éĻį 温 +åĬĽ æ±Ĥ +ä¸ī çĻ¾ +çī© ä»· +丢 失 +å¢Ļ ä¸Ĭ +éĥ¨ 份 +æł· æĿ¿ +ä¹ĭ æĦı +ç½ij å°ıç¼ĸ +ä¸ĸ ä¸Ĭ +è°ĥ è¯ķ +污æŁĵ éĺ²æ²» +å½± éĻ¢ +å®Įåħ¨ åı¯ä»¥ +éĢļ åħ³ +ä¹īåĬ¡ æķĻèĤ² +没æľī åĬŀæ³ķ +èĢ ¿ +å¦ ³ +æĹł æĥħ +å¾Ĺ çĽĬ +å¾ĹçĽĬ äºİ +æľŁ çĽ¼ +娱ä¹IJ åľº +çĶ² æĸ¹ +ä¸Ģ æ±½ +çĹ ° +çĸij ä¼¼ +æĸ°æµª å¾®åįļ +强 è¡Į +å½ĵ ä»ĸ +èĥ º +çĶ¨æĪ· æıIJä¾Ľ +åĮº å§Ķ +æĦ¿ æĻ¯ +æĬĺ æī£ +失 踪 +è¿« åĪĩ +åŃĹ æ¯į +åĴ ¯ +èªį èŃĺ +ä»Ģä¹Ī æĦıæĢĿ +çĽĴ åŃIJ +å½ķ éŁ³ +建设 å·¥ç¨ĭ +ä¸ļ ä½Ļ +å®ŀè·µ æ´»åĬ¨ +羣 空 +çĤ ĸ +åľ¨ è·¯ä¸Ĭ +主è¦ģ åĮħæĭ¬ +该 æĢİä¹Ī +æĢ» æľī +æĢ§ æĦŁ +æ°ij èĪª +å¼Ģ åºĹ +欺 éªĹ +çªģ åĩ» +缺 失 +æī§ ä¸ļ +åľ° éģĵ +并 æĹł +æ°ij åĬŀ +ç»Ħç»ĩ çĶŁæ´» +æĪij å¦Ī +è¨ĺ èĢħ +管 åĪ¶ +æī¾ 个 +èĹ » +çĤİ çĹĩ +äºĴ åĬ© +æµıè§Ī åĻ¨ +çݩ家 æĿ¥è¯´ +éĻįä½İ äºĨ +è£ Ķ +æĮ£ éĴ± +åķĨ æľº +æĶ¹ è£ħ +æµģ 浪 +æĶ¿ æ³ķ +èĢģ 头 +çĶŁäº§ åĴĮ +ç© Ĺ +亲 çĪ± +亲çĪ± çļĦ +å±¥ èģĮ +åŁİ éĩĮ +ç»Ĩ åĪĨ +åĬ³åĬ¨ åIJĪåIJĮ +åľ¨ æĹ¥æľ¬ +å¨ģ å°Ķ +åį« è§Ĩ +éĢ£ çµIJ +çĿĢ éĩį +æĬĺ 磨 +åĽ¾ 为 +çľ · +å·¥ åºı +æĵ ģ +æĵģ æľī +ç½ijç«Ļ åľ°åĽ¾ +çļĦä¸Ģ 大 +ç»Ħç»ĩ å®ŀæĸ½ +æĬĽ å¼ĥ +åĴĮ æĶ¯æĮģ +æ³ķ åĪĻ +浪 æ½® +çİ° æľīçļĦ +åĩł çİĩ +为 客æĪ· +åįģ ä¸ĩ +è ¹Ħ +çªģåĩº éĹ®é¢ĺ +åıĥ åĬł +éĥ½ä¼ļ æľī +çĽ ¤ +è°ģ éĥ½ +æīĭ åĬ¨ +缴 è¾¾ +çĤ¹ å¤ļ +éĺ¶ å±Ĥ +ä¸į ä½³ +éĤ£ 段 +滨 æµ· +æĺ¯ åĽ½åĨħ +æĪij å¸ĮæľĽ +åIJĽ åŃIJ +è§Ĥ éŁ³ +åģļ é¥Ń +æ±½ è»Ĭ +åħ³ ç¨İ +çľ¼åīį çļĦ +æ°´ éĿ¢ +èĢ³ æľº +追 踪 +æݨ éĢģ +éĴ± åĮħ +æģ¶ å¿ĥ +æµ· åŁŁ +å· į +å¼Ģ æĿ¥ +表 æĢģ +仪 表 +å¹³ åİŁ +åįģ å¤ļå¹´ +ä¹Ł æĹłæ³ķ +åħ¼ 顾 +è¡£ æŁľ +æł½ åŁ¹ +æĪ¿ æºIJ +设ç«ĭ äºĨ +ä¸ĩ åIJį +æķ° é¢Ŀ +è¦ģ åĿļæĮģ +åIJīæŀĹ çľģ +请 èģĶç³» +ç»ıåİĨ è¿ĩ +çļĦ æľ¬è´¨ +åħ¥ éŨ +æľ¬ æ¡Ī +çİĩ è¾¾åĪ° +åı° éĺ¶ +éĴ ŀ +æĪij èĥ½ +èݲ èĬ± +éĴ ł +ä¸Ģ äºĭ +åİŁ æľīçļĦ +æ¯ı åĢĭ +æ¯Ķäºļ 迪 +æ£ĭçīĮ 游æĪı +ä¸įä¼ļ æľī +å½Ĵ æĿ¥ +äºĶ çĻ¾ +è¿ĩ é«ĺ +鼷 è¾¾ +ä¸Ģèµ· åİ» +æķĻ 导 +å°± è¯Ĭ +å°± å¾Ī +ä¸įåIJĮ äºİ +ä¿ º +å¸ĸ åŃIJ +æĶ¿åįı å§Ķåijĺ +çĸ«æĥħ å½±åĵį +åĪĨ è£Ĥ +为ä»Ģä¹Ī ä¼ļ +äºĶ æĺŁ +å°ij åĦ¿ +æĬ¢ éĻ© +梦 è§ģ +è®°èĢħ éĩĩ访 +å±± è·¯ +æĪij 个人 +æ²Ļ 滩 +è¹ Ń +æĶ¹ è®Ĭ +æĸ°åŀĭ åĨł +æĸ°åŀĭåĨł çĬ¶ +åĮ» æĬ¤ +åĮ»æĬ¤ 人åijĺ +æµ· å°Ķ +åħ³äºİ æĪij们 +éĻ¤ å¤ĸ +åº ļ +宣 åijĬ +ä¸ī åįĥ +æ¦ ¨ +ç§ijæĬĢ 大åѦ +ä¸ĥ åħ« +顺 åºĶ +çĪ¸çĪ¸ å¦Īå¦Ī +éĢī åıĸ +åī§ çĥĪ +乡æĿij æĹħ游 +积æŀģ æݢ索 +表çİ° 为 +å¾Ī æ¸ħæ¥ļ +大 åĨĽ +æĿ¥ çĶµ +å¥Ĺ æĪ¿ +çİ° è¡Į +享 åıĹåĪ° +çľĭ çĤ¹ +åĽºå®ļ èµĦ产 +以 人为 +以人为 æľ¬ +ä¸į å®Į +éĻį 鼨 +åģļçļĦ äºĭæĥħ +并 äºİ +顽 强 +èĢ ¸ +åĺ´ å·´ +缸åħ³ ä¿¡æģ¯ +æĪij 没 +æĪĺçķ¥ æĢ§ +æĢĿ 念 +åĪĺ å¤ĩ +åĬ© æĶ» +é£İ è²Į +éĿ¢å¯¹ éĿ¢ +积æŀģ å¼Ģå±ķ +çĸĹ æķĪ +çľĭ 书 +缺 åı£ +åĽ½æ°ij ç»ıæµİ +使çĶ¨ æĿĥ +éģ¥ è¿ľ +å¡« è¡¥ +第ä¸ī 人 +åįĬ å¤ľ +æѦæ±ī å¸Ĥ +æĪij åıijçİ° +ä¼ĺæĥł æĶ¿çŃĸ +é£İ åı£ +å°± ä¸įèĥ½ +为 主è¦ģ +æµģ åĩº +å´ĩ æĭľ +并 ä¸įèĥ½ +é«ĺ ä¸ī +ä¸ĸçķĮä¸Ĭ æľĢ +æĥ³ å¿ħ +åħ¶ æīĢ +åĢĻ éĢī +åĢĻéĢī 人 +ä¸į çĪ± +åī¯ ä½ľçĶ¨ +人æ°ij æĹ¥æĬ¥ +æĪij ä¸įæĺ¯ +å®ŀ çī© +çĶµ åİĤ +ä¹Ł ç®Ĺæĺ¯ +æľī éĹľ +æľī èĥ½åĬĽ +æĮĤ åľ¨ +çľ¼ ä¸ĭ +约 ç¿° +å°ı åѦçĶŁ +èµ· åĪ°äºĨ +å·¥ 夫 +åIJĮ å¿ĥ +åĿ¦ è¨Ģ +çł Į +åıijæĮ¥ äºĨ +èģĮä¸ļ éģĵå¾· +è¿ĻäºĽ å¹´ +念 头 +èĢģ é¼ł +åħ¨ èµĦ +åħ¨èµĦ åŃIJ +ä¸Ģ åij³ +å¤ļ ä¸ĩåħĥ +æł¼ æľĥ +éķ¿ éĢĶ +带 èµ° +èĭ± 寸 +æĸĩ ä½ĵ +对 ä»ĸ们 +åĵŃ äºĨ +å¡« æĬ¥ +çīĪæĿĥ 声æĺİ +çĶµ 线 +è´Ńçī© ä¸Ńå¿ĥ +饱 满 +ä½İ 头 +强 è¿« +ä¿Ŀ æ´ģ +欧 åĨł +缸 è¿ŀ +认 è´Ń +çģ« æĺŁ +é«ĺ å°Ķ +é«ĺå°Ķ 夫 +èij« èĬ¦ +æłĩ 注 +çļĦ çIJĨæĥ³ +æł¸ éħ¸ +æł¸éħ¸ æ£Ģæµĭ +åĬ ī +ä¸ĢèĪ¬ æĺ¯ +æĢĿ ç´¢ +轨 迹 +çĥŃ å¸¦ +éĻ £ +åĩĨç¡® æĢ§ +æĪ´ çĿĢ +åľ¨ çĶŁæ´»ä¸Ń +æīĢ èĥ½ +æľ¯ åIJİ +带 ä½ł +ç¥ ł +æ®ĭ éħ· +ä¹Ł åıªæĺ¯ +çĶ³ è´Ń +举åĬŀ äºĨ +æľī æĦıä¹ī +æĹº 缼 +åľ¨ 網 +åľ¨ç¶² è·¯ä¸Ĭ +å¾Ī大 ç¨ĭ度 +管 è¾ĸ +çĸ«æĥħ æľŁéĹ´ +触 æij¸ +éĺ¶æ®µ æĢ§ +ä¼ļ è§īå¾Ĺ +çļĦ çĶ»éĿ¢ +æİ¥åıĹ äºĨ +表达 äºĨ +éĤĵ å°ı +éĤĵå°ı å¹³ +åħļ é£İ +åħļé£İ å»īæĶ¿ +åķĨ åѦéĻ¢ +åħij æį¢ +é£Łåĵģ èį¯åĵģ +éĿŀ常 好çļĦ +çľ ¯ +纳 ç±³ +åĬ¨ æijĩ +åĽŀ éģ¿ +çľĭ èijĹ +款 项 +åħ« å¹´ +åģļ 个 +æĸĩ æ¡£ +éĩijèŀį ç§ijæĬĢ +åħ¶ä¸Ń æľī +äºĨä¸Ģ ç³»åĪĹ +æĹĹèĪ° åºĹ +称 èµŀ +éĽ¢ éĸĭ +åĪ¶ åĨ· +家 éŨåı£ +åįģ å¤ļ +ä¼´ ä¾£ +çľĭ çĹħ +æĭī çĿĢ +æī Ĵ +çĸ² æĥ« +å°ijæķ° æ°ijæĹı +åĽ¾ å½¢ +è½ § +å¢ŀ éĩı +饲 åħ» +çģ« å±± +æ¯ı 个æľĪ +ä½ľä¸º ä¸ĢåIJį +è½´ æī¿ +æĸĩ 书 +ç¼ ķ +åħ·ä½ĵ æĥħåĨµ +çĹĽ çĤ¹ +缴 éĶĢ +å¡ Ĭ +ä¹Ł æľĥ +çĥŃ æ½® +å¹³ æ°ij +æ¼ĶåĶ± ä¼ļ +æķĻ çłĶ +éĢĥ éģ¿ +ä¸Ģ è´¯ +å°± è¶Ĭ +å®ŀ å®ŀåľ¨ +å®ŀå®ŀåľ¨ åľ¨ +ä¹łè¿ijå¹³ æĢ» +æº º +å¿ĥ åºķ +éķ¿ å¾ģ +媽 媽 +第ä¸ī 次 +åĩº æ¼Ķ +çĭĢ æ³ģ +å°Ķ æĸ¯ +代çIJĨ åķĨ +çĨ ı +çļĦ 对象 +çĶµ éĩı +è¡Į åĪĹ +åĽ½ 人 +è·ij äºĨ +åįĶ åĬ© +èIJ¥ è¿IJ +å¸Ī åħĦ +æ¦ ® +æĥ³ åĥı +æĢ§ 强 +ç§ijåѦ çłĶ究 +延 å®ī +ä¸¥æł¼ èIJ½å®ŀ +é¢Ĩ ä¼ļ +缸 å·® +è·¯ 人 +çĶ « +æľī ä»·åĢ¼ +æľīä»·åĢ¼ çļĦ +ç¾İ åĽ¢ +æ°ij主 çĶŁæ´» +æĪij æīį +ç¾İåĽ½ 人 +æ°Ķ åij³ +åıį å°Ħ +çļĦ åĨ³å¿ĥ +大 è±Ĩ +交 代 +è¿Ľ åĩº +åıį æĬĹ +æĮĩ çļĦæĺ¯ +ä»· ä½į +è¿Ľ é©» +ä¸Ĭ çĻ¾ +ä½į åĪĹ +ä¸ŃåĽ½ ä¼ģä¸ļ +çļĦ好 å¤Ħ +主 ç¼ĸ +æ±½ æ²¹ +ä½Ĩ æĪij们 +æĢİä¹Ī çľĭ +é»Ħ å±± +å¤ļ åªĴä½ĵ +åIJİ åį« +èİ·å¾Ĺ æĽ´å¤ļ +åĬ¡ å¿ħ +为 å¥ijæľº +é¦ĸ 饰 +ä¸ĩ åįļ +è¶ĬæĿ¥è¶Ĭ 大 +ä¸ĵ项 è¡ĮåĬ¨ +å¥ĭ è¿Ľ +ä»į çĦ¶æĺ¯ +è´¨ æĦŁ +å¦Ĥæŀľ ä¸įæĺ¯ +ç«Ļ èµ·æĿ¥ +ä¹¾ éļĨ +åı¯æĢķ çļĦ +å¯Į è´µ +æ¸ħ ç®Ĺ +åIJij ä¸ĭ +åĢ ļ +çļĦ çŃĶæ¡Ī +èĪ¹ ä¸Ĭ +çļĦ羣å®ŀ æĢ§ +çŃī åĬŁèĥ½ +åĸľ åī§ +å¨ģ åĬĽ +æĸ° é¢ĸ +æł¸ çĶµ +æĬ¥ éĶĢ +æķħ 乡 +ä¼´ éļı +éŀ Ń +å¦Ĭ å¨ł +åĪĨ åĮĸ +æľī å¾Ī大 +æĢİä¹Ī 说 +æĻĤ 代 +产 åĩº +ä»ĭç»į 说 +å¤ĦçIJĨ åĻ¨ +èĨ¨ èĥĢ +åī¯ å¸Ĥéķ¿ +çļĦ 妻åŃIJ +æł· åĵģ +åIJĮæ¯Ķ ä¸ĭéĻį +åħĥ å·¦åı³ +çĶ¨ èĩªå·±çļĦ +é«ĺ éĽĦ +æĺ¥ æĻļ +ä¹Ł æľīå¾Īå¤ļ +çľ¼ çIJĥ +æķ£ æŃ¥ +ä»ĸ们 éĥ½ +第ä¸Ģ 家 +åĬŀ 好 +å®ī éĺ² +ä¸Ģ ä¸ĩ +åľ¨ éĩĮéĿ¢ +éŁ³ é¢ij +åı£ åı· +ä¸Ģ è¶Ł +ç¦ı çī¹ +é³ ŀ +æĥĬ èī³ +æĸ° å¨ĺ +绿èī² åıijå±ķ +ä¸Ń å¼ı +ä¹Ł åıªæľī +çİ° 身 +åı¯ ä¾Ľ +æ¯ı ä¸Ģ个人 +第ä¸ī èĢħ +åľ° å½¢ +éĴ¢ ç»ĵæŀĦ +çĽijçĿ£ æ£ĢæŁ¥ +åı« æĪij +èĩ´ æķ¬ +æ´Ĺ æīĭ +ä¸ĭ è°ĥ +康 çĨĻ +æĪIJ交 éĩı +ä¹Ł æĪIJ为 +åħī æ»ij +å®Įæķ´ æĢ§ +çģ ¼ +網 éłģ +éķ¿ 寿 +éģ© çĶ¨ +çļĦä¸Ģ 项 +çŀ© 缮 +æĬĬ èĩªå·±çļĦ +éĵ¶è¡Į åį¡ +å°± å¿ħé¡» +ç¾İ çĻ½ +éŀį å±± +æľ¬ é¢Ĩ +ä¸Ģ ç¢Ĺ +æīĵ æ³ķ +æĤ¨ 好 +对 åŃ©åŃIJ +æĬ¥éģĵ 称 +ä¼ł åĩº +大 èĩ£ +ç¬ ĭ +çĽ ı +é¾ ļ +缴 线 +æĻº åºĵ +秣 车 +é£İ åij³ +çľĭ ä¸Ģä¸ĭ +æݨ éĶĢ +éĥ¨ éĥ¨éķ¿ +è´¨éĩı åĴĮ +åĪĬ çĻ» +å·¥ä¸ļ åĮĸ +çİĩ 为 +鼶 件 +硬 åĮĸ +ä¸Ĭ åįĥ +ç»ıéªĮ åĢ¼ +å¹³ è¡Į +声 éģĵ +æľįåĬ¡ è´¨éĩı +çĶŁ çĶ¢ +æľĢ 容æĺĵ +ä¸Ģ æŀļ +å¹´ æĬ¥ +åħ¬ ç½ij +åħ¬ç½ij å®ī +åħ¬ç½ijå®ī å¤ĩ +çļĦ èĥ½éĩı +å®ŀéĻħ è¡ĮåĬ¨ +è¦ģ ä¸įè¦ģ +æĹ¥æľ¬ 人 +èĢ¶ 稣 +ç¼ĸ åī§ +æ¶ © +åį° å°¼ +ä¸Ĭä¸ĭ 游 +åĩł åı¥ +ä¸Ń éĵģ +ç°¡ åĸ® +èĩª 带 +çĶŁ äºİ +ä¸Ģ åı£æ°Ķ +åĭ¤ å¥ĭ +éĻį ä»· +å±ķçİ° äºĨ +å¸ĥ æĭī +ä¼ļ éĢīæĭ© +çļĦ ç»ıåħ¸ +好 æľĭåıĭ +车 éģĵ +æķ´ åĢĭ +åľ ĵ +éķ¿æľŁ 以æĿ¥ +æĬķ å½± +çļĩ åĨł +è¿ĩ 大 +åijĬè¯ī ä»ĸ +ä¼ģä¸ļ æıIJä¾Ľ +æĬ½ 象 +éĢĤ 度 +çļĦ 女åŃ© +èµ· ä¼ı +çļĦ åĬŁæķĪ +ä¸ĵ项 æķ´æ²» +åı¯ éĢļè¿ĩ +ä¸įåIJĮ ç¨ĭ度 +å¼Ĥ è®® +åĩĢ èµĦ产 +åij Ĺ +ä»Ģä¹Ī åij¢ +å·¡ éĢ» +è¸ı ä¸Ĭ +ä½Ĩ å®ĥ +ç²¾ 度 +管 å±Ģ +第ä¸Ģ åIJį +åĨħ åŃĺ +æijĨ åľ¨ +åī© ä¸ĭ +主ä½ĵ 责任 +çĤ¹ åįĬ +以 èĩ³äºİ +åħ»èĢģ ä¿ĿéĻ© +æĦŁåıĹ åĪ°äºĨ +çŁ¥åIJį çļĦ +å¯Į 豪 +妥 åĸĦ +åŃĻ åŃIJ +éĵ Ĥ +说 èĩªå·± +让 æĤ¨ +æķ° æݧ +çļĦçľ¼ åħī +注 éĶĢ +çļĦ çģµéŃĤ +è¿ĺ ä¸įéĶĻ +éĹ® ä»ĸ +èĩªä¸» çłĶåıij +èĵ ĭ +ç´« èī² +åĽ½å®¶ å®īåħ¨ +è¾½å®ģ çľģ +ä¹Ł æ¯Ķè¾ĥ +ç¾İ èĤ¡ +ä¸įç¡®å®ļ æĢ§ +å¿ĥ 头 +æĪ ³ +级 åĪ«çļĦ +论 è¿° +çļĦ åĽŀçŃĶ +ä¿Ŀè¯ģ éĩij +çŃī è¡Įä¸ļ +幸ç¦ı æĦŁ +æѧ è§Ĩ +æľº 票 +æ´¾ 人 +èĩ´ åij½ +åĺ´ è§Ĵ +æĸ°éĹ» ä¸Ńå¿ĥ +æĶ¾å¼ĥ äºĨ +å®ľ å±ħ +åĨĻ ä¸ĭ +éĹ® çŃĶ +è¿ĻéĩĮ æĺ¯ +å¤ļ åľ° +åĮºåŁŁ åĨħ +åīµ æĸ° +çľĭ ä»ĸ +æī§æ³ķ 人åijĺ +åĬ¨ æľº +éŁ³ åĵį +çļĦ åij½è¿IJ +顶 éĥ¨ +åĵ Ł +éĥ½ æľĥ +æīĵéĢł æĪIJ +æĦı åĽ¾ +çļ ĸ +åĢĴ åħ¥ +å·´ èIJ¨ +åĬ© åѦ +å¤į åı¤ +åIJ¯ çĶ¨ +åĽ½éĻħ å¸Ĥåľº +åĤ¨ èĥ½ +é»ijé¾Ļæ±Ł çľģ +ä¹ĺ 车 +è¿IJåĬ¨ ä¼ļ +ä¿Ŀ åĪ© +çŁ³ æĿIJ +çµ ® +çĤĴ ä½ľ +çļĦ ä¿¡ä»» +å°± æĪIJäºĨ +åı¯ è§Ĥ +çļĩ ä¸Ĭ +è¿Ļ åĩłå¤© +ä¸Ģ éĶ® +åĨ· åĨ» +ä¿Ŀ åį« +æł¸ æ¡ĥ +åIJĪä½ľ åħ³ç³» +éĢģ åĩº +æĹĹ ä¸ĭçļĦ +åľ¨ ä¹İ +为 广大 +åįĪ é¤IJ +ä¸ĵ 访 +æĪĸ å°Ĩ +éĿĴå²Ľ å¸Ĥ +å¥Ķ è·ij +æĹ¥ æĬ¥éģĵ +å¥ij åIJĪ +æĸ° æĺ¥ +ä¸į å°ıå¿ĥ +两 ä¸ī +æĦıæĢĿ æĺ¯ +åĨ· èĹı +çļĦ çĹĩçĬ¶ +æĢ§ åij½ +è¶ħ æłĩ +å¯Ĩ 碼 +ç§ijæĬĢ èĤ¡ä»½ +äºĨä¸Ģ æī¹ +çĿ£ å¯Ł +åªĴ ä»ĭ +å°Ħ æīĭ +ä¿® åħ» +çīĩ åĪ» +éĢĤåIJĪ èĩªå·± +åıªè¦ģ æĺ¯ +åIJĥ è¿ĩ +éĩij éĵ¶ +缴 å±ŀ +åѦ éĹ® +åİĭ åĪ¶ +çªĹ å¤ĸ +æĶ¶ åĪ°äºĨ +åħ¨åĽ½ 人大 +ä½Ĩæĺ¯ å¯¹äºİ +åľ¨ æķ´ä¸ª +çļĦ èĥĮåIJİ +åĩıå°ij äºĨ +åıį èħIJ +åıįèħIJ åĢ¡ +åıįèħIJåĢ¡ å»ī +æĹ · +åĪĨ æľŁ +åľ¨ æ·±åľ³ +æīĵ çĿĢ +æī« ä¸Ģ +æī«ä¸Ģ æī« +æĶ¿åºľ éĥ¨éŨ +æİ¥ è¿ŀ +å±ŀäºİ èĩªå·± +åŃIJ å¼¹ +åIJĮæł· æĺ¯ +æĢ» åħ± +车 ä¼ģ +æ¢ ĵ +åħ¬ é¡· +åıij 声 +éĴ Ľ +èµ°åĬ¿ åĽ¾ +主 èIJ¥ +åĸ Ķ +æķ°æį® åĪĨæŀIJ +ä¸į è¿ľ +æľī åIJį +æľīåIJį çļĦ +åģ¿ è¿ĺ +å¾Ī ä½İ +è®ĵ 人 +èĿ ī +é«ĺ è´µ +å°ij 许 +æ° Ł +å¹ ¢ +亲 æĥħ +è¿Ļ件 äºĭæĥħ +çĶ¨ é¤IJ +缸åħ³ æĸ°éĹ» +å°± åºĶ该 +ç»Ī çĤ¹ +æĺ¯ å¤ļå°ij +çĻ» åľº +è¯ķ 管 +è¯ķ管 å©´åĦ¿ +åģļ 大 +åģļ大 åģļ强 +çļĦ ä¾ĭåŃIJ +åħ« 个 +æĺİ æĹ¥ +çĤ ³ +èµ° åİ» +éģ º +å¢ © +ä½ĵä¼ļ åĪ° +åĴ ı +ä¸ĭ è¾¾ +å¤į åıij +追 éĢIJ +æīĵ åĵį +çļĦ éļ±ç§ģæ¬Ĭ +åħ·æľī ä¸Ģå®ļ +è¿Ļä¹Ī å¤ļå¹´ +æłij æŀĹ +æľĢ éķ¿ +åIJĮ èĥŀ +åħī æ³½ +åŁŁ åIJį +æĮĩ åIJij +åıĹ害 èĢħ +æłij èĦĤ +æľīå¤ļ 大 +大 éĿ¢ç§¯ +æĹł ç¼Ŀ +æĶ¹ æŃ£ +æĽ´å¤ļ çļĦæĺ¯ +æľŁ æľ« +æŃ ¼ +ä¹ī ä¹Į +éĤ£ ä½ł +çļĦ 第ä¸Ģ个 +èĮ µ +å° § +èį « +ä¸įä»ħ åı¯ä»¥ +æ¶Į çİ° +æĢ» éĿ¢ç§¯ +æĸ°éĹ» åıijå¸ĥ +æ°ij çĶ¨ +å°± 读 +æīĵ è´¥ +å¤ĸ è¯Ń +æĪij们 ä¸Ģèµ· +é¢Ħ å®ļ +çĥ¹ 饪 +æľĢ 主è¦ģ +æľĢ主è¦ģ çļĦ +çīĮ çħ§ +åĽł åħ¶ +ä½İ ä¸ĭ +ä¼ļ åIJĮ +è§ģ 解 +éĹ´ éļĶ +æķĻ ç¨ĭ +å° ī +å¸Ĥ ä¸Ńå¿ĥ +åħ³éĶ® æĺ¯ +æµ· åįĹçľģ +çī¹åĪ« æĺ¯åľ¨ +ä¸ŃåĽ½ 大éĻĨ +åħħ足 çļĦ +æĹ¢ èĥ½ +åĤ³ çµ± +çijľ ä¼½ +åħ¥ åĽ´ +æħ¢æħ¢ åľ° +æĬ¥ éħ¬ +æī¹ å¤į +å·¥ä¸ļ åĽŃåĮº +ä¸İ åıijå±ķ +èĥ¸ éĥ¨ +åľ¨ ç½ij绾 +åľ¨ç½ij绾 ä¸Ĭ +交 è°Ī +æĽ´ æĶ¹ +åįłæľī çİĩ +ä¸Ŀ绸 ä¹ĭè·¯ +è¡ Ľ +çłĶ åĪ¤ +åĪ ª +åĪª éĻ¤ +è¿Ļ åıª +çļĦ æ°Ķæģ¯ +åĬł å·ŀ +éĴ § +çIJĨäºĭ éķ¿ +ä¸ĸ 家 +æµģè¡Į çļĦ +å¾Ī æľīåı¯èĥ½ +们 éĥ½ +ç»ıèIJ¥ 模å¼ı +è¡Įä¸ļ ä¸Ń +éĢļçŁ¥ 书 +åij½ é¢ĺ +æľ¬ 網ç«Ļ +æ²Ļ çī¹ +åıij åħī +é«ĺ ä»· +å·² çĦ¶ +åıĮ åįģä¸Ģ +ä¸Ĭ è¯ī +ç¿ħ èĨĢ +è¿Ļä¸Ģ å¹´ +大ä¼ļ ä¸Ĭ +éĩ ī +å®Įåħ¨ æĺ¯ +å¾Ĺ 太 +ä¸ĢèĪ¬ 人 +è¿ĺ ç®Ĺ +æĬĺ åıł +æĬķ æľº +çĤ¹ çĩĥ +çİ°éĩij æµģ +åħĶ åŃIJ +ç½ij æł¼ +æİ¥ è¿ĩ +ä¾Ľ è´§ +éĺ´ å½± +åİŁ åħĪ +æį £ +å·¦ 侧 +åħĭ æĭī +æīĵ åį¡ +ç§ij æ¯Ķ +æ±ĩ éĽĨ +åľ°çIJĨ ä½įç½® +è¯Ħ å§Ķ +ç»ĵåIJĪ èµ·æĿ¥ +è¿Ľåħ¥ åĪ° +åı¯ è¡Į +åı¯è¡Į æĢ§ +让 å®ĥ +åĪ¶åº¦ æĶ¹éĿ© +çĶĺèĤĥ çľģ +åĵ Ĺ +åģı åģı +è¡£ çī© +ç¥Ŀ è´º +æºIJ èĩª +并ä¸į 代表 +åĽ½ 度 +好 åĿı +æĿ ĸ +æĿŃ å·ŀå¸Ĥ +湿 度 +é² ¸ +åįļ 彩 +æ³° å±± +æĿij èIJ½ +æĸ° èģŀ +èĤ ĭ +åı¤èĢģ çļĦ +çļĦ ç§ĺå¯Ĩ +ä¸Ģ个 éĹ®é¢ĺ +éģı åĪ¶ +åįĥ 亿 +è¿ĩ 硬 +å°Ħ åĩ» +èĩªçĦ¶ æĺ¯ +产 åĮº +çĤ¹ çĤ¹å¤´ +åı¯ä»¥ 帮åĬ© +说 å®ŀ +说å®ŀ è¯Ŀ +æĪij åıªæĺ¯ +ä¹ĭ ä½Ļ +åIJĮæŶ ä¹Łæĺ¯ +ä¸ŃåĽ½ éĺŁ +建æĪIJ åIJİ +ä¹IJ è§Ĩ +åij¨ å²ģ +èį¯ åºĹ +éĩij åįİ +严éĩį å½±åĵį +è´¨ åľ° +æĹħ éģĬ +åħµ åĻ¨ +æķĻèĤ² æķĻåѦ +离 åİ» +åIJĦå¼ı åIJĦæł· +ä»ĭ ç´ +ä»ĭç´ ¹ +å¼Ģ 头 +å°Ĩ èĩªå·±çļĦ +åIJ¬ åĬĽ +ä¿¡æģ¯ ç³»ç»Ł +ä»İ æł¹æľ¬ +ä»İæł¹æľ¬ ä¸Ĭ +æİĮ 声 +欢 åĸľ +å±ķ åĮº +åķ ¸ +太å¤ļ äºĨ +éĹ² ç½® +èĥ¡ èIJĿåįľ +å§Ķ å®£ä¼ł +å§Ķå®£ä¼ł éĥ¨ +åįĹ éĺ³ +å·ŀ åĮº +ä¸İ æŶ +ä¸İæŶ 俱 +ä¸İæĹ¶ä¿± è¿Ľ +å«Įçĸij 人 +èī¯ å¿ĥ +头 顶 +è´¢ æĬ¥ +ä½Ľ æ³ķ +å¾ µ +åİŁ 件 +åĭ ŀ +çĶ· 篮 +å¤ĸåĽ½ 人 +è¿Ŀ 纪 +æī¾ äºĨ +æįķ æįī +缸 è¯Ĩ +æIJľ éĽĨ +çļĦ ä¼Łå¤§ +ä¸ī ç»´ +å°±è¡Į äºĨ +çĭIJ æľĪ +çĭIJæľĪ å±± +å¸ĮæľĽ éĢļè¿ĩ +èĢĮ 对äºİ +éĿ¢ å°į +åĨĽ åĽ¢ +è¡Ĺ åĮº +æĤ¬ æĮĤ +便 ç§ĺ +æľīä¸Ģ çĤ¹ +ä¼ļè®® ä¸Ĭ +ä¸ĭ æīĭ +廣 åijĬ +äºĶ è¡Į +çŃī åĢĻ +紧紧 åĽ´ç»ķ +æĭ¿ äºĨ +æ¡Į éĿ¢ +ç¥ŀ æĥħ +éĽĦ åİļ +çŀ ³ +楼 ä¸ĭ +å½ ª +äºĭ åıij +åĨį è§ģ +é¤ ĺ +é¢Ħ åĶ® +åİ» çľĭçľĭ +æĪij们 åºĶ该 +ä¸ī 家 +æµ Ĭ +ä¹IJ éĺŁ +çľĭ ä¸įè§ģ +èĦij åŃIJ +æĮģ æľīçļĦ +çĻ½ èıľ +éĹª çĥģ +åĸĿ æ°´ +æݧåĪ¶ ç³»ç»Ł +ä¸ĵ åĮº +æľĿ å»· +æĪij å¿ĥéĩĮ +å±ķ åİħ +èľĺ èĽĽ +åĨ» ç»ĵ +ç² ª +åº IJ +åIJij 社ä¼ļ +åĨ³çŃĸ éĥ¨ç½² +çŁŃ æľŁåĨħ +æĸ° ä¸ļæĢģ +æľ Ķ +æŶ æĬ¥ +使 ä¹ĭ +åĽł åŃIJ +åıĤä¸İ èĢħ +çļĦ 年轻人 +æīĭ 表 +å°ģ éĶģ +为ä»Ģä¹Ī ä¸į +åIJ¸ çĥŁ +æ¯Ĵ ç´ł +åĪij æ³ķ +磫 æŃ£ +身 æĹģ +åİŁ è°ħ +çĽij æĬ¤ +æѤ å¤Ħ +éļ¨ æĻĤ +æŀľ å®ŀ +åĮ»çĸĹ æľįåĬ¡ +ä¸į åIJĪçIJĨ +æIJŀ 好 +çļĦ èĦļæŃ¥ +å¤ĸ å¥Ĺ +ç¶ĵ éģİ +æĶ¾ ç¼ĵ +åģľ çķĻ +æĺŁ çIJĥ +çļĦä¸Ģ éĿ¢ +åĩł ä½ķ +è½® åĽŀ +æ¯Ľ å·¾ +ä¿® çIJĨ +ä¸įçŁ¥ ä¸į +ä¸įçŁ¥ä¸į è§ī +æķ´ 个人 +æ¯ģ çģŃ +åı° å·ŀ +使çĶ¨ 寿åij½ +é»ij çĻ½ +æij¸ ç´¢ +é¼ł æłĩ +éĿ© æĸ° +éº µ +ä¸ĵéŨ 为 +å¾Īå¤ļ æľĭåıĭ +å·¥ä½ľ ç»Ħ +åIJĪ å½± +çĤº ä»Ģ麼 +æŀģ 度 +çļĦ è¿ĽæŃ¥ +å½ĵ ä¹ĭ +å½ĵä¹ĭ æĹł +å½ĵä¹ĭæĹł æĦ§ +è´´ è¿ij +å°º 度 +åľ¨ çİ°åľº +éĻį 临 +åħ»èĢģ éĩij +ç£ ķ +åı¯ä»¥ 使 +管çIJĨ æ°´å¹³ +æľ¬æĬ¥ è®°èĢħ +æ³ķ 令 +åį¡ 车 +举 æµ· +å¤ļ éĩį +åħ¶ éĹ´ +ç´ Ļ +éĩį大 é¡¹çĽ® +æ±Ĺ æ°´ +ç»Ħ å§Ķä¼ļ +ä¿¡æģ¯ åħ¬å¼Ģ +ä¸į论 æĺ¯ +ä¸Ģ åIJ¬ +èĴ¸ æ±½ +æıŃ ç§ĺ +è¶ħ éģİ +触 åıij +å© ¦ +åħ³èģĶ 交æĺĵ +å°± ç»Ļ大家 +好 ä¹ħ +åĢŁ è´· +游æĪı è§Ĵèī² +å¼ĢåIJ¯ äºĨ +æİ ł +åħļçļĦ åįģä¹Ŀ +ä¸ĭ 鼨 +çŁŃ æŶéĹ´åĨħ +å¯ ħ +导 åħ¥ +å·¥ä½ľ ç»ıéªĮ +ä¹Ł åıªèĥ½ +鼷 éľĨ +è·Ł è¿Ľ +åį¡ éĢļ +é¢ĩ æľī +æľº ä½ĵ +æĪĺ士 èģĮä¸ļ +女 主 +ä½ĵåĪ¶ æľºåĪ¶ +足 åįı +èĪĴéĢĤ çļĦ +åĢŁ åı£ +æī¹ åĪ¤ +æķ° åĢ¼ +è« ¾ +éĺ¿æĭī 伯 +åĺ İ +æħ ¶ +è¾¾ 人 +å¼Ģ æ°´ +大 鼨 +温 室 +ä½İ è¿· +ä»į æŧ +éªĹ åŃIJ +亲 å±ŀ +çIJĨ æĻº +æľ¬ åŁºéĩij +å¨ ħ +åĨĻåŃĹ æ¥¼ +å¢Ļ å£ģ +å® µ +èĻ½ çĦ¶æĺ¯ +顺 çĿĢ +åħ« åį¦ +åķĨ çĶ¨ +ä¸į 失 +è¿· èĮ« +顺 便 +æļij æľŁ +欺 è´Ł +é¢ij é¢ij +该 æł¡ +æĸĻ çIJĨ +æ·± æĥħ +åīį éĶĭ +ä¿Ŀ èŃī +èģĮä¸ļ çĶŁæ¶¯ +åħ¬ å¼Ģåıij +åħ¬å¼Ģåıij è¡Į +åħ¥ æĪ· +éł ĵ +åĢ¾ åIJ¬ +éŃ ģ +æĦī æĤ¦ +åĽŀ åIJĪ +åħ¨åĬĽ 以 +åħ¨åĬĽä»¥ èµ´ +åĥ¹ åĢ¼ +èĥ½åĬĽ 强 +ç»ı å¼Ģ +ç»ıå¼Ģ åĮº +è¿ľ æĸ¹ +çļĦ éģĵçIJĨ +缴 åįĩ +缴åįĩ æľº +为主é¢ĺ çļĦ +ç»Ļ æĤ¨ +è¿ĺ æĥ³ +æ¯Ķ æĪij +åĨľ çī§ +æµ· åºķ +çŃ¾è®¢ äºĨ +对äºİ æĪij们 +æŶ 许 +éĶ® çĽĺ +å®ŀéĻħ æݧåĪ¶ +çļĦ æ¨¡æł· +åıįæĺł äºĨ +代 åĬŀ +åĮ» çĶ¨ +éĽĨ ç»ĵ +åıijå±ķ åīįæĻ¯ +æĮĩ çĿĢ +åįİ åĮĹ +è¿Ļ åĩłä¸ª +åIJį æ°Ķ +åĤį æĻļ +èĩª åıij +æ³¢ åħ° +大åĬĽ æİ¨è¿Ľ +èĩª 称 +èįĨ å·ŀ +æIJį 害 +äºĨä¸Ģ åı¥ +æľĢåĪĿ çļĦ +éĩijèŀį åį±æľº +æĢĢ 念 +è¡Į åĭķ +女 æİĴ +ä¸į 解 +ä¼ł éĶĢ +转载 请 +饰 åĵģ +åıª 为 +ä¸İ ä¼Ĺ +ä¸İä¼Ĺ ä¸įåIJĮ +èĥ½ èĢĹ +èı© æıIJ +è¿ij 两年 +è¿Ķ 乡 +马ä¸Ĭ å°± +äºĮ çŃīå¥ĸ +æ°´ 管 +æ³ķ åѦ +çģŃ çģ« +大 å§IJ +åij¨ 转 +æľī æľŁ +æľīæľŁ å¾Ĵ +æľīæľŁå¾Ĵ åĪij +å°į æĸ¹ +ç¥ŀ èī² +æ²¹ èĦĤ +ä¸ī çĤ¹ +ä¸į åĪ©äºİ +äºĭä¸ļ éĥ¨ +å°± è·Ł +å¼Ģ æĶ¯ +å°ı 女åŃ© +åħ±åIJĮ åĬªåĬĽ +çĶļèĩ³ è¿ĺ +è¿Ļ åIJį +è¿Ļ ç¬Ķ +çݯ åį« +æľī ç§į +è§Ĩ åĬĽ +çĨŁ çŁ¥ +åħ¬ç§¯ éĩij +æ¶Īéĺ² å®īåħ¨ +é¢ĩ 为 +大 èħ¿ +éĿ ¶ +çī¹ æķĪ +æľįåĬ¡ åĮº +å¼Ģ åĩº +深度 èŀįåIJĪ +æĹł 忧 +æŁ¥ éĺħ +ç»Ī ç»ĵ +ä¿Ŀ ç¨İ +è¨İ è«ĸ +å½ĵ åģļ +è·³ èĪŀ +å¯ § +女 çİĭ +è®°èĢħ åľ¨ +åħ¨ 产ä¸ļéĵ¾ +è´¯ éĢļ +åħ´ ä¸ļ +éĻį åĪ° +å°ģ éĿ¢ +åħ¨éĿ¢ æİ¨è¿Ľ +奶 èĮ¶ +éĢī åĿĢ +äºĨä¸Ģ åľº +åIJĮ ä¼´ +è®® 论 +æIJ ĵ +诸 èijĽ +诸èijĽ 亮 +å¹² åĺĽ +æµģ æĦŁ +ä¸ĵä¸ļ çŁ¥è¯Ĩ +çĶµ ç«Ļ +åĩı å¼± +åĩº åħ¥ +åIJĦ çľģ +éĿŀ常 é«ĺ +åľ° 毯 +åıij æĸĩ +çĦ ī +çĥ§ çĥ¤ +å£ģ 纸 +æģ¶ åĮĸ +èĬ ¸ +èĥĸ åŃIJ +çĩ Ĵ +çľģ éĴ± +çĻ¾ 强 +çIJĨå·¥ 大åѦ +éĴ¢ æĿIJ +åĽ½æľī èµĦ产 +æĪĺ æľº +æ³Ħ éľ² +åIJİéĿ¢ çļĦ +æ°´ èµĦæºIJ +æ¢ħ èĬ± +åĨĻ çĿĢ +ä¹ĭ 声 +æĹł åı¯ +æĺİ æľĿ +ç«ĭæĸ¹ ç±³ +ç· £ +æĶ¾ è¿ĩ +ç¦ı çĶ° +å¾Ĺ ä½ı +åıĹ ä¼Ĺ +ä¸Ń 级 +çĹħ åıĺ +ä¸Ģ çŀ¬éĹ´ +æĿĥ éĩį +人æĢ§ åĮĸ +åĮ»çĸĹ åį«çĶŁ +ä¸įåĪ° ä½į +æĻºèĥ½ 家å±ħ +饮 çĶ¨ +æ¼Ķ åıĺ +é«ĺ ç´łè´¨ +ä¹Ļ æĸ¹ +åģľ çķĻåľ¨ +èİ· æī¹ +ç©¿ æ¢Ń +客 åľº +æĮ½ åĽŀ +京 åŁİ +çĶŁåij½ åĬĽ +實 éļĽ +çĩ Ī +åĨį çİ° +çİ°å®ŀ ä¸Ń +æľī ä¿¡å¿ĥ +çĸı éĢļ +åĺ´ åĶĩ +鼷 éĶĭ +èıľ åįķ +éħ ¯ +è¶ħ é«ĺ +å¾Ī é«ĺåħ´ +çĶŁ æ®ĸ +éĢł ä»· +误 åĮº +æĨ ĭ +好 æ¶Īæģ¯ +å´ Ń +以 èĩ´ +å¼Ģ çİ©ç¬ij +çĽij è§Ĩ +å·¡ å¯Ł +å¾· å·ŀ +æĹ© æĹ© +éĹª çĶµ +æĪª åĽ¾ +åı¯ä»¥ æł¹æį® +æīĭ èīº +æİ¥ 轨 +ç§į æĹı +æĢĢ éĩĮ +åİ» åĮ»éĻ¢ +ä¸Ģ äºĮ +å¼Ģ éĺĶ +åĩı éĢŁ +ä½Ĩ ä»İ +éĢĻ ä¸Ģ +åĩı åħį +主é¢ĺ æķĻèĤ² +å¼Ģå·¥ 建设 +è¹ ¦ +æľĪ 饼 +ä¸ĭ æ²ī +å°Ĭ 严 +éĻ ĩ +å®ŀ æľ¨ +å»ł åķĨ +声 称 +èĢĥ åľº +å¸ĥ é²ģ +èĩª æĿ¥ +èĩªæĿ¥ æ°´ +éĴ ¾ +å¹´ 以ä¸Ĭ +大 åıĶ +ä»ĸ å·²ç»ı +åħ¨ æĿij +èģĶç³» çĶµè¯Ŀ +为 导åIJij +åĪ¤ å¤Ħ +对 éĺµ +缮 æ¨Ļ +åIJį é¢Ŀ +客 æ°Ķ +横 åIJij +çŃī åĨħ容 +åĩł çĤ¹ +è°Ī 论 +ä¸į ä¹ı +å±ķ çİ°åĩº +è¾ĥ éķ¿ +éĢĨ 转 +å°ı æĻĤ +æĺ¯ å¤ļä¹Ī +æľ¬ æľĪ +è¿ij è§Ĩ +æĪIJç«ĭ 以æĿ¥ +代表 çĿĢ +æĬ¥ å¤į +æĪı æĽ² +è¨Ń åĤĻ +åħ¥ èĤ¡ +å¾ģ æľį +é«ĺ åĩº +èĪŀåı° ä¸Ĭ +å¿ĥ åĬ¨ +两 çĤ¹ +缸 çķ¶ +èĻ Ľ +主 页 +åĩł 家 +æĹł ä¸į +åįı å®ļ +æĸ IJ +å¯ĵ æĦı +åħ¨ 线 +æįķ é±¼ +åı¯ä»¥ ä»İ +æľī è¿Ļæł·çļĦ +æģ¶ éŃĶ +åĮħ åŃIJ +æģ ¤ +å¼Ģå¥ĸ ç»ĵæŀľ +ä¸į æŃ» +èĹ į +弯 æĽ² +æµ· 峡 +éĶĢ æ¯ģ +çļĦ çĭ¬çī¹ +示 æĦı +ä¸įèĥ½ åĨį +èĥ½ æĬĬ +éĺ² çº¿ +ä¸įå°ij äºİ +æ± Ģ +çļĦ éĤ£ä¸Ģ +羣 æĥħ +åŀ ® +被 æīĵ +åĽ½ å®ī +ç¾İ å¦Ļ +è¿Ļ åĩł +åĩº éģĵ +æľįåĬ¡ äºİ +æĪIJæŀľ 转åĮĸ +æīį åįİ +天 é¹ħ +åĩł 个人 +åĢĺ èĭ¥ +èĢ½ 误 +æĬĹ æĪĺ +è¡Į éĬ· +æĿ¥ è¢Ń +åĢŁ éĮ¢ +èįī èİĵ +ä¸¥æł¼ æī§è¡Į +举è¡Į äºĨ +å¤ĸ ç±į +å·² è¾¾ +æĿij åħļæĶ¯éĥ¨ +è¡ Ŀ +éĻį èĩ³ +æµ· éĩı +é¤IJ é¦Ĩ +æĢ¥ å¿Ļ +æ·± è¿ľ +å¾Ģ è¿Ķ +ç¨İåĬ¡ å±Ģ +å¹¿æ³Ľ åºĶçĶ¨ +è®® åijĺ +æĹł æķĮ +çľ¼ åħī +çĥŃè¡Ģ ä¼łå¥ĩ +æŃ IJ +äºĨ äºĽ +è¿Ŀ èĥĮ +è¿Ļ æĺ¯ä¸Ģç§į +ä¸į 稳å®ļ +大家 åĪĨ享 +表 çı¾ +åīį åįģ +è·¯ è¿ĩ +æĴ © +åIJĮ æĥħ +ä¹ł ä¿Ĺ +åıij è´¢ +åºĶ æľīçļĦ +æĿİ æŁIJ +èĤ Ľ +马 åħĭ +éĢļ åijĬ +å·¨ 人 +ä¸Ģ åĽ¢ +éĢĻ 次 +ä¸į äºĨ解 +æĸ½ è¡Į +èij¡èIJĦ çīĻ +åıĺå¾Ĺ æĽ´åĬł +æı £ +åĪĽæĸ° èĥ½åĬĽ +çķħ éĶĢ +表 æī¬ +æ¯Ķ åĪ© +æ¯ĶåĪ© æŶ +åĮ»çĸĹ ä¿ĿéĻ© +æĵį 纵 +伤 亡 +æµİ å®ģ +åıĺ äºĨ +æľ¬æ¬¡ æ´»åĬ¨ +åľŁ 豪 +æĥ³ åĬŀæ³ķ +æĺ ķ +å½ĵ æĻļ +åĩº å±Ģ +çĥŃ è®® +è°Ī è°Ī +æĻĭ åįĩ +åĬ¿ å¿ħ +çĻ» å±± +éĤ£ åĦ¿ +åIJĥ åĪ° +ä¹ĭ åŁİ +å¿« æĿ¥ +æ¹Ľ æ±Ł +第ä¸ī 个 +åħ¨éĿ¢ æıIJåįĩ +å¥ĸ åѦ +å¥ĸåѦ éĩij +æĬķåħ¥ 使çĶ¨ +é½IJ é²ģ +åı¯ä»¥ æĬĬ +åĴĮ ä»ĸçļĦ +è´ŃæĪ¿ èĢħ +æŃ£å¼ı åIJ¯åĬ¨ +åįİ æ¶¦ +ä¸įæĸŃ å®ĮåĸĦ +éĴ¢ æĿ¿ +ç´¯ 积 +满 èĦ¸ +åĽĽ æĸ¹ +è´¢ çī© +ä»ĸ们 ä¼ļ +å¤ı æĹ¥ +éĤ£ 个人 +éĿł çĿĢ +çĤ¹ äºĨ +çĤ¹äºĨ çĤ¹å¤´ +æ© ĭ +åıĪ 好 +åıĪ好 åıĪ +åıĪ好åıĪ å¿« +éĺµ éĺµ +å°ģ 建 +æľ¬ çĶ° +çī©ä¸ļ æľįåĬ¡ +èĩªè´¸ åĮº +åIJ ı +便åĪ© åºĹ +åĽ½å®¶ æłĩåĩĨ +éĿ¢ ç²ī +èī° è¾Ľ +æĶ» åħ³ +æīĵ åĮħ +车 éĺŁ +人 éĢī +åı¯ ä¸įæĺ¯ +äºĮ åįģå¹´ +åIJį å¸Ī +浦 举 +åħ¬ è¯ģ +è¿IJ éĢģ +æĺ¯ æľĢ好çļĦ +æŁĶ åĴĮ +çİĭ æŁIJ +çĹħ æĪ¿ +åĨ¶ éĩij +ä¸Ģ件 äºĭæĥħ +åį ¤ +åı¯ æݧ +çī Ł +æĭ Ĥ +å·² äºİ +人 éĢł +çĶŁçī© åĮ»èį¯ +ä½ĵ çİ°åĩº +èĤ² åĦ¿ +èĢģ å®ŀ +åľĸ çīĩ +è« ¸ +ç´¯ äºĨ +æĦŁåħ´è¶£ çļĦ +åĽ¾çīĩ æĿ¥æºIJ +ä¹Ł æĺ¯ä¸Ģç§į +æ¾İæ¹ĥ æĸ°éĹ» +æŶ 表示 +åħī è¾ī +æĬ¥ åºŁ +å²ģ æŶ +éħ ® +æ£Ģ ä¿® +åıĺ éĢŁ +åıĺéĢŁ ç®± +åľ¨ èģĮ +éı ¡ +æį Ĥ +çĿ£ åĬŀ +æ°¸ ä¸į +åģļ ä¸ĢäºĽ +åİĨ æŶ +å·¥ç¨ĭ æľºæ¢° +æģ° å½ĵ +å°± åľ¨äºİ +称 åij¼ +éĢļ常 æĺ¯ +æł· å¼ı +åij¨ ä¸Ģ +èĭ± éķij +åĿĩ 线 +ä¼ł éĹ» +çĶ¨æĪ· ä½ĵéªĮ +èµŀ åIJĮ +骨 æĬĺ +为主 ä½ĵ +æ±Ł å±± +æ¸ħ æľĿ +æĶĢ åįĩ +ä¸į çĽ¸ä¿¡ +éĿ ´ +æѦ åĬŁ +åĭ¤ åĬ³ +æĿ¥ æī¾ +å°Ĩ æĮģç»Ń +丫 头 +æ¨Ļ æºĸ +è£ ´ +深深 çļĦ +åŃķ èĤ² +è§ĦåĪĴ 建设 +æ¸ħ çĪ½ +ç²¾åĩĨ æī¶è´« +æīĵçł´ äºĨ +è¿Ļä¸Ģ 天 +å·¥ä½ľ æĢ»ç»ĵ +æĹħ ç¨ĭ +举 èIJ¥ +æĶ¾ å°Ħ +æľī åĩłä¸ª +éĿŀ çī©è´¨ +åIJĥ å¾Ĺ +åĹ ¨ +ä¼ļ åıijçĶŁ +篮 æĿ¿ +å¼Ģ å°ģ +麻 å°Ĩ +èıı æ³½ +ä¸į åIJĪ +ç³»åĪĹ äº§åĵģ +èѬ å¦Ĥ +ç¾İ èªī +èĩªå·± åĸľæ¬¢ +交æĺĵ ä¸Ńå¿ĥ +åIJĪ åĶ± +使 æĪij +åĥı ç´ł +带 éĺŁ +ä½Ĩ 对äºİ +æĬĬ è¿Ļ个 +èĤĿ èĦı +åįķ纯 çļĦ +æĶ»åĿļ æĪĺ +缼 ä¼ļ +åijµ æĬ¤ +æª Ģ +赶 ä¸Ĭ +æ¥ Ĭ +ä¹ħ äºĨ +ç¡ Ŀ +çŃĶ é¢ĺ +ä¿ĿæĮģ çĿĢ +è§ģ è¯Ĩ +çĤ¹ åĦ¿ +åįĬ 个æľĪ +æ» ĩ +浸 泡 +ä¼ł éĢģ +åľ¨ å¸Ĥåľºä¸Ĭ +ä¹ĭ 乡 +çī¹ éķ¿ +éĽ ŀ +èª ł +身 å¤Ħ +æŁł 檬 +身 ç©¿ +çľģ åħ¬å®ī +çľģåħ¬å®ī åİħ +åıĻ åĪ©äºļ +åĩł åĪĨéĴŁ +人 åĢij +åľ° 段 +èĩª åѦ +ä¹Ł è¶ĬæĿ¥è¶Ĭ +èģĮ æĿĥ +æĸ § +èĩ » +å½Ĵ 纳 +驾 é©Ń +éĥ¨åĪĨ åľ°åĮº +没æľī æĥ³åĪ° +æĴ ĩ +ä¹Į é²ģ +ä¹Įé²ģ æľ¨ +ä¹Įé²ģæľ¨ é½IJ +èĤ² 人 +çļĦ æŃ¥ä¼IJ +延 æľŁ +æ²¹ æ°Ķ +åģļ å®Į +åľ£ åľ° +丰 åİļ +宽 带 +åı¯éĿł çļĦ +åºŃ éĻ¢ +åŃ ľ +å°ı康 社ä¼ļ +å®īåħ¨ 管çIJĨ +å¹´ 第 +æİĴ 污 +èĥĮ åĮħ +家 ä½ı +åħ¶å®ŀ å°±æĺ¯ +ä¼ļ è§ģ +帮åĬ© ä¼ģä¸ļ +ç½ij è´Ń +æĺ¯ ä¸įä¼ļ +飯 åºĹ +æŃ» åİ» +åħįçĸ« åĬĽ +æľ ķ +åĸĿ äºĨ +è½» å¾® +个æľĪ åĨħ +ç»Ħ åĽ¢ +åĴĮ å®ĮåĸĦ +é¸ ½ +æıIJ éĢŁ +西å®ī å¸Ĥ +ä¸Ńå¿ĥ 主任 +æŶéĹ´ 为 +æľŁ æĿĥ +è¶ ķ +ä¸įä»ħ è¦ģ +æľį ä»İ +é¡ĺ æĦı +ä¸į å°ı +ä¸įå°ı çļĦ +ç° ĩ +çª ¦ +åĪĩ æĪIJ +åĵĪ åĪ© +天 羣 +ä¸Ģ次 次 +éĩij å¸ģ +æĢİä¹Ī èĥ½ +ç½ij è´· +ä¼ļ计 å¸Ī +çŁŃ 缺 +对 æłĩ +åıĺå¾Ĺ æĽ´ +åīį åĩłå¤© +éĺ² æ±Ľ +彩 èĻ¹ +åĵģ ä½į +表 æł¼ +严 å¯Ĩ +æ¯Ľ åĪ©çİĩ +çļĦ åį±å®³ +å½ķ åĪ¶ +æ°´ åĬ¡ +èĥ½å¤Ł 让 +å¹³ æĿ¿ +ä¹³ æĪ¿ +è¸ı å®ŀ +é¦ĸ åĪĽ +é¦Ļ èķī +æĬ¥ 表 +ä¸Ģ æĬ¹ +åĩºçĶŁ äºİ +è²» çĶ¨ +åĩº 让 +åIJĪæ³ķ æĢ§ +å°¼ åħĭ +åĨ° åĨ· +é¦Ļ æ°Ķ +åı· 称 +èµ· çłģ +åŁİ åİ¿ +çİ© èĢį +ä¸Ĭ éĻIJ +ä¼ļè®® ç²¾ç¥ŀ +æĹģè¾¹ çļĦ +便 ä¼ļ +æıŃ æĻĵ +çİ© æĦı +éĽª å±± +åIJij çĿĢ +ä½ĵèĤ² åľ¨çº¿ +说æĺİ ä¹¦ +åĮĸ èĤ¥ +åħļç»Ħ 书记 +åĬ¨ 人 +ä¹ĭ æīĢ +æľĪ èĩ³ +æľĢå¿« çļĦ +èĬĤ åģĩæĹ¥ +ä¸ĵ åľº +èĢĥ ä¸Ĭ +çª Ł +é²ľ è¡Ģ +è¾ĥ强 çļĦ +æĤĦ çĦ¶ +å¤ļ个 åĽ½å®¶ +çªĹ å¸ĺ +æŀģ å¤§åľ° +ä¸įçĶ¨ æĭħå¿ĥ +è¿Ļä¹Ī åģļ +åĥ¹ æł¼ +ç¾İ丽 乡æĿij +å°ıæŶ åĨħ +ç´§ è¿« +大 çģ« +èĥ³ èĨĬ +æĵįä½ľ ç³»ç»Ł +æ®ĭ çķĻ +åĨĻ åĩº +ç¦ģ å¿Į +åĬłçĽŁ åºĹ +è¿ij çĻ¾ +便 åı¯ +æķ´æĶ¹ æݪæĸ½ +éĩĩ访 æŶ +åĶIJ 代 +æ·±åĮĸ æĶ¹éĿ© +çŁ ¢ +éĥ½ åĸľæ¬¢ +è¶ĬæĿ¥è¶Ĭ é«ĺ +èĬ± æľµ +头 çĸ¼ +å®ī 康 +å¢ŀéķ¿ çİĩ +çľ¼ çľĭ +å°±æĺ¯ ä¸ºäºĨ +èĢĮ 导èĩ´ +åĬłå¿« 建设 +èĬ± æł· +åĨħå¿ĥ çļĦ +æĺĨ å±± +è³ĩ æºIJ +åĽŀåĪ° 家 +èıĬ èĬ± +æ°´ éĩı +å¾ģ ä¿¡ +è¡ĮæĶ¿ åĮº +ä¹ĥ æĺ¯ +æĬķèµĦ é¡¹çĽ® +å«ģ ç»Ļ +ç¥ŀ åľ£ +ç¨ ł +æľ¬æĿ¥ å°± +éĢIJ ä¸Ģ +èģĮä¸ļ æĬĢæľ¯ +ä¸įèī¯ ä¿¡æģ¯ +æīĺ è¿IJ +åIJ¯ 示 +ä¹ĭ åħ§å®¹ +éŁ ¶ +奢 åįİ +æıŃ ç¤º +æĪIJ为 ä¸ŃåĽ½ +æ¶Īè´¹ åĵģ +åħ¬ çĶ¨ +æIJŀ å®ļ +请 ä½ł +æŁ ļ +åĨħ è¡£ +ä½Ĩ ä»ĸ们 +ä¿Ŀ 湿 +该 åİ¿ +饱 åĴĮ +æݨ åIJij +èµĦæĸĻ æĺ¾ç¤º +ä¸į å½±åĵį +人 人éĥ½ +åıijå±ķ 壮大 +åħ»èĢģ æľįåĬ¡ +çĶŁæ´» æ°´å¹³ +åIJĦ åİ¿ +ä½ł éľĢè¦ģ +说 çļĦæĺ¯ +å¤ĸ åªĴ +æѤ 人 +次 è¦ģ +追 赶 +åºĶ该 å¦Ĥä½ķ +æĹ¥ åĩĮæĻ¨ +çķ¥ æľī +éĥ½ æĥ³ +游 ä¹IJ +è¿Ļ款 游æĪı +å¹³ æ·¡ +æĺ¯ä¸Ģ åĢĭ +å¤ĩ èĢĥ +åĪ¶ æŃ¢ +ä¸Ģå®ļ èĥ½ +å¾Ĵ å¼Ł +以 çĤº +åįĥ åħĥ +äºĶ åħŃ +迪 士 +迪士 å°¼ +éĺ³ æĢ§ +åĨ¬å¥¥ ä¼ļ +å°±æĺ¯ åĽłä¸º +æĮĤ éĴ© +æ¦Ĥ åĨµ +åıªè¦ģ æľī +æ²¹ çĶ» +åľ° æłĩ +ä¸Ĭ è°ĥ +产ä¸ļ åĽŃåĮº +åħ« åįģ +æ£ ± +液 æĻ¶ +æĿij å§Ķä¼ļ +çŃ¾çº¦ 仪å¼ı +è¿Ļ åħ¶ä¸Ń +åĨĻ éģĵ +示èĮĥ åŁºåľ° +éĩİçĶŁ åĬ¨çī© +鼻åŃIJ ä¿¡ç®± +åĽ½éĻħ è´¸æĺĵ +人 æĿĥ +ä¿Ŀ 管 +èĭ¥ æĤ¨ +åİĭ æĬij +é» Ľ +åľ° çľĭçĿĢ +éĻ ° +ä¸Ģå¹´ å¤ļ +ä»İ 容 +ä¸Ń æĸŃ +å¯Ł è§ī +移 交 +éĶ ¯ +æĪĸ许 æĺ¯ +ç¶ ł +两 项 +æľĢ åĸľæ¬¢ +æľĢåĸľæ¬¢ çļĦ +å¤ľ éĩĮ +åIJĮ ä»ģ +åĪĽæĸ° 驱åĬ¨ +è°ģ èĥ½ +é£ ¾ +åħī åѦ +åİ Ħ +èĦ± é¢ĸ +èĦ±é¢ĸ èĢĮåĩº +è¿ ¦ +æĺ¯ ä¸įåı¯èĥ½ +çª ¥ +èĥ½ 满足 +宽 度 +伦 çIJĨ +åı¯ä»¥ èİ·å¾Ĺ +转 ä¼ļ +å±± æĿij +éĵº 设 +åĩº åĩ» +æĸĩåĮĸ èīºæľ¯ +ä¼ļè®® 室 +æŃĮ 声 +æ» Ķ +èIJİ ç¼© +æľįåĬ¡ åijĺ +åıij表 äºĨ +æĸ¼ æĺ¯ +æĺİç¡® è§Ħå®ļ +ç»´ å¥ĩ +æ°´ 产 +æĬķ ä¿Ŀ +éĺ´ éģĵ +赶 å¿« +夺 å¾Ĺ +ä¸ĭ åįķ +çī©æµģ åħ¬åı¸ +çݯ ç»ķ +å½ Ī +ä½ľé£İ 建设 +æĹħ游 æĻ¯åĮº +æľī æĽ´å¤ļçļĦ +丰å¯Į å¤ļ彩 +çIJĨè´¢ 产åĵģ +åĩº å·® +ä»İ严 æ²» +ä»İ严治 åħļ +缸 å¹² +æ»ĭ 润 +主åĬŀ æĸ¹ +åī§ åľº +æ»ļ çIJĥ +æ©Ħ æ¦Ħ +èĩªä¸» åĪĽæĸ° +éĢļ å¾Ģ +æł¼ å°Ķ +çļĦ ä¼ĺçĤ¹ +èĥĮ ä¸Ĭ +çª ľ +çĪĨ åĩº +å¹³ æķ´ +ä¸Ģ èĦļ +åħ¨ä½ĵ åijĺå·¥ +éĻIJ å®ļ +åŁİéķĩ åĮĸ +æ· ³ +éĢ® æįķ +è¡ĮåĬ¨ 计åĪĴ +æīĵ å¾Ĺ +åİļ éĩį +纪å½ķ çīĩ +åĿļ ä¿¡ +央 ä¼ģ +åĨį ä¹Łä¸į +天 涯 +åıĤèĢĥ èµĦæĸĻ +æľī æ¯Ĵ +åIJ¸ 纳 +è¶Ĭ åıij +éĩįè¦ģ æĦıä¹ī +åĽ½éĺ² éĥ¨ +è¿Ļ个 è¡Įä¸ļ +æĻ® æŁ¥ +å¼Ĥ æĢ§ +延 è¿Ł +å°ı å¹ħ +èī² æĥħ +综åIJĪ æ²»çIJĨ +æŃ£æĺ¯ åĽłä¸º +产ä¸ļ ç»ĵæŀĦ +çłĶ究 æĬ¥åijĬ +åģľ ä¸ĭ +éķ¿ èĢģ +éĩĿ å°į +åįĹ京 å¸Ĥ +çģĮ æºī +转 è¿IJ +欺 è¯Ī +éĢł åģĩ +åĪĨå¸ĥ å¼ı +æĦŁ 触 +æĪij å½ĵæŶ +åıij è§ī +åĽ¾ 纸 +æĶ¹ èī¯ +çĭł çĭł +åĨ² åĪº +æĸ° 京 +æĸ°äº¬ æĬ¥ +ç¥ŀ åĻ¨ +秸 ç§Ĩ +çĪ º +å°Ĩ è¿İæĿ¥ +å·¥ ä¿¡ +工信 éĥ¨ +éĻIJ éĩı +æŃ¢ æįŁ +åѦä¼ļ äºĨ +åįİ çĽĽ +åįİ缼 é¡¿ +å¾Į ä¾Ĩ +ä¸ĭéĿ¢ æĺ¯ +ä¸ĭéĿ¢æĺ¯ å°ı +æIJ¬ è¿IJ +ç¾İæľ¯ é¦Ĩ +æ¸ħ åĩī +å¤ļå¹´ åīį +è© ŀ +åįĥ ç±³ +表 è¿° +æ±Ł éŨ +åĬłæ²¹ ç«Ļ +æľ¬ èĥ½ +导 读 +åĽ´ è§Ĥ +并 åIJij +åŁºæľ¬ æĥħåĨµ +æīĵ å¼ĢäºĨ +è¿Ļ ä¸ī个 +æ±ķ 头 +强 æľīåĬĽ +强æľīåĬĽ çļĦ +è¿Ľ åľº +ä¹Ŀ æ±Ł +çIJĥ æĺŁ +好çľĭ çļĦ +大 æĪ· +æ¹ ¯ +å¥ĩ å¦Ļ +ä¹IJ åĻ¨ +æĪijçļĦ å¿ĥ +çľī 头 +åĨľä¸ļ çĶŁäº§ +ç¼ĸ çłģ +åŁº ç¤ +åŁºç¤ İ +天 æĸĩ +åĢĭ人 è³ĩè¨Ĭ +åİ» è¿ĩ +èģĨ åIJ¬ +æĶ¾ åģĩ +ä¸į åħ·å¤ĩ +æ·Ģ ç²ī +大 佬 +åħ¨ 天 +åħ¨éĿ¢ 建æĪIJ +éļIJ å½¢ +ç¼ħ çĶ¸ +åIJ ³ +è¡ĮæĶ¿ æī§æ³ķ +åŁİ åł¡ +èİ« æĸ¯ +èİ«æĸ¯ ç§ij +æīĢæľī æĿĥ +éĽĨ åľĺ +å±Ģ åī¯å±Ģéķ¿ +åĩłä¹İ 没æľī +æ´ģ åĩĢ +çĶµå½± èĬĤ +åŃ© ç«¥ +æīĢ åģļçļĦ +æ¸ħ 代 +æĸ° çīĪ +éĵĿ åIJĪéĩij +为 æĬĵ +为æĬĵ æīĭ +åĪ¤ å®ļ +çī¹ 产 +æīĭ æ©Ł +ä¸įåı¯ æĪĸ +ä¸įåı¯æĪĸ 缺 +å¸Ĥåľº è§Ħ模 +åĿ ¯ +åĮ» åѦéĻ¢ +å¿« è¦ģ +èĮ ľ +æĬĺ èħ¾ +äºĨ è¿ĩæĿ¥ +æĬ¥åijĬ æľŁåĨħ +çī© ç§į +ç»Łè®¡ å±Ģ +æī© 建 +æ¶ ħ +责任 人 +éĺ İ +è¯Ħ è®® +å¾Ģ äºĭ +æīĢ 示 +æķ´ æ´ģ +éĹº èľľ +æĹħ éĢĶ +å®ŀ è®Ń +ä¹ĭ 称 +å·´ 士 +éĢŁåº¦ å¿« +ä¸įä»ħ å¦ĤæѤ +å®Ŀè´µ çļĦ +åºŁ çī© +æ²³ æ°´ +æİ¥ 纳 +ç²¾ æ¹Ľ +åħ¶æ¬¡ æĺ¯ +顺 å¾· +åħ¬åħ± åį«çĶŁ +è¤IJ èī² +ä¸į æĥľ +æĬĢæľ¯ æľįåĬ¡ +æİ · +æ±Ĥ èģĮ +ä¸ī 峡 +æĬķåħ¥ åĪ° +太 åIJİ +åIJ¯åĬ¨ 仪å¼ı +缴æİ¥ å½±åĵį +æĸ° 款 +个 乡éķĩ +çĻ¾ 亿 +åº « +ä¹Ł æŃ£æĺ¯ +åı¶ çīĩ +æľĢæĹ© çļĦ +æĪĺ 绩 +å·¥ æľŁ +æĻļ æľŁ +è¿Ļæł· 说 +è¯į è¯Ń +ä¾ Ħ +æķ£ çĥŃ +éĽĨæĪIJ çĶµè·¯ +åIJį è¯į +æĻº åķĨ +æĭ¥ åłµ +çĭĤ 欢 +è¿Ļ èĪ¬ +æµ´ 室 +åijķ åIJIJ +æľªæĿ¥ åıijå±ķ +ä¸īä½į ä¸Ģä½ĵ +åªĴ é«Ķ +ä¸įå¾Ĺ 转载 +åĽłä¸º 她 +æĺ¾ç¤º å±ı +ä¾Ľ æļĸ +éĨ« éĻ¢ +æľī æĦıæĢĿ +æľīæĦıæĢĿ çļĦ +娱ä¹IJ åŁİ +åįµ å·¢ +åĪĽéĢł åĬĽ +ç«ł èĬĤ +人大 常å§Ķ +èĢĮ çİ°åľ¨ +å¤ĸ å©Ĩ +å¢ŀ æĮģ +äºĶ åįĥ +èĢģå¸Ī 们 +æ´Ľ æĿī +æ´ĽæĿī 磶 +æİĮæı¡ äºĨ +ä¸ŃåĽ½ æĸĩåĮĸ +æĸ° æĶ¿ +主è¦ģ çĶ¨äºİ +åıij çĥ§ +类似 äºİ +åĮĹ æŀģ +æĪij们 认为 +å¼¥ 漫 +åħ¨çIJĥ ç»ıæµİ +é¢ IJ +ä¸Ģèµ· è£ħä¿® +æĶ Ĵ +æĭī èIJ¨ +帶 ä¾Ĩ +åĨ· æ°´ +ä¸ī åĨľ +æĿ¿ æĿIJ +è¿ŀ è¿ŀ +éĵ ® +ç»ıèIJ¥ çIJĨ念 +å±± 顶 +å¾Ī æĥ³ +çĺ « +å§ĭç»Ī ä¿ĿæĮģ +åľ¨ 广å·ŀ +ä¸įåIJĮ æĦı +åıĺ åİĭ +åıĺåİĭ åĻ¨ +产 éĶĢ +表 éĿ¢ä¸Ĭ +æīĢ以 ä»ĸ +ç»ıéªĮ 丰å¯Į +éĥ¨ å§Ķ +åħµ åĽ¢ +æīĢ è¿° +æķ¦ çħĮ +ç»ıèIJ¥ èĮĥåĽ´ +åı£ è¯Ń +失 ä¿¡ +æ¯ı个人 çļĦ +æīĭ æĮģ +æģIJ æħĮ +åł¡ åŀĴ +é¦ ħ +éĵ¸ éĢł +æĭ¿ åĩºæĿ¥ +æİ¢ æµĭ +大家 ä¸Ģèµ· +å¥ § +å®ŀè´¨ æĢ§ +å°ı åĦ¿ +èĩº åįĹ +èĩºåįĹ å¸Ĥ +å¼Ģåıij èĢħ +åı¯ æł¹æį® +ç®± åŃIJ +饺 åŃIJ +å¿Ļ çĿĢ +æĿ¥ ä¸įåıĬ +缸 ä¼ł +åĽ½ ç½ij +èħ¹ æ³» +è¿ĻéĩĮ æľī +é£İ æĻ¯åĮº +åıĤ ä¿Ŀ +æŃ» èĢħ +æĪ´ ä¸Ĭ +æ©Ł æ§ĭ +è¯ķéªĮ åĮº +ä¼ł æİĪ +æµ· è¾¹ +泪 æ°´ +缸åħ³ åĨħ容 +éĥij å·ŀå¸Ĥ +åħij çİ° +两 åij¨ +èĬľ æ¹ĸ +çĶµåŃIJ ä¿¡æģ¯ +红 å¤ĸ +æĹħ游 å±Ģ +å¾Ģå¾Ģ ä¼ļ +è¿ħ çĮĽ +ä¼ł 羣 +æ¸ħ æ¾Ī +å°± è¿ij +微信 群 +ç³»åĪĹ æ´»åĬ¨ +ç»ı常 ä¼ļ +è§Ĥ æµĭ +å¿ĥå¾Ĺ ä½ĵä¼ļ +éĻĪ åĪĹ +åĮĹ æĸĹ +è« ® +è«® è©¢ +è¿ĺæĺ¯ ä¼ļ +æµĭ ç®Ĺ +æĺŁ ç©º +宽 容 +çī©ä¸ļ åħ¬åı¸ +æĪĴ æĮĩ +å¸ħ æ°Ķ +ä¸ĢæŃ¥ æŃ¥ +åħ± 鸣 +åĨ³ ä¸į +æİ¥ 管 +å¦ĩ èģĶ +æ¯Ķ åĸ» +é²ģ è¿ħ +æĮģ çºĮ +缸 亲 +å¨ģå°¼æĸ¯ 人 +ç«ĭ 项 +åĪ Ŀå§ĭ +èĩª åĪ¶ +è¿Ī è¿Ľ +ä¸Ĭ æ±½ +å®ı ä¼Ł +æł¹æľ¬ 没æľī +æĸ°åĨł çĹħæ¯Ĵ +åĵª ç§į +康 åħ» +è¡° èĢģ +å½ķ åĥı +é«Ķ é©Ĺ +ç»ij å®ļ +é¢Ŀ 头 +äºĶ æľĪ +èĬ± å¼Ģ +ä¸Ģ线 åŁİå¸Ĥ +åĪ° åľº +æĬķ éĻį +çĹĺ çĹĺ +åıĹ ä¸įäºĨ +æīİ æł¹ +æĽ´ ä½ķåĨµ +æĬ½ æŁ¥ +åĩº è·¯ +审议 éĢļè¿ĩ +ä¸į åĥħ +èī² è°ĥ +çĻ¾ ä½Ļ +èĤł éģĵ +æ·±åİļ çļĦ +马 åĬĽ +æĹ© æĻļ +æŃĮ èĪŀ +éĺ² æĻĴ +æľĢåIJİ ä¸Ģ个 +樱 èĬ± +å°ıä¼Ļ åŃIJ +åľ¨ å½ĵåľ° +å°ıä¼Ļä¼´ 们 +èµ· æºIJ +åħ¨ åªĴä½ĵ +ç° ½ +éħ± æ²¹ +æĹłè®º å¦Ĥä½ķ +裤 åŃIJ +åģľ 产 +ä¸įçĶ± å¾Ĺ +çīµ å¼ķ +ä¼ł åĬ¨ +ä¹Ŀ é¾Ļ +åĬł åĽº +ä¹Łä¸į æķ¢ +æĬĢæľ¯ æĶ¯æĮģ +ä¸Ĭ å²Ĺ +ç»ıéªĮ åĴĮ +æł¼ æŀĹ +åIJ¸ éĻĦ +æľªæĪIJ å¹´ +奢ä¾Ī åĵģ +追 æį§ +好 ä¸į容æĺĵ +èķ´ åIJ« +ä¿Ŀ å®ļ +æĬ¥ ä¸ļ +æµ· åĨħå¤ĸ +ä½ł çİ°åľ¨ +æ²¹ èĢĹ +è´¨éĩı 管çIJĨ +æ½ľ æ°´ +丽 æ±Ł +转 åħ¥ +è¿Ļä¹Ī ä¹ħ +æĺİ ä»£ +责任 åĪ¶ +éĩį å·¥ +大 å·´ +触 åıĬ +èµ· åĪĿ +大 å¦Ī +æĸ¯ å¡Ķ +åĨĽ å·¥ +书 éĻ¢ +å³ ¨ +æݨ çIJĨ +è¿Ļç¯ĩ æĸĩç«ł +è¿ģ 移 +åľ¨ åIJĮä¸Ģ +ç»Ĩ ç»Ĩ +åīĬ å¼± +书 æĪ¿ +ç¶ĵ 常 +è¯ķ é¢ĺ +æĤ£ ä¸Ĭ +çĻ«çĹ« çĹħ +åĨ² æ´Ĺ +å¤ĸ æı´ +åħĭ åĪ¶ +åįģ æľĪ +åģļ ä¸įåĪ° +ç¾İ åĮĸ +å¦Ĥ æľŁ +è¿ĺ éľĢ +天 åºľ +å°± æĦıåij³çĿĢ +çļĦç¡® æĺ¯ +éªĹ å±Ģ +å°ıç»Ħ èµĽ +è© © +ä¹Ŀ å¹´ +æĻĵ å¾Ĺ +çłĶ究 人åijĺ +大 éħĴåºĹ +ç§ij åѸ +åħŃ åIJĪ +çķĮ å®ļ +车 è½½ +å¼Ģ çĿĢ +毫 æĹłçĸij +毫æĹłçĸij éĹ® +è¿IJ ç»´ +ç¦ģ åĮº +èĦ± èIJ½ +讲 å¸Ī +产ä¸ļ åŁºåľ° +é«ĺ æĢ§èĥ½ +åħī 彩 +çİ° éĺ¶æ®µ +åĩ ¿ +è¾ĥ å·® +饮 çĶ¨æ°´ +éĸĭ çĻ¼ +ç½ij åIJ§ +çĮ´ åŃIJ +æѦ æŀĹ +å®ī åİ¿ +ä¸įåı¯ æĢĿ +ä¸įåı¯æĢĿ è®® +éĬ· åĶ® +è´« ç©· +为 åķ¥ +éº ĵ +å¹¾ åĢĭ +è§Ħ模 以ä¸Ĭ +æı ļ +被 åĽ° +缺 å¸Ń +å¿« é¤IJ +æĬ¢ åįł +æĻ Ł +å¤į æ´» +æľ¬æĬ¥ 讯 +åĪĽ ä¸ĭ +æµ· 滩 +éĩı 产 +å¦Ĥä½ķ åİ» +车 ä½į +å¯ ĩ +äºĮ åįģåĽĽ +ç»ıæµİ æįŁå¤± +éħįå¥Ĺ 设æĸ½ +åŁºæľ¬ éĿ¢ +äºī 论 +就好 åĥı +çłĶ究 æĪIJæŀľ +éĻĪ è¿° +æīĵ åĬ¨ +ä¸ĭ å·´ +ç§Ĵ éĴŁ +对 人ä½ĵ +æĬĢæľ¯ çłĶåıij +åİŁ åŃIJ +æĺ¯ä¸Ģ 项 +äºĨä¸Ģ 份 +æĮĩ çĶ² +çĶ¨ éĩı +è¿ĺä¸į å¤Ł +æĶ¿åºľ éĩĩè´Ń +çŁ¥è¯Ĩ çĤ¹ +ä¸ŃåĽ½ 梦 +å¾Ī å¼Ģå¿ĥ +礼 è²Į +éĿŀ常 å¤ļ +éĿŀ常å¤ļ çļĦ +åĽ ļ +æĹħ é¦Ĩ +å°½ æĥħ +æŃĮ åĶ± +æ²Ļ é¾Ļ +车 åİ¢ +客 æµģ +åģı å·® +积累 äºĨ +æ¡ Ķ +çĶ» çĶ» +ä¹Ł åºĶ该 +åºĶçĶ¨ ç¨ĭåºı +èĥĥ èĤł +以 å¾Į +豪 å®ħ +æ·± åĬłå·¥ +缴 è¨Ģ +åĮĸ çŁ³ +åĽ½ éģĵ +ä¸ĥ 个 +ä»İèĢĮ 使 +èĤł èĥĥ +æĹ¥ è¶ĭ +çĪ¶ åŃIJ +ç· © +æĭĽ çīĮ +产 å¦ĩ +çķª èĮĦ +æĪij éĻ¢ +建çŃij å·¥ç¨ĭ +å±ķè§Ī ä¼ļ +家éķ¿ 们 +åĨľ ä½ľçī© +æĹ¥ å¤ľ +æĶ» æĵĬ +è§Ħ éģ¿ +èĪŁ å±± +便 æ°ij +åħ« åŃĹ +ä¸į æĽ¾ +æĶ¯ éħį +çĨ¬ å¤ľ +人 é¡ŀ +ç´Ģ éĮĦ +ç»ıèIJ¥ æ´»åĬ¨ +大 涨 +å¸Ĥå§Ķ 常å§Ķ +åĪĨ éIJĺ +ä¸Ģ个 èģĮä¸ļ +çĹħ åĽł +è¿Ļ 对äºİ +ä¸įå¾Ĺä¸į 说 +åıijçĶµ æľº +æľīæīĢ 帮åĬ© +缮æłĩ ä»»åĬ¡ +åĽł åľ° +åĽłåľ° åĪ¶ +åĽłåľ°åĪ¶ å®ľ +å°Ĩ è¾¾åĪ° +ç²Ĺ ç³Ļ +稳 åĽº +å« £ +çİ°åľ¨ å¾Īå¤ļ +ä¸ĸçķĮ 级 +å¼ł æŁIJ +çĤ¹ ç¼Ģ +èij µ +社ä¼ļ ç»Ħç»ĩ +å¾Ģ åIJİ +åĬł æģ¯ +åĻª 声 +æľī åħ´è¶£ +为æĤ¨ æıIJä¾Ľ +æ²¹ æ¼Ĩ +ç¬¬åĽĽ å±Ĭ +çļĩ 宫 +ä¹Ĵ ä¹ĵ +ä¹Ĵä¹ĵ çIJĥ +éļ¨ èijĹ +éģ© åIJĪ +åįĹ éĿŀ +æĵ ´ +西 æ´ĭ +åĬł å¯Ĩ +æĪIJåĬŁ 举åĬŀ +åı£ æ°´ +æĪIJ 年人 +æīĢ æıIJä¾ĽçļĦ +éļĶ å£ģ +åľ¨ 京 +å½ĵåľ° æŶéĹ´ +çŃī åIJĦç§į +é£İ æ°Ķ +å±ĭ éĩĮ +ä¸Ģ åŃĹ +çļĦæŶéĹ´ éĩĮ +åĺ¿ åĺ¿ +å¿« 讯 +ä¸Ń åľº +ä¸Ģ çĵ¶ +æ» ķ +é¢Ĩ è·ij +好 èݱ +好èݱ åĿŀ +没 åħ³ç³» +åĩº å¢ĥ +ä¸įæĺ¯ ä¸Ģ个 +éĥ½æĺ¯ éĿŀ常 +éľĩ åĬ¨ +èİ· èĥľ +åįļ å¼Ī +æĬļ åħ» +对 ç«ĭ +æľįåĬ¡ æľºæŀĦ +è°£ è¨Ģ +社ä¼ļ ç§ijåѦ +åIJ¬è¯´ è¿ĩ +æī ³ +æīĵ 磨 +åı£ æľį +好 åĥıæĺ¯ +以åıĬ åħ¶ä»ĸ +çī¹ è´¨ +亲 è¿ij +ä¸Ģ ç»ı +æ¶ Ŀ +éŃĶ æľ¯ +éģĵè·¯ 交éĢļ +è§Ħ模 æľĢ大 +å®ŀæĸ½ æĦıè§ģ +ä¹ ŀ +ä¸Ģ ä¸ĸ +åŁ· è¡Į +è±Ĩ çĵ£ +åĪĹ ä¸º +æķħ 宫 +çĶŁ åij½åij¨æľŁ +ä¸īç§į èģĮä¸ļ +详ç»Ĩ ä»ĭç»į +å®Į å¤ĩ +岩 çŁ³ +éļı æīĭ +é£ ² +æķĪæŀľ åĽ¾ +ç§ĭ åĨ¬ +åĬŁ å¾· +è§Ħç«ł åĪ¶åº¦ +æĹ¥ æ¸IJ +æīĢ éľĢè¦ģ +æīĢéľĢè¦ģ çļĦ +å²Ľ ä¸Ĭ +åĩº åľŁ +åĽ¾ æĸĩ +ç§ijæĬĢ è¿ĽæŃ¥ +éĢļ èĥĢ +èĢģ 太太 +èĭĹ æľ¨ +éĵ¶ å·Ŀ +å¸IJ 篷 +éĿŀ è¦ģ +éħį çĶµ +å¤Ħ å¢ĥ +èĤ¡æĿĥ æĬķèµĦ +ä¸Ģ缴 åĪ° +åĿĩ çĶ± +æĬĹ æĹ¥ +æį® ä»ĭç»į +ä½ł åĸľæ¬¢ +åĪĽæĸ° åŀĭ +åıĺ è¿ģ +è§Ĩ å¯Ł +å®Įåħ¨ 没æľī +åħĥ æŦ +åı¯ ä¿¡ +åı¦ è¡Į +æĿij 级 +åħ¥ åľº +æIJŃ æ¡£ +ä¹Ł åĽłæѤ +æį¢ æĪIJ +ä¸į è´Ł +äºĨ 大éĩıçļĦ +éģĶ åĪ° +å¸Ĥ åİ¿ +å¹´ è¼ķ +å¿« æīĭ +å¸Į å°Ķ +èĩª èIJ¥ +éĽª èĬ± +æIJ ģ +çľ¼ ç§ij +æŃ£ 確 +çļĦ 姿æĢģ +åĿļå®ŀ çļĦ +æĮĩ 纹 +æªĶ æ¡Ī +ç½® äºİ +佩 æľį +豪 éŨ +åĵ Ĵ +æģ° 好 +檢 æŁ¥ +åĪĿ è¡· +大 åĶIJ +约 ä¼ļ +èĴ¸ åıij +çŃ¹ åĪĴ +å¹´ ç»Ī +è¡Į æ¥Ń +åħ± éĿĴ +åħ±éĿĴ åĽ¢ +ä¼ļ å¼ķèµ· +ä¸Ń ç§ij +ä¸Ńç§ij éĻ¢ +æĮ¯ åĬ¨ +åį´ åıijçİ° +ä¸įåĬ¨ 产 +èĮ ¹ +æĪ¿éĹ´ éĩĮ +è´§å¸ģ æĶ¿çŃĸ +æ²» çĻĤ +æħİ éĩį +å¡ŀ å°Ķ +åĽ½ ç±į +åĽł æŀľ +çŃī çī¹çĤ¹ +å±± è°· +ä¸ĭ è¼ī +è®ĵ æĪij +饮 éħĴ +è¿Ļ个 游æĪı +ç»Ŀ 大éĥ¨åĪĨ +åĴ¨è¯¢ æľįåĬ¡ +å¹² æ´» +è®® ä¼ļ +æ¦Ĥ è¿° +åĪĨ åĮº +æŃ» åIJİ +ç«Ļ çĿĢ +主è¦ģ é¢Ĩ导 +åIJĮ åŁİ +大 æłij +对 åѦçĶŁ +社ä¼ļ ä¿ĿéĻ© +å¢ŀ èµĦ +主人 åħ¬ +å®£ä¼ł æķĻèĤ² +æĸĩåĮĸ 交æµģ +客 æĪ¶ +çŁ¥åIJį åĵģçīĮ +æ»ŀ åIJİ +äºĴ è¡¥ +æĦŁ 人 +åī ¿ +åIJİ ä»£ +äºī 龸 +æķĻèĤ² åŁ¹è®Ń +éĿĻ èĦī +ä¹ı åĬĽ +说 åĩºæĿ¥ +çİĭèĢħ èį£èĢĢ +åĢ « +åįĩ èµ· +éķ ģ +åĩº 游 +éĢļè¡Į è¯ģ +å·¥ä½ľ å²Ĺä½į +åĮł å¿ĥ +æĭ¿ æĿ¥ +æ´Ĺè¡£ æľº +æĪijä¸į æĥ³ +é¢Ħ è§ģ +æ¼Ķ 示 +ä¸Ģ缴 没æľī +è·Ł 她 +对çħ§ æ£ĢæŁ¥ +ç° ¿ +ä¸ĵ å¿ĥ +è®® äºĭ +åīį 端 +åį¡ å°Ķ +è¨Ń å®ļ +设置 äºĨ +å©ļ 纱 +åľ¨ åĽ½å¤ĸ +åı³ 侧 +è³¼ çī© +å¥ĩ èij© +å¢ŀåĬł åĢ¼ +好 è¿IJ +åĽ½éĻħ æľºåľº +ä¸ĭ 称 +缮åīį 为æŃ¢ +ç¥ŀ ä»Ļ +å®ĥ åı¯ä»¥ +æ¾Ħ æ¸ħ +èĥ½ 使 +游 åĩ» +游åĩ» éĺŁ +åĩ ¹ +ä¸įè¦ģ åĨį +åĨ³ èĥľ +åĨ³ æĪĺ +æĭ ½ +缼 åħ¸ +å¾Ī好 åľ° +æľĢ ç¾İçļĦ +åĥ ļ +å·´ åŁº +å·´åŁº æĸ¯åĿ¦ +æľĢ éĢĤåIJĪ +é«ĺ èģĮ +ä¿Ŀ å§Ĩ +æİĪ æ¬Ĭ +说åĪ° è¿ĻéĩĮ +æݨ å¼Ģ +çİĩ è¾¾ +ä¸īåĪĨ ä¹ĭä¸Ģ +管çIJĨ ä¸Ńå¿ĥ +交 æ±ĩ +森æŀĹ åħ¬åĽŃ +å¾Ģ ä¸Ĭ +éªij è¡Į +æį® æѤ +纽 带 +ç» ŀ +ä¸ī æĸ¹ +æĦıä¹ī ä¸ĬçļĦ +æݨ è¿Ł +å¤ļæł· æĢ§ +æĥ³ èµ·äºĨ +æİĴåIJį 第 +å·¨ é¢Ŀ +æĿŁ ç¼ļ +å®ī å®ļ +äºĭ 實 +çļĦ æĦ¿æľĽ +è£ħå¤ĩ åĪ¶éĢł +人 å±ħ +人å±ħ çݯå¢ĥ +å¿ĺè®° äºĨ +该 游æĪı +楼 ä¸Ĭ +å¼Ģ ä¼ļ +æģ ³ +åıĭæĥħ éĵ¾æİ¥ +ç¡ Ĵ +ç»ĻäºĪ äºĨ +åģı 好 +åĵ ī +交éĢļ å®īåħ¨ +éĽ Į +æ²» çĹħ +è§īå¾Ĺ å¾Ī +衬 è¡« +å¿ĥ æĦ¿ +æ´ŀ å¯Ł +æ°ij æ£Ģå¯ŁéĻ¢ +æıIJ çĤ¼ +è¦ģ è¿Ľä¸ĢæŃ¥ +驾 车 +æĻ® æĥł +æķ ĸ +ç¦ı éŁ³ +éĢģ è¾¾ +è§ĦåĪĴ 设计 +æīĭ å¥Ĺ +å®ī ä¿Ŀ +è¿ĺä¸į å¦Ĥ +åīį è¿° +æłĩ è®° +ç´§ æİ¥çĿĢ +æ§ IJ +深深 åľ° +满满 çļĦ +æĺ¥ è¿IJ +æĹ¥ 产 +çĪ± æĬ¤ +åħ¨ æĹ¥ +åħ¨æĹ¥ åĪ¶ +转 åĬ¨ +ç¥Ń ç¥Ģ +ä¹° ä¸ľè¥¿ +对 æľªæĿ¥ +æ¶Ī失 äºĨ +åļ´ éĩį +ä¸ī æĿ¡ +éħ¸ 奶 +éĽĨåĽ¢ èĤ¡ä»½ +西 è·¯ +åıª å¾Ĺ +éĢģ åİ» +çĭł æĬĵ +åĪ©çĶ¨ çİĩ +ä¸ĭ åij¨ +å¥ĭ æĪĺ +æĺ¥èĬĤ æľŁéĹ´ +è´Ł 责任 +æĺĤ è´µ +å°¾ å·´ +ç¯ĩ æĸĩç«ł +åħ ® +è®Ĭ æĪIJ +å¹ ¹ +çĻ» éĮĦ +ä½ Ī +å·¥ åĮł +åĵªæĢķ æĺ¯ +åıį åĵį +ç§ ĥ +åĩº 轨 +æĹ¥ åĨĽ +åIJį èªī +æķı éĶIJ +æľįåĬ¡ æ°´å¹³ +çħ§ å°Ħ +ä¼Ĭ æĭī +ä¼Ĭæĭī åħĭ +åĨħ éĺģ +èĬĴ æŀľ +ä¸ĩ åĪĨ +éĢĢ 款 +缴æĴŃ éĹ´ +æĭ¿ åĪ°äºĨ +å°İ èĩ´ +空æ°Ķ ä¸Ń +客æĪ· æľįåĬ¡ +è¿IJ åĬ¿ +ç»ĵ çŁ³ +ä¸į å¿ħè¦ģçļĦ +èĥ¶ åĽĬ +çIJĨ ä¼ļ +æĬ½ åĩº +空æ°Ķ è´¨éĩı +æ¯ķ ç«Łæĺ¯ +åĨ· æ¼ł +ä¸Ģ å¦Ĥ +ä¸Ģå¦Ĥ æĹ¢ +ä¸Ģå¦ĤæĹ¢ å¾Ģ +æĤ£ çĹħ +åĬł æĮģ +èµŀ åĬ© +é« ® +åij½ ä¸Ń +æĦıä¹ī ä¸Ĭ +ä¸į èĪį +åģļ 梦 +æīĵ æī« +æĺŁ åħī +æĸŃ è£Ĥ +åħ¨ å¥Ĺ +è£ģ å®ļ +马 åħĭæĢĿ +骨 骼 +ä¸Ģ è·¯ä¸Ĭ +å®ļ æŶ +å·¥ç¨ĭ æĬĢæľ¯ +å½¼ å¾Ĺ +æ±² åıĸ +ä¸Ģ è§Ī +åIJµ æŀ¶ +ä¿Ĺ 称 +æłª æ´² +åºŁ æŧ +è¡Į æĺŁ +åıijçĶŁ åıĺåĮĸ +é¦ĸ ä»ĺ +åįģåĪĨ éĩįè¦ģ +æĬĬ è¿ĻäºĽ +ç¥ŀ å·ŀ +æıIJä¾Ľ åķĨ +æ¥ · +å± İ +çĬ¶ åħĥ +åŁİ å¢Ļ +çľĭ ä¸Ģçľĭ +çĶŁäº§ èĥ½åĬĽ +åŁºæľ¬ä¸Ĭ éĥ½ +æīĵ æī° +åĪĿ 次 +åĩº 示 +åħ¶ä¸Ń ä¸Ģ个 +çĶŁæĢģ ç³»ç»Ł +æīĭ æİĮ +æµİåįĹ å¸Ĥ +åľĭ åħ§ +æŃ£ åĢ¼ +å¹¾ ä¹İ +æݨèįIJ éĺħ读 +è¿Ń 代 +è°ĥ ä¾ĥ +饮 åĵģ +å¢Ļ ä½ĵ +åıĺ çİ° +äºĨ 好 +äºĨ好 åĩł +ä¸į çķĻ +çĪ ² +å°½ æĹ© +æŃ£åľ¨ è¿Ľè¡Į +åĩº éĻ¢ +æĿĢ 害 +æıIJ 款 +åıijå±ķ 空éĹ´ +åīį 身 +ä¸įæĸŃ å¢ŀ强 +æ·± å±Ĥ次 +容 纳 +éĤ£ 份 +å·¥ä½ľ æķĪçİĩ +æľ¬ åĽ½ +失 èIJ½ +æŃ£ åĽłä¸º +èĬĤ æ°´ +ä¸ĭ ä¸Ģ代 +çłĶåıij ä¸Ńå¿ĥ +ä¸į çIJĨ +å®Į 好 +ä¿ĿæĬ¤ åĮº +ç»ĵæŀĦ è°ĥæķ´ +å¥ł å®ļ +宣 称 +éĺ» æĮ¡ +æĴ¤ 离 +ä¸į æĸ¹ä¾¿ +åĴ ķ +ç¬ijäºĨ ç¬ij +çݯå¢ĥ 污æŁĵ +ä½ı æĪ· +ç»Ŀ ç¼ĺ +éĻ¤ å°ĺ +é«ĺ å°ļ +æĢİä¹Ī åı¯èĥ½ +éĿ¢ èī² +åķĨ æ¥Ń +çĸ ¹ +èµĦæºIJ ä¼ĺåĬ¿ +è¾ĸåĮº åĨħ +èĢĢ çľ¼ +æij§ æ¯ģ +ä¸ĸçķĮ ç»ıæµİ +å¼ķ æĿ¥ +ä¸Ģ åĪĻ +æĭĩ æĮĩ +æĬµ 御 +éĽ į +åĩĨå¤ĩ å·¥ä½ľ +çıł ä¸īè§Ĵ +ç¨Ģ åľŁ +èİ·å¾Ĺ æĦŁ +æĪIJåĬŁ çİĩ +ç½ij 约 +ç½ij约 车 +èĦ IJ +æķ¬ ä¸ļ +éĩij ä»· +ç²¾ é«ĵ +ä¹° 车 +åħ³ åı£ +åĨį å¤ļ +æŀģ åĵģ +åIJĦ 家 +举æĬ¥ çĶµè¯Ŀ +èļ Ĭ +æĸ¹ å½¢ +ç§ijæĬĢ æĪIJæŀľ +æľĢ好 æĺ¯ +éĹ® åĢĻ +红 éħĴ +åĽĽ ç§į +ç¿Ĵ æħ +ç¿Ĵæħ £ +åŀ ¦ +éĤ£ åıª +é¢Ĩ æĤŁ +çľ¼ éĥ¨ +æ³° å®ī +ä»» æľŁ +磨 æįŁ +æĽ¿ æį¢ +åħ¸ 礼 +符åIJĪ æĿ¡ä»¶ +è¿ĺæľī ä»Ģä¹Ī +åħ±äº« åįķ车 +åı¯ åĪĨ为 +åŃ£ åIJİ +åŃ£åIJİ èµĽ +举èİŀ å¸Ĥ +å¿ĥ æĦı +æīŃ æĽ² +ä½ľä¸º ä¸Ģç§į +è¿Ļ éĥ¨åĪĨ +åıĤä¸İ åĪ° +ç½ij çIJĥ +實 çı¾ +ç»Ħ è£ħ +åIJij å¤ĸ +å·¥ä½ľ æĸ¹æ¡Ī +åįģ æĿ¡ +課 ç¨ĭ +颤 æĬĸ +åĵ © +éĤ® å¯Ħ +äº ¢ +åħį è²» +ç§ ¤ +åºĶæĢ¥ 管çIJĨ +åĽĽ äºĶ +éºĴ éºŁ +å¾Ĵ æŃ¥ +è¨ĺ å¾Ĺ +çĴ IJ +æĺ¯åIJ¦ ä¼ļ +æĦıè§ģ åıįé¦Ī +éļ¾ æĢª +çª į +交 æİ¥ +两 åįĥ +æĩī çĶ¨ +æľŁ éĸĵ +æIJ¬ åĪ° +è®® é¢ĺ +碧 æ¡Ĥ +碧æ¡Ĥ åĽŃ +åģļ çĶŁæĦı +éĻĽ ä¸ĭ +è· ĭ +èĢģ人 家 +带 åĽŀ +æŀ¸ æĿŀ +è¡Į éķ¿ +åĨħ容 ç®Ģä»ĭ +æ¢ ¢ +æĮĩ æݧ +éĩį çĹĩ +ç½ijåıĭ 们 +çı¾ 代 +ç±» 产åĵģ +å¥Ķ æ³¢ +æ¸ º +ç²ī ç¢İ +è¿Ļ åıªæĺ¯ +æ£Ģå¯Ł æľºåħ³ +é½ Ĭ +æĪ¿ 秣 +å¾· æĭī +å²ģ 以ä¸Ĭ +纯 åĩĢ +åĪĨå¸ĥ åľ¨ +èĥ½ å¾ĹåĪ° +ä¸į å°½ +ç«ŀ ä»· +çļĦ 带é¢Ĩ +çļĦ带é¢Ĩ ä¸ĭ +ä¸Ńèį¯ æĿIJ +æĿij éķĩ +ä¸įåı¯ éģ¿åħį +éľ² 天 +å°ı å§ijå¨ĺ +çī© 件 +èijĹä½ľ æĿĥ +æĭĺ çķĻ +éĥ½ è§īå¾Ĺ +æĽ² æĬĺ +æ·»åĬł åīĤ +åı¬ åĽŀ +æīİå®ŀ æİ¨è¿Ľ +æĬĦ è¢Ń +åĮĸ 身 +缴 èIJ¥ +ä¹Ł å¸ĮæľĽ +èį£èªī 称åı· +åįĸ ç»Ļ +æľī ä¸įåIJĮçļĦ +å¥ĩ çī¹ +éĥ½ 认为 +å¦ ŀ +æĪIJéķ¿ 为 +辩 æĬ¤ +主 æķĻç»ĥ +æ³ķå¸Ī èģĮä¸ļ +æ¤į åħ¥ +ç´¢ å°¼ +åIJ¬ è¿ĩ +ä¹łæĥ¯ äºĨ +夺 åıĸ +éŁ ĵ +æľ¬è´¨ ä¸Ĭ +æİ¥ åĬĽ +äºij 端 +è¦ģ åģļ好 +è·¯ çģ¯ +åįıåIJĮ åıijå±ķ +æľī å¾ħ +æ°´ åŁŁ +æIJľçĭIJ é¦ĸ页 +è´¨éĩı å®īåħ¨ +åįģäºĮ äºĶ +åĵ® åĸĺ +èĵ¬åĭĥ åıijå±ķ +åIJį 声 +身 亡 +çİĭ åºľ +åİŁåĪĻ ä¸Ĭ +çĥĺ å¹² +éģĹ æ¼ı +éĿ¢ 缮 +åĽ½ ä¼ļ +ä¸Ģ缴 éĥ½æĺ¯ +æľīä¸Ģ ä½į +éħį æľī +éĻª çĿĢ +ä¼ģ åĽ¾ +æĮī ä¸ĭ +èĵĿ åĽ¾ +æ© ĺ +大å¤ļ æĺ¯ +辩 论 +æĹĭ å¾ĭ +æĬ¥ éĢģ +æĿ¡ è§Ħå®ļ +åĬ¨ éĿĻ +åĮĪ 奴 +æĭľ 访 +ä¸Ģ åĪĢ +ä»ĸ çŁ¥éģĵ +主 æĿĥ +ä»ĸ æĽ¾ +æĴŃ ç§į +å£ģ åŀĴ +çī¢è®° 使åij½ +åľ¨è¿Ļ æĸ¹éĿ¢ +æīĭ èħķ +æĶ¯ æŀ¶ +ä¾Ĩ èĩª +éĩį å¡ij +å¤ļ å±Ĥ次 +ä»ĭ è´¨ +éĿ¢ åŃĶ +æ½® 湿 +åİ¿ åŁŁ +游æĪı å½ĵä¸Ń +å£ ŀ +åĪĹ åĩº +èµĽ åĮº +å¤ļ åįĬ +éĩįçĤ¹ å·¥ä½ľ +æĪij们 å¿ħé¡» +æŁı æŀĹ +é²ģ èĥ½ +æĸ½ å±ķ +åIJĦ åĮº +åħį ç¨İ +èµĽ åIJİ +æľĢ éĩįè¦ģ +ä¸Ģ个 好çļĦ +è¿Ŀæ³ķ è¿Ŀè§Ħ +äºĨ解 æĽ´å¤ļ +æķ¬ 请 +ç¬ijçĿĢ 说 +ä¸įæĸŃ åıijå±ķ +æijĦå½± å¸Ī +以 éĺ² +çĤ¸ å¼¹ +声 åĵį +ç¤ ģ +æĩ ¿ +èĪĨ æĥħ +èĩªçĶ± è´¸æĺĵ +æķı æį· +ä¸ī大 éĺ¶æ®µ +èĭ Ķ +æĹº åŃ£ +ä¸į 满æĦı +微信 åı· +ä¿® 为 +çł´ è£Ĥ +éĢĥ 离 +æ¯ı èĤ¡ +è¾¾ ä¸įåĪ° +æ¯ıå¹´ éĥ½ +çģ¯ 笼 +æѤ åŁºç¡Ģä¸Ĭ +åĥı 个 +åĪĨ 娩 +æĻ ¾ +ä¸į èĩ³äºİ +红 线 +误 解 +举 è·¯ +æ·® å®ī +产 åѦ +产åѦ çłĶ +èī¾ æ»ĭ +èī¾æ»ĭ çĹħ +åīįæıIJ æĺ¯ +æ¯ı ä¸Ģ天 +ä¸ĥ 大 +æłij åı¶ +èµ° å¾Ĺ +è¿Ļ 两ç§į +æİı åĩº +æİ IJ +é¢Ĩ导 èĢħ +ä¸Ģ æľµ +个å¤ļ æľĪ +ä¸Ń åħ³ +ä¸Ńåħ³ æĿij +课åłĤ æķĻåѦ +大 åĴĸ +éģĭ çĶ¨ +è¯ļ æĦı +ç»Ħ åĽ¾ +è¯ķ çĿĢ +ä¹Ķ æ²» +è¿ĺ ä¸įæĺ¯ +æľī æĽ´å¥½çļĦ +åIJİ å¤ĩ +æĸ°çĶŁ åĦ¿ +æ°Ķ è¡Ģ +æ²¥ éĿĴ +å±ı éļľ +æ¥Ń åĭĻ +æĪij 以为 +éķ¿ 缸 +èĢģ çĪ¸ +éķĩ æ±Ł +æľºæ¢° 设å¤ĩ +ä½Ĩæĺ¯ å¦Ĥæŀľ +åĿļå®ļ ä¸į +åĿļå®ļä¸į 移 +åĨ² éĶĭ +ç®Ģ缴 æĺ¯ +åĤ¨ èĵĦ +纯 çĶµåĬ¨ +漫 æŃ¥ +举 èµ· +æģ¶ æĢ§ +è¨ĺ éĮĦ +èģĮèĥ½ éĥ¨éŨ +åħ¨ éķ¿ +鼻 è¦ĸ +ä¹³ èħº +ä½ķ å¤Ħ +æ¶Ī æŀģ +æŃ£ å¤Ħäºİ +å®ī å®ģ +æĪIJ éķ· +åıĻ è¿° +æºĥ çĸ¡ +ä½Ĩ çİ°åľ¨ +女 æĺŁ +å©´ å¹¼åĦ¿ +æĬķ èŀįèµĦ +éĹ® éĹ® +æıŃ å¼Ģ +è¯ ı +åIJį å½ķ +èĺij èıĩ +åIJĬ 顶 +æ¹ĸ åĮº +åįĸ åľº +建 ç¯ +å»ºç¯ ī +èİ ½ +åIJ¬ åIJ¬ +ç«ŀäºī ä¼ĺåĬ¿ +åĩº ä»» +æľī 两ç§į +橱 æŁľ +è¤ ª +è¯ķ åį· +ç»ıæµİ æĬĢæľ¯ +æ·± å±Ĥ +éĩįè¦ģ åĨħ容 +é£İ æݧ +çĬ¶æĢģ ä¸ĭ +éĥ¨ éĸĢ +广 æ±½ +è§Ĥ æij© +éģĹ çķĻ +转 è´¦ +æĮģ ä»ĵ +æĢ» 计 +åľĺ éļĬ +æĪ¿ 举 +éĺĢ éŨ +åħ¬ åħ³ +åħ³ åĪĩ +èĤ ĺ +æķ¸ æĵļ +ä¸ī åįģå¹´ +è§ģè¯ģ äºĨ +å± Ĩ +çģ° å°ĺ +æ¦ľ é¦ĸ +è¦ĨçĽĸ çİĩ +ä»Ļ 女 +çĶŁäº§ æĢ» +çĶŁäº§æĢ» åĢ¼ +æĪ¿ è´· +æ±Ł åĮº +åħħçĶµ æ¡© +çĻ¾ åIJĪ +確 èªį +转 移åĪ° +éĥ½ æĹłæ³ķ +纪念 é¦Ĩ +çŃ¾ç½² äºĨ +并ä¸į å¤ļ +æĮ ł +ä¸į太 好 +ä¸ĸ 代 +误 导 +é«ĺå³° 论åĿĽ +åħ¼ 容 +龸 æ°Ķ +æĿ¥ 访 +æīĢ 带æĿ¥çļĦ +æĺ¯ä¸Ģ éĥ¨ +æĻļ é¥Ń +åİĨ 代 +åIJ¦ åīĩ +ä¹ħ ä¹ħ +æľīæķĪ æľŁ +诱 åıij +æĢ» èµĦ产 +æľ¬èº« å°±æĺ¯ +çĶŁäº§ åİĤ家 +æŶ 髦 +èĢIJ çĶ¨ +ä»İå°ı å°± +æĿ¡ 约 +èĭ± åĭĩ +ä¿Ĺ è¯Ŀ说 +寺 åºĻ +å¿ĥçIJĨ åģ¥åº· +ä»Ģä¹Ī äºĭæĥħ +æ±ī åŃĹ +çķĻ ä½ı +åįĹ è·¯ +ä¸ī 项 +丢 äºĨ +æĥ³ åĪ°äºĨ +çŃ¹ éĽĨ +éĻĦåĬł åĢ¼ +西 è£ħ +ä¹ĭ ä½ľ +åģļçļĦ äºĭ +çķ¶ æĤ¨ +çķ¶æĤ¨ åľ¨ +é¦ĸ 款 +ä¸įåľ¨ ä¹İ +å·¥ç¨ĭ æĸ½å·¥ +éļIJ éļIJ +åıĺ 身 +沿 éĢĶ +æĤł æĤł +ä¿Ŀ æļĸ +çĶŁæ´» åŀĥåľ¾ +渤 æµ· +æѦ ä¾ł +女 主è§Ĵ +举 ä¾ĭ +æ ·¨ +çĻ½ é¢Ĩ +è£Ļ åŃIJ +è¿Ķ è¿ĺ +è¿Ī åĩº +é¾Ļ éŨ +ç»ıæµİ ä½ĵ +æĶ¶ å®ĺ +çķĮ éĻIJ +è·³ åĩº +åįĩ åĢ¼ +绵 éĺ³ +çĸ¤ çĹķ +çľĭ æ¸ħ +æĭĴ çµķ +è¥Ħ éĺ³ +课 å¤ĸ +åŃIJ åŃĻ +æŃĮ è¯į +æĪIJ åIJį +溶 液 +åĦĴ 家 +åķĨä¸ļ åĮĸ +辨 åĪ« +å¤ļ è¾¾ +ç½ij åºĹ +ä¹Ŀ 大 +ä¹Ŀ大 ç²¾ç¥ŀ +æѤ 举 +è¿ŀ è½½ +ä¸Ģ åĢĭ人 +èī² æ³½ +涵çĽĸ äºĨ +è¦ı åĬĥ +åĽ½ æĥħ +åį«çĶŁ åģ¥åº· +积æŀģ åĵįåºĶ +æĭ Ļ +åĪ¶ åĬ¨ +æĥ³è±¡ åĬĽ +çļĦ ä¹IJ趣 +å¼łå®¶ çķĮ +å´ İ +éĩį åŀĭ +å¤ĸ å¢Ļ +æĶ¾ åѦ +è®¤çľŁ åŃ¦ä¹ł +è´¬ åĢ¼ +æ³ķ æ¡Ī +æĬ¤èĤ¤ åĵģ +éĻ·åħ¥ äºĨ +请 æĤ¨ +åŀ ¢ +æķĻèĤ² èµĦæºIJ +交æĺĵ å¹³åı° +æŶ è£ħ +ä¼łæŁĵ çĹħ +æ¹ĸ æ³Ĭ +èµĦ 管 +åݨ å¸Ī +éĹľ éį +éĹľéį µ +åĵĪåĵĪ åĵĪ +çĽĹ çªĥ +çĶľ ç¾İ +åºĦ åĽŃ +缮åīį å·²ç»ı +è¾¹ ä¸Ĭ +çģ« èĬ± +æĬ¥ è®°èĢħ +æģĭ æĥħ +ç´§ åĩij +æ°´ æµģ +è¿Ļæĺ¯ æĪij们 +æ³¥ åľŁ +æĽ¾ ä»» +æĸ¹ è¨Ģ +åij¨ åħŃ +åı· 楼 +ä¼ij åģĩ +误 ä¼ļ +åĽ½ åĢº +åīį å¤ķ +两 å¼ł +éĹ « +éŃĶ 鬼 +æĬĬ æĮģ +èĬĤèĥ½ çݯä¿Ŀ +æ¸ħæ´ģ èĥ½æºIJ +èĤ¥ æĸĻ +é«ĺ é¢ij +å°± æľīäºĨ +交 ä¼ļ +没 éĴ± +éĽħ æĢĿ +è¦ģ åıĬæŶ +åŁ¹åħ» åѦçĶŁ +欣 åĸľ +çĥŃæ°´ åĻ¨ +é¾Ļ æ¹ĸ +äºĮ 楼 +æĸ°æµª è´¢ç»ı +æĸ° åĬ¨èĥ½ +èµ£ å·ŀ +æĭ³ 头 +æµģ åIJij +ä¹Łæĺ¯ å¾Ī +åıij åĶ® +ä¸Ń åIJ«æľī +åIJĵ å¾Ĺ +å·¨ æĺŁ +æĹł æīĢè°ĵ +æ¯Ľ åŃĶ +åħ¬åħ± 交éĢļ +çĤİ çĥŃ +èµ· èįī +åĬłçĽŁ åķĨ +说 ä¸įåĩº +大åѦ æ¯ķä¸ļ +å·¥ä¸ļ åĽŃ +éłĺ åŁŁ +åºĨ åħ¸ +æµģ 产 +èģ² éŁ³ +ä¼¼ä¹İ æĺ¯ +è´§ æºIJ +æ·± åĪĩ +æ²»çĸĹ æĸ¹æ³ķ +èµĦæºIJ éħįç½® +網 åıĭ +çĶ £ +äº ¥ +躲 åľ¨ +社 ç§ij +è»Ł é«Ķ +女 è£ħ +æŃ¡ è¿İ +综åIJĪ å®ŀåĬĽ +æł¼ å°ĩ +åħļåı² åŃ¦ä¹ł +æľĢ åŁºæľ¬ +æľĢåŁºæľ¬ çļĦ +çľĭ æľĽ +åıĹ è´¿ +ä¸įä»ħ èĥ½ +ä½ķ å¿ħ +ä¸Ģ个 å°ıæŶ +ç¾ Į +æĭĽ æĶ¶ +çĤĴ èĤ¡ +æĿij å¹²éĥ¨ +缸 çĪ± +æ½ľ èĥ½ +ä¹ į +æŶ è¾° +欣 æħ° +éĵ¶ è¡Įä¸ļ +çĭŃ çªĦ +éĩįçĤ¹ é¢ĨåŁŁ +çİ°å®ŀ çĶŁæ´» +éĮ¯ 誤 +æĸ° è§Ħ +滥 çĶ¨ +æŶ ä¸į +æŶä¸į æŶ +帳 èĻŁ +ç¨Ģ 缺 +åIJij 举 +ä¿Ŀåģ¥ åĵģ +çıŃ éķ¿ +äºĴ åĭķ +笼 罩 +æ½ Ľ +æļĸ å¿ĥ +è½° çĤ¸ +åºĨ 幸 +è²Į ä¼¼ +æĵ º +èĢIJ 磨 +ä¸ĵä¸ļ 人士 +ä¸ĢèĪ¬ éĥ½æĺ¯ +æ¼³ å·ŀ +åħ¨ èĩªåĬ¨ +å½ķ çĶ¨ +大 è·Į +æľīæķĪ æĢ§ +èĩª åĭķ +ä¸ī个 æĸ¹éĿ¢ +港 åĮº +ä¿¡ 貸 +éĢļ è¯Ŀ +é«ĺ 涨 +æ³Ħ æ¼ı +éħį ä¸Ĭ +åħļ å·¥å§Ķ +被 认为 +被认为 æĺ¯ +ä¸įä¼ļ åĨį +è°ĥ åīĤ +åıĤ èĤ¡ +èĦ± åıij +å¿ł å®ŀ +åĨħ åĪĨæ³Į +ç¹ģ å¿Ļ +åıĮ åĪĽ +é©» æĿij +åĪĴ ç®Ĺ +éģİ ä¾Ĩ +åľ£ ç»ı +èıľ 鸣 +æĭ¼ å¤ļå¤ļ +ä¸ŃåĽ½ 汽车 +çĥŁ èįī +缴 æµģ +äºĨä¸Ģ åı£æ°Ķ +ä½İ æĪIJæľ¬ +æī¾ åĽŀ +èĩª åįij +總 æĺ¯ +æĸĩåĮĸ åĪĽæĦı +天 æ²³ +樱 æ¡ĥ +éªij åħµ +éĩĮéĿ¢ æľī +çİ ® +èĥ½ æī¾åĪ° +éĢĥ è·ij +åĪĩ å°Ķ +åĪĩå°Ķ 西 +以ä¸ĭ æĺ¯ +å²³ éĺ³ +çļĦ æ¦Ĥçİĩ +æĬµ åĪ¶ +å¸Ī äºĭåĬ¡ +å¸ĪäºĭåĬ¡ æīĢ +åĩĨ æŶ +屬 æĸ¼ +订 è´Ń +åįłæį® äºĨ +ä¸Ń éĢĶ +å° ĭ +é»ij 马 +åİ¿ åħ¬å®īå±Ģ +ä¸ĥ æľĪ +èī² ç´ł +å¿ĥèĦı çĹħ +æŶ éĻIJ +æ¯į åħ¬åı¸ +å¹ķ åIJİ +ä¸Ĭ æ¦ľ +åĢ¾åIJij äºİ +纸 ä¸Ĭ +æ¡ ĵ +éĽĨä½ĵ ç»ıæµİ +æĥħ å¢ĥ +è¦ģ åģļåĪ° +ç©į 極 +åıª æĢķ +æ¹ĺ 西 +çļ± 纹 +åħ¨ åľĭ +çĦ¡ è«ĸ +好 æĦŁ +åįķ ä»· +è¿Ľç¨ĭ ä¸Ń +æĺĨ ä»ij +åĪĽ 客 +åħħ æĸ¥ +åħĪ æĬĬ +该 æĢİä¹ĪåĬŀ +åĵģ å¾· +åħ¨éĿ¢ åıijå±ķ +è¨Ī åĬĥ +æĢ» å·¥ä¼ļ +ä½Ľå±± å¸Ĥ +æĬĹ è¡¡ +å¼Ģ åľº +éĴ± å¸ģ +åıĭ 们 +å«ī å¦Ĵ +ç´¢ èµĶ +è®Ĭ åĮĸ +æĮ¤ åİĭ +æĮij è¡ħ +çŃī ä¸Ģæī¹ +æĿ¨ 欢 +ä¸ĵ家 åѦèĢħ +èĥ½ è¾¾åĪ° +èµ° è¿ij +è´«åĽ° åľ°åĮº +éĻIJ æľŁ +ä¸į 平衡 +åĽ½åĨħ å¸Ĥåľº +èµĽ åľº +éħį èµĦ +è¦ģ èĢĥèĻij +ä¸ĩ åı° +æľĪ æľ« +éĶ ¥ +åŃ « +æݥ触 åĪ° +åĩº 产 +æķĻ åѸ +ä½ľ å¼Ĭ +çļĦ æľĢåIJİä¸Ģ +ä¿ĥ æĪIJ +åIJ¸ åıĸ +æ½ľ èīĩ +被 éªĹ +è¾ĵ äºĨ +çĭIJ çĭ¸ +åįĩ éĻį +è¿ĻäºĽ ä¸ľè¥¿ +æĬķèµĦ åŁºéĩij +çĶŁçī© åѦ +ç½ij绾 èIJ¥éĶĢ +åIJij è®°èĢħ +èįī åľ° +æĢ ¯ +æľįåĬ¡ èĥ½åĬĽ +éĥģ éĹ· +åįķ åĵģ +å¾Ĺ 罪 +æĺĵ äºİ +个å¤ļ å°ıæŶ +éĩį ä»» +ä¸Ĭ å®ĺ +æľ¬ éĩij +çı¾ åł´ +溢 ä»· +æĺŁ è¾° +æ´»åĬ¨ çİ°åľº +丹 麦 +å¸Ŀ çİĭ +æŁ¥ æĺİ +åŃĺåľ¨ äºİ +é¦Ļ æ°´ +æĬ½ æ£Ģ +å®ŀéĻħä¸Ĭ æĺ¯ +æĸ° å¾ģç¨ĭ +è´¢åĬ¡ 管çIJĨ +æİ Ľ +åĨľ åİĨ +éĥ½ èĥ½å¤Ł +éĤ¯ éĥ¸ +羣 實 +ç» Ĭ +åĨµ ä¸Ķ +ç½® 身 +ç¥Ī 祷 +çĿģ å¼Ģ +æĮĩ çĤ¹ +å¼Ģ æľº +西 å®ģ +åĮĹ çº¦ +积 æ°´ +åĩº åĬ¨ +åıijå±ķ 模å¼ı +转 æĬĺ +èĢĥ çĤ¹ +æľī ç½ijåıĭ +è´«åĽ° æĿij +æĪij们 çŁ¥éģĵ +åĪĨ éĶĢ +å±± èĦī +æ¯Ķ æĭŁ +ä¼° ç®Ĺ +æĶ¹ 建 +壮 è§Ĥ +ç§ī æĮģ +æı ª +ç¦ Ģ +åĮĸåѦ åĵģ +ä¸ŃåĽ½ åĪ¶éĢł +ä¸Ģ æŀ¶ +æīį è¡Į +æĭĽ å¾ħ +åıĺ æį¢ +åīį 线 +幸 好 +è¿Ļæł· çļĦè¯Ŀ +å¿ĥ è¡Ģ管 +æĢ§ çĸ¾çĹħ +åħ¨ èĥ½ +åĪij 侦 +ä¿¡æģ¯ åıijå¸ĥ +æĺ¾ çĦ¶æĺ¯ +éĿĴ éĵľ +åIJĥ ä»Ģä¹Ī +çĶµ ä»· +æ³ķå¾ĭ è§Ħå®ļ +çħ ² +çĵ· åĻ¨ +èĤī ç±» +æıĴ åħ¥ +åĹ ľ +è¿Ł è¿Ł +ä¸ĢçĤ¹ éĥ½ä¸į +è¿ĺ åĮħæĭ¬ +èĪį ä¸įå¾Ĺ +æłĩå¿Ĺ æĢ§ +æľĪ 以æĿ¥ +ç³ĸ æŀľ +éĥ½ åºĶ该 +çݯå¢ĥ åį«çĶŁ +èĪª è¡Į +éĥij éĩį +ç½ij æĬķ +åįģ ä½³ +ç§ģ ä¸ĭ +æļ´ è·Į +åĬłå¿« åıijå±ķ +产åĵģ çłĶåıij +åĪĽéĢł åĩº +æĢ» è§īå¾Ĺ +åºķ çĽĺ +èķ Ĭ +åĩºå¸Ń ä¼ļè®® +主 æĿ¿ +æĹ¥æĻļ éĹ´ +å®ĺæĸ¹ å¾®åįļ +å¼ķçĶ¨ æĹ¥æľŁ +åī¯ æķĻæİĪ +çĶµåŃIJ 产åĵģ +è¡° éĢĢ +çķĻ åŃĺ +çģ« åĬĽ +çĴ § +çļ Ĥ +åħ¼ åħ· +éĩį è¿Ķ +é¢Ĩ çķ¥ +åĪĩ éĻ¤ +åĨįçĶŁ èĥ½æºIJ +å®ŀåľ¨ 太 +çIJĨ论 ä¸Ĭ +ä¸ī å±Ĥ +ä¸ĸçķĮ åIJĦåĽ½ +å®ľ æĺĮ +èĢ³ è¾¹ +宽 æķŀ +æ±ī æĹı +çĻ½ çĻ½ +è¿ĻéĩĮ éĿ¢ +çĶŁæ´» ä¹łæĥ¯ +èµŀ èµı +çĶ· 士 +ä¸Ń ä¿Ħ +车 祸 +åīĤ éĩı +éĻ¤ åİ» +å·¦ è¾¹ +çŃij çī¢ +çīĽ å¸Ĥ +家 åĬ¡ +åķ ĥ +ç½® æį¢ +ç´« å¤ĸ +ç´«å¤ĸ 线 +å¾Ģ åīį +åĬĽ åѦ +ç´§ è·Ł +缮çļĦ åľ¨äºİ +ç» ® +ç¥ Ĥ +宣 è¨Ģ +äºĮ æ°§åĮĸ +äºĮæ°§åĮĸ 碳 +æĹł ç¼ĺ +ç²¾ éĢļ +è¨ º +å¼ķåıij äºĨ +æľĢ åħĪ +æ´¾ é©» +ä¸į å¿į +æĪij çĪ¸ +å¹´ ä¸ĭåįĬå¹´ +æ·ĭ å·´ +没 éĹ®é¢ĺ +åºĹ åĨħ +è·Ł æĪij说 +çĶŁäº§ çĶŁæ´» +è§Ĥ æľĽ +æ¸ į +被 æī§è¡Į +被æī§è¡Į 人 +èĪ ľ +æİ º +ä¸Ģ ç§Ĵ +èįī åĿª +åij¼ åĴĮ +åij¼åĴĮ 浩 +åij¼åĴĮ浩 çī¹ +人æ°ij éĵ¶è¡Į +çĦķ åıij +è¯ģåĪ¸ 交æĺĵ +çķ Ķ +æľº èĥ½ +å¦ ¾ +æĻļ å¹´ +å·¥åķĨ èģĶ +åİŁ åŀĭ +è§Ĵ度 çľĭ +æĬ¥ 社 +è¯į æĿ¡ +躲 éģ¿ +éĩį åIJ¯ +å¤ķ éĺ³ +èĤ¡æĿĥ 转让 +åľ¨ ä¸Ģ +åľ¨ä¸Ģ æĹģ +社ä¼ļ åĮĸ +åıijå±ķ åİĨç¨ĭ +æĭĸ æ¬ł +使 èĢħ +ä¸İ åIJ¦ +æĸ° å±ĢéĿ¢ +ä»Ĭ天 æĪij们 +é½IJ èģļ +对 æĪij说 +éĢĴ 交 +æľª æĽ¾ +èİ Ĭ +éĸ ī +亲 æīĭ +è§Ĵ éĢIJ +æľī é»ŀ +ç¨İ çİĩ +ä½İ 声 +é»ĺ å¥ij +æĻ® æ³ķ +大 ä¸ĵ +第äºĮ 大 +ä½ı åĿĢ +æĶ¾ è¿Ľ +äºĮ æĪĺ +亲 身 +åĽº åĮĸ +ä¸ĭ 乡 +åħ³éĶ® æĬĢæľ¯ +åĽŀ æĥ³ +æĬ¥ åĪĬ +æ¶Ĥ æĬ¹ +èĹı çĿĢ +ç¥Ŀ æĦ¿ +åįĩ 温 +çĶļèĩ³ è¿ŀ +åħ¬åħĥ åīį +ç¾İ æĸ¹ +è¯ļ å®ŀ +æĹł åģ¿ +åīµ æ¥Ń +å°ıå¿ĥ 翼 +å°ıå¿ĥ翼 翼 +两 æīĭ +温馨 æıIJ示 +仿 羣 +æĥ ¶ +èĥ¡ åŃIJ +å·¥ä½ľ ç«Ļ +硬 çĽĺ +ç« ¿ +åĤ³ éĢģ +åħ¨ æł¡ +é²ľ æ´» +çĴĢ çĴ¨ +ç»ĵ å°¾ +æį¢ æĿ¥ +æĪ Ģ +ä½İ ä½į +ä¸ĩåħĥ 以ä¸Ĭ +åĬł åĪĨ +æݨä»ĭ ä¼ļ +çIJĨ èµĶ +å¾· å°Ķ +æĬĹ è®® +æ´ ¼ +åĸ § +åŁİ éĻħ +å¾Ī æ£Ĵ +人 æŃ»äº¡ +ä¼ļå±ķ ä¸Ńå¿ĥ +äºĴèģĶ äºĴéĢļ +èĸĦ èĨľ +éĩį é»ŀ +ç¦ģ æ¯Ĵ +åĨ· ç¬ij +大家 åı¯ä»¥ +é¦ĸ 缸 +è¿ij è·Ŀ离 +æµ® çİ° +ç§ĺ è¯Ģ +èµ· é£ŀ +æIJ ¶ +羣 åģĩ +æģ ķ +å°ı åºĹ +æ°ij çľ¾ +åıijå¸ĥ åħ¬åijĬ +侧 éĩį +å¾ĺ å¾Ĭ +æĢ Ķ +æª IJ +æķ° 缮 +åī¯ ç§ĺ书éķ¿ +两 åı¥ +éļIJ çŀĴ +åıĮ åıĮ +æīĭ æĦŁ +èij¡ 京 +éģĹ å¿ĺ +é¬ ¥ +è¿Ļ个 åľ°æĸ¹ +说 çļĦè¯Ŀ +å·¡ åĽŀ +è¿Ŀ ç«ł +æī¾ å·¥ä½ľ +æĶ¯ çIJĥéĺŁ +裡 éĿ¢ +æĺ¾ç¤º åĩº +èĩ³ å°Ĭ +两 级 +åīį 段æŶéĹ´ +çĺ¦ èº« +èĤ¢ ä½ĵ +æ¯į 親 +æīĭç»Ń è´¹ +汽车 è¡Įä¸ļ +æİ© çĽĸ +æݧèĤ¡ éĽĨåĽ¢ +åı£ å¾Ħ +æĶ¿çŃĸ æݪæĸ½ +æµ· 绵 +åħ¨ éķĩ +äºĭ åħ³ +å¸Ń æī§è¡Į +å¸Ńæī§è¡Į å®ĺ +éĤ£ 次 +åı¯èĥ½ åĩºçİ° +ä¸Ńå¿ĥ åŁİå¸Ĥ +ç¿» 身 +ä¹Ł ç®Ĺ +ä¾µ çķ¥ +åĸĩ åıŃ +æ¯ı次 éĥ½ +è§ ħ +éĻ¢ éĻ¢éķ¿ +å§ĭ äºİ +èѦ åĬ¡ +èį¯ æĿIJ +å±ł æĿĢ +æľ¬èº« å°± +éļıæŶ éļı +éļıæŶéļı åľ° +åĶ® åįĸ +æĹłäºº 驾驶 +é¢ ħ +åĵģ 質 +åĺ² ç¬ij +è·ij åİ» +åħĭ éĩĮæĸ¯ +çķ¸ å½¢ +ä¿® 饰 +磩 éĺµ +éŁ³ä¹IJ ä¼ļ +æŁ³ å·ŀ +é½ ¡ +ä¼ļ è°Ī +æŃ£ çīĪ +ä¹Ł åIJĮæł· +æļ§ æĺ§ +è¡ĮæĶ¿ éĥ¨éŨ +ä¹ĸ ä¹ĸ +èĤ¤ èī² +æŶ ä»» +羣 åĪĩ +æľĪ ä¸ĭ +æľĪä¸ĭ æŬ +举æĸ¹ è´¢å¯Į +è£ħä¿® åħ¬åı¸ +éĢĢ è¿ĺ +åĭĺ å¯Ł +åĵ¥ 伦 +åĵ¥ä¼¦ æ¯Ķäºļ +çĭ¬ ä¸Ģ +çĭ¬ä¸Ģ æĹł +çĭ¬ä¸ĢæĹł äºĮ +è°ĥ åij³ +åİĭ è¿« +åħ¨çIJĥ æľĢ大 +åī¯ æł¡éķ¿ +æĽ´ ä½İ +åĪĨéĴŁ åIJİ +åĽŀ ä¾Ĩ +åĪ¶ åīĤ +åijĬè¯ī 大家 +çĤ¹ éĴŁ +åįģä¸ī å±Ĭ +åij¨ åĽĽ +è¿Ļæł· ä¸Ģ +è¿Ļæł·ä¸Ģ æĿ¥ +èĭ Ł +æľĽ åİ» +æĪIJ è¯Ń +å½ĵ åį³ +ç¬ij 声 +ä¹ĭ åĬ¿ +åĪijäºĭ æ¡Ī件 +æĮĤ çĿĢ +ä½ķ ç§į +å°ı 游æĪı +åĽ½å®¶ æĪĺçķ¥ +åĨ· åĨ· +å®ľ 宾 +æIJº ç¨ĭ +è¶ĭ äºİ +åıį çľģ +常 说 +ä¸ĩ æĪ· +åĥµ å°¸ +åįĥä¸ĩ åĪ« +åıijçİ° éĹ®é¢ĺ +åı¯ çŁ¥ +éŨæĪ· ç½ijç«Ļ +åģ¥åº· 产ä¸ļ +åı³ è¾¹ +æµ· è¿IJ +è¿ij ä¹İ +åĮ» æ²» +æĢ» ç®Ĺ +ä¸Ģ åĪĨéĴŁ +æĭ § +ä¹Ł æľīä¸ĢäºĽ +ä¾ĽçĶµ åħ¬åı¸ +å»ī ä»· +帮 ä»ĸ +æŃ¤æ¬¡ æ´»åĬ¨ +åıªèĥ½ 说 +èĬ ĭ +çīĩ 段 +åŃĺåľ¨ éĹ®é¢ĺ +ä½łä¼ļ åıijçİ° +è½® å»ĵ +ç½ij éĢļ +滨 æ±Ł +æİĪ ä¿¡ +é»İ æĺİ +ä¸į å±ŀäºİ +约 åįł +éķ¿æ²Ļ å¸Ĥ +èĥļ èĥİ +åħĥ 件 +éĻĨ åĨĽ +è³¼ è²· +æĮĩ æľĽ +å®ŀä¹ł çĶŁ +çī¹çĤ¹ æĺ¯ +çıł æ±Ł +çľĭ ä¸įåĩº +ä¸įè§ģ äºĨ +ç¼ ī +éĺµ èIJ¥ +åĶIJ æľĿ +没 å¿ħè¦ģ +åĽ½åľŁ èµĦæºIJ +ç»ıæµİåѦ 家 +åIJĪèĤ¥ å¸Ĥ +çIJ¢ 磨 +ç¡® åĪĩ +åŁİå¸Ĥ åıijå±ķ +çŃ· åŃIJ +人æ°ij æľįåĬ¡ +满 åĪĨ +è¿· ä¿¡ +ä½ľèĢħ æľ¬äºº +æĸĩç«ł æĿ¥æºIJ +ç«Ļ ç«ĭ +æŀĦ æĪIJäºĨ +è¾Ľ åĭ¤ +è¶ħ 强 +éĶ ļ +åīįä¸ī åŃ£åº¦ +å°± è§īå¾Ĺ +å´ĩ é«ĺ +è¶Ĭ ä¾Ĩ +è¶Ĭä¾Ĩ è¶Ĭ +å¸Ĥåľº èIJ¥éĶĢ +综åIJĪ ç´łè´¨ +åŃ ļ +ä¾® è¾± +äºĮ åŃĹ +å·¥ä½ľ ä»»åĬ¡ +åı²ä¸Ĭ æľĢ +æľĢ ä¼ĺ +åIJ© åĴIJ +表 çĻ½ +èİ« åIJį +èİ«åIJį åħ¶ +èİ«åIJįåħ¶ å¦Ļ +å¹ £ +åIJĮå¿Ĺ 们 +建设 çĶ¨åľ° +åĦ Ģ +éħį åģ¶ +å¼ © +åĶ± çīĩ +æīĭ èĦļ +åħ¼ ä»» +åģľ æĶ¾ +æŃ£ å®Ĺ +æĸ° åĨľæĿij +åĤ¬ çĶŁ +æīĢ åŃ¦æł¡ +念 ä½Ľ +åĶ¤ éĨĴ +åħ± åĪĽ +æĭī ä¸ģ +èĥĮ çĿĢ +çĶŁæĢģ ä¿ĿæĬ¤ +åı£ 头 +æĸ¹åIJij çĽĺ +調 æķ´ +æĭĽèģĺ ä¿¡æģ¯ +åħ¶ä»ĸ åĽ½å®¶ +ç®Ģ æĺĵ +åĮ¿ åIJį +è¯Ħ æµĭ +æĺ¯ä¸Ģ 座 +çīµ æīĭ +足 迹 +çIJĨ解 åĴĮ +æľĢ åıĹ +å¿ĥ è·³ +çĪ¶ 親 +éĿŀ常 åĸľæ¬¢ +èĭ¦ éļ¾ +æĬĢ å¸Ī +æ°ij æĦı +æĪĺ åĽ½ +æĽ¿ è¡¥ +æ´¥ è´´ +ä¸ŃåĽ½ ä¼łç»Ł +åIJĦ è¡Į +åIJĦè¡Į åIJĦ +åIJĦè¡ĮåIJĦ ä¸ļ +第äºĶ å±Ĭ +èį· èĬ± +æĦı èŃĺ +票 ä»· +åĪĨ æµģ +æĿİ çĻ½ +æ±Ł åĮĹ +æİĴ æĸ¥ +ä½ĵ éĩı +åĮħåIJ« äºĨ +åĪĺ æŁIJ +çİ° å¦Ĥä»Ĭ +å·¥èīº åĵģ +è¿Ļç§į æĸ¹æ³ķ +åĬŀåħ¬ 楼 +çĶµ å·¥ +çħ Ļ +åį¡ çīĩ +å¹´ å¹´åºķ +ä¸ĵ项 èµĦéĩij +åĮ» ç§ij +åĮ»ç§ij 大åѦ +åĽŀ头 çľĭ +ä¸į å±ij +èĩª 驾 +没 æĶ¶ +æīĵ çĮİ +èĦ¸ éĥ¨ +åıĥ èĢĥ +å°Ĩ 士 +è´«åĽ° 人åı£ +çIJĨæĥ³ 信念 +é£İ å°ļ +人æīį éĺŁä¼į +çij ¾ +æĿ¥ è¿ĻéĩĮ +æ´Ĺ 涤 +å¹´ èĸª +èĭį çĻ½ +ä¸ĩ äºĭ +课 æľ¬ +åºĵ éĩĮ +çī¹ æ´¾ +çī¹æ´¾ åijĺ +èµŀ ç¾İ +ç©¿ æĪ´ +製 ä½ľ +èµŀ æĪIJ +ä¸Ģ 侧 +å½ĵåľ° 人 +æĭ İ +纸 è´¨ +ä½Ļ 个 +éĶĤ çĶµæ±ł +æľº åŀĭ +éĻ¢ éĻ¢å£« +åģļ å·¥ +å¼ł è´´ +ç¥Ľ æĸij +æ®ĸ æ°ij +å¥ij 约 +æ¹ĺ æ½Ń +æIJ ĸ +åŃĺ è´§ +交éĢļ 大åѦ +è¶ģ çĿĢ +æĸĩçī© ä¿ĿæĬ¤ +å¤ĩ æĪĺ +éĩĩ 纳 +åįĬ æľĪ +æľĢ åħ³éĶ® +æľĢåħ³éĶ® çļĦ +æİ¥ éĢģ +æĶ¶ åī² +åıį åĢĴ +çĥ Ľ +æ ½Ķ +ä¼Łå¤§ å¤įåħ´ +çļĦè¯Ŀ è¯Ń +容 å¿į +å®ļ éĩı +æķ Ĺ +åĵģçīĮ 形象 +æīŃ è½¬ +åĽ½å®¶ éĩįçĤ¹ +èĨĿ çĽĸ +ä¸Ģ 楼 +大 éĻ¸ +éĤª æģ¶ +åĽŀ åij³ +çĮ ¿ +çĿ¡ åīį +æĹł è¾ľ +çĹħæ¯Ĵ æĦŁæŁĵ +æľºæ¢° åĮĸ +çĤ¹ 亮 +溶 解 +åĩłä¹İ æīĢæľī +è·ij éģĵ +çĶµè§Ĩ æľº +åı ¨ +æijĩ äºĨ +æijĩäºĨ æijĩ头 +èĩª è´Ł +综åIJĪ åĪ©çĶ¨ +èĩª å¦Ĥ +åİŁ ä¾Ĩ +ä¹Łä¸į æĥ³ +èĬĤ 课 +è¿ĩ åī© +çĶ² çĬ¶ +çĶ²çĬ¶ èħº +æĸ° ä¸ĸ纪 +èĩªä¸» åĵģçīĮ +é«ĺ å±Ĥ次 +ä¸Ģ è§Ĵ +è¡Į äºĭ +ç¥ĸ åħĪ +å©ļ åIJİ +éĹ´ éļĻ +ç¼Ŀ éļĻ +è¿Ļ æĶ¯ +ä¸įæĸŃ åĪĽæĸ° +å¾® åŀĭ +æĽĻ åħī +享 çĶ¨ +ä¸ŃåĽ½ 移åĬ¨ +éĹŃ çݯ +æī§ æĦı +åıijå±ķ æł¼å±Ģ +æł¸å¿ĥ åĮº +éªļ æī° +åħļåĴĮ åĽ½å®¶ +ä¸ŃåĽ½ æĶ¿åºľ +帶 èijĹ +ä¸ĩåįĥ çĵ¦ +åħ© 人 +äºİæĺ¯ æĪij +åĽº ä½ĵ +çªģ å¦Ĥ +çªģå¦Ĥ åħ¶ +çªģå¦Ĥåħ¶ æĿ¥ +éĩĮç¨ĭ ç¢ij +çĪ± ç¾İ +æŁ¥ éªĮ +åıĮ èµ¢ +éĹª åħī +楼 å®ĩ +æĻ ı +æľī è¶³å¤ŁçļĦ +æŁĶ æĢ§ +ä¿¡æģ¯ å®īåħ¨ +管 线 +并 ä¸įä¼ļ +åĻ¨ 件 +ä½ł åºĶ该 +çĿĢ å®ŀ +æĺİ æ¸ħ +æĬĹ çĶŁç´ł +æīĵ æŃ» +å®Įåħ¨ ä¸įåIJĮ +èĬ± æ¤Ĵ +æĶ¾ 宽 +ä½İ 端 +åĽĽ èĤ¢ +åĮĹ京 èµĽè½¦ +éĽĨ å¸Ĥ +æľª å©ļ +大å¹ħ æıIJåįĩ +建çŃij 设计 +çĭ¬ æľīçļĦ +æİ¢ éĻ© +æ²³æµģ åŁŁ +æħķ 容 +被 çĽĹ +åĵº ä¹³ +èı ģ +æĥ¬ æĦı +è¶ĬæĿ¥è¶Ĭ 好 +广大 群ä¼Ĺ +å¾· èĤ² +å¸Ĥåľº ä»·æł¼ +奥 å·´ +奥巴 马 +èĬĤ缮 ä¸Ń +两 款 +ä¸ĩä½Ļ åħĥ +ç»´ å°Ķ +çĶŁçī© ç§ijæĬĢ +åIJ¬ èµ·æĿ¥ +çł ļ +æĭŁ å®ļ +æ²¹ çĶ° +声 èªī +建çŃij ä¸ļ +éĻIJ è´Ń +çīĩ åŃIJ +çķľ 禽 +ç½ij é¦ĸ页 +ä¼Ĺ çŃ¹ +æĴŀ åĩ» +åīį ä¸įä¹ħ +åīį ä¸ĸ +åĽĽä¸ª æĦıè¯Ĩ +æµĭ ç»ĺ +éĺ² ç©º +漫éķ¿ çļĦ +æ²IJ æµ´ +æ¯Ķè¾ĥ ç®Ģåįķ +æµĭ å®ļ +åĽŀ è°ĥ +让 人们 +èĴĭ ä»ĭ +èĴĭä»ĭ çŁ³ +ç»ĵ æĻ¶ +å¢ŀæ·» äºĨ +æĿ¡ è¯Ħ论 +åī¯ ä¼ļéķ¿ +ä½ı æīĢ +ç»Ļ åĩºäºĨ +è°ĥ éħį +æ² ĸ +æľī çĶ¨ +æľīçĶ¨ çļĦ +ä¸ĢæĿ¡ é¾Ļ +éĩİ å¤ĸ +ç¼ĺ åĪĨ +æ°¸è¿ľ ä¸įä¼ļ +æŀľ æłij +大åıij å¿«ä¸ī +麻 éĨī +äºij éĽĨ +åİ» åĵªéĩĮ +åħ¥ å¸Ĥ +ä»» æĢ§ +建 æ¡£ +建档 ç«ĭ +建档ç«ĭ åį¡ +ä¸Ģ 棵 +社 åįĢ +缸 ä¼´ +åļ · +å¡« åħħ +ä¸Ģ æĹı +ç¾ ģ +åıĸ è¯ģ +èĪ° éĺŁ +åİĤ åĮº +è¡· å¿ĥ +åıijå±ķ éĺ¶æ®µ +é«ĺ 强度 +åĹĵ åŃIJ +é¢Ĩ è¡Ķ +楼 主 +大 èĴľ +æŀķ 头 +ç²® æ²¹ +é»Ħ çĵľ +æĵ Ĵ +å°ı çĭĹ +æĶ¹éĿ© å§Ķ +åįģ åĪĨéĴŁ +é²ľ èī³ +åħ³ ç¾½ +çĭĢ æħĭ +å®ŀçĶ¨ æĢ§ +å°ij è§ģ +é£ŀ æī¬ +çĶ° éĩİ +æIJ Ĥ +è¿Ļ个 è¯į +åºĶæĢ¥ é¢Ħæ¡Ī +è§Ĵ度 æĿ¥çľĭ +æķ¬ çķı +æ³ķ å®Ŀ +åĸĦ æĦı +æīĵ æĸŃ +对 åĨ³ +çµķ å°į +åĢŁ æѤ +å¼Ģ æºIJ +å°ı 說 +ç¥ º +å²ģ 以ä¸ĭ +éĢĢå½¹ åĨĽäºº +ä¸įä¹ħ åīį +åĩº åİĤ +讽 åĪº +æĿ¥çľĭçľĭ åIJ§ +éŃĶ åħ½ +çķĻ ä¸ĭæĿ¥ +å±ħ 室 +åłħ æĮģ +çľĭ äºĨä¸Ģ +çľĭäºĨä¸Ģ çľ¼ +éĽĨåĽ¢ æĹĹä¸ĭ +æĪĺ æĪĺç»ĦåIJĪ +è®¤çľŁ èIJ½å®ŀ +汽车 产ä¸ļ +çī©çIJĨ åѦ +æķ µ +éĴ Ŀ +åĽ¢ éķ¿ +ä¸įæĸŃ æī©å¤§ +èĤ© è´Ł +åıijå±ķ 缮æłĩ +è³ĩ éĩij +åīį ç½® +ä¸ŃåĽ½ åı¤ä»£ +æŃ» åĪij +åħħåĪĨ ä½ĵçİ° +åħ³ éŨ +ç¾İ æĦŁ +æīĵ åħ¥ +æĬijéĥģ çĹĩ +å°ij çĪ· +æłij æŀĿ +æ¶Īæģ¯ 称 +æ´Ľ åħĭ +åį ¯ +è¿Ī åIJij +æݨ åĭķ +ä»İä¸ļ èĢħ +åİ» ä¹° +欢 å¿« +æĭ¥ æĮ¤ +马 桶 +æĬĬ æݧ +æĶ¿ åħļ +å¼ł æī¬ +客 æłĪ +红 æĺŁ +éĢģ æĿ¥ +åħ¨åŁŁ æĹħ游 +èĩª ç§ģ +åįģäºĮ æĿ¡ +åı¹ æģ¯ +ä¸Ģ èīĺ +ä¿Ŀ è´¹ +æĸ½å·¥ çİ°åľº +æľī 幸 +ç»Ń èĪª +åı¯èĥ½ æľĥ +èĥĮ åıĽ +ä½£ éĩij +ä¸ī çŃīå¥ĸ +å¾Ī 满æĦı +游æĪı åī¯æľ¬ +群 éĩĮ +æŀĦ 件 +åºı å¹ķ +太 æ¹ĸ +æľ¨ è´¨ +æĻĭ æ±Ł +çµĤ æĸ¼ +è·³ è·ĥ +åĢºæĿĥ 人 +çŃī 诸å¤ļ +æĶ¾ åĩº +åħ³éĶ® æŶåĪ» +æĦŁæŁĵ èĢħ +é£ŀè¡Į åijĺ +èĥĨ åĽº +èĥĨåĽº éĨĩ +æĬ± æŃī +åij¨ äºĮ +æĸ° æĹ¶æľŁ +åĨ·éĵ¾ çī©æµģ +è¿Ļç§į æĸ¹å¼ı +该 æĿij +åĽŀ é¦Ī +åŁºçĿ£ æķĻ +人 åıĤ +æŀ¯ çĩ¥ +æī¹åıij å¸Ĥåľº +åħħåĪĨ èĤ¯å®ļ +å¸Ĥ æĶ¿åįı +äºĭ æ¥Ń +龸 çİĭ +çĥŃ æIJľ +åįģä¹Ŀ 大 +ä¼´ æľī +ç¾İåĽ½ æĢ»ç»Ł +åŁİå¸Ĥ 管çIJĨ +ä¸ĭ 令 +èĥ¸ åı£ +åıª çŁ¥éģĵ +åij¨ ä¸ī +çĶ¨ æĪ¶ +éŃ ¯ +å¿ĥ è¡Ģ +带头 人 +åĮ» åĬ¡ +åĮ»åĬ¡ 人åijĺ +æݧåĪ¶ åĻ¨ +ä½ľåĵģ åĨħ容 +æĪĺ åıĭ +åİĨ å¹´ +ä¸į åħĭ +ä¸įåħĭ ä¸įåıĬ +æĹ¥ æŃ£å¼ı +è±IJ å¯Į +ç¨İ è´¹ +æŶ æķĪ +å±ķ ä½į +è¡¡ éĺ³ +æĪ¿ 貸 +çĪĨ 款 +ä¹IJ æĦı +çĶ· 主 +å¯ ¬ +æľĥ èŃ° +ä¹ĭ å¤ľ +åIJĮ 樣 +ä¸įè¦ģ 太 +ä¼Ĭ æĸ¯ +ä¼Ĭæĸ¯ åħ° +åŁºæľ¬ åİŁåĪĻ +åİ» æİī +ä½İ ä¿Ŀ +个 交æĺĵ +个交æĺĵ æĹ¥ +èģĬ èģĬ +åĽĽ ä½į +åħļç»Ħ æĪIJåijĺ +主è¦ģ ä»İäºĭ +å½± éŁ³ +åĨĴ åĩº +åij¼åIJ¸ éģĵ +è¾¾ å°Ķ +æľ¨ åľ°æĿ¿ +诡 å¼Ĥ +çģ¯ åħ· +çģ« çĥ§ +解 èĦ± +æĦĪ åıij +æ¹ĸ å·ŀ +é£İ ä¿Ĺ +æĸ° å½¢åĬ¿ +æĸ°å½¢åĬ¿ ä¸ĭ +è² Ŀ +èĦ ĵ +åĬ¨åĬĽ çĶµæ±ł +é£ŀ èĪ¹ +飧 æĢ§ +åĪ© çī© +åĪ©çī© 浦 +ä¸į 认è¯Ĩ +ç¼ĸ ç»ĩ +ä½ľ åĿĬ +èģĮä¸ļ æĬĢèĥ½ +çľĭ è¦ĭ +åĽ´ æ£ĭ +æĺı è¿· +å½Ĵ å±ŀäºİ +æĤ¬ å´ĸ +éĨ« çĻĤ +å®ĭ 代 +åºĦ æĿij +èĹ ķ +çĮĽ çĦ¶ +çĩĥæĸĻ çĶµæ±ł +å®ŀä½ĵ åºĹ +ä¸į足 以 +æĥħ ç· +æĥħç· Ĵ +å»Ĭ åĿĬ +çĶµ åı° +åºĶ åĬĽ +ä¸Ńå°ı åѦçĶŁ +èĥ¡ åIJĮ +éī´ åĪ« +åĨħ ç½® +ä¹± 象 +æ¬Ĭ çĽĬ +å¼ĢæĶ¾ å¼ı +åįļ æĸĩ +讲 课 +çŃī åİŁåĽł +ç©· 人 +交 æĽ¿ +æĬ¤ çħ§ +åıijå±ķ æľºéģĩ +客 åķĨ +åıį ä¹ĭ +ç±³ é¥Ń +并 åıij +并åıij çĹĩ +æ±ī åŃIJ +æŀľ åĽŃ +对æĪij æĿ¥è¯´ +åģı åIJij +æī¹ 示 +读 åIJİ +读åIJİ æĦŁ +æĺİ æĻº +åĽ´ çĿĢ +åıį 转 +æĿ¨ å¹Ĥ +ä¸ĵ åįĸ +ä¸ĵåįĸ åºĹ +åıĹ éĻIJ +åºŁ è¯Ŀ +æŀģ å°ij +åįĪ åIJİ +è¿Ľ ä¿® +åīĬ åĩı +æľ¬ç§ij çĶŁ +ä¼ĺ éĢī +åħī çħ§ +åıĻ äºĭ +åıĸ æļĸ +åĮĹ è·¯ +æ¦ ķ +èİĨ çĶ° +楼 å±Ĥ +天 èĬ± +天èĬ± æĿ¿ +çĤ ľ +å·²ç»ı æľīäºĨ +è¶ ¾ +çĶ³ åįļ +çĶµ éĺ» +åĬŁ 课 +æŃ¥ æŃ¥ +éĤ£ä¹Ī 容æĺĵ +æѤ æĸĩ +ä½ ° +计 è¾ĥ +çīĩ éĿ¢ +çĶµå½± éĻ¢ +ä¸į åħ¬å¹³ +ä¸ī æľŁ +æĹħ游 èµĦæºIJ +å¤ļç§į å½¢å¼ı +è£Ĥ ç¼Ŀ +åIJİ æİĴ +硬 度 +åĽŀ æļĸ +éģĵ æķĻ +è´« è¡Ģ +æ¸ħ é¦Ļ +伤 çĹħ +æĦı 義 +çļĦ ç¼ĺ +çļĦç¼ĺ æķħ +åºĦ 严 +åıªæĺ¯ ä¸ºäºĨ +æīĵ æĬĺ +以 ä¾Ĩ +滿 足 +çİĽ 丽 +風 éļª +æĸĩ ç§ij +éħįå¤ĩ äºĨ +è¿Ľ é£Ł +æ¶ ¡ +è·¯ ç¨ĭ +åı« 声 +ä¸Ńå¿ĥ åŁİåĮº +æľīæīĢ ä¸įåIJĮ +å¼µ è²¼ +é¢Ħ æĬ¥ +æľīå¤ļ ä¹Ī +è¿Ľè¡Į åħ¨éĿ¢ +æĽ¾ ç¶ĵ +ä¸ī 代 +å®ı 大 +æ¸ħ æī« +éĢī åĩº +åĵª ä¸Ģ个 +主 義 +ä¾Ŀ æĵļ +çļ® éĿ© +赶 æĿ¥ +çŃĽ æŁ¥ +æ¨ Ł +ä¿Ŀ èįIJ +åIJĥ æĥĬ +æľĭåıĭ们 对 +ä»ĸ æĺ¯ä¸Ģ个 +åºŁ æ°Ķ +æ» ħ +è´¢ ç¨İ +æĿij æĿijæ°ij +èµĦ产 è´ŁåĢº +å®ī å¨ľ +缮åīį åĽ½åĨħ +æĦŁè§ī èĩªå·± +çµIJ åIJĪ +éĶ¦ æłĩ +éĶ¦æłĩ èµĽ +æĽ´ æ·± +åŁº æķ° +éħ¿ éħĴ +çī¹èī² 产ä¸ļ +åİĭ å®ŀ +ä¾Ŀæ³ķ 追究 +æ·¡ å®ļ +ç®Ģ缴 å°±æĺ¯ +å£ĵ åĬĽ +æ°ij å¿ĥ +ä¸į åIJĪéĢĤ +çĶ±æѤ åı¯è§ģ +èµŀ èªī +æ¾ ¤ +åĩłå¹´ åīį +åIJī ä»ĸ +çł´ æįŁ +轻轻 åľ° +å²Ľ 屿 +æĦı å¢ĥ +ä»Ģä¹Ī åı« +åģĩ è£ħ +éĢģ è´§ +å¹ķ å¢Ļ +妥 åįı +åĽ½ æĹĹ +äºĨ å¾Īä¹ħ +åĪĨ辨 çİĩ +ç´ Ķ +éĺ³ åĮº +åĩŃ çĿĢ +åģľè½¦ ä½į +京 éĥ½ +éĶ £ +æĵ ¾ +è¿Ľ éŨ +åĪĺ æµ· +åĽĽ 级 +女 足 +è¡ĮæĶ¿ 审æī¹ +éģ¥ æݧ +ä¸į éĮ¯ +å¾Ĺ å¾Ī好 +为 缮çļĦ +ä»į æľª +ç²¾ è£ħ +éĢį éģ¥ +å°½ 头 +çºł ç¼ł +éłĺ å°İ +æĭħ è´Ł +æĪĸèĢħ åħ¶ä»ĸ +åıªä¸įè¿ĩ æĺ¯ +åı® åĺ± +åģĩ åĨĴ +æļĸ æ°Ķ +çĽIJ åŁİ +被 è§Ĩ为 +诺 è´Ŀå°Ķ +ç»ĻäºĨ æĪij +è¿ij åįĥ +éĩį åĽŀ +éĨĴ äºĨ +çĶµ 解 +忽çķ¥ äºĨ +èĥĮ éĥ¨ +æĸĩæĺİ åŁİå¸Ĥ +æº ħ +è² ĵ +æĬµ æĮ¡ +åĸľæ¬¢ åIJĥ +éĿĻéĿĻ åľ° +å¾Ī æ·± +åŁºç¡Ģ çŁ¥è¯Ĩ +è¿ĩ éĶĻ +çIJĨ ç§ij +交æµģ åIJĪä½ľ +èĪ Ķ +調 æŁ¥ +æħĪ æĤ² +éĴ ° +èĩ´ çĶµ +å®£ä¼ł æ´»åĬ¨ +åıĺ éĩı +çļĦ人 æĿ¥è¯´ +æŶ éļĶ +ä¸į管 ä½ł +缸 è¿ij +è´µ éĩijå±ŀ +ä¹Łä¸į åı¯èĥ½ +ç²ī æľ« +åįĹ çĵľ +çĻ½ 马 +åħī æºIJ +éĩij å¥ĸ +çĭ¬ è§Ĵ +çĭ¬è§Ĵ åħ½ +妨 ç¢į +ç»Ļ åĬĽ +ä½Ĩ ä»į +å¼łå®¶ åı£ +èIJ¬ åħĥ +渲 æŁĵ +éķ¿å¤§ äºĨ +è®°èĢħ äºĨ解 +æĢĢ çĿĢ +è¦ģ åѦä¼ļ +游æĪı 代 +游æĪı代 ç»ĥ +äºĮ çĻ¾ +æĦıè¯Ĩ å½¢æĢģ +çİ º +计åĪĴ çĶŁèĤ² +æī¾ åĩĨ +åħ° èĬ± +è¿Ļ座 åŁİå¸Ĥ +污 æ³¥ +å®ĺæĸ¹ 微信 +å½Ĵ å±ŀ +æ°§ æ°Ķ +éģİç¨ĭ ä¸Ń +åį°è±¡ æ·±åĪ» +稳 妥 +çµIJ æĿŁ +åŃķ æľŁ +çī¹ æĿĥ +åĿļ åĽº +顺 åĬ¿ +æŀľ èĶ¬ +éĨ« 師 +åİ ® +ä¹Łæĺ¯ å¦ĤæѤ +é¦Ĵ 头 +缸 åĬ© +å¹² 线 +ä¸Ģ æľ¬ä¹¦ +ç» ¥ +æĮ¯ å¥ĭ +èĤ¾ èĦı +åĭķ çī© +é£ŀ è·ĥ +èıľ åĵģ +å¤ļ ä½Ļ +å¤ļä½Ļ çļĦ +éĢĿ ä¸ĸ +æģĭ 人 +å¼Ģåıij åĪ©çĶ¨ +顺 丰 +éĩİ å¿ĥ +æł¡ å¤ĸ +æģIJ é¾Ļ +éĿ¢ åħ· +éķ¿ è¾Ī +éļı å¤Ħ +éļıå¤Ħ åı¯è§ģ +ç´§ 缺 +éĩį ä¸Ń +éĩįä¸Ń ä¹ĭ +éĩįä¸Ńä¹ĭ éĩį +奥 æĸ¯ +奥æĸ¯ åį¡ +ä¸Ģ个 å¤ļ +ä¸Ģ个å¤ļ æľĪ +ä¸įåı¯ 缺å°ij +æĸ° æł¼å±Ģ +æıIJ æĮ¯ +è¡Į è´¿ +æ¼Ĥ æµģ +èģĬ åŁİ +åħ´ 建 +è´¨ æ£Ģ +ç§ģæľį 游æĪı +æĽ´ éĩįè¦ģ +è´ ® +çħ ľ +转åıĺ 为 +è¿Ļ 两年 +ä¿Ŀ é²ľ +æī§ æķĻ +çĥ ¨ +å¼Ģåıij 建设 +è¿IJèIJ¥ 管çIJĨ +误 å·® +京 åī§ +å¸IJ åı· +å·¥ä½ľ ä½ľé£İ +ä¸ĸ ä¿Ĺ +çĻ½ 宫 +天 åĽ½ +å¤©åĽ½ 继ç»Ń +å·´ æĸ¯ +èIJ¥ åĪ© +åĵģ æł¼ +æĿijæ°ij 们 +æĪ¿ 车 +çŃī çĹĩçĬ¶ +å¦Ĥ å®ŀ +å® ¸ +å±Ĥ 级 +éĶĻ è¿ĩäºĨ +ç»ĵ å®ŀ +ç¬ij èĦ¸ +羣å®ŀ æĢ§ +éĥ½å¸Ĥ æĬ¥ +é¥Ń èıľ +åºĶ 注æĦı +æĬ½ çĥŁ +伪 éĢł +åīį ä¸Ģ天 +éŃĶ é¾Ļ +éŃĶé¾Ļ 令çīĮ +约 è°Ī +绣çŃ¹ æİ¨è¿Ľ +让 çĶ¨æĪ· +åħ¨éĿ¢ èIJ½å®ŀ +å¼Ħ å¾Ĺ +è°Ī æģĭçĪ± +鸣 æĪIJéķ¿ +鸣æĪIJéķ¿ è®° +æ´ĭ æ´ĭ +çĸı æķ£ +éĿ¢ç§¯ 约 +æµĵ 缩 +æĸ¯ é¡¿ +çĶŁæĢģ åľĪ +æī§ 导 +移 éĢģ +齿 è½® +æł¹æľ¬ å°±ä¸į +缩 åĩı +èµ° ä¸ĭåİ» +çĿ« æ¯Ľ +ä¹Łä¸į éĶĻ +åıįæĺł åĩº +èĭ¦ æģ¼ +缸åħ³ æĶ¿çŃĸ +é«ĺ 楼 +ç²ī èī² +æĬķèµĦ é¢Ŀ +ä¸į ç»ı +ä¸įç»ı æĦı +å®ģ æĦ¿ +èĪĮ 头 +æ»ĭ çĶŁ +å®ģ åİ¿ +åīįåĪĹ èħº +åĩ ³ +é£Ł 欲 +åıĸ èĥľ +éĻ¢ åŃIJ +ç´łè´¨ æķĻèĤ² +滨 å·ŀ +æĬ¢ æĬĵ +å¼Ĥ åij³ +åĴ ļ +åĬ į +宽 éĺĶ +æļ´ 涨 +æĥł åıĬ +è§Ħ ç¨ĭ +ä¾Ľ åħ» +éĢģ å¾Ģ +å±± åºĦ +举 äºļ +å±ķ é¦Ĩ +解 éĶģ +æĹł è§Ĩ +éĻį èIJ½ +è¿ŀ äºij +è¿ŀäºij 港 +åıĤ è°ĭ +çİ ĸ +ç¬ ĥ +èĢĹ è´¹ +æī¿ å¾· +社ä¼ļ æķĪçĽĬ +åįĹæµ· ç½ij +åĪĽ 伤 +èIJ ± +åħħ æ²Ľ +ç½ijç«Ļ 建设 +大 åºĨ +åĨį éĢł +åŃĹ æł· +åħ¨æ°ij åģ¥èº« +èĮ« èĮ« +æµ® åĬ¨ +åīį åı° +å¢ŀ 设 +éĢĽ è¡Ĺ +åĢĴ éĹŃ +æ³ķå¾ĭ 顾éĹ® +çĸ ® +çĹħ çĹĩ +空 åīį +请 æķĻ +èĥľ ä»» +æĿĢ èıĮ +æĪĺæĸĹ æľº +ç»ĺ åĪ¶ +å¤Ħ æĸ¹ +çªģ åĽ´ +çĮ« åĴª +æĬ¥åijĬ æĺ¾ç¤º +ç¿ Ł +çķ¶ åľ° +æľĢ éļ¾ +纪 å§Ķ书记 +ä½İ åİĭ +èĻļ 空 +è¿Ļéĥ¨ çĶµå½± +产ä¸ļ åįĩ级 +è°· çĪ± +è°·çĪ± åĩĮ +æĬ¼ éĩij +女 æĸ¹ +éĴ» çłĶ +æļĹ æļĹ +è¿· ä½ł +æīĢ è¬Ĥ +å¨ģ å»ī +å¼Ģ æľĹ +å² Ķ +çģ« çĤ¬ +åIJĪçIJĨ æĢ§ +åħ¬ åĬŀ +ä¼ļ ä¼ļéķ¿ +éĺ´ è°ĭ +å¼Ģ å±Ģ +æĻ®éĢļ è¯Ŀ +åį¡ æĭī +å°ij åIJĥ +éĹª èĢĢ +æŀľ æ±ģ +æī§è¡Į åĬĽ +è° Ľ +æĬ¢ åĬ« +é«ĺéĢŁ åıijå±ķ +éŁ ¬ +åįĹ æ²Ļ +é«ĺçŃī åŃ¦æł¡ +æį¢ 个 +åı¯èĥ½ åŃĺåľ¨ +æĬ Ĵ +è°± åĨĻ +被 æĬĵ +æĿ¯ åŃIJ +èĬĤèĥ½ åĩıæİĴ +æ°ĶåĢĻ åıĺåĮĸ +åĪĨ åĪ¥ +ä¸Ń æŀ¢ +欢 åij¼ +åħī 纤 +è¿Ļ 群 +çľ¼ çķĮ +åħ±åIJĮ åıijå±ķ +çİ° ä»Ĭ +éĹ» è¨Ģ +çī¹èī² å°ıéķĩ +æķij 人 +éĻį æ°´ +ä¸ĸçķĮ ä¸Ģæµģ +å°± é¤IJ +çŀ ¥ +å¤į ä»ĩ +ç¾½ æ¯Ľ +ç¾½æ¯Ľ çIJĥ +è´© åįĸ +æºIJ æ³ī +æĢ»ä½ĵ è§ĦåĪĴ +åĬ¨ æĦŁ +ä¸Ģ 审 +åĢŁ éĴ± +è§ģ æķĪ +èĬ± èįī +åIJĮ ä¸ļ +æŁ¥ è©¢ +åĽ½éĻħ åIJĪä½ľ +ä¾Ľ åĽ¾ +åģ ´ +æł ĵ +缸 éĢļ +è°Ī åıĬ +è¿ĩç¨ĭ å½ĵä¸Ń +é¦Ļ èıĩ +åįģåĽĽ æĿ¡ +ä¸Ģå¼Ģå§ĭ å°± +ä¸ĵ åijĺ +æĺİ é¡¯ +æīĵéĢł åĩº +ä¸ĭéĿ¢ æĪij们 +æľº æ²¹ +åı° è¯į +åŃIJ å¼Ł +æľĢ 常è§ģçļĦ +æĪij è®°å¾Ĺ +ç» ° +æĤ¬ æµ® +è¿ĺ 羣æĺ¯ +æĮĤ åı· +åıĭ åĸĦ +éĩį 伤 +çħ§ 亮 +æѦ èѦ +åĩºçİ° éĹ®é¢ĺ +è¸Ĭ è·ĥ +åľ°çIJĥ ä¸Ĭ +å¸Ĥ 人大 +åıĹ害 人 +å² IJ +åIJĮ åѸ +éĩijèŀį å¸Ĥåľº +æľīçļĦ çݩ家 +å¸Ĥ æķĻèĤ² +å¸ĤæķĻèĤ² å±Ģ +åIJĦ å¼Ĥ +ç·ļ ä¸Ĭ +æģ º +æľī 大éĩıçļĦ +åķĨ æĬ¥ +åįķ åįķ +åħ¨ é¢Ŀ +ä¾Ŀæŧ æĺ¯ +好 åĩłä¸ª +åĸ µ +éĩį æķ´ +çĶŁæ´» è´¨éĩı +æİ¢ 访 +åį° èĬ± +缼 è¡Į +å¾® è§Ĥ +èĪį å¾Ĺ +åºŁå¼ĥ çī© +积 èĵĦ +å®ļ å±ħ +æĤ ¼ +èĮ ¸ +çļĦ 帮åĬ© +çļĦ帮åĬ© ä¸ĭ +亿 åIJ¨ +åŃĶ éĽĢ +è¿ĻæĿ¡ è·¯ +é¥ µ +æĦĪ åĬł +éķ į +ä½ľ æ¡Ī +èįĶ æŀĿ +太 å°ij +è·» 身 +åħ¬çĽĬ æ´»åĬ¨ +çĻ½ æĸij +æĬĢæľ¯ æ°´å¹³ +å¸ § +æĹł çŁ¥ +åºĶ该 æĢİä¹Ī +éĢĢ å¸Ĥ +æ¸ Ń +åħ» çĮª +é© ¼ +群 å²Ľ +大 åį« +ä¹ĺ çĶ¨è½¦ +èı² å°Ķ +è´´ åIJ§ +åģľ ä¸ĭæĿ¥ +æľīæľº ç»ĵåIJĪ +åĪ» èĭ¦ +çļĦ åľ° +çļĦåľ° æŃ¥ +è¯Ĭ æīĢ +å¼Ģ æĪĺ +èĢģ çīĮ +çŃ¹ çłģ +åħ«å¤§ 以æĿ¥ +楼 æĪ¿ +åŃĻ æĤŁ +åŃĻæĤŁ 空 +åħĴ åŃIJ +第ä¸Ģ æĿ¡ +社交 åªĴä½ĵ +æĥ³ èµ·æĿ¥ +大 æ´ĭ +æĭ¼ éŁ³ +è¿Ľ åįļä¼ļ +è¿ĩ åħ³ +æ² ¼ +ç©¿ æIJŃ +éĤ£ ä¸Ģ天 +çł´ éŨ +æĬķæłĩ 人 +èµ¢ 家 +èĻļ å¼± +æ¿ ĥ +å®ī æ£Ģ +客 家 +çĭ¬ç«ĭ èij£äºĭ +æīĭ åĬ¿ +åīµ éĢł +åľĨ满 å®ĮæĪIJ +为主 线 +好å¥ĩ å¿ĥ +é¢Ĩ åľŁ +çª ĸ +åħ¸åŀĭ æ¡Īä¾ĭ +çªģåıij äºĭ件 +åºķ æ°Ķ +头 æĻķ +å®Ľ å¦Ĥ +è§ ¸ +æ¸ħ æ·¡ +åļ ¼ +åģľ çĶµ +ç²ī å°ĺ +éĻįä½İ æĪIJæľ¬ +æĶ¾ æīĭ +è®°èĢħ 表示 +æĭĸ 延 +éª ĩ +æ®ĭ å¿į +çľģ æķĻèĤ² +çľģæķĻèĤ² åİħ +é«ĺ é¢Ŀ +éĦ Ļ +æ¥ ŀ +åĨħ ç§ij +èIJ¥ä¸ļ é¢Ŀ +åŁº çŁ³ +æµģ æ·Į +主 æŨ +éĺIJ éĩĬ +建 åįİ +æĥĬ åı¹ +çī¢åĽº æłijç«ĭ +æĺ¯åIJ¦ åŃĺåľ¨ +建 åĨĽ +éĽ¾ éľ¾ +åħ¬ 认 +åħ¬è®¤ çļĦ +æ°¨ åŁº +æ°¨åŁº éħ¸ +åīį åĩłå¹´ +åĪ¹ éĤ£ +æ±Ł 举 +å·¥ æ¥Ń +ä¸ĢçĤ¹ ä¹Łä¸į +ä¿® 士 +äºĨä¸Ģ éģį +åĪ ģ +æ»ļ æ»ļ +åĪĨ æł¡ +羣 çĪ± +è¡Ģ èĦī +æĢ¥ åī§ +ä¸Ģ群 人 +ç¾ ¯ +æĪIJ é¾Ļ +ç²¾ç¥ŀ çĹħ +缸åħ³ 人åijĺ +éĿĵ 丽 +ä¸ī åŃ£åº¦ +åĪĴ å®ļ +ä¸ĸçķĮ 第ä¸Ģ +éĢļ ä¿Ĺ +åķĨä¸ļ åľ°äº§ +åĬŁèĥ½ æĢ§ +èµĦæľ¬ 主ä¹ī +详 è§ģ +æĬĵ æįķ +æĸĩ æĺĮ +å®Ŀ å®ī +è£ħéħį å¼ı +æºIJ æºIJ +æºIJæºIJ ä¸įæĸŃ +çĶŁ æĢķ +纵 åIJij +å£ ½ +çľ¼ è¢ĭ +èĤī ä½ĵ +åı¤ ä»Ĭ +èŀį åªĴä½ĵ +åģ ī +æł¼ æľĥåĵ¡ +çĥ · +åĬŁ çĶ¨ +æīŃ çŁ© +绿èī² éĢļéģĵ +åī§ ç»Ħ +å¼± åĬ¿ +è´¨éĩı éĹ®é¢ĺ +éĻIJ é¢Ŀ +éª Ĩ +éģµ ä¹ī +å¯Ŀ 室 +æĥ³ 念 +åł± åijĬ +ä»ħ 次 +ä»ħ次 äºİ +èŀį åĪĽ +æĭĽèģĺ ä¼ļ +åºĬ åŀ« +转åŀĭ åıijå±ķ +ä¸ŃåĽ½ çĶµä¿¡ +åIJ¬ è¯Ŀ +è«ĭ æ±Ĥ +大éĥ¨åĪĨ 人 +æ´» å¾Ĺ +åĵŃ æ³£ +è¶ Ļ +åıijçĹħ çİĩ +ä¸į 符 +åĨĽ å®ĺ +é¢Ī æ¤İ +æĸ°åĨł çĸ«æĥħ +æŁ¬ åŁĶ +æŁ¬åŁĶ 寨 +ä»»ä½ķ å½¢å¼ı +人 éĻħ +人éĻħ åħ³ç³» +æĢ» æī¿åĮħ +å¹³åĿĩ æ¯ı +æģŃ åĸľ +åĦ ĺ +åħµ 马 +è¿Ł åĪ° +å·¥ 伤 +çīĪæĿĥ å½Ĵ +çīĪæĿĥå½Ĵ åİŁ +æĭ¥ æĬ¤ +ç³Ĭ æ¶Ĥ +å¹² æ¶ī +å°ij ä¸įäºĨ +æĥ³ æī¾ +è´¹ çİĩ +该 éĻ¢ +èŀį åĮĸ +è¿İ åIJĪ +è§ĨåIJ¬ èĬĤ缮 +æł¼ 網ç«Ļ +çľī æ¯Ľ +欢è¿İ 大家 +家åºŃ æķĻèĤ² +ä¾µ èļĢ +ç»Ļ ä½łä»¬ +è¡Ģ液 循çݯ +å¯Ħ æīĺ +å°ĸ åı« +以ä¸ĭ åĩłä¸ª +è¿ĺ 以为 +åħ¶ä»ĸ çݩ家 +ç¬ij ç¬ij +æīĵ åIJ¬ +èĩªçĦ¶ ç§ijåѦ +åŁº ç«Ļ +ä¹Ŀ å·ŀ +ä¿Ŀ 驾 +ä¿Ŀ驾 æĬ¤ +ä¿Ŀ驾æĬ¤ èĪª +æĶ¾ çľ¼ +çŁ¥åIJį ä¼ģä¸ļ +ç¸ ® +ç¨ ½ +æļ ĩ +使çĶ¨ 網路 +é¢Ħ çķĻ +大 象 +åıijæĺİ ä¸ĵåĪ© +æĸĩ 娱 +éĢł ç¦ı +湿 润 +éĿ¢ æĿ¡ +æ¶Īè´¹ åįĩ级 +è®Ĭ å¾Ĺ +åĩł åIJį +ä» Ħ +认 æ¸ħ +è¿ľ æĻ¯ +æıĴ 座 +诸 侯 +åıĺ æĢģ +ç¦ı 彩 +è´§ æŀ¶ +失 æݧ +移åĬ¨ 端 +ä¸Ĭ åı¸ +éĢł 纸 +å¸ĥ æľĹ +çĴ ĩ +åı° åįĹ +åĮĹ京 åĨ¬å¥¥ +èĵĿ çīĻ +éķ¿ çŁŃ +æĬĺ å°Ħ +ç»ij æŀ¶ +å¯Ĵ åģĩ +转 åŁºåĽł +æĢ¥ äºİ +æŃ£ åĵģ +åħħ 滿 +大 纲 +æĬĹ ä½ĵ +è¨ĵ ç·´ +æĶ¶ ç´§ +æ¯Ķ è³½ +åħµ åĬĽ +æľ¬ æĽ¸ +äºĮ 代 +æĢ¥ è¯Ĭ +æĸĩ æ¡Ī +ç»ı åķĨ +æĻ¨ æĬ¥ +æ£ ĺ +æĢ»ä¹¦è®° åľ¨ +åıĹ éĤĢ +äºĶ åĽĽ +å²Ń åįĹ +çĪ± åIJĥ +åŁĥ å°Ķ +å¿ĥ å¢ĥ +è¦ĨçĽĸ éĿ¢ +å®ŀåľ¨æĺ¯ å¤ª +æł¹ åºķ +纷纷 表示 +åĹ ħ +éļıçĿĢ æŶéĹ´ +åİĨåı² æĤłä¹ħ +éħ ī +æĢ» éĺŁ +主é¢ĺ æ´»åĬ¨ +éĹ® åį· +é©¿ ç«Ļ +æı¡ ä½ı +åı¯èĥ½ 导èĩ´ +æ°ij éĸĵ +éĸĭ åķŁ +ä½Ĩ ä¸įéĻIJ +ä½Ĩä¸įéĻIJ äºİ +åįģ éĩĮ +å¨ ¥ +æįŁ èĢĹ +çĸı 导 +çݯ æ°§ +ç¥ŀ éĢļ +çĪ± å°Ķ +çĪ±å°Ķ åħ° +æľ´ å®ŀ +å¿« æĬ¥ +æĶ¶ åıĹ +æĪĸ 許 +èĥĮ éĿ¢ +æĸĩåĮĸ ä¼łåªĴ +ä¸ī åĢĭ +æĶ» åĬ¿ +å®ī 举 +å®ī举 å°¼ +åĿĩ å·² +顾 èĻij +éĦ Ń +è¿Ļ家 åħ¬åı¸ +åħ¬åijĬ 称 +æıIJä¾Ľ ä¼ĺè´¨ +稳æŃ¥ æİ¨è¿Ľ +å¤į è¯ķ +å°Ĩ é¢Ĩ +è°Ī èµ· +å¨ Ħ +è¿ŀ 线 +æ©Ł éĹľ +åºĶçĶ¨ åľºæĻ¯ +çĶ» åĥı +è´¢ è¿IJ +ä¿Ŀ éļª +çĹħ çIJĨ +æ¯Ľ 主å¸Ń +ä¸Ŀ 毫ä¸į +çĪ± å¥ĩ +çĪ±å¥ĩ èīº +ä¸ĵ家 ç»Ħ +åij¼ åĶ¤ +éĭ ¼ +çģ ¸ +é¢ĨåħĪ åľ°ä½į +æıIJ æĭĶ +龸 éģĵ +å±± åĿ¡ +èĿ İ +沸 èħ¾ +该 项 +ä»Ĭ çĶŁ +ä¸Ģç¯ĩ æĸĩç«ł +æĸ¹å¼ı è¿Ľè¡Į +é»ij 客 +æĶ¹ åĬ¨ +主 é¡Į +æķ£ å¸ĥ +ä»Ģä¹Ī åľ°æĸ¹ +åĮĸ åIJĪ +åĮĸåIJĪ çī© +éĿĻ çĶµ +æĢ» æĶ¶åħ¥ +å§Ķ ç»Ħç»ĩ +å§Ķç»Ħç»ĩ éĥ¨ +éĿĻ æĢģ +èĢģ åŃĹåı· +室 åıĭ +éĥ½ä¸į æķ¢ +æŀ¶ åŃIJ +çģµ æķı +审 è§Ĩ +æĤ£ åĦ¿ +å±± 寨 +èĸª èµĦ +é©° æı´ +éĥ¨åĪĨ åĨħ容 +好 ä¼¼ +æĪIJåijĺ åĽ½ +åľ¨æĪij çľĭæĿ¥ +åħ³æ³¨ 度 +éĻĪ æŁIJ +è¿Ļç§į äºĭæĥħ +éĢī å®ļ +ç²¾ åŃIJ +å£ģ çĶ» +æ±Ł æ·® +é«ĺ æĺĤ +æł¼ åĬĽ +è¼ © +åѦ åłĤ +æĤ¨ åIJĮæĦı +ä¸ĢåĪĩ éĥ½æĺ¯ +æ½ ¤ +éĸ ĥ +å¸ĮæľĽ èĩªå·± +ä¿ ĺ +æ±Ł åİ¿ +æ³ ¾ +ç§ij æķĻ +æīĵ è¿Ľ +ä¸į æħİ +å¯Ĵ åĨ¬ +æ¸Ķ æ°ij +鼷 æĸ¯ +主 å®° +æĹħ游 度åģĩ +çĶµåŃIJ éĤ®ä»¶ +æ±Ĥ å©ļ +éļİ æ®µ +åģ¥èº« æĪ¿ +注æĺİ åĩºå¤Ħ +äºĭæķħ åıijçĶŁ +级 以ä¸Ĭ +åŃĺ æ´» +æĸ½ èĤ¥ +èľľ èľĤ +åµ © +æĮĸæİĺ æľº +æĬĹ æĭĴ +ä¼ł 导 +æĺ¯ä»Ģä¹Ī åij¢ +ä¸Ĭå¹´ åIJĮæľŁ +建 åħļ +çĶŁ æħĭ +ä¿Ŀ ä½ı +款 车åŀĭ +人 èĦī +éļIJ èĶ½ +失 æķĪ +éģ¿ åŃķ +ç®Ģ 便 +谢谢 ä½ł +å®Ī ä½ı +æĶ¾ æĺł +è¨Ī çķ« +çݰ代 çī©æµģ +é¤IJ 廳 +æķħ å±ħ +大 大å°ı +大大å°ı å°ı +çī¹åĪ« 声æĺİ +éģį åıĬ +å¿ĥçIJĨ åĴ¨è¯¢ +è³ ´ +çĮ® è¡Ģ +å·²ç»ı è¾¾åĪ° +æīĵ æĭĽåij¼ +åıĮ è¾¹ +ä¸Ģæĸ¹éĿ¢ æĺ¯ +å´ĩ å°ļ +éĺ¿ å¯Į +éĺ¿å¯Į æ±Ĺ +æĮģ æľī人 +è± ģ +é£İ çŃĿ +åĬ¨ èį¡ +äºĨä¸Ģ ä¼ļ +äºĨä¸Ģä¼ļ åĦ¿ +ä¸ĩ 象 +çľĭ çĶµè§Ĩ +åįģä¸ī æĿ¡ +çĮĽ çĥĪ +è¦ģ ä¸įçĦ¶ +太æŀģ æĭ³ +å¼ķ çĪĨ +ç»ıè¿ĩ å¤ļå¹´ +游æĪı éĩĮçļĦ +é¾Ļ æ³ī +æłĩ éħį +è®ĵ ä»ĸåĢij +éĢł æŀĹ +åĮºåŁŁ æĢ§ +亿 ä¸ĩ +æĪĺçķ¥ å¸ĥå±Ģ +éķĩ æĶ¿åºľ +åĶ® 票 +çĶŁäº§ å·¥èīº +éķĩ åħļå§Ķ +ä¸Ńå°ı åŀĭ +æľ¨ èĢ³ +æ²³ è¾¹ +èĦ¾ èĥĥ +欢è¿İ æĤ¨ +åıĺ å¼Ĥ +缤 纷 +åŀĥåľ¾ 桶 +辩 è¯ģ +车 åºĵ +æ¯Ķ çİĩ +åħ´ æĹº +详ç»Ĩ äºĨ解 +å®ī å±ħ +çħ§ æĸĻ +æĸ¹ æīį +èµ ¦ +åĨ ķ +å¥Ķ èµ´ +å®Ŀ 鸡 +åľº åĿĩ +缮åīį æŃ£åľ¨ +åIJŀ åĻ¬ +è¿° èģĮ +æĩ µ +å¥ĩ çijŀ +ä»į å°Ĩ +èĪī 辦 +å·¥åķĨ å±Ģ +å¡ij èĥ¶ +åĬŀ å®ŀäºĭ +æĸ¹ æĸ¹éĿ¢ +æĸ¹æĸ¹éĿ¢ éĿ¢ +æĸĩåĮĸ èĬĤ +åħ¥ èģĮ +é¸ ¥ +ç©¿ éĢı +以 ä¹łè¿ijå¹³ +åį± éļª +æľ¦ èĥ§ +åİĨåı² æĢ§ +æķŀ å¼Ģ +ä¼Ļä¼´ åħ³ç³» +çŁ¿ åĮº +åĽ½éĻħ åľ¨çº¿ +ä¼łå¥ĩ éĩĮéĿ¢ +è¿ij äºĽ +è¿ijäºĽ å¹´ +åĬ£ åĬ¿ +æĶ»åĩ» åĬĽ +æĻº éĢł +ç¦ § +çİĭ åħĪçĶŁ +éĨ« çĶŁ +åĽĽ 项 +å®ŀ æĻ¯ +åĪĿ åĪĽ +å¿ĥ 裡 +æĻ¶ ä½ĵ +交 éĻħ +让 æ¶Īè´¹èĢħ +课 æĸĩ +æİĴ æ°Ķ +并ä¸į æĦıåij³ +缸 声 +第ä¸Ģ å±Ĭ +åİŁ èijĹ +éĽ ľ +没æľī 太大 +è¡¥ æ°´ +çī©æµģ ä¼ģä¸ļ +第äºĮ æī¹ +åħ¶å®ĥ éĹ®é¢ĺ +æİĮ éŨ +责任 å¿ĥ +é¤IJ åħ· +ç¾Ĭ æ¯Ľ +没æľī å¿ħè¦ģ +ä¹IJ åĽ¢ +è¿Ľ åŁİ +ä¸ĢçĤ¹ åĦ¿ +身 å½¢ +çļ®èĤ¤ çĹħ +æĺ ± +å¢ŀ èĩ³ +èģ² æĺİ +æıIJ è´¨ +ä½ĵèĤ² åľº +çŃ¹ 建 +é¬ Ĩ +车 çīĮ +éļĶ éŁ³ +è´Łè´£ åIJĮå¿Ĺ +丰 ç¡ķ +ä½Ľ éĻĢ +äºī åIJµ +åº ¶ +æ·¡ æ°´ +å°ı çĶ·åŃ© +ç§ģ èĩª +åĮĸ è¿Ľç¨ĭ +æĪĺ士 æĿ¥è¯´ +æ²¹ èħ» +èĦ±è´« èĩ´å¯Į +æĹ¥å¸¸ å·¥ä½ľ +交 èŀį +åĨľ è´¸ +åĨľè´¸ å¸Ĥåľº +åĵĪ çĻ» +çĶµ è´¹ +èµ ĺ +åıĮ èħ¿ +æĵĶ å¿ĥ +æĿ¥ 形容 +使åij½ æĦŁ +éĤ£ä¹Ī ç®Ģåįķ +èĬĻ èĵī +åĢŁæ¬¾ 人 +ç§Ģ 丽 +è®ĵ ä»ĸ +严åİī æīĵåĩ» +è³ ŀ +æļ « +çħ¤ æ°Ķ +çĪ¬ ä¸Ĭ +æ½ĩ æ´Ĵ +太 ä¹ħ +åij½ åIJį为 +è·¯ çĶ± +è·¯çĶ± åĻ¨ +é© ¯ +æıIJ æĹ© +æĬĹåĩ» çĸ«æĥħ +åĩ Ľ +交 åıĭ +éĶĢåĶ® æ¸łéģĵ +毫ä¸į çĬ¹è±« +èIJ¥ åľ° +çłĶ究 表æĺİ +é±¼ ç±» +æį¢ å±Ĭ +æİ¡ åıĸ +çī Ĩ +缼 å¼Ģ +沧 æ¡ij +åºŃ 审 +ç»ı æŁ¥ +åĬł å¼· +缸æ¯Ķ äºİ +ä¸ĵ çıŃ +ä½ĵ åŀĭ +被 害 +被害 人 +æĶ¶ 款 +åħ·æľī èī¯å¥½ +é«ĺå³° æľŁ +åģı ä½İ +åĦ Ł +åĨľä¸ļ ç§ijæĬĢ +çī¹æ®Ĭ æĥħåĨµ +å¦Ĥæŀľ çݩ家 +éķ¿ 约 +第åħŃ å±Ĭ +åħ¬å¼Ģ æĭĽèģĺ +åĪĩ æĸŃ +è¿« 使 +çĸĹ ç¨ĭ +第äºĮ ç§į +ä¸į åħį +å¹² èѦ +çŁ³ 榴 +åĹ £ +两 ç±» +çĪµ 士 +åŁİ乡 å±ħæ°ij +æѤ 项 +缴 è¾ĸ +缴è¾ĸ å¸Ĥ +åij¼ åºĶ +éĴ ¯ +ç¦ı å¾· +æľº 身 +æĵį åľº +æ¿Ĵ 临 +人群 ä¸Ń +èĤ¡ æ°ij +åŃ ½ +æ³ķ åħ° +é¨ İ +糯 ç±³ +æĢ» çļĦ +æĢ»çļĦ æĿ¥è¯´ +åħ¸ éĽħ +æĸ° éĻĪ +æĸ°éĻĪ 代谢 +缮 çĿ¹ +é¢Ħ è¨Ģ +è·Į çł´ +æĸ° ç¯ĩç«ł +æ¯Ĵ æĢ§ +åĸĿ èĮ¶ +æŁ¥ èİ· +亮 丽 +çĶŁäº§ åķĨ +æĶ¹ æĪIJ +为äºĨ æĽ´å¥½ +æ·± 交 +深交 æīĢ +æİ ĥ +ä¹Ļ èĤĿ +泸 å·ŀ +åħĪè¿Ľ æĬĢæľ¯ +è¾ĵ ç»Ļ +æķ£ æĪ· +æĢĿç»´ æĸ¹å¼ı +åºĹ 主 +è°ĭ æ±Ĥ +游æĪı æĬĢå·§ +ä¸Ģå¹´ 级 +çľ¼ è§Ĵ +ä¸Ńä»ĭ æľºæŀĦ +å·§ åIJĪ +éĺ² çĽĹ +导 è´Ń +æĪ Ĭ +æĽ´ éĢĤåIJĪ +åŁºæľ¬ ä¿¡æģ¯ +马 ä¸ģ +åħ»æ®ĸ åľº +åıį è¿ĩæĿ¥ +æݨ å´ĩ +å¯ĨåĪĩ åħ³æ³¨ +åŁºéĩij ç»ıçIJĨ +æĮī éĶ® +åĨħéĥ¨ æݧåĪ¶ +æĪIJåijĺ åįķä½į +æľ¯ è¯Ń +åĪ¶ æľį +åĪļ éľĢ +æ£Ģ ç´¢ +大大 æıIJé«ĺ +åģ¥åº· 管çIJĨ +èĩª æѤ +客æĪ· éľĢæ±Ĥ +丰 èĥ¸ +èµ· éĩį +èµ·éĩį æľº +æ¬ł 缺 +æ¡Ī åŃIJ +æĥħ人 èĬĤ +åħļ æł¡ +è¢ ľ +该 åī§ +迷失 ä¼łå¥ĩ +ç»ļ 丽 +åķ ª +æĹł ç§ģ +éĢ² ä¸ĢæŃ¥ +第ä¸Ģ ç«ł +åĻ¨ åħ· +åĨľ èµĦ +確 實 +åºı åĪĹ +娱ä¹IJ å¹³åı° +èŀįèµĦ 秣èµģ +èµĦæºIJ åħ±äº« +èģ½ åĪ° +æIJŀ å¾Ĺ +继ç»Ń ä¿ĿæĮģ +åIJ¯ èĴĻ +çľ º +ä¸Ŀ è·¯ +设æĸ½ 建设 +æİ¥ åľ° +æİ¥åľ° æ°Ķ +第ä¸ī åŃ£åº¦ +åŁº è°ĥ +åıij éŁ³ +社ä¼ļ èµĦæľ¬ +éĽĩ 主 +è¿ŀ èĥľ +没 åķ¥ +å» ¢ +赶 èµ´ +æ¼Ķ åĮĸ +åı¤ æĢª +çİĭ çĪ· +é¢Ħ åħĪ +å¼Ģ åħ· +åĽŀ é¦ĸ +åľ°ä¸ĭ æ°´ +å°ıç¼ĸ ä¸Ģèµ· +èµİ åĽŀ +åľ° è²Į +åĪĿ ä¸ī +åı¯ çĶ¨äºİ +éģĹ è¿¹ +è¿Ļ æī¹ +èĸª æ°´ +å¿ħçĦ¶ ä¼ļ +æ² ½ +éį ĭ +第ä¸Ģ éĥ¨ +åĪĬ çī© +å®ŀ ä¾ĭ +æ¸ħ åĩĢ +ä¸Ĭ èµĽåŃ£ +åĽ¾ 表 +éĤ® è½® +åĵª 裡 +缸 è§ģ +æī° ä¹± +æ¯ı æ¯ı +è¿Ļ è¾ĪåŃIJ +ç¡« éħ¸ +äºī 缸 +溯 æºIJ +åĩº ä¼Ĺ +çİī çŁ³ +åħ± çĶŁ +æŶéĹ´ 段 +éĩįè¦ģ æĮĩ示 +æ¶Īè´¹ éľĢæ±Ĥ +éķ¿ éķ¿ +éķ¿éķ¿ çļĦ +å®ī æĬļ +å¢ŀ é«ĺ +æľ¬ è½® +亲 çľ¼ +é£İ æ³¢ +èĢģ å¦Ī +æĶ¶è´¹ æłĩåĩĨ +åĨħ éĻĨ +æĮ¥ åıij +åįĩ åѦ +èĥ¸ åīį +åģı è¿ľ +纯 æ´ģ +æĸ½å·¥ åįķä½į +身 ä»· +è´¢ åĬĽ +çº ¶ +è£ħ çĶ² +æĺ¾ç¤º åĻ¨ +毫 åįĩ +æ·± çŁ¥ +èĢ¶ ç© +èĢ¶ç© Į +è¾ĥ éĩı +åľ¨ è¿ĩ渡 +åľ¨è¿ĩ渡 æľŁ +èĮ Ĺ +ä¸Ģ个 æĺŁæľŁ +èĬ · +è´¿ èµĤ +æ¿ ķ +æĩĤ äºĭ +ç§ § +åħħ å½ĵ +åĽ½ ç«ĭ +èĬ± çĵ£ +éĤĦ è¦ģ +åħ¬ åľĴ +触 åĬ¨ +æ³° å·ŀ +ä»Ģä¹Ī æł· +æ»ĭ åħ» +è¯Ħ åĪ¤ +æĮ¥ æīĭ +èĦ Ī +姥 姥 +è¿IJ è´¹ +æ¯ħ åĬĽ +å¿ĥ æĻº +ä¸į æİĴéĻ¤ +第ä¸ī 代 +éĢĢ è´§ +æĺŁ éĻħ +æ°¸ åĪ© +æĬ¤ åį« +çıŃ è½¦ +è¨Ģ è¡Į +ç¹ ª +主åĬ¨ æĢ§ +å·¥ç¨ĭ è´¨éĩı +éĥĬ åĮº +ä¸Ģ æłĭ +ä½Ĩ å®ŀéĻħä¸Ĭ +ä¸ī大 èģĮä¸ļ +åij¼ åı« +女 åħĴ +è¯ģåĪ¸ æĬķèµĦ +èĢĥ æħ® +çĤ« èĢĢ +æ²» 好 +åĺ ¶ +èĥ ¤ +åħīä¼ı åıijçĶµ +åĩł æŃ¥ +æīĢ æīĢ +æīĢæīĢ éķ¿ +çħ§ æł· +åĵ¥ 们 +è¯ Ľ +è¿Ļä¸Ģ åĪ» +çŁ¿ çī©è´¨ +ä¸įå¾Ĺ å·² +åIJĮ 缣 +ç»Ĩ å¾® +è·¯ èĻİ +çĻ¾ èĬ± +æ·· æ²Į +ä¸Ĭæµ· è¯ģåĪ¸ +éĢĢ ç¨İ +èµŀ åı¹ +æī®æ¼Ķ 游æĪı +åIJį åĪĹ +åIJįåĪĹ åīį +åIJįåĪĹåīį èĮħ +ç±³ å°Ķ +ä»Ģä¹Ī åİŁåĽł +å®īåħ¨ ä¿Ŀéļľ +ä¸Ģåıª æīĭ +ä¹³ ä¸ļ +ä¸į çĶĺ +æĥħ åķĨ +æĮ¡ ä½ı +åİŁåĽł ä¹ĭä¸Ģ +è¿Ļ 两天 +çĥĺ çĦĻ +è± ¬ +ä½ł 以为 +没 è§ģè¿ĩ +åĵªå®¶ 好 +åīį ä»» +è¿Ľ è´§ +éĢĢ åĽŀ +串 èģĶ +èĩ³ æĸ¼ +åĨ° æ·ĩ +åĨ°æ·ĩ æ·ĭ +æŁ¥çľĭ 详æĥħ +çı¾ 實 +æݨ æµĭ +æİ¥ æīĭ +éļ¶ å±ŀäºİ +åŁİå¸Ĥ 群 +æĿİ åħĪçĶŁ +çŁ¿ æ³īæ°´ +çī¹ ä»· +æĽ´å¤ļ 精彩 +ç¨ĭ å¼ı +读 æĩĤ +å±ı èĶ½ +奥 æŀĹ +奥æŀĹ åĮ¹ +奥æŀĹåĮ¹ åħĭ +红 èĸ¯ +å¥ ® +å®Ŀ çİī +網 絡 +è² § +欧 å¼ı +çĻ½ ç³ĸ +èĩªçĦ¶ çģ¾å®³ +åijĬè¯ī 她 +å» ļ +çĤ¹åĩ» æŁ¥çľĭ +é£İ 湿 +èµĦ产 éĩįç»Ħ +ä¹Łä¸į ä¾ĭå¤ĸ +åįĬ 个å°ıæŶ +åIJ¸å¼ķ æĽ´å¤ļ +æŶéĹ´ èĬĤçĤ¹ +æĶ¶ 纳 +åIJ¸ æ¯Ĵ +èĢģ 乡 +çIJ ħ +æľĢ çµĤ +åıį æĦŁ +çĶ¨ 微信 +çĶ¨å¾®ä¿¡ æī« +éĢŁ çİĩ +大 çĨĬçĮ« +åı¯ æĥ³ +åı¯æĥ³ èĢĮ +åı¯æĥ³èĢĮ çŁ¥ +åĴ § +èµ° åħ¥ +碳 éħ¸ +èĮĥ åĨ° +èĮĥåĨ° åĨ° +被 åĪ¤ +积æŀģ æݨåĬ¨ +足 足 +ç²Ĵ åŃIJ +大 å®Ĺ +大å®Ĺ åķĨåĵģ +ç½ij绾 ç§ijæĬĢ +æĽ¼ åŁİ +å·² ä¹ħ +å·²ä¹ħ çļĦ +秦 çļĩ +秦çļĩ å²Ľ +ä»» æķĻ +åĶ¯ ç¾İ +æ·¡ åĮĸ +æ¡Ĥ èĬ± +çŁ¥è¯Ĩ åĪĨåŃIJ +æĩĴ å¾Ĺ +主 åħ¬ +设计 çIJĨ念 +è³ º +æīĢ æıIJä¾Ľ +æīĢæıIJä¾Ľ ä¹ĭ +æĶ» åħĭ +åĤ ¾ +è¯Ń æ³ķ +åįĥ åı¤ +éĸĭ æĶ¾ +第ä¸Ģ èĬĤ +éĤĦ æ²Ĵ +éĢĥ çĶŁ +æ³ Ĺ +åİ¿ å§Ķ书记 +ä½ľèĢħ æīĢæľī +çħ ½ +ç» ħ +æł ħ +æľ´ ç´ł +çijķ çĸµ +åĮħ åĮħ +æ°ij主 åħļ +ä¸į è¿ľå¤Ħ +å¥ĩ å¼Ĥ +åĺ» åĺ» +æī ¼ +ç¿» å¼Ģ +æĢİ èĥ½ +éģ´ éĢī +解 éĩĭ +å¹¼ ç¨ļ +è¦ģ 好好 +趴 åľ¨ +ç´¢ åıĸ +ç»Ī çĶŁ +åħ¨ æµģç¨ĭ +éģ© çķ¶ +åįıè°ĥ åıijå±ķ +æĬ¥ ä»ĩ +ç§ijæĬĢ åĽŃ +ä»Ģä¹Ī éĥ½ä¸į +æľĢåIJİ ä¸Ģ次 +ç»Ļ人 ä¸Ģç§į +æł¸ å®ļ +被 åĪĹåħ¥ +æĦı æĥ³ä¸įåĪ° +èĢĥ æŁ¥ +åľ¨æѤ ä¹ĭåīį +æīĵ çIJĥ +è¶ĬæĿ¥è¶Ĭ å°ij +å®ļ å¾ĭ +è¡ĮæĶ¿ æľºåħ³ +ä½ıæĪ¿ åħ¬ç§¯ +å°ıå§IJ å§IJ +ä¸ī èı± +ä¿® è¡¥ +èŀĥ èŁ¹ +西 çĶ² +æĢ ł +çŃī å¤ļ项 +产ä¸ļ éĽĨèģļ +ä»·æł¼ ä¸Ĭ涨 +åħ¬åħ± åľºæīĢ +è¢ĭ åŃIJ +æĨ§ æĨ¬ +çļĦæĸ¹å¼ı æĿ¥ +åĪ° è´¦ +çģ ½ +å·´ èı² +å·´èı² çī¹ +æ¼Ķ ä¹ł +èŃ¦ç¤º æķĻèĤ² +çķı æĥ§ +å¼ķ æµģ +æĶ¶ æĶ¯ +å±Ĥ åĩº +å±Ĥåĩº ä¸į +å±Ĥåĩºä¸į ç©· +æijĩ æ»ļ +辦 çIJĨ +纵 è§Ĥ +æķij æµİ +家 éĥ½çŁ¥éģĵ +åĮ ¯ +å°ı 鸣 +ä»» åĭĻ +计 åħ¥ +ç«ŀ éĢī +å¼ĢèįĴ æĹ¶æľŁ +åij¨ æģ© +åij¨æģ© æĿ¥ +交 ç»ĩ +çķ¢ æ¥Ń +æł¹æį® èĩªå·± +æĸ°äºº çݩ家 +åѵåĮĸ åĻ¨ +éĩĩ æļĸ +å¹³åĿĩ æ°´å¹³ +åħ¬å¼Ģ 课 +失 åĪ© +伺 æľį +çĬ ģ +忽 æĤł +主è¦ģ éĽĨä¸Ń +æ¤į æłij +æ¯Ĺ éĤ» +èĩº çģ£ +åĩºåĽ½ çķĻåѦ +æĬĹ éľĩ +æĥ© æĪĴ +å¹´åºķ åīį +åĴ¸ éĺ³ +æ°ij å±ħ +大çIJĨ çŁ³ +éĿ ³ +éķ ĸ +æ¸ħ è¿ľ +è£ħ è½½ +èĩ Ģ +å½± ä¸ļ +å¼Ł åħĦ +æĤ² è§Ĥ +çĿĢçľ¼ äºİ +æįį åį« +åī¥ 夺 +ç¯ Ĩ +å¾Ī éķ¿æŶéĹ´ +è¥ Ł +第ä¸Ģ çĻ¾ +ä¸ĢåĪĨ éĴ± +æĸ°éĹ» è®°èĢħ +éķ· æľŁ +æ³ķ æĪĺç»ĦåIJĪ +è°ģ çŁ¥éģĵ +èħ° éĥ¨ +æ±ī åł¡ +åħ¥ çĿ¡ +åįĸ æİī +æ¶Īè²» èĢħ +æĥ¯ ä¾ĭ +æĥ³ äºĨ +æĥ³äºĨ æĥ³ +èĢģæŧ å°ıåĮº +ä¼ł è¨Ģ +åĪĨæķ° 线 +æµģ 泪 +ç»Ħç»ĩ é¢Ĩ导 +äºļ åĨĽ +å¢ŀåĢ¼ æľįåĬ¡ +å¾ ¹ +ä¼ ¶ +äºĽ 许 +å¸ĥ èݱ +强 æĤį +宫 å»· +绿 èĮ¶ +åĮ ¡ +å¾Ī æŃ£å¸¸ +æĺ¥ å¤ı +æ¯ Ļ +è¯Ħ æ¯Ķ +åĩ¡ äºĭ +æĬī æĭ© +åĢĴ éľī +éĩį 度 +åįıä¼ļ ä¼ļéķ¿ +忧 èĻij +ä¸ĭ ä¸Ģç¯ĩ +沪 æ·± +æĪ İ +æīĵ ä»Ĺ +åįĪ é¥Ń +å¹´é¾Ħ 段 +ä¸ŃåĽ½ 足çIJĥ +设计 æĸ¹æ¡Ī +åºĶçĶ¨ æŁ¥çľĭ +é¢Ħ æĸĻ +åĹ ¡ +ç¥ĸ çĪ¶ +çļĦä¸Ģ åijĺ +æ´Ĺ å¹²åĩĢ +åİĨåı² æĸ° +åİĨåı²æĸ° é«ĺ +çĭ¬ åħ· +æħĭ 度 +æīĵ 交 +æīĵ交 éģĵ +é»Ħ çŁ³ +çĽ¼ æľĽ +çī§ åľº +转 弯 +åįĩ åįİ +åĨį ä¹Łæ²¡æľī +èĭ± æīį +æĽ´ åIJį为 +åĢŁ çĶ¨ +çºł éĶĻ +ç»Ŀ对 ä¸įä¼ļ +çİĭ çīĮ +çĽĨ åľ° +失 è°ĥ +好 象 +é³ ¥ +ä¿Ŀ ä¿® +åĽĽä¸ª èĩªä¿¡ +头 çļ® +åİŁ åīĩ +æĬ¥ æ¡Ī +奴 éļ¶ +å³ Ļ +è°ĥ æĸĻ +ä¹Ł 許 +èIJ½ åĪ° +èIJ½åĪ° å®ŀ +èIJ½åĪ°å®ŀ å¤Ħ +çĦļ çĥ§ +çĶŁæ´» çݯå¢ĥ +åºĶ åıĬæŶ +è¶Ĭ è¿ĩ +æĦŁ è¬Ŀ +æĻ¯ å¾· +æĻ¯å¾· éķĩ +çĬ Ģ +身 éĤĬ +ç¨İåĬ¡ æĢ»å±Ģ +åĩĢ åľŁ +ä¾µ åįł +åĬ¨ å·¥ +å¹´ ä¹ĭ +å¹´ä¹ĭ ä¹ħ +第äºĮ èĬĤ +åĬ¨çī© åĽŃ +第ä¸Ģ 书记 +éħ ļ +çĶŁäº§ 设å¤ĩ +æŁIJç§į ç¨ĭ度 +åľ Ń +åĩŃåĢŁ çĿĢ +éĺħ è§Ī +çĻ½ æ²Ļ +æ²¹ çĥŁ +çªģçł´ åı£ +åıĹ å½±åĵį +åı¯ä»¥ æĽ´å¥½ +å³° åĢ¼ +æĿĤ è´¨ +宿 è¿ģ +çĽĺ æ´» +æ¿Ģ èµ· +åĦ¿ ç§ij +åĿIJ èIJ½åľ¨ +æĮª å¨ģ +æµ· å²Ľ +绣 绣 +éĻ ¨ +ä¼ĺ äºİ +å°Ī 家 +ä¸Ģ éĤĬ +èIJ Ĭ +äºĨä¸Ģ åı£ +æ²ĥå°Ķ æ²ĥ +æŃ£å¸¸ 使çĶ¨ +æĻ®éģį åŃĺåľ¨ +丰 满 +çĶ» åį· +åºĶ æĶ¶ +åºĶæĶ¶ è´¦ +åºĶæĶ¶è´¦ 款 +å®Įæķ´ çĥŃ +å®Įæķ´çĥŃ æ¦ľ +注 è§Ĩ +çĨ Ħ +èº ¬ +éĶĢåĶ® 人åijĺ +è¶ĭ åIJij +çĦ¦ æĢ¥ +åįģå¹´ åīį +ä¼łç»Ł 产ä¸ļ +質 éĩı +åĩ¤åĩ° ç½ij +èµĦæºIJ æķ´åIJĪ +æ¶Į åħ¥ +æĸĩåĮĸ ä¼łæĴŃ +çķĮ 第ä¸Ģ +æ°´ æ³µ +宫 殿 +æİ¢ 寻 +ä¿® åīª +æĦı è¦ĭ +ç´Ĭ ä¹± +æĽ ī +çĻ½ è¡£ +èĻİ åį« +ç´§ æī£ +å¤Ħå¤Ħ éķ¿ +åĪĽå»º å·¥ä½ľ +红 æŀ£ +饼 å¹² +äºĨ åįĬ天 +ä¼ļå½±åĵį åĪ° +çĽ¸ä¿¡ 大家 +èħ¾ é£ŀ +å°± å¦ĤåIJĮ +ä¸ĭéĿ¢ å°ıç¼ĸ +æ°ijèIJ¥ ç»ıæµİ +æĻ ¦ +è£ħ æī® +é»ij å¤ľ +常 å¾· +å·¥ä¸ļ 大åѦ +æĺİ çŁ¥ +éĺŁåijĺ 们 +åIJ¬ 课 +æ¯ı éļĶ +羣æĺ¯ å¤ª +åIJĪä½ľ åħ±èµ¢ +çIJĨ åıij +æīį å¹² +çľĭ èµ·ä¾Ĩ +殿 ä¸ĭ +å®ī éĺ³ +æīĢ 产çĶŁçļĦ +éĽĩ ä½£ +æĬ¬èµ· 头 +æį® æĬ¥éģĵ +éļĨéĩį 举è¡Į +交 éĶĻ +è¶ħ é¢Ŀ +åĮĸ çĸĹ +é¡ Ĩ +纵 æ·± +çĪ±åĽ½ 主ä¹ī +éĻ¢ åī¯éĻ¢éķ¿ +è® ³ +羣æŃ£ åģļåĪ° +åѤ åįķ +èĩªçĦ¶ èĢĮ +èĩªçĦ¶èĢĮ çĦ¶ +ä¿® 身 +èĬ ¹ +æģ¯ æģ¯ +æģ¯æģ¯ 缸åħ³ +驾 æł¡ +æİ© 饰 +æ³½ è¿ŀ +æ³½è¿ŀ æĸ¯åŁº +举 æŃ¢ +管çIJĨ ä½ĵåĪ¶ +åħ¶ä¸Ń ä¹ĭä¸Ģ +æĿ¾ å¼Ľ +æĭ¦ æĪª +åį« åģ¥ +åį«åģ¥ å§Ķ +ä»İ åݻ年 +åĤ ¢ +è´Ń 票 +åĽ¾ æłĩ +æ²³ 西 +æ°ijæĶ¿ å±Ģ +ç§ģ èIJ¥ +å¤ĸåĽ½ è¯Ń +å¹² è´§ +æĵ¦ æĭŃ +åľ° ä¸Ń +åľ°ä¸Ń æµ· +æµĵ æµĵ +æµĵæµĵ çļĦ +å§ĭ 建 +å§ĭ建 äºİ +ç¶ĵ æŃ· +è·¯ æ¼Ķ +æļ´ é£İ +åŁº è¾ħ +æī¶è´« å·¥ä½ľ +ä¸Ģ缴 å¤Ħäºİ +æĥħ 趣 +äºĮ åŃ£åº¦ +åİĮ æģ¶ +顺åĪ© å®ĮæĪIJ +æŁ¥ å°ģ +顶 端 +ä¸į åŃķ +ä¸Ģ大 åłĨ +被 æ·ĺæ±° +æĺ¯ çĶ¨æĿ¥ +æľĢ åIJĪéĢĤ +亮 çľ¼ +并ä¸įæĺ¯ å¾Ī +ç§ijçłĶ éĻ¢ +ç§ijçłĶéĻ¢ æīĢ +ç² Ł +é¢Ī éĥ¨ +é»ĺé»ĺ åľ° +é«ĺä¸Ń çĶŁ +æĹıèĩªæ²» åİ¿ +æķĻåѦ è´¨éĩı +æĪĺ çģ« +åĿİ åĿ· +æIJŃ ä¹ĺ +è¯Ĺ æĦı +åĪij èѦ +åĩº æ±Ĺ +åįģåħŃ æĿ¡ +请 åıĬæŶ +åĨľä¸ļ 大åѦ +èIJ½ åı¶ +æĢ» èĢĮè¨Ģ +æĢ»èĢĮè¨Ģ ä¹ĭ +æĿľ åħ° +æĿľåħ° çī¹ +éĻª ä½ł +åħ¬ æĬ¥ +çķĻè¨Ģ æĿ¿ +éĺħ åİĨ +競 çĪŃ +ç»Ļ åĪ«äºº +æĹ¥æĬ¥ 社 +åĿIJ èIJ½ +åĿIJèIJ½ äºİ +éĩij åŃĹ +éĩijåŃĹ å¡Ķ +åĽ ¤ +è¯Ŀ åī§ +æĮģç»Ń æİ¨è¿Ľ +æ¼ı æ°´ +詳 ç´° +æĢĢ æĬ± +åıĺ å¹» +饥 饿 +éļIJ 身 +个 èµĽåŃ£ +åĵ¡ å·¥ +æģ¢å¤į æŃ£å¸¸ +äºĨ 好å¤ļ +æĺŁ å·´ +æĺŁå·´ åħĭ +åħī çݯ +å¸ħ åĵ¥ +çĻ½ éĽª +ç¨į ç¨į +计 æıIJ +æĦĽ æĥħ +éİ ĸ +ä¿¡ éĺ³ +è§Ģ å¯Ł +å¦Ĥæŀľä½ł æĥ³ +缸æ¯Ķ ä¹ĭä¸ĭ +解 å¼Ģ +æīĵåį° æľº +身 躯 +ç²¾ç¥ŀ æĸĩæĺİ +èĤ¡ æĮĩ +å¾® åĪĽ +红 èĮ¶ +èĩ´ çĻĮ +æģ© æĸ½ +èħ¿ éĥ¨ +大åŀĭ å¤ļ人 +å®ī åĢį +è¾ħ导 åijĺ +èĪª éģĵ +å¸ĥ å°Ķ +åįĹå®ģ å¸Ĥ +ä¸ĬçıŃ æĹı +侧 ç»ĵæŀĦæĢ§ +追 éļı +å½ĵåľ° æĶ¿åºľ +èµ° åĩºæĿ¥ +éĩijèŀį ä¸ļ +丼 书 +é¡¹çĽ® ç»ıçIJĨ +è¿ĩ æĪ· +骨 æŀ¶ +è¡ Ļ +ä»Ģ 麽 +èħ ĭ +è¦ģ 害 +åľ¨ åºĬä¸Ĭ +代è¨Ģ 人 +並 å°ĩ +åIJĦ个 æĸ¹éĿ¢ +è°´ è´£ +åħ± æĮ¯ +åį³å°Ĩ åĪ°æĿ¥ +èĤº çĻĮ +ä¾Ľ éĶĢ +丼 æŀĹ +èµ ĥ +åįģä½Ļ å¹´ +åĭĺ æİ¢ +飵 åij³ +èĭ¦ ç¬ij +æľĢ大 ç¨ĭ度 +éĩįçĤ¹ åħ³æ³¨ +ä¹ĭ 举 +满 æĢĢ +åıĹåĪ° å½±åĵį +æĭĽ æĬķæłĩ +è¡¥ é½IJ +西 红 +西红 æŁ¿ +é¬ § +è£ħ åį¸ +éĤ» éĩĮ +èĤĩ äºĭ +æİĴ æ¯Ĵ +åѤ åĦ¿ +鼶 è·Ŀ离 +å®ŀ å¹² +çľĭ æŁ¥çľĭ +æĶ¶è´¹ ç«Ļ +ç» · +åħ¬çĽĬ æĢ§ +éĢĴ ç»Ļ +æĶ» æīĵ +æĺŁçº§ éħĴåºĹ +æĺİ åªļ +çį¨ ç«ĭ +è¯Ŀè¯Ń æĿĥ +ä¸ĢæŃ¥ ä¸ĢæŃ¥ +书æ³ķ 家 +æľªç»ı æİĪæĿĥ +çŁ³ èĨı +åĩŃ ä»Ģä¹Ī +çļĦ æĹ¥ +çļĦæĹ¥ åŃIJéĩĮ +诱 人 +çĻ¾åĪĨ çĻ¾ +èĪĪ 趣 +å¼ł åħĪçĶŁ +èĢģçĪ· åŃIJ +æ³¢ çī¹ +åŁºéĩij 份é¢Ŀ +æ²Ļåıij ä¸Ĭ +å¥ĭæĸŠ缮æłĩ +æ°¢ èĥ½ +æ²ĥå°Ķ çİĽ +義 åĭĻ +éŁ³ ç®± +æ²ī 浸 +æ²ī浸 åľ¨ +èĭ± åľĭ +çģ¯ çģ« +è¿Ľ 项 +两 端 +ä¹Ķ 丹 +èĦ¸ é¢Ĭ +åıijå±ķ æ½ľåĬĽ +åĭķ ä½ľ +åĵĪ ä½Ľ +å®´ ä¼ļ +æ§ į +ç«ĭ å¿Ĺ +ç¡ķ士 åѦä½į +åĭĭ ç«ł +è¿Ļ åľºæ¯ĶèµĽ +æĮģ å¹³ +éķĢ éĶĮ +èĭ± çī¹ +èĭ±çī¹ å°Ķ +æķĻ èģĮå·¥ +åĬŁ åĬĽ +该 æ¡Ī +ä¸Ģ æ¢Ŀ +åĺī å¹´ +åĺīå¹´ åįİ +è¿« ä¸įåıĬ +è¿«ä¸įåıĬ å¾ħ +è¿Ļ个 æĹ¶ä»£ +精彩 æĴŃæĬ¥ +人 èĦ¸ +人èĦ¸ è¯ĨåĪ« +æ£Ģå¯Ł å®ĺ +å°ı èħ¿ +éĨĴ 缮 +åħļ æĢ» +åħļæĢ» æĶ¯ +æĪ Ł +èĮ« çĦ¶ +è±Ĩ æµĨ +主 æ²» +éĿĴæµ· çľģ +åĪijäºĭ 责任 +çł ° +ä¹ĭ æ¬ĬåĪ© +äºĶ å®ĺ +è¿· æĥij +åħ¥ åºĵ +家 纺 +å¼¹ ç°§ +åįģäºĶ æĿ¡ +ç»Ļ å®Ŀå®Ŀ +èĪªç©º èĪªå¤© +å¾Ģ å¤ĸ +å¼ķ åĬĽ +çľ¼ çļ® +æ¶ī 足 +æĿ¥ 宾 +åľ¨çº¿ è§Ĵèī² +çĥŃ éĶĢ +æµģ éĢĿ +泡 泡 +éĻį å¹ħ +è´ŁéĿ¢ å½±åĵį +红 楼 +红楼 梦 +éļĶ çĿĢ +ä¾¥ 幸 +许 ä¹ħ +åĴĮ çĿ¦ +èŃ ½ +使çĶ¨èĢħ æĪĸ +ä¹° åįķ +è¿ ´ +é£İ æīĩ +æķĻ 師 +æ¡ĮåŃIJ ä¸Ĭ +å¾Ī æ¼Ĥ亮 +åł± å°İ +第ä¸Ģ åŃ£åº¦ +ç©© å®ļ +æĤ² åĵĢ +çĿĢåĬĽ æīĵéĢł +æĮ Ł +è·¯ æ¡¥ +åij IJ +åľ£è¯ŀ èĬĤ +çļĩ åŃIJ +ä»ĩ æģ¨ +éħĿ éħ¿ +ä¸į éĹ´ +ä¸įéĹ´ æĸŃ +æĮĩ å°ĸ +ä¸ŃåĽ½ ç½ij游 +åŀ £ +æĦıè§ģ 建议 +æ¯ħ çĦ¶ +亮 度 +èģĶ è°Ĭ +å½ķ åħ¥ +åĦ ² +å¨ĺ 家 +ç§ij å°Ķ +ä¹Łæ²¡ ä»Ģä¹Ī +æł¹æį® ä¸įåIJĮ +åı¶ ä¿® +åĢ¼ å®Ī +æľ« 端 +åĪ ¨ +åĤµ åĭĻ +èģ¯ åIJĪ +å¥ĩ å¹» +èĻļ æŀĦ +é»Ħ æĺı +å¹³ åĿ¦ +æµģ æ°ĵ +æĸ° åŁºå»º +æĮ½ æķij +åįİ å°Ķ +åįİå°Ķ è¡Ĺ +æľĢ åıĹ欢è¿İ +ç»Ń 约 +å¼Ĭ 端 +éŃĶ æ³ķå¸Ī +éŃĶæ³ķå¸Ī åĴĮ +åħ·ä½ĵ åĨħ容 +çIJī çĴĥ +æī© 容 +èĮ¶ åĽŃ +主ä¹ī èĢħ +ç«ĭ éĿ¢ +æİ¥åıĹ éĩĩ访 +åĩº åħ¥å¢ĥ +ç§ij åįı +éĴ ³ +çµIJ æ§ĭ +ç»ĵæŀľ æĺ¾ç¤º +åı° è´¦ +å°± æĿ¥çľĭçľĭ +èĩª æķij +åıį æĩī +åİ» åĵªåĦ¿ +è¿Ļ é¦ĸ +è¿Ļé¦ĸ æŃĮ +åIJ¬ ä¼Ĺ +å¤ĸ 壳 +ä½ĵèĤ² é¦Ĩ +實 æĸ½ +èŀº ä¸Ŀ +æĭī åįĩ +çĮĽ åľ° +åħ¨åĽ½ 人æ°ij +æĤī å°¼ +æĹı 群 +åĽ¢ åijĺ +两个 å°ıæŶ +åľ¨ çݩ家 +åľ¨çݩ家 ä¸Ń +çĶľ çĶľ +æĬķ è¡Į +åįĶ æľĥ +éĻ ¡ +åĬłå·¥ åİĤ +æ¦Ĩ æŀĹ +æŃ» è§Ĵ +åĨħ å¹ķ +æīĢæľī æĥħèĬĤ +åĪ· åį¡ +æ°´ èĤ¿ +èĥĥ åı£ +å«Į å¼ĥ +æ²® 丧 +ä¸īå¹´ 级 +æ¶Ĥ å±Ĥ +å¿ĥ 仪 +å¿ĥ仪 çļĦ +å¤ Ń +é¦ĸ è½® +æĹłè®ºæĺ¯ åħ¶ +éĢı æ°Ķ +äºĮ åįģäºĶ +ç® « +åĬŁ åĬ³ +çŃ¾ ä¸ĭ +æ²ī è¿· +æķij åij½ +éĹª éĹª +åIJĥ äºı +å±ķ åĵģ +åį³æŶ åıijçĶŁ +ç¶ ľ +綾 åIJĪ +æłĩ æĺİ +çľĭ çĶµå½± +åħ¬ ç«ł +éĺ¿ æ£® +éĺ¿æ£® 纳 +身 åĪĽéĢł +身åĪĽéĢł çļĦ +æ¸Ľ å°ij +åĢ¼å¾Ĺ åħ³æ³¨ +鼶åĶ® åķĨ +æįĨ ç»ij +è¸ı åħ¥ +èĽ Ł +æŁ´ 纳 +èĢģ åħµ +绿èī² çݯä¿Ŀ +é¹ Ń +麻 æľ¨ +æıŃ çīĮ +è¿Ļ款 车 +ç¾İ å¾· +ç¾İå¾· åħ¬åı¸ +æ¶ § +è°ģ çŁ¥ +æ´ĭ èij± +æ¯į æł¡ +ä¸Ģ éĹª +çĶ· 主è§Ĵ +æĹłçº¿ çĶµ +å±ł å®° +æĺ¯ éŁ©åĽ½ +æĺ¯éŁ©åĽ½ 娱 +容 è²Į +åĿĩ 使åħ¶ +太 å¿« +å¹´ çĶ± +å¹´çĶ± 缼 +èĭ¦ èĭ¦ +åĬĽ è¿ĺæĺ¯ +åĬĽè¿ĺæĺ¯ èĩª +æĨ © +èģ¯ 絡 +åĶ ¾ +åħ·æľī æĪĺ士 +追 éĹ® +åłĨ æĶ¾ +åıį 驳 +å®ŀäºĭ æ±Ĥ +å®ŀäºĭæ±Ĥ æĺ¯ +åѸ éĻ¢ +åįģ åĩłä¸ª +æķij æĬ¤ +æķijæĬ¤ 车 +ç½ij绾 ä¼łæĴŃ +åįģåħ« å±Ĭ +éĥ¨ åī¯ +éĥ¨åī¯ éĥ¨éķ¿ +çĹ´ è¿· +管çIJĨ æĿ¡ä¾ĭ +èŀį 为ä¸Ģä½ĵ +æĢ» 产åĢ¼ +è³ ĵ +ä¸ĥ æĺŁ +çıŃ ç»Ħ +绣 é¢Ĩ +请 大家 +éĩij éĻµ +èĪħ èĪħ +æµ· æ¹¾ +æĸ½ çŃĸ +享 èªī +éº ¥ +端 åįĪ +绿 åŁİ +確 ä¿Ŀ +å·´ æĭī +åĨĴ çĿĢ +æħ· æħ¨ +个人 è§ĤçĤ¹ +ä¹Ļ çĥ¯ +ç¡ħ è°· +éĸĭ å±ķ +å°ļ 书 +åĿļ 飧 +åº µ +èĢģ é¾Ħ +èĢģé¾Ħ åĮĸ +羨 çľ¼ +绿 æ°´ +绿水 éĿĴå±± +书 é¦Ļ +主åĬĽ åĨĽ +æīįæĺ¯ çľŁæŃ£ +æĬ¢ åħĪ +æĪIJå°± æĦŁ +éĩį æŀĦ +éĴ¢ åİĤ +æĪIJ 份 +èĬ± 纹 +ä¹ĭ äºī +å¹² ç»Ĩèĥŀ +æĹ¢ åı¯ä»¥ +ç¹ģ çIJIJ +æĦļ èł¢ +éĿŀ常 æĺİæĺ¾ +ä½ĵ 彩 +æĬĢ æ³ķ +æĿĨ èıĮ +å¹¿æ³Ľ åħ³æ³¨ +åĮĹ å®ĭ +å§Ĭ 妹 +åįı åĬŀ +æ·® åįĹ +çĥ ı +æ´Ĺ èĦ¸ +åıĹ è®¿ +åıĹ访 èĢħ +éĩįè¦ģ åĽłç´ł +å½±è§Ĩ åī§ +综èīº èĬĤ缮 +èľķ åıĺ +äºĮ 线 +äºĮ线 åŁİå¸Ĥ +ä¼Ĭ å§ĭ +çıĬ çijļ +èĩª æŁ¥ +åħ¥ åĽŃ +åĩ¶ æīĭ +åħ¬ è¯ī +éģĩ éļ¾ +éĩĩçŁ¿ çŃī +èĩª çIJĨ +åĸ· æ¶Ĥ +æī© åħħ +éĢı è§Ĩ +é«ĺéĢŁ å¢ŀéķ¿ +åĽ¾ çĶ» +ç¾ ¹ +èĤĩ åºĨ +è¾ľ è´Ł +èµĶ ä»ĺ +è· ¡ +åģ¥åº· æĪIJéķ¿ +以ä¸Ĭ åѦåİĨ +åıĸå¾Ĺ 以åıĬ +æ²ī 积 +åįģä¹Ŀ å±Ĭ +缸éĹľ æľįåĭĻ +æī§ åĭ¤ +åī¯ åİ¿éķ¿ +å¯ ° +åģľ æ»ŀ +æ·¹ 没 +çŁ³ çģ° +çį ¸ +åĢ ¦ +ç¾İ åªĴ +æķĻ æ¡Ī +åĬł çĽĸ +åħ¬å¼Ģ èµĽ +å¥ł åŁº +æĺĨ èĻ« +çŀ ħ +磷 éħ¸ +äºī åĪĽ +çİĭ æĻĵ +ç¼ĵ åĨ² +åİļ åİļ +åİļåİļ çļĦ +æŀ£ åºĦ +ç²¾ çĽĬ +ç²¾çĽĬ æ±Ĥ +ç²¾çĽĬæ±Ĥ ç²¾ +åĪĨæĶ¯ æľºæŀĦ +å®ŀæĸ½ ç»ĨåĪĻ +æĸ° èµĽåŃ£ +總 çµ± +éĢł è¡Ģ +é¢ĩ åħ· +é»Ħ åŁĶ +è¡Ģ èĦĤ +交éĢļ å·¥åħ· +å³ ¥ +æĹıèĩªæ²» å·ŀ +寺 éĻ¢ +確 å®ļ +æ¦Ĥ念 èĤ¡ +æĦŁ å®ĺ +æŁľ åı° +åĶ Ķ +çŀŃ解 並 +æĢ» ä»· +åIJ¸ åħ¥ +æĢ ¼ +æĻļ éĹ´ +å±Ĭ æ¯ķä¸ļçĶŁ +çĶŁ å§ľ +éĺħ读 åħ¨æĸĩ +å¾ĹåĪ° æľīæķĪ +æIJľ æķij +åİĨ æĿ¥ +èŃī æĺİ +åĥ » +èĨ³ é£Ł +åĦĦ åħĥ +æīĵ åİĭ +宾 客 +åķ ¼ +ä¸ĢçĻ¾ å¤ļ +æ·±åħ¥ 人å¿ĥ +æ¢ħ å·ŀ +çłĶ åѦ +åħ³ ä¹İ +è¼ Ľ +亲 åıĭ +éħį æĸĻ +æĪij çĪ±ä½ł +è´¸æĺĵ æĪĺ +æľī èī² +æľīèī² éĩijå±ŀ +æįIJ åĬ© +为 é¦ĸ +为é¦ĸ çļĦ +å¯Į åĬĽ +çĶ· ç¥ŀ +é³ ³ +æµĩ æ°´ +åIJ ± +æĺİç¡® æıIJåĩº +åı¹ äºĨ +åı¹äºĨ åı£æ°Ķ +礼 æĭľ +è¿Ļ个 åIJįåŃĹ +ä¿¡ å¾Ĵ +å¿Ĺ 强 +éĻIJ æŶ +æĶ¶ è²» +åĨľå®¶ ä¹IJ +å°ıé¾Ļ èĻ¾ +èIJ½ å¹ķ +æ§ Ł +åѦ 龸 +æĪĸ å¤ļ +æĪĸå¤ļ æĪĸ +æĪĸå¤ļæĪĸ å°ij +座è°Ī ä¼ļä¸Ĭ +æ¶ ¼ +éŃĶ çİĭ +å² ± +顶 å±Ĥ +顶å±Ĥ 设计 +èĦij åŃIJéĩĮ +éĻ¢ åŃIJéĩĮ +轩 è¾ķ +身å¿ĥ åģ¥åº· +èħ ij +éĹľ 注 +åıĤåĬł ä¼ļè®® +ä¸Ńåįİ æĸĩåĮĸ +追 寻 +å®ī çĦ¶ +é£Ļ åįĩ +éŁŃ èıľ +é¸ ¦ +åĤ¨ éĩı +çĶ· æĸ¹ +å¤ĩ 份 +æijĶ åĢĴ +润æ»ij æ²¹ +éĢ¼ è¿ij +çĶ³ è¯ī +鸣 ç±» +çŁ³æ²¹ åĮĸå·¥ +åĿļ æŀľ +è¿Ļ家 ä¼Ļ +æĭĴ ä¸į +羣 çļ® +è·Ŀ éĽ¢ +è¿ĺ æĮº +éĽķ åĥı +åĪĿ æģĭ +æıIJä¾Ľ æĽ´å¤ļ +æŁ¥çľĭ åħ¨æĸĩ +æķ°åŃĹ è´§å¸ģ +åĸī åĴĻ +åı¦ä¸Ģ ä½į +åĤ¬ åĮĸ +åĤ¬åĮĸ åīĤ +ä»İæĿ¥ 没 +å¯ĨåĪĩ 缸åħ³ +éĥ¨ 主任 +产åĵģ ç»ıçIJĨ +並 åIJĮæĦı +èIJ½ åħ¥ +å±ıå¹ķ ä¸Ĭ +åħ¬åı¸ ç«łç¨ĭ +æį¢ åı¥è¯Ŀ +æį¢åı¥è¯Ŀ 说 +ä½į æĸ¼ +ä½ Ķ +åĩ» æĿĢ +缸 è¾ĥ +缸è¾ĥ äºİ +ç²½ åŃIJ +åįĹ æŀģ +宫 é¢Ī +è£ģ åijĺ +æĺİ ç»Ĩ +ä»·åĢ¼ éĵ¾ +åĽĽä¸ª æĸ¹éĿ¢ +æĥħåĨµ æĿ¥çľĭ +æĮij åīĶ +æ® ĺ +æŀģ åĬĽ +çĸij éļ¾ +æĬµæĬĹ åĬĽ +æĢ¥ éĢŁ +æĪ Į +ä½İ ä¼° +éĹª è¿ĩ +æģ ¬ +èµŀ æī¬ +ä»ĸ å¦Ī +æĪIJ为 ä¸ĢåIJį +æ´Ĺ 礼 +é¢Ħ计 å°Ĩ +åħĪè¿Ľ åįķä½į +è¼ Ķ +éĢĥ èĦ± +çİ° åŃĺ +èĢģèĻİ æľº +åįģä¸ĥ æĿ¡ +åı¦ä¸Ģ åįĬ +温 æĥħ +åī¥ 离 +ä¸ĸ è´¸ +å®ĺ åı¸ +å¾Ī å·® +éĹ´ è·Ŀ +请 注æĦı +åı² è¯Ĺ +åĪ© åĻ¨ +è¿IJ ç®Ĺ +沦 为 +該 使çĶ¨èĢħ +èĮ ¬ +éĶ¦ 绣 +åı² æĸĻ +çģµ æ´»æĢ§ +èģĶ 社 +æĹł åĬ© +æĬĹ æ°§åĮĸ +èıľ èĤ´ +éĢł èĪ¹ +æİī èIJ½ +å¤į æŁ¥ +åĭĥ åĭĥ +åij¼ 声 +給 äºĪ +åIJĮäºĭ 们 +ç½ ° +è¯ķ æİ¢ +åħ³éĶ® åŃĹ +æįIJ çĮ® +ç»Łè®¡ æķ°æį® +åĪĽ ä½ľèĢħ +ä¸ĭ åįĬ +ä¸ĭåįĬ åľº +æī¿æĭħ 责任 +端 æŃ£ +ç©¿ è¡£ +ä¼ł çIJĥ +åĬ© éķ¿ +åĩ ± +éķ¶ åµĮ +é£ŀ ç¿Ķ +è¾ĵ åįµ +è¾ĵåįµ 管 +ä¸ĩ åħ¬éĩĮ +æݨ广 åºĶçĶ¨ +å¿« æ¨Ĥ +ç§ ½ +èī° å·¨ +åIJ¬ å®Į +åĿļ 硬 +奥 åľ° +å¥¥åľ° åĪ© +é¢ ĵ +èĻIJ å¾ħ +ä¾Ľ æ±Ĥ +éľī ç´ł +伪 è£ħ +乡 åľŁ +åĩ¡ æľ¬ç½ij +åĩ¡æľ¬ç½ij 注 +ä¼Ĭ åĪ© +è¡¡ æ°´ +æĽ´ åĥıæĺ¯ +åĪĨéĴŁ å·¦åı³ +è¦ı 模 +äºĶ åĪĨéĴŁ +åºĹ åĬłçĽŁ +åĽ° éĽ£ +åħ³ åģľ +æĢĿ 绪 +åĴ½ åĸī +缸 符 +çĥ¦ èºģ +æĻĤ æľŁ +åijĪ çı¾ +解 æķ£ +诱 导 +éļĶ çĥŃ +çĮ ¶ +åįĹ å®ĭ +æ·±åħ¥ äºĨ解 +çŃĶ çĸij +æĺ¼ å¤ľ +åįĥ ä¼ı +åĬ³åĬ¡ æ´¾éģ£ +红 è±Ĩ +åĿı äºĭ +çĤ¹ æ»´ +å°±ä¸ļ å²Ĺä½į +约 åIJĪ +åħį éĻ¤ +éĢĨ åĬ¿ +éĩį éĩijå±ŀ +å®ĺ 宣 +ä½İ å»ī +æģ¨ ä¸įå¾Ĺ +å¾Ĺ 天 +å¾Ĺ天 çĭ¬ +å¾Ĺ天çĭ¬ åİļ +ä¸Ģå°ģ ä¿¡ +æĬ½ å¥ĸ +è¾Ĺ 转 +çķĻ å®Ī +çķĻå®Ī åĦ¿ç«¥ +çŃĶ åį· +å·¨ åŀĭ +æľĢ好 ä¸įè¦ģ +æµĻæ±Ł 大åѦ +æĨ ¨ +æı¡ æīĭ +éĴĪ ç»ĩ +æİĴ 骨 +çĤ ½ +å°ģ è£ħ +åįĢ åŁŁ +空æ°Ķ åĩĢåĮĸ +åħī å½± +åĢĴ å¡Į +å§ļ æĺİ +æ¤į 被 +åѦ åīį +åѦåīį æķĻèĤ² +èĬĿ åĬł +èĬĿåĬł åĵ¥ +缩 æ°´ +ä½ Ł +åľ¨çº¿ åĴ¨è¯¢ +èµı æŀIJ +éĿĴ èĽĻ +æĬ± ä½ı +èĮĤ åIJį +åħ¨åĬĽ æīĵéĢł +åįļ士 åѦä½į +沧 å·ŀ +åĻ ¢ +æĿĤ çī© +åĪ» çĶ» +æį ħ +å¾® éĩı +å¾®éĩı åħĥç´ł +ä¸Ģ åĽŀäºĭ +鸡 èĤī +åĪ©æ¶¦ çİĩ +æīį ç®Ĺ +å¾® å¦Ļ +棵 æłij +è´ª 婪 +åĩı åĢ¼ +梦 å¢ĥ +åı¯ è§Ĩ +åı¯è§Ĩ åĮĸ +广大 å¸Ĥæ°ij +ä¸ĵä¸ļ ä»İäºĭ +ç»ı 纬 +ç´§ çĽ¯ +çŁ¥ å·± +è¤ ļ +æĸĩåĮĸ åºķèķ´ +åݦéŨ å¸Ĥ +临 港 +对åħ¶ 羣å®ŀ +岸 è¾¹ +è¦ĸ çĤº +æĬĹ çĻĮ +åĶIJ å®ĩ +ä¸įå¾Ĺ è¶ħè¿ĩ +å¨ģ æħij +æ¡Ĩæŀ¶ åįıè®® +èµ° ç§ģ +åĽ¢ å§Ķ +夸 大 +æ¬ Ħ +ç¥ŀç»ı ç³»ç»Ł +æijĦå½± ä½ľåĵģ +èĬ ¥ +å®ī åºĨ +æµ· 滨 +æŀĦ æĢĿ +çīµ æĮĤ +åı © +éĺIJ æĺİ +éģ ģ +ç²¾ æ²¹ +ç©´ ä½į +æĬ¤ 身 +æĬ¤èº« 符 +æĮĩ å°İ +åŃĺåľ¨ ä¸Ģå®ļ +å¯Ĥ éĿĻ +æµ·å¤ĸ å¸Ĥåľº +éĿ ¡ +综åIJĪ å¾ģ +ä¿ IJ +è¨Ī ç®Ĺ +æĺİ æľĹ +äºļ è¿IJ +äºļè¿IJ ä¼ļ +åīįçŀ» æĢ§ +åĮ® ä¹ı +产ä¸ļ æī¶è´« +èĦij æµ· +èĦijæµ· ä¸Ń +åħļçļĦ é¢Ĩ导 +åĪĺ éĤ¦ +æµģ æĺŁ +æĵ Ĥ +æĶĢ çĻ» +åĴ Ķ +ä¸Ģä¸ĭåŃIJ å°± +è¯Ĭ æ²» +使 åĬ² +åīµ ä½ľ +éĵŃ è®° +éĴ± è´¢ +æĹ¥æĬ¥ è®°èĢħ +çĥŁ çģ« +èĥľ è´Ł +åįļ 主 +ä¸ŃåĽ½ èģĶéĢļ +ç½ijç«Ļ é¦ĸ页 +å°± å¤Ł +å°±å¤Ł äºĨ +æīij åħĭ +å±ħ å§Ķä¼ļ +è° ¬ +å®īåħ¨ äºĭæķħ +åķĨ çĶ¨è½¦ +循çݯ ç»ıæµİ +æ· ¤ +èĢĥ è¯ģ +å®Ŀ èĹı +å®Į ç»ĵ +çłĶåıij æĬķåħ¥ +å² ij +æģŃ æķ¬ +离 éĢĢä¼ij +æ°´ 墨 +å© ¶ +è¯Ĺ åı¥ +å®ģæ³¢ å¸Ĥ +å¼± çĤ¹ +åģľ çīĮ +奶 æ²¹ +å¥ĩ纳 æ²³ +æĨ Ĥ +社ä¼ļ å®ŀè·µ +è´Ŀ 壳 +çłĤ æµĨ +èĪ¹ åıª +宣 æī¬ +综åIJĪ æķ´æ²» +åĤ ij +æ°ijæĹı æĸĩåĮĸ +éĩį çİ° +积 æ·Ģ +åħ¬ çĦ¶ +çħ ī +缸 èģļ +æ± ¾ +纹 çIJĨ +çĩĥ çħ¤ +æѤ ç§į +ç¾İ å¦Ĩ +åįĥ çĵ¦ +çIJ Ľ +驾驶 è¯ģ +éĺ¶ æ¢¯ +ä¸Ŀ ä¸Ŀ +å¾Īå¤ļ äºĭæĥħ +åħī éĺ´ +èijĹä½ľ æ¬Ĭ +åħ§ éĥ¨ +çĽ¸å¯¹ æĿ¥è¯´ +éĸ Ĵ +éľĩ æħij +說 話 +æĨ ij +ç«¥ è£ħ +ä½ıæĪ¿ åĴĮ +ä½ıæĪ¿åĴĮ åŁİ +å·²ç»ı è¶ħè¿ĩ +侦 å¯Ł +çŁ¿ çī© +ä¾Ľ 大家 +çī¹ éĤĢ +ç¨ĭåºı åijĺ +çķľçī§ ä¸ļ +æ° ª +çij ª +åĢĴ åľ¨ +åĢĴåľ¨ åľ° +æ¯ Ģ +梯 éĺŁ +æİ¥ èijĹ +æĬĹ èıĮ +è¤ ĩ +ç¬ Ļ +æ¯Ķ ä¸Ĭå¹´ +鸡 汤 +åŃ¦ä¹ł æĪIJ绩 +æĸij æĸĵ +åħĪ 导 +åĪĹ ä¸¾ +è°ĥæŁ¥ æĺ¾ç¤º +æ© « +ä¹Ŀ åįģ +è°¢ 飵 +è·¨è¶Ĭ å¼ı +女æĢ§ æľĭåıĭ +èIJ¥åħ» ä»·åĢ¼ +å®ŀè·µ ç»ıéªĮ +èĭı å·ŀå¸Ĥ +çĵ¶ åŃIJ +æĸ° çļĦä¸Ģ +æĸ°çļĦä¸Ģ å¹´ +æĺİ æĻ° +å®ł çĪ± +åŃŠ第 +æľĹ 诵 +纳 æĸ¯ +éĢĨ è¡Į +è«ĭ æĤ¨ +è«ĭæĤ¨ æıIJä¾Ľ +èĥ¸ æĢĢ +第ä¸ĥ å±Ĭ +强 壮 +代 åŃķ +汶 å·Ŀ +家 åĸ» +家åĸ» æĪ· +家åĸ»æĪ· æĻĵ +èħ ® +åIJ¯ 迪 +æĹł éļľç¢į +èĻķçIJĨ åıĬ +æĿ¥ åİĨ +å®ŀ åĬ¡ +ä¹Ł éļıä¹ĭ +æĬĢèĥ½ åŁ¹è®Ń +åѤ ç«ĭ +åī ģ +éĥ´ å·ŀ +æĶ¶ æķĽ +éł» éģĵ +èį£ 幸 +èİ« è¿ĩäºİ +æѤ æĻĤ +纪å§Ķ çĽij +纪å§ĶçĽij å§Ķ +缸 éĤ» +åı¦ä¸Ģ è¾¹ +çªĴ æģ¯ +æľīå¾Īå¤ļ ç§į +æ¯ı éĢ¢ +éĹ® ä¸ĸ +ç´¯ ç´¯ +éĿĴæĺ¥ æľŁ +è·¯ åĨµ +åħĭ èݱ +è¿Ħä»Ĭ 为æŃ¢ +æĥĬ å¥ĩ +è·¨ 度 +éħ¿ éĢł +åĩ ĭ +è¿ij ä¸īå¹´ +åĨħ 马 +åĨħ马 å°Ķ +æı į +è¿Ľå±ķ æĥħåĨµ +èĮ § +æľīåºı æİ¨è¿Ľ +æĢ» åĨłåĨĽ +æĪIJ绩 åįķ +éĽ»è©± åıĬ +ç´§å¯Ĩ ç»ĵåIJĪ +åºĬ ä½į +é¹ Ĭ +æķ£åıij çĿĢ +åĭŁ èµĦ +æ°¨ éħ¸ +彩 ç¥ŀ +è®Ģ åıĸ +éĩį 温 +ä¸Ń åŃĺåľ¨çļĦ +ç¾İ éºĹ +ä¸įæĸŃ å¢ŀåĬł +è½® æµģ +æİ¥ åIJ¬ +å¹´ 产åĢ¼ +åįĥ åħĭ +æĪĺåľº ä¸Ĭ +çħ§ 顧 +å¹²éĥ¨ éĺŁä¼į +åį° ç«ł +ä¸Ģèĩ´ æĢ§ +è¿ŀ å¤ľ +åħħ è£ķ +é»ij åIJįåįķ +åĩĢ æ°´ +ä¸Ģ大 æĹ© +åĮħ 袱 +çĬ¯ è§Ħ +çIJĨ è«ĸ +æŀģ æĺĵ +éª ¸ +å¨ĺ å¨ĺ +åĽ¢ åľĨ +亿åħĥ 以ä¸Ĭ +åĪ©çĶ¨ æĤ¨çļĦ +带æĿ¥ æĽ´å¤ļ +ä¸Ń央 空è°ĥ +æľĪ èĸª +çĮľ æĥ³ +åĪº 客 +ä½ľ æģ¯ +åįķ è°ĥ +äºĴ åĪ© +å¦Ĥæľī ä¾µæĿĥ +å°ı å·§ +åįģ åł° +åĵĪåĵĪ åĵĪåĵĪ +è¾¹ éĻħ +æłĩ è¯Ń +åĪĩåħ¥ çĤ¹ +éĢĨ è¢Ń +è¯ķ åīĤ +绿 è±Ĩ +è® ļ +åŁºçĿ£ å¾Ĵ +å£ ¬ +åħ¨ æĺİæĺŁ +éĢī ç§Ģ +èĪĮ å°ĸ +ä¸įåIJĮ ç±»åŀĭ +çĥŁ åĽ± +çģµ æ°Ķ +åĮº 管å§Ķä¼ļ +åĨľ åī¯ +åĨľåī¯ 产åĵģ +èĶļ æĿ¥ +沪 æĮĩ +åħ»æ®ĸ æĪ· +æĸĹ å¿Ĺ +é¦ĸ é¢Ĩ +è¡Ģ èħ¥ +åĬł ç´§ +ä¸Ģèĩ´ 好è¯Ħ +第ä¸ī èĬĤ +æī¬ å°ĺ +交éĢļ æŀ¢çº½ +鼶 ç¢İ +é»ij æ´ŀ +çľĭ ä¸įæĩĤ +å±ŀ å®ŀ +主 åŁİåĮº +å¨ Ľ +å¨Ľ æ¨Ĥ +ç¬ij æĦı +èĻ¹ æ¡¥ +åIJĦ个 çݯèĬĤ +çķ¥ å¾® +èĢķ èĢĺ +æľ¬ åľºæ¯ĶèµĽ +æĪIJ è´¥ +éĢī èĤ¡ +èªŀ è¨Ģ +çŃĶ 辩 +èĩª ä¹ł +æ£ º +ä¸ĩ 欧åħĥ +åģľ å·¥ +对åħ¶ è¿Ľè¡Į +积æŀģ éħįåIJĪ +ä¹¾ åĿ¤ +å¦ĸ æĢª +èļĮ åŁł +èµĦ产 è¯Ħä¼° +è°ĥ çļ® +éĻ¤ å¤ķ +åĽ´ å¢Ļ +æľį å½¹ +æ·± æ¸Ĭ +é¢Ħ åĪ¶ +ç ĥ½ +å®ī 稳 +建 æŀĦ +çĭĻ åĩ» +主åĭķ 註åĨĬ +éĥ½æľī èĩªå·± +æİĴåIJį 第ä¸Ģ +麻 è¾£ +çĢ ļ +çĥŁèĬ± çĪĨ +çĥŁèĬ±çĪĨ 竹 +èĩªçĦ¶ ä¿ĿæĬ¤ +ä»Ļ å¢ĥ +为äºĨ éģ¿åħį +åĨ· åºĵ +解æĶ¾ æĢĿæĥ³ +åĪĿ äºĮ +ä½ĵ è´´ +é¦ĸ å¯Į +迪 æĭľ +æļĤ ç¼ĵ +æĶ¯æĮģ åĬĽåº¦ +侦 æİ¢ +马 åĪº +åĮĹ æ±½ +ç¹ ŀ +è°İ è¨Ģ +éĢ£ çºĮ +å· ³ +ä»»ä½ķ æŶåĢĻ +车 èģĶç½ij +åįķ 项 +å¸Ń åį· +建çŃij æĿIJæĸĻ +ä¸Ńç§ĭ èĬĤ +ç¡ķ士 çłĶ究 +ç§ģ ç«ĭ +åħļåĴĮ æĶ¿åºľ +æľ¬æ¬¡ 交æĺĵ +èººåľ¨ åºĬä¸Ĭ +ç½ijåıĭ è¯Ħ论 +å¦ Ŀ +害 ç¾ŀ +åħ¬ç«ĭ åĮ»éĻ¢ +ä¸ ŀ +çĶŁçī© è´¨ +åºĶ éĤĢ +æĬ½ åıĸ +åĩł å¼ł +æijĺ ç¼ĸ +ç»ĺ æľ¬ +详 解 +强 硬 +æľĢ åħĪè¿ĽçļĦ +æĭĽ èĤ¡ +æĭĽèĤ¡ 书 +åįĥ æĸ¹ +åįĥæĸ¹ çĻ¾ +åįĥæĸ¹çĻ¾ 计 +éħį éŁ³ +驾 çħ§ +å¾ģ æĪĺ +èªĵ è¨Ģ +æĭľ å¸Ī +æĭľå¸Ī åѦ +æĭľå¸ĪåѦ èīº +æĬ± åĽ¢ +ç±³ ç²ī +éĿŀ常 éĢĤåIJĪ +èĪª æµ· +å±¥ 约 +åįģåħ« æĿ¡ +éĶ» éĢł +éĩįè¦ģ 举æݪ +åıijæĮ¥ ä½ľçĶ¨ +æ· ļ +人 社 +人社 å±Ģ +è¯ķçĤ¹ å·¥ä½ľ +éĺľ éĺ³ +æ¡ĥ åľĴ +æ°ij ä¼ģ +æ´ģ çĻ½ +è´µ 宾 +åħ¬ 社 +è§ī æĤŁ +è®°å¿Ĩ åĬĽ +æľĥåĵ¡ 註åĨĬ +æѤ æ¡Ī +麻 çĹ¹ +çı Ģ +æĸ© èİ· +çĶ· åŃ©åŃIJ +å±ĢéĻIJ äºİ +åĭĺ æŁ¥ +åIJĥ 饱 +èĬ¬ åħ° +æ£ķ èī² +ç¦ı ç¥ī +çĶ³ èĬ± +æµ· çĽĹ +èĶ ij +æĸĩ åѸ +æ´»æĢ§ çĤŃ +缴 éĢļ车 +è°¢ éĤĢ +躺 çĿĢ +åľ ĥ +æ¯ıæĹ¥ ç»ıæµİ +åħ¬åħ± æĸĩåĮĸ +讲 æķħäºĭ +å¯Ł çľĭ +æĤł éĹ² +åľ° åĿª +æ¶Į çİ°åĩº +é«ĺçŃī éĻ¢æł¡ +èĮĦ åŃIJ +éĺ² åį« +ä¾ĭ è¡Į +æĺ¾ éľ² +æĸ° 常æĢģ +ç»Ŀ ä½³ +å¯Į æ°ij +以 人æ°ij +以人æ°ij 为 +éĤ¢ åı° +å±ķ æ¼Ķ +çĻ¼ å¸ĥ +è´Ł è½½ +åģı 离 +æ°¸ éģł +éĩįè¦ģ åİŁåĽł +åįıä¼ļ ä¼ļåijĺ +éļ¾ æ°ij +çĶŁäº§ 车éĹ´ +çģµ åĬ¨ +两年 åīį +æĸ¹ åľĨ +æ´» ä¸ĭåİ» +ä¸ĸçķĮ è§Ĥ +éªĹ åıĸ +ç¾İ è²Į +èĥ½ çľĭåĩº +çĻ¼ æı® +è§Ĥ å½± +åī ĥ +åIJĪèµĦ åħ¬åı¸ +å© § +å¹² æű +åħŃ ä¸ªæľĪ +尤为 éĩįè¦ģ +èĤ ½ +秦 åĽ½ +æīĺ ç¦ı +建çŃij å¸Ī +åįĩ级 æĶ¹éĢł +å°ı é¢Ŀ +å°ıé¢Ŀ 贷款 +两个 ç»´æĬ¤ +æĭį æĭį +åı¯ çĸij +æį¢ åıĸ +æѦ 士 +èµĸ 以 +èµĸ以 çĶŁåŃĺ +æĮ ļ +殿 åłĤ +èĩªçĦ¶ çķĮ +ç£ģ åľº +å¦Ĥä½ķ çľĭå¾ħ +ä»ĬæĹ¥ 头æĿ¡ +西 åŁŁ +èİ· è¯Ħ +風 æł¼ +ä¿Ħ åĽ½ +æīĵ æĭ¼ +å®£ä¼ł çīĩ +å¾Ī æĸ¹ä¾¿ +ä¾Ľç»Ļ 侧 +纪念 ç¢ij +毫 åħĭ +èĬ³ é¦Ļ +å·¥åķĨ éĵ¶è¡Į +请 çĤ¹åĩ» +ç¼ ª +æĹłæķ° 次 +èį¯ å¸Ī +èħ ¸ +游 èīĩ +åĮ ¾ +å·¡ èĪª +æ²»çIJĨ ä½ĵç³» +èIJ¥éĢł èī¯å¥½ +æ·· æ·Ĩ +éĢļ çķħ +åĬ³ ç´¯ +ä»ĵ ä½į +å¢ŀ éķ· +éļIJ 约 +æĿĤå¿Ĺ 社 +åħ» èĤ² +åı¯èĥ½ åıijçĶŁ +èĢĥ 試 +西 侧 +åĬł åĢį +主æĮģ åı¬å¼Ģ +çķ¢ ç«Ł +éĹ® 询 +æµ· æ£ł +èĹ © +注æĺİ æĿ¥æºIJ +æ£Ģ çĸ« +请 åģĩ +æĬļ æij¸ +èĵĦ çĶµæ±ł +è·Ł ä¸įä¸Ĭ +çݰ代 社ä¼ļ +çŃ¹ èµĦ +ä½ĵèĤ² 彩票 +延 误 +è¾Ľ è¾£ +éĿ¢ 容 +åį° è®° +çģŃ äº¡ +ç´ł é£Ł +åħ´ èĩ´ +éľĢè¦ģ çĶ¨ +éľĢè¦ģçĶ¨ åĪ° +å®Ŀ å¦Ī +ç£ĭ åķĨ +éļ¶ å±ŀ +è´¡çĮ® åĬĽéĩı +åħ¬åħ± èµĦæºIJ +大 éĺª +åĨĽ è®Ń +æĤ¬ 念 +社ä¼ļ 稳å®ļ +å¹²äºĭ åĪĽä¸ļ +æľī æĿ¡ä»¶ +æľīæĿ¡ä»¶ çļĦ +ä¸Ģå¹´ ä¸Ģ度 +åİ ¥ +强 奸 +豪 车 +æİĮ æŁľ +æ°´åĪ© å·¥ç¨ĭ +å³ ª +积æŀģ ä½ľçĶ¨ +æµ· æ·Ģ +æµ·æ·Ģ åĮº +çĥŃ æĴŃ +åĿļæĮģ ä¸įæĩĪ +åıĮ èĦļ +绣 æĪĺ +ä»»ä½ķ 人éĥ½ +åľ°ä¸ĭ 室 +åĨ¶ çĤ¼ +è°ħ 解 +æ¸Ķ èĪ¹ +太éĺ³ åŁİ +被 æįķ +计ç®Ĺ åĻ¨ +西 åĮ» +èĪĴ å¿ĥ +æ¡ ¦ +éģ ² +åĬ ij +è¨ Ĺ +èİ º +åĸ ¬ +çĵ ¯ +åĺ ĺ +åł ķ +æķ Ŀ +åij ¦ +èĭ ŀ +æŃ ¹ +æĵ ¬ +æ£ Ħ +èĪ µ +å¥ ª +çļ ĭ +æĶ ¸ +åľ © +ç¤ Ļ +ç¢ ĺ +éı Ī +æĦ ķ +ç¹ ³ +èĺ ¸ +è² Ĥ +æ¼ ² +æij ¹ +æĶ Ŀ +åŃ ¢ +èķ Ń +é¨ ° +æ½ ¼ +éħ ° +æĴ ¥ +è¹ ¬ +é¨ Ļ +è¸ ¹ +éģ IJ +çĺ Ģ +èĽ ¤ +æĤ ĸ +çĴ ŀ +ç£ IJ +æİ ° +è¾ Ĭ +å¾ ij +æİ ĸ +éģ ŀ +éĤ ¸ +éĽ ı +æĨ İ +æľ ½ +çį » +ç® Ķ +è¤ ¶ +æļ ¢ +æĺ µ +çı Ĥ +æĤ ¸ +åģ µ +åĻ ľ +å£ ¯ +æĴ ® +æģ į +å© ķ +ç¯ ± +éĺ Ļ +çī ł +è£ ĺ +è³ ¢ +éĩ ľ +éĵ ł +èİ ĺ +æ® Ĩ +çĻ ¸ +è´ ı +ç² ± +å« ¡ +åĨ ¢ +è¤ Ĵ +æĩ Ĭ +éľ ĵ +å¡ µ +æĭ £ +å» Ł +é£ ½ +é¢ Į +åļ İ +æ· º +èĨ ł +åİ Ń +åļ ĩ +åij ĥ +çĴ ĭ +çŃ ± +æĭ · +èį § +éĶ ° +åŃ ° +èĵ ĵ +èĨ ½ +æŀ ī +åĸ ½ +çĽ Ķ +çŃ IJ +ç¾ ļ +è ħĮ +è¾ « +æ³ ĵ +çĶ ¬ +èŁ ² +åĸ ª +å¦ ĵ +è¬ Ģ +çĤ Ĭ +æĽ ľ +æ± IJ +è´ Ī +èį Ģ +æĬ ł +ç¢ ¾ +æ« ĥ +éŀ ł +èij Ĩ +ç¥ ¯ +å½ Ŀ +é¦ į +åĮ £ +æľ Ń +åĿ Ĥ +ä¿ ij +èĵ ® +çij Ľ +æī ī +èĩ Ł +è² « +çİ ¥ +æ· ¼ +åİ ² +é³ Į +å³ Ń +åij Ľ +é § +é§ IJ +éģ · +ä¿ ª +æĢ Ĥ +è¾ į +å± į +åĭ ģ +å¥ ļ +éļ ħ +éĴ ´ +è¼ Ŀ +å® ¦ +èIJ ĥ +çĺ ĭ +æĨ ¶ +æĤ ħ +è¾ Ļ +åij ľ +çł º +éĢ ŀ +æµ ļ +éĸ £ +èĸ © +éĻ ĭ +çĤ Ļ +èª ķ +ä¸ Ł +é¹ ½ +ç± Į +è´ ° +éĭ ª +çľ © +æĴ IJ +èĨ º +éŀ ĺ +ç¾ ² +çª ® +ç´ IJ +æ® ´ +çº ¾ +èº į +ç´ ĭ +çĦ ĸ +çĶ º +çī ½ +çĤ ¯ +ç¼ Ķ +æ¯ ĵ +å¬ ° +æ¢ § +äº Ł +è¢ ħ +çį Ħ +è¿ ¥ +æ¼ ¾ +çĿ ij +ç¸ ¾ +é¦ ĭ +é¤ ħ +æ ¹Ħ +æĺ ĩ +æŀ Ń +èĸ ° +æŁ ij +æ¦ » +åĻ Ĺ +åĻ ´ +æ£ £ +åĶ § +çĨ ¹ +è¼ ¯ +å¢ Ł +é² ² +æĪ Ľ +èī ¦ +èĬ ® +åĺ Ł +å¸ ¥ +å¿ » +çĮ Ŀ +å¯ µ +è³ ¦ +èĽ ¾ +æ» ¾ +çĤ ķ +éĵ ¬ +èĴ ¿ +éĴ ¨ +çĥ Ļ +ç² ķ +æĥ ¦ +æº § +é¢ į +éħ £ +å³ ¦ +ç± ģ +çĥ ĥ +åĨ Ĺ +åı ģ +çĽ § +ç½ µ +éĴ Ĺ +å¬ ī +è° ı +ç³ § +è¾ Ń +æ· ¬ +èŁ Ĵ +è¯ © +è¦ ĥ +çĻ ĸ +é½ Ĵ +çĪ IJ +ç® į +ç¼ İ +ç£ º +è¯ « +è¤ ² +æĵ ł +èIJ ¦ +çĿ ¬ +è° į +éĦ ° +æł ¾ +é¡ ı +ç¸ ± +æ¡ ¨ +éĨ ¬ +è¥ ² +è® ª +å© º +èį Ł +åĮ Ŀ +çĨ ł +èĽ Ĭ +æ¸ ļ +å´ ½ +é² ¤ +åķ ° +åĮ ķ +ä¸ IJ +è® ¥ +åı ½ +åı ¼ +çļ ¿ +è¿ Ĥ +åIJ Ĩ +å± ¹ +èĩ ¼ +è® ¹ +é© ® +çº « +æ± ŀ +æĬ ¡ +èĭ ĩ +åIJ ł +åIJ Ń +åIJ ® +å² ĸ +ä½ ĥ +çĭ Ī +åº ĩ +åIJ Ŀ +éĹ ° +æ± ¹ +å¿ ± +æĭ Ħ +æĭ Ĺ +èĮ ī +èĭ Ľ +èĮ ģ +çŁ ¾ +èĻ ı +åij » +åĴ Ħ +å¿ ¿ +èĤ ® +çĭ ŀ +çĸ Ł +çĸ Ļ +çĸ ļ +æ³ ŀ +å¸ ļ +å± ī +è¿ ¢ +é© ¹ +ç İ· +çıĬ ó +çıĬó ł +çıĬół Ħ +çıĬółĦ ģ +æĮ İ +æĭ ´ +åŀ Ľ +èį ¤ +æ® ĥ +çĽ ¹ +åĵ Ĩ +è´ » +æ¯ ¡ +çĭ ° +çĭ ¡ +æŁ Ĵ +æģ ĥ +è¯ ¬ +è¢ Ħ +è¯ ² +èļ ¤ +èĢ Ļ +åŁ Ĥ +æį İ +æį Į +æ¢ Ĩ +é ħĮ +çł ¾ +æ® ī +åĶ ł +æĻ Į +èļ £ +èļ ª +èļ ĵ +é¸ ¯ +åĶ ģ +åĶ Ĩ +åĢ Ķ +èĪ Ģ +è± º +èĥ ° +é¸ µ +é¸ ³ +é¦ ģ +ç¾ Ķ +æ¶ £ +æ¶ ķ +æĤ ¯ +è¯ ½ +è° Ĩ +ç¥ Ł +ç» ¢ +æį º +æį ¶ +æį » +æİ Ĥ +èı ł +èIJ ¤ +éħ Ĺ +çľ ¶ +åķ Ħ +èļ ¯ +èĽ Ģ +åĶ ¬ +å¸ · +éĵ IJ +éĵ Ľ +åģ İ +å¾ Ļ +èĦ ¯ +è± ļ +çĮ ĸ +çĹ Ĭ +æ¶ ® +æĥ Ń +æĤ ´ +æĥ ĭ +è° ļ +æı © +æIJ Ģ +æIJ Ķ +æ¦ Ķ +æ¤ Ń +éĽ ³ +åĸ ³ +è· Ľ +èľ ĵ +èľ Ĵ +é¹ ĥ +éĶ Ħ +çĶ ¥ +çŃ ı +çĮ © +çĮ ¬ +çĮ ¾ +çĹ ¢ +çĹ ª +æĥ ° +çª ĺ +è° ¤ +éļ ĺ +å© ¿ +é¹ ī +çij Ļ +æĸ Ł +æ¤ ¿ +éħ ª +éĽ ¹ +åĹ ¦ +è· · +è· º +è· ¤ +èľ Ī +èľ Ĺ +å¹ Į +é¦ ı +èª Ĭ +æ¼ ĵ +è¤ Ĥ +èĶ Ĺ +èĶ ¼ +åħ ¢ +è£ ³ +èľ » +èĿ ĩ +åĺ Ģ +éĶ ¹ +ç® ķ +ç® © +çĺ © +çĺ Ł +æ¼ ± +å¯ ¥ +éª ¡ +æĴ µ +æĴ ¬ +è± Į +åĺ ¹ +èĿ ł +èĿ Į +èĿ Ĺ +èĿ Ļ +éķ IJ +ç¨ ¼ +ç¯ ĵ +èĨ Ľ +é² « +çĺ ª +é² ¨ +æĨ Ķ +ç¿ © +è¤ ¥ +ç¼ Ń +åĻ © +çĵ ¢ +éľ İ +è¸ ± +è¹ Ĥ +èŁ Ĩ +é¹ ¦ +ç¯ ¡ +çĺ ¸ +çª ¿ +ç¼ ° +èĹ IJ +è¹ ĭ +èŁ ĭ +èŁ Ģ +èµ ¡ +èĩ Ĭ +é³ Ħ +ç³ ł +æĩ ¦ +åļ £ +éķ ° +é³ į +ç° ¸ +çĻ £ +é³ ĸ +é¬ ĵ +èł ķ +éľ ¹ +èº ı +é» ¯ +çĵ ¤ +çŁ Ĺ +ä¹ Ĥ +ä¹ ľ +åħ Ģ +å¼ ĭ +åŃ ij +åŃ ĵ +å¹ º +äº ĵ +å »¿ +ä¸ ı +åį ħ +ä» ĥ +ä» ī +ä» Ĥ +åĪ Ī +çĪ » +åį ŀ +éĹ © +è® £ +å¤ ¬ +çĪ ¿ +æ¯ ĭ +éĤ Ĺ +éĤ Ľ +èī ½ +èī ¿ +åı µ +ä¸ ķ +åĮ ľ +åĬ ¢ +åį Ł +åı ± +åı » +ä» ¨ +ä» Ł +ä» ¡ +ä» « +ä» ŀ +åį ® +æ° IJ +çĬ ° +åĪ į +éĤ Ŀ +éĤ Ļ +è® ¦ +è® § +è® « +å° » +éĺ ¡ +å° ķ +å¼ ģ +èĢ Ĵ +çİ İ +çİ ij +åľ ¬ +æī ¦ +åľ ª +åľ ¹ +æī ª +åľ ® +åľ ¯ +èĬ Ĭ +èĬ į +èĬ Ħ +èĬ ¨ +èĬ ij +èĬ İ +èĬ Ĺ +äº ĺ +åİ į +å¤ ¼ +æĪ į +å° ¥ +ä¹ © +æĹ ¯ +æĽ ³ +å² Į +å± º +åĩ ¼ +åĽ ¡ +éĴ ĩ +ç¼ ¶ +æ° ĺ +æ° ĸ +çī Ŀ +ä¼ İ +ä¼ Ľ +ä¼ ¢ +ä½ ¤ +ä» µ +ä¼ ¥ +ä¼ § +ä¼ ī +ä¼ « +åĽ Ł +æ± Ĩ +åĪ ĸ +å¤ Ļ +æĹ ® +åĪ İ +çĬ · +çĬ ¸ +èĪ Ľ +åĩ « +é Ĥ¬ +é¥ § +æ± Ķ +æ± ľ +æ± Ĭ +å¿ ĸ +å¿ ı +è® ´ +è® µ +è® · +èģ ¿ +èī ® +åİ ¾ +å¦ ģ +çº ¡ +çº £ +çº ¥ +çº ¨ +çİ ķ +çİ Ļ +æĬ Ł +æĬ Ķ +åľ » +åĿ į +æĬ ĥ +㧠IJ +èĬ « +èĬ ¾ +èĭ Ī +èĭ £ +èĭ ĭ +èĬ ¼ +èĭ Į +èĭ ģ +èĬ © +èĬ ª +èĬ ¡ +èĬ Ł +èĭ Ħ +èĭ İ +èĭ ¡ +æĿ Į +æĿ ĵ +æĿ Ī +å¿ ij +åŃ Ľ +éĤ ´ +éĤ ³ +å¥ ģ +è± ķ +å¿ Ĵ +æ¬ ¤ +è½ « +è¿ ĵ +éĤ ¶ +å¿ IJ +åį £ +éĤ º +æĹ ° +åij ĭ +åij Ĵ +åij ĵ +åij Ķ +åij ĸ +æĹ ¸ +åIJ ¡ +èĻ ¬ +åIJ ½ +åIJ £ +åIJ ² +å¸ ı +å² Ī +å² ĺ +åħ ķ +åĽ µ +åĽ « +éĴ Ĭ +éĴ ĭ +é ĴĮ +è¿ ķ +æ° Ļ +æ° ļ +çī ¤ +ä½ ŀ +ä½ ļ +ä½ Ŀ +ä½ Ĺ +å½ · +ä½ ĺ +ä½ ¥ +è± ¸ +åĿ Į +èĤ Ł +å¥ Ĥ +åĬ ¬ +çĭ ģ +é¸ ł +é¥ ¨ +é¥ © +é¥ « +é¥ ¬ +åº ij +åº ĭ +çĸ Ķ +çĸ ĸ +èĤ ĵ +éĹ ± +éĹ ³ +çĤ Ģ +æ² £ +æ² ħ +æ² Ķ +æ² ¤ +æ² ı +æ² ļ +æ± © +æ± ¨ +æ² ¨ +æ± ´ +æ² Ĩ +æ² © +æ³ IJ +æĢ ĥ +æĢ Ħ +å¿ ¡ +å¿ ¤ +å¿ ¾ +æĢ ħ +å¿ ª +æĢ Ĩ +å¿ Ń +å¿ ¸ +è¯ Ĥ +è¯ ĥ +è¯ ħ +è¯ ĭ +è¯ Į +è¯ Ĵ +éĻ Ĥ +éĻ ī +å¦ © +å¦ ª +å¦ £ +å¦ Ĺ +å¦ « +å§ Ĵ +å¦ ¤ +åĬ Ń +åĪ Ń +éĤ ° +çº Ń +çº ° +çº ´ +çİ ¡ +çİ Ń +çİ ł +çİ ¢ +çİ ¦ +çĽ Ĥ +å¿ Ŀ +åĮ ¦ +åĿ © +æĬ ¨ +æĭ ¤ +åĿ « +æĭ Ī +åŀ Ĩ +æĬ » +åĬ ¼ +æĭ ĥ +æĭ Ĭ +åĿ ¼ +åĿ » +㧠Ł +åĿ ¨ +åĿ Ń +æĬ ¿ +åĿ ³ +èĭ · +èĭ ¤ +èĮ ı +èĭ « +èĭ ľ +èĭ ´ +èĭ Ĵ +èĭ ĺ +èĮ Į +èĭ » +èĭ ĵ +èĮ ļ +èĮ Ĩ +èĮ ij +èĮ ĵ +èĮ Ķ +èĮ ķ +è ĮĢ +èĭ ķ +æŀ ¥ +æŀ ĩ +æĿ ª +æĿ ³ +æŀ § +æĿ µ +æŀ ¨ +æŀ ŀ +æŀ ĭ +æĿ » +æĿ · +æĿ ¼ +çŁ ¸ +ç łĢ +åĪ ³ +å¥ Ħ +æ® ģ +éĥ ı +è½ Ń +éĥ ħ +é¸ ¢ +çĽ ± +æĺ Ļ +æĿ ² +æĺ ĥ +åĴ Ĥ +åij ¸ +æĺ Ģ +æĹ » +æĺ ī +çĤ ħ +çķ Ģ +èĻ ® +åĴ Ģ +åij · +é» ¾ +åij ± +åij ¤ +åĴ Ĩ +åĴ Ľ +åij ¶ +åij £ +åĴ Ŀ +å² ¢ +å² ¿ +å² ¬ +å² « +å¸ Ļ +å² £ +å³ ģ +åĪ ¿ +å² · +åī Ģ +å¸ Ķ +å³ Ħ +æ² ĵ +åĽ ¹ +ç½ Ķ +éĴ į +éĴ İ +éĴ ı +éĴ Ĵ +éĴ ķ +éĤ ¾ +è¿ ® +çī ¦ +ç« º +è¿ ¤ +ä½ ¶ +ä¾ ij +ä¾ ī +èĩ ¾ +ä¾ Ĺ +ä¾ ı +ä¾ © +ä½ » +ä½ ¾ +ä¾ ª +ä½ ¼ +ä½ ¯ +ä¾ ¬ +å¸ Ľ +ä¾ Ķ +å¾ Ĥ +åĪ ½ +éĥ Ħ +ç± ´ +çĵ ® +æĪ Ĺ +èĤ ¼ +äı Ŀ +èĤ ± +èĤ « +è¿ © +éĥ ĩ +çĭ İ +çĭ į +çĭ Ĵ +åĴ İ +é¥ ¯ +é¥ ´ +åĨ ½ +åĨ ¼ +åº ĸ +çĸ ł +çĸ Ŀ +åħ ĸ +åĬ ¾ +ð¬ ī +ð¬ī ¼ +çĤ ĺ +çĤ Ŀ +çĤ Ķ +æ³ Ķ +æ² Ń +æ³ · +æ³ ± +æ³ ħ +æ³ ł +æ³ º +æ³ ĸ +æ³ « +æ³ ® +æ² ± +æ³ ¯ +æĢ Ļ +æĢ µ +æĢ ¦ +æĢ Ľ +æĢ ı +æĢ į +ã ¤ +㤠ĺ +æĢ © +æĢ « +æĢ ¿ +å® ķ +ç© ¹ +å® ĵ +è¯ ĵ +è¯ Ķ +è¯ ĸ +è¯ ĺ +æĪ ¾ +è¯ Ļ +æĪ ½ +éĥ ĵ +è¡ © +ç¥ Ĩ +ç¥ İ +ç¥ ĩ +è¯ ľ +è¯ Ł +è¯ £ +è¯ ¤ +è¯ § +è¯ ¨ +æĪ ķ +éĻ Ķ +å¦ ² +å¦ ¯ +å§ Ĺ +å¸ ij +åŃ ¥ +é© ½ +èĻ ± +è¿ ¨ +ç» Ģ +ç» ģ +ç» Ĥ +é© · +é© ¸ +ç» ī +ç» Į +éª Ģ +çĶ ¾ +çı ı +çı IJ +çı ij +çİ ³ +é¡ ¸ +çı ī +çı Ī +æĭ ® +åŀ Ń +æĮ Ŀ +æĮ ŀ +åŀ ¤ +èµ ³ +è´ ² +åŀ ± +åŀ Į +åŀ § +åŀ ĵ +æĮ ¦ +åŀ ł +èį ļ +èį ij +è´ ³ +èį ľ +èİ Ĵ +èĮ ¼ +èĮ ´ +èĮ ± +èİ Ľ +èį ŀ +èĮ ¯ +èį ı +èį ĩ +èį ĥ +èį ł +èĮ Ń +åŀ © +èį ¥ +èį ¦ +èį ¨ +èį © +åī ĭ +èį ª +èį ¬ +èį ® +æŁ ° +æł ī +æŁ ĺ +æł Ĭ +æŁ © +æŀ ° +æł Į +æŁ Ļ +æŀ µ +æŀ ³ +æŁ ŀ +æŁ Ŀ +æł Ģ +æŁ ¢ +æł İ +æŁ Ī +æŁ ģ +æŀ · +æŁ ½ +åī Į +éħ Ĭ +éĥ ¦ +çĶ Ń +çł Ĺ +çł ĺ +çł Ĵ +æĸ « +çł Ń +çł ľ +èĢ · +èĻ º +æ® Ĥ +æ® ĩ +æ® Ħ +è½ ± +è½ ² +è½ ³ +è½ ¶ +è½ ¸ +èĻ ¿ +æ¯ ĸ +è§ ĩ +å° ľ +åĵ IJ +çľ Ħ +çľ į +ðł ³ +ðł³ IJ +éĥ ¢ +çľ ĩ +çľ Ĭ +çľ Ī +ç¦ º +åĵ Ĥ +åĴ ´ +æĽ · +æĺ ´ +åĴ ¦ +åĵ ĵ +åĵ Ķ +çķ İ +åij ² +èĥ Ħ +çķ ĭ +çķ Ī +èĻ ¼ +èĻ » +çĽ ħ +åĴ £ +åĵ ķ +åī IJ +éĥ § +åĴ » +åĽ ¿ +åĴ ¿ +åĵ Į +åĵ Ļ +åĵ ļ +åĴ © +åĴ ¤ +åĵ Ŀ +åĵ ı +åĵ ŀ +å³ £ +ç½ ĺ +å³ Ĵ +å³ ¤ +å³ ĭ +è´ ¶ +éĴ ļ +éĴ ¡ +éĴ £ +éĴ ¤ +éĴ « +æ° ¡ +çī ¯ +éĥ ľ +ç§ ķ +ç§ Ń +ç« ½ +ç¬ Ī +ä¿ ¦ +ä¿ ¨ +ä¿ ħ +åı Ł +åŀ ¡ +çī ® +ä¿ £ +ä¿ ļ +çļ Ī +ä¿ Ł +éĢ ħ +å¾ ĩ +å¾ ī +èĪ ¢ +éĥ Ĺ +ä¿ İ +éĥ ¤ +çĪ ° +éĥ Ľ +çĵ ´ +èĥ ¨ +èĥ ª +èĥ Ľ +èĥ Ĥ +èĥ Ļ +èĥ į +èĥ Ĺ +è ĥĿ +æľ IJ +èĥ « +é¸ ¨ +åĮ į +çĭ ¨ +çĭ ¯ +é£ ij +çĭ © +çĭ ² +è¨ ĩ +éĢ Ħ +æĺ Ŀ +é¥ · +é¥ ¸ +é¥ ¹ +åŃ ª +å¨ Ī +åº ¥ +çĸ ¬ +çĸ £ +çĸ ¥ +çĸ Ń +åº ł +ç« ij +é£ Ĵ +éĹ ¼ +éĹ ¾ +éĹ ¿ +éĺ Ĥ +ç¾ ij +è¿ ¸ +ç± ¼ +éħ ĭ +çĤ » +çĥ Ģ +çĤ · +æ´ ± +æ´ ¹ +æ´ § +æ´ Į +æµ ĥ +æ´ ĩ +æ´ Ħ +æ´ Ļ +æ¶ İ +æ´ İ +æ´ « +æµ į +æ´ ® +æ´ µ +æµ Ĵ +æµ Ķ +æµ ķ +æ´ ³ +æģ ¸ +æģ ĵ +æģ ¹ +æģ « +æģ » +æģ Ĥ +æģ ª +æģ ½ +å® ¥ +æī ĥ +è¡ ² +è¡ ½ +è¡ ¿ +è¢ Ĥ +ç¥ ľ +ç¥ ĵ +ç¥ ļ +è¯ ® +ç¥ Ĺ +ç¥ ¢ +è¯ ° +è¯ ³ +é¸ © +æĺ ¶ +åĴ « +å¼ Ń +çī ģ +èĥ ¥ +éĻ Ł +å§ ® +å¨ Ĩ +å§ Ŀ +å§ £ +å§ ĺ +å§ ¹ +ç¾ ¿ +çĤ ± +çŁ ľ +ç» Ķ +éª ģ +éª ħ +ç» Ĺ +ç» Ľ +éª Ī +èĢ ĸ +æĮ Ī +çı ¥ +çı Ļ +é¡ ¼ +çı ° +çı © +çı § +çı £ +çı ŀ +çIJ ¤ +çı ² +æģ ļ +åŁ ķ +åŁ ĺ +åŁ Ļ +åŁ ļ +æĮ ¹ +èĢ Ĩ +èĢ Ħ +åŁ Ĵ +æį ĭ +è´ ½ +åŀ ¸ +æį ĥ +çĽ į +èį ¸ +èİ ³ +èİ ´ +èİ ª +èİ ł +èİ ľ +èİ ħ +èį ¼ +èİ © +èį ½ +èİ ¸ +èį » +èİ ¨ +é¸ ª +èİ ¼ +æł ² +æł ³ +æ¡ ¡ +æ¡ İ +æ¡ ¢ +æ¡ ¤ +æ¢ ĥ +æł Ŀ +æ¡ ķ +æ¡ ģ +æ¡ § +æ¡ ħ +æł Ł +æ¡ ī +æł © +éĢ ij +éĢ ĭ +å½ § +é¬ ² +è± ĩ +éħ IJ +éĢ ¦ +åİ Ŀ +åŃ ¬ +çł Ŀ +çł ¹ +çł § +çł · +çł Ł +çł ¼ +çł ¥ +çł £ +åī ŀ +çł » +è½ ¼ +è½ ¾ +è¾ Ĥ +é¸ « +è¶ ¸ +é¾ Ģ +é¸ ¬ +èĻ Ķ +çľ ¬ +åĶ Ľ +çľ Ļ +åĵ § +åĵ ½ +æĻ ģ +é¸ ® +è¶ µ +è¶ ¿ +çķ Ľ +èļ ¨ +èļ ľ +èļ į +èļ ĭ +èļ ¬ +èļ Ŀ +èļ § +åĶ ¢ +åľ Ħ +åĶ £ +åĶ ı +çĽ İ +åĶ ij +å´ Ĥ +å´ ĥ +ç½ ¡ +ç½ Ł +è§ Ĭ +èµ ħ +éĴ ² +éĴ µ +éĴ ¹ +éĴ º +éĴ ½ +éĴ ¼ +éĴ ¿ +éĵ Ģ +éĵ Ħ +éĵ Ĩ +éĵ Ī +éĵ ī +éĵ Ĭ +éĵ ĭ +éĵ Į +é ĵį +ä ¥ +ä¥ ½ +éĵ İ +æ° © +æ° ¤ +æ° ¦ +æ¯ ª +èĪ IJ +ç§ £ +ç§ « +çĽ ī +ç¬ Ħ +ç¬ ķ +ç¬ Ĭ +ç¬ ı +ç¬ Ĩ +ä¿ ¸ +ä¿ µ +åģ Į +ä¿ ³ +ä¿ ¶ +åĢ ¬ +åĢ ı +æģ ģ +åĢ Ń +ä¿ ¾ +åĢ ľ +éļ ¼ +éļ ½ +åĢ Į +åĢ ¥ +èĩ ¬ +éĥ « +åĢ ¨ +è¡ Ħ +é¢ Ģ +å¾ ķ +èĪ « +è¡ ¾ +èĥ ¯ +èĥ ± +èĥ ´ +èĥ Ń +èĦ į +èĥ ¼ +èĦ Ĵ +é¸ ± +é¸ ² +çĭ · +çĮ ģ +çĭ ³ +çĮ ĥ +çĭ º +éĢ ĸ +æ¡ Ģ +é¥ ½ +åĩ ĩ +æĮ Ľ +äº ³ +çĸ ³ +çĸ ´ +çĸ ¸ +çĸ ½ +çĹ Ī +çĸ ± +çĹ Ĥ +çĹ ī +è¡ ® +é¢ ĥ +æģ £ +æĹ Ĩ +æĹ Ħ +æĹ ĥ +éĺ ĥ +éĺ Ħ +è¨ ļ +éĺ Ĩ +æģ Ļ +ç² ij +çĥ ľ +çĥ © +çĥ Ĭ +åī ¡ +éĥ ¯ +çĥ ¬ +æ¶ ij +æµ ¯ +æ¶ ŀ +æ¶ Ł +å¨ ij +æ¶ ł +æµ ŀ +æ¶ ĵ +æµ ¥ +æ¶ Ķ +æµ ľ +æµ ł +æµ £ +æĤ ļ +æ ĤŃ +æĤ Ŀ +æĤ Ĵ +æĤ Į +æĤ Ľ +çª Ī +åī ľ +è¯ ¹ +è¯ ¼ +è¢ Ĵ +è¢ ¢ +è¯ ¿ +è° Ģ +è° Ĥ +è° Ħ +è° ĩ +å± IJ +å± Ļ +éĻ ¬ +åĭ IJ +å¥ ĺ +çī Ĥ +èļ © +éĻ ² +å¨ Į +å¨ ī +å¨ ² +å¨ ´ +å¨ £ +å¨ ĵ +å© Ģ +çķ ļ +éĢ ¡ +ç» ł +éª Ĭ +ç» ¡ +éª ĭ +ç» ¦ +ç» ¨ +éª İ +éĤ ķ +é¸ ¶ +å½ Ĺ +èĢ ľ +çĦ ĺ +èĪ Ĥ +çIJ ı +çIJ ĩ +éº ¸ +æı ¶ +åŁ ´ +åŁ ¯ +æį ¯ +æİ ³ +æİ ´ +åŁ ¸ +åŁ µ +èµ § +åŁ ¤ +æį Ń +éĢ µ +åŁ Ŀ +åł ĭ +åł į +æİ ¬ +é¸ · +æį ½ +æİ Ĭ +åł ī +æİ ¸ +æį © +æİ ® +æĤ « +åŁ Ń +åŁ ½ +æİ ĩ +æİ ¼ +èģ ĥ +èIJ ģ +èı ĺ +åł ĩ +èIJ ĺ +èIJ ĭ +èı ½ +èı ĸ +è IJľ +èIJ ¸ +èIJ ij +æ£ » +èı Ķ +èı Ł +èIJ ı +èı ¹ +èı ª +èı ħ +èı Ģ +èı ° +èı ¡ +æ¢ ¿ +æ¢ ı +è§ ĭ +æ¡ ´ +æ¡ · +æ£ ģ +æ¡ « +æ£ Ĥ +åķ ¬ +éĥ ¾ +æķ ķ +è± ī +éĦ Ħ +éħ ŀ +ç¡ İ +ç¡ Ń +ç¡ ĸ +ç¡ Ĺ +ç¡ IJ +ç¡ ĩ +ç¡ Į +é¸ ¸ +çĵ ł +åĮ ı +åİ © +æ® Ĵ +æ® ĵ +æ® į +èµ ī +éĽ © +è¾ Ħ +åł ij +çľ Ń +çľ ¦ +åķ § +æĻ ¡ +æĻ ¤ +çľ µ +åľ Ĭ +åĸ ı +åķ ī +åĭ ĸ +æĻ ŀ +åĶ µ +æĻ Ĺ +åķ Ń +çķ ¦ +è¶ º +åķ ® +è· Ħ +èļ ¶ +è ĽĦ +èĽ İ +èĽ Ĩ +èļ ° +åľ ī +èļ ± +èĽ ī +èĽ ı +èļ ´ +åķ ģ +åķ ķ +åĶ ¿ +åķ IJ +åĶ ¼ +åĶ · +åķ ĸ +åķ µ +åķ ¶ +åķ · +åĶ ³ +åĶ ° +åķ ľ +å¸ » +å´ ļ +å´ ¦ +å¸ ¼ +å´ ® +å´ ¤ +å´ Ĩ +èµ ĩ +èµ Ī +èµ Ĭ +éĵ ij +éĵ Ĵ +éĵ Ĺ +éĵ Ļ +éĵ Ł +éĵ ¡ +éĵ ¢ +éĵ £ +éĵ ¤ +éĵ § +éĵ ¨ +éĵ © +éĵ ª +éĵ « +éĵ ¯ +éĵ ° +éĵ ± +éĵ ³ +éĵ µ +éĵ · +çī ¾ +é¸ ¹ +ç§ ¾ +éĢ ¶ +ç¬ º +çŃ ĩ +ç¬ ¸ +ç¬ ª +ç¬ ® +ç¬ ł +ç¬ ¥ +ç¬ ¤ +ç¬ ³ +ç¬ ¾ +ç¬ ŀ +åģ ¾ +åģ ĥ +åģ ķ +åģ Ī +åĤ Ģ +åģ ¬ +åģ » +çļ ij +çļ İ +é¸ » +å¾ ľ +èĪ ¸ +èĪ » +èĪ ´ +èĪ · +é¾ Ľ +ç¿ İ +èĦ ¬ +èĦ ĺ +èĦ ² +åĮ IJ +çĮ Ĺ +çĮ ¡ +çĮ ŀ +æĸ Ľ +çĮ ķ +é¦ Ĺ +é¦ ĥ +é¦ Ħ +é¸ ¾ +åº ¹ +åº ¾ +çĹ Ķ +çĹ į +ç¿ Ĭ +æĹ Į +æĹ İ +è¢ ¤ +éĺ ĩ +éĺ Ī +éĺ ī +éĺ Ĭ +éĺ ĭ +éĺ į +éĺ ı +ç¾ Ł +ç² Ŀ +çĦ IJ +çĦ ĵ +çĦ Ĺ +æ· ħ +æ· ŀ +æ¸ İ +æ¶ ¿ +æ· ĸ +æĮ ² +æ· ł +æ¶ ¸ +æ¸ ij +æ· ¦ +æ· Ŀ +æ¶ ª +æ· Ļ +æ¶ « +æ¸ Į +æĤ » +æĤ ± +æ ĥĿ +æĥ ĺ +æĥ Ĩ +æĥ ļ +æĥ ĩ +æĥ ® +çª ķ +è° Į +æī Ī +çļ ² +è° ij +è£ Ĩ +è¢ · +è£ ī +è° Ĵ +è° Ķ +è° ķ +è° ĸ +è° Ĺ +è° Ļ +è° Ŀ +éĢ ¯ +éĥ ¿ +éļ Ī +ç² ľ +éļ į +éļ Ĺ +å© Ĭ +å¨ ¼ +å© ¢ +å© µ +èĥ ¬ +è¢ Ī +ç¿ Į +æģ ¿ +æ¬ ¸ +ç» « +éª IJ +ç» ¯ +ç» ± +éª Ĵ +ç» ² +éª ĵ +ç» ¶ +ç» º +ç» » +ç» ¾ +éª ĸ +ç¼ ģ +èĢ ł +çIJ « +çIJ µ +çIJ ¶ +çIJ ¥ +çIJ ¨ +çIJ ° +çIJ ® +çIJ ¯ +çIJ ¬ +çIJ ļ +è¾ ĩ +é¼ ĭ +æı ³ +åł ŀ +æIJ ½ +æı ¸ +æı ł +åł Ļ +è¶ Ħ +æı ĸ +é¢ ī +å¡ Ħ +æı ¿ +èĢ ĭ +æı Ħ +èĽ © +èĽ ° +å¡ Ĩ +æij Ĵ +æı Ĩ +æİ ¾ +èģ Ĵ +èij ij +èij ļ +éĿ ° +éĿ ¸ +èij ³ +èij º +èij ¸ +èIJ ¼ +èij ¶ +è ĴĮ +èij Ń +æ¥ ® +æ £¼ +æ¤ Ł +æ£ ¹ +æ¤ ¤ +æ£ ° +èµ į +æ¤ ĭ +æ¤ ģ +æ¤ ª +æ¤ IJ +é¹ ģ +éħ ¤ +éħ ¢ +éħ ¡ +é¹ Ĥ +æ® ļ +æ® Ľ +éĽ ± +è¾ ĭ +æ¤ ł +è¾ İ +çĿ Ħ +çĿ ĩ +çĿ ĥ +æĪ ¢ +åĸ ĭ +åĹ Ĵ +åĸ ĥ +åĸ ± +åĸ ¹ +æĻ · +åĸ Ī +è· ĸ +è· Ĺ +è· ŀ +è· ļ +è· İ +è· ı +è· Ĩ +èĽ ± +èĽ ² +èĽ Ń +èĽ ³ +èĽ IJ +èĽ Ķ +èĽ ŀ +èĽ ´ +èĽ ĺ +åĸ ģ +åĸ Ł +åķ ¾ +åĹ ĸ +åĸ ij +åĹ Ł +åĹ ŀ +åĸ Ļ +åµ ĺ +åµ ĸ +å´ ´ +éģ Ħ +è© Ī +åµ İ +å µ¬ +åµ Ľ +åµ ¯ +åµ Ŀ +åµ « +å¹ Ħ +åµ ĭ +èµ ķ +éĵ » +éĵ ¼ +éĵ ¿ +éĶ ĥ +éĶ Ĩ +éĶ ĩ +éĶ ī +éĶ ı +éĶ ij +éĶ Ĵ +éĶ Ķ +éĶ ķ +æİ £ +çŁ ¬ +æ° ° +æ¯ ³ +æ¯ ½ +çĬ Ĭ +çĬ Ħ +çĬ ĭ +é ¹Ħ +çĬ į +åµ ĩ +é» į +ç¨ ĥ +ç¨ Ĥ +çŃ ļ +çŃ µ +çŃ Į +åĤ £ +åĤ Ī +èĪ Ħ +çī į +åĤ ¥ +åĤ § +éģ ij +åĤ © +å¾ ¨ +åª Ń +çķ ² +å¼ ij +ç¿ ķ +é¹ Ĩ +èħ Ī +èħ ĵ +èħ Ĩ +èħ ´ +èħ ļ +èħ ± +é± ¿ +é² Ģ +é² Ĥ +çĮ ¢ +çĮ ¹ +çĮ ¥ +é£ ĵ +è§ ŀ +è§ ļ +çĮ ± +é¢ İ +é£ § +é¦ ĩ +é¦ Ĭ +äº µ +èĦ Ķ +è£ Ĵ +çĹ £ +çĹ ¨ +çĹ ¦ +çĹ ŀ +çĹ ¤ +çĹ § +èµ ĵ +ç« ¦ +çĵ ¿ +åķ » +é¢ ı +é¹ ĩ +éĺ ij +éĺ Ĵ +éĺ ķ +ç² ŀ +éģ Ĵ +åŃ ³ +çĦ ¯ +çĦ ľ +çĦ ± +é¹ Ī +æ¸ « +æ¹ ® +æ¹ İ +æ¹ ľ +æ¹ į +æ¹ « +æº ² +æ¹ Ł +æº Ĩ +æ¹ ² +æ¹ Ķ +æ¹ ī +æ¸ ¥ +æ» ģ +æĦ ł +æĥ º +æĦ ¦ +æĥ ´ +æĦ Ģ +æĦ İ +æĦ Ķ +åĸ ¾ +å¯ IJ +è° Ł +è£ ¢ +è£ İ +è£ ¥ +ç¥ ¾ +è° ł +è° ¡ +è° ¥ +è° § +åŃ ± +å¼ ¼ +å· ½ +éª ĺ +åª ª +å· ¯ +ç¿ ļ +çļ ´ +éª Ľ +ç¼ Ĥ +ç¼ ĥ +ç¼ Ħ +å½ ĺ +ç¼ ĩ +ç¼ Ī +ç¼ Į +ç¼ ij +ç¼ Ĵ +ç¼ Ĺ +é£ ¨ +èĢ ¢ +çij ģ +çij Ĺ +çij Ħ +éģ ¨ +éª ľ +éŁ « +é« ¡ +å¡ ¬ +éĦ ¢ +è¶ Ķ +è¶ ij +æij ħ +æij ģ +èľ ĩ +æIJ ĭ +æIJ ª +æIJ IJ +æIJ Ľ +æIJ ł +æij Ī +å½ Ģ +æ¯ Ĥ +æIJ ¦ +æIJ ¡ +èĵ ģ +æĪ ¡ +è ĵį +éĦ ŀ +èĵ IJ +èĵ ¦ +é¹ ĭ +èĴ ½ +èĵ ĸ +èĵ Ĭ +èĴ ¯ +èĵ Ł +èĵ ij +èĴ º +èĵ ł +èĴ Ł +èĴ ¡ +èĴ ¹ +èĴ ´ +èĴ Ĺ +èĵ ¥ +æ¥ Ķ +æ¥ Ĥ +æ¥ Ŀ +æ¥ « +æ¥ ¸ +æ¤ ´ +æ§ Į +æ¥ ¯ +çļ Ļ +æ¦ Ī +æ§ İ +æ¦ ī +æ¥ ¦ +æ¥ £ +æ¥ ¹ +æ¤ ½ +åī ½ +éħ © +èľ ĥ +ç¢ Ľ +ç¢ ĵ +ç¡ ¼ +ç¢ ī +ç¢ ļ +ç¢ ĩ +ç¢ ľ +é¹ Į +è¾ ı +é¾ ĥ +é¾ ħ +è¨ ¾ +ç² ² +çĿ ļ +åĹ ª +éŁ ª +åĹ · +åĹ ī +çĿ ¨ +çĿ ¢ +éĽ İ +çĿ ¥ +åĹ ij +åĹ « +åĹ ¬ +åĹ Ķ +åĹ Ŀ +æĪ ¥ +åĹ Ħ +çħ ¦ +æļ Ħ +éģ ¢ +æ ļĮ +è· ¬ +è· ¶ +è ·¸ +è· IJ +è· £ +è· ¹ +èĽ ¸ +èľ Ĭ +èľ į +èľ ī +èľ £ +çķ ¹ +èĽ ¹ +åĹ ¥ +åĹ ² +åĹ ³ +åĹ Į +åĹ į +åĹ IJ +åĹ ¤ +åĹ µ +ç½ ¨ +åµ Ĭ +åµ ´ +éª ° +éĶ Ĺ +éĶ Ľ +éĶ ľ +éĶ Ŀ +éĶ ŀ +éĶ Ł +éĶ ¢ +éĶ ¨ +éĶ © +éĶ Ń +éĶ ± +éĽ ī +æ° ² +çĬ ı +æŃ ĥ +ç¨ ŀ +ç¨ Ĺ +ç¨ Ķ +çŃ ł +çŃ ¢ +çŃ ® +çŃ ² +çī Ĵ +æķ « +å¾ Ń +æĦ Ĩ +èī Ħ +è§ İ +æ¯ ¹ +è² Ĭ +è² ħ +è² ī +é¢ Ķ +èħ ł +èħ © +èħ ¼ +èħ Ń +è ħ§ +å¡ į +åª µ +é² ħ +é² Ĩ +é² ĩ +é² Ī +é² ĭ +é² IJ +èĤ Ħ +é¹ IJ +é£ ķ +è§ ¥ +éģ Ľ +é¦ IJ +é¹ ij +äº ¶ +çĺ ĥ +çĹ ± +çĹ ¼ +çĹ ¿ +çĺ IJ +çĺ ģ +çĺ Ĩ +éº Ĥ +æŃ Ĩ +æĹ Ĵ +éĺ ĸ +éĺ Ĺ +ç¾ § +è± ¢ +ç² ³ +çĮ · +çħ ³ +çħ ¨ +çħ ħ +çħ Ĭ +çħ ¸ +çħ º +æ» Ł +æº ± +æº ĺ +æ¼ Ń +æ» ¢ +æº ¥ +æº ½ +è£ Ł +æº » +æº · +æ» Ĺ +æ» « +æº ´ +æ» ı +æ» ĥ +æ» ¦ +æº ı +æ» Ĥ +æ» ĵ +æº Ł +æ» ª +æĦ « +æħ Ĭ +é² İ +éª ŀ +çª ł +çª £ +è£ ± +è£ ¨ +è£ ¾ +è£ ° +ç¦ Ĭ +è° © +è° ª +åª ¾ +å« « +åª ² +å« Ĵ +å« Ķ +åª ¸ +ç¼ Ļ +ç¼ ľ +ç¼ Ľ +è¾ Ķ +éª Ŀ +ç¼ Ł +ç¼ ¡ +ç¼ ¢ +ç¼ £ +éª Ł +èĢ ¥ +çĴ Ī +çij Ń +çį Ĵ +è§ ı +æħ Ŀ +å« ł +åı Ĩ +æij ½ +å¢ ģ +æĴ Ĥ +æij ŀ +æĴ Ħ +ç¿ ¥ +è¸ ħ +æij Ń +å¢ ī +å¢ Ĵ +æ¦ ĸ +ç¶ ¦ +èĶ « +èĶ · +éĿ º +éĿ ¼ +éŀ ħ +éĿ ¿ +çĶ į +èĶ ¸ +èĶ Ł +èĶ º +æĪ ¬ +èķ ĸ +èĶ » +èĵ ¿ +æĸ ¡ +é¹ ķ +èĵ ¼ +æ¦ Ľ +æ¦ § +æ¦ « +æ¦ Ń +æ§ Ķ +æ¦ ± +æ§ ģ +æ§ ł +æ¦ · +åĥ ° +éħ ½ +éħ ¹ +ç¢ ¡ +ç¢ ´ +ç¢ £ +ç¢ ² +èĩ § +è± ¨ +æ® ¡ +éľ ģ +èľ ļ +é¾ ĩ +é¾ Ī +ä ģ +äģ ĸ +çĿ ½ +åĺ ŀ +åĺ Ī +åĺ Į +åĺ ģ +æļ Ŀ +è¸ Į +è¸ ī +èľ ŀ +èľ ¥ +èľ ® +èĿ Ī +èľ ´ +èľ ± +èľ © +èľ · +èľ ¿ +èŀ Ĥ +èľ ¢ +åĺ ¡ +é¹ Ĺ +åĺ £ +åĺ ¤ +åĺ ļ +åĹ ¾ +åĺ § +ç½ ´ +ç½ ± +å¹ Ķ +å¶ Ĥ +å¹ Ľ +èµ Ļ +ç½ Ĥ +éª · +éª ¶ +é¹ ĺ +éĶ ² +éĶ ´ +éĶ ¶ +éĶ · +éĶ ¸ +éĶ µ +éķ Ĥ +çĬ Ĵ +ç® IJ +ç® ¦ +ç® § +ç® ¸ +ç® ¬ +ç® ħ +ç® ª +ç® ľ +ç® ¢ +ç® ĵ +åĥ ĸ +åĦ Ĩ +åĥ ³ +åĥ Ń +åĬ ģ +åĥ ® +éŃ ĥ +éŃ Ĩ +çĿ ¾ +èī ĭ +éĦ ± +èĨ Ī +èĨ ij +é² ij +é² Ķ +é² ļ +é² Ľ +é² Ł +çį IJ +è§ « +éĽ Ĵ +å¤ ¤ +é¦ ij +éĬ ® +å¡ ¾ +çĺ Į +çĺ Ĭ +çĺ ĺ +çĺ Ļ +æĹ ĸ +èĨ Ĥ +éĺ ļ +éĦ ¯ +é² ŀ +ç² ¿ +ç² ¼ +ç³ ģ +æ§ Ĭ +é¹ ļ +çĨ ĺ +çĨ ¥ +æ½ ¢ +æ¼ ķ +æ» ¹ +æ¼ ¯ +æ¼ ¶ +æ½ ĭ +æ½ ´ +æ¼ ª +æ¼ ī +æ¼ © +æ¾ ī +æħ µ +æIJ ´ +çª ¨ +å¯ ¤ +ç¶ ® +è° ® +è¤ ¡ +è¤ Ļ +è¤ ĵ +è¤ Ľ +è¤ Ĭ +è° ¯ +è° ° +è° ² +å± £ +é¹ Ľ +å« ± +å« ĸ +å« ¦ +å« ļ +å «ĺ +é¼ IJ +çŀ Ģ +é¹ ľ +éª ł +ç¼ ¥ +ç¼ ¦ +ç¼ § +ç¼ ¨ +éª ¢ +ç¼ « +èĢ ¦ +èĢ § +çĴ ľ +çĴ İ +çĴ ģ +å¥ Ń +é« ¯ +é« « +æĴ · +æĴ ħ +èµ Ń +æĴ ¸ +éĭ Ĩ +æĴ Ļ +æĴ º +å¢ Ģ +èģ © +è§ IJ +éŀ ij +èķ Ļ +éŀ Ĵ +èķ Ī +èķ ¨ +èķ ¤ +èķ ŀ +èķ º +çŀ ¢ +èķ ĥ +èķ ² +èµ ľ +æ§ ¿ +æ¨ ¯ +æ§ Ń +æ¨ Ĺ +æ¨ ĺ +æ§ ² +éĨ Į +éĨ ħ +éĿ ¥ +éŃ ĩ +é¤ į +ç£ Ķ +ç£ Ļ +éľ Ī +è¾ ĺ +é¾ ī +é¾ Ĭ +è§ ij +çŀ Į +ç ŀĭ +çŀ ij +åĺ Ń +åĻ İ +åĻ ¶ +é¢ Ļ +æļ ¹ +åĻ ĺ +è¸ Ķ +è¸ Ŀ +è¸ Ł +è¸ Ĵ +è¸ ¬ +è¸ ® +è¸ ¯ +è¸ º +è¸ ŀ +èĿ ½ +èĿ ¾ +èĿ » +èĿ ° +èĿ ® +è ŀĭ +èĿ ĵ +èĿ £ +è Ŀ¼ +åĺ ¬ +é¢ ļ +åĻ į +åĻ Ļ +åĻ Į +åĻ Ķ +é¢ Ľ +å¹ ŀ +å¹ ¡ +å¶ Ļ +å¶ Ŀ +éª º +éķ Ĭ +éķ ī +éķ Į +éķ ı +éķ Ĵ +éķ ĵ +éķ Ķ +ç¨ · +ç® ´ +ç¯ ij +ç¯ ģ +ç¯ Į +çī ĸ +åĦ ĭ +èĻ ¢ +é¹ ŀ +èĨ ĺ +é² ł +é² ¡ +é² ¢ +é² £ +é² ¥ +é² § +é² © +çį Ĺ +çį ł +è§ ¯ +é¦ ĵ +é¦ Ķ +éº ¾ +å» Ľ +çĺ Ľ +çĺ ¼ +çĺ ¢ +çĺ ł +é½ ij +ç¾ ° +𥠻 +ð¥» Ĺ +ç³ Į +ç³ į +ç³ ħ +çĨ ľ +ç Ĩµ +æ¾ į +æ¾ Į +æ½ ¸ +æ½ ¦ +æ½ ² +éĭ Ī +æ½ Ł +æ½ º +å¯ ® +çª ³ +è° ³ +è¤ ´ +è¤ Ł +è¤ « +è° µ +çĨ ¨ +å± ¦ +åĭ ° +æĪ ® +èĿ ¥ +ç¼ ¬ +ç¼ ® +ç¼ ¯ +éª £ +çķ ¿ +èĢ © +èĢ ¨ +èĢ ª +çĴ Ł +éĿ Ľ +çĴ ł +çĴ ĺ +èģ ± +èŀ ¯ +é« » +é« Ń +é« ¹ +æĵ Ģ +çĶ ı +æĵ ŀ +ç¸ ł +ç£ ¬ +é¢ ŀ +èķ » +é¢ Ł +èĸ ¤ +èĸ ¨ +æª ł +èĸ ı +èĸ ® +èĸ ľ +èĸ ħ +æ¨ ¾ +æ© Ľ +æ© ĩ +æ¨ µ +æª İ +æ© ¹ +æ¨ ½ +æ¨ ¨ +æ© ¼ +å¢ ¼ +æ© IJ +ç¿ ® +éĨ IJ +éĨ į +éĨ ļ +ç£ ² +èµ Ŀ +æ® ª +éľ ı +éĮ ¾ +è¾ ļ +éģ ½ +æ° ħ +çŀ Ł +çŀ ł +çŀ ° +åļ Ħ +åļ Ĩ +åĻ ¤ +æļ ¾ +è¹ Ģ +è¸ µ +è¸ ½ +è¹ ī +è¹ ģ +èŀ ¨ +èŀ Ī +èŀ ħ +èŀ Ń +èŀ ł +èŀ Ł +åĻ ± +åĻ « +åĻ » +åĻ ¼ +ç½ ¹ +åľ ľ +ä ¦ +ä¦ ĥ +éķ Ĺ +éķ ĺ +éķ ļ +éķ Ľ +éķ Ŀ +éķ ŀ +éķ ł +æ° ĩ +æ° Ĩ +ç© ij +ç¯ Ŀ +ç¯ ¥ +ç¯ ¦ +ç¯ ª +ç¯ Ļ +çĽ ¥ +åĬ ĵ +ç¿ ± +éŃ ī +éŃ Ī +å¾ ¼ +æŃ Ļ +èĨ ¦ +èĨ Ļ +é² ® +é² ± +é² ³ +é² ´ +é² µ +é² · +é² » +çį ´ +çį Ń +çį ¬ +éĤ Ĥ +é¹ § +å» ¨ +èµ Ł +çĺ ° +å» ª +çĺ ¿ +çĺ µ +çĺ ´ +çĻ ĥ +çĺ ³ +éº ĩ +éº Ī +å ¬´ +å£ ħ +ç³ Ĺ +çĶ ij +çĩ İ +çĩ ł +çĩ Ķ +çĩ § +æ¿ ij +æ¿ ī +æ½ ŀ +æ¾ § +æ¾ ¹ +æ¾ ¥ +æ¾ ¶ +æ¿ Ĥ +è¤ ° +çª ¸ +å¬ ĸ +çĬ Ł +éļ ° +å¬ Ĺ +é¢ ¡ +ç¼ ± +ç¼ ² +ç¼ ³ +çĴ © +çĴ ª +èŀ « +æĵ ¤ +å£ ķ +è§ ³ +ç½ Ħ +æĵ ¢ +èĸ ¹ +éŀ ¡ +éŀ ¬ +èĸ · +èĹ ĵ +èĹ ģ +æª Ħ +æª © +æĩ ĭ +éĨ ¢ +ç¿ ³ +ç¤ ħ +ç£ ´ +é¹ © +é¾ ĭ +é¾ Į +è± ³ +å£ ij +é» » +åļ ı +åļ ħ +è¹ ij +è¹ Ĵ +è¹ Ĭ +è Ł¥ +èŀ ¬ +èŀ µ +çĸ ĥ +èŀ ³ +èŁ ij +åļ ĵ +ç½ ½ +ç½ ¾ +å¶ · +é» ľ +é» Ŀ +é« ģ +é« Ģ +éķ ¡ +éķ ¢ +éķ £ +éķ ¦ +éķ § +éķ © +éķ ª +éķ « +ç½ ħ +ç° Į +ç¯ ¾ +ç¯ ¼ +ç° ĸ +ç° ĭ +é¼ ¢ +åĦ ¡ +é¹ ª +é¼ ¾ +çļ ¤ +éŃ į +é¾ ł +ç¹ ĩ +è² ĺ +éĤ Ī +è² Ķ +èĩ Į +èĨ » +èĩ Ĩ +èĩ ĥ +é² ¼ +é² ½ +é³ Ģ +é³ ĥ +é³ ħ +é³ ĩ +é³ Ĭ +èŀ ½ +çĩ ® +é¹ « +ç³ ľ +ç¸ » +çĻ į +éº ĭ +æĩ ij +æ¿ ¡ +æ¿ ® +æ¿ ŀ +æ¿ ł +æ¿ ¯ +è¹ ĩ +è¬ ĩ +éĤ ĥ +è¥ ģ +æª Ĺ +æ ĵĺ +åŃ º +éļ ³ +å¬ · +èŁ Ĭ +é¹ ¬ +éį ª +éı Ĭ +é¬ Ī +é¬ ĥ +çŀ ½ +éŀ ¯ +éŀ ¨ +éŀ « +éŀ § +éŀ £ +èĹ ľ +èĹ ł +éĨ ª +è¹ Ļ +ç¤ ĵ +çĩ ¹ +é¤ ® +çŀ ¿ +æĽ Ľ +é¢ ¢ +èº ĩ +è¹ ļ +èŁ Ľ +èŁ ª +èŁ ł +èŁ ® +é¹ ® +é» ł +é» Ł +é« ħ +é« Ĥ +éķ ¬ +éķ Ń +éķ ¯ +é¦ ¥ +ç° Ł +ç° ª +é¼ ¬ +éĽ ł +èī Ł +é³ İ +é³ ı +é³ IJ +çĻ ŀ +çĻ Ķ +ç³ ¨ +è¹ © +éİ ı +éĤ ĭ +é¬ ı +æĶ ī +éŀ ² +éŀ ´ +èĹ ¿ +èĺ § +èĺ ħ +éĨ ® +éĨ ¯ +éħ ĥ +éľ ª +éľ Ń +éľ ¨ +é» ¼ +åļ ¯ +è¹ ° +è¹ ¶ +è¹ ½ +è¹ ¼ +è¹ ´ +è¹ ¾ +è¹ ¿ +èł ĸ +èł ĵ +èŁ ¾ +èł Ĭ +é» ¢ +é« ĭ +é« Į +éķ ² +ç± Ģ +é½ ģ +éŃ ij +èī ¨ +é³ ĵ +é³ Ķ +é³ ķ +é³ Ĺ +é³ Ļ +éı ĸ +ç¾ ¸ +㸠Ĩ +çĢ £ +çĢ Ľ +è¥ ¦ +è° ¶ +è¥ ŀ +éª ¥ +ç¼ µ +çĵ Ĵ +æĶ ĺ +èĺ © +èĺ ĸ +éĨ ´ +éľ ° +éħ Ĩ +çŁ į +èº ħ +é¼ į +å· ī +é» © +é» ¥ +é» ª +éķ ³ +éķ ´ +é» § +çº Ĥ +çĴ º +é¼ ¯ +èĩ ľ +é³ ľ +é³ Ŀ +é³ Ł +çį ¾ +åŃ Ģ +éª § +ç ĵĺ +é¼ Ļ +éĨ º +ç¤ ´ +é¢ ¦ +æĽ © +é³ ¢ +éº Ŀ +å¤ Ķ +çĪ Ŀ +çģ ı +ç¦ ³ +éIJ ¾ +ç¾ ¼ +èł ¡ +èĢ ± +é¹ ³ +æ° į +é¥ ķ +èº IJ +é« ij +éķ µ +ç© ° +é¥ Ķ +é¬ » +é¬ Ł +è¶ ± +æĶ « +æĶ ¥ +é¢ § +èº ľ +é¼ ¹ +çĻ ¯ +èł ² +èł ¹ +èº ŀ +è¡ ¢ +çģ ŀ +è¥ » +çº Ľ +é¬ £ +æĶ ® +åĽ Ķ +é¦ ķ +æĪ Ĩ +çĪ ¨ +é½ ī +äº į +å° ¢ +å½ ³ +åį ¬ +æ® ³ +ðł Ļ¶ +æ¯ Į +éĤ ĺ +æĪ ĭ +åľ ¢ +æ° ķ +ä¼ ĭ +ä» Ŀ +åĨ ® +æ° ¿ +æ± Ī +æ° ¾ +å¿ ī +å® Ħ +𬣠Ļ +è® ± +æī ŀ +åľ ² +åľ « +èĬ ı +èĬ ĥ +æľ ³ +æľ ¸ +ð¨ Ļ +ð¨Ļ ¸ +éĤ ¨ +åIJ Ĵ +åIJ ĸ +å± ¼ +å± ¾ +è¾ ¿ +éĴ Ĩ +ä» ³ +ä¼ £ +ä¼ Ī +çĻ ¿ +çĶ ª +éĤ ł +çĬ ´ +åĨ ± +éĤ ¡ +ð¬ĩ ķ +æ± ĭ +ä ľ +äľ £ +è® » +𬣠ŀ +åŃ ĸ +ð¬ĺ ĵ +çº © +çİ Ĵ +çİ ĵ +çİ ĺ +çİ ļ +åĪ ¬ +ð«Ń Ł +åĿ ľ +åĿ ī +æī ½ +ð«Ń ¢ +åĿ ĭ +æī º +㧠ij +æ¯ IJ +èĬ ° +èĬ £ +èĭ Ĭ +èĭ ī +èĬ ĺ +èĬ ´ +èĬ ł +ð« ĩ +ð«ĩ Ń +èĬ ¤ +æĿ ķ +æĿ Ļ +æĿ Ħ +æĿ § +æĿ © +å° ª +å° ¨ +è½ ª +ð«IJ Ħ +åĿ Ĵ +èĬ Ī +æĹ ´ +æĹ µ +åij Ļ +ã ķ +ãķ ® +å² į +ð« µ +𫵠· +å² ł +å² ľ +åij ĩ +åĨ ı +è§ ĥ +å² Ļ +ä¼ ¾ +ãij ĩ +ä¼ Ń +ä½ ĸ +ä¼ ² +ä½ ģ +é£ ı +çĭ ĥ +éĹ ¶ +æ± § +æ± « +𣲠ĺ +ð£² Ĺ +æ² Ħ +æ² ĺ +ð¬ĩ Ļ +æ± Ń +ã³ ĩ +æ² ĩ +å¿ ® +å¿ ³ +å¿ º +𬣠¡ +ç¥ ĥ +è¯ ĩ +éĤ ² +è¯ İ +è¯ IJ +å± ĥ +ð« ¸ +𫸠© +å² Ĭ +éĺ ½ +ä¢ º +éĺ ¼ +å¦ § +å¦ ĺ +ð¨ ļ +ð¨ļ ķ +çº ® +é© ² +ð«ĺ ľ +çº » +ð¬ĺ ĺ +ð«ĺ Ŀ +çº ¼ +çİ ¤ +çİ ŀ +çİ ± +çİ Ł +éĤ ½ +éĤ ¿ +åĿ ¥ +åĿ ° +åĿ ¬ +åĿ ½ +å¼ Ĩ +èĢ µ +ä¢ ¼ +ð¦ Ń +ð¦Ń ľ +èĮ ĭ +èĭ § +èĭ ¾ +èĭ ł +æŀ ħ +ãŃ İ +æŀ ĺ +æŀ į +çŁ ¼ +çŁ » +åĮ ¼ +𬨠Ĥ +ð¬Ģ © +ð¬Ģ ª +æĹ ¿ +æĺ Ħ +æĺ Ĵ +æĺ Ī +åĴ ī +åĴ ĩ +åĴ į +å² µ +å² ½ +å² ¨ +å² ŀ +å³ Ĥ +ã Ł +ãŁ ĥ +åĽ · +𬬠© +éĴ IJ +éĴ Ķ +éĴ ĸ +çī ¥ +ä½ ´ +åŀ Ī +ä¾ ģ +ä¾ ¹ +ä½ ¸ +ä½ º +éļ ¹ +ãij Ĭ +ä¾ Ĥ +ä½ ½ +ä¾ ĺ +éĥ Ī +èĪ ł +éĥ IJ +éĥ ĥ +æĶ ½ +èĤ Ń +èĤ ¸ +èĤ · +çĭ ī +çĭ Ŀ +é¥ ³ +å¿ ŀ +çĤ Į +çĤ Ĩ +æ³ Ļ +æ² º +æ³ Ĥ +æ³ ľ +æ³ ĥ +æ³ ĩ +æĢ Ĭ +å³ ĥ +ç© ¸ +ç¥ ĭ +ç¥ Ĭ +ð«į £ +𬣠³ +𬠩½ +é¸ ¤ +å¼ ¢ +å¼ ¨ +éĻ ij +𬮠¿ +éĻ İ +𬯠Ģ +åį º +ä¹ ¸ +å¦ Ń +å§ Ī +ð« ° +ð«° Ľ +è¿ ³ +åı ķ +𬳠µ +é© µ +𬳠¶ +ä Į +äĮ ¹ +é© º +ð«ł Ĭ +ç» ĭ +ç» IJ +çł ī +èĢ Ķ +ãĽ ĥ +çİ ¶ +çı ĩ +çı ħ +ð¬į Ľ +çı ĭ +çİ ¹ +çı Į +çİ ¿ +éŁ ¨ +åŀ ļ +åŀ ¯ +åŀ Ļ +åŀ ² +åŁ ı +åŀ į +èĢ ĩ +é¿ į +åŀ İ +åŀ ´ +åŀ Ł +åŀ ŀ +æĮ ĵ +åŀ µ +åŀ ı +æĭ ¶ +èį ĸ +èį ģ +èį Ļ +èį Ľ +èĮ Ī +èĮ ½ +èį Ħ +èĮ º +ð¬ľ ¬ +èį ĵ +èĮ ³ +𦠰 +𦰠¡ +èĮ Ľ +èį Ń +ãŃ ķ +æŁ · +æŁ ĥ +æŁ Ĭ +æŀ ¹ +æł IJ +æŁ ĸ +éĥ ļ +åī ħ +ä´ ĵ +è¿ º +åİ ĸ +çł Ĩ +çł ij +çł Ħ +èĢ ı +å¥ ĵ +ä ¶ +ä¶ ® +è½ µ +è½ · +è½ ¹ +è½ º +æĺ º +𪠾 +𪾠¢ +æĺ ½ +çĽ · +åĴ ¡ +åĴ º +æĺ ³ +æĺ £ +æĺ ¤ +æĺ « +æĺ ¡ +åĴ ¥ +æĺ ª +èĻ · +èĻ ¸ +åĵ ĥ +å³ ĺ +èĢ ij +å³ Ľ +𪨠° +å³ Ĺ +å³ § +å¸ ¡ +éĴ ĺ +ð«ĵ § +éĴ ľ +𬬠® +𬬠± +ð¬¬ Ń +éĴ ª +éĴ ¬ +éĴ Ń +çŁ § +ç§ ¬ +ä¿ « +èĪ ģ +ä¿ ľ +ä¿ Ļ +ä¿ į +åŀ ķ +è¡ İ +èĪ £ +å¼ ĩ +ä¾ ´ +é¸ § +äı ¡ +èĥ ł +ð¦ Ļ¶ +èĥ Ī +èĥ © +èĥ £ +æľ ı +é£ IJ +è¨ Ħ +é¥ » +åº ¤ +çĸ ¢ +çĤ £ +çĤ Ł +ã ¶ +㶠² +æ´ Ń +æ´ ĺ +æ´ ĵ +æ´ ¿ +ã³ ļ +æ³ ļ +æµ Ī +æµ ī +æ´ ¸ +æ´ ij +æ´ ¢ +æ´ Ī +æ´ ļ +æ´ º +æ´ ¨ +æµ IJ +ã³ ĺ +æ´ ´ +æ´ £ +æģ Ķ +å® ¬ +çª Ģ +æī Ĥ +è¢ Ĩ +ç¥ ı +ç¥ IJ +ç¥ ķ +åı ļ +éĻ § +éĻ ŀ +å¨ Ģ +å§ ŀ +å§ ± +å§ ¤ +å§ ¶ +å§ ½ +æŀ ² +ç» ĸ +éª ĥ +ð¬ĺ ¡ +𬳠½ +ð¬ĺ © +ð«Ħ § +å½ ĸ +éª ī +æģ Ŀ +çı ª +çı Ľ +çı ¹ +çIJ Ĭ +çİ ¼ +çı ĸ +ðª Ł +ðªŁ Ŀ +çı ½ +çı ¦ +çı « +çı Ĵ +ð¬į ¤ +çı ¢ +çı ķ +çı Ŀ +ð«Ń ¼ +åŁ Ĺ +åŀ ¾ +åŀ º +åŁ Ĩ +åŀ ¿ +åŁ Į +åŁ ĩ +èİ ° +èĮ Ŀ +ð¬ľ ¯ +éĦ Ģ +èİ ¶ +èİ Ŀ +äĵ ĸ +èİ Ļ +æł » +æ¡ ł +ð¬ Ĥ +ð¬Ĥ © +æ¡ Ħ +æ¢ ł +æł ´ +æ¢ ´ +æł Ĵ +éħ İ +éħ ı +ð«ł Ĩ +çł µ +çł ł +çł « +çł ¬ +ç¡ ģ +æģ § +ç¿ ĥ +éĥ ª +ð¨ IJ +ð¨IJ Ī +è¾ Ģ +è¾ ģ +ð¬ Į +ð¬Į Ĺ +åī ķ +èµ Ģ +åĵ ¢ +æĻ ħ +æĻ Ĭ +åĶ Ŀ +åĵ ³ +åĵ ± +åĨ Ķ +æĻ Ķ +æĻ IJ +çķ ĸ +èļ Ħ +èļ Ĩ +ð« ij +ð«ij ¡ +å¸ ± +å´ ģ +å³ ¿ +𪨠¶ +å´ Ħ +å¸ ¨ +å ´Ģ +èµ Ĩ +𬠬¸ +éĴ · +𬬠» +𬬠¹ +𬬠¿ +ð¬Ń ģ +çľ ļ +çĶ ¡ +ç¬ « +åĢ » +åĢ ´ +èĦ © +åĢ ® +åĢ ķ +åĢ ŀ +ð« ¢ +ð«¢ ¸ +åĢ ĵ +åĢ § +è¡ ĥ +èĻ Ĵ +èĪ Ń +èĪ ¯ +èĪ ¥ +çĵ ŀ +é¬ ¯ +é¸ ° +èĦ İ +æľ ĵ +èĥ ² +èĻ ĵ +é± ½ +çĭ ´ +å³ ± +çĭ » +çľ ¢ +ð«Ĺ § +åĭ į +çĹ Ħ +çĸ ° +çĹ ĥ +ç« ĺ +ç¾ ĸ +ç¾ ĵ +æ¡ Ĭ +æķ ī +çĥ ł +çĥ Ķ +çĥ ¶ +çĥ » +ð¬Ĭ Ī +æ¶ į +æµ ¡ +æµ Ń +æµ ¬ +æ¶ Ħ +æ¶ ¢ +æ¶ IJ +æµ ° +æµ Ł +æµ Ľ +æµ ¼ +æµ ² +æ¶ ĺ +æĤ Ī +æĤ ĥ +æĤ ¢ +ð¬Ĵ Ī +å® § +çª ħ +çª Ĭ +çª İ +æī ħ +æī Ĩ +è¢ ª +è¢ Ĺ +è¢ ¯ +ç¥ § +éļ º +åł ² +çĸ į +𨠺 +𨺠Ļ +éĻ ´ +ç ĥĿ +çł ® +ãĽ ļ +åĵ ¿ +ç¿ Ģ +ç¿ Ĥ +åī Ł +𬳠¿ +ð«Ħ ¨ +ç» ¤ +éª į +ð¬ĺ « +ä Ĥ +äĤ ® +çIJ İ +çı ¸ +çı µ +çIJ Ħ +çIJ Ī +çIJ Ģ +çı º +æİ Ń +åł İ +åł IJ +åŁ ¼ +æİ İ +åŁ « +åł Į +æĻ ¢ +ð« ® +ð«® ĥ +æİ ŀ +åŁ ª +å£ ¸ +ãĻ į +èģ į +èı Ŀ +èIJ ļ +èı ¥ +èİ ¿ +äĵ « +åĭ ļ +äĵ ¬ +èIJ Ĩ +èı Ĥ +èı į +èı ¼ +èIJ £ +äĵ ¨ +èı ī +äĵ Ľ +æ¢ ¼ +æ¢ ½ +æ¡ ² +æ¢ ¾ +æ¡ ¯ +æ¢ £ +æ¢ Į +æ¡ ¹ +æķ Ķ +åİ £ +ç¡ Ķ +é¿ İ +ç¡ Ļ +ç¡ ļ +ç¡ Ĭ +ç¡ į +åĭ Ķ +ä´ ķ +é¾ ģ +éĢ ´ +åĶ ª +åķ « +ç¿ Ī +ã « +ã« ° +æĻ Ļ +çķ ¤ +𬱠ĸ +è¶ ¼ +è· Ĥ +èĽ ĥ +èļ ² +ð¬Ł ½ +èļ º +åķ ´ +äİ ĥ +å´ § +å´ Ł +å´ ŀ +å´ Ĵ +å´ Į +å´ ¡ +éĵ ı +ð«ĵ ¯ +ð«Ł ¹ +éĵ ķ +ð«Ł ¼ +éĵ ĸ +éĵ ĺ +éĵ ļ +éĵ ŀ +éĵ ¥ +éĵ ´ +çī » +çī ¿ +ç¨ Ĩ +ç¬ ± +ç¬ ¯ +åģ ° +åģ ¡ +é¸ º +åģ Ń +åģ ² +åģ ģ +ã ¿ +ã¿ ł +éĦ ħ +åģ ĵ +å¾ Ľ +è¡ Ĵ +èĪ ³ +èĪ ² +é¸ ¼ +æĤ Ĩ +éĦ ĥ +çĵ » +ä Ŀ +äĿ Ļ +èĦ ¶ +èĦ ŀ +èĦ Ł +äı ² +é± ¾ +çĮ ĩ +çĮ Ĭ +çĮ Ħ +è§ ĸ +ðł ħ +ðłħ ¤ +åº ± +åº ¼ +åº ³ +çĹ ĵ +ä´ Ķ +ç« « +åł ĥ +éĺ Į +ç¾ Ŀ +ç¾ ķ +çĦ Ĩ +çĥ º +çĦ Į +æ· ı +ð¬ĩ ¹ +æ· Ł +æ· ľ +æ· ´ +æ· ¯ +æ¹ ´ +æ¶ ´ +ð¬į ¡ +ã ¥ +㥠Ħ +æĥ Ľ +æĥ Ķ +æĤ ° +æĥ Ļ +å¯ ģ +éĢ Ń +𬤠ĩ +ð«į ¯ +è¢ ¼ +è£ Ī +ç¥ ² +𬤠Ĭ +ð«į ² +è° ŀ +èī ´ +å¼ ¸ +å¼ ¶ +ð¬¯ İ +éļ ĥ +å© ŀ +å¨ µ +å© ¼ +åª ĸ +å© ³ +å© į +å© Į +å© « +å© ¤ +å© ĺ +å© ł +ð¬ĺ ¬ +ð¬ĺ Ń +𬴠Ĥ +ð«ĺ ¦ +ç» ¹ +ð«Ł ħ +ð¬ĺ ¯ +éª ķ +ð«ĺ § +çµ ľ +çı · +çIJ ² +çIJ ¡ +çIJ Ł +çIJ Ķ +çIJ Ń +åł ¾ +åł ¼ +æı ķ +ãĻ ĺ +åł § +åĸ Ĩ +åł ¨ +å¡ ħ +åł ł +çµ · +𪠣 +𪣠» +ð¡ İ +ð¡İ ļ +è ijľ +æĥ İ +èIJ ³ +èij Ļ +éĿ ¬ +èij ´ +èĴ ĩ +èĴ Ī +éĦ ļ +èĴ ī +èĵ ĩ +èIJ © +èij ° +èij İ +éĦ ij +èĴ İ +èij ĸ +èĴ Ħ +èIJ ¹ +æ£ ¤ +æ£ ½ +æ£ « +æ¤ ĵ +æ¤ ij +ð¬ ĥ +ð¬ĥ Ĭ +é¹ Ģ +æ¤ Ĩ +æ£ ĵ +æ£ ¬ +æ£ ª +æ¤ Ģ +æ¥ Ĺ +𬠷 +𬷠ķ +çĶ ¦ +éħ ¦ +è§ Į +å¥ ¡ +çļ ķ +ç¡ ª +æ¬ ¹ +è© Ł +ð«IJ IJ +è¾ Į +æ£ IJ +é¾ Ĥ +𬠹 +𬹠¼ +é» ¹ +çī ļ +çĿ İ +æĻ « +æĻ ª +æĻ ± +ð § +𧠿 +𧿠¹ +èĽ ij +çķ ¯ +æĸ Ŀ +åĸ ¤ +å´ ¶ +åµ ģ +ð« ¶ +𫶠ĩ +å´ ¾ +åµ ħ +å´ ¿ +åµ ļ +ç¿ Ļ +ð«ĸ ® +åľ Į +åľ IJ +èµ ij +èµ Ĵ +é¿ ı +éĵ ¹ +ð¬Ń Ĭ +éĵ ½ +𨱠ĩ +ð«ĵ ¶ +éĶ Ĭ +éĶ į +éĶ İ +ð¬Ń İ +éĶ ĵ +çĬ ĩ +é¢ ĭ +ç¨ Į +çŃ Ģ +çŃ ĺ +çŃ ľ +çŃ ¥ +çŃ ħ +åĤ ĥ +åĤ ī +ç¿ Ľ +åĤ Ĵ +åĤ ķ +èĪ ¾ +çķ ¬ +ð«ĸ ¯ +èĦ ¿ +èħ ĺ +ä IJ +äIJ ĥ +èħ Ļ +èħ Ĵ +𬱠Ł +é² ĥ +çĮ ° +ð« Ľ +ð«Ľ Ń +çĮ ¯ +ã º +㺠Ħ +é¦ ī +åĩ ĵ +éĦ Ĺ +ð« · +ð«· · +å» ĭ +å» Ĩ +éĦ Į +ç² ¢ +éģ Ĩ +æĹ IJ +𬮠± +çĦ ŀ +ð¬Ĭ ¤ +æ¬ » +𣠸 +𣸠£ +æº ļ +æº ģ +æ¹ Ŀ +æ¸ ° +æ¹ ĵ +ã ´ +ã´ Ķ +æ¸ Ł +æº ł +æ¸ ¼ +æº ĩ +æ¹ £ +æ¹ ij +æº ŀ +æĦ IJ +æĦ ĥ +æķ © +çĶ ¯ +æ£ ¨ +æī Ĭ +è£ £ +ç¥ ¼ +å© » +åª Ĩ +åª ŀ +ãĽ ¹ +åª ĵ +åª Ĥ +åª Ħ +æ¯ µ +çŁ ŀ +𬴠ĥ +ð«ĺ ¨ +ç¼ Ĭ +ç¼ IJ +éª Ļ +çij ĥ +çij ĵ +çij ħ +çij Ĩ +ä´ ĸ +çij ĸ +çij Ŀ +çij Ķ +çij Ģ +𤠧 +𤧠Ľ +çij ³ +çij Ĥ +å¶ ħ +çij ij +éģ ĺ +é« ¢ +å¡ ¥ +åł ½ +èµ ª +æij Ľ +å¡ Ŀ +æIJ Ĵ +æIJ Į +èĴ ± +èĴ ¨ +èĵ ı +èĶ Ģ +èĵ ¢ +èĵ Ĥ +èĴ » +èĵ £ +æ¤ ¹ +æ¥ ª +æ¦ ĥ +æ¦ ħ +æ¥ Ĵ +æ¥ © +æ¦ ĩ +æ¤ ¸ +æ¥ Ļ +æŃ ħ +𬠪 +𬪠© +ç¢ ĥ +ç¢ ı +ð¬Ĵ Ķ +ç¢ Ī +äĥ ħ +ç¡ ¿ +éĦ ł +è¾ Ĵ +ð¬¨ İ +ð«IJ ĵ +é¾ Ĩ +è§ ľ +ä £ +ä£ ĺ +æļ ķ +é¹ į +ð« « +ð«« ĩ +㬠Ĭ +æļ ħ +è· ± +èľ IJ +èľ İ +åµ ² +èµ Ĺ +éª ± +éĶ ĸ +ð«ĵ ¹ +éĶ ĺ +éĶ ³ +éĶ § +éĶ ª +ð¬Ń ļ +éĶ « +éĶ ¬ +ð¬Ń Ľ +ç¨ ij +ç¨ Ļ +ä ħ +äħ Ł +ð¬ ķ +ð¬ķ Ĥ +çŃ » +çŃ ¼ +çŃ ¶ +çŃ ¦ +çŃ ¤ +åĤ º +é¹ İ +åĥ ĩ +èī ħ +èī ī +è° ¼ +è² Ĩ +èħ ½ +èħ ¨ +èħ ¯ +é² ī +é² Ĭ +é² Į +ä² Ł +𬶠ĭ +𬶠į +é² ı +éĽ Ĭ +çĮ º +é£ Ķ +è§ Ł +ð¦ Ŀ¼ +é¦ Į +è£ Ľ +å» Ĵ +çĺ ħ +éĦ ĺ +é¹ Ĵ +éĦ ľ +éº Ģ +éĦ £ +éĺ ĺ +ð«Ķ ¶ +çħ ģ +çħ ĥ +çħ ´ +çħ ĭ +çħ Ł +çħ ĵ +æ» ł +æº į +æº ¹ +æ» Ĩ +æ» ī +æº ¦ +æº µ +æ¼ · +æ» § +æ» ĺ +æ» į +æĦ Ń +æħ ¥ +æħ Ĩ +å¡ ± +ð« ĮĢ +è £¼ +ç¦ ĭ +ç¦ Ķ +ç¦ ĺ +ç¦ Ĵ +è° « +é¹ Ķ +ð«ĸ ³ +æĦ į +å« Ħ +åª ± +æĪ ¤ +åĭ ł +æĪ £ +ð«ĺ ª +ð«ĺ ¬ +ç¼ ŀ +èĢ ¤ +çij § +ð« ŀ +ð«ŀ © +çij ¨ +çij ± +çij · +çij ¢ +æĸ ł +æij ı +å¢ ķ +å¢ Ī +å¢ IJ +å¢ ĺ +æij ´ +éĬ İ +ð¡ IJ +ð¡IJ ĵ +å¢ ļ +æĴ ĸ +𪠤 +ðª¤ Ĺ +éĿ ½ +éŀ ģ +èĶ Į +èĶ Ī +èĵ ° +èĶ ¹ +èĶ Ĭ +åĺ ı +æ¦ ° +æ¦ ij +æ§ ļ +ð£ Ĺ +ð£Ĺ ĭ +æ§ ľ +æ¦ į +çĸ IJ +𬸠ĺ +éħ º +éħ ¾ +éħ ² +éħ ´ +ç¢ ¶ +äĥ İ +ð¬Ĵ Ĺ +ç¢ ¨ +ð¥ Ķ +ð¥Ķ ² +ç¢ ¹ +ç¢ ¥ +åĬ Ĥ +ð«ļ ĸ +ä´ Ĺ +å¤ ¥ +çŀ į +é¹ ĸ +ã¬ İ +è· ½ +èľ ¾ +å¹ ĸ +å¶ į +åľ Ļ +𨱠ı +éĶ º +éĶ ¼ +éĶ ½ +ð¬Ń ¤ +éĶ ¾ +éĶ ¿ +éķ ĥ +éķ Ħ +éķ ħ +é¦ Ŀ +é¹ Ļ +ç® ¨ +ç® ĸ +åĬ Ħ +åĥ ¬ +åĥ ¦ +åĥ Ķ +åĥ İ +æ§ ĥ +ãĻ ¦ +é² Ĵ +é² ķ +ð«ļ ķ +é² ĸ +é² Ĺ +é² ĺ +é² Ļ +𬶠IJ +𬶠ı +ð ©½ +𩽠¾ +å¤ IJ +çį į +é£ Ĺ +𬸠ļ +åĩ ĺ +å» ij +å» Ļ +çĺ Ĺ +çĺ ¥ +çĺ ķ +é² Ŀ +éĦ « +çĨ ĩ +æ¼ ¹ +æ¼ ĸ +æ½ Ĩ +æ¼ ¤ +æ½ © +æ¼ ¼ +æ¼ ´ +ã ½ +ã½ ı +æ¼ Ī +æ¼ ĭ +æ¼ » +æħ ¬ +çª ¬ +çª Ń +ã ® +ã® ¾ +𬤠Ŀ +è¤ ķ +ç¦ Ľ +ç¦ ļ +éļ © +å« ķ +å« Ń +å« ľ +å« ª +ð¬ ĻĤ +ã » +ã» ¬ +éº ¹ +çĴ Ĩ +æ¼ ¦ +åı ĩ +å¢ £ +å¢ ¦ +å¢ ¡ +åĬ IJ +èĸ ģ +èķ ° +èĶ ĥ +é¼ Ĵ +æ§ ± +é¹ Ŀ +ç£ ı +ç£ ī +æ® £ +æħ Ń +éľ ħ +æļ µ +æļ ² +æļ ¶ +è¸ ¦ +è¸ £ +äĹ ĸ +èĿ ĺ +èĿ ² +èĿ ¤ +åĻ ĩ +å ĻĤ +åĻ Ģ +ç½ ¶ +å¶ ² +å¶ ĵ +ãł ĩ +å¶ Ł +å¶ Ĵ +éķ Ĩ +éķ Ī +éķ ĭ +éķ İ +ð¬Ń © +éķ ķ +ç¨ ¹ +åĦ ĩ +çļ ŀ +çļ Ľ +ä´ ĺ +èī İ +èī ı +é¹ Ł +𩾠ĥ +é² ¦ +é² ª +é² ¬ +æ© ¥ +è§ Ń +é¹ ł +é¹ ¡ +ç³ ĩ +ç³ Ī +ç¿ ¦ +é¹ ¢ +é¹ £ +çĨ Ľ +æ½ ĸ +æ½ µ +ã µ +ãµ IJ +æ¾ Ĥ +æ¾ Ľ +çij ¬ +æ½ ½ +æ½ ¾ +æ½ ı +æĨ Ń +æĨ ķ +𬸠£ +æĪ Ń +è¤ ¯ +ç¦ ¤ +ð«į ½ +å« ½ +éģ ¹ +𬴠Ĭ +çĴ ¥ +çĴ ² +çĴ Ĵ +æĨ Ļ +æĵ IJ +éĦ ¹ +èĸ ³ +éŀ Ķ +é» ĩ +ð¬ ŀ +ð¬ŀ Ł +èķ Ĺ +èĸ ¢ +èķ ¹ +æ© ŀ +æ© ij +æ© ¦ +éĨ ij +è§ ± +ç£ ¡ +ð¥ ķ +ð¥ķ ¢ +ç£ ľ +è± ® +ð«Ł ¦ +𬺠Ī +ð«ł ľ +é¹ ¾ +èĻ ¤ +æļ ¿ +æĽ Į +æĽ Ī +㬠ļ +è¹ ħ +è¸ ¶ +äĹ Ľ +èŀ Ĺ +çĸ ģ +ãł ĵ +å¹ ª +𪠩 +𪩠ĺ +å¶ ¦ +ð¬Ń ¬ +𨱠ij +ð¬Ń ¯ +é¦ ŀ +ç© Ħ +ç¯ ļ +ç¯ ¯ +ç° ī +é¼ ½ +è¡ ł +çĽ ¦ +èŀ £ +ç¸ ¢ +é² Ń +é² ¯ +é² ° +é² º +é² ¹ +ð«Ĺ ´ +äº ¸ +çĻ Ģ +çĺ Ń +𬸠¦ +ç¾ ± +ç³ Ĵ +çĩ ĭ +çĨ » +çĩ Ĭ +çĩ ļ +çĩ ı +æ¿ © +æ¿ ĭ +æ¾ ª +æ¾ ½ +æ¾ ´ +æ¾ Ń +æ¾ ¼ +æĨ · +æĨ º +æĩ Ķ +é» ī +å¬ Ľ +é¹ ¨ +ç¿ ¯ +ð«Ħ · +çĴ ± +𤠩½ +çĴ ¬ +çĴ ® +é« ½ +æĵ ¿ +èĸ ¿ +èĸ ¸ +æª ij +æ« Ĩ +æª ŀ +éĨ ¨ +ç ¹Ħ +ç£ ¹ +ç£ » +çŀ « +çŀ µ +è¹ IJ +èŁ ı +ã ĺ +ãĺ İ +ð¬Ń ³ +éķ ¤ +ð¬Ń ¶ +ð«Ķ į +éķ ¥ +éķ ¨ +ð¬Ń ¸ +𨱠Ķ +ð¬Ń ¼ +ð«Ķ İ +çŁ ° +ç© Ļ +ç© ľ +ç© Ł +ç° ķ +ç° ĥ +ç° ı +åĦ ¦ +éŃ ĭ +æĸ ¶ +èī ļ +𬸠ª +è° ¿ +ä² ł +𬶠Ł +é² ¾ +𬶠ł +é² ¿ +é³ ģ +é³ Ĥ +é³ Ī +é³ ī +çį ¯ +äĹ ª +é¦ ĺ +è¥ ķ +è¥ ļ +𬶠¨ +èŀ ± +çĶ ĵ +å¬ ¬ +å¬ ¥ +ð¦ Ī +ð¦Ī ¡ +ð«Ħ ¸ +çĵ Ģ +éĩ IJ +é¬ ¶ +çĪ ĩ +éŀ ³ +éŀ ® +ð¬Ł ģ +èĹ Ł +èĹ ¦ +èĹ ¨ +é¹ ² +æª « +é» ¡ +ç¤ ŀ +ç¤ Į +ð¥ ĸ +ð¥ĸ ¨ +è¹ ¢ +è¹ ľ +èŁ « +äĹ ´ +åļ ļ +é« ĥ +éķ ® +éķ ± +éħ Ĥ +é¦ § +ç° ł +ç° Ŀ +ç° ° +é¼ « +é¼ © +çļ ¦ +èĩ ij +ä² ¢ +é³ ij +é³ Ĵ +é¹ ± +é¹ ¯ +çĻ Ĺ +ð¦ Ĵ +ð¦Ĵ į +æĹ ŀ +ç¿ · +åĨ ģ +äİ ĸ +çĢ Ķ +çĢ į +çĢ Į +è¥ ľ +ä´ Ļ +ð¬Ļ Ĭ +åļ Ń +ã ° +ã° Ģ +é¬ · +éĨ Ń +è¹ ¯ +èł ĭ +ç¿ ¾ +é³ ĺ +åĦ ³ +åĦ ´ +é¼ Ĺ +ð¬¶ Ń +𩾠Į +é³ ļ +é³ Ľ +éº ij +éº ĸ +èł ĥ +å½ Ł +å¬ ¿ +é¬ Ĵ +èĺ ĺ +æ¬ Ĥ +é Ĩµ +é¢ ¥ +çĶ Ĺ +ð¨ Ł +ð¨Ł ł +å· ĩ +éħ ħ +é« İ +çĬ ¨ +𬶠® +ð¨ Ń +ð¨Ń ī +㸠Į +çĪ Ķ +çĢ ± +çĢ ¹ +çĢ ¼ +çĢ µ +è¥ « +åŃ ħ +éª ¦ +ð¬Ļ ĭ +èĢ ° +𤠫 +𤫠ī +çĵ ĸ +é¬ ĺ +è¶ ¯ +𬺠ĵ +ç½ į +é¼ ± +é³ ł +é³ ¡ +é³ £ +çĪ Ł +çĪ ļ +çģ Ī +éŁ Ĥ +ç³ µ +èĺ ¼ +ç¤ µ +é¹ ´ +èº Ķ +çļ Ń +é¾ ¢ +é³ ¤ +äº ¹ +ç± ¥ +é¼ · +ð«ļ Ń +çİ ĥ +éĨ ¾ +é½ ĩ +è§ ¿ +èł ¼ +× § +× ¤ +× Ľ +×ķ× ª +× ¡ +×Ļ× Ŀ +× ¦ +× Ĵ +× ĺ +×ķ× ¨ +× Ŀ +×ķ× ľ +× ĸ +๠Ĥ +ï º +ðŁ į +ðŁ IJ +×Ļ× ¨ +ï » +ðŁ ij +ðĿ IJ +ðŁ ı +ðŁ Ķ +ðŁ Į +ðŁ İ +ðŁ ĵ +× Ł +ðĿ ij +×ķ× ĵ +ï ¦ +Ġ× ķ +×ķ× ij +à¸Ń à¸ĩ +ðĿ ĺ +×Ļ× ª +ðĿ ķ +à¸Ĺ ีà¹Ī +Ø§Ø ¦ +ðŁ ¤ +×ķ× Ł +ر ÙĬ +×Ļ× ľ +ร ะ +า ย +ï ¯ +ï ® +า ม +â ĩ +ðŁ ¥ +ï Ń +ðĿ Ļ +×ķ× ł +á ½ +Ġ× Ľ +ðŁ ļ +â ļ +ï § +×ij ר +×Ļ× ł +á ´ +Ġ× Ĺ +á ¼ +ðĿ Ĺ +Ġ× ¢ +×Ļ× Ķ +ãģ£ ãģŁ +ãģĵ ãģ¨ +á ¸ +ÙĬ ÙĨ +ãģª ãģĦ +ا ع +ภ¨ +à¹Ī à¸ĩ +×Ļ× ĵ +×ŀ ש +á Ī +׳ ×Ļ +×Ļ× ij +ï ¥ +ðĿ ĵ +Ġ× Ļ +× ļ +ั à¸ĩ +â ĵ +ï ¤ +ĠاÙĦ Ø£ +า à¸ģ +à¹ī à¸Ļ +à¹Ģ ร +×ķ× Ŀ +á ¹ +ภ¶ +×Ļ× § +ภĭ +à¸Ħ ร +ภĺ +ั à¸ģ +ðŁ ķ +ÙĪ ÙĨ +à¸Ń ย +â Ĭ +ðĿ Ĵ +ĠاÙĦ ع +า à¸Ļ +×Ļ× Ł +ÙĦ ÙĬ +×Ļ× © +à¸Ľ ระ +à¹Ģ à¸Ľ +Ġ× ł +×ķ× ¡ +ภł +Ùħ ÙĨ +×ķ× ¢ +×ķ× ŀ +â Į +ðŁ § +à¹ĩ à¸Ļ +ภį +ã İ +á µ +ĠاÙĦ س +×ķ× § +ห ล +ðŁ ĩ +â ı +ðŁ ¦ +Ġ×Ķ ×ŀ +ÙĪ ا +Ġ× ª +ר ×IJ +à¸Ń à¸Ļ +ภ© +à¹Ī ว +×ķ× ¦ +í Ĺ +ã Ħ +ï ¨ +ï ¹ +â İ +ï ² +ðĿ ļ +ð IJ +à¸Ħ ว +ห à¸Ļ +Ġ× ¨ +ب ÙĬ +ร à¹Į +ر ا +Ø´ ر +×ķ× Ĺ +×ķ× ¤ +×ķ× © +×ķ× Ĵ +í Ŀ +â Ľ +à¸ķ ิ +à¹Ģ à¸ģ +ï ³ +ï ± +à¸Ķ à¹ī +ë ¹ +ï ¬ +á ¿ +ðŁ Ľ +ðĿ ĸ +à¹Īา à¸ĩ +ู à¹ī +Ġ×Ķ ×IJ +ĠاÙĦ ØŃ +פ ר +ÙĪ Ùħ +à¹Ģ ล +í ĸ +×Ļ× ¢ +ì Ī +í ĵ +ðŁ ħ +á ł +à¸Ħว าม +à¸Ī ะ +׳ ×Ķ +Ġ× § +ภŁ +à¹ī à¸ĩ +ห ม +ت Ùħ +׾ ×Ļ +ÙĬ د +à¹Ī à¸Ļ +׊ר +ש ר +à¹Ģ à¸Ĺ +×ŀ ר +ë ĸ +ع ÙĦ +×ŀ ×¢ +â ² +׾ ×Ķ +Ġ× ¤ +à¸Ń à¸ģ +س ÙĦ +×Ļ× ŀ +ÙĤ ÙĬ +í İ +ت ØŃ +×Ļ× ¡ +×Ļ× Ĺ +í Ľ +ï ° +â ½ +á ī +á Ĭ +á ¨ +Ùĩ ا +Ġ׾ ×Ķ +×ķ× IJ +Ùħ ا +à¹īà¸Ń à¸ĩ +ر ب +ĠاÙĦ ج +×ŀ ×ĵ +Ùħ ÙĦ +ت ر +à¹Ģ à¸Ķ +ק ר +í ħ +ì ¼ +ê ¿ +ã Ī +á IJ +ðŁ Ĺ +ê ¦ +á ĭ +ðĿ Ķ +à¹Ģà¸Ľ à¹ĩà¸Ļ +à¹ĥ ห +ม า +ว à¹Īา +ม ี +ี à¹ī +à¹Ħม à¹Ī +ÙĨ ÙĬ +Ø ¤ +ร า +×ķ ×Ļ +ãĤĪ ãģĨ +ิ à¸Ķ +×Ļ× ¤ +׊׾ +ÙĤ د +à¹Ģ ส +×Ļ× ĺ +à¸ģ ล +ר ׼ +×ķ× Ľ +×Ļ× Ľ +ë Ī +ë ĥ +ðŁ ĸ +á ħ +â ¼ +ã ī +à¹Ħ à¸Ķà¹ī +ת ×Ļ +×Ļ× IJ +ĠاÙĦ Ø¥ +à¸ł า +ร ิ +ÙĤ Ø© +ØŃ Ø¯ +ê » +ì ± +ת ×Ĺ +ì º +â ĭ +á Ħ +á ¾ +â µ +â ¾ +ĠÙĪ اÙĦ +׳ ×ķ +Ù Ģ +ÙĬ ا +à¸ģ à¹ĩ +×ŀ ×Ķ +ãģĦ ãĤĭ +ع د +ĠاÙĦ ÙĨ +Ġ×Ķ ש +Ø ¦ +ั à¹īà¸ĩ +ร ัà¸ļ +ÙĪ ÙĤ +ãģ§ ãģį +à¹Ģ à¸ŀ +׼ ׾ +×ĺ ר +ั à¸Ķ +à¸Ń า +ì ¢ +à¸Ń à¸ļ +à¸ķ ร +à¹Ģ à¸Ĭ +ì Ķ +ãģĹ ãģ¾ +ë ģ +ë ķ +ðŁ Ļ +â Ĵ +á ¶ +à¹ģ ล +ÙĨ ا +à¹ĥห à¹ī +à¹Ħ à¸Ľ +× £ +ั ว +า à¸ĩ +×ĵ ר +×ij ׾ +פ ×Ļ +Ġ× ĵ +ĠاÙĦ Ùģ +à¹Ģ à¸Ĥ +ש ×Ķ +×IJ ר +ë ¬ +ãģ« ãģª +ÑĢ о +ว ิ +Ùħ ر +×IJ ת +Ùĥ ر +س ب +ÙĨ ت +ãģĹ ãģĦ +ا ج +à¸Ń รà¹Į +Ùĥ ÙĦ +س Ùħ +ส ิ +×Ļ× ¦ +ë Ŀ +í ľ +ì ī +á Ĩ +Ùĩ Ùħ +à¸Ļ ีà¹ī +ãģĤ ãĤĭ +ãģĦ ãģ¦ +س ÙĬ +׾ ×IJ +د ر +ãģ ļ +ÙĪ ج +ĠاÙĦ Ø® +ص ر +í ı +à¹īา à¸ĩ +ุ à¸Ķ +×ķ× ĺ +×ij ×¢ +í Ĩ +à¸Ĭ า +ร ม +ש ×ŀ +×ŀ ס +ê ´ +ì ´ +ë ľ +ì ¿ +ì © +ë » +â ¤ +ðŁ Ĩ +á Į +á ķ +Ø° ا +à¸Ĺ ำ +à¸ķ à¹Ī +ĠاÙĦ ÙĤ +ÙĦ Ùĥ +ู à¹Ī +à¸Ħ ุ +ÙĬ Ùħ +׳ ×Ļ×Ŀ +ืà¹Ī à¸Ń +ÙĪ ع +ãĤ ĩ +ا ÙĤ +Ġ×ij ×¢ +à¹Ģ ม +ج Ùħ +á» « +ãģĵãģ¨ ãģĮ +ب د +×ķ× Ķ +ש ׾ +Ùĩ ر +à¹Ģ à¸Ļ +ãģ ¹ +í ĭ +ì » +ì ½ +ë Ń +ì Į +í Ģ +ë Į +ë º +ã Ĭ +à¹ĥ à¸Ļ +Ġ× Ĵ +๠Ĩ +à¸Ī าà¸ģ +ว ย +à¹ĥ à¸Ĭ +à¸ĩ าà¸Ļ +ĠاÙĦ Ø´ +ا ØŃ +à¹īา à¸Ļ +ืà¹Ī à¸Ńà¸ĩ +×IJ ×Ļ +ب ÙĦ +ãģ¨ æĢĿ +׳ ס +ãģ¾ ãģĽ +Ùĥ ÙĨ +×¢ ר +ĠاÙĦ د +ש ת +í ŀ +Ùħ س +ص ÙĦ +×ķ׳ ×Ķ +ار Ø© +ÙĦ Ùħ +ส ม +Ø£ ÙĨ +ת ר +×IJ ×ŀ +ع ب +Ø® ت +ãĤ ĥ +ì ¡ +ì £ +ив а +ส ั +ึ à¸ģ +ì ¸ +ë Ĩ +алÑĮ н +ì ³ +ì į +ê ¼ +ê ½ +ì ı +ã Į +ã ı +ï © +ê ª +á İ +Ġ× ĸ +à¸ģ ัà¸Ļ +×Ļ ×ķ +à¸Ħ à¸Ļ +׳ ×ķת +à¸ľ ูà¹ī +à¹ĥ à¸Ī +ãģĦ ãģŁ +Ùģ ر +×ĺ ×Ļ +צ ×Ļ +ãĤĤ ãģ® +ĠاÙĦ ص +ãģ¾ãģĽ ãĤĵ +د Ø© +×ij ×Ļ +ĠاÙĦ ر +Ġ×ŀ ×IJ +ส ำ +à¹Ģ ห +ع ر +ãģª ãģı +à¸ģร ะ +×ij ×ĵ +à¹Ģ à¸Ī +×Ļ× ļ +×Ĺ ×Ļ +ÙĬ ع +ש ×ij +ÙĨ Ø© +ÙĪ ض +ÙĦ Ùģ +ÙĢ ÙĢ +פ ×¢ +í Ī +×ŀ ק +ภIJ +ØŃ Ø© +ا ص +Ñĭв а +à¸Ħ ม +ว ั +à¸Ľ ล +ì Ł +í ļ +ë ´ +ë ij +ë ī +ë ĩ +ì ¨ +ë ± +ë İ +â ¬ +á ¥ +á Ĺ +á Ľ +á į +Å © +à¸Ķ ี +ô i +Ġ× ¡ +׾ ×ķ +á»Ŀ i +à¸Ħุ à¸ĵ +â y +à¸Ļ า +×Ĺ ×ĵ +×ĵ ×Ļ +ห า +ج ÙĦ +à¹Ģ ว +ãĤĩ ãģĨ +Ùħ Ø© +ĠاÙĦ Ùĥ +Ġ×Ķ ×¢ +ج ر +×ĸ ר +ا Ø· +׼ ת +×ķ׳ ×Ļ×Ŀ +ØŃ Ùħ +ê ¶ +ر Ùĥ +Ġ׾ ×¢ +×ķ× ĸ +ส ร +צ ׾ +Ø ¢ +ا ست +à¹Ī ม +Ø® ر +צ ×¢ +×Ļר ×ķת +اد Ø© +Ø´ ار +×ŀ ×Ĺ +í Ĵ +à¹Ģร ีย +׊ק +Ø§Ø « +ร à¸ĩ +à¹Ģ à¸ķ +à¸Ī ำ +ภĿ +à¹Īา ย +à¸Ħ ล +ÙĤ ÙĪ +иÑĩеÑģ к +à¸ĵ à¹Į +ั ย +Ùħ ع +ë ¨ +ë ¿ +ë ® +ï ´ +ì ¥ +ì « +ë µ +á ¡ +â į +ð ĵ +â ° +à¸Ĥ à¸Ńà¸ĩ +Ù ĭ +à¸ģ ัà¸ļ +ãģ® ãģ§ +à¹ī ว +à¸Ńย à¹Īาà¸ĩ +ãģ Ń +á»ĩ t +à¸ķ à¹īà¸Ńà¸ĩ +×ŀ ×Ļ +à¹ģ à¸ļ +×Ĵ ר +ÙĪ Ùģ +ÙĤ ÙĦ +à¸łà¸² à¸ŀ +ר ×Ļ +ล า +ÙĬ س +Ġ× ¦ +ÙĬ Ùģ +Ġ× ĺ +à¸ľ ล +á ng +ร ว +Ġ×ŀ ש +×IJ ×ķת +×ĸ ×Ķ +ู à¸ģ +à¸Ļ ัà¸ģ +اÙĨ ÙĬ +د ا +ãģ ³ +׼ ף +ãĤī ãĤĮ +ãĤĮ ãģ° +ת ק +ú c +ÙĪ ز +×Ļר ×Ķ +Ġn gh +án h +Ġ×ķ ×IJ +á» ħ +ส ุà¸Ķ +ë į° +ا ض +اÙĦ ÙĬ +ب ار +ع Ùħ +à¸ļ า +ت ج +à¸ŀ ร +×ķר ×Ķ +ả ng +Ø® ÙĦ +ภī +ắ c +ש ×Ļ×Ŀ +í Ķ +Ùģ س +×Ļ× Ĵ +п ÑĢ +ĠاÙĦ Ø« +س Ø· +ร ูà¹ī +ีà¹Ī ย +à¸Ń à¸Ķ +ãģª ãĤĬ +×Ĵ ×ĵ +ãģĦ ãģ¾ãģĹãģŁ +ס ק +Ø® ص +la ÅŁ +ен но +ب ØŃ +ส à¸Ļ +ภ® +ר×IJ ש +Ùħ ÙĪ +دÙĬ د +ษ า +×ķ× ļ +ãĥ§ ãĥ³ +à¸ķ ุ +Ġê µ +ĠÑģв о +צ ×ij +à¸Ń ม +à¸Ľ ร +ت ع +×Ķ ת +اÙħ ÙĦ +×ŀ ׳ +ç ¶ļ +ภ¤ +í į +ë ĺ +ë ¤ +ì ij +â ´ +ã ĭ +Ġب اÙĦ +á»ģ u +ĠاÙĦ ÙĦ +à¸ķ ัว +Ø° Ùĩ +ึ à¸ĩ +à¹ĥà¸Ĭ à¹ī +á»ĵ ng +à¸Ļ ั +ม าà¸ģ +ãĥ Ł +×ŀ ×ķ +à¸Ĺ ย +á»Ļ i +Ạ± +ả o +à¹Ĥ à¸Ķ +×IJ ׾ +ส าม +ÙĪ ب +à¸Ĺ ุ +ย ัà¸ĩ +×¢ ת +×ķ׳ ×ķת +à¸Ĥ ึ +à¸Ĥึ à¹īà¸Ļ +à¸ģ à¹Ī +Ạ« +á»ij c +ãģĹ ãĤĩãģĨ +á»ĭ ch +Ġ×IJ ×ķת +Ġש ×IJ +׼ ×ķ׾ +á»Ļ c +ع Ø© +à¸Ĺ ี +à¹Ģ à¸Ń +Ùĥ ت +ãģ » +Ạ» +ìĹ ħ +à¸Ń à¸Ńà¸ģ +اÙĨ ت +à¹Ħ ร +Ġ×IJ ×Ĺר +Ø· ر +ÙĨ د +ื à¹īà¸Ń +Ø· ÙĦ +×IJ ×Ķ +uy ên +í ĸī +×ij ×Ķ +à¸Ħ à¹Ī +à¸Ĭ à¹Īว +ãģĤãĤĬ ãģ¾ãģĻ +ÙĬ ب +ק ׾ +ãĥ Ļ +Ä © +س ر +า ว +ãĤ ± +à¸ļ ริ +ר ×Ĵ +á»ĥ u +ØŃ Øª +×ķ×ŀ ×Ļ +ب ÙĨ +êµ IJ +ÄŁ u +ãģª ãĤĵ +×ij ק +Ġפ ר +ắ n +ØŃ ÙĦ +×ij ×Ĺ +ấ u +×ij ×ķ×ĵ +ãĥ ¯ +Ġ׾ ק +ั à¸į +à¸ŀ ิ +×Ĺ ×Ķ +×ĸ ׼ +ãĥ¼ãĥ ł +ÑĤ елÑĮ +×ŀ ×Ļ×ĵ +ÙĬ Ø® +Ạ³ +ت ص +à¸ĺ ิ +è¾ ¼ +ì ĵ +Ùĥ Ø© +ÙĤ ب +à¸Ħ à¹Į +à¹īา ย +à¸ĵ ะ +า ะ +ë Ĵ +ê ¾ +ë · +ì ĩ +ê º +ì ģ +ë Ģ +ì ¾ +ë ½ +ë ļ +ì Ń +ì İ +á ij +ë Ĺ +ê Ĵ +à ¡ +à ¬ +ðIJ Į +ã ĩ +ðĿ Ħ +Ġ׾ ×IJ +ãģ¨ ãģĦãģĨ +Ġn hi +×Ļ ×ķת +Ġש ×Ķ +à¹ģล à¹īว +Æ°á»Ľ c +à¸Ķà¹ī วย +à¸Ĺ าà¸ĩ +׳ ת +פ ת +à¹ģ à¸ķà¹Ī +Æ° ng +à¸Ńย ูà¹Ī +à¹ī ำ +Ġ×IJ ׾ +Ùĥ Ùħ +ấ p +ล à¸ĩ +ãģŁ ãĤģ +×Ĵ ׾ +ห ร +ĠÑĢ е +à¹Ģà¸Ĥ à¹īา +ÙĤ ر +Ġ×Ķ ס +ÙĪ ÙĬ +สาม าร +สามาร à¸ĸ +Äĥ n +à¸Ń ี +פ ×ķ +×Ļ׳ ×ķ +ว ัà¸Ļ +ặ c +íķ Ļ +×ŀ ת +ê u +Ạ¹ +Ùģ ÙĬ +×ŀ צ +à¸Ħ า +ãģĿ ãģĨ +ãĢ ħ +ا ز +ا Ùĩ +ר ×Ļ×Ŀ +ấ n +ห าร +ạ t +ÙĨ Ùĩ +à¹Ģ à¸Ħร +ج Ùĩ +׼ ×Ļ +ắ t +à¸Ħ à¹īา +ر Ø© +ãĥ ı +Ùĥ ÙĪÙĨ +ứ ng +Ġìļ ° +ย à¹Į +à¹Īว à¸Ļ +à¸ģ ำ +Ø« ر +Ñģ и +ĠاÙĦ Ø· +Ġ×Ķ צ +ĠØ · +ĠاÙĦ ÙĪ +ê¹ Į +ØŃ ÙĬ +ار ات +à¹Ģ à¸ĭ +ب ا +г ÑĢ +ร ี +ืà¸Ń à¸Ļ +ع ت +ÙĤ اÙĦ +د Ùħ +Ø ¡ +Ġ×ŀ ק +×ĵ ×Ļ×Ŀ +×¢ ׾ +ãģ Ĵ +ëĭ ĺ +×¢ ×Ķ +Ġìĸ ´ +Ñģ ÑĮ +ÙĤ Ø· +ãĥ Ľ +èĢĥ ãģĪ +à¹ģ à¸Ļ +ÙĪ ات +â u +ĠìĤ¬ ëŀ +ห ว +ĠاÙĦØ£ Ùħ +Ġ×Ķ ×ŀש +ب ÙĪ +à¸Ĭ à¸Ļ +ãĤĵ ãģ§ãģĻ +ว à¸Ļ +à¸ģร รม +×ŀ ×ķ×ĵ +Ùĥ اÙĨ +×ķ× £ +ол ог +ت ÙĨ +à¸ķ à¹Į +ê² ĥ +ר ×ĺ +ừ ng +×ķ×ij ×Ķ +Ùħ ØŃ +ĠÐ § +פ ×Ĵ +ส à¸ĸ +ãģĭ ãĤĬ +ını z +à¹Ģ ย +ãĥ¼ ãĥ³ +ãģĬ ãĤĬ +פ ש +ิ à¸ķ +Ø· ÙĨ +×Ļת ×Ļ +×IJ ׳ +ç ek +ì ª +×ŀ ×ij +ศ า +ãĤ¹ ãĤ¿ +à¸ļ ุ +×ĵ ×ijר +ãģĦ ãģı +ส ะ +à¹Ģ หล +ิ à¸ĩ +à¸ŀ ัà¸Ļ +ãģĦ ãģŁãģł +ãĤĤ ãĤī +à¹ī ม +ãģĵãģ¨ãģĮ ãģ§ãģį +าร à¹Į +ุ à¸ĩ +í ij +ì ¯ +ë ¼ +í Ĥ +ì · +ê ¡ +á ı +á Ĵ +ðĿ ľ +á © +ðŁ Ħ +ðIJ ¤ +Ġש ׾ +Ġ×ŀ ×Ķ +à¹ģล ะ +Ġ׼ ׾ +Ạ½ +á»Ļ ng +Ø° ÙĬ +л е +× ¥ +ãģª ãģ© +ĠÙĪ Ø£ +หà¸Ļ à¹īา +ãģ¾ ãģ§ +à¸ķà¹Ī à¸Ń +à¸Ĺ ัà¹īà¸ĩ +ãģł ãģij +à¹ģà¸ļ à¸ļ +à¹Ģร า +פ ׾ +ãģŁ ãģĦ +à¹Ģล ย +ãģ£ãģ¦ ãģĦãĤĭ +ế p +ึ à¹Īà¸ĩ +ê ´Ģ +ê³ Ħ +׼ ×ķ +à¹Ģร ืà¹Īà¸Ńà¸ĩ +ק ×Ļ +êµ Ń +פ ס +ت ÙĬ +ãĥ Ħ +Ġ×Ķ ×Ĺ +г и +ר×IJ ׾ +×ŀ ׾ +ĠØ£ ÙĬ +Ġع ÙĦÙĬ +ãģĭ ãģ£ãģŁ +ש ×Ļ +д Ñĥ +×ŀ ף +׳ ×ĺ +׳ ×Ļת +mi ÅŁ +׼ ×Ŀ +Ġ×ij ר +Ġ׾ ×ij +ĠÐ Ľ +ç e +×ķ׳ ×Ļ +ãĤĪãģĨ ãģ« +פ ×ķר +ãĥ į +Ùĥ ÙĬ +×Ĺ ×ª +Ùģ ÙĦ +Ġ×Ķ ק +Ġ×Ķ ×ij +Ġ×ŀ ס +à¹Īา à¸Ļ +п еÑĢ +à¹Īา ว +Ġ×ij ×IJ +ĠÙĪ Ùĩ +à¸Ļ ำ +Ġ×ij ש +׳ ק +ãģ© ãģĨ +ש ×ķת +×ĵ ×Ķ +à¹Ģ à¸ļ +ÙĨ س +Ġìļ° 리 +ส à¹Īวà¸Ļ +ล ัà¸ĩ +ج ز +Ġ×Ĺ ×Ļ +Ùĥ ثر +ล ะ +Ùĩ د +ĠÙĪ ب +اÙĦ Ùħ +à¹ģ ม +Æ¡ i +Ġ×ij ×Ĺ +ữ a +à¹Ģà¸Ĺ ศ +à¸ķ ัà¹īà¸ĩ +ог да +׾ ק +د د +สร à¹īาà¸ĩ +à¸Ĭ ี +Ùģ ض +à¹ģ ห +uy á»ĩn +ร ัà¸ģ +á»ĩ m +ส า +פ ק +ีย à¸ĩ +à¸ķ à¹Īาà¸ĩ +à¸Ħร ัà¹īà¸ĩ +ØŃ ÙĤ +à¹Ģ à¸Ńà¸ĩ +ائ ÙĬ +×ĺ ×¢ +اÙĦ Ø© +ิ à¹Īม +ãĤ ½ +د Ùī +Ġר ×IJ +ãģ£ ãģ¨ +ãĥĥ ãĥĹ +ÙĬر Ø© +ê± ´ +×ŀ ×IJ +×ķ ×ķ +ب ع +ãģ ² +ร าย +×ĵ ×Ŀ +ت Ùģ +à¸ķ à¸ģ +ạ ng +ãĤĴ è¦ĭ +à¸Ĭ ั +Æ°á» Ł +Æ°á»Ł ng +ج ب +×ķ×ŀ ר +ĠìĤ¬ëŀ Į +ó ng +ร ั +Ġ×Ķ ×ĸ +ר צ +Ġ×Ĺ ×ĵ +Ø° ÙĦÙĥ +×ķר ×Ļ +ãģ¡ ãĤĥ +Ùģ ع +Ġ׾ צ +á i +à¹ĩ à¸ļ +ãģ İ +à¸ģ ิ +ạ c +ë© ° +ãģª ãĤĭ +×ķ׾ ×Ŀ +à¹ģ à¸Ĺ +×ķ× ¥ +м еÑĤ +ü ÅŁ +ÑĢ Ñı +ภĴ +ÑģÑĤ оÑı +ع ÙĪد +Ùħ ار +Ø· Ø© +à¸ŀ ื +к ÑĢ +à¹ģ à¸ģ +à¹Ĥ รà¸ĩ +×ij ×Ļ×ĺ +ê² ł +×ķ׾ ×Ķ +ØŃ Ø± +ืà¹Ī à¸Ńà¸Ļ +×ķ×ij ר +׊ש +ãĥķãĤ ¡ +×ŀ ×ĺ +ú t +Ġd ön +ắ ng +ëł ĩ +ẳ ng +ว à¸ģ +ص د +Ø® Ø· +à¸Ń ั +ãĤı ãĤĮ +سÙĦ اÙħ +à¹Ģร à¹ĩ +×Ļש ×Ļ +ج اÙĦ +ãģij ãĤĭ +à¸Ĭา à¸ķิ +ÙĪا ÙĤ +à¹Ĥ à¸Ļ +ãģ¦ ãģĹãģ¾ +اع Ø© +ãĤŃ ãĥ£ +à¸į า +ÙĦا ÙĤ +ิ à¸ģ +ĠÑģ ов +ÑĢаРº +×Ļ׳ ×Ļ +ü ÄŁ +Ã¼ÄŁ ü +ק ×ij +à¹Ī à¸Ńà¸ĩ +Ġger çek +à¸Ĺ ั +ов аниÑı +×ŀ ׼ +س Ø© +×Ļ× £ +le ÅŁ +Ùħ ؤ +ĠìĿ ĺ +à¸IJ าà¸Ļ +ĠÑģ об +Ġêµ Ń +×¢ צ +з в +ส à¸ĩ +ز ÙĦ +ãģı ãĤĮ +и ÑĢÑĥ +ت Ø£ +п олн +ìĺ Ģ +ÙĨ Ø´ +׼ ×IJ +Ùħ Ø´ +à¸Ķ à¹Į +ÙĪ ÙĬÙĦ +à¹ģ à¸Ĥ +ãģ£ãģ¦ ãģĹãģ¾ +но ÑģÑĤ +в л +Ùħ ÙĤ +را ج +å¤ ī +ë Ľ +â ¸ +ì IJ +à » +á ļ +â » +ê Ļ +â § +ð Ĵ +ðĿ ĩ +Ġ×IJ ת +ĠÙĦ ÙĦ +ĠØ£ ÙĨ +Ġ×ķ ×Ķ +ãģ« ãģ¯ +Ġ×Ļ ש +ت Ùĩ +ÃŃ nh +ÙĬ ات +Ġ×ij ×ŀ +à¸Ļั à¹īà¸Ļ +à¸Ļ à¹īำ +Ãł o +à¸ķ าม +ãģ® ãģ¯ +d ır +Ġn ghi +ặ t +×ŀ ×Ļ×Ŀ +ãģ¦ ãģĦãĤĭ +Ġ×ij ת +หร ืà¸Ń +Ġس ÙĬ +ãģª ãĤī +à¹Ĥà¸Ķ ย +ı yor +à¸Ńี à¸ģ +á»ĩ nh +Ñĭ м +à¸Ĺุ à¸ģ +Ġ׾ ×Ĺ +Ġ×Ķ ר +Ġ×Ķ ×Ļ +à¸ŀ ระ +à¹Ģว ลา +ĠØ º +ẫ n +m Ä±ÅŁ +׼ ×Ķ +á»ij n +ãģ§ ãģĹãĤĩãģĨ +ãĥ ¢ +à¸Ľ ี +ס ×Ļ +ãģĵ ãĤį +Ġ׾ פ +ร à¸ĸ +ê¸ Ī +à¸ģ วà¹Īา +ë ¬´ +á»į ng +ãĤĵ ãģ§ +ãĤĪãģĨ ãģª +á»ĵ i +ãĤ ¬ +ส à¹Īà¸ĩ +×Ļ׳ ×Ķ +à¸ĸ ูà¸ģ +à¸Ī ัà¸Ķ +Ġ×Ķ ×Ĵ +ãĥ ľ +×ŀ ×ķת +ÙĪ Ùĥ +ëĭ ¨ +ĠØ « +ãģ® ãģĮ +à¹Ģห à¹ĩà¸Ļ +ع ا +à¸Ļ ิ +Å ŀ +à¸Ń ะ +ãģĪ ãĤĭ +Ø« ÙĦ +ØŃÙħ د +à¹Ģà¸ģ ิà¸Ķ +פ שר +פ ×Ķ +ม ิ +ئ ÙĬس +à¸Ĺำ à¹ĥหà¹ī +×¢ ×ĵ +ìĭ ¤ +à¸Ĭà¹Īว ย +ĠاÙĦÙħ ÙĨ +ز ÙĬ +ع ÙĬ +Ġ׼ ×IJ +ạ nh +á» ¹ +ãĤĵ ãģª +ส ู +צ ר +Æ°á»Ľ ng +×ķ ×ķ×Ķ +à¹Ĥ ล +ĠاÙĦ Ùĩ +ว า +หล าย +Ñī е +à¸Ĥ à¹īà¸Ń +à¹īà¸Ń ย +ب Ø· +ка Ñı +ĠØ ¢ +Ġи Ñģ +ĠاÙĦ غ +à¸ģ า +à¸Ļ à¹Īา +ÙĬ ÙĪ +×ij ×ķר +á»ħ n +ว à¸ĩ +×Ļ× ĸ +ì² Ń +н им +ëŁ ° +×Ĵ ×ķר +ص ØŃ +ÙĦ ÙĪ +×Ĺ ×ķת +ส ุ +رÙĬ ÙĤ +ס ×ĺ +Ġ×ŀ ×¢ +ãĥĨ ãĤ£ +à¸Ħ ิà¸Ķ +ãĤį ãģĨ +à¹Ħ ล +à¸Ļ à¹Į +á»ı i +ÑģÑĤÑĢ о +ส à¸Ķ +ส าร +ÙĪÙĦ Ø© +ầ m +ร à¹Īว +รà¹Īว ม +ร ุ +ĠاÙĦس ÙĬ +ìĺ ģ +Ġ×ŀ ×ij +פ ×ĺ +à¸ķิ à¸Ķ +×ĺ ×Ļ×Ŀ +Ġë ¬´ +ÙĤد Ùħ +Ġdü ÅŁ +ائ ÙĦ +м Ñĭ +ØŃ Ø³ +ÙĪ ص +×Ļק ×Ķ +ãģ§ãģ¯ ãģªãģĦ +à¹Ģ หม +оÑĢ ÑĤ +í Ĩµ +ãģ IJ +к ÑĢа +ีย ว +ع ار +ئ Ø© +íĥ Ģ +ãģ«ãģª ãĤĬ +ج Ø© +ÙĪÙĤ ع +ÑĮ Ñı +×ķצ ×Ķ +ש ×Ŀ +ب ÙĤ +Ġ×Ļ ×Ķ +ÙĬ Ø· +ım ız +д еÑĢж +×Ļש ר×IJ׾ +غ ÙĬر +ร à¸Ńà¸ĩ +à¹Ģรีย à¸Ļ +Ġ×Ķ ×ĺ +หม าย +Ùħ Ùĩ +اÙģ Ø© +Ġо ÑĢг +ÙĪ Ùī +ãĥ© ãĤ¤ +×ŀ ׳×Ķ +ĠÄij o +Ġг оÑĢ +اÙħ Ø© +æ¥ ½ +Ø« ÙĬر +à¸ģิ à¸Ī +á»ĵ n +ÙĨ ب +ÑĢÑĥ д +ìĹ Ī +Ġ×Ĺ ×ijר +ÑĢаР¶ +ạ ch +ت ÙĪ +à¹Ĥ ม +×ij ×Ļ×ij +Ġí Ĩµ +aca ģı +جÙĦ س +à¹Ģà¸Ľ ล +ว à¸Ķ +à¸Ń ล +ãģŁ ãĤĬ +à¸Ľ ัà¸į +Ġìķ Į +عر Ùģ +à¹Ħ à¸Ł +Ø£ Ø® +å¤ļ ãģĦ +à¸Ķ ัà¸ĩ +Ø´ Ùģ +ãģ£ãģ¦ ãģĦãģ¾ãģĻ +׼ ×ł×¡ +ÑĨ е +еÑģ п +Ùħ اÙħ +à¸ŀื à¹īà¸Ļ +иÑĩеÑģ ки +Ø® د +Ùĥ ÙĪÙħ +Ġ×Ķ ר×IJש +ت اب +é£Ł ãģ¹ +ื à¸Ļ +оÑĢ о +Ġb öl +×ķ×Ĺ ×ĵ +دÙĬ ر +ắ m +د ع +ãģķ ãģĽ +à¸ĺ ร +à¸ĺร รม +ãģĭ ãĤĤ +å¤ļ ãģı +r ä +س ع +×Ļ׾ ×Ķ +ض ر +ĠاÙĦ شر +×ĸ ×ķר +×¢ ×ijר +ạ m +алÑĮ но +ر ÙĨ +اÙħ ج +׼ ×ļ +d ıģ +д ен +ض ا +ÙĦÙĬ Ùħ +Ġê·¸ 룬 +تÙħ اع +ار ÙĬØ® +à¹Ĥ à¸ķ +ĠÑģ ÑĢед +Ġ׳ ×ķס +ÙĤ بÙĦ +оÑĤ ов +le ÅŁtir +Ġм еÑģÑĤ +سÙĦ Ùħ +Ġ×¢ צ +ĠاÙĦس ÙĦ +еÑĤ ÑĮ +اب Ø© +н ак +สà¸ĸ าà¸Ļ +Ġ×ij ׳ +à¸ļ ัà¸Ļ +׼ ׳ +Ġö ÄŁ +ãģ¨ è¨Ģ +uy ến +di ÄŁ +áºŃ u +ÑĢ аÑģ +ãĤ· ãĥ§ãĥ³ +n ız +×ķ×ĵ ×Ķ +ت س +Ùħ اÙĦ +à¹Ģห à¸ķุ +ย ว +à¸ŀ ัà¸ģ +ãģĦ ãģªãģĦ +Ġк аÑĩ +ล à¹Į +ר׼ ת +ÅŁt ur +×ŀ ×ķס +ãģ ¥ +б ол +عÙħ اÙĦ +×ķר ת +ÑĨи он +ศ ึà¸ģ +ภı +ÑĢ ен +اس ÙĬ +ائ ر +à¹Ĥ à¸Ľà¸£ +Ġse ç +غ ÙĬ +Ñį ÑĤ +ен н +ãģª ãģ® +×Ļש ×Ķ +×Ļפ ×ķר +ãģŁãĤģ ãģ« +ز Ø© +Ġç oc +ãĤ¯ ãĥª +ÑĪ ен +ãĤı ãģij +رÙĬ د +ĠÑĢ аÑģÑģ +Ùĥ ات +ส à¸Ńà¸ļ +ce ÄŁi +ãĤ¿ ãĤ¤ +à¸ļ ร +ĠاÙĦ بر +׳ ×ķ×¢ +r ün +را ض +ศา ส +à¸ķ รà¹Į +ãģį ãģŁ +×ķ׾ ×ĵ +еÑĢ и +íĹ ĺ +ắ p +ت عÙĦ +Ùĥ د +иÑĤелÑĮ но +Ø· Ùģ +Ġав ÑĤом +Ġ×ŀ צ +ÑĪи Ñħ +ات Ùģ +ĠÑħ оÑĤ +Ùİ Ø§ +ãģı ãĤĭ +×Ķ פ +à¹Ĥ à¸Ĺ +à¹ģ à¸ŀ +à¹Ī à¸Ńย +ĠاÙĦÙħ Ø´ +à¸ģาร à¸ĵà¹Į +ани з +×Ķ ׾ +ظ Ùħ +ย ุ +li ÄŁ +à¹Ħ à¸Ĥ +à¸ĸ ืà¸Ń +ö z +ãģij ãģ¦ +à¹Ģ à¸ľ +ุ ม +ãĥĹ ãĥ¬ +Ġ×Ķ×IJ ×Ĺר +خت ÙĦÙģ +à¸ İ +ÙĦا ØŃ +Ġdü zen +צ ×Ķ +س اء +×ķר ×ļ +×ķ×ĵ ×Ļ +ÑĢа ÑĦ +ÅŁt ır +ãģ« åħ¥ +ãģĪ ãģ° +ص ÙĪÙĦ +ĠÐľ оÑģ +ا Ùĩر +ãģ£ ãģ +ĠлÑİ Ð± +×Ļ×¢ ×Ķ +Ġ×Ķ×ŀ ק +สิ à¸Ĺ +สิà¸Ĺ à¸ĺิ +×Ļ׳ ×Ŀ +ÙĦا Ùģ +à¸ŀัà¸Ļ à¸ĺ +×ķ×IJ ×Ķ +ม ั +à¸Ĥ à¸ĵะ +д оÑĢ +ãģ¨ ãģª +à¸ģระ à¸Ĺ +ac ı +×ķ׾ ×ķ×Ĵ +Ñĥ ÑĪ +ãĥ¥ ãĥ¼ +ãĥ ¦ +Ùħ ست +Ġa ÅŁ +ש ק +פ ת×Ĺ +าย à¸Ļ +í ĩ +ë ¢ +ï · +í ī +ì µ +ì ¬ +ðĿ Ľ +ì Ĵ +ë Ļ +ê § +á ĸ +â ¨ +â ± +á ĺ +ð ĸ +à ł +á Ķ +ðIJ Ń +ữ ng +Å© ng +Ġ×Ķ ת +ĠاÙĦ ا +Ġ×ŀ ת +à¸ĸ ึà¸ĩ +ò n +á»ĭ nh +нÑĭ м +Ġc ả +à¸Ķ ู +Ġ à¹ģà¸ķà¹Ī +Ġ×ij ×Ķ +ó i +ãģ¨ ãģĹãģ¦ +ú ng +ĠØ ° +Ġ×Ķ ׳ +Ġب ÙĨ +ÙĦ اÙĦ +à¹Ħ à¸Ĺย +á»ĩ p +t ı +ม ัà¸Ļ +ằ ng +á»ij t +к ом +à¸ĭ ึà¹Īà¸ĩ +à¸Ħร ัà¸ļ +à¸ļ à¹īาà¸Ļ +ĠاÙĦ ÙĬ +l ü +ÙĪ س +ãģł ãģ£ãģŁ +à¹Ģ à¸ĩ +Ġê³ µ +н Ñĥ +ãĤĪ ãĤĬ +м Ñĥ +à¹Ģà¸Ĥ า +ãĤ Ģ +ни е +ãģ«ãģª ãĤĭ +áºŃ y +ĠÙĪ ا +ëł ¤ +ש ×ķ +á p +×ĵ ×ķ +ãģ§ ãģĹãģŁ +ع ض +Ñģк ой +æĦŁ ãģĺ +ÑİÑĤ ÑģÑı +Ġ×Ļ ׼×ķ׾ +ãĤĵ ãģł +в и +à¹Ģล à¹Īà¸Ļ +ìĿ´ ëĭ¤ +ĠÙĦ Ùĩ +à¸Ħ ืà¸Ń +ت Ùĥ +Ùħ ÙĥÙĨ +a ģı +׳ ×ĵ +ë¯ ¼ +à¹Ħ ว +สำ ห +สำห รัà¸ļ +Ñģл ед +t ır +ĠÙĦ ÙĬ +ĠاÙĦع ÙħÙĦ +×ij ×ķת +×ij ×Ļ×Ŀ +à¸Ħ ำ +à¹Ģà¸Ħร ืà¹Īà¸Ńà¸ĩ +lı ģı +ืà¸Ń à¸ĩ +ج د +íŀ Ī +ìĭ ¬ +×¢ ×ķת +ส ิà¸Ļ +Ñĩ и +ر ض +à¹Ģà¸Ľ ิà¸Ķ +à¸Ħ à¹Īา +ìĦ ł +ÙĪر Ø© +ק ×ĺ +ìľ ł +ع ÙħÙĦ +×IJ ×Ļ×Ŀ +׾ ×Ļ×Ŀ +à¹ĥห à¸į +à¹ĥหà¸į à¹Ī +ừ a +á»į i +ãģ ¶ +ÃŃ ch +ãĥĩ ãĤ£ +×ķר ×Ļ×Ŀ +Ñģ о +ìķ ½ +ов а +Ñĩ аÑģÑĤ +à¹Ģà¸Ī à¹īา +п ÑĢо +Ġ×ŀ ×Ĺ +ãĥ İ +×ķ×Ļ ×ķת +Ġд е +ë§ Ī +ì§ ģ +×Ļפ ×Ķ +ĠاÙĦع اÙĦÙħ +ë¥ ´ +ר×IJ ×Ķ +uy á»ĥn +×¢ ×Ļ +ม ืà¸Ń +Ø¥ ÙĨ +ร ู +ĠØ ² +×Ļ ×ķ×Ŀ +à¸ķ à¹īà¸Ļ +ãģ¦ ãģĦãģ¾ãģĻ +Ùħ اÙĨ +ĠÐ ¥ +à¸Ľà¸£à¸° à¹Ģà¸Ĺศ +á» ³ +׾ ×ij +à¹Ģà¸Ķ à¹ĩ +ãģŁ ãģ¡ +à¸Ĺี ม +à¸Ļ ะ +ìĹ ° +Ġìł Ģ +ÙĦ Ùĩ +ợ i +ĠاÙĦ ز +د ار +ãĤ³ ãĥ³ +м ин +à¹ģห à¹Īà¸ĩ +à¸Ķ ัà¸ļ +׼ ר +ж а +íĸ Ī +×ŀ ×ĸ +ợ i +à¸Ķ า +Ġع بد +à¹ģ ร +×IJת ר +×¢ ׳×Ļ +à¹Ģ à¸Ħ +×ķצ ר +ì§Ģ ë§Į +ائ Ùħ +Ø£ س +uy á»ģn +Ġ×IJ ׳ +׊׳×ķ +×ĸ ×Ļ +ร à¹īาà¸Ļ +ĠÐł оÑģ +ĠÐłÐ¾Ñģ Ñģ +رب ÙĬØ© +t ür +ãĤĭ ãģĵãģ¨ +ظ ر +б Ñĭ +à¸Ĺีà¹Ī สุà¸Ķ +Ġצ ר +èĩª åĪĨ +л аÑģ +ĠÑı в +ĠÑıв лÑı +à¸ŀร à¹īà¸Ńม +à¸Ńา à¸Ī +à¸ļริ à¸ģาร +Ġç ı +ëį ĺ +ĠاÙĦÙħ ست +ت Ø´ +ש ×ķ×ij +ãĤ ´ +Ġyap ıl +ĠاÙĦ Ø° +ุ à¹Īม +à¸ĸ à¹īา +ìĦ ¤ +ì° ¨ +в аÑĢ +à¹Ģà¸ŀ ิà¹Īม +Æ°á»Ľ i +Ùĥ س +à¸Ńย าà¸ģ +ãģ¦ ãĤĤ +Ġг од +ÙĬ ار +à¸ķ à¸Ńà¸Ļ +Ġиг ÑĢ +à¹Ħà¸Ķà¹ī รัà¸ļ +ĠاÙĦÙħ ر +ÙĤ ت +Ġë ĺ +Ġëĺ IJ +ẩ n +ãģĻãĤĭ ãģĵãģ¨ +×Ĵ ×Ŀ +Ġ×ij ×ij +ت د +ÙĪ ار +ãĤ ® +п ол +Ġм ог +تر Ùĥ +ÙĪ Ø« +Ġç ık +ا Ø© +à¹Ģà¸Ķ ียว +มี à¸Ħวาม +Ġ×ŀ ×Ĵ +ص Ùģ +ĠТ ак +Ġ׼ ת +×Ļ×ĵ ×Ļ +ов оÑĢ +ầ y +สิ à¹Īà¸ĩ +ب ت +ür ü +ÙĨ ج +หล ัà¸ģ +×Ļ×Ķ ×Ŀ +ÙĤ ص +з Ñĭ +×Ľ×ª ×ij +Æ° u +m ız +ĠìĦ ¸ +л ог +Ùħ ÙĬÙĦ +ÙĬ ج +íĴ Ī +à¸ŀ à¸ļ +ห ัว +з на +ר ק +à¹Ĥ ร +Ġ×ij ס +ĠBaÅŁ kan +ĠëĶ ° +à¸Ń ัà¸Ļ +ีà¹Īย ว +н еÑģ +à¹Ģà¸Ķ ิà¸Ļ +ÙĬ اÙĨ +×ķ׾ ×Ļ +ا خت +צ ×ķת +ãģĵ ãģĵ +ĠاÙĦ اÙĨ +ĠпÑĢо ÑĨ +ãģ¾ ãģł +׼ ס +ĠاÙĦ Ø¢ +ÙĬ ز +ĠاÙĦد ÙĪÙĦ +Ġíķĺ ëĤĺ +ض ع +ê» ĺ +ÅĽ wi +ย ิ +ãģ¡ãĤĥ ãĤĵ +ĠÙħ Ø´ +à¸ĺ ี +ãģ¨ ãģį +׳×Ļ ×ķת +Ġë ¯ +Ġë¯ ¸ +Ġs ı +ëĭĪ ê¹Į +Ġп л +غ ÙĦ +à¹ģ รà¸ĩ +ب ÙĬر +ãģĤãĤĬ ãģ¾ãģĽãĤĵ +ê· ¼ +Ġy üz +ĠdeÄŁ er +åł´ åIJĪ +á» ¡ +м аÑĤ +รา à¸Ĭ +ÙĪر ÙĬ +ж ен +ãģ¾ ãĤĬ +ãģ® ä¸Ń +×Ļ×ĵ ×¢ +à¸Ń ุ +à¸ļ à¸Ńล +à¸Ľà¸±à¸į หา +ز Ùħ +ÄŁ a +à¸Ń ืà¹Ī +à¸Ńืà¹Ī à¸Ļ +п л +Ġне обÑħодим +׼ ×ij +à¹Ģ ศ +קר ×Ķ +ì² ĺ +ëł ¨ +×ŀק ×ķ×Ŀ +jÄħ c +Ùĩ ÙĦ +Ġ×¢ ×ij×ķ×ĵ +à¹Ħม à¹ī +à¸ģล ัà¸ļ +×ķ׼ ׾ +ק ×ĵ +اÙĦ ÙĬØ© +ر Ùĩ +ãģij ãĤĮãģ° +ĠÙĨ Ùģس +ãĤ¢ ãĥ« +ìĹ Īëĭ¤ +ק ×ķר +н еÑĢ +ب اب +ãĤ ¶ +سب ب +ÙĦ ÙĬÙĦ +ص ÙĨ +ص در +ế m +à¸Ĭà¹Īว à¸ĩ +ØŃ ÙĨ +Ġ×ij ×Ĵ +×ŀ ×ķ×¢ +׾ ×Ĺ +大 ãģį +ت ب +н еÑĤ +×Ļ×ij ×Ķ +б л +ãĥĹ ãĥª +اص Ø© +ãģ¤ ãģij +×Ļ×ŀ ×ķש +ãģĮ ãģĤ +ëĭ ´ +ãģĭãĤĤ ãģĹ +ãģĭãĤĤãģĹ ãĤĮ +ãģ¡ ãĤī +×ij ×ĺ +Ġba ÄŁ +×Ļ×Ĺ ×¡ +×ij ×ķ×¢ +ล ี +פע ×Ļ׾ +им и +g ÅĤ +Ġим е +خد اÙħ +×IJ ×Ļר +Ġy apt +ãģ¨ ãģĦ +à¸ĩ à¹Īาย +׾×Ļ ×ķ +ØŃد Ø« +را ÙĤ +ĠÄIJ i +اد ر +ãģĵãģ¨ ãĤĤ +×ij ×Ļר +Ġв з +ض اÙģ +ת ×ķ׼ +ÑĢ ом +ر ات +à¹Ģà¸Ĺ à¹Īา +ãģĺ ãĤĥ +ãģĿ ãģĵ +اج تÙħاع +à¹īà¸Ń à¸Ļ +ÙĤ Ùħ +ë³ ¸ +Ä ŀ +ש ×Ļ×ķ +×ij ׳×Ļ +ìľĦ ìĽIJ +à¹ģ à¸Ī +×Ĺ ×ķר +دÙĬ ÙĨØ© +ت Ø· +ằ m +ò a +ย à¸Ńà¸Ķ +Ġëĭ ¹ +สุ à¸Ĥ +×ĵר ×ļ +د ÙĨ +س ÙĬÙĨ +ÙĪÙĤ Ùģ +ÑĨ Ñĭ +г оÑĤов +еж дÑĥ +à¸ŀ วà¸ģ +اÙĤ تص +اÙĤتص اد +cz ÄĻ +ni ÄĻ +ÑĢ еб +ØŃ ÙĪ +à¸Ĺ à¹Į +ãĤĪ ãģŃ +д ж +à¸ģล à¹Īาว +دÙĬ Ø« +ãĤ³ ãĥŁ +ÙĤ ÙĪÙħ +Ġت ØŃ +à¹Ģ à¸ķิ +اÙģ ظ +à¸Ī ุ +رÙĬ اض +×ŀש ×ļ +à¹Ĥ ย +еÑĢ е +ãģ¿ ãģŁãģĦ +ìĿ´ ëĿ¼ +ĠاÙĦÙħ ÙĪ +ĠÑģÑĤ о +à¹Ģรà¹ĩ ว +Ġд еÑĤ +ĠÑģ дел +à¹Ģà¸Ĭ ืà¹Īà¸Ń +פ ׳×Ļ +ÙĪض ÙĪع +×ij ס +à¹ģ à¸Ķ +ó c +ริ ม +ÑĢаР´ +ìĪ ł +ãĥ¼ãĤ º +ãģ« ãģĬ +и но +פ ×Ļ׾ +à¸Ĭั à¹Īà¸Ļ +×Ĺ×ĵ ש +à¹Ģà¸Ļ ืà¹Īà¸Ńà¸ĩ +׳ ×Ļס +غ رب +ãĤ¸ ãĥ£ +ส ัà¸ĩ +à¹Ģ à¸Ĺีà¹Ī +à¹Ģà¸Ĺีà¹Ī ยว +ëŁ ¼ +à¹ģ à¸Ł +ãĥ¼ãĤ · +ãĥ¼ãĤ· ãĥ§ãĥ³ +Ġвоз мож +جÙħ ÙĪع +×ijר ×Ļ×Ŀ +ãĥĪ ãĥ© +ĠкаÑĩ еÑģÑĤв +Ø· ÙĬ +ÑĤ Ñı +צ ×ķ×¢ +ÄŁ ını +ع ÙĦÙī +ا Ø° +ÙĪاÙĤ ع +Ùħ ÙĪا +ائ ÙĬÙĦ +к ол +á»ģ m +à¸ľà¸¥ ิà¸ķ +×Ļ׳ ×ĺר +س Ùĥ +ש ×Ļר +ศึà¸ģ ษา +à¸ļ ั +Ñĩ аÑģ +×ķפ ×Ķ +×Ļפ ×ķ׾ +ĠاÙĦس اب +رÙĬ ب +ĠاÙĦ بÙĬ +ãĤ¹ ãĥĨ +Ñĩ ен +à¹ģ à¸ľ +Ġ׳ ש +ز ÙĬد +ØŃ Ø§Ø¯ +ëį Ķ +رÙĪ ع +à¸Ĺุ à¸Ļ +ส มา +c zeÅĦ +×Ļ×ĵ ×Ķ +ãģ§ ãģĤ +Ġçoc uk +Ø® ب +à¸ļ าย +à¸Ľà¸£à¸° à¸Ĭา +×ŀש ׾ +ãģª ãģĭ +à¸ģ าย +ãĥģ ãĥ£ +аÑĢ и +ĠÑĩ а +à¸Ķ ำ +à¸Ĺั à¹Īว +Ñĥ Ñħ +Ġö z +Ġì¢ ĭ +ج رÙĬ +ائ ÙĤ +à¸ł ัย +Ø· ار +د ارة +Ä© nh +Ø« ÙĨ +zell ik +اÙĦ ت +Ġg eli +ãĥķãĤ © +ол од +رب ع +שת ×ŀש +à¸ļร ร +íĿ ¬ +Ġü rün +Ġê·¸ ëłĩ +ศาส à¸ķรà¹Į +ãģ ľ +×Ļ×ij ׾ +ĠпÑĢед ÑģÑĤав +سط ÙĬÙĨ +ãĤĴ 使 +Ġпом оÑī +×ķק ר +ãĥ¯ ãĥ¼ +Ġyö net +×Ļק ר +à¸Ĥ า +еÑĢи ал +ØŃ Ùģ +Ġ×Ļ צ +à¸Ĺ ิ +å£ ² +à¸Ļ à¸Ńà¸ģ +×ķ׼ ר +íĻ ľ +ủ y +ĠاÙĦÙĤ ر +×Ļ×ij ×ķת +ÅĽ ni +Ùħ شار +ượ t +ĠÙĦ دÙĬ +ÑĤ ел +ĠØ¥ ÙĦÙĬ +عÙĦ ÙĪÙħ +ìķ ĺ +в иÑĤ +à¸Ħ ะ +yr ı +ãģ¨ ãģ£ãģ¦ +à¹Ģ à¸ī +à¸ĸ าม +ÙĤ ار +عÙĦ اÙħ +ặ ng +Ùħ ÙĴ +×Ļ×ŀ ת +سب Ø© +ãĤ¯ ãĥ© +×ķס ×£ +ĠпÑĢ ин +ãģĦ ãĤį +س اس +عت بر +วิ à¸Ĺย +วิà¸Ĺย า +س Ùĥر +ãĤ· ãĥ§ +ãģ ģ +ัà¸ģ ษ +×ij ×ķ×Ķ +ห ย +ãģ¾ ãĤĮ +ĠоÑĢг аниз +каз ал +ĠÑģв Ñıз +uy ết +ĠпÑĢо из +Ġק ×ĺ +à¹ģà¸ģ à¹ī +п ÑĥÑģ +Ġê·¸ ê²ĥ +ëĬ IJ +л екÑģ +ãĥ¼ãĥ Ĺ +à¸ķ ำ +ת×Ĺ ×Ļ׾ +à¸Ńà¸ĩ à¸Ħà¹Į +Ạµ +׳ צ +Ø£ Ø´ +Ø´ Ùĩ +ย ะ +à¸ģ à¸İ +ĠاÙĦØ¥ سÙĦاÙħ +ед ÑĮ +ãģ² ãģ¨ +ëıĦ ë¡Ŀ +ãģ© ãģ® +Ñĥ в +еÑĩ ение +ĠاÙĦت ج +ãģ« è¡Į +Ġп озв +ãĤı ãĤĬ +ÙĦ اث +íķĺ ìĺĢ +Ġм аÑĢ +Ġkon uÅŁ +ãĥ¬ ãĤ¹ +ãĤĴ æĮģ +ĠоÑģ нов +×Ĺ ×ij +ÙĪج ÙĪد +פ ×ķף +в оÑĢ +Ġн ик +ãģĭ ãĤĭ +ÅŁtır ma +×Ļס ×ĺ +Ø£ ÙĦ +ห à¹Į +и она +лÑĮ н +Ġг оÑģ +ĠÐľÐ¾Ñģ к +ÑĢ об +×ķ×IJ ×Ļ +ãģĬãĤĬ ãģ¾ãģĻ +ãģ£ãģ ± +к л +à¸Ļ à¸Ķà¹Į +رÙĬ Ùģ +اس ب +ĠÑĢ еÑĪ +Ġд ол +ãģ¹ ãģį +×Ļ×ij ×ķר +м еÑī +Ġна ÑĪ +à¹ģ à¸Ľà¸¥ +ÑĢ иÑĤ +кÑĥ Ñģ +и ÑĢа +аÑĤ ÑĥÑĢ +ÙĪا صÙĦ +à¹Ģà¸ľ ย +à¸Ń ำ +à¹Ģà¸ģ ิà¸Ļ +غ Ùħ +ãģĻ ãģİ +lı kl +ÅĦ sk +ê² ¬ +×Ļ׼ ×Ķ +׊ש×ij +ÙĪر ÙĬØ© +Ġд ейÑģÑĤв +×Ĺ׾ ×ĺ +Ġ׾ ×ŀ×¢ +צ׾ ×Ļ×Ĺ +еÑĩ а +Ùģ اع +×Ĵ ×Ļ×ĵ +áºŃ m +ÄĻ b +Ø´ ع +ãģı ãĤĬ +à¸ŀ ุ +ед еÑĢ +à¸Ĥ à¸Ļ +à¸Ħ าร +ĠболÑĮ ÑĪ +ãģı ãģªãĤĬ +à¸ĵ า +×ĵ ×ķ×Ĵ +Ġм н +ä¸Ĭ ãģĮ +ç¶ļ ãģį +ฤ ษ +ภĨ +Ø® ÙĬ +à¹Ģà¸Ĺ à¸ŀ +สั ม +à¹Ģส à¸Ļ +à¹Ģสà¸Ļ à¸Ń +ãĥ ´ +Ġи ÑģÑĤ +با شر +ĠÑĥ ÑĢов +×ŀ ×ķ×ĸ +ab ı +wa ż +×ķצ ×IJ×Ķ +ÑĤ веÑĢ +à¸ŀัà¸Ļà¸ĺ à¹Į +׳ ×Ĵ×ĵ +ãĤĭ ãģĵãģ¨ãģĮãģ§ãģį +ĠÑĤÑĢ еб +à¸ģร ุà¸ĩ +ØŃت اج +à¹Ģ à¸Ħล +ã Ĩ +ÄĻ tr +Ġszcz eg +Ġר ש +à¸Ĺ à¸ĺ +Ġн ек +Ġнек оÑĤоÑĢ +в ÑĪ +Ð ¬ +à¹Īว ย +ล ุ +б ÑĢÑı +หม ูà¹Ī +à¹ģ à¸ķà¸ģ +ר׼ ×Ļ×Ŀ +Ġí ĸī +ã i +Ùĥر Ø© +â Ń +í IJ +ã į +á ģ +â ® +â ¥ +ì ® +à ¿ +â ¿ +á Ĥ +á ¤ +â ł +í Ł +ðIJ į +ðIJ ° +ðĿ Ĩ +ðŁ Ī +Ġ×¢ ׾ +Ġع ÙĨ +ĠÙħ ع +Ġ×ĸ ×Ķ +ĠÙħ ا +Ġm Ãł +Ġd ụ +á»ĩ c +а Ñħ +s ı +íķĺ ê³ł +Ġ×ķ ×ij +ĠÐŁ о +×ķת ר +ĠÙĦ Ùħ +Ġ×ķ ׾ +ãģĹãģ¦ ãģĦãĤĭ +Ġ×ŀ ×Ļ +Ġب ÙĬÙĨ +з а +ĠÙĥ اÙĨ +Ġ×Ķ ×Ļ×Ķ +ëħ Ħ +×IJ ×ķ +д и +ĠпеÑĢ е +d ı +Ġ׾ ש +Ġש ×ŀ +ãģĮ ãģĤãĤĭ +ãģĦ ãģĦ +ÑĢ е +ק ×ķ +и ли +м е +ÙĬ ت +ãģ§ ãģĤãĤĭ +Ġв о +à¹ĥ หม +à¹ĥหม à¹Ī +Ġש ×ij +Ġ à¹Ĥà¸Ķย +ÙĬ Ùĩ +ãģ§ãģĻ ãģĮ +ãģ¨ ãģ¯ +ר ×ķ +Ġ à¸ĭึà¹Īà¸ĩ +ãģ§ãģį ãĤĭ +м о +à¹Ģà¸ŀ ืà¹Īà¸Ń +צ ×ķ +×ĺ ×ķ +ìķ Ī +Ġh á»į +à¹Ģà¸ĩ ิà¸Ļ +ĠاÙĦ ب +Ġ มี +ë¬ ¼ +Ñģ е +ëĵ¤ ìĿ´ +Ġë§ IJ +Ġl Ỽ +a ÅĤ +×Ĺ ×ijר +Ġd á»± +ÙĬ Ø« +Ġth á»ĭ +à¸ģà¹Ī à¸Ńà¸Ļ +Ġ×ij ׼׾ +ãģ ¸ +ãģ¨æĢĿ ãģĦãģ¾ãģĻ +ả nh +ย า +Ùģ ا +ส ี +à¸ķ า +ë² ķ +ãĥª ãĥ¼ +รา à¸Ħา +Ġ×ķ ׾×IJ +ãģ¨ ãģĵãĤį +à¹Ģล ืà¸Ń +di ÄŁi +ÙĪ اÙĨ +Ġ׾×Ķ ת +รว ม +פ ×Ļ×Ŀ +à¸ľ ม +ж и +c ı +ÑĢ од +Ġkar ÅŁÄ± +×Ĵ ×ķ +ãģ« ãģ¤ +ãģ«ãģ¤ ãģĦãģ¦ +r Ãł +×Ļ×ķת ר +ĠìĨ Į +ק ×Ķ +ÑģÑĤв о +ãģij ãģ© +g é +à¸Ķ à¹īาà¸Ļ +çļĦ ãģ« +ĠÙĬ ÙħÙĥÙĨ +ìĨ į +ÙĬ Ùĥ +à¹Ħว à¹ī +Ñģки й +ì m +Ġ׾×IJ ×Ĺר +à¸Ńา หาร +Ġà¹Ģ à¸ŀ +รา ะ +ล ูà¸ģ +ÑģÑĤ а +Ġìľ ł +ÙĤ ÙĪÙĦ +б оÑĢ +Ñģк ого +หล ัà¸ĩ +à¸Ĥ à¹Īาว +à¹Ģม ืà¸Ńà¸ĩ +ê° ģ +t Ãł +ÙĬ ÙĬÙĨ +عر ض +ë° © +Ġëı Ļ +Ġà¹Ģ à¸Ľ +Ġà¹Ģà¸Ľ à¹ĩà¸Ļ +ç i +li ÄŁi +ìĹIJ ê²Į +ãĤ¿ ãĥ¼ +Ġ׾ ת +פ ×ķת +à¸Ĥ à¸Ń +ر س +ìł IJ +à¸ľ à¹Īาà¸Ļ +ÑĦ и +ج ÙĨ +ì¢ ħ +Ġ×Ķ פ +Ġn go +á»ĭ a +Ġtá» ķ +Ġê·¸ 리 +à¹Ģม ืà¹Īà¸Ń +Ø° Ùĥر +ìĸ ij +ìĹ Ń +×ĺ ׾ +k ı +Ġع ÙħÙĦ +Ġع ÙĨد +à¸ĭ ืà¹īà¸Ń +Ġê± ° +в е +r ü +à¹Ģ à¸Ńา +ส à¹Į +à¸Ī à¸Ļ +ס ת +Ġgi ả +ãĤĭ ãģ¨ +à¸ģำ ลัà¸ĩ +н ей +à¸Ī ริ +à¸Īริ à¸ĩ +Ġë į +Ġëį Ķ +à¸Ħà¹Ī ะ +ì n +Ġsü re +Ġqu y +à¸ļ าà¸ĩ +åıĸ ãĤĬ +ר ×Ĺ +×ij ת +ãģĮ ãģĤãĤĬãģ¾ãģĻ +ר ש +ìĹIJ ëĬĶ +Ġ×IJ פשר +ay ı +ãģĮ ãĤī +ØŃ Ø¨ +ан Ñģ +س ÙĪ +ĠпÑĢ е +د ÙĪ +ãģ« ãĤĪ +à¹Ģà¸ģ ม +สู à¸ĩ +m akt +makt ad +maktad ır +Ġön em +×Ļ×ŀ ×Ļ×Ŀ +б о +ÙĪ ÙĬØ© +รู à¸Ľ +à¹Ĥล à¸ģ +Ùħ ÙĬع +ÑģÑĤ Ñĥп +à¹Ĥ à¸Ń +دÙĬ ÙĨ +ì¤ ij +ãģĹãģ ı +à¹Ģส ีย +в Ñĭ +Ùħ ت +íĺ Ħ +ãĥIJ ãĥ¼ +ا Ø´ +ק ס +Ġtá» ¥ +ล à¸Ķ +Ùģ Ø© +í ijľ +ر ج +k ÅĤad +ĠÅŁ ey +ĠØ£ Ùħ +Ġà¹Ģ ม +Ġب ÙĦ +Ñģ каÑı +ãģ¨ ãģ® +Ġìĭ ¤ +ấ m +ห à¹īà¸Ńà¸ĩ +à¸Ĭ ม +d ü +Ġç ek +Ġê³ ł +×Ĵ ×ij +à¸Ĭี วิ +à¸Ĭีวิ à¸ķ +Ùģض ÙĦ +ภ¯ +ç ı +Ġب Ø´ +ĠÙĩ ÙĨا +ãģį ãģ¾ãģĹãģŁ +t ü +Ġìĺ ģ +ĠTür k +к ÑĤ +פר ס +ãģ¨ãģĦãģĨ ãģĵãģ¨ +í ĶĦ +à¹ģร à¸ģ +ר ×ķף +Ġar as +×ŀצ ×IJ +Ġtá» ī +س ا +à¸ŀ à¸Ń +ĠاÙĦÙħ ØŃ +ãĥ ¤ +ĠاÙĦ است +Ùģ ÙĨ +×Ļ×ŀ ×Ķ +ر ت +ãģ¨ ãĤĤ +Ġна Ñģ +п ÑĢи +Ġ×Ĺ ×ķ +и ла +ÙĬ Ø´ +Ġgö z +Ġ×ij ׳×Ļ +ım ı +ĠÑĤ еÑħ +Ġh á»Ļ +غ ر +к он +اØŃ Øª +Ġ à¸ŀ +à¸Ń à¸Ńà¸Ļ +à¸Ńà¸Ńà¸Ļ à¹Ħล +à¸Ńà¸Ńà¸Ļà¹Ħล à¸Ļà¹Į +Ñħ о +Ñı в +à¹ģ สà¸Ķ +à¹ģสà¸Ķ à¸ĩ +à¹Ģà¸ŀ ียà¸ĩ +ÑĤ ов +ا ÙĬ +Ġ×Ķ ×ĵ +Ġ×ķ ׼ +ãĤī ãģĦ +×ķפ ף +Ġë ¶Ī +ล à¸Ńà¸ĩ +Ø· اÙĦ +Ġн и +ĠÙħ ست +ế c +Ġש ׼ +ĠëķĮ 문 +วัà¸Ļ à¸Ĺีà¹Ī +×Ļ׾ ×ĵ +ØŃ Ø§ +е ÑĨ +Ġc ứ +×ĵ ×ķר +ĠÙħ ØŃ +ר׼ ×ij +بÙĬ ع +ни и +ĠاÙĦØ£ ÙĪÙĦ +à¸Ħว ร +ãģ¨æĢĿ ãģĨ +ĠС о +ائ ÙĬØ© +ر اء +оÑģ об +Ġب Ø£ÙĨ +×¢ ×ķ×ĵ +ĠÑĤ е +ãģĵ ãģĨ +ÑģÑĤ ÑĢа +ай н +Ġsö z +ت ÙĨا +à¸Ń ิ +ặ p +ĠìķĦ ëĭĪ +íķ Ń +Ġר×IJ ש +Ġ à¹Ħà¸Ķà¹ī +Ġ×Ĵ ×ĵ +Ġס פר +обÑī е +ĠÙĪ Ø¥ +ada ÅŁ +ãģ¡ ãĤĩ +ק ×ķ׾ +ÑĢ ез +ĠdÃ¼ÅŁ ün +Ġ×ij ×IJ×ŀ +Ġìĸ´ ëĸ +ער ×ij +н ее +ĠÑģÑĤÑĢ ан +س اÙĨ +yn ı +ĠاÙĦر ئÙĬس +ãģĹãģ ª +Ġ׳ ת +ãģ«ãģª ãģ£ãģŁ +g ü +åıĹ ãģij +׾ ת +ìł Ī +ëĬĶ ëį° +Ø® ÙĬر +à¸ķà¹īà¸Ńà¸ĩ à¸ģาร +ĠÙĦ Ø£ÙĨ +Ġch á»ĭ +ÙĪ Ø© +à¹ĥ ส +ë¶Ģ íĦ° +íķĺ ë©´ +ữ u +à¹Ģหม ืà¸Ńà¸Ļ +б еÑĢ +ĠìĿ´ ìļ© +ĠÑģ еб +wiÄĻ ks +Ġ׳ ×¢ +ÑĤ ÑĥÑĢ +Ġngh Ä© +ש ×ķ×ĺ +ti ÄŁi +Ġde ÄŁi +×IJ ×ij +Ġ×ŀ ×ŀ +ãĥĹ ãĥŃ +wa ÅĤ +à¸Ī ึà¸ĩ +Ø® دÙħ +×IJ ×Ŀ +Ä±ÅŁ ı +cz Äħ +ר ×ĵ +ĠÑĢ Ñĥб +خر Ùī +ãģ® æĸ¹ +Ġд енÑĮ +×Ĺ ×Ļ×Ŀ +еÑĤ е +ëĤ ľ +×IJ ×Ĵ +×¢ ×ķר +ë³ Ħ +åIJĮ ãģĺ +ãĤ ² +ר ×ļ +×ķש ×IJ +ìľ ¡ +ا Ø® +צ ×Ļ×Ķ +á»± a +ãģĪ ãģ¦ +ש×Ķ ×ķ +ан ÑĤ +ลา à¸Ķ +ин г +ë¡ ł +اع د +ÙĪ سط +Ġв оп +Ġвоп ÑĢоÑģ +Ùħ ÙĬÙĨ +à¸Ħ à¸ĩ +×Ļר ×Ļ×Ŀ +c ów +ê² © +Ġê·¸ 룰 +Ġì§ Ħ +Ġש ׾×Ķ +à¹Ģร ิà¹Īม +à¸Ĭ à¸Ńà¸ļ +д еÑĤ +ÑİÑī иÑħ +à¸ļ à¸Ńà¸ģ +æĢĿ ãģĦ +ع ÙĬد +ס ×ŀ +×Ĵ ×Ļ×¢ +צ ×ĵ +ب ات +ĠëĶ° ëĿ¼ +à¸Ī ัà¸ĩ +ãģłãģij ãģ§ +×¢ ×Ļר +ĠÑĩ ел +ĠÑĩел ов +ĠÑĩелов ек +ãĥĥ ãĥģ +à¹Ģà¸ģ ีà¹Īยว +à¸Ķ ิ +Ġפ ×¢ +×Ļ×ŀ ×Ļ +ë° ĺ +Ø® ار +×ij ×Ļת +×¢ ×Ļ×Ŀ +ü yor +ãĤģ ãģ¦ +к лад +Ġ à¸Īาà¸ģ +à¹Ģà¸Ħ ย +ส à¸Ńà¸ĩ +à¹ģ à¸Ħà¹Ī +ẫ u +หà¸Ļ ัà¸ĩ +ש׾ ×ķ×Ŀ +اÙĨ ÙĬØ© +åĩº ä¼ļ +åĩºä¼ļ ãģĦ +à¸ł าย +à¸ļา à¸Ĺ +à¸Ĭา ว +mu ÅŁ +Ġ׾ק ×ij׾ +ãĤ· ãĥ£ +ĠÄ° ÅŁ +×Ĵ×ĵ ×ķ׾ +ج عÙĦ +ë³ Ģ +ยิ à¹Īà¸ĩ +à¸Ļ าย +à¸Ļ ีà¹Ī +วิ à¸ĺี +ãĤī ãģªãģĦ +ëł Ī +Ġ문 ìłľ +Ġ à¸ģ +à¸Ĺำ à¸ĩาà¸Ļ +à¹Ģว à¹ĩà¸ļ +ÑĦ е +楽 ãģĹ +สำ à¸Ħ +สำà¸Ħ ัà¸į +ر Ùħ +ãģķãĤĮ ãģ¦ +Ġоб ла +ר×IJ ×Ļ +หม à¸Ķ +ÙĨ ÙĬØ© +ли н +Ġe ÄŁ +it im +ëł ¹ +ص اÙĦ +ÅĽ l +à¸ľ ิà¸Ķ +ãĥŀ ãĥ³ +åħ¥ ãĤĮ +à¹Ģà¸ķ à¸Ńรà¹Į +ار ÙĬ +ĠÐ ¦ +d ür +ส วย +ë¦ ½ +رÙĥ Ø© +Ġh ã +×Ļת ×Ķ +à¸Ĥ à¸Ļา +à¸Ĥà¸Ļา à¸Ķ +à¸Īำ à¸Ļ +à¸Īำà¸Ļ วà¸Ļ +ש ×ķק +Ġд ом +ì± ħ +ãģĭ ãģij +פ ×ķ׾ +à¸Ĭ าย +Ñģ моÑĤÑĢ +Ñģл Ñĥж +ש ×IJ׾ +кÑĢÑĭ ÑĤ +Ġìŀ ĺ +é«ĺ ãģĦ +ĠÑĢ Ñĥк +ÙĨ ص +д ав +Æ°á» ¡ +ưỡ ng +ر اÙħ +×Ļ׳ ×Ļ×Ŀ +ãĥ© ãĥ¼ +ëĦ ¤ +Ġت ع +l ke +好 ãģį +æĮģ ãģ¡ +Ġë§ İ +Ġy ük +ĠÑģоÑģÑĤ ав +енÑĤ ÑĢ +pe ÅĤ +à¹Ģà¸Ľà¸¥ ีà¹Īย +à¹Ģà¸Ľà¸¥à¸µà¹Īย à¸Ļ +íı ī +ãĤĦ ãģĻ +×Ĺ ×ĸ +×ijר ×Ķ +ë£ ¨ +ìĶ Ģ +بØŃ Ø« +à¹Ģà¸ķ à¹ĩ +ów i +ب Ùĩ +ãģį ãģ¾ãģĻ +Ġ×¢ ×ŀ +×Ĵ ×ķ׾ +ез д +ÙĬÙģ Ø© +สà¸Ļ à¹ĥà¸Ī +Ġת ׾ +Ñı Ñī +Ġس ÙĨ +ĠÙĪا ØŃد +ĠÑģ м +lad ı +ı ld +×Ļר ת +ีย à¸Ļ +ת×Ĺ ×ª +Ġж из +à¸ŀ ั +à¸ŀั à¸Ĵ +à¸ŀัà¸Ĵ à¸Ļา +à¸Ĭ ิ +ا Ø®ÙĦ +ãģ£ãģ¦ ãģĦãģŁ +รั à¸IJ +ãĤģ ãĤĭ +à¹Ĥ à¸ģ +ĠT á»ķ +Ġh akk +ر Ùģ +ìł Ģ +Ñģ об +ãģª ãģijãĤĮãģ° +Ùĩ ÙĪ +Ġë² ķ +ãĤ Ĩ +ĠاÙĦس عÙĪد +Ġ×IJ תר +Ø§Ø º +Ġ׾ ×ĵ +à¹ģ à¸ķ +à¹ģà¸ķ à¹Īà¸ĩ +íĮ Į +Ñĥп иÑĤÑĮ +à¸ŀืà¹īà¸Ļ à¸Ĺีà¹Ī +×ij ת×Ļ +à¹ĩ à¸ģ +ÅĤ at +Ġê°ľ ìĿ¸ +ìłķ ë³´ +ÑĤ ал +Ġgü ven +ĠÄ° l +Ġê° ģ +Ġب ت +×ŀ ×ķ׳×Ķ +ĠاÙĦØŃ ÙĥÙĪÙħ +ÙĤ ات +à¹ģ à¸ģà¹Ī +ห าà¸ģ +н ÑĮ +à¸Ľ รัà¸ļ +มา à¸ĵ +Ġне Ñģк +ĠØ ¶ +สม ั +สมั à¸Ħร +ãģĮ ãģĤãĤĬ +м еÑģÑĤ +Ġ×IJ צ׾ +Ġкомп ани +ס ר +ÙĬÙħ Ø© +ĠÑħ оÑĢо +ĠÑħоÑĢо ÑĪ +Ġ×Ļ ×ķ×ĵ +ü s +×Ĵ ×Ļש +à¸ļ à¸Ĺ +تÙĨ ظ +ว าà¸ĩ +ม หา +Ġ׼ ×ķ׾ +à¸Ĥ à¹īาà¸ĩ +ë° ľ +г од +д ан +ãģĭãĤĤãģĹãĤĮ ãģ¾ãģĽãĤĵ +ãģĵ ãģ¡ãĤī +ãĥIJ ãĤ¤ +ece ÄŁi +دÙĬ دة +ÙĨ Ùī +Ġëĭ¤ ìĿĮ +ว ี +غ ا +ли з +à¹Ģà¸Ķ ิ +à¹Ģà¸Ķิ ม +ĠÙĬ ست +Ġy ılı +ko ÅĦ +ãģ§ãģĹãĤĩãģĨ ãģĭ +ãģĤ ãģª +ãģĤãģª ãģŁ +ÑĨ ен +ĠÙĪ ز +×IJ ×Ļש +à¹Ī à¸Ń +ر ØŃ +ê´ ij +ÑĢа ÑģÑĤ +Ġ×Ķ ׾ +ãģĹãģ¦ ãĤĤ +×ŀר ׼ +×ŀר׼ ×ĸ +éģķ ãģĦ +ãģŁ ãģı +ĠÑģ Ñĥд +в еÑģÑĤи +ĠíķĦ ìļĶ +ãĥķ ãĤ§ +ÑĤелÑĮ но +à¹Ģà¸ŀ ืà¹Īà¸Ńà¸Ļ +ÅĤu ż +à¹Ģà¸Ķิà¸Ļ à¸Ĺาà¸ĩ +ש ×ķר +Ġ×ŀ ×ĵ +×ķ×¢ ׾ +ÙĦ اÙħ +à¹Ħ à¸ĭ +л ей +кÑĥ ÑĢ +Ạ¢ +à¸Ĺ าà¸Ļ +ì§ ij +ĠгоÑĢ од +ר ס +׾ ×ķ×Ĵ +mas ını +Ġл ÑĥÑĩ +ล à¹Īา +ìļ ¸ +ש ×ĺ +ĠÐĺ н +í Ĥ¤ +ÙĪÙĦ ا +ìķ ł +ĠØ£ÙĬ ضا +Ùĥ ار +ĠاÙĦت ع +ส ูà¹Ī +ãĤ ¼ +×ij ×Ļ×IJ +ย à¸ģ +ĠØŃ ÙĤ +ر بÙĬ +ãģĺãĤĥ ãģªãģĦ +รัà¸ģ ษา +Ñħод иÑĤ +à¸ķ à¸Ńà¸ļ +׳ ×ĺ×Ļ +ĠاÙĦÙħ ج +تÙħ ع +ов аÑĤÑĮ +ÙĦ ÙĬÙĨ +×Ļ×ŀ ×ķת +Ġm ù +n ÄĻ +Ġد ÙĬ +׼ ש×Ļ×ķ +Ġhi ç +ë ijIJ +ÙĪ اء +ÙĪ Ø· +ĠاÙĦ بÙĦ +à¹ģม à¹ī +ק ×ķת +ÙĪج د +å§ĭ ãĤģ +ÙĬ ئة +Ġë§ ¤ +ص بØŃ +פ ×IJ +г оÑĢ +ס ×Ķ +بÙĬ ÙĤ +ย าà¸ģ +Ġн ад +ÙĬ Ùij +Ġب ÙĪ +ס ×ķר +Ùħ ÙĥاÙĨ +ר ×ij +×Ĵ ×ĸ +צ ת +b ilit +л аг +ĠN go +×IJ ×ķר +à¸ķ à¸Ļ +íĬ ¹ +à¸Ĺีà¹Ī à¸Ķี +à¸Ľà¸£à¸° à¸Īำ +ов ание +ãģĦ ãģ¤ +ãĥĥãĤ¯ ãĤ¹ +åIJĪ ãĤı +åIJĪãĤı ãģĽ +×Ļ׳ ×ķ×Ļ +ạ y +Ø« ÙĤ +ĠпÑĢ об +ĠпÑĢоб лем +ÅŁ eh +ÅŁeh ir +ع ادة +اÙĨ ÙĪÙĨ +à¸ķัว à¹Ģà¸Ńà¸ĩ +ì¶ ķ +ı lan +б ан +ãĥ³ ãĥī +à¸Ī ี +Ġ×Ķש ׳×Ļ +п оÑĤ +×ķ׾ ×Ļ×Ŀ +ล ัà¸ļ +ĠÑį ÑĤи +×ijק ש +ë¹Ħ ìĬ¤ +à¸Ńยà¹Īาà¸ĩ à¹Ħร +×Ļ׾ ×Ļ +à¹ĥà¸Ĭ à¹Ī +ĠاÙĦ ÙĥÙĦ +ãĥļ ãĥ¼ãĤ¸ +ص Ø© +ÑĤи ÑĢ +ãĤĵ ãģ© +зÑĭ к +wy ż +Ùĩ ÙĬ +ĠÙħ ÙĦÙĬ +Ġвид е +ظ اÙħ +دا ÙĪÙĦ +×ŀ ת×Ļ +Ġs ık +à¹Ģà¸ķิ ม +ãĤ¢ ãĤ¤ +ка Ñħ +צ ×Ļ׾ +à¹Ģà¸Ĭ à¹Īà¸Ļ +м аг +маг аз +магаз ин +à¸Ľ ั +à¸Ľà¸± à¸Ī +Ġש ×Ļר×ķת +ีย ม +ãĥĸ ãĥ« +Ġد ÙĪÙĦ +קר ×Ļ×Ŀ +Ùĩ Ùı +ов о +Ġü ret +د ÙĪÙĨ +à¹ģà¸Ļ ว +à¹Ģà¸Ļ ืà¹īà¸Ń +ĠÑĦ оÑĤ +ãĥ ĺ +ãģ¤ ãģĭ +Ñı Ñģ +ĠíķĺëĤĺ ëĭĺ +ائ ع +Ġп лаÑĤ +ìĺ Ī +Ġdost ÄĻp +ÙĪج Ùĩ +Ġ×Ķ ×Ĺ×Ļ +׳ ×Ļק +д ей +í ĽĦ +ı y +بØŃ Ø± +à¹Ģส ริม +Ġ׾ ×Ĵ +Ø°Ùĩ ب +ج ÙĬÙĦ +رÙĥ ز +Ġë ħ +Ġëħ ¸ +פ×Ļ׾ ×ķ +ãģ¾ ãģļ +iri ÅŁ +ĠÙĥ ÙĬÙģ +Ġ×ij צ +Ġêµ IJ +ÑĢоÑģ Ñģ +ĠØ´ ÙĬ +Ġiç er +×Ĵ ×ķ×ij×Ķ +мен но +×¢ ×ij×Ļר +×ķ×ŀ ×Ķ +ãĤī ãģĹãģĦ +ãģ ¼ +Ñī ин +è²· ãģĦ +جÙħÙĪع Ø© +Ġdön em +Ġ×ij ×IJר +в еÑģÑĤ +×ķר ×ķת +س Ùģ +à¹ģà¸Ĺ à¸Ļ +Ġд окÑĥменÑĤ +Ġا ÙĬ +ج اÙĨ +צ×ķ×¢ ×Ļ +ĠоÑģ об +ĠاÙĦÙħ س +ÑĢаР± +à¸ł ู +à¸Ķ าว +л екÑĤ +ع ÙĤ +×ķ×ĵ ×ķת +Ġol u +Ġolu ÅŁtur +ãģ¾ ãģ¾ +ед ин +à¹Ģ à¸Ńà¸ģ +ãĤµ ãĤ¤ +ëĦ Ī +Ø· ÙĨÙĬ +Ø· ÙĤØ© +ĠÐł аз +ÙĦ Ùij +Ñĩ ем +Ġ׾ ×ĺ +สั à¹Īà¸ĩ +سر ائÙĬÙĦ +Ġפר ×ĺ×Ļ +д еÑģÑĮ +Ġ׳ ׼ +اÙĨ ب +ÙĬا Ø© +Ùħ بر +Ġk ı +à¸Ľ à¸ı +à¸Ľà¸ı ิ +à¸ļั à¸ķิ +׳ ת×Ļ +ìĨ ¡ +ر اب +à¹ĥ à¸ķ +à¹ĥà¸ķ à¹ī +×Ļ׳ ת +ÙĪ ÙĬر +Ġ×Ķ×ŀ ×Ļ +ей ÑĩаÑģ +ק ×ķ×ij +در اس +ĠÙħ ÙĤ +رÙĬ ÙĨ +Ø® اص +ãģĬ éĩij +Ġج دا +ãģĨ ãģ¡ +ëħ ¸ +ır ım +æ§ ĺ +ãģ« å¯ +ãģ«å¯ ¾ +ÑĨ ев +Ġv ard +ĠÐIJ н +e ÄŁ +ÑģÑĤв енно +Ð ¨ +س د +à¸ģ ุ +à¹ģà¸ľ à¸Ļ +รูà¹ī ส +รูà¹īส ึà¸ģ +ات ØŃاد +Ñij ÑĤ +×Ĺ ×ķק +ãģĻ ãģIJ +Ø· ÙĦاÙĤ +Ġק ×ķ×ĵ +à¹ĥà¸Ĭ à¹īà¸ĩ +à¹ĥà¸Ĭà¹īà¸ĩ าà¸Ļ +ãĥ¼ãĤ ¿ +Ġs ür +ÑĢ ок +ë³ ij +สมา à¸Ĭ +สมาà¸Ĭ ิà¸ģ +ãĥķ ãĥ¬ +è¾¼ ãģ¿ +ãĤ» ãĥ³ +Ġê°Ģ ì§Ģ +à¸ľ à¹īา +ÑįÑĤ омÑĥ +иÑĤ ел +à¸ł ั +ภij +ãĥĸ ãĥ© +×Ľ×ª ×ķ×ij +׳ ×Ŀ +ен нÑĭе +×¢ ×¨×Ľ×ª +Ġì Ĥ +ĠìĤ ´ +à¸Ĥ à¹īา +׳ ×ķס +ãĥ¬ ãĥĵ +ÑĢ еÑģ +à¹Ģล à¸Ĥ +Ø« اÙĦ +ìĹ Ĩ +ĠÑĩ аÑģÑĤ +า ศ +ãĥª ãĤ¢ +u ç +×Ļ׼ ×ķת +ล à¹īาà¸Ļ +i ë +ãĤ¸ ãĤ§ +à¸Ī à¸Ń +ÙĪ ØŃد +×Ļצ ×ķ×ij +Ġ×ij ש׾ +ок о +ض Ø© +Ø° ر +ĠÑĥ д +Ä° L +×ķצ ×Ļ×Ŀ +×ĸ ×ŀף +à¸Ľ à¸ģ +íķĻ êµIJ +س اÙħ +à¹Ħ à¸Ķ +ละ à¹Ģà¸Ń +ละà¹Ģà¸Ń ีย +ละà¹Ģà¸Ńีย à¸Ķ +ả y +аÑĨи он +ãĤ¹ ãĤ¯ +פ ×ķס +ร à¹Īาà¸ĩ +ен нÑĭй +ع ÙĨ +عÙĦ ÙĨ +ائ Ùģ +d ÄĻ +ؤ ÙĪÙĦ +׾×ķ ×ķ +Ġ×ij ש×ij +ä»Ĭ åĽŀ +ĠاÙĦج ÙĨ +د اد +wa Äĩ +ãĥª ãĥ³ +ĠìŀIJ ìĭł +اÙĨ ÙĬا +ãĥ¡ ãĥª +ÙĦ ÙĪÙĨ +à¸Ĺ à¹Īà¸Ńà¸ĩ +à¸Ĺà¹Īà¸Ńà¸ĩ à¹Ģà¸Ĺีà¹Īยว +اÙģ ÙĬ +Ġли ÑĪ +Ùħ ÙĬØ© +оÑĤ веÑĤ +Ñĩ ин +à Ĭ +ãĥ¡ ãĥ³ +å® Ł +éļĽ ãģ« +ĠÑĢаР¹ +ãĤ¦ ãĥ³ +×Ļר ×ķש +×Ļר×ķש ׾×Ļ×Ŀ +ม ะ +Ġar a +каз аÑĤÑĮ +à¸ķ ัà¸Ķ +ÑĥÑİ ÑĤ +Ġü st +×Ĵ ×ķ×ij +×Ĵ×ķ×ij ×ķת +mal ı +ег од +егод нÑı +اÙģ ÙĤ +à¸Ĭ à¹Īà¸Ńà¸ĩ +Ġö zellik +×Ļצ ×ķר +Ġmi ÄĻd +Ġili ÅŁ +Ġна Ñħод +×¢ ×ĸר +׾ ×Ľ×ª +ÙĨت اج +ĠÑģ ем +à¸Ī à¹Īาย +à¸ķร ว +à¸ķรว à¸Ī +פר ×ķ +à¸Ĥ ัà¸ļ +ãģ ŀ +Ġп ло +к олÑĮ +×ŀ×¢ ×ĺ +íķĺ ìĭľ +jÄħ ce +ÙĨ اÙĨ +ลี à¸ģ +н ÑĥÑĤ +Ġоб ÑĢаз +Ùĥ بر +ĠاÙĦÙĪ Ø·ÙĨ +ãģķãģĽ ãģ¦ +ÙĤ اء +×ŀ×ĵ ×Ļ׳ +y ü +פ ×Ļת +׳ ×ķף +ÙħÙĨ ظ +หà¸Ļ ัà¸ģ +ìŀ Ī +ãĤ« ãĥ¼ãĥī +ع ÙĨÙĬ +п од +ض اء +à¸Ļ à¸ķà¹Į +×ŀש פ +ว à¹Į +ר ×ķק +ส ืà¹Īà¸Ń +פק ×Ļ×ĵ +ãģªãĤī ãģªãģĦ +ĠìŬ 룬 +ÙĦ ج +Ñī иÑĤ +ãĥĥ ãĤ· +ÙĦÙĬ س +ĠÙĦ Ùħا +ìł ij +×ij ×Ļף +ãĥģ ãĤ§ +Ġgü ç +Ġch ứ +×ķצ ×IJ +קר ×ij +à¹Ĥ à¸ŀ +оÑĩ но +סק ×Ļ +ש׾ ×Ŀ +صر Ùģ +ĠL Ãł +×¢ ×Ļת +á» · +à¹Ĥ à¸Ńà¸ģ +à¹Ĥà¸Ńà¸ģ า +à¹Ĥà¸Ńà¸ģา ส +Ġ×Ķ ×ĵ×ijר +à¸Ļั à¹Īà¸Ļ +ز ر +нак о +íļ į +ãĤĤ ãģ¡ +ãĤĤãģ¡ ãĤį +ãĤĤãģ¡ãĤį ãĤĵ +اÙħ ت +عد اد +и нÑĭ +ÅĤy w +à¸Ħ à¸ĵะ +à¸Ĺ ะ +kt ör +×Ļ×Ĺ ×Ķ +Ġм е +Ġме ÑģÑı +׳×Ķ ×Ĵ +ĠÑģ ÑĥÑīеÑģÑĤв +à¸Ļ ัà¸Ļ +ÑĦ ÑĦ +ек ÑĤив +عÙĦÙĪÙħ ات +б Ñĥд +à¸Ļัà¸ģ à¸ĩาà¸Ļ +หà¸Ļà¹īา à¸Ĺีà¹Ī +ÙĤÙĬ ÙĤ +ãĤ· ãĥ³ +ãģ« éĸ¢ +×IJר ×Ĵ +ĠпÑĢ оÑĤ +ĠпÑĢоÑĤ ив +ĠìŀĪ ìĸ´ +ÙĤÙĬ ÙĤØ© +ìĹ ĩ +k ür +ãģ«ãģªãĤĬ ãģ¾ãģĹãģŁ +Ġде ÑıÑĤ +ĠдеÑıÑĤ елÑĮ +פ×ķר ×ĺ +à¸Ł à¹īา +à¹Ģ à¸ł +ĠавÑĤом аÑĤ +×ĸ ×Ļק +Ġold uk +ع اÙħ +ĠÑĤ оÑĢ +yrı ca +ê Ì +ãĤŃ ãĥ³ãĤ° +ãģ« ãģ¨ãģ£ãģ¦ +à¹Ģà¸ī à¸ŀ +à¹Ģà¸īà¸ŀ าะ +ãģ¯ ãģļ +×ŀ ×IJ×Ļ +สะ à¸Ķ +สะà¸Ķ วà¸ģ +ìľ¼ ë©° +à¸ģ ี +ภ¬ +Ġ×¢ ×ķש +à¸łà¸² ษา +à¸Ĺ ัà¸Ļ +ac akt +acakt ır +اع دة +ĠÑĥÑģл Ñĥг +ס ר×ĺ +×ķ×ŀ ×ķת +×Ķ ×ķר +×ŀ ×ķ×ij +×ŀ×ķ×ij ף +سÙĬ اس +اتÙģ اÙĤ +×Ķ צ׾ +Ùħؤ س +Ġp ó +Ġк ни +×Ļ׼ ×ķ׾ +à¹Ģหล ืà¸Ń +׼׾ ׼ +׳ ×ĸ +ÑĪи е +r ès +ĠاÙĦØŃ ÙĤ +лÑı ÑĢ +ห à¸į +หà¸į ิà¸ĩ +ר×Ĵ ×Ļש +à¹Ģส à¹īà¸Ļ +ש×ij ×ķף +ô tel +ап ÑĢ +апÑĢ имеÑĢ +اب ÙĦ +ĠÑĢаз виÑĤ +Ġп олÑĮз +ĠС еÑĢ +×ķ×ij ×Ļ +r óż +ìĭ Ń +ãĤ¯ ãĥĪ +ãģĹ ãĤĪãģĨ +à¸ģร ม +ØŃ ÙĥÙĪÙħ +à¹Ĥ à¸ļ +à¸Ĺ à¹īาย +ĠM á +ĠÑĤ Ñĭ +à¸Ħร ัว +ÑĢÑĥ б +ạ p +Ġm ÅĤ +ĠmÅĤ od +Ġgör Ã¼ÅŁ +Ġgeli ÅŁ +Æ°Æ¡ i +×ŀש ק +ÙĢÙĢ ÙĢÙĢ +รา ว +ãģĹãģ £ +ãģĹãģ£ ãģĭãĤĬ +ĠÐļ он +Ġk ê +à¹Ĥà¸Ĺ ร +èIJ½ ãģ¡ +åĩº ãģ¦ +ล ัà¸ģษ +Ġ×Ĵ ×ij×ķ×Ķ +ãĥĻ ãĥ« +ê±° ëĤĺ +ë§ IJ +×Ļ׾ ×ĵ×Ļ×Ŀ +ĠëĦ Ī +×ŀר ×Ļ +ร ส +ãĥŃ ãĥ³ +и ло +ноÑģÑĤÑĮ Ñİ +×ĸר ×Ĺ +п он +Ġ×Ķש ׾ +ê²ł ìĬµëĭĪëĭ¤ +Ġki ÅŁ +ĠÐļ и +ว ร +د اع +ÅŁ im +ÙĨ Ùij +в аÑĤ +را Ùĥ +ب اÙĦ +ид е +Ġ×Ķ×ŀ ×Ĺ +ìĸ µ +تÙģ اع +Ø£ ت +ëĬ ĺ +ש ×Ļת +ست Ùħر +ĠÑĦ ак +ĠاÙĦØ£Ùħ رÙĬ +ëŀ ¨ +اس Ùħ +Ġa ÄŁ +Ġç ev +Ùĥ ÙĪر +ãģķ ãģ¾ +Ġç öz +Ġر س +Äħ da +สà¸Ļ ุ +ãģĹãģ¦ ãģıãĤĮ +н Ñİ +leÅŁ me +ãĤª ãĥ³ +ãģ¨ ãģªãĤĬ +ava ÅŁ +×ĺ ×Ļ×ij +ØŃ Ø¶ +×ķצ ×IJ×ķת +ÙĨ ÙħÙĪ +ı t +ĠÑħ а +ĠÑħа ÑĢак +ĠÑħаÑĢак ÑĤеÑĢ +Ġd ÅĤ +ãĥĹ ãĥ© +à¸Ĭ ุม +à¹Ī à¸Ńà¸Ļ +×ķ×ij ׾ +Ñģ ол +×ĵ ×Ĵ +аÑĢ аÑĤ +n ivers +Ġgerçek leÅŁtir +ĠاÙĦ ÙĦÙĬ +ระ ยะ +ĠÙħ ختÙĦÙģ +Ġgö nder +Ùģ ار +do ÄŁ +doÄŁ an +ص ÙĦاØŃ +Ġyay ın +ãĥĨ ãĥ³ +รว à¸Ī +×Ļ×Ĺ ×Ļ×ĵ +ünk ü +ÑĨи алÑĮн +à¸ļ ู +ม ุ +h ä +Ø® Ùģ +å¢ Ĺ +å¢Ĺ ãģĪ +еÑĩ но +ĠاÙĦس ÙĨ +à¸Ĥ าว +im di +Ð « +à¸Ļà¸Ńà¸ģ à¸Īาà¸ģ +à¸ļา ล +ת ש +Ġdüzen le +мÑĭ Ñģл +ãģı ãģª +ż u +Ġwsp óÅĤ +Ġн аз +ınd aki +تر Ø© +ÅŁ ek +Ġö d +ĠÙĪ Ùĥ +Ġпозв олÑı +Ġת ×ķ׼ +ÙħÙĨ تج +ë§ ī +ĠاÙĦØ« ÙĦاث +аÑĨи Ñİ +ÙĪر ÙĪ +Ñĭв аеÑĤ +خص ص +ĠاÙĦÙģ ÙĦ +ĠاÙĦÙģÙĦ سطÙĬÙĨ +Ø¥ جر +إجر اء +اÙĨت Ø® +اÙĨتخ اب +ار ÙĬØ© +×ķ Ö +Ø¢ ÙĨ +×ŀ×¢ ×ķת +Ġм ал +Ġ×IJ ×Ĺ +à¸Ĺ à¹īà¸Ńà¸ĩ +ze ÅĽ +Ġë§Į ëĵ¤ +رÙĬ ع +äºĭ ãĤĴ +à¸ļริ หาร +׾ ×ŀ×Ļ×ĵ +Ġм Ñĥж +ت رÙĪ +ĠباÙĦ Ø¥ +פ ×Ļק +ز ÙħØ© +ĠÃ¶ÄŁ renc +ãĥ ¶ +اÙħ عة +ק×ij ×ķצ +×ŀ ׳×ķת +رÙĬ Ùħ +Ġо каз +ãģłãģij ãģ© +Ġh ız +Ġש ×IJת +ãĤ¢ ãĥ¼ +Ġmożli wo +ìĦ ¼ +ÙĪ اب +ог ÑĢаÑĦ +Ġعبد اÙĦ +ãĤĴ è¡Į +ب ÙĬÙĦ +ĠÄ° ç +ย าย +ĠÑĥ ÑĩаÑģÑĤ +ÑĦ еÑģÑģ +ÑĦеÑģÑģ иона +Ạ¤ +ÙĨ ÙĬÙĨ +عد ÙĦ +สร ร +دÙĬ ÙĦ +×ij ×Ļק +czy ÅĤ +ÑĢом е +Ġм ед +ìĻ Ķ +ãĥ© ãĤ¤ãĥ³ +ĠÑĤ еп +еÑĢ ÑĮ +i ÄŁi +в ели +ÑĢи ÑģÑĤ +ס ×ķפ +×ŀ׾ ×Ĺ +ĠاÙĦØ¥ ÙĨ +Ġ׾×Ķ ש +è¶Ĭ ãģĹ +ĠÑĢ Ñĭ +×ķ×IJ ר +رÙĩ اب +פ ×ķ×IJ×Ļ +ĠгоÑģ Ñĥд +ĠгоÑģÑĥд аÑĢ +ĠгоÑģÑĥдаÑĢ ÑģÑĤв +ĠاÙĦØ£Ùħ ÙĬر +Ùħ ج +à¹Ģหม าะ +ÑĢ ев +à¸Ĭี à¸ŀ +ãĥķ ãĥĪ +иÑĩ но +ĠاÙĦÙħ ؤ +Ġi ht +íħ ľ +د ÙĨÙĬ +ر ص +ла ÑģÑĤ +à¹Ģหล à¹Īา +ılı r +ร à¸ĵà¹Į +×ŀש ×Ļ×ļ +Ġd á»ĭ +Ø·Ùģ اÙĦ +×ĺ ×ķף +Ġ×ij ×Ļ׳ +ãģ¾ ãģ£ãģŁ +лож ениÑı +تØŃ Ø± +ب اØŃ +à¹Ģส ืà¹īà¸Ń +ãģĻ ãģĶ +lt ür +à¸ĩ าม +Ġt ü +ĠпÑĢ им +ĠпÑĢим ен +Ġhay at +ëĥ IJ +ëĭ Į +׳×Ļ ×ķ +вед ен +ìħ ¨ +à¸Ī ัย +à¸ģà¹Ī à¸Ń +Ġв од +оÑģÑĤ оÑı +н аÑĤ +à¹ģ หล +سÙħ ÙĬ +à¸Ķำ à¹Ģà¸Ļ +à¸Ķำà¹Ģà¸Ļ ิà¸Ļ +w ód +ö yle +ãĥĢ ãĤ¤ +ÑĪи й +меÑī ен +ãģĹãģ¾ ãģĨ +ãĥī ãĥ© +ÙĪض ØŃ +à¸Ńà¸Ļ ุ +ĠاÙĦ اجتÙħاع +laÅŁ ma +à¸Ħ à¸Ńà¸Ļ +×ŀר ×Ļ×Ŀ +ÙĨ اÙħج +שר ×ķת +اÙĦ Ø£ +Ġksi Äħż +Ġа н +ÑĢаР¹ +اÙĩر Ø© +×ŀ×ĵ ×Ķ +ä¸Ģ ç· +ä¸Ģç· Ĵ +ä¸Ģç·Ĵ ãģ« +ÑĢиÑĤ оÑĢ +d ıkl +à¹ģ à¸ĸ +à¹ģà¸Ĥ à¹Īà¸ĩ +екÑĤ оÑĢ +×ŀס ×¢ +ÑĢак ÑĤи +u ÄŁu +×ķ×ij ת +สู à¸ķร +ĠçalÄ±ÅŁ m +ĠçalÄ±ÅŁm alar +Ġа на +ãĥĽ ãĥ¼ãĥł +Ġböl üm +Ġب ص +ол оÑģ +ĠìķĬ ëĬĶ +à¹Ī ะ +ÙĪ تر +ä¹ Ĺ +ست خداÙħ +פ×Ļ ×Ļס +פ×Ļ×Ļס ×ij +פ×Ļ×Ļס×ij ×ķק +Ġк ÑĢаÑģ +ли к +رÙĬ ØŃ +×ŀש ׾×Ķ +à¹Ģย ีà¹Īย +à¹Ģยีà¹Īย ม +в иÑģ +ом н +ÄŁ un +ãĥŃ ãĥ¼ãĥ³ +Ø£ تÙĬ +à¸ķร ี +çĶ³ ãģĹ +تÙħ ر +ìĹ ĪìĬµëĭĪëĭ¤ +ĠÙĪ غÙĬر +red ni +ĠاÙĦص Ùģ +Ġна ÑģÑĤоÑı +ĠнаÑģÑĤоÑı Ñī +à¸ķ รา +ĠÑĥÑģл ов +ĠÑĥÑģлов иÑı +ÑĨ еп +×Ķ ×Ĺ׾×ĺ +Ø· ÙĬع +ĠB akan +ĠاÙĦ رÙĪ +илÑĮ но +Ġм еÑĤ +à¸Ķ à¸Ńà¸ģ +ãģĭãĤī ãģªãģĦ +Ġпо ÑģÑĤоÑı +ĠпоÑģÑĤоÑı н +ĠÑĩ аÑģ +ü c +wr ó +б ÑĥÑĢ +ãĥIJ ãĥĥãĤ¯ +ãĥ©ãĥ³ ãĥī +Ġо гÑĢ +สั à¸į +สัà¸į à¸įา +มั à¹Īà¸Ļ +à¸Ħ à¸Ńม +al ık +Ġн ед +üm üz +ĠÅĽ wie +é rio +×Ļ×IJ ×Ķ +دÙħ ات +ı rl +ĠоÑĤ з +ĠоÑĤз Ñĭв +ä»ĺ ãģį +Ġkaż de +мин иÑģÑĤ +ãĤ° ãĥ« +ë° ĸ +ез н +اÙĦ Ùģ +Ġש ק׾ +Ùħ ض +ãĥĿ ãĥ¼ãĥĪ +ÙħÙĨ ت +ÙĤÙĬ اÙħ +Ø´ ÙĨ +×Ļר ×ķ×¢ +ãĤŃãĥ£ ãĥ³ +доÑĢ ов +×ŀ ×Ļת×Ļ +ÙĪÙĦ ÙĪج +Ùĥ اÙģ +ĠÑĢаз лиÑĩ +иÑĤ еÑĤ +н олог +ลà¸ĩ à¸Ĺุà¸Ļ +Ġyak laÅŁ +ãĥ¬ ãĤ¤ +ê²ł ëĭ¤ +æ±Ĥ ãĤģ +رÙĪ Ùģ +Ġí Ĭ +ĠíĬ ¹ +ãģ£ ãģıãĤĬ +à¸Ħวาม à¸Ħิà¸Ķ +×Ķ ×Ļס×ĺ +Ø¥ ÙĤ +ãģ¦ ãģĦ +à¹Ĥ à¸Ĭ +ĠBü yük +ĠФ едеÑĢ +ÑĨи н +ÑĢов а +ĠاÙĦ اÙĤتصاد +Ġch á +à¸ĺ าà¸Ļ +ë¥ ł +à¹Ħ à¸ķ +ÃŃ pio +Ùĭ ا +Ġоб Ñıз +Ùĩ ج +Ġì¤ij ìļĶ +ãģ® ãģ§ãģ¯ãģªãģĦ +بار اة +ãĤ¤ ãĥ« +Ġн оÑĢм +á»ī nh +m ö +mö glich +ÑĨи п +ãĤ¢ ãĤ¯ +×Ķ ×Ļ +ÑĨи алÑĮно +ĠÅĽ wi +ت ÙĤ +ĠÑģÑĤо им +بÙĬ عÙĬ +Ġ׾ ש×ŀ +г лÑı +глÑı д +ãģ¦ ãģıãĤĮ +ÄĻd zi +à¸Ĥ ั +à¸Ĥั à¹īà¸Ļ +Ø· ÙĤ +ĠìĹ Ń +ãģ£ãģ¦ãģĹãģ¾ ãģĨ +ĠdeÄŁer l +ĠdeÄŁerl endir +Ġü lk +Ġмн ог +๠ĭ +ë¿ IJ +ĠУ кÑĢа +ÄŁ ini +Ġбез оп +Ġбезоп аÑģ +à¸Ńà¸Ńà¸ģ à¹ģà¸ļà¸ļ +Ø§Ø ¸ +ØŃد اث +л еÑĢ +×Ļ× ¥ +×Ļ׳×ĺר ׳×ĺ +lar ınız +ØŃÙĬ ØŃ +ż eli +à¸Ń ัà¸ĩ +à¸Ńัà¸ĩ à¸ģ +à¸Ńัà¸ĩà¸ģ ฤษ +ĠоÑĤ лиÑĩ +ั ส +ëŀ į +ож но +ãĤ¹ ãĥĿ +ĠÑħ оÑĩ +Ġк ап +еÑĩ ен +ØŃÙĦ Ø© +ÙĬا Ùĩ +на л +×ķצ ר×Ļ×Ŀ +Ġk ald +åĥ į +ĠاÙĦØ´ خص +Ġз на +Ġwz gl +ż ycz +ê° Ŀ +à¸ŀ ลัà¸ĩ +íģ ¼ +Ġö l +Ġb ụ +Ø´ Ùĩر +Ġз ам +Ġд ев +×Ļ×ĺ ת +تعÙĦ ÙĤ +ÙĪÙħ Ø© +ãĤĴ ä½ľ +ãģį ãģ¦ +í ĥĿ +ras ında +ãĤĴ æİ¢ +ĠÙħ باشر +راج ع +Ġв озд +ÙħØŃ Ø§ +×ķש ר +ĠиÑģÑĤ оÑĢ +ม ัà¸ģ +t ıģ +Ø« ار +تر ÙĨت +à¹ģà¸Ĥ à¹ĩ +à¹ģà¸Ĥà¹ĩ à¸ĩ +п оÑĩ +Ġ×ij ×IJ×ķת +ë¯ Ģ +ëĿ¼ ëıĦ +à¸Ĭ ัà¸Ķ +ส à¸ķà¹Į +ãĥĭ ãĥĥãĤ¯ +ид енÑĤ +Ġг ÑĢÑĥпп +ت Ø® +Ạł +ย ืà¸Ļ +ย ัà¸Ļ +ó ry +T Ãľ +ãģĹ ãĤĥ +ĠпÑĢов ед +лÑı еÑĤ +Ùħ Ø® +ย à¸Ńม +×Ľ×ł×¡ ת +ĠاÙĦÙħ ÙĨت +Ġol mad +ר׼ ×ĸ×Ļ +Ġв ÑģÑĤÑĢ +ĠиÑģ Ñģлед +ÑĤвеÑĢ ж +بد ÙĪ +еÑĢ ÑĤ +ï» · +± ħ +สัม à¸ŀัà¸Ļà¸ĺà¹Į +ิ à¹Īà¸Ļ +צ ×Ļ×ij +wiÄĻ t +Ġì° ¸ +Ġz wiÄħz +سب ÙĪع +ãĥĥ ãĤ° +à¸Ľà¸¥ à¸Ńà¸Ķ +à¸Ľà¸¥à¸Ńà¸Ķ à¸łà¸±à¸¢ +ãĤĤ ãĤĬ +ÙĤد س +Ġspr z +Ġsprz eda +Ġist edi +Ġk hu +Ġд ен +Ġko ÅĦ +Ġ×ij ×Ĺ×Ļ +à¹Ģà¸Ĺ à¹īา +×ķס ×Ļ×£ +ãĥĭ ãĥ¥ãĥ¼ +ĠпÑĢед оÑģÑĤ +ĠпÑĢедоÑģÑĤ ав +à¹Ĥ à¸Ł +é v +ĠاÙĦص ØŃ +صØŃ Ø§Ø¨ +à¹Ģà¸Ī à¹ĩà¸ļ +вл ек +วั à¸ķ +à¸ĸ ุ +ãģĵãģ¨ãģĮãģ§ãģį ãģ¾ãģĻ +ÙĤÙĬ ÙĤÙĬ +×ķ׊ר +Ñĭ ÑĪ +ĠоÑĤ но +ĠоÑĤно ÑĪ +об илÑĮ +Ùģ ØŃ +ı nt +ınt ı +Ġ׾ ×ij×ĵ +í İĺìĿ´ì§Ģ +ãĥĬ ãĥ« +ĠÙħ ساء +×Ļ×ĺ ×ij +ÑĮ еÑĢ +ëĦ · +Ñĭ ÑĤа +ĠоÑĩ еÑĢ +à¸Ķ ืà¹Ī +à¸Ķืà¹Ī ม +ĠN gh +ت عب +ÙĦاÙĤ ات +×ķ׾×ķ×Ĵ ×Ļ×Ķ +ĠìĿ´ ê²ĥ +Ġ×Ķ ×ijר +ìľ µ +à¹Ģà¸Ħล ืà¹Īà¸Ńà¸Ļ +Ùĩ Ø© +à¸Īำ à¹Ģà¸Ľà¹ĩà¸Ļ +å¤ī ãģĪ +wi ÅĽcie +ch od +chod zÄħ +в ÑĢо +×ŀ×Ĺ ×Ļר +Ġy ı +Ġyı ll +ì¡ Į +à¹Ħ หว +ãģªãģı ãģª +Ġзав иÑģ +ĠìĺĪ ìĪĺ +Ùģ Ø° +ủ ng +à¸ŀุ à¸Ĺà¸ĺ +з н +lay an +ãĤ ¡ +à¸ģà¹ĩ à¸ķาม +ĠsaÄŁ lam +ร à¸ĵ +ĠÑģ иÑĤ +ĠÑģиÑĤ Ñĥ +ĠاÙĦت ÙĨ +×Ķ ×ĸ +ĠØ· ÙĪÙĬÙĦ +ta ÅĤ +Ġgö rd +å¤ī ãĤı +ëĥ ¥ +à¸Ħà¹Ī à¸Ńย +×IJ ×ķ×ĺ +ëħ IJ +ãĥ©ãĥ³ ãĤ¹ +วั à¸Ĵ +วัà¸Ĵ à¸Ļ +Ġol uÅŁ +פע ×ķ׾ +Ġszczeg óÅĤ +à¸Ħา สิ +à¸Ħาสิ à¹Ĥà¸Ļ +pow ied +ĠÑĤ еб +หà¸Ļ à¹Īวย +Ġм ил +ØŃ Ùĥ +à¸Ĺ à¸Ķ +ĠмаÑĤ еÑĢиал +ÅĤ ow +à¹Ģà¸ģ ีย +ĠÑģов еÑĢ +ãĤ © +à¸Ľ ริ +Ġи Ñİ +наÑĩ ен +ÑĢен д +mu ÅŁtur +ĠпÑĢод Ñĥк +з д +Ñı ÑĤи +ÑıÑĤи Ñı +à¹Ģม ีย +رات ÙĬج +Ġam acı +ש ×ķ׾ +ש×ķ׾ ×Ĺ +สะ à¸Ńา +สะà¸Ńา à¸Ķ +פ×Ĵ ×¢ +عب Ø© +d ın +íħ Ķ +Ġ×ŀש ×Ĺק +Ġfi yat +Ġз аÑı +ĠзаÑı в +à¹Ĥ หล +à¹Ĥหล à¸Ķ +à¸ģรุà¸ĩ à¹Ģà¸Ĺà¸ŀ +צ×Ļ ×Ļף +ìļ ± +Ùħ ب +Ùħب اد +land ır +Ġв еÑģÑĮ +Ġh ük +ĠÐĴ оз +ÑĩиÑĤ Ñĭва +ว ล +×ķצ ×¢ +à¸Ĥà¸ĵะ à¸Ĺีà¹Ī +ĠaÅŁ aģı +׾×IJ ×ķ×ŀ×Ļ +tr zym +Ã¤ÃŁ ig +owo ÅĽci +ãģĿ ãĤĤ +Ġroz wiÄħz +ĠgÅĤ ówn +м онÑĤ +×ŀ ×ķ×ŀ +ĠÑģÑĤ ан +ÙĦا ÙĤØ© +p rowad +prowad zi +ĠÑģоÑģÑĤ оÑı +×Ļ×IJ ×ķת +r ı +g ı +ãĥij ãĥij +Ġна лиÑĩ +×Ķ צע +Ġ׳ ×Ķ +à¸Ħ ัà¸ļ +ع راض +и ж +Ùĩ ائÙĬ +ãĤī ãģı +ож еÑĤ +Ġоб оÑĢ +ĠобоÑĢ Ñĥд +Ø£ سÙĦ +à¹ĩ à¸Ķ +ÑĢÑĥ ÑĤ +دÙĬ ÙħÙĤ +دÙĬÙħÙĤ را +Ġjest e +×ķ×ķ ×Ļר +×ij×ĵ ×Ļק +деÑĢж ива +ãģĬ ãģı +ewn ÄĻtr +ewnÄĻtr zn +à¸ŀ ฤ +Ġ×IJ ×ķ×Ķ +ת×Ĺ ×ķש +Ġz ob +д Ñĥм +ĠÑģ Ñĭ +ÙĬر ا +ĠwiÄĻ ks +à¹ģà¸ķà¸ģ à¸ķà¹Īาà¸ĩ +lar aras +lararas ı +íĺ Ģ +ëī ´ +×ķ×Ĵ ׾ +ĠоÑĤ меÑĤ +ĠÑĢ ан +ت ÙĥÙĦ +иÑĤелÑĮ н +à¸Ľà¸£à¸° วั +à¸Ľà¸£à¸°à¸§à¸± à¸ķิ +ìŀ ĸ +мож но +pie czeÅĦ +pieczeÅĦ st +ëª » +ìĬ ¨ +×ŀס ×ŀ +á» ¦ +ศ ิ +ศิ ล +ศิล à¸Ľ +ĠÅļ w +ãĥĥ ãĤ·ãĥ§ãĥ³ +unit Ãł +Ġmiesz ka +Ġmieszka ÅĦ +pr zed +przed si +przedsi ÄĻb +przedsiÄĻb ior +à¸Ľà¸£à¸° สิà¸Ĺà¸ĺิ +à¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺิ à¸łà¸²à¸ŀ +ย à¹Ī +ìķ Ļ +รว à¸Ķ +รวà¸Ķ à¹Ģรà¹ĩว +å½ĵ ãģŁãĤĬ +äl le +Ñĥ еÑĤÑģÑı +ã n +ëł µ +th è +ãĤĴ åĪ©çĶ¨ +ì µľ +íĵ ¨ +à¸Ĺ ัà¸ļ +า à¸Ħม +ãģ ĩ +ëĤ Į +à¹Ģà¸Ľà¸¥ à¹Īา +â ¦ +ë ¾ +ê Ģ +ê ĩ +â ¡ +ðŁ Ł +ã IJ +â º +á Ń +á Ļ +á ĵ +á ² +ðĵ ı +á ¬ +â ¯ +ä ¨ +ê Ŀ +ê « +ð ij +ðĵ ĥ +ðĿ ħ +< unk + + + +Ġع ÙĦÙī +Ġm á»Ļt +Ġv Ỽi +Ġng Æ°á»Ŀi +ĠØ¥ ÙĦÙī +Ġnh ững +Ġth á»ĥ +Ġ×IJ ×ķ +Ġ×¢ ×Ŀ +ا Ùĭ +Ġ à¹ģละ +ĠÙĦ ا +Ġnh Æ° +ĠاÙĦت ÙĬ +Ġ×Ķ ×ķ×IJ +ĠÄij ến +ĠØ£ ÙĪ +Ġv á»ģ +ĠlÃł m +Ġs ẽ +Ġc Å©ng +Ġ ợ +ĠÄij ó +Ġnhi á»ģu +Ġt ại +Ġtr ên +Ġ×Ĵ ×Ŀ +Ġnh Ãł +Ġ׼ ×Ļ +Ġs á»± +ĠÄij ầu +Ġb á»ĭ +ĠÙĩ ذا +Ġnh ất +Ġph ải +Ġhi á»ĩn +Ġdụ ng +ĠÄij á»Ļng +ĠاÙĦÙĦ Ùĩ +ĠØ Į +ĠÙĥ ÙĦ +Ġvi á»ĩc +Ġn Äĥm +Ġth ì +Ġh á»įc +ĠÙĪ ت +t é +Ġا ÙĨ +Ġt ôi +Ġ×IJ ׳×Ļ +Ġ׾ ×Ļ +Ġ×ŀ ×ķ +Ġng Ãły +Ġn Æ°á»Ľc +Ġ×Ķ ×Ļ×IJ +Ġ×IJ ×Ļ +Ġh Æ¡n +ĠÙĩ Ø°Ùĩ +ĠÙĪ ÙĬ +ĠاÙĦ Ø°ÙĬ +Ġ×ķ ×ŀ +Ġgi á +Ġnh ân +Ġch ÃŃnh +Ġm ình +ĠÐĿ а +Ġth ế +Ġ×Ļ ×ķתר +Ġ×IJ ×Ŀ +Ġn ên +Ġh ợ +Ġhợ p +Ġc òn +ĠÙĩ ÙĪ +Ġc Æ¡ +Ġr ất +ĠVi á»ĩt +Ġب عد +Ġש ×Ļ +Ġth á»Ŀi +Ġc ách +ĠÄij á»ĵng +Ġн о +Ġtr Æ°á»Ŀng +Ø Ł +ĠÄij á»ĭnh +ĠÄiji á»ģu +×Ļ ×Ļ×Ŀ +Ġth á»±c +n ın +Ġh ình +Ġn ói +Ġc ùng +Ġ×Ķ ×Ķ +ĠØ¥ ÙĨ +Ġ×IJ ×ij׾ +Ġnh Æ°ng +Ġbi ết +Ġж е +Ġch úng +ĠÄij ang +ĠØ° ÙĦÙĥ +Ġl ên +Ġkh ách +Ġn Ãło +Ġs á»Ń +Ġkh ác +Ġë° ı +Ġl ý +×Ļ ×Ļ +ĠÄij ây +Ġ׾ ×ŀ +Ġc ần +Ġtr ình +Ġph át +ãģ« ãĤĤ +п о +Ġn Äĥng +Ġb á»Ļ +Ġv ụ +ĠÄij á»Ļ +Ñĩ е +Ġnh áºŃn +Ġtr Æ°á»Ľc +Ġ×¢ ×ĵ +Ġh Ãłnh +ĠØ® ÙĦاÙĦ +Ġl ượng +Ġc ấp +Ġtá» ± +Ġv ì +Ġt Æ° +Ġch ất +Ġ׼ ×ŀ×ķ +Ġg ì +Ġש ׳ +Ġt ế +ת ×ķ +Ġnghi á»ĩp +Ġm ặt +ĠÙĥ Ùħا +Ġ×ij ×Ļף +Ġר ק +Ġth ấy +Ġmá y +ĠÙģ Ùī +Ġd ân +Ġ×IJ ×Ĺ×ĵ +Ġt âm +Ġ׼ ×ļ +Ġ׾ ×ķ +в о +Ġt ác +Ġto Ãłn +ĠÙĪ Ùħ +Ġk ết +Ġ หรืà¸Ń +ĠÙĪاÙĦ Ùħ +ĠÄiji á»ĥm +Ġ×ĸ ×ķ +Ġ×ij ×ķ +׼ ×ķת +Ġh á»Ļi +Ġb ằng +ت Ùĩا +Ġ׼ ×ĵ×Ļ +Ġ×Ķ ×Ŀ +Ġxu ất +ĠÙĤ د +Ġb ảo +Ġt á»ijt +Ġt ình +ĠÙĩ ÙĬ +ĠÄij á»iji +Ġthi ết +Ġhi á»ĩu +Ġti ếp +Ġt ạo +ת ×Ķ +Ġch ủ +o ÅĽÄĩ +Ġgi ú +Ġgiú p +Ġà ½ +Ġqu ả +Ġlo ại +Ġc ô +Ġà ´ +Ġô ng +Ġ×Ķ ×ķ +ĠاÙĦÙĬ ÙĪÙħ +ĠtÃŃ nh +г а +Ġph òng +Ġ Äĥn +Ġع اÙħ +Ġv á»ĭ +lar ını +r ÃŃa +Ġt Ỽi +ĠÄij Æ°á»Ŀng +Ġgi Ỽi +Ġb ản +Ġc ầu +Ġnhi ên +Ġb á»ĩnh +Ġth Æ°á»Ŀng +Ġ×IJ ×Ļף +ĠÄij á»ģ +Ġh á»ĩ +Ġ×Ļש ר×IJ׾ +Ġqu á +ĠÐĹ Ð° +ãģ® ãģ§ãģĻãģĮ +ĠÐŁ ÑĢи +Ġph ần +ĠÙĪ ÙĦا +ĠlỼ n +Ġtr á»ĭ +Ġcả m +Ġм о +Ġd ùng +ĠاÙĦ Ùī +ĠعÙĦÙĬ Ùĩ +ĠìŀĪ ìĬµëĭĪëĭ¤ +ÙĬ ÙĤ +ĠÙĤ بÙĦ +Ġho ặc +ĠØŃ ÙĬØ« +Ġ à¸Ĺีà¹Ī +Ġغ ÙĬر +ĠÄij ại +Ġsá»ij ng +нÑĭ ми +Ġth ức +Ġפ ×Ļ +ĠÄiji á»ĩn +ãģª ãģĭãģ£ãģŁ +Ġgi ải +Ġv ẫn +Ġи Ñħ +Ġö nce +Ġv áºŃy +Ġmu á»ijn +Ġ ảnh +à¹ĥà¸Ļ à¸ģาร +ĠQu á»ijc +Ġk ế +׳ ×IJ +Ġס ×Ļ +Ġy êu +ãģ® ãģĭ +ĠÄij ẹ +ĠÄijẹ p +Ġch ức +Ġy ıl +ĠTür kiye +d é +ĠÙĤ اÙĦ +Ġd á»ĭch +ĠolduÄŁ u +Ġch á»įn +Ġت Ùħ +หà¸Ļ ึà¹Īà¸ĩ +ãģķãĤĮ ãģŁ +Ġph áp +ìĽ Ķ +Ġti á»ģn +ãģĹ ãģ¾ãģĹãģŁ +Ġש ׾×IJ +ÙĦ Ø© +Ġ׾פ ׳×Ļ +Ġ×ij ×Ļת +ĠH Ãł +ĠØŃ Øª +ĠØŃت Ùī +Ġ×¢ ×ķ×ĵ +Ġn ó +Ġth áng +à¹Ģลืà¸Ń à¸ģ +ר ×Ķ +Ġt Äĥng +Ġcá i +Ġtri á»ĥn +Ġ×IJ×ķת ×ķ +ìłģ ìĿ¸ +ĠC ông +Ġ׾×Ķ ×Ļ×ķת +Ġг ода +и Ñİ +Ġب عض +Ġ à¸ģาร +èī¯ ãģĦ +ÙĪ ت +Ġli ên +ĠÐĿ о +ĠÐĿ е +çļĦ ãģª +ĠÙħ ت +ĠÑĤак же +ĠкоÑĤоÑĢ Ñĭе +Ġ×Ļ ×ĵ×Ļ +Ġtr á»įng +ãĤµ ãĤ¤ãĥĪ +ìłģ ìľ¼ë¡ľ +Ġt áºŃp +Ġש ׾×Ļ +íķĺ ê²Į +Ġt Ãłi +ĠÐ ¯ +Ġr á»ĵi +ا Ùĥ +Ġth Æ°Æ¡ng +Ġ×Ķ ×ĸ×Ķ +ĠÙĪ ÙħÙĨ +à¸Ĺีà¹Ī มี +Ġcu á»Ļc +Ġbü yük +ãģ¨ ãģĭ +Ġ×ij ×Ļ×ķתר +Ġl ần +Ġgö re +Ġtr ợ +Ġ×ĺ ×ķ×ij +ÑĤÑĮ ÑģÑı +Ġth á»ijng +Ġ׼ ש +Ġti êu +Ġ×ŀ×IJ ×ķ×ĵ +Ø Ľ +k Äħ +Ġ à¹ĥà¸Ļ +Ġv ấn +Ġש ׾×ķ +ĠÄij á»ģu +Ùģ ت +Ġê²ĥ ìĿ´ +Ġh óa +ĠاÙĦع اÙħ +ĠÙĬ ÙĪÙħ +к ой +Ġbi á»ĩt +ÑģÑĤ о +Ġ×Ķ ×Ļ×ķ +à¸Ĺีà¹Ī à¸Īะ +Ġ×ĵ ×Ļ +Ġ×IJ ×ļ +Ġá n +ص ÙĪر +Ġtr ÃŃ +ĠÐŁÑĢ о +Ġl á»±c +ãģĹãģ¦ ãģĦãģ¾ãģĻ +Ġb Ãłi +Ġ×ĸ ×IJת +Ġb áo +à¸ļ à¸Ļ +ĠëĮĢ íķľ +Ġti ế +Ġtiế ng +Ġb ên +ãģķãĤĮ ãĤĭ +s ión +Ġt ìm +×¢ ×ķ +m é +ни Ñı +ãģ» ãģ© +Ġà¹Ģà¸ŀ ราะ +ب Ø© +Ġë¶ Ħ +Ġ×IJ ×ĸ +à¸Ĺ à¹Īาà¸Ļ +ת ×Ŀ +Ġth êm +Ġho ạt +y ı +×ĸ ×ķ +Ġgi á»Ŀ +Ġb án +à¸Ĥ าย +Ñĩ а +Ġ à¹Ĩ +ĠاÙĦÙħ ت +ĠоÑĩ енÑĮ +Ġb ất +Ġtr ẻ +ÑĤ ÑĢ +ĠØ£ ÙĨÙĩ +ĠØ« Ùħ +Ġ׼ ×ŀ×Ķ +Ġkh ó +Ġr ằng +ĠÙĪ ÙģÙĬ +ни й +Ġho Ãłn +t ó +Ġ×IJ שר +ĠìĥĿ ê°ģ +Ñģ а +Ġ׼ ×ijר +ĠÑįÑĤ ом +lar ının +Ġch Æ°a +з и +Ġd ẫn +ĠÐļ ак +ج ÙĪ +ĠбÑĭ ло +ĠÙĬ ت +n ı +ÅĤ am +ĠÙĪÙĩ ÙĪ +×ij ×ķ +п и +ר ת +Ġqu á»ijc +ж д +ĠÄij Æ¡n +Ùĥت ب +Ġm ắt +ระ à¸ļ +ระà¸ļ à¸ļ +ĠÙĥ اÙĨت +Ġth ân +สิà¸Ļ à¸Ħà¹īา +×Ĵ ×Ļ +Ġph Æ°Æ¡ng +à¹Ħมà¹Ī à¹Ħà¸Ķà¹ī +ĠìĦ ± +ĠC ác +Ġ×Ķ×ŀ ×ķ +ĠÑĤ ем +Ġ×ĵ ×ķ +à¸Ńะ à¹Ħร +Ġv Äĥn +ãģª ãģ®ãģ§ +ĠN á»Ļi +Ġ×¢ ×ķ +ãĤīãĤĮ ãĤĭ +Ġs áng +Ġgö ster +ãģĵãģ¨ ãĤĴ +Ġtaraf ından +Ġм а +ĠпоÑģл е +Ġ׳ ×Ļת +Ġ׳×Ļת ף +Ġл еÑĤ +Ġ׾ ׳×ķ +Ñģ Ñģ +Ġ×Ļ ×ķ +п е +ĠÙĪ ÙĦÙĥ +ĠÙĪÙĦÙĥ ÙĨ +Ġngo Ãłi +ĠÄij á»ĭa +r zÄħd +dz iaÅĤ +ĠÙħ ر +иÑĤÑĮ ÑģÑı +Ġ×IJ×Ĺר ×Ļ +Ġ׾ ׼׾ +à¸Ĥ à¹īà¸Ńม +à¸Ĥà¹īà¸Ńม ูล +Ġб ол +Ġбол ее +جÙħ ع +л еÑĤ +Ġl á»ĭch +ĠÙħ Ø«ÙĦ +Ġ그리 ê³ł +Ġth ứ +ĠdeÄŁ il +ÙĪ ØŃ +Ġש׾ ×ļ +ĠÙħ ØŃÙħد +Ġn ếu +ĠÄij á»ķi +Ġv ừa +Ġm á»įi +Ġо ни +Ġl úc +ĠÙĬ ÙĥÙĪÙĨ +ì§ Ī +Ġש׾ ׳×ķ +ĠÐĶ о +Ġש ׳×Ļ +ล ิ +×IJ פשר +Ġs ức +ê¶ Į +Ġ ứng +à¹Ħมà¹Ī มี +Ø·ÙĦ ب +ĠÑĩ ем +Ġch uyên +Ġth ÃŃch +Ġ×ķ ×Ļ +íķ © +ĠÙħ صر +д о +ĠÄij ất +Ġch ế +à¸Ĭ ืà¹Īà¸Ń +Ġìĭ ł +ĠØ¥ ذا +Ġر ئÙĬس +Ġש ×Ļש +Ġgiả m +Ñģ ка +lar ında +Ġs ợ +ĠtÃŃ ch +ĠÙĦ ÙĥÙĨ +Ġب Ùħ +×¢ ×ķ×ij +×¢×ķ×ij ×ĵ +ÅĤÄħ cz +ları na +Ġש ×Ŀ +ĠÙĦ ت +Ġש×Ķ ×ķ×IJ +t ów +Ġëĭ¤ 른 +ĠØ£ Ùĥثر +ãģ® ãģ§ãģĻ +׼ ×Ļ×Ŀ +ĠolduÄŁ unu +ãģĭ ãģª +ãĤĤ ãģĨ +ÙĬ ØŃ +Ġnh ìn +Ġngh á»ĩ +ãģ«ãģª ãģ£ãģ¦ +п а +Ġquy ết +ÙĦ ÙĤ +t á +Ġlu ôn +ĠÄij ặc +Ġ×IJ ר +Ġtu á»ķi +s ão +ìĻ ¸ +ر د +ĠبÙĩ ا +Ġ×Ķ×Ļ ×ķ×Ŀ +×ķ ×ķ×Ļ +ãģ§ãģĻ ãģŃ +ĠÑĤ ого +Ġth ủ +ãģĹãģŁ ãģĦ +ر ÙĤ +Ġb ắt +г Ñĥ +Ġtá» Ń +ÑĪ а +Ġ à¸Ľà¸µ +Ġ×Ķ×IJ ×Ŀ +íı ¬ +ż a +Ġ×IJת ×Ķ +Ġn á»Ļi +Ġph ÃŃ +ĠÅŁek ilde +Ġl á»Ŀi +d ıģı +Ġ׼×IJ ף +Ġt üm +Ġm ạnh +ĠM ỹ +ãģĿ ãĤĵãģª +Ġnh á»ı +ãģª ãģĮãĤī +Ġb ình +ı p +à¸ŀ า +ĠÄij ánh +ĠÙĪ ÙĦ +ר ×ķת +Ġ×IJ ×Ļ×ļ +Ġch uyá»ĥn +Ùĥ ا +ãĤĮ ãĤĭ +à¹ģม à¹Ī +ãĤĪ ãģı +ĠÙĪ ÙĤد +íĸ Īëĭ¤ +Ġn Æ¡i +ãģ«ãĤĪ ãģ£ãģ¦ +Ġvi ết +Ġà¹Ģà¸ŀ ืà¹Īà¸Ń +ëIJĺ ëĬĶ +اد ÙĬ +ĠÙģ Ø¥ÙĨ +ì¦ Ŀ +ĠÄij ặt +Ġh Æ°á»Ľng +Ġx ã +Ġönem li +ãģł ãģ¨ +Ġm ẹ +Ġ×ij ×Ļ +Ġ×ĵ ×ijר +Ġv áºŃt +ĠÄij ạo +Ġdá»± ng +ĠÑĤ ом +ĠÙģÙĬ Ùĩا +Ġج ÙħÙĬع +Ġthu áºŃt +st ÄĻp +Ġti ết +Ø´ ÙĬ +Ġе Ñīе +ãģĻãĤĭ ãģ¨ +ĠmÃł u +ĠÑįÑĤ ого +Ġv ô +ĠÐŃ ÑĤо +Ġth áºŃt +Ġn ữa +Ġbi ến +Ġn ữ +Ġ׾ ׼×Ŀ +×Ļ ×Ļף +Ġس ت +ĠÐŀ ÑĤ +Ġph ụ +ê¹Į ì§Ģ +Ġ׾ ×ļ +Ġk ỳ +à¹ĥ à¸Ħร +Ġg ây +ĠÙĦ ÙĦÙħ +Ġtụ c +ت ÙĬÙĨ +Ġtr ợ +Ġ׾ פ×Ļ +Ġb á»ij +ĠÐļ а +ĠÄij ình +ow Äħ +s ında +Ġkhi ến +s ız +Ġк огда +ס ׾ +ĠбÑĭ л +à¸Ļ à¹īà¸Ńย +обÑĢаР· +Ġê²ĥ ìĿ´ëĭ¤ +ëĵ¤ ìĿĢ +ãģ¸ ãģ® +Ġà¹Ģม ืà¹Īà¸Ń +Ġph ục +Ġ׊׾ק +Ġh ết +ĠÄij a +à¹Ģà¸Ķà¹ĩ à¸ģ +íĺ ķ +l ÃŃ +ê¸ ī +Ġع دد +ĠÄij á»ĵ +Ġg ần +Ġ×Ļ ×ķ×Ŀ +Ġs Ä© +ÑĢ Ñıд +Ġquy á»ģn +Ġ×IJ ׾×IJ +Ùĩ Ùħا +׳ ×Ļ×Ķ +׾ ×ķת +Ġ×Ķר ×ij×Ķ +Ġti ên +Ġal ın +Ġd á»ħ +人 ãģĮ +но Ñģ +л ÑģÑı +ĠÄij Æ°a +ส าว +иÑĢов ан +Ġ×ŀס פר +×Ĵ ף +Ġki ến +ĠÐ ¨ +p é +б Ñĥ +ов ой +б а +ĠØ¥ ÙĦا +×IJ ׾×Ļ +Ġx ây +Ġb ợi +Ġש ×ķ +人 ãģ® +ק ×Ļ×Ŀ +à¹Ģà¸Ķ ืà¸Ńà¸Ļ +Ġkh á +Ġ×ķ ׾×Ķ +×ĵ ×ķת +Ġ×¢ ×ij×ķר +Ġبش ÙĥÙĦ +ĠÙĩÙĨا Ùĥ +ÑĤ ÑĢа +Ġ íķĺëĬĶ +ร à¸Ńà¸ļ +owa ÅĤ +h é +Ġdi á»ħn +Ġ×Ķ ׼׾ +ĠØ£ س +Ġch uyá»ĩn +ระ à¸Ķัà¸ļ +ĠNh ững +Ġ×IJ ×Ĺת +ĠØŃ ÙĪÙĦ +л ов +׳ ר +Ġ×ķ ׳ +Ġch Æ¡i +Ġiç inde +ÑģÑĤв Ñĥ +Ġph á»ij +ĠÑģ Ñĥ +ç§ģ ãģ¯ +Ġch ứng +Ġv á»±c +à¹ģ à¸Ń +Ġl áºŃp +Ġtừ ng +å°ij ãģĹ +ĠNg uy +ĠNguy á»ħn +ĠÙģÙĬ Ùĩ +Ġб а +×Ļ ×Ļת +Ġ×ľ×¢ ש×ķת +Ġ×ŀ ׼ +Ġnghi á»ĩm +Ġм ного +Ġе е +ëIJĺ ìĸ´ +Ġl ợi +Ġ׾ ׾×IJ +Ġ׼ ף +Ġch ÃŃ +ãģ§ ãģ® +×Ĺ ×ķ +ש ×ķ×Ŀ +Ġ×ŀ ר +ĠÐĶ лÑı +Å ģ +Ġ׼×IJ שר +ĠM á»Ļt +ĠÙĪاÙĦ ت +ĠìĿ´ 룰 +ÅŁ a +Ġchi ến +Ġaras ında +Ġ×ij ×IJתר +ãģķãĤĮ ãģ¦ãģĦãĤĭ +Ø´ ÙĥÙĦ +Ġt ượng +Ġت ت +ĠC ó +Ġb á»ı +Ġtá»ī nh +Ġkh ÃŃ +ĠпÑĢ оÑģÑĤ +ĠпÑĢоÑģÑĤ о +ĠÙĪ ÙĤاÙĦ +Ġgi áo +ĠN ếu +×IJ ×ŀר +×¢×ł×Ļ ×Ļף +íİ ¸ +Ùĩد Ùģ +ĠB á»Ļ +Ġb Ãłn +Ġng uyên +Ġgü zel +ส าย +ì² ľ +×ŀ ×ķר +Ġph ân +ס פק +ק ×ij׾ +ĠاÙĦÙħ تØŃ +ĠاÙĦÙħتØŃ Ø¯Ø© +ائ د +Ġ×IJ ×ŀר +Ġki ÅŁi +ì¤ Ģ +Ġtr uyá»ģn +ĠÙĦ Ùĩا +ĠÐľ а +à¸ļริ ษ +à¸ļริษ ั +à¸ļริษั à¸Ĺ +Ġש ׳×Ļ×Ŀ +Ġмен Ñı +ÅŁ e +Ġdi á»ĩn +Ġ×IJ׳ ×Ĺ׳×ķ +k ü +Ġc á»ķ +Ġm á»Ĺi +w ä +Ùħ ÙĬ +Ġhi á»ĥu +ëĭ ¬ +Ġ×Ķ ×Ĺ׾ +Ġt ên +Ġki á»ĩn +ÙĨ ÙĤÙĦ +Ġv á»ĩ +×ĵ ת +ĠÐłÐ¾ÑģÑģ ии +л Ñĥ +ĠاÙĦع ربÙĬØ© +ĠØ· رÙĬÙĤ +Ġ×Ķ×ij ×Ļת +Ñģ еÑĢ +Ġм не +ä u +Ġtri á»ĩu +ĠÄij ủ +Ġר ×ij +ت ÙĩÙħ +à¸ĭ ี +Ġì§Ģ ê¸Ī +li ÅĽmy +د عÙħ +ãģł ãĤįãģĨ +Ñģки е +Ġh á»ıi +Ġק ×ķ +ÑĢÑĥ Ñģ +ÙĨ ظر +ãģ® ãĤĤ +Ġ×Ķ ׼×Ļ +ĠìĽ IJ +ÙĪ Ùĩ +ĠÙĪ Ùİ +ĠB ạn +п лаÑĤ +Ġ×ŀ ×ŀש +лÑİ Ð± +ĠнÑĥж но +Ġth Æ° +ãģ µ +ãģı ãĤīãģĦ +ر Ø´ +ר ×ķ×Ĺ +ĠÙĬ تÙħ +Ġצר ×Ļ×ļ +Ġph á +ม à¸Ńà¸ĩ +Ġ×ij×IJ ×ķפף +Ġcả nh +Ġíķľ ëĭ¤ +Ġ×Ķ×ŀ ת +à¸ķà¹Īาà¸ĩ à¹Ĩ +มี à¸ģาร +Ñģки Ñħ +ĠÐĴ Ñģе +Ġا ÙĪ +ج ÙĬ +ãģĵãģ¨ ãģ¯ +Ġd Ãłi +Ġh á»ĵ +èĩªåĪĨ ãģ® +à¹Ħ หà¸Ļ +ëĵ¤ ìĿĦ +ĠV Äĥn +Ġд аж +Ġдаж е +Ñĭ ми +лаÑģ ÑĮ +ÙĬ ÙĪÙĨ +ÙĨ ÙĪ +c ó +ãģĹãģ¦ ãģĦãģŁ +ãģł ãģĭãĤī +طاÙĦ ب +Ġc á»Ńa +п ÑĢоÑģ +ãģªãģ© ãģ® +รุ à¹Īà¸Ļ +Ġchi ếc +л Ñĭ +ĠÑıвлÑı еÑĤÑģÑı +Ġn á»ķi +ãģ® ãģĬ +Ġ×IJת ×Ŀ +ĠëķĮ문 ìĹIJ +à¸ģล าà¸ĩ +ĠbaÅŁ ka +ìĦ Ŀ +ĠÑĨ ел +Ùģ ÙĤ +ãģ«ãĤĪ ãĤĭ +ÙĤ ا +Ġçı kar +Ġcứ u +Ø· ا +Ġש ת +à¹Ĥ à¸Ħ +Ġ×ŀ ׾ +Ġ×Ķ פר +Ġг де +ĠØ® Ø· +åīį ãģ« +c jÄĻ +Ġ׊ש×ķ×ij +ר×Ĵ ×¢ +Ġkho ảng +ĠÄij á»Ŀi +ĠÐł е +Ġо на +Ġ×IJ ׳×ķ +ãģ® ãģ« +ĠاÙĦØ° ÙĬÙĨ +кÑĥ п +ãĤµ ãĥ¼ãĥ +ãĤµãĥ¼ãĥ ĵ +ãĤµãĥ¼ãĥĵ ãĤ¹ +в ал +г е +Ġgi ữa +ĠKh ông +ĠâĹ ĭ +à¸ģล ุà¹Īม +ĠÙħÙĨ Ø° +à¸Ń à¹Īาà¸Ļ +ĠÑģп оÑģоб +ĠÄij á»Ļi +Ġdi ÄŁer +Ġ à¸ĸà¹īา +Ùħ Ø«ÙĦ +Ġ×Ķ×IJ ×Ļ +Ġد ÙĪÙĨ +ÙĬر اÙĨ +Ñī и +بÙĨ اء +ĠØ¢ خر +ظ Ùĩر +Ġ×ij ׼ +ĠاÙĦÙħ ع +ãĥ Ĵ +Ġt ất +Ġm ục +ĠdoÄŁ ru +ãģŁ ãĤī +Ġס ×ķ +Ġx ác +ร à¸Ń +ĠcÄĥ n +Ġон л +Ġонл айн +Ġk ý +Ġch ân +Ġ à¹Ħมà¹Ī +اØŃ Ø© +r án +׳×Ļ ×Ļ×Ŀ +Ġ×ij ף +ĠÐ ĸ +à¸ķร à¸ĩ +д Ñĭ +Ġs ắc +ÙĦ ت +ãĥŃ ãĥ¼ +ĠÙĦ ÙĨ +Ġר ×ķ +Ġd Æ°á»Ľi +à¹Ģ à¸ĺ +à¹Ģà¸ĺ à¸Ń +e ÄŁi +Ġ×ķ ש +ĠÙĦ Ø£ +Ġg ặp +Ġc á»ij +ãģ¨ ãģ¦ãĤĤ +رÙĪ س +Ġ׾×Ķ ×Ļ +Ġë³ ¸ +ä¸Ĭ ãģĴ +Ġm ức +Ñħ а +Ġìŀ ¬ +à¸ī ัà¸Ļ +ÑĢÑĥ ж +Ġaç ık +ÙĪ اÙĦ +Ġ×ĸ ×ŀף +人 ãģ¯ +ع ÙĬÙĨ +Ñı Ñħ +Ġ×Ĵ×ĵ ×ķ׾ +ר ×ķ×ij +g ó +ëĿ¼ ê³ł +Ġark adaÅŁ +ÙĨ شر +Ġгод Ñĥ +ĠболÑĮ ÑĪе +ãģ¡ãĤĩ ãģ£ãģ¨ +Ġcâ u +Ġs át +íĶ ¼ +Ġti ến +íķ´ ìķ¼ +ĠÙĪ Ø£ÙĨ +à¸Ļ าà¸Ļ +Ġ×ij×IJ×ŀ צע +Ġ×ij×IJ×ŀצע ×ķת +Ġ׾ ר +Ġqu ản +ĠÙĪاÙĦ Ø£ +Ġ×IJ×ķת ×Ķ +Ġìĸ´ëĸ ¤ +Ġê²ĥ ìĿĢ +ØŃس ÙĨ +Ġm ất +à¸Ħ ูà¹Ī +ãĥ¬ ãĥ¼ +ĠÐĶ а +Ġol ması +Ġthu á»Ļc +׳ ×Ĺ +íĨ ł +Ġsö yle +ãģĿãģĨ ãģ§ãģĻ +Ġت ÙĥÙĪÙĨ +л ÑĥÑĩ +׾ ×Ļ×ļ +ĠØ£ ØŃد +ли ÑģÑĮ +ĠвÑģ его +Ġ×Ķר ×ij +Ġëª » +o ÄŁ +oÄŁ lu +ĠìĦ ł +Ġк аÑĢ +à¸łà¸² à¸Ħ +e ÅĦ +Ġ à¸ģà¹ĩ +Ġa ynı +Ġb Ãł +ãģªãĤĵ ãģ¦ +Ġ모 ëĵł +ÙĤر ار +ãģĹãģª ãģĦ +ĠÐĴ о +ĠÙĪÙĩ ÙĬ +ни ки +ãĤĮ ãģŁ +Ġchu ẩn +ר ×¢ +Ùģ رÙĬÙĤ +ãĤĴ åıĹãģij +ĠÄij úng +б е +׼ ×ķ×Ĺ +п Ñĥ +Ġ×ķ ×Ĵ×Ŀ +×ŀ ׳×Ļ +íĸ ¥ +צ ×Ļ×Ŀ +à¸ĭ ิ +Ùĩ ÙĨ +н ем +Ġ×ij×ij ×Ļת +ر ع +Ġ ส +ĠÄIJ Ãł +íķĺ ëĭ¤ +Ġ ấy +×Ĺ ×ķ×ĵ +×Ĺ×ķ×ĵ ש +ĠÑĩеÑĢ ез +Ñĥ л +ĠB ình +Ġê²ĥ ìĿĦ +Ġ×Ĵ ר +ä»ĺ ãģij +×Ĺ׾ ק +Ġت ÙĦÙĥ +à¹ĥส à¹Ī +sz Äħ +ÙĤ اÙħ +د ÙĪر +ĠÙģ ÙĤØ· +Ġh ữu +Ġмог ÑĥÑĤ +Ġg á»įi +Ġק ר +à¸Īะ มี +ت ÙĤدÙħ +Ġع بر +Ġ׾×Ķ ×Ŀ +ĠÑģам о +ס ×ĵר +Ġc Ãłng +r ÃŃ +Ġìŀ ¥ +ëĵ¤ ìĿĺ +ĠÙĦ Ùĥ +п оÑĢÑĤ +Ġkh ả +ĠÑģеб Ñı +׳ ף +Ġد ÙĪر +Ġm ợ +Ġcâ y +Ġf ark +Ġfark lı +а ÑİÑĤ +Ġtr á»±c +wiÄĻks z +Ġthu á»ijc +Ġت ØŃت +ت ÙĦ +ов Ñĭе +ëĤ ł +Ġв ам +بÙĦ غ +Ġê°Ļ ìĿĢ +íĮ IJ +ÙĦ ب +Ġnas ıl +Ġод ин +м ан +ĠعÙĦÙĬ Ùĩا +б и +Ġפ ש×ķ×ĺ +×ijר ×Ļ +Ġש ׳×Ķ +Ġëı Ħ +ĠÄIJ ại +Ġ×IJ×ķת ×Ŀ +ĠاÙĦØŃ Ø± +Ġб о +à¸Ī ุà¸Ķ +Ġr õ +ĠdeÄŁi ÅŁ +Ġëĭ ¨ +ĠÑģлÑĥÑĩ а +ĠÑģлÑĥÑĩа е +Ġ×IJ׳ ש×Ļ×Ŀ +×ĵ ×£ +ש×ij ת +Ġש׾ ׼×Ŀ +Ġch ú +nik ów +Ġtan ı +Ġcá o +ĠÄij á +Ġ×IJ ×ĵ×Ŀ +Ġê° ķ +Ġnhi á»ĩm +Ġ׾ ס +Ġ×Ľ×ª ×ij +Ġ×Ķס פר +ĠÄij Äĥng +Ġë ijIJ +à¸ľ ิ +à¸ľà¸´ ว +ج ا +Ġê° IJ +ر Ø£ +ست خدÙħ +ãģ«ãģªãĤĬ ãģ¾ãģĻ +Ġtá» · +×ĺ ×ķר +г овоÑĢ +Ġв оÑģ +ĠÙħÙĨ Ùĩا +иÑĢов аÑĤÑĮ +ĠÄij ầy +׳ ×Ĵ +ĠÙħ ÙĪ +ĠÙħ ÙĪÙĤع +ר׼ ×Ļ +ت Ùı +ëª ¨ +Ġת ×ķ +ÙĬا Ùĭ +à¹ĥ à¸Ķ +ãĤĬ ãģ¾ãģĻ +à¸Ńยูà¹Ī à¹ĥà¸Ļ +ĠØ£ ÙĪÙĦ +ĠØ£ خرÙī +Ġc Æ° +ص ار +×ŀ׊ש×ij +б ÑĢа +ÅĦ ski +б ÑĢ +ĠÙĬ Ùı +à¸ģ ิà¸Ļ +Ġch á»ijng +Ùħ Ùı +Ġ à¸Ħืà¸Ń +Ġت ÙĨ +t ÃŃ +y Äĩ +Ġm ạng +Ùģ ÙĪ +Ġdü nya +ק ר×IJ +Ġק ׾ +ĠØŃ Ø§ÙĦ +c ÃŃa +Ġà¹Ģ รา +Ġר ×ķצ×Ķ +Ġá p +ë° ķ +ا ÙĤØ© +ни Ñİ +Ġ×IJ ׾×ķ +Ġ×ŀס ×ķ +ãģ§ãģ¯ ãģªãģı +Ġtr ả +Ġק שר +mi ÅŁtir +Ġl Æ°u +Ġh á»Ĺ +ĠбÑĭ ли +Ġl ấy +عÙĦ Ùħ +Ġö zel +æ°Ĺ ãģĮ +Ġ×ĵ ר×ļ +Ùħ د +s ını +׳ ×ķש×IJ +r ów +Ñĩ еÑĢ +êµIJ ìľ¡ +ĠÐľ о +л ег +ĠV Ỽi +วัà¸Ļ à¸Ļีà¹ī +ÑİÑī ие +ãģĬ ãģĻ +ãģĬãģĻ ãģĻ +ãģĬãģĻãģĻ ãĤģ +ëı ħ +Ġ×Ļ×Ķ ×Ļ×Ķ +×ŀ ×ĺר +Ñı ми +Ġl á»±a +ĠÄij ấu +à¹Ģส ียà¸ĩ +Ġt Æ°Æ¡ng +ëĵ ± +ĠÑģÑĤ аÑĢ +à¹ĥ à¸ļ +ว ัà¸Ķ +ĠÄ° stanbul +Ġ à¸Īะ +à¸ķ ลาà¸Ķ +Ġب ÙĬ +à¹ģà¸Ļ ะ +à¹ģà¸Ļะ à¸Ļำ +س اعد +Ġب Ø£ +Ġki á»ĥm +ØŃ Ø³Ø¨ +à¸Ĭั à¹īà¸Ļ +Ġ×ķ ×¢×ķ×ĵ +ов ÑĭÑħ +оÑģ нов +Ġtr Æ°á»Łng +צ ×ij×¢ +ĠÃŃ t +Ġk ỹ +cr é +Ñı м +êµ ° +ãģĮ ãģªãģĦ +ÙĬÙĦ Ø© +ãĥķ ãĤ£ +ر Ùī +ĠÙĬ جب +Ġ×IJ ×£ +Ġc á»±c +ãĤīãĤĮ ãģŁ +Ġ à¸ľà¸¹à¹ī +Ġ à¸Ń +lar ımız +Ġkad ın +Ġê·¸ ëŀĺ +Ġê·¸ëŀĺ ìĦľ +ĠëĺIJ ëĬĶ +ĠÄij ả +ĠÄijả m +Ġ×IJ ×ķ×ŀר +Ġy ếu +ci Äħ +ciÄħ g +Ġt á»ij +Ġש×IJ ׳×Ļ +Ġdz iaÅĤa +Ñī а +ĠÄij Ãłn +s ına +ãģĵãĤĮ ãģ¯ +Ġ×ij ׾×Ļ +Ġ×ij ×Ļשר×IJ׾ +л оÑģÑĮ +Ġgi ữ +ê° IJ +ÑĢ он +تج ار +г лав +в ин +Ġh ạn +Ġyapı lan +ب س +Ġ à¸ŀรà¹īà¸Ńม +ê´Ģ 리 +mÄ±ÅŁ tır +b ü +r ück +ĠBaÅŁkan ı +ĠÙĦ ÙĬس +Ġs Æ¡ +à¸Īัà¸ĩ หว +à¸Īัà¸ĩหว ัà¸Ķ +د اء +Ġ×Ķ ׼ +v ÃŃ +ש ×IJר +Ġh Æ°á»Łng +Ġb óng +ĠCh ÃŃnh +Äħ c +à¹Ģà¸ģีà¹Īยว à¸ģัà¸ļ +Ġtá» © +Ġtứ c +ĠÑĨ веÑĤ +Ġt á»iji +ĠnghÄ© a +ÙĦا عب +د ÙĦ +Ġפע ×Ŀ +h ör +à¸Ĭ ุà¸Ķ +à¸ŀ ู +à¸ŀู à¸Ķ +п аÑģ +ĠÅŁ u +Ġt Æ°á»Łng +خار ج +Ġâ m +ĠинÑĤеÑĢ еÑģ +ен нÑĭÑħ +×IJ ׳×Ļ +بد Ø£ +ëĿ¼ ëĬĶ +ì¹ ´ +æĸ¹ ãģĮ +ли в +Ġ à¸Ħà¸Ļ +ער ×ļ +à¸Ĥà¸Ńà¸ĩ à¸Ħุà¸ĵ +п ад +Ġc ạnh +ĠëĤ ¨ +ĠÄij âu +Ġbi á»ĥu +ãĤĤ ãģĤãĤĭ +׾ ×Ĵ +Ġ สำหรัà¸ļ +Ġxu á»ijng +ס ×ķ +ĠØ° ات +ĠÐľ е +ع اÙĦÙħ +×IJ ס +ب ÙĬØ© +Ø´ ا +и ем +ĠNg Æ°á»Ŀi +íĺ ij +Ñģл ов +Ġп а +Ġm ẫu +ĠпÑĢоÑĨ еÑģÑģ +ĠNh Ãł +пÑĢо из +пÑĢоиз вод +à¸łà¸²à¸¢ à¹ĥà¸Ļ +Ġ à¸ļาà¸Ĺ +×ŀ ׳×ķ +ĠоÑĢг ан +רצ ×ķ +×ķ×ŀ ×Ļ×Ŀ +Ġyaz ı +Ġd ù +ãĥ¬ ãĥ³ +ÙĪÙĦ ÙĬ +ย ู +Ġtr ò +à¹Ģà¸ŀ ลà¸ĩ +Ġ×ŀ ׾×IJ +à¸ķ ล +à¸ķล à¸Ńà¸Ķ +ĠÄij ạt +Ġ×Ĺ×ĵ ש +p óÅĤ +Ġ×ŀ ×ĵ×Ļ +ujÄħ c +×ŀ׳×Ķ ׾ +Ġש×ij ×ķ +Ġ×Ķ×ŀש פ×ĺ +Ġ×IJ ׾×Ķ +ĠÙĪ Ø°ÙĦÙĥ +à¹Ģà¸ŀ ราะ +ĠÄijo Ãłn +Ġíķ¨ ê»ĺ +Ġd ục +Ø´ ت +Ġ ula +Ġula ÅŁ +Ġqu ý +Ġ×Ķ ×Ĵ×ĵ×ķ׾ +à¸ķัà¹īà¸ĩ à¹ģà¸ķà¹Ī +Ġש ר +Ø´ Ùĩد +׳ ש×Ļ×Ŀ +à¸ŀ ล +رÙĪ ا +ãĤĮ ãģ¦ +Ġн иÑħ +Ġдел а +ãģ§ãģį ãģªãģĦ +ÅĤo ż +×IJ ×Ĺר +ì ½Ķ +ãĤ¢ ãĥĥãĥĹ +د Ùģع +Ġti á»ĩn +Ġkh á»ı +Ġkhá»ı e +ĠاÙĦع اÙħØ© +ãģ« ãģĤãĤĭ +ĠÄij á»Ļc +ì¡ ± +Ġc ụ +й ÑĤе +Ġзак он +ĠпÑĢо екÑĤ +ìĸ ¸ +ÙĦ ØŃ +ĠçalÄ±ÅŁ ma +ãĤĴ ãģĻãĤĭ +Ñħ и +ع اد +Ġ׳ ×ŀצ×IJ +Ġר ×Ļ +à¸Ńà¸Ńà¸ģ มา +ĠT ôi +Ġth ần +ĠÙĬ ا +ล าย +Ġав ÑĤо +Ġsı ra +ĠÙĥ Ø«ÙĬر +Ùħ ÙĬز +ĠاÙĦع ÙĦÙħ +æĸ¹ ãģ¯ +×ķ×¢ ×ĵ +Ġобла ÑģÑĤи +×Ļ׾ ×Ļ×Ŀ +ãģĮ åĩº +à¸ĺ ุ +à¸ĺุ ร +à¸ĺุร à¸ģิà¸Ī +ÙĤت ÙĦ +ר×IJ ×ķ +Ġng u +Ġngu á»ĵn +Ġ มา +Ġпл ан +t ório +Ġcu á»iji +Ñģк ом +ĠاÙĦÙħ اض +ĠاÙĦÙħاض ÙĬ +Ġ×ij×¢ ׾ +Ġר ×ij×Ļ×Ŀ +Ġlu áºŃn +Ùĥ ÙĪ +à¸Ĺัà¹īà¸ĩ หมà¸Ķ +в ан +Ġtho ại +à¹Ħ à¸Ń +б иÑĢ +ĠاÙĦ ض +ت ا +ĠÑĢ од +ĠV Ãł +×ŀ ×Ļף +ĠбÑĭ ла +к ами +ĠÐĶ е +t ık +קר ×Ļ +ĠeÄŁ itim +ĠÙĥ بÙĬر +ب Ùĥ +ĠÙĦ ÙĪ +в ой +Ġ ãģĵãģ® +ĠÑĤ ÑĢÑĥд +my ÅĽl +Ġs Æ° +à¸ŀ ีà¹Ī +Ġ à¹ģลà¹īว +×¢ ק +Ġ×Ĺ×ijר ת +ระ หว +ระหว à¹Īาà¸ĩ +×Ļ ×Ļ×Ķ +ĠاÙĦÙĨ اس +ün ü +Ġ׾ ×ŀ×Ķ +Ġch Æ°Æ¡ng +ĠH á»ĵ +ار ت +ãĤĪãģĨ ãģ§ãģĻ +l á +ק×Ļ ×Ļ×Ŀ +æľ¬ å½ĵ +æľ¬å½ĵ ãģ« +ãģĵãĤĵ ãģª +Ñģ ов +Ġ×ķ ×Ĺ +à¹Ģà¸ģ à¹ĩà¸ļ +Ġк ÑĤо +à¹Ĥร à¸Ħ +ĠØ´ رÙĥØ© +ع زÙĬ +عزÙĬ ز +Ø·ÙĦ ÙĤ +п ÑĥÑģÑĤ +Ùģ تØŃ +ëŀ Ģ +Ġhã y +ض Ùħ +ë¦ ° +åł´åIJĪ ãģ¯ +ãĤª ãĥ¼ +Ġh ắn +Ġ×IJ ×ij×Ļ×ij +Ġש׾×Ķ ×Ŀ +Ġ×Ķ×Ļ ×Ļת×Ķ +ĠاÙĦد ÙĪÙĦØ© +ĠاÙĦ ÙĪÙĤ +ĠاÙĦÙĪÙĤ ت +ãģĤ ãģ¾ãĤĬ +Ġta ÅŁÄ± +Ä° N +×¢ סק +ãģ¦ ãģĦãģŁ +Ġtá»ķ ng +ĠاÙĦØ¥ ÙĨس +ĠاÙĦØ¥ÙĨس اÙĨ +ÑĢ еÑĪ +Ġg ái +ĠÑĨ ен +ĠÙģ ÙĤد +Ùħ ات +ãģķãĤĵ ãģ® +Ġph ù +×ĺ ×Ķ +ĠÙĪاÙĦ تÙĬ +Ġب Ùĥ +ìĿ´ ëĤĺ +к Ñģ +Ùħ ÙĬر +Ġv ùng +ĠاÙĦØ´ عب +ĠNh Æ°ng +ãĥĢ ãĥ¼ +Ġ×Ĺ×Ļ ×Ļ×Ŀ +ĠØ´ خص +ק ×ķ×ĵ +ê² Ģ +×¢ ש +×¢ ×ķ׾×Ŀ +צ ×ķר +ع ÙĤد +ĠiÅŁ lem +Ġ×Ķ×ij ×IJ +Ġd ưỡng +à¸Ł รี +Ġph ÃŃa +ãģ®ä¸Ń ãģ§ +Ġп и +Ġng Ãłnh +ним а +ĠÙĩ ÙĦ +Ġ×ķ ×IJת +ĠÄij áng +é quipe +ĠÑįÑĤ оÑĤ +Ġgö rev +ë§ ¤ +Ġqu ân +å¼ķ ãģį +æĻĤ ãģ« +Ġب Ùħا +×ŀ ×Ļת +Ġü lke +Ġ×ŀק ×ķ×Ŀ +×ij ף +æ°Ĺ æĮģãģ¡ +Ġë§İ ìĿĢ +Ġyük sek +ÑĨ енÑĤÑĢ +ĠÙħ جÙĦس +ç§ģ ãģ® +ÙĤد ر +Ġë¶Ģ ë¶Ħ +Ġì° ¨ +خر ج +ãģĭ ãģªãĤĬ +ë³´ ëĭ¤ +Ġ×ŀ ×Ļ×ĵ×¢ +peÅĤ ni +Ġx á»Ń +ìĹIJìĦľ ëĬĶ +ĠباÙĦ Ùħ +ĠÙĪ Ùħا +ĠÑįÑĤ ой +ب ÙĬÙĨ +n ü +ØŃ Ø² +ØŃز ب +ĠÑĢабоÑĤ а +ĠNh áºŃt +ÙĦ اء +Ġëĵ ¤ +Ġëĵ¤ ìĸ´ +ãĤĦãģĻ ãģĦ +×Ĺ×ĸ ק +Ġ×Ķ×Ĺ ×ijר×Ķ +п иÑĤ +ãģĭãĤī ãģ® +Ġë§IJ ìĶĢ +Ġפ ×ķ +ÙĦ Ùİ +à¹Ģà¸ķà¹ĩ ม +ĠÐļ о +Ġm ówi +Ġt ÃŃn +ר×Ĵ ש +פר ק +Ġtr ạng +ĠÐŀ н +×Ĺ ×ķ×¥ +ĠعÙĨد Ùħا +Ġب ر +使 ãģĦ +Ġr á»Ļng +ëĮĢ ë¡ľ +íĪ ¬ +Ġktóry ch +в ид +ลูà¸ģ à¸Ħà¹īา +Ġmog Äħ +Ġש ×Ĺ +×ij ×Ĺר +ãĥĸ ãĥŃãĤ° +ĠTh Ãłnh +Ġ×Ķ ר×Ļ +ĠÑģÑĤ аÑĤÑĮ +ĠH á»Ļi +à¸ļ à¹īาà¸ĩ +çī¹ ãģ« +ĠÄIJ ức +èĢħ ãģ® +×¢ ×ŀ×ķ×ĵ +×ĺר ×Ķ +Ð ¥ +ĠÙħ Ùħا +Ġe ÅŁ +ĠнеобÑħодим о +ник ов +Ġüzer inde +a ÅĤa +Ġchá»ĭ u +ĠاÙĦ دÙĬÙĨ +أخ بار +ĠÄij au +ãģĮ å¤ļãģĦ +jÄħ cych +د Ø®ÙĦ +ları nd +larınd an +Ġs ẻ +à¸ŀิ à¹Ģศ +à¸ŀิà¹Ģศ ษ +ת ף +t ıģı +Ġlu áºŃt +ĠÅŀ e +ãĤ« ãĥ¼ +ãģ® ãģĤãĤĭ +Ġ×Ķ×IJ תר +ĠاÙĦØ¢ ÙĨ +ıld ı +Ġá o +ĠнаÑĩ ал +Ġvi á»ĩn +Ġ×ij×¢ ×ķ׾×Ŀ +з наÑĩ +×Ļ×ĺ ×Ķ +к ам +ĠÐĺ з +à¹Ģà¸Ĥ ียà¸Ļ +à¸Ļ à¹īà¸Ńà¸ĩ +ÑĤ ÑĢо +à¹Ģ à¸Ł +Ġжиз ни +Ġ สà¹Īวà¸Ļ +Ġv áºŃn +Ġê´Ģ 볨 +Ġl âu +ס ×ĺר +ק ש +س ÙĬر +Ġ×IJ×ķת ×Ļ +Ġm ôi +ائ ب +Ġо ÑģÑĤа +Ġm ón +Ġ×ij ×ŀק×ķ×Ŀ +Ġد اخÙĦ +Ġ×IJ ×ķר +Ġв аÑģ +Ùĥ Ø´Ùģ +ìĺ ¨ +à¸ĸ à¹Īาย +Ġkullan ıl +Ġt ô +ãģ« ãĤĪãĤĬ +ĠëĺIJ íķľ +Ġ×¢×ij×ķ×ĵ ×Ķ +Ġri ê +Ġriê ng +Ġyak ın +ز ا +Å » +×IJ ×ķ׼׾ +شار Ùĥ +Ġб еÑģ +× ´ +Ġا بÙĨ +ĠTá»ķ ng +ÙĨ ظ +ÅĽwi ad +ãĤµ ãĥ¼ +ห าย +ĠG ün +Ġhakk ında +à¹Ģà¸Ĥà¹īา มา +ز ÙĨ +ĠÐł о +Ġbi á»ĥn +ãģ© ãģĵ +Ùģ عÙĦ +ز ع +פר ×ĺ +Ġ×Ķ ף +Ø£ ÙĩÙĦ +Ġth ất +ØŃ ÙħÙĦ +Ñĩ Ñĥ +ĠìĤ¬ ìĭ¤ +ì° ¸ +ĠìľĦ íķ´ +ÙĪ ظ +ĠÐŁ од +Ġkho ản +ÑĤ ен +ĠÙģ اÙĦ +Ñģ ад +à¸Ļ à¸Ńà¸Ļ +ĠاÙĦسعÙĪد ÙĬØ© +" ØĮ +ĠاÙĦ ÙĴ +ãĤī ãģļ +Ġto án +Ġch ắc +׼ ×Ļר +m éd +méd ia +ز ÙĪ +Ġyan ı +פ ׳×Ļ×Ŀ +ØŃ Ø¸ +Ġб еÑģп +ĠбеÑģп лаÑĤ +ĠбеÑģплаÑĤ но +ĠØ£ ÙħاÙħ +à¸Ń าย +à¸Ńาย ุ +ר שת +Ġg á»ĵ +Ġgá»ĵ m +Ġu á»ijng +ص ب +k ır +ãĥij ãĥ¼ +Ġ׾×ĵ עת +Ġк ÑĥпиÑĤÑĮ +׾ ×ķ×Ĺ +ÙĪض ع +ÙĤÙĬ Ùħ +à¸Ľ า +ж ив +à¸Ķ ิà¸Ļ +×IJ ×ķפ +à¹Ģล à¹ĩà¸ģ +ãĥĥ ãĥī +иÑĩеÑģки Ñħ +ĠCh ủ +кÑĢ аÑģ +ÙĪ صÙĦ +p ÅĤat +м оÑĢ +Ġ×Ķ×IJ ×ķ +à¸Ń ิà¸Ļ +Ġíķľ êµŃ +гÑĢ е +Ġìłľ ê³µ +ì° ½ +Ġê°ľìĿ¸ ìłķë³´ +Ġngh á»ĭ +à¸ĭ า +ØŃس اب +Ġby ÅĤa +ÙħÙĦ Ùĥ +иÑĩеÑģки е +Ġb ác +ض ØŃ +ê¸ ¸ +ש ×ŀ×¢ +Ġìĸ´ëĸ » +Ġìĸ´ëĸ» ê²Į +ìĽ Į +ات Ùĩ +à¹Ĥรà¸ĩ à¹ģ +à¹Ĥรà¸ĩà¹ģ รม +خد ÙħØ© +ĠÐł а +׼×ķ׾ ×Ŀ +×ŀש ×Ĺק +ĠÙĪ ÙĥاÙĨ +ס ×ķ×£ +ĠاÙĦØŃÙĥÙĪÙħ Ø© +Ġ×ij ×ĺ +Ġtr áºŃn +Ġ×Ķ×¢ ×ķ׾×Ŀ +ĠÃŃ ch +t Äħ +ש×ŀ ×ķ +Ġ×Ķר×IJש ×ķף +Ġíķĺ ê³ł +ãģķ ãĤī +ãģķãĤī ãģ« +ãģ« ãģĹãģ¦ +Ġ à¸ľà¸¡ +ãģ® ãĤĪãģĨãģª +ĠÙĪ ÙĤت +ãĥį ãĥĥãĥĪ +ÙĦ عب +ÙĪ Ø´ +ìĺ ¬ +Ġ หาà¸ģ +Ġm iaÅĤ +à¸Ĺ à¸Ńà¸ĩ +иÑĤ а +ا صر +ил ÑģÑı +з е +à¸Ľà¸£à¸° มาà¸ĵ +ãģĿãĤĮ ãģ¯ +Ġb ır +Ġbır ak +صÙĨ اع +Ð ® +Ø´ عر +Ġ׳ ×Ĵ×ĵ +Ġب سبب +ãĥĿ ãĤ¤ +ãĥĿãĤ¤ ãĥ³ãĥĪ +ĠاÙĦج ÙĪ +ĠнеÑģк олÑĮко +Ġki ếm +Ùģ Ùİ +Ġض د +×ij×Ļ×ĺ ×ķ×Ĺ +تاب ع +ÙĨ ز +ĠB ản +Ġaç ıkl +Ġaçıkl ama +Ġ à¸Ħุà¸ĵ +à¸Ĺ า +ÅĤ ów +Ø· ب +ÙĨ ØŃÙĨ +Ġ×ŀק ×ķר +ĠÄ° s +Ġдом а +Ġ วัà¸Ļ +Ġd Ãłnh +Ñı н +ми ÑĢ +Ġm ô +ĠvÃł ng +ص اب +s ının +à¸Ħ ืà¸Ļ +Ø® بر +×ĸ׼ ×ķ +Ġ×ŀ ש×Ķ×ķ +m ü +Ġкомпани и +Ġ×Ķ×¢ ×Ļר +ĠÙĥ ÙĪ +ÙĤÙĦ ب +ĠlỼ p +и ки +׳ ×ij +à¹Ĥ à¸Ħร +à¹Ĥà¸Ħร à¸ĩ +à¹Ĥà¸Ħรà¸ĩ à¸ģาร +×ŀ×ķ×¢ ×ĵ +ÑıÑĤ ÑģÑı +หลัà¸ĩ à¸Īาà¸ģ +ени Ñİ +Ġש ×¢ +Ġb Æ°á»Ľc +ãĥ¡ ãĥ¼ãĥ« +ãĤĦ ãĤĬ +Ġ×Ļ×ķ×ĵ ×¢ +Ġê´Ģ íķľ +ĠاÙĦØ£ Ùħر +Ġböl ge +ĠÑģв ой +ÙĦ س +Ġ×ŀ×Ļ ×ķ×Ĺ×ĵ +ĠëĤ´ ìļ© +ĠØ£ جÙĦ +ĠÄIJ ông +Ġ×ŀ ×ł×ª +Ġìĭľ ê°Ħ +Ùĥ Ùİ +ãģ¨ãģĦãģĨ ãģ®ãģ¯ +Ġnale ży +تÙĨظ ÙĬÙħ +ĠÑģозд а +Ġph é +Ġphé p +ãģ§ãģį ãģ¾ãģĻ +Ġع ÙĦÙħ +大ãģį ãģª +ãĤ² ãĥ¼ãĥł +í ħĮ +Ġ׼×ķ׾ ׾ +ĠинÑĤеÑĢ неÑĤ +ĠT ừ +ãģ¨ ãģªãĤĭ +ز اÙĦ +Ġktóry m +Ġnh é +ìĪ ľ +н ев +д еÑĢ +ãĤ¢ ãĥĹãĥª +i á»ĩu +×ij ×Ļ׾ +Ġت س +ĠÄIJ ây +ĠاÙĦØ® اصة +Ġà¹Ģ à¸Ĭ +Ġà¹Ģà¸Ĭ à¹Īà¸Ļ +ص اد +Ġd ạng +س عر +Ġש ×Ļ×ŀ×ķש +×Ĵ ×Ļ×Ŀ +ãģĮãģĤ ãģ£ãģŁ +п ÑĢов +пÑĢов од +Ġ×IJ ×Ļ׳×ķ +Ġ׾ ר×IJ +Ġ׾ר×IJ ×ķת +ĠØ£ ÙģضÙĦ +ĠØŃ ÙĦ +ĠØ£ بÙĪ +ê° ķ +Ġì§ ij +ãģ® ãĤĪãģĨãģ« +Ġפ ׳×Ļ +ס ×Ļ×Ŀ +ĠÙĪÙĩ ذا +Ġka ç +Ġé én +Ġê± ´ +ë° Ķ +Ñĥ з +à¸Ĥà¸Ńà¸ĩ à¹Ģรา +i ÅĤ +ĠÐľ Ñĭ +Ġch ết +ĠاÙĦØ« اÙĨÙĬ +×IJ ק +Ġ×ķ ×¢×ľ +ĠاÙĦØ· ب +×ij×ĺ ×Ĺ +Ġج دÙĬدة +Ġع دÙħ +ع ز +สิà¹Īà¸ĩ à¸Ĺีà¹Ī +ãģĻ ãĤĮãģ° +ĠÄij ô +ì£ ł +د ÙĤ +н омÑĥ +Ġk á»ĥ +ãĤ¢ ãĥ³ +å¤ļãģı ãģ® +à¸Ľà¸£à¸° à¸ģ +à¸Ľà¸£à¸°à¸ģ à¸Ńà¸ļ +פע×Ļ׾ ×ķת +ĠÑģÑĤ ол +may ı +ãģ¤ ãģĦ +Ġyılı nda +Ġ à¸Īึà¸ĩ +koÅĦ cz +ĠTh ông +Ġак ÑĤив +н ÑģÑĤ +нÑģÑĤ ÑĢÑĥ +ĠÃĸ z +Ġת ×ŀ×Ļ×ĵ +ĠÙĥ ÙĨت +Ñģ иÑģÑĤем +pr és +prés ent +Ġn â +Ġnâ ng +gÅĤ os +ĠÙĪز ÙĬر +ØŃ ØµÙĦ +Ġиме еÑĤ +ØŃ Ø±ÙĥØ© +à¸ŀ à¹Īà¸Ń +ãĤĴ ãģĬ +Ġاست خداÙħ +×IJ×Ļר ×ķ×¢ +ä»ĸ ãģ® +Ġש×Ķ ×Ŀ +ãģĹãģŁ ãĤī +ש×ŀ ×Ļ +Ñģ ла +m ı +Ġbaz ı +Ġíķĺ ì§Ģë§Į +×ĵ ׾ +Ġyapt ıģı +ãĥĬ ãĥ¼ +׾ ×Ļ׾×Ķ +ãģ¨ãģĦ ãģ£ãģŁ +änd ig +ĠÅŁ a +ĠÙģÙĬ Ùħا +иÑĤ елÑı +×ŀ ×ķש +à¸Ĥ à¸Ńà¸ļ +l ük +Ġh á»ĵi +Ġëª ħ +ĠاÙĦÙĥ Ø«ÙĬر +צ ×IJ +Ġhaz ır +طر Ùģ +ا ÙĬا +ĠÄij ôi +ен д +ÙĦ غ +×Ĺ ×ĸ×ķר +ĠвÑģ ег +ĠвÑģег да +ëIJĺ ê³ł +×ĵ ×ķ×ĵ +ан а +د ÙĪÙĦØ© +Ġho ạch +ع ÙĦا +عÙĦا ج +Ġ×ķ ×¢×ĵ +×Ķ ×Ŀ +ки й +ÙĦ ÙIJ +Ġ×¢ ׾×Ļ×ķ +ÑİÑī ий +Ġng ủ +صÙĨ ع +ĠاÙĦع راÙĤ +à¸ķà¹Īà¸Ń à¹Ħà¸Ľ +ãģŁãģı ãģķãĤĵ +Ġph ạm +ÙĦ اÙĨ +ات Ùĩا +Ġbö yle +تÙĨ ÙģÙĬ +تÙĨÙģÙĬ Ø° +Ġש×Ķ ×Ļ×IJ +Ñģ Ñĥ +ย าว +Ġש ×ķ׳×Ļ×Ŀ +Ġ×ŀ ×ķ׾ +ĠÑģ ил +Ġ×IJ×Ĺר ×Ļ×Ŀ +Ġph ủ +ÙĤØ· ع +ĠTh ủ +à¸Ľà¸£à¸°à¹Ģà¸Ĺศ à¹Ħà¸Ĺย +ÙĨ ÙĤ +ĠÄijo ạn +Ġب Ø¥ +п ÑĢедел +×ķת ×ķ +Ġy arı +пÑĢ е +ĠczÄĻ ÅĽci +ØŃ ÙĥÙħ +×ķ׳ ×Ļת +פע ׾ +ãĤĴ ãģĹãģ¦ +Ġktó rzy +׾ ×Ŀ +ĠÄIJi á»ģu +ĠкоÑĤоÑĢ аÑı +ĠìĿ´ ìĥģ +ãģĤ ãģ£ãģŁ +Ġ×ŀ×ĵ ×ķ×ijר +פ ×ķ×¢×ľ +d ım +éĢļ ãĤĬ +ĠбÑĥд ÑĥÑĤ +à¹Ģวà¹ĩà¸ļ à¹Ħà¸ĭ +à¹Ģวà¹ĩà¸ļà¹Ħà¸ĭ à¸ķà¹Į +ا خر +×Ĺ ×Ļ׾ +Ġ×Ļ ׾ +Ġ×Ļ׾ ×ĵ×Ļ×Ŀ +×Ĺ ×Ļפ +×Ĺ×Ļפ ×ķש +Ġd òng +Ġש ×ĸ×Ķ +ÑĮ е +ãģĤ ãģ¨ +ìŀIJ ê°Ģ +×IJ ×ĵ +Ġü z +Ġüz ere +ظ ÙĦ +Ġ×IJ ×ķ׾×Ļ +Ġ×ij ×Ļ×ķ×Ŀ +ÙĦ ات +Ġm ê +ì¹ ¨ +تØŃ Ø¯ +تØŃد Ø« +ĠØ® اصة +Ġب رÙĨ +ĠبرÙĨ اÙħج +ĠH Ãłn +×Ĺ ×¡ +ĠÙĪ ÙĦÙħ +×¢ ×Ŀ +Ġm ı +à¸Ł ัà¸ĩ +ש ×¢×Ķ +ÙĪÙģ ÙĤ +ס ×ij×Ļר +алÑĮ нÑĭй +×Ĺש ×ķ×ij +Ġn Ãłng +ë³ ¼ +ĠкоÑĤоÑĢ ÑĭÑħ +Ġ×Ĺ ×ķק +t ör +ĠлÑĥÑĩ ÑĪе +ãĥij ãĥ³ +ลà¹Īา สุà¸Ķ +Ġج دÙĬد +ÙĬد Ø© +à¸Ĺ รà¸ĩ +ãĤĪãĤĬ ãĤĤ +ÙĦ ÙĦ +ãĤĤ ãģ£ãģ¨ +ש×ĺ ×Ĺ +Ġ×ķ ×IJ×Ļ +Ġgi á»ijng +Ø¥ ضاÙģ +ק ת +ë§ Ŀ +Ġzosta ÅĤ +ÑĢ оз +×Ļפ ×Ļ×Ŀ +Ġ׼׾ ׾ +ת×ķ׼ ף +dıģ ını +ÙĤ سÙħ +ĠÑģ ÑĩиÑĤ +ĠÑģÑĩиÑĤ а +×ĺ ×ķת +Ġ Æ°u +ĠØ¢ ÙĦ +Ġм ом +Ġмом енÑĤ +ĠاÙĦتع ÙĦÙĬÙħ +×¢×ľ ×ķת +Ġch ữa +Ġy ön +Ġtr Ãł +ĠØŃ ÙĬÙĨ +à¸ĭ ั +ĠC á +×¢ ×ĸ +ĠاÙĦØ£ ÙħÙĨ +c ÃŃ +Ġv á»ijn +Ġ à¸Ļาย +об ÑĢа +ק ×IJ +Ġthi ếu +ãĥŀ ãĥ¼ +ส วà¸Ļ +Ġg á»Ń +Ġgá»Ń i +Ġê ¹ +Ġê¹ Ģ +Ġthi á»ĩn +ÙĤ ع +w ÄĻ +Ġн ам +ÑĤ ол +Ġs ân +ס ×ķ×Ĵ +Ġgeç ir +ÑĤ он +ев а +ĠÙĪ ضع +Ġع شر +Ñģ ло +à¸Ī ัà¸ļ +ãĤ· ãĥ¼ +ãĤĤ ãģĤãĤĬãģ¾ãģĻ +Ġv ẻ +ĠÄIJ á»ĥ +ر Ùģع +ĠاÙĦØ£ÙĪÙĦ Ùī +ÑĤ аÑĢ +ãģªãģı ãģ¦ +Ùħ Ùİ +qu ÃŃ +×¢×ł×Ļ ×Ļ׳ +г ен +Ġh ôm +à¸Ī า +Ġnh Ỽ +ĠاÙĦع ربÙĬ +×IJ ף +Ġl á»Ļ +Ġje ÅĽli +à¹Ģà¸Ĺà¹Īา à¸Ļัà¹īà¸Ļ +ĠØ£ÙĨ Ùĩا +Ġt uy +Ġtuy á»ĩt +Ġت ص +Ġتص ÙĨÙĬ +ĠتصÙĨÙĬ Ùģ +Ġê·¸ëŁ¬ ëĤĺ +о ÑĨен +à¸ģิà¸Ī à¸ģรรม +ãĤĦ ãģ£ãģ¦ +Ġkh á»ıi +Ġl á»ĩ +ĠاÙĦÙħج تÙħع +à¸Ńาà¸Ī à¸Īะ +à¸Īะ à¹Ģà¸Ľà¹ĩà¸Ļ +ов Ñĭй +ר ×Ŀ +ร à¹īà¸Ńà¸Ļ +ש ×ŀש +人 ãģ« +Ġüzer ine +פר ×Ļ +du ÄŁu +Ñĩ ик +Ġmù a +Ġ×ŀת ×ķ×ļ +Ġc áºŃp +Ġت ارÙĬØ® +×ij׾ ת×Ļ +Ġì¢ Ģ +ÙĦ ع +ب اÙĨ +Ġch út +Ġ×Ķ×ĸ ×ŀף +n ée +ĠLi ên +ĠÙĦÙĦ Ø£ +ØŃد ÙĪد +Ġ×¢ ׼ש×Ļ×ķ +в оз +Ġyapt ı +Ġоб о +à¹ĥหà¹ī à¸ģัà¸ļ +Ġ×ij×Ķ ×Ŀ +ãģı ãģ¦ +ر أس +ĠÑģÑĢед ÑģÑĤв +ĠB Ãłi +ãģĵãģ¨ ãģ« +ĠìĤ¬ íļĮ +Ġ모 ëijIJ +×ij ×IJ +Ġtr ắng +ĠاÙĦبÙĦ د +ĠHo Ãłng +ли бо +ĠдÑĢÑĥг иÑħ +Ä° R +Ñĥм а +ĠJe ÅĽli +ãĤĤ ãģĹ +Ġv òng +Ġ×IJתר ×Ļ×Ŀ +ĠÄij á»įc +Ġв оÑĤ +ãģł ãģĮ +ë° ° +à¸Ķู à¹ģล +Ġ×ŀ ׼׾ +ìĹIJ ëıĦ +г аз +Ġ׳×ķס פ×Ļ×Ŀ +ãģĵãģ¨ ãģ§ +Ġت ÙĪ +ãģ§ ãģĤãĤĬ +à¸Ļั à¹Īà¸ĩ +ĠможеÑĤ е +sz ÄĻ +ãģ® ãģł +ĠÙħÙĨ Ùĩ +Ġb á»ķ +Ġb üt +Ġbüt ün +ë³´ ê³ł +Ġch á»ĵng +à¹ģà¸Ī à¹īà¸ĩ +ĠV ì +ĠØŃ Ø± +Ġgi ản +ĠÙħ دÙĬÙĨØ© +تط بÙĬÙĤ +à¸Ī ิ +æĹ¥ ãģ® +б ил +à¸ģ à¸Ńà¸ĩ +ê³ ³ +ĠØ£ Ùħا +ìĨ IJ +Ġtr ái +ĠвÑģ ем +Ġس ÙĨØ© +ĠÑģай ÑĤ +Ġг оÑĤов +п Ñĭ +ĠëIJ ł +ĠاÙĦØ® Ø· +ĠاÙĦرئÙĬس ÙĬØ© +Ġíķ ©ëĭĪëĭ¤ +ĠìķĦëĭĪ ëĿ¼ +ĠìĿ´ ëłĩ +ĠìĿ´ëłĩ ê²Į +) ØĮ +h ält +ĠØ£ Ùħر +Ġع Ùħر +à¸ģà¹ĩ à¸Īะ +Ġ à¸Ĺำà¹ĥหà¹ī +Ġc ân +Ġ×ij ׾ +Ġ×ij׾ ×ij×ĵ +פ סק +ĠÙĬ ÙĤÙĪÙĦ +н ÑĥÑĤÑĮ +à¹ģ à¸Ħ +Ġק צת +Ġn ằm +Ġh òa +bilit Ãł +ĠìĹĨ ëĭ¤ +Ġ׼ פ×Ļ +ÑĢ ож +лаг а +Ġ×Ķש ×Ļ +ĠNgo Ãłi +ĠÙĪ ج +ĠÙĪج ÙĪد +ĠìľĦ íķľ +Ġus ÅĤug +Ġtu ần +d ź +×ŀ ×ķף +ĠاÙĦع دÙĬد +Ġch ẳng +สุà¸Ĥ à¸łà¸²à¸ŀ +Ġ×ij ×ĵר×ļ +ĠÑģеб е +ĠìŀĪ ìĿĦ +ĠاÙĦØŃ Ø§ÙĦ +Ġd á +Ġc Æ°á»Ŀi +Ġnghi ên +ie ÅĦ +ĠD Æ°Æ¡ng +ï¼ ħ +Ø´ د +ãģĦãģ¤ ãĤĤ +ĠвÑĭб оÑĢ +Ġc á»Ļng +ש ×Ļ׳×ķ×Ļ +Ġch ạy +Ġ×ij×¢ ׾×Ļ +اخ بار +íķĺ ë©° +ż Äħ +ج از +Ġ׳ ר×IJ×Ķ +ศ ู +ศู à¸Ļ +ศูà¸Ļ ยà¹Į +×Ĵ ×¢ +Ġ×¢ ×ĵ×Ļ +Ġ×¢×ĵ×Ļ ×Ļף +بر ا +ÑĨи й +ĠÄIJ á»ĵng +ÙĤ اÙĨÙĪÙĨ +ĠÄij ứng +ãģĹãģŁ ãĤĬ +Ġ×Ĺ×Ļ ×Ļ +Ġë IJľ +ĠëIJľ ëĭ¤ +Ġм еждÑĥ +à¸ŀวà¸ģ à¹Ģà¸Ĥา +ĠB ắc +ล ำ +ë° ± +ĠíĻ ķ +มาà¸ģ ม +มาà¸ģม าย +бан к +à¸Ńา à¸ģาร +Ġh Ãł +Ġ׾ ׳ +à¸Ń à¸Ń +Ġë°Ķ ë¡ľ +л ом +m ática +ĠØŃ Ø¯ +اب ت +à¸Ĺีà¹Ī à¸Ļีà¹Ī +Ġco ÅĽ +ÙģÙĬ دÙĬ +ÙģÙĬدÙĬ ÙĪ +ĠмеÑģÑĤ о +Ġph út +มาà¸ģ à¸ģวà¹Īา +×IJ פ +ب ÙIJ +ĠPh ú +ì± Ħ +ĠÙĪ سÙĦÙħ +à¸Īี à¸Ļ +поÑĤ ÑĢеб +Ġ×Ĺ×ĵ ש×ķת +Ø´ ÙĪ +Ġעצ ×ŀ×ķ +ĠعÙħÙĦ ÙĬØ© +à¸Ħุà¸ĵ à¸łà¸²à¸ŀ +ãģ¾ãģĻ ãģĮ +دع ÙĪ +طر ÙĤ +à¹Ħมà¹Ī à¸ķà¹īà¸Ńà¸ĩ +ë² Ķ +ìĬ ¹ +Ġk ÃŃch +ĠìĹĨ ëĬĶ +ĠÑĤ ам +ĠÙĨ ØŃÙĪ +ĠاÙĦÙĤ اÙĨÙĪÙĨ +×Ĺ ×ķ×Ŀ +Ġk ız +Ġ×ĵ ×Ļף +ĠвÑĢем ени +ãģ£ãģŁ ãĤĬ +ĠØ´ Ùĩر +ĠìĦľ ë¹ĦìĬ¤ +×¢ ש×Ķ +Ġgi ác +ĠاÙĦسÙĦ اÙħ +Ġ×IJ ש +ĠполÑĥÑĩ а +à¸Īัà¸Ķ à¸ģาร +к оÑĢ +Ġ×Ķ×ĺ ×ķ×ij +ราย à¸ģาร +주 ìĿĺ +à¹ģà¸ķà¹Ī ละ +Ġê·¸ëŁ° ëį° +à¸Ĺีà¹Ī à¹Ģà¸Ľà¹ĩà¸Ļ +Ġת ×ķ×ļ +بÙĬ اÙĨ +Ð Ļ +oÅĽci Äħ +ÑĤ ок +Ġà Ķ +ĠÃĶ ng +à¹Ħมà¹Ī à¹ĥà¸Ĭà¹Ī +ãģ¿ ãģ¦ +ÐŁ о +ĠЧ ÑĤо +íĻ © +×ĺ ×ij×¢ +меÑĤ ÑĢ +Ġ×ij ×ŀ×Ķ +Ġ×ij×ŀ×Ķ ׾ +Ġ×ij×ŀ×Ķ׾ ×ļ +Ñĩ ÑĮ +ק ש×Ķ +з нак +знак ом +uj ÄĻ +×Ļצ ר +ĠاÙĦÙħ ÙĦÙĥ +ı yla +×IJ×ŀ ת +à¸Ľ ิà¸Ķ +×IJ ×Ĺ×ĵ +ر اد +Ġm áºŃt +ëĭ¤ ëĬĶ +Ġl ạnh +ש׾ ×ķש +ØŃ Ø¯ÙĬØ« +ت ز +å¹´ ãģ® +Ġк ваÑĢ +ĠкваÑĢ ÑĤиÑĢ +ä½ľ ãĤĬ +رÙĪ ب +ов ан +ĠТ е +à¸Īำ à¸ģ +à¸Īำà¸ģ ัà¸Ķ +ب اط +×Ĵ ת +Ġм аÑĪ +ĠмаÑĪ ин +×Ļצ ×Ķ +ãģ» ãģ¨ +ãģ»ãģ¨ ãĤĵãģ© +ÃŃ do +ĠÑı зÑĭк +à¸ļ ิà¸Ļ +สà¸ĸาà¸Ļ à¸Ĺีà¹Ī +ĠìĹ ´ +ãĤ¦ ãĤ§ +Ġc Ãł +п ан +åı£ ãĤ³ãĥŁ +Ġر د +اÙĤ ت +ĠÙĥ ب +ĠÙĥب ÙĬرة +ÑģÑĤ ал +ש×ŀ ×Ĺ +pos ición +ĠÙħÙĦÙĬ ÙĪÙĨ +ĠìĿ´ ìķ¼ +ĠìĿ´ìķ¼ 기 +Ġh út +ĠÅĽw iat +Ġë°© ë²ķ +ĠÑģв еÑĤ +Ġвиде о +ĠاÙĦÙĨ ظاÙħ +Ġtr á»Ŀi +ĠëĮĢ íķ´ìĦľ +ר ×ŀת +ت داÙĪÙĦ +×ķר ×ĵ +ת ×ŀ +ת×ŀ ×ķ׳×ķת +Ġ×ŀ ף +Ġдв а +Ġ×Ķק ×ķ +æĹ¥ ãģ« +Ġ×Ķ×Ĵ ×Ļ×¢ +à¹Ģà¸ŀิà¹Īม à¹Ģà¸ķิม +Ùħار س +Ġê²ĥ ìŀħëĭĪëĭ¤ +ãģªãģĦ ãģ¨ +Ġnhi á»ĩt +ëIJ ©ëĭĪëĭ¤ +Ġ×ij׳ ×ķש×IJ +Ġê°Ģ ìŀ¥ +Ġv ợ +ĠÄij óng +צ×Ļ׾ ×ķ×Ŀ +ê´Ģ ê³Ħ +в аÑı +×IJ ×Ļ×ĸ +×IJ×Ļ×ĸ ×Ķ +ĠÙĨ ظاÙħ +ÙħØŃ Ø§Ùģظ +Ġt ải +기 ëıĦ +à¸Ľà¸±à¸Ī à¸Īุ +à¸Ľà¸±à¸Īà¸Īุ à¸ļัà¸Ļ +׼ ×ĵ×ķר +ĠìķĦ ìĿ´ +׼׳ ×Ļס +à¹Ģ à¸ķร +à¹Ģà¸ķร ียม +Ġngo ại +ĠدÙĪÙĦ ار +Ġr ẻ +Ġkh Äĥn +عد د +Ø´ عب +czy Äĩ +ĠاÙĦ Ùĥر +ĠÑĩеловек а +ĠÙĪ Ø¥ÙĨ +×IJ ×ĺ +Ġth Æ¡ +ĠاÙĦ رÙĬاض +оп ÑĢедел +опÑĢедел ен +×Ķ ×ŀש×ļ +ĠÐĿ ово +з Ñĭва +ĠاÙĦدÙĪÙĦ ÙĬ +ĠÄij áp +Ġк ÑĢед +ĠкÑĢед иÑĤ +ов ого +Ġm ôn +à¸Ľà¸£à¸° à¹Ĥย +à¸Ľà¸£à¸°à¹Ĥย à¸Ĭà¸Ļ +à¸Ľà¸£à¸°à¹Ĥยà¸Ĭà¸Ļ à¹Į +ÑģÑĤ е +ĠTh á»ĭ +د ÙĬØ© +×ŀצ ×ķ +Ùģ ات +ק ×ĵ×Ŀ +ìĿ´ëĿ¼ ê³ł +ÙĪ Ø® +Ġ×Ĺ ×ĸ +ĠÑĦоÑĤ о +׾ ×Ļת +ت Ùİ +ÙĪ بر +й ÑĤи +ĠÃ¶ÄŁ ren +Ġ×Ķ×ĸ ×ķ +Ġv á»įng +ÙĤÙĪ Ø© +ĠT ây +ĠÐĿ и +Ġש ×ķ×ij +ãģ¨è¨Ģ ãĤıãĤĮ +ãģ© ãĤĵãģª +׊צ×Ļ +ï½ ľ +Ġ×ķ×Ķ ×ķ×IJ +ä¸Ģ ãģ¤ +ĠÑģÑĤо иÑĤ +ni Äħ +×ĺר ×Ļ +ĠдеÑĤ ей +нÑı ÑĤÑĮ +ĠÑģдел аÑĤÑĮ +Ġë§İ ìĿ´ +ä½ķ ãģĭ +ãģĽ ãĤĭ +à¹Ħ หม +à¸ķิà¸Ķ à¸ķà¹Īà¸Ń +Ġ×ij ת×Ĺ +Ġ×ijת×Ĺ ×ķ×Ŀ +ìĻ Ħ +ì§Ģ ëĬĶ +ÑģÑĤ аÑĤ +ÑıÑģ н +ü b +Ġth ả +Ġ×ij×IJ×ŀ ת +Ġt uyến +×ĵ ×Ļר×Ķ +Ġ×IJ ×Ļש×Ļ +×ĸ׼ ר +ãģ° ãģĭãĤĬ +Ġx ét +׼ ×Ļ×ķ +׼×Ļ×ķ ×ķף +diÄŁ ini +ĠاÙĦÙħ ÙĪضÙĪع +Ġh áºŃu +à¸Īาà¸ģ à¸ģาร +×ijס ×Ļס +Ġ×ŀ×Ĵ ×Ļ×¢ +×ij ×Ļ×¢ +ĠÙĪ جÙĩ +à¹ģà¸Ķ à¸ĩ +à¸Ļ าà¸ĩ +ĠÅŀ a +ì ¡´ +ë¡ Ģ +à¸ķ ะ +Ġ×Ķ×Ĺ×Ļ ×Ļ×Ŀ +Ùģ ÙĬد +ãģ§ãģĻ ãģĭãĤī +ê· ľ +ź ni +ĠлÑİ Ð´ÐµÐ¹ +Ġyüz de +ıy orum +ĠاÙĦ بØŃر +e ño +п аÑĢ +ÙĬ ÙĤØ© +об ÑĢ +ר ×ķ×ļ +ت ÙĪÙĤع +ĠاÙĦØ´ ÙĬØ® +åĪĿ ãĤģãģ¦ +ĠÑĤ елеÑĦ +ĠÑĤелеÑĦ он +Ġth ôi +Ġ×Ļ׼×ķ׾ ×Ļ×Ŀ +ĠÅŁ irk +ĠÅŁirk et +Ġìļ°ë¦¬ ê°Ģ +ĠÄij ông +Ġת ×ķ×ĵ×Ķ +ÑģмоÑĤÑĢ еÑĤÑĮ +ĠÙĦ ÙĩÙħ +Ġ׾ ׼ +ĠN ó +ĠØŃ Ø§ÙĦØ© +ãģĦ ãģij +קר ×ķ +az ı +ãĤ³ ãĥ¼ +ĠÙĦÙĦ ت +s ınız +ĠH ải +기 ìĪł +ยัà¸ĩ à¹Ħมà¹Ī +ëĭ¤ ê³ł +פ ×Ĺ +Ġ׾×Ĵ ×ij×Ļ +Ġع ÙĨÙĩ +Ġк аз +Ġказ ино +ب ÙĪر +ÑĦ еÑĢ +Ġê°Ļ ìĿ´ +تس جÙĬÙĦ +ĠاÙĦÙħ رÙĥز +ĠTh ái +д аÑĤÑĮ +×ŀ×Ļ ×Ļ׾ +Ġpay laÅŁ +ãģ¤ ãģ® +à¹Ģร ืà¸Ń +n ça +׳ ×ķ×Ĺ +Ġ×IJ פ×Ļ׾×ķ +ãģ¨ èĢĥãģĪ +ãģ¨ãģĹãģ¦ ãģ¯ +à¹Ģà¸Ī à¸Ń +×ŀ פ +Ġg iriÅŁ +л иÑĤ +ÑĤ елÑı +Ñij н +æ°Ĺ ãģ« +Ġg ó +Ġgó p +åĪĩ ãĤĬ +Ġ×Ķ ×Ĺ×ĵש +ж ал +Ġ×ĵ עת +éģķ ãģĨ +à¹Ģà¸Ĥà¹īา à¹Ħà¸Ľ +Ġס ר×ĺ +e ña +æĸ° ãģĹãģĦ +ر Ùİ +ĠÐIJ ÑĢ +Ġph ản +à¸Īะ à¹Ħà¸Ķà¹ī +Ġ×ijצ ×ķר×Ķ +Ø´ اÙĩ +شاÙĩ د +ÙĪر د +à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ à¸Īาà¸ģ +или ÑģÑĮ +à¹ģละ à¸ģาร +Ġ×Ķ ×ĸ׼ +Ġ×Ķ×ĸ׼ ×ķ×Ļ×ķת +ei ÃŁ +ãĥ ¨ +ìĥ Ī +ĠÃĩ a +Æ ¯ +ש ×Ĵ +ÙĬÙĨ Ø© +ร à¹īà¸Ńà¸ĩ +ãĤµ ãĥ³ +ÑĢоÑģÑģ ий +ÑĢоÑģÑģий Ñģк +a ÄŁa +ĠнаÑĩ ина +Ġص ÙĦÙī +à¸Ĺุà¸ģ à¸Ħà¸Ļ +íļĮ ìĤ¬ +Ġли ÑĨ +Ø´ ÙĬر +ĠØ´ÙĬ Ø¡ +ÙĬÙĨ ا +Ġפ ×Ĺ×ķת +Ġiçer is +Ġiçeris inde +ĠØ£ ØŃÙħد +Ġże by +ì´ Ŀ +Ġп оказ +Ġи менно +หà¸Ļัà¸ĩ ส +หà¸Ļัà¸ĩส ืà¸Ń +ĠÑĤÑĢ е +สัà¸ĩ à¸Ħม +Ø¥ ÙIJ +ãģĮ å¿ħè¦ģ +ÙĬÙij Ø© +פ צ +íĭ ° +ĠÙħ جاÙĦ +׳ פש +к ан +×Ĺ ×ķפ +×Ĺ×ķפ ש +ì²ĺ ëŁ¼ +ов аÑı +з ов +Ġh ạ +Ġdzi ÄĻki +×Ļר ×ķ +Ġ׾ ×ŀצ +Ġ׾×ŀצ ×ķ×IJ +×Ļ×ĵ ×ķ +Ġs ợ +Ġ׾×Ķ ×Ĵ×Ļ×¢ +ק ×ij×¢ +Ġchi á»ģu +ãĥŀ ãĤ¤ +Ġd Ãłng +à¹ģà¸Ł à¸Ļ +Ġü ye +×Ļ׳ ×Ĵ +à¹Ģรีย à¸ģ +ç§ģ ãģĮ +th é +ĠÑĦ илÑĮ +ĠÑĦилÑĮ м +ĠNg Ãły +Ġж ен +Ġжен Ñīин +ج ÙĬد +n ç +à¸Ľ รา +×Ļ×ŀ ×ķ +Ġn á»ģn +×IJ ×ķ׾×Ŀ +Ġвозмож ноÑģÑĤÑĮ +Ġëĭ¤ ìĭľ +è¦ĭ ãģŁ +à¸ĸ à¸Ļ +à¸ĸà¸Ļ à¸Ļ +mız ı +ĠÙħ جÙħÙĪعة +c jÄħ +ĠÐł Ф +à¸ģำ หà¸Ļ +à¸ģำหà¸Ļ à¸Ķ +ĠìŬ 기 +land ı +ни ÑĨ +ÑģÑĤв е +Ġ×ĵ ×ijר×Ļ×Ŀ +Ġsk ÅĤad +ãĤĬ ãģ¾ãģĹãģŁ +ĠоÑĤ кÑĢÑĭÑĤ +нÑı ÑĤ +ĠÑģво ей +à¸Ī ิà¸ķ +ĠкаÑĩеÑģÑĤв е +Ġet tiÄŁi +ìĤ¬ íķŃ +ĠاÙĦÙĬ ÙħÙĨ +иÑĩеÑģки й +ë¸ Į +Ġ×ij×IJר ×¥ +Ġا سÙħ +Ġиз веÑģÑĤ +r ão +Ġatt ivitÃł +à¹Ģà¸Ľà¹ĩà¸Ļ à¸ģาร +ĠاÙĦد Ùĥت +ĠاÙĦدÙĥت ÙĪر +ĠÙĪاØŃد Ø© +ĠÑģ ÑĩеÑĤ +ĠпÑĢ иÑĩ +ĠпÑĢиÑĩ ин +ĠÙĪز ارة +Ġh uyá»ĩn +ĠÙĥ تاب +à¹ģà¸Ļ à¹Īà¸Ļ +à¹ģà¸Ļà¹Īà¸Ļ à¸Ńà¸Ļ +Ġgün ü +г ÑĢÑĥз +ĠاÙĦØ® اص +Ġgör ül +׾ ×ŀ×ĵ +Ġìłķ ëıĦ +×ķ×ij ×Ļ׾ +Ġ×ŀק צ×ķ×¢×Ļ +ĠоÑģоб енно +à¸Ľà¸£à¸° à¸ģา +à¸Ľà¸£à¸°à¸ģา ศ +aca ģını +ë¶ ģ +à¸łà¸¹ มิ +ĠÑį лекÑĤ +ĠÑįлекÑĤ ÑĢо +Ġק ש×Ķ +سÙĦ Ø· +à¸Ĭà¸Ļ ะ +×¢ ×Ļ׾ +ĠЧ е +à¹ģà¸Ļ à¹Ī +lı ÄŁ +lıģ ın +Ġ×ŀ×¢ ×¨×Ľ×ª +好ãģį ãģª +มาà¸ģ à¸Ĥึà¹īà¸Ļ +×ŀ×¢ ×ijר +ĠاÙĦÙħ غرب +ĠпеÑĢ и +ĠпеÑĢи од +Ġnh ạc +ا ÙĪÙĬ +ĠÙĪ عÙĦÙī +أخ Ø° +ĠC ô +תר ×ij×ķת +×Ĵ ×Ķ +Ġktóre j +×IJ ×Ļת +×ij ×ķ×IJ +д елÑĮ +รี วิ +รีวิ ว +ж Ñĥ +Ġ×ij×Ĺ ×ķ +еÑĪ ÑĮ +ĠØ£ ÙĦÙģ +ĠاÙĦÙĪ Ø·ÙĨÙĬ +ĠاÙĦÙħÙĨ Ø·ÙĤØ© +nÄħ Äĩ +Ġthi ên +иÑĩеÑģк ой +ĠاÙĦÙħ ÙĦ +Ġع Ùħ +ס פר +Ġnh óm +ÙĪص Ùģ +ĠCh úng +Ġر ÙĤÙħ +ãģ¾ãģĹãģŁ ãģĮ +al ité +ล ม +ĠëĤ´ ê°Ģ +׾ק ×ķ×Ĺ +ĠS Æ¡n +pos ição +mi ÄĻ +Ġtr ánh +ĠÄIJ á»Ļ +׼ ×Ĺ +ãģĤ ãģ£ãģ¦ +à¸Ńย à¹Īา +Ġ×ŀ×Ĺ ×Ļר +Ġ×Ķ ×Ļת×Ķ +à¸Ľ à¹Īา +à¸Ńืà¹Īà¸Ļ à¹Ĩ +Ø´ ÙĤ +×ł×¡ ×Ļ +ë¦ ¼ +ãģ¦ãģĹãģ¾ ãģĨ +Ġ×ŀ צ×ij +ãģ« åĩº +ÙħÙĪا Ø·ÙĨ +ยัà¸ĩ มี +алÑĮ нÑĭе +san ız +Ø¥ سرائÙĬÙĦ +ĠvÃł i +ì¤ Ħ +ãģ¨æĢĿ ãģ£ãģ¦ +×Ļ ×ķ׳×Ļ +çĶŁ ãģį +Ġs âu +Ñĩ иÑģÑĤ +Ġl á»ħ +ĠGi á +à¸Ńุ à¸Ľ +à¸Ńà¸¸à¸Ľ à¸ģร +à¸Ńà¸¸à¸Ľà¸ģร à¸ĵà¹Į +Ġnh ẹ +r ö +ס ×ĺ×Ļ +ãģķãĤĵ ãģĮ +Ġd ầu +ع Ùİ +ت را +×Ĵ×ĵ ׾ +Ġtécn ica +׼ ׳×Ļ×Ŀ +תק ש +תקש ×ķרת +Ġн его +ét ait +Ġm á»ģm +Ñģ еÑĤ +Ġnh áºŃt +Ġ×ŀ ×¢×ľ +Ġ×Ķ×¢ ×ij×ķ×ĵ +Ġ×Ķ×¢×ij×ķ×ĵ ×Ķ +Ġ×Ĵ ×Ļ׾ +ãģ¯ ãģªãģĦ +ائ ØŃ +Ġз деÑģÑĮ +×IJ ×Ļ׳×ĺר +Ùħ ÙIJ +Ġ×Ļ ×Ĺ×ĵ +ر اÙģ +ì²ĺ 리 +×ĵ ×¢×ķת +ì¹ ľ +ĠТ о +ĠTh ế +ì¶ © +Ġ׳׼ ×ķף +عÙĬ Ø´ +ни з +Ġج اÙĨب +×ŀק צ×ķ×¢ +à¹Ĥ à¸ĭ +Ñģ ÑĥÑĤ +ìĸ´ ìļĶ +ãĤĴè¦ĭ ãģ¦ +ار د +Ġaç ıl +ĠاÙĦØŃ ÙĬاة +à¸ģà¹ĩ à¹Ħà¸Ķà¹ī +ãģĿãĤĮ ãĤĴ +عض ÙĪ +Ġг ÑĢаж +ĠгÑĢаж дан +à¸Īะ à¸ķà¹īà¸Ńà¸ĩ +ĠìĿ´ 룬 +ĠìĿ´ëŁ¬ íķľ +Ġtr ách +ÙĨ Ùİ +Ġkı sa +à Ķ +ÑĪ ка +ãģ® 人 +ĠÐŁ оÑģ +ĠÐŁÐ¾Ñģ ле +Ñĥ лÑĮ +ÙĪا جÙĩ +ÙĤ رب +à¸Ľà¸ıิ à¸ļัà¸ķิ +ê° Ļ +Ġ×ŀ ׳ +ĠÑģво и +بر اÙħج +Ġر ÙĪ +пÑĢ од +пÑĢод аж +Ġby ÅĤy +วั ย +Ġgör ün +Ġà Ī +ÑİÑī им +ĠÑĤак ой +Ùģ ÙĪر +ĠÙģ عÙĦ +Ġб ел +ëIJ ł +er ÃŃa +ĠÑģво Ñİ +Ġl ã +Ġlã nh +à¹Ģà¸ŀืà¹Īà¸Ń à¹ĥหà¹ī +ÙĤ ÙĨ +تط ÙĪÙĬر +Ġsay ı +ĠÑģ ейÑĩаÑģ +Ġ×IJ×Ĺר ת +ק ×ķפ×Ķ +ק×ķר ס +Ġس Ùħ +Ġ×ĺ ×Ļפ×ķ׾ +ìĿ´ëĿ¼ ëĬĶ +دراس Ø© +èµ· ãģĵ +×Ĺ ×Ļ׳ +×Ĺ×Ļ׳ ×ķ×ļ +×ĵ ק +Ġë§ ŀ +Ġком анд +ĠÐij о +Ġиг ÑĢÑĭ +à¸ļ ี +ĠØ£ Ùİ +в ен +ĠاÙĦج دÙĬد +ĠÙĦ Ø¥ +Ġ×ķ×IJ ׳×Ļ +Ġ×Ķס ×Ļ +иÑĩеÑģк ого +رÙĪ ØŃ +à¸ģาร ศึà¸ģษา +ĠTr Æ°á»Ŀng +иг ÑĢа +ıl ması +Ġм аÑģÑģ +ãģ¨ãģį ãģ« +à¸Ĺีà¹Ī à¸ľà¹Īาà¸Ļ +à¸Ĺีà¹Īà¸ľà¹Īาà¸Ļ มา +ĠاÙĦساب ÙĤ +Ġ×ŀ×¢ ×ĺ +в аÑĤÑĮ +m Ã¼ÅŁ +Ġ׾ ׼×ļ +Ġt á»ĭch +Ùģ ÙĩÙħ +تد رÙĬب +Ø´ Ùĥ +Ġ×ij ×ŀ×Ļ +Ġ×ij×ŀ×Ļ ×ķ×Ĺ×ĵ +ÙĤØ· اع +ãģª ãģĹ +×ķצ ×Ļ×IJ +ĠÙĪ سÙĬ +з Ñĥ +Ġy at +Ġyat ırım +ë§ İ +Ġth ắng +ãģĬ 客 +ãģĬ客 æ§ĺ +ĠThi ên +ãģ«å¯¾ ãģĹãģ¦ +ÑĢ иÑģ +ÙĨت ائ +ÙĨتائ ج +Ġ×ŀ שר +Ġ×ŀשר ×ĵ +Ġتع اÙĦ +ĠتعاÙĦ Ùī +ש ׳×Ļ +Ùĩ اÙħ +×IJ׳ ש×Ļ×Ŀ +Ġżyc ia +ĠÑĢÑĥб лей +ÙĬ ض +Ġkat ıl +ĠÙħ ÙĪضÙĪع +Ġvard ır +ĠÙħÙĨ Ø·ÙĤØ© +ĠTr ần +Ġв еÑģ +ü p +Ùħ ÙĪÙĨ +ÑĪ ли +Ġn óng +Ø® ÙĦÙģ +ĠС ÑĤа +Ġд оÑĢ +ĠдоÑĢ ог +ĠwÅĤa ÅĽnie +eÄŁ in +Ġhi á»ĥm +ĠС ам +ê»ĺ ìĦľ +ĠÑĦ а +ãģ» ãģĨ +ãģ»ãģĨ ãģĮ +×ķפ ×Ļ×¢ +ê° Ī +د ÙĪÙĦ +Ġthu ê +Ġch á»Ĺ +Ġëĭ¹ ìĭł +ãģij ãĤĮ +ãģijãĤĮ ãģ© +ë³´ íĺ¸ +ãģķãĤĮ ãģ¦ãģĦãģ¾ãģĻ +Ġнад о +ĠìĤ¬ëŀĮ ëĵ¤ +à¹Ģà¸Ĥ à¸ķ +สม ัย +z ÅĤ +ت ÙĪر +Ġש ת×Ļ +v ê +Ġ×ijת ×ķ×ļ +à¸Ĭ ัย +ãģĦ ãģ£ãģŁ +ìĿ ij +Ġt ầ +Ġtầ ng +ש ׼ר +Ġê¸ Ģ +Ġ×Ķש ׳×Ķ +Ġا ÙĨÙĩ +ç«ĭ ãģ¡ +r és +füh ren +ر ØŃÙħ +ê· ¹ +ĠâĢ « +Ġsu ất +à¸Ł ิ +ÙĬ Ùĩا +ĠاÙĦ اتØŃاد +Ġt uyá»ĥn +ãģ¾ ãĤĭ +Ġm ại +Ġng ân +ãĤ° ãĥ© +欲 ãģĹãģĦ +س ار +ãĤĤãģ® ãģ§ãģĻ +ки е +Ġseç im +åħ¥ ãĤĬ +ãģªãģ© ãĤĴ +ÑĤ ÑĢи +ĠÑģп еÑĨ +ĠØ£ د +Ġод но +ÑĪ ел +ãĥĩ ãĥ¼ãĤ¿ +ãĤ· ãĤ¹ãĥĨ +ãĤ·ãĤ¹ãĥĨ ãĥł +è¡Į ãģį +ãģ¨æĢĿ ãģ£ãģŁ +à¹Ģà¸ģิà¸Ķ à¸Ĥึà¹īà¸Ļ +ĠÑĤ ож +ĠÑĤож е +Ġs ạch +ĠÑģ ÑĢок +Ġкли енÑĤ +ĠÙħØ´ رÙĪع +Ġalt ında +Ġì ·¨ +ä¸Ń ãģ® +ãģķãģĽ ãĤĭ +ãģĻ ãģ¹ +ãģĻãģ¹ ãģ¦ +ê°ľ ë°ľ +ĠÄij êm +ãģªãģĦ ãģ®ãģ§ +ì² ł +×¢ ×ij×ĵ +Ġd ấu +à¸Ħà¸Ļ à¸Ĺีà¹Ī +ĠC ách +تع ÙĦÙĬÙħ +Ġh ại +ãĤ» ãĥķãĥ¬ +ĠÙĨÙģس Ùĩ +ĠíĨµ íķ´ +ÑĪ ло +Ġнап ÑĢав +ĠнапÑĢав лен +ÑĢÑĥ Ñĩ +íĶ Į +Ġ×ijר ×Ļ×IJ +ãģ® ãģ¿ +ãģ«ãģĬ ãģĦãģ¦ +×ij ׳ק +ãĤ¨ ãĥ³ +Ø«ÙĦ اث +Ġm ỹ +ĠÑģай ÑĤе +Ġе мÑĥ +ت غÙĬ +تغÙĬ ÙĬر +خص ÙĪص +ÑĤе ли +Ġ×ķ׾ ׼ף +פע ×Ŀ +Ġпо ÑįÑĤомÑĥ +ر اÙĨ +иÑĤел ей +пиÑģ ан +×¢ ×¥ +ĠìĤ¬ ìĹħ +Ùħ ز +جÙħ ÙĬع +ë©´ ìĦľ +à¸ľà¸¥à¸´à¸ķ à¸łà¸± +à¸ľà¸¥à¸´à¸ķà¸łà¸± à¸ĵ +à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵ à¸ij +à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵà¸ij à¹Į +ĠпÑĢ имеÑĢ +ãĤŃ ãĥ¼ +l â +Ġch Äĥm +缮 ãģ® +ãģĦ ãģĭ +ãģ¨è¨Ģ ãģĨ +×ĸ ×ķ×Ĵ +Ġ×ij ×ĵ×Ļ +Ġ×ij×ĵ×Ļ ×ķק +ãģĬ åºĹ +à¸ķà¸Ńà¸Ļ à¸Ļีà¹ī +Ġph á»iji +п ÑĤ +สà¸Ļ าม +Ø· ÙĪ +ص اØŃ +صاØŃ Ø¨ +ĠD ü +ĠDü nya +Ġп ока +п ал +ĠÄij ảo +ĠاÙĦÙģ ÙĪر +ĠاÙĦÙģÙĪر Ùĥس +Ġmá u +кÑĢ еп +ĠاÙĦس اعة +ĠгоÑĢ ода +Ùģ صÙĦ +ай ÑĤе +Ġд ог +Ġдог овоÑĢ +ĠØ¥ Ø° +Ġ×ij׼׾ ׾ +ÙĬ تÙĩ +×Ĵ ×ijר +Ġbir ç +Ġbirç ok +문 íĻĶ +ãģĿãģĨ ãģª +را ØŃ +ĠÙħ رة +ĠденÑĮ ги +f ä +à¸Ĥà¹īา ว +ĠÑģов ÑĢем +ĠÑģовÑĢем енн +׾×Ĺ ×¥ +èī¯ ãģı +ĠÙģ Ø£ +Ġ×ķ ×ĸ×Ķ +Ġз ани +Ġзани ма +Ġê°Ģì§Ģ ê³ł +Ġh Æ¡i +ãģªãģ® ãģĭ +ãĥĨ ãĥ¬ãĥĵ +Ġר ×ij×ķת +à¸ķ ี +Ġ×ijש ×ł×ª +ĠT ại +Ġthu áºŃn +Ñģ ел +Ñij м +dzi Äĩ +ĠÑģ ка +ĠÑģка Ñĩ +ĠÑģкаÑĩ аÑĤÑĮ +×ķ×ŀ ×ķ +г ла +Ġмин ÑĥÑĤ +åĩº ãģĻ +Ġ×Ĺ×Ļ ×Ļ×ij +Ġת ×Ĵ×ķ×ij×Ķ +à¸£à¸¹à¸Ľ à¹ģà¸ļà¸ļ +ни ÑĨа +ĠÄ° n +ĠØ£ ع +Ġض ÙħÙĨ +Ùħ ثاÙĦ +ĠyaÅŁ an +ĠìĹ° 구 +ĠL ê +ש׾ ×Ĺ +ãģı ãģªãĤĭ +ìĹĨ ìĿ´ +ĠÑĤ ÑĢи +ĠÑĩаÑģÑĤ о +Ġоб ÑĢаÑĤ +п ло +د Ø® +دخ ÙĪÙĦ +س Ùĩ +à¸Ń าà¸ģ +à¸Ńาà¸ģ าศ +Ġ׼ ×ĸ×Ķ +Ġ×Ķ×¢ סק +ĠاÙĦØ£ ÙĨ +å¹´ ãģ« +×¢ ש×ķ +Ġש ×¢×ķת +Ġm Ãłn +×IJר ×Ļ +sı yla +Ùģر ÙĤ +ни Ñħ +Ġت ست +è¦ĭ ãģ¦ +ØŃا ÙĪÙĦ +×IJ ×Ļ׼×ķת +ĠbaÅŁ ladı +st Äħ +stÄħ pi +à¸Ĺีà¹Ī à¹Ģรา +ÙĤر ر +ج اب +Ġ×ijר ×ķר +à¹Ģà¸Ĥà¹īา à¹ĥà¸Ī +×ŀ׊קר +al ım +Ġס ×Ļפ×ķר +ãģ§ãģĤ ãĤĮãģ° +Ġש×ŀ ×ķר×ķת +Ġ×ķ ×ŀ×Ķ +ãģĵ ãģĿ +id ée +ä¸ĭ ãģķãģĦ +تÙĨا ÙĪÙĦ +Ġ ลà¹īาà¸Ļ +Ġìļ°ë¦¬ ëĬĶ +اÙĨ ا +ÑģÑĤ ой +б оÑĤ +ĠyaÅŁ am +kö y +Ø¥ ÙĦ +ÑĢ Ñĭв +기 ìĹħ +Ġ×Ķ×ŀ ×ĵ +Ġ×Ķ×ŀ×ĵ ×Ļ׳×Ķ +د ب +×¢ ×Ļ׳×Ļ +×ŀ ת×Ĺ +Ġפ ר×Ļ +ãĥĭ ãĥ¼ +اÙħ ÙĬ +Ġnh ằm +ãĤĮ ãģªãģĦ +ت عرÙģ +Ġë§Ī ìĿĮ +ìĵ ° +Ġh ấp +ר×Ĵ ×Ļ׾ +ب Ùİ +Ġr Äĥng +gl Äħd +ĠÑģиÑģÑĤем Ñĭ +Ġkh óa +ãģ§ãģĻ ãĤĪãģŃ +大ãģį ãģı +기 를 +Ġké o +ÙĪ Ø¡ +ج اÙħ +جاÙħ ع +Ġ×¢ ×Ļצ×ķ×ij +t éri +Ġת ש +Ġ×IJ ×ij×Ļ +ĠCh Æ°Æ¡ng +à¸ļริ à¹Ģว +à¸ļริà¹Ģว à¸ĵ +ãģ¤ ãģı +Ġ×Ĺ ×ķ׾ +עת ×Ļ×ĵ +ש ×Ļ×ŀ×Ķ +ëĤ ¨ +Ġש×IJ ×Ļף +ĠÙĪاÙĦ Ø¥ +ÑĦ а +Ġkh ám +Ġ×ĺ ×ķ×ij×Ķ +ĠвÑĭ Ñģ +ĠвÑĭÑģ око +ĠاÙĦØŃ Ø¯ÙĬØ« +人 ãĤĤ +d Ã¼ÄŁÃ¼ +×Ļ×Ĺ ×ķ×ĵ +تع ÙĦÙĬ +تعÙĦÙĬ ÙĤ +l ö +تØŃ Ø¯ÙĬد +н его +ĠÑĥд об +Ġ׾ ×ŀ×Ļ +Ġר ×ķצ×Ļ×Ŀ +Ġج اء +Ġ×ij ×ĸ×ŀף +à¸Ľà¸ģ à¸ķิ +é«ĺ ãģı +à¸Ľà¸¥ า +Ġart ık +Ġbug ün +ק ׳×Ļ +Ġkho á +ĠÙħ رÙĥز +ĠìŀIJ 기 +در جة +×ŀש ר×ĵ +Ġgi ấy +Ġch óng +ק פ +ÙĬب Ø© +ĠczÄĻ sto +в али +Ùĥ ب +ìŁ ģ +ส à¸ļาย +à¸Ľà¸£à¸°à¸Ĭา à¸Ĭà¸Ļ +×Ĵ ×ķ×£ +ëŁ ī +ãģ® ãģĵãģ¨ +ล à¸Ń +Ġngh á»ī +åŃIJ ãģ© +åŃIJãģ© ãĤĤ +à¹Ħà¸Ķ à¹īà¸Ńย +à¹Ħà¸Ķà¹īà¸Ńย à¹Īาà¸ĩ +×ĵ ×¢ +ĠاÙĦت Ùī +ĠÑģов еÑĤ +Ġqual itÃł +åĩº ãģĹ +ĠÑĢÑĥк ов +ĠÑĢÑĥков од +ราย ละà¹Ģà¸Ńียà¸Ķ +ãģªãģĭ ãģªãģĭ +기 ê´Ģ +Ġ×Ĺ ×ķש +Ġ×Ĺ×ķש ×ij +л оÑĤ +à¸Ļะ à¸Ħรัà¸ļ +ק×ij ×ķצ×Ķ +Ġth ái +Ġש ×ij×Ķ +ĠÑĪ кол +ĠÙĦ ÙĥÙĦ +à¹ĥà¸Ļ à¸Ĭà¹Īวà¸ĩ +ĠÙħ ÙĥاÙĨ +ë ķĮ +Ġc ải +ĠCh ÃŃ +ÑĥÑĩ а +ìĿ µ +Ġx ảy +à¸Ĭà¸Ļ ิà¸Ķ +Ġc áºŃu +к ÑĢов +ss é +ĠÙĨ ÙĪع +ĠТ а +Ø® Ùħس +פ×ķס ×ĺ +Ġm ắc +ĠÄij em +à¸ģาร à¹ĥà¸Ĭà¹ī +ר ×ķס +ĠÐĽ е +Ġth á»Ń +รà¹Īาà¸ĩ à¸ģาย +üz ü +æĹ¥æľ¬ ãģ® +ê³¼ ìłķ +ש ×Ļ×IJ +ĠìŀĪ ê³ł +×ij ×ķ׾ +ìķ ħ +ĠÙĪاÙĦ ا +ĠÐĽ и +ĠвÑģ Ñij +Ġużytk ow +×Ĺ ×ķ׾ +ر Ùģض +Ġson uç +ãģĦ ãģ¾ãģĽãĤĵ +ìĤ¬ ìĹħ +ëĪ Ħ +ÑĤ ек +Ġud ziaÅĤ +л ез +Ġ×Ķ×Ļ ×Ļת×Ļ +ãĤīãĤĮ ãģ¦ +Ùħس ؤÙĪÙĦ +ر ار +ÑĤ ан +ĠÄij Ãło +Ġר ×ķ×ij +Ġ×ijש×ij ×Ļ׾ +ä»ĬåĽŀ ãģ¯ +ãĤ¸ ãĥ¥ +Ġ×¢ ×ijר +ãģĽ ãģ¦ +п олÑĮ +ak lı +Ġk ÃŃnh +د ت +лож ение +ĠاÙĦÙħ ص +ĠاÙĦÙħص رÙĬ +à¸Īริà¸ĩ à¹Ĩ +ĠاÙĦشر ÙĥØ© +ĠÄij á»ı +ãĥĽ ãĥĨ +ãĥĽãĥĨ ãĥ« +Ñį кон +Ñįкон ом +ĠÙĪ عÙĨ +Ġת ׳ +Ġ×ª×ł ×IJ×Ļ +ĠاÙĦدÙĪÙĦ ÙĬØ© +Ġì§Ģ ìĹŃ +ãģ§ãģĻ ãģĭ +Ġв аÑĢи +ĠваÑĢи анÑĤ +ĠاÙĦع رب +ел а +Ġt Æ°á»Ľng +sk Äħ +Ġm ặc +ส ัà¸ģ +ãĥĵ ãĥ¼ +Ġ×ij ×Ĵ׾ +Ġ×ij×Ĵ׾ ׾ +ãĥķãĤ¡ ãĥ³ +×ij ×Ļצ +×ij×Ļצ ×ķ×¢ +ли ÑģÑĤ +à¸Ł ุ +à¸Łà¸¸ à¸ķ +à¸Łà¸¸à¸ķ à¸ļà¸Ńล +à¸Ŀ à¹Īาย +ìŀIJ ìĿĺ +Ġس ÙĪÙģ +Ġש ×Ķת +Ġê± ¸ +×¢ ×ij×ķ×ĵ +ãģĻãĤĭ ãģĵãģ¨ãģĮ +ĠÑĩа ÑģÑĤÑĮ +ãĤ¢ ãĥ¡ãĥª +ãĤ¢ãĥ¡ãĥª ãĤ« +Ġtak ım +Ġs Ỽ +ĠsỼ m +שר ×Ķ +è¨Ģ ãģĨ +л ан +ì» ¤ +׼ ׳×Ķ +ÙĪÙģ ÙĬ +íĹ Ī +lu ÄŁu +ĠëĮĢ íķ´ +Ġ׾×ij ×Ļת +Ġ×Ķר×IJש ×ķ׳×Ķ +ص Ùħ +Ġsö yled +Ġsöyled i +à¸Ľ าà¸ģ +Ġard ından +ãģĪ ãģŁ +à¸Ĺัà¹Īว à¹Ħà¸Ľ +Ġ׳×ķס ×£ +б олÑĮ +ãĤĵãģ§ãģĻ ãģijãģ© +ĠлиÑĪ ÑĮ +Ġ×ij ×IJ×Ļ +ĠбÑĭ ÑģÑĤÑĢо +ส ัà¸Ļ +Ġ×ij פ׳×Ļ +л еÑĩ +ĠاÙĦØ® بر +Ġsó c +Ġth ú +Ġп ÑıÑĤ +ãģĬ é¡ĺ +ãģĬé¡ĺ ãģĦ +ÑĤ ин +ãģ«ãģ¤ãģĦãģ¦ ãģ¯ +פ ף +Ġдв ÑĥÑħ +à¸į ีà¹Ī +à¸įีà¹Ī à¸Ľ +à¸įีà¹Īà¸Ľ ุ +à¸įีà¹Īà¸Ľà¸¸ à¹Īà¸Ļ +оп еÑĢ +ĠاÙĦب شر +ĠاÙĦÙħ اÙĦ +ıyor uz +تØŃ ÙħÙĬÙĦ +à¸ģ ะ +éĸĵ ãģ« +×Ĺ ×ķש +ĠNg uyên +ãģĦãģ¦ ãģĦãĤĭ +дÑĥ ÑĪ +ש פע +ÑĪ Ñĥ +å®Ł éļĽãģ« +ĠÑĢай он +ĠCh á»ī +ÙĨ صر +Ġìļ ´ +Ġìļ´ ìĺģ +Ġ×Ķ×ĵ ×Ļף +ØŃد د +ر ز +ĠاÙĦد Ùħ +ĠPh áp +ÑĤ ÑģÑı +è¦ĭ ãģĪ +Ġti á»ĥu +Ġs á»Ńa +а ÑİÑĤÑģÑı +ĠB á +Ġ×ķ ׼׾ +Ð ĸ +ÑĪ им +ìĿ´ ëĬĶ +л ев +d ık +Ġprés ente +Ġara ç +صد ÙĤ +Ġпом ог +ĠاÙĦشر ÙĤ +ĠÙĪاÙĦ Ø°ÙĬ +رÙĬ ا +×ij ׳×ķת +Ġng á»ĵi +ר ×ķפ +ר×ķפ ×IJ +Ġth ấp +ãĤĦ ãģ¯ +ãĤĦãģ¯ ãĤĬ +ĠاÙĦج دÙĬدة +éĿŀ常 ãģ« +ÙĬÙĦ ÙĬ +ìª ½ +تع اÙħÙĦ +ãģł ãģ¨æĢĿãģĦãģ¾ãģĻ +Ùħ Ùħ +иÑĤе ли +ãĤµãĤ¤ ãĤº +اد ات +ĠاÙĦÙħ اÙĦÙĬØ© +Ùĥات ب +к ли +веÑĢ Ñħ +ни Ñĩ +Ġ×ľ×¢ ×ij×ķ×ĵ +׾ ×Ļ×Ķ +ØŃ Ùİ +ãĤ¤ ãĥĻ +ãĤ¤ãĥĻ ãĥ³ãĥĪ +Ġת ×Ĵ×ķ×ij×ķת +ÑĦ он +ĠдÑĢÑĥг ие +×IJ ×ĸ×ķר +Ġper ò +ìķ ŀ +åĢŁ ãĤĬ +ר צ×Ļ +×IJ ×ĸ +алÑĮ нÑĭÑħ +Ġê²ĥ ìľ¼ë¡ľ +ĠпÑĢав о +ĠاÙĦØ£ رض +à¹Ģà¸Ĺ à¸Ħ +à¹Ģà¸Ĺà¸Ħ à¹Ĥà¸Ļ +à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļ à¹Ĥล +à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥล ย +à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥลย ี +צ ר×Ļ +ĠÐļ Ñĥ +ıl ma +決 ãĤģ +ا ÙĪ +Ġ×ĵ ק×ķת +à¸Ħร ู +ĠÙħست ÙĪÙī +à¸Ľ à¹īà¸Ńà¸ĩ +à¸Ľà¹īà¸Ńà¸ĩ à¸ģัà¸Ļ +×ĵ ×ķ×ŀ×Ķ +ĠÑģ егоднÑı +س ÙĪÙĤ +ר×Ĺ ×ķ×ij +ĠØ¥ دارة +Ñħ ож +éģİ ãģİ +à¸Ħ à¸Ń +нÑĥ л +×ķ׼ ×Ķ +ÙĪ اÙģÙĤ +׼׾ ׾ +Ġ×Ķ ×ĵ×ķ +Ġl Ä©nh +Ġkh ảo +×IJ×ŀ צע +ë¨ ¸ +Ġ׼ ×Ļצ +Ġ׼×Ļצ ×ĵ +Ġдолж нÑĭ +หว ัà¸ĩ +ãĥĩ ãĤ¶ +ãĥĩãĤ¶ ãĤ¤ãĥ³ +Ġng á»Ŀ +ä¸Ń ãģ« +à¸ģลัà¸ļ มา +جÙħ اÙĦ +à¸Ķัà¸ĩ à¸ģลà¹Īาว +س ÙĥÙĨ +س ÙĨ +Ġözellik le +з еÑĢ +rz ÄĻ +×ŀ ×ķר×Ķ +Ġl ạ +×ŀ ×Ļ׳×Ļ +ר ×Ļת +ãģĿãĤĮ ãģĮ +ãģĭ ãĤĮ +ĠÙĬÙħÙĥÙĨ Ùĥ +öff entlich +г ан +ĠاÙĦØŃ ÙĦ +ĠmiÄĻd zy +ĠÑĩа ÑģÑĤи +ujÄħ cy +ĠbaÄŁ lı +ĠiliÅŁ ki +Ùģ اء +ãĥª ãĥ³ãĤ° +Ġhã ng +ĠконÑĤ ÑĢ +ĠконÑĤÑĢ ол +к оп +ש ×Ļ×¢ +ש×Ļ×¢ ×ķר +ĠÐĴ аÑĪ +Ġ×Ķ תק +ÙħÙĨ ع +ĠpolÃŃt ico +Ġг олов +ĠØ¥ ÙĬ +Ø¥ ÙĨتاج +à¸ļ ิ +Ġг овоÑĢ +ĠговоÑĢ иÑĤ +Ġph á»ķ +ĠÑģем ÑĮ +ãģ¯ ãģĤãĤĬãģ¾ãģĽãĤĵ +ĠÙĪ است +×ŀש פ×ĺ +з ем +×ŀ×ĵ ×ijר +Ġíģ ° +ĠìĿ´ ë²Ī +ê°Ģ ëĬĶ +Ġì§Ģ ìĽIJ +Ġca ÅĤy +Ġgeli ÅŁtir +Ñģк ое +pos é +Ġkh ô +à¸ķิà¸Ķ à¸ķาม +miss ão +Ġ׾ ×ŀר +Ġ׾×ŀר ×ķת +Ġb ó +à¸ķรวà¸Ī สà¸Ńà¸ļ +Ġngh á»ģ +Ġб из +Ġбиз неÑģ +ÑģÑĤ еÑĢ +ÙĪ Ùİ +楽 ãģĹãģ +楽ãģĹãģ ¿ +ãģĵãĤĮ ãģĭãĤī +wiÄħ zan +ส à¸Ńà¸Ļ +Ùħ ÙĪر +׳×ĵ ׾ +Ġ×Ķ×IJ ×ĵ×Ŀ +Ġм олод +ØŃ Ùħا +ØŃÙħا ÙĬØ© +ÑģÑĤ ÑĢан +Ġbu á»ķi +ת×Ļ ×Ļ×Ŀ +abile ceÄŁi +L Ä° +à¹Ģย à¸Ńะ +à¸Ī ร +س ÙĥاÙĨ +à¸Ļ ัà¸Ķ +Ġm ấy +ĠÐij а +s ÅĤaw +ĠÙģ ÙĦا +ĠкоÑĤоÑĢ ой +Ġпло Ñī +ĠплоÑī ад +ãĤĤ ãģĤãĤĬ +sz czÄĻ +×Ļפ ×ķ +ש×ŀ ת +owa ÅĤa +Ġn ông +צ×ij ×IJ +ĠìŀĪ ìĹĪ +ãģ¾ ãģ¨ +ãģ¾ãģ¨ ãĤģ +ÙĤÙĪ ات +ãģ¿ ãĤĵãģª +Ġ׼ ×ŀ×¢×ĺ +Ġx úc +ï¼ Ĩ +r ÄĻ +rÄĻ cz +×ĵ ×ŀ×Ļ +Ġt áºŃn +à¸Ķ วà¸ĩ +ê²½ ìłľ +п ÑĥÑĤ +Ø£ ربع +Ġ×ŀ שת×ŀש +ãĤ¿ãĤ¤ ãĥĹ +Ġìłľ ê°Ģ +Ġ׾ ׼ף +ĠобÑĢаз ом +ÙĬÙĥ ا +w ÅĤ +wÅĤ asn +ĠاÙĦÙĪØ·ÙĨ ÙĬØ© +بÙĬ ب +×ŀ ׾×Ļ +к ÑĢаÑĤ +기 ìĹIJ +ÙĤ اد +ĠÙĦ دÙī +à¸Ħวาม รูà¹ī +×ŀ×ĵ×Ļ׳ ×Ļ×ķת +ê² ¨ +Ġíĺ Ħìŀ¬ +ש ת×Ļ +м ол +Ġmá i +à¸ŀิ ม +à¸ŀิม à¸ŀ +à¸ŀิมà¸ŀ à¹Į +หล วà¸ĩ +Ġx uyên +×Ĺ ×¡×¨ +رÙĪ ÙĨ +ãģĿãģĨ ãģĦãģĨ +ãģĿãĤĮ ãģŀ +ãģĿãĤĮãģŀ ãĤĮ +Ġ׼ ש×Ķ +ÐŁ ÑĢав +×ŀ×ij צע +ع رب +Ġbü yü +פ×Ļת ×ķ×Ĺ +à¸Ī à¸ļ +ĠØ£ Ùĥبر +שר ת +×ŀ׼ ש×Ļר +ĠÙĪ Ùħع +ãģ® ãģŁãĤģãģ« +à¸Ļ ัà¸ļ +ì° ° +ãĥª ãĥķãĤ© +ãĥªãĥķãĤ© ãĥ¼ãĥł +Ġc Æ°á»Ŀng +ĠìłĢ íĿ¬ +ÙħÙĨظ ÙħØ© +Ġhiç bir +ãģ§ãģ¯ ãģĤãĤĬãģ¾ãģĽãĤĵ +ร à¸Ńย +ëIJľ ëĭ¤ +ãģĻãģIJ ãģ« +к ла +Ġürün ler +Ġki á»ĥu +ĠëĤĺ ëĬĶ +ÑĤ ки +Ñģ им +Ġchá»ī nh +ãĤĤ ãģªãģĦ +ศ รี +æĽ¿ ãģĪ +ta ÅŁ +Ġب ÙĥÙĦ +Ġ×ķ ×Ļש +vis ão +ä¼ Ŀ +ä¼Ŀ ãģĪ +ÙĦ د +׾ ×Ļ×ŀ +׾×Ļ×ŀ ×ķ×ĵ +t ória +د Ùij +اÙħ ر +Ġê·¸ëłĩ ê²Į +Ġmateria ÅĤ +à¸Ĺ รา +à¸Ĺรา à¸ļ +ãģ®æĸ¹ ãģĮ +ãģ¦ ãģįãģŁ +ض غ +ضغ Ø· +ĠÙĬ عÙĨÙĬ +ел о +×IJ×Ķ ×ij×Ķ +×¢ ×ŀ +ÅŁ ık +ìŀIJ ëĬĶ +ãĤ¿ ãĥ³ +Ġb áºŃt +×ŀשפ ×Ĺ×Ķ +к ÑĢи +б ли +สั à¸ķ +สัà¸ķ วà¹Į +ĠسÙĨ ÙĪات +ĠPh Æ°Æ¡ng +ãģ¦ãģĹãģ¾ ãģ£ãģŁ +ãģª ãģľ +Ġ×ij×IJ ×ķ +Ġc án +س جÙĦ +Ġl ẽ +ãĤ± ãĥ¼ãĤ¹ +Ġק ×Ļ×ij׾ +à¸ļà¸Ĺ à¸Ħวาม +Ġ×ķ ׼ף +ĠпÑĢедÑģÑĤав лен +Ġn á»iji +Ġcoment ário +ени ем +Ġtá» ı +l Ãł +Ġש×Ķ ×Ļ×Ķ +Ñģл ав +ĠاÙĦ ÙĪÙĦا +ĠاÙĦÙĪÙĦا ÙĬات +ÙĦج ÙĨØ© +ק×ķר ×IJ +бÑĭ ÑĤ +Ġì ¦ +Ġì¦ ī +ãģ§ãģĻ ãģĹ +หรืà¸Ń à¹Ħมà¹Ī +за ÑīиÑĤ +ÙģÙĦ سطÙĬÙĨ +Ġmi á»ħn +à¹Ģย à¹ĩà¸Ļ +ĠçalÄ±ÅŁ an +×Ļ×Ĵ ×Ķ +ĠE ÄŁ +ĠEÄŁ itim +ãĥĥãĤ· ãĥ¥ +Ġоп Ñĭ +ĠопÑĭ ÑĤ +ر غ +رغ ب +ĠÑģво иÑħ +à¸Ľà¸£à¸° à¸ķ +à¸Ľà¸£à¸°à¸ķ ู +Ġ×ŀ×IJ ×ĵ +׼ ×ķ׳×Ļ×Ŀ +à¸Ļ ี +ĠвÑĭ Ñħод +ãģ®ä¸Ń ãģ« +פ ׾×IJ +ĠÙĪ ÙĦÙĬس +פ×ķר ס +פ×ķרס ×Ŀ +Ùħ سÙĦÙħ +Ġng ôi +×ĵ ×ŀ×ķת +ãĤĴ使 ãģ£ãģ¦ +ĠпомоÑī ÑĮÑİ +Ø£ سر +бл ок +ÙĤ Ùĩ +ãģĹãģ¾ ãģĦ +ãģ¨ ãģĹãģŁ +Ġп еÑģ +ãĥī ãĥ« +×Ĺ ×Ŀ +ãģĹãģª ãģĮãĤī +ĠÐŁ ÑĢед +ãĥģãĤ§ ãĥĥãĤ¯ +å¼· ãģĦ +ש ×Ļר×ķת +д аеÑĤ +×Ļ×ij ×ķ +Ġgen ç +ил аÑģ +илаÑģ ÑĮ +ĠبÙĦ د +æĤ ª +æĤª ãģĦ +Ġ×ŀ שת +æ§ĺ ãĢħ +æ§ĺãĢħ ãģª +à¸ĺรรม à¸Ĭาà¸ķิ +ĠÙĥ اÙħÙĦ +ĠاÙĦس Ùħ +×ij×ĺ ×Ļ×Ĺ +c á +g ência +ãĤ¹ãĤ¿ ãĥ¼ +à¸Ĺำ à¸ģาร +×Ļ׾ ת +Ġ×Ļ ×ķצ×IJ +w ój +à¸ļุ à¸Ħ +à¸ļุà¸Ħ à¸Ħล +ع تÙħ +عتÙħ د +ãģĿãĤĮ ãģ« +ĠاÙĦت ارÙĬØ® +ÙĤر اء +Ġyönet im +ק שר +ĠÑģп оÑĢÑĤ +Ġר×IJש ×ķף +Ġseñ al +Ġch ắn +çĦ¡ ãģĦ +ĠдоÑģÑĤ аÑĤ +ĠдоÑģÑĤаÑĤ оÑĩно +Ġá gua +à¸ģร à¸ĵ +à¸ģรà¸ĵ ี +Ġ×ŀש ×ķ +Ġtr ải +ë² Į +ujÄħ cych +Ùģر د +à¹ĥ à¸ģล +à¹ĥà¸ģล à¹ī +ãĤĭ ãģ®ãģ¯ +ר×ķ ×ķ×Ĺ +ÙĨ Ùĥ +ĠاÙĦÙĨ ÙĤ +ãģ®ãģ§ ãģĹãĤĩãģĨ +ãģ®ãģ§ãģĹãĤĩãģĨ ãģĭ +Ùħ عرÙģ +ÙħعرÙģ Ø© +ÑĥÑī е +Ġ×ij×¢ ×Ļקר +ت صÙĦ +Ġ×Ķ×IJ ר +Ġ×Ķ×IJר ×¥ +ĠÅŀ i +à¸Ĥา à¸Ķ +íŀ ĺ +ãģªãĤĵ ãģ¨ +ĠìĤ¬ëŀ ij +l Ã¼ÄŁÃ¼ +ب اء +ĠاÙĦØ¢ خر +Ġfam ÃŃlia +ĠTh áng +Ñī ениÑı +ãĤ¯ ãĥŃ +ĠTh ứ +æĽ¸ ãģį +ен ной +ìŀ ¡ +бл аг +благ о +п ов +à¹ģ ว +à¸ĩ à¸Ħà¹Į +à¸Ńัà¸Ļ à¸Ķัà¸ļ +ãģĤ ãģĴ +ร à¹īาย +ün ün +Ġ×Ļ׼×ķ׾ ×Ķ +з он +ĠÐľ и +маÑĤ еÑĢиал +Ġë³´ ë©´ +ØŃÙģ ظ +ê Ìģ +ãģ« ãģĻãĤĭ +Ġת ×IJ +Ġ×Ķס ×ķ +ĠÑģÑĤ оÑĢ +ĠÑģÑĤоÑĢ он +ãĥĪ ãĥĥãĥĹ +ÅĤo ÅĽÄĩ +ëħ ¼ +ëĵ Ŀ +ĠÙĪاÙĦ ع +ì¶ Ķ +Ġ×Ļצ ×IJ +ĠÑĢаз дел +алÑĮ наÑı +×IJ׳ ש×Ļ +spo ÅĤ +spoÅĤ ec +spoÅĤec zn +Ø¥ عÙĦ +إعÙĦ اÙĨ +ÙĤÙĪ Ùī +íķĺë©´ ìĦľ +تط ÙĪر +Ġsi êu +Ỽ t +д ви +дви ж +Ġqu ần +k ıl +ĠпÑĢи зна +ĠH ã +ĠHã y +ĠباÙĦ ت +man ın +ãĤ« ãĥ« +Ġk á»· +ק ׾×Ļ +ëIJĺ ì§Ģ +تعÙĦ Ùħ +ìĭľ ìĦ¤ +ìĭ ¶ +íĺ ¼ +Ùĥ ÙĬÙģ +売 ãĤĬ +วิ à¸Ĭา +б ал +ĠØ£ ØŃ +Ġдолж ен +รา à¸ĩ +ราà¸ĩ วั +ราà¸ĩวั ล +Ùħ اء +ج ار +Å ļ +Ġ×ŀ×IJ ×ĸ +ר ×ŀ×Ķ +ãģĭãĤĤãģĹãĤĮ ãģªãģĦ +ét ude +czÄħ c +Ġg ór +×ł×¡ ×Ķ +Ùħ ÙĬد +ĠÐŁ еÑĢе +Ø£ خر +ãģĿãģ® å¾Į +à¹Ģà¸Ķียว à¸ģัà¸Ļ +×ŀ ×Ĵ×ķ +×ŀ×Ĵ×ķ ×ķף +д ов +mas ına +×¢ ׳×Ķ +ãĤ± ãĥĥãĥĪ +ס ×¢ +סע ×Ļ×£ +ĠT Æ° +Ġt óc +íĻľ ëıĻ +ĠÐŀ д +ĠÐŀд нако +Ġdol ayı +ؤ Ùĥد +ê³Ħ íļį +׾ ר +в еÑĩ +Ġkh ợi +Ġth ủy +×ĵ ף +ร à¸ģ +à¸ļั à¸ķร +à¹Ģà¸ģ à¹Īา +ĠاÙĦØ« اÙĦ +ĠاÙĦثاÙĦ Ø« +Ġpod rá +ער ×Ļ +ÙĨج اØŃ +Ġkh ắc +ì¸ ¡ +Ä° M +ãĤ» ãĥĥãĥĪ +ż enia +Ġ׾×Ĺ ×ijר +er Ãł +ì ´Ī +Ġkü ç +Ġküç ük +ات ÙĩÙħ +à¸ĭ à¹Į +Ùħشار ÙĥØ© +ĠاÙĦ بط +Ġd ây +ен нÑĭм +à¸Ĺีà¹Ī à¹Ħมà¹Ī +ÙĤ Ùİ +Ġv ượt +Ġtr ì +Ġwp ÅĤyw +A Åŀ +з о +ĠاÙĦس ÙĬد +à¸Ĺะ à¹Ģล +ĠÑģодеÑĢж а +ع Ø·ÙĬ +ĠاÙĦع ÙĨ +èĢħ ãģĮ +à¹Ģ หà¸Ļ +à¹Ģหà¸Ļ ืà¸Ń +Ġb ÃŃ +Ġüzer inden +ĠV Å© +Ġnu ôi +ÙĨ Ùħ +алÑĮ ного +×¢ ×Ļף +ØŃ Ø¶Ø± +ĠоÑĤ дел +ëª ĩ +ìķ ¡ +ĠÙĦدÙĬ Ùĩ +ìĻ ľ +Ġse ktör +Ġвозмож но +ĠÐĶ ж +Ġh ô +äºĭ ãģĮ +иÑĢов ание +алÑĮ ной +Ġ미 êµŃ +ر ØŃÙĦ +ĠÑįк Ñģ +пÑĢав лÑı +Ġnh á»Ŀ +ĠÄij ẩ +ĠÄijẩ y +Ùģ Ùĥر +ĠÙĪØ£ ضاÙģ +ãĥIJ ãĤ¹ +ת×ķ׼ ׳×Ļת +ÑĤел ей +ĠØ¥ÙĦÙĬ Ùĩ +ãģ¨è¨Ģ ãģ£ãģ¦ +Ġдв е +Ġch ấp +ĠL ö +à¸Ħล ิ +à¸Ħลิ à¸Ľ +Ġس ÙĪر +ĠسÙĪر ÙĬا +×ŀ×Ĺ ×ķ +st ä +д об +Ġni á»ĩm +ãģ® 大 +פר×ķ ×Ļק +פר×ķ×Ļק ×ĺ +ĠCh âu +Ġ×ŀ×Ķ ×Ŀ +Ñģк им +ĠполÑĥÑĩ иÑĤÑĮ +ÙĬ ÙĪÙħ +Ø« ÙĪر +פ×ķ׾ ×Ļ×ĺ +פ×ķ׾×Ļ×ĺ ×Ļ +ĠмеÑģÑı ÑĨ +åħ¨ ãģ¦ +ĠاÙĦÙħ جÙĦس +ĠاÙĦت اÙĦÙĬ +Ġ׊ר +åIJij ãģij +׼ ×ŀ×Ķ +б ед +Ø£ عض +أعض اء +ÙĪÙĦ د +วà¹Īา à¸Īะ +Ġb ánh +à¸Ļิ ย +à¸Ļิย ม +à¸Ľà¸£à¸° à¸ģัà¸Ļ +ÑģÑĤав иÑĤÑĮ +à¸ŀ à¸Ļัà¸Ļ +ĠÑį ÑĦÑĦ +ĠÑįÑĦÑĦ екÑĤив +Ġав ÑĤоÑĢ +ĠÄIJ Äĥng +Ġth Æ°á»Łng +ãĤĴ æĦŁãģĺ +à¸ģัà¸ļ à¸ģาร +å¾Į ãģ« +Ġya ÄŁ +ست اÙĨ +Ġli á»ģn +ãģĦ ãģ¾ +i êu +à¹Ĥà¸Ķ à¸Ļ +ĠÙĦ Ø°ÙĦÙĥ +à¹Ĥรà¸ĩ à¹Ģรียà¸Ļ +צ ×Ļ×Ĵ +ĠاÙĦÙħ عÙĦÙĪÙħات +ç§ģ ãģŁãģ¡ +à¸Ĺีà¹Ī à¸Ħุà¸ĵ +ãģ«ãģª ãģ£ãģ¦ãģĦãĤĭ +×ŀ×ĵ ×Ļ׳×Ķ +ס ׼×Ŀ +Ġв не +à¸ŀ à¸Ļัà¸ģà¸ĩาà¸Ļ +ÑĢ ей +à¹Ģà¸Īà¹īา หà¸Ļà¹īาà¸Ĺีà¹Ī +ĠHi á»ĩn +Ġméd ico +ĠتØŃ ÙĤÙĬÙĤ +ÑĮ ÑĤе +miÅŁ ti +ÙĤÙĬ ادة +ãĤı ãģĭãĤĬ +มา à¸Īาà¸ģ +ëħ Ģ +ãģ«éĸ¢ ãģĻãĤĭ +×IJר×Ĵ ×ķף +m ètre +Ġעצ ×ŀ×Ļ +ĠCh úa +รูà¹ī à¸Ī +รูà¹īà¸Ī ัà¸ģ +ì£ Ħ +ëĭ µ +à¹ģà¸Ĺ à¹ī +Ġgeç en +Ġlan ça +ĠاÙĦ بØŃØ« +×ĵ ×ŀ×ķ +ãģ¯ ãģĺ +ãģ¯ãģĺ ãĤģ +Ġdön Ã¼ÅŁ +è¿ij ãģı +à¹Ģส ม +à¹Ģสม à¸Ń +ëĿ ½ +Ġü ç +á» ŀ +ÑĪ аÑı +à¸Ĺ ร +ØŃ ÙĤÙĬÙĤØ© +à¸Ĥà¸Ńà¸ĩ à¸ģาร +Ġ무 ìĹĩ +Ġ×Ķ ׼ר +ĠاÙĦص ÙĬÙĨ +ĠлÑİ Ð´Ð¸ +à¸ķ าย +ب ÙĪÙĦ +Ġvi êm +Ġthi á»ĩu +à¸ģ à¸Ķ +Ġ׾ ×ĵ×ijר +פ ׳×Ķ +×IJר ×ij×¢ +س Ùī +ĠاÙĦسÙĬ اس +ĠاÙĦسÙĬاس ÙĬØ© +yd ı +ÙĪØŃد Ø© +ĠдеÑıÑĤелÑĮ ноÑģÑĤи +Ġ×ķ×Ķ ×ŀ +п еÑĩ +пеÑĩ аÑĤ +иÑĢов аниÑı +ĠÑģ ог +ĠÑģог лаÑģ +Ġ׼ ×ĵ +Ġ׼×ĵ ×IJ×Ļ +ĠиÑģполÑĮзов аÑĤÑĮ +ס פ×ķר×ĺ +Ġil çe +exp érience +ĠTh á»Ŀi +Ä° K +à¹Ħà¸Ł à¸Łà¹īา +ëĵ¤ ìĹIJê²Į +à¸Ľà¸£à¸° à¹Ģà¸ł +à¸Ľà¸£à¸°à¹Ģà¸ł à¸Ĺ +Ġmü mk +Ġmümk ün +Ġ×IJ×ķת ׳×ķ +ìĦ± ìĿĦ +ĠìĿ´ ìľł +زÙĬ ارة +Ġolduk ça +r ób +ĠØ£ ÙĨا +Ġ×Ķ ×ij×Ļ +Ñģ ен +×¢ ×Ļקר +×Ļ×ĵ ×ķ×¢ +d zÄħ +Ùħ عÙĦÙĪÙħات +Ø´ اب +Ġpar ça +à¸Ļะ à¸Ħะ +ب اس +ĠÑĤоÑĢ г +ĠÑĤоÑĢг ов +Ġ×Ĺ ×ĵר +׼ ר×ĺ +׼ר×ĺ ×Ļס +ĠA yrıca +ÃªÌ £ +ìľ ¨ +ĠÑĤак ие +Ġ×ŀצ ×ķ×Ļ +ãĥ©ãĥ³ ãĤŃãĥ³ãĤ° +ש×Ļ×ķ ×ķק +åīį ãģ® +ĠB ảo +Ñī Ñĥ +æĹ© ãģı +ĠPh òng +à¸ŀระ ราà¸Ĭ +פ ×Ĺ×ķת +Ġг л +Ġгл аз +à¸Ĺ à¹Īา +Ġd ạy +ÑĢ оÑģÑĤ +à¹Ĥà¸Ķย à¹Ģà¸īà¸ŀาะ +Ġqu áºŃn +Ġ×Ĺ×ijר ×ķת +m ême +mÄ±ÅŁ tı +ĠاÙĦت داÙĪÙĦ +Ġn ạn +Ġ×Ķ ×ĵ×Ļ +ĠاÙĦØ· رÙĬÙĤ +×Ĵ ×ķת +Ġ×Ķ ×ĵר×ļ +ujÄħ ce +Ġch ữ +ãĤĤãģ® ãģ® +ë° Ľ +ãģķãĤĵ ãģ¯ +Ġyard ım +ĠاÙĦع Ùħ +Ġì§Ħ íĸī +Ġ×Ļ ×Ĺ +Ġ×Ļ×Ĺ ×¡×Ļ +ĠاÙĦÙħ دÙĬÙĨØ© +Ġc ú +à¸ģี ฬ +à¸ģีฬ า +Ġni ên +mis ión +׳×Ļס ×Ļ +׳×Ļס×Ļ ×ķף +Ġвоз ÑĢаÑģÑĤ +Ġ×¢×ķש ×Ķ +ĠÙħ دÙĬر +Ñı ÑģÑĮ +ØŃ Ø¬Ùħ +íĻĺ ê²½ +ĠاÙĦØ£ خرÙī +u ÃŁer +ĠاÙĦعاÙĦÙħ ÙĬØ© +ĠNg á»įc +êµIJ íļĮ +ä¸Ĭ ãģ§ +×Ļ×Ķ ×ķ×ĵ +×Ļ×Ķ×ķ×ĵ ×Ļ×Ŀ +Ùħس اعدة +Ġжиз нÑĮ +ĠпоÑĤ омÑĥ +ĠاÙĦÙħ ÙħÙĦ +ĠاÙĦÙħÙħÙĦ ÙĥØ© +ĠG ör +ر ÙIJ +×ŀק ×ķ×ŀ×ķת +åĩºæĿ¥ ãĤĭ +ÑĦ ÑĤ +ĠìĿ´ ìłľ +ĠÑĢ ем +ĠÑĢем онÑĤ +ת ×ķ×ļ +æĻĤ ãģ¯ +ãĤīãĤĮ ãģªãģĦ +alt ı +家 ãģ® +ĠاÙĦØ¥ عÙĦاÙħ +리 ëĬĶ +ãģĭãĤī ãģ¯ +ĠH ạ +ãģĤ ãģ® +×ĵ×Ļ ×ķף +رÙĬ س +Ġsoci etÃł +ĠاÙĦÙĥ بÙĬر +Ġ×ij ×ŀס +Ġ×ij×ŀס ×Ĵר +Ġ×ij×ŀס×Ĵר ת +ĠìŀĪ ìľ¼ë©° +Ġn ặng +Ùĩ Ùī +ĠB Ãł +×ŀר ×ķ +Ġj ÄĻ +ĠjÄĻ zy +ĠjÄĻzy k +Ġ׼ ×ŀ×ķ×ijף +×¢ ׾×Ķ +à¸Ĺีà¹Ī à¹Ħà¸Ķà¹ī +ãģ¾ ãģĹãĤĩãģĨ +×ŀס פר +Т Ðŀ +سÙĬاس Ø© +Ġкажд Ñĭй +ë² ł +t ım +y á»ĩn +ร ีà¹Ī +ĠдеÑĤ Ñģк +วิà¸ĺี à¸ģาร +m ówi +×ĺ×¢ ×Ŀ +×Ķצ׾ ×Ĺ×Ķ +ض ÙĬÙģ +ĠÑħоÑĤ Ñı +ãĤĵãģ§ ãģĦãĤĭ +à¸Ħา à¸Ķ +à¸Ħร à¸ļ +Ġк ÑĥÑĢÑģ +ĠbaÅŁ arı +×ijר ×ķ +ÙĬع Ø© +ĠÐĿ Ñĥ +à¸Ħวาม à¹Ģà¸Ľà¹ĩà¸Ļ +Ġ׾ ×ŀש׾ +Ġì¢ĭ ìĿĢ +Ùħؤس س +Ùħؤسس ات +Ġpréc is +Ġth ảo +à¸ģà¹ĩ à¸Ħืà¸Ń +Ġש ׼׾ +führ ung +ãģĦ ãģ§ +à¹ģละ มี +à¸ģà¹ĩ มี +Ġש ש +м ел +Ġкни г +ĠباÙĦ ÙĨ +ĠباÙĦÙĨ سبة +Ġald ı +ÑĤ ай +Ġ×Ĺ×ĵ ש×Ļ×Ŀ +å®Ł ãģ¯ +ع ÙĪا +ĠìĿĺ 미 +из м +ÑĢабоÑĤ аÑĤÑĮ +Ùģ ص +Ġ×ij׳ ×ķסף +ãģ¨ãģĹãģ¦ ãĤĤ +à¹Ģà¸Ľà¹ĩà¸Ļ à¸Ĺีà¹Ī +ĠÑģлед ÑĥеÑĤ +èĢĥãģĪ ãģ¦ +Ġ׼ ×Ļ×ķ×Ŀ +ÑģÑĤ Ñĭ +׼׾׼ ׾×Ļ +æµģ ãĤĮ +ãĤĴ ãģ¤ãģij +Ñĩ аÑĤ +×Ļ׼ ×ķף +×Ļר ×Ļ +ları yla +ãĤ¤ ãĥ¡ +ãĤ¤ãĥ¡ ãĥ¼ãĤ¸ +׳×ĸ ק +Ġci ò +Ġs ın +Ġsın ır +à¸Ļ à¸Ħร +к аÑĤ +Ġl á»Ĺi +ëŀ Į +تÙģ اص +تÙģاص ÙĬÙĦ +ëĨ ĵ +ĠÙħ ض +il miÅŁ +بار Ùĥ +ÐĿ Ðĺ +Ġth ẩm +Ġ×IJ×ķת ×ļ +ĠпÑĢин им +ĠпÑĢиним а +Ġyö nt +Ġyönt em +Ġ×ŀק ×ij׾ +Ġktó rego +ê· Ģ +شر Ùģ +د اÙħ +ãģĦãĤį ãģĦãĤį +ĠAl ém +Ġgör ü +Ġgörü nt +Ġgörünt ü +د س +ÑĪ ки +г ÑĢад +Ġl ạc +Ġs ữa +ãĤīãĤĮ ãģ¾ãģĻ +o Ãłi +Ñī ен +ãģĭ ãģªãģĦ +Ġп оп +Ġпоп Ñĥ +ĠпопÑĥ лÑıÑĢ +ĠاÙĦÙħ ÙĪÙĤع +rä g +ï¼ ¡ +íķ Ħ +ãĤĴè¦ĭ ãĤĭ +اÙħ ا +ĠاÙĦØŃ Ø±Ø¨ +ĠÐŁ а +Ġ׾ ×IJתר +Ġt á»ijc +×ij ׾×Ķ +ر ئÙĬس +в Ñĥ +ÙĬ دÙĬ +каз ан +Ġ׊ש×ij×ķף +h ôtel +×¢ ×ķ׳×Ķ +ب ÙĨÙĬ +×ŀ ×ķ׾ +Ġд нÑı +éĽ£ ãģĹãģĦ +вед ениÑı +Ġ×ķ ×ŀת +н апÑĢимеÑĢ +ÙĤ ابÙĦ +Ġrésult at +ĠÑĢазвиÑĤ иÑı +ر Ùij +ìłĦ 문 +ĠاÙĦÙħ زÙĬد +ĠìľĦ íķ´ìĦľ +ëĨ į +íĻ ķ +ĠThi ết +íĮ ¨ +malı dır +Ġcz ÅĤ +ĠczÅĤ owie +ĠczÅĤowie k +ĠÙĦ بÙĨ +ĠÙĦبÙĨ اÙĨ +üs ü +ãģªãĤĵ ãģł +Ġżyc ie +ĠÑħоÑĢоÑĪ о +æĸ¹ ãģ« +ëĭ¤ ë©´ +иÑĩеÑģ каÑı +ער ×Ļ׼ +ער×Ļ׼ ת +ãģ¾ãģĽãĤĵ ãģ§ãģĹãģŁ +ĠÑģоб ой +Ġg á»Ĺ +Ġдел аÑĤÑĮ +da Äĩ +аÑĢ а +róż ni +à¹Ģล ีà¹ī +à¹Ģลีà¹ī ย +à¹Ģลีà¹īย à¸ĩ +à¸Ŀ าà¸ģ +Ġت ÙĤ +ĠتÙĤ دÙĬ +ĠتÙĤدÙĬ Ùħ +หà¸Ļ ุà¹Īม +Ġmü cade +Ġmücade le +ì§Ģ 를 +ãĤ¤ ãĤ¹ +ĠØ£ ساس +jÄħce go +ĠÅŁ eh +н ÑĤеÑĢ +ÑĨи Ñİ +ï» » +ÑİÑī его +à¹Ĥà¸Ľà¸£ à¹ģ +à¹Ĥà¸Ľà¸£à¹ģ à¸ģรม +Ġmie Äĩ +ØŃÙĥÙĪÙħ Ø© +ãģ§ãģĹãģŁ ãģĮ +×Ļס ×Ķ +ãĤĤãģ® ãĤĴ +Ġ×ŀ ×IJת +สุà¸Ķ à¸Ĺà¹īาย +Ġc Å© +ÙĨ سب +ĠпÑĢ оÑĩ +Ġд ней +ĠÑįÑĤи Ñħ +׾ ×ŀת +нÑı Ñı +Ñį к +Ġì§Ģ ëĤľ +มหา วิà¸Ĺยา +มหาวิà¸Ĺยา ล +มหาวิà¸Ĺยาล ัย +d ão +ĠMá y +ĠêµŃ ê°Ģ +à¸ļุ รี +×Ĵ ×Ļ׾ +ĠÑĤÑĭ ÑģÑı +ĠÑĤÑĭÑģÑı Ñĩ +Ùģ Ùĥ +ĠÐĺ Ñģ +è¡Į ãĤıãĤĮ +פר ×ĵ +ãģ¤ ãģį +à¸Ħร à¸Ńà¸ļ +à¸Ħรà¸Ńà¸ļ à¸Ħรัว +à¸Ĥึà¹īà¸Ļ มา +ä»ĬæĹ¥ ãģ¯ +ĠìĤ¬ëŀĮ ìĿ´ +עצ ×ŀ×Ķ +п оÑĢ +ĠK ỳ +Ġ Æ¡n +Ġth Äĥm +Ùģ اÙĤ +ãģļ ãģ« +Ġ׾ קר +Ġ׾קר ×ķ×IJ +اÙģ ÙĬØ© +Ùħ Ùİا +г аÑĢ +ص ÙĦا +صÙĦا Ø© +Ġ×ŀ ×ĸ×Ķ +lı ģını +Ġ×IJ ×Ļ׳×Ķ +к ÑĢо +Ġng Æ°Æ¡i +Ġв ним +Ġвним ание +jÄħ cy +ÙĢÙĢÙĢÙĢ ÙĢ +Ñģ Ñħод +ãģªãĤĵ ãģĭ +×ŀ ×Ļ׾ +Ġ×Ķ×IJ ×Ĺ +ãĤı ãģªãģĦ +ع سÙĥر +ĠìĦ¸ ê³Ħ +ĠÑĩ его +ĠÑģÑĢед ÑģÑĤва +ĠÐł аÑģ +ãģª ãģģ +ÙĨ Ùģس +ר×Ļ ×ķף +Ñģ Ñĥд +ĠìĿ¸ ê°Ħ +ĠاÙĦÙħ ÙĤبÙĦ +ÙĨ عÙħ +تÙĪ Ùģر +ש ×ij×¢ +ı lm +ılm Ä±ÅŁ +Ġ×ľ×ª ת +تص Ùģ +×Ķפ ×ķ×ļ +à¹ĥà¸Ļ à¸Ľà¸µ +ìĿ´ ê³ł +Ùģ ÙĪز +à¸ľà¸¥ à¸ĩาà¸Ļ +ĠGi áo +à¸ļà¸Ńà¸ģ วà¹Īา +Ġd Ä±ÅŁ +ĠdÄ±ÅŁ ında +ì£ ½ +Ġdzie ÅĦ +к ÑĨии +и ÑĨе +ãģ® ä¸Ģ +ع Ø´ +пÑĢ еÑģÑģ +หà¸Ļ à¹Īà¸Ńย +ลัà¸ģษ à¸ĵะ +Ġpossibilit Ãł +à¹Ħà¸Ķà¹īรัà¸ļ à¸ģาร +หย ุà¸Ķ +Ġphi ên +çĶŁ ãģ¾ãĤĮ +Ø· ÙĪÙĦ +ÑĦ ин +f ür +ØŃ ÙĬاة +íĸ ĪìĬµëĭĪëĭ¤ +׼ ׳×ķת +à¸Ľà¸£à¸° ส +à¸Ľà¸£à¸°à¸ª à¸ļ +à¸Ľà¸£à¸°à¸ªà¸ļ à¸ģารà¸ĵà¹Į +ëIJĺ ìĹĪ +Ġkaż dy +Ġl uyá»ĩn +ĠоÑĢганиз аÑĨии +å°ij ãģªãģı +ÑģÑĤÑĢо ен +Ġtécn ico +ק ×Ķ׾ +Ġ×ķ×IJ ×Ĺ +ĠعÙĦÙĬ Ùĥ +Ñī ение +Ġ×Ķ ×Ļ׾×ĵ×Ļ×Ŀ +ÙĪس ائÙĦ +Ġ×ķ ×Ķת +تÙħ ÙĬز +ĠÑģ казал +Ġпол и +Ġ×Ķ×ŀ ס +ÙĦÙij Ùİ +Ùħؤس سة +Ġ×ŀ ×Ļ×ĵ +ãģ£ ãģ¡ +ĠëĦĪ 무 +à¸ŀ ี +Ġt ặng +Ġt ấn +ר ש×Ŀ +Ġméd ica +Ġ×¢ ×ķ×ŀ +Ġ×¢×ķ×ŀ ×ĵ +ÑĦ оÑĢ +Ùħر Ø© +Ġvat anda +Ġvatanda ÅŁ +Ġдел о +à¸Ļ ม +ãģ¨ åIJĮãģĺ +Ùģ Ùī +Ñģ оÑĢ +Ġ×Ķס ר×ĺ +Ġép oca +ìłķ ì±ħ +ĠÑģвÑıз ан +ض رب +ĠÙĦ ÙĨا +Ġuży wa +ĠاÙĦج ÙĬØ´ +Ñİ ÑĢ +×ijס ×ķ×£ +Ġм Ñĥ +ĠмÑĥ зÑĭк +bilit é +Ġma ç +س Ùİ +ت ÙĦÙĥ +ãģ ¬ +ÙĬ ÙĦا +ÑĪ ла +ÙĢÙĢ ÙĢ +Ġод ной +зв ан +ĠÑģ ÑĢаз +ĠÑģÑĢаз Ñĥ +ÙĨ ظÙħ +را Ùĩ +ĠÙĦÙĩ ذا +׼ ×ķר +Ġ×Ķש ×ij×ķ×¢ +Ġ×Ķש ת +ĠQu ảng +ãĥ« ãĥ¼ +ãģĪ ãģªãģĦ +×ĺ ×IJ +Ġmi á»ģn +ĠPh áºŃt +ĠاÙĦس ÙĪÙĤ +Ä Ĥ +ĠاÙĦج Ùħع +ĠاÙĦجÙħع Ø© +ÑİÑī ей +a ÅĤem +عت ÙĤد +Ø£ ÙĦÙħ +Ñģ ке +ĠìĿ´ íķ´ +ÙĨس Ø® +è¨Ģ ãģĦ +д обав +سب ÙĤ +×¢×ķר ר +ÑĤи п +ãģĿãģĵ ãģ§ +vis ión +عÙĪد Ø© +ë¨ ¹ +×ŀ ×ĸר×Ĺ +ĠØ¥ ØŃ +Ġ׾×ij ×Ļף +Ġ׾צ ×IJת +Ġyard ı +Ġyardı mc +Ġyardımc ı +Ä° Z +ק פ×Ķ +tr é +liÄŁ ini +клÑİÑĩ а +Ġüret im +Ġa yrı +ĠkiÅŁ iler +à¸Ħ à¹īà¸Ļ +à¸Ħà¹īà¸Ļ หา +ĠS á»± +Ġ׼ ס +Ġ×Ľ×¡ ×£ +ĠÑĤак иÑħ +ĠXu ân +Ġл ег +Ġлег ко +Ø«ÙĤ اÙģØ© +ÐĿ Ðŀ +ãĤ¹ãĤ¿ ãĥĥ +ãĤ¹ãĤ¿ãĥĥ ãĥķ +åIJĪ ãģĦ +Ġ×Ķש ×Ļ×ŀ×ķש +man ız +ĠÐĴ аÑģ +g ün +ìľĦìĽIJ íļĮ +Ġwsp óln +ĠÑģв ое +í ĥģ +à¹Ģà¸Ļ ีย +ÙĪب Ø© +в Ñıз +ı dır +ëIJĺ ìĹĪëĭ¤ +ĠdeÄŁi ÅŁtir +ãĤĭ ãģĵãģ¨ãģĮ +Ġ×Ĺ×ĵ ש×Ķ +ãĤīãĤĮ ãģ¦ãģĦãĤĭ +×Ĺ×Ļ ×Ļ×ij +ĠÐļ аÑĢ +׳×Ļת ×ķ×Ĺ +Ġק×ĺ ף +ר ×ĸ +ÙĪ غ +èªŃ ãģ¿ +Ġت ÙĤÙĪÙħ +ĠÙĥ اÙĦ +à¸Ŀ ึà¸ģ +Ġë°ľ ìĥĿ +ológ ico +ر اع +à¹ģà¸ģà¹ī à¹Ħà¸Ĥ +ĠÑĢабоÑĤ Ñĥ +ÙĨÙij Ùİ +à¸Ńยูà¹Ī à¸Ĺีà¹Ī +ĠاÙĦØ« اÙĨÙĬØ© +ĠNh ân +Ñħ ваÑĤ +ö ne +Ġع دة +à¹ģ สà¸ĩ +ÑĤ оп +пÑĥÑģ ка +شر اء +ĠÐļ ом +Ġפע ×ķ׾×Ķ +ìĤ¬ ìĿ´ +ìĤ¬ìĿ´ íĬ¸ +è¡Į ãģ£ãģ¦ +Ġ×Ķ ×Ķת +ĠÑģÑĤ оÑĢо +ĠÑģÑĤоÑĢо нÑĭ +در س +à¸ĭ ู +à¸ķà¹Ī ำ +ĠØ£ بÙĬ +под об +ãģ« ãģ¦ +ار تÙģاع +ĠÙħ ؤ +ик ов +ge führt +มืà¸Ń à¸ĸืà¸Ń +ĠÙĦ ÙĤد +ĠØ£ÙĨ Ùij +سÙĬ طر +ãģ¾ãģļ ãģ¯ +ס ×ĵ +Ñģк олÑĮко +ãģ¿ãģŁãģĦ ãģª +×ĵר ×Ĵ +×¢ ×Ļ×ĵ +à¹ĥหà¹ī à¸ļริà¸ģาร +ĠÐĶ и +×ij×¢ ×Ļ×ķת +Ġ×Ķ×Ĺ ×ķ +пиÑģ ÑĮ +ĠاÙĦØ® ÙĦ +б ав +ĠÄ° lk +ĠاÙĦØ® Ùħ +ĠاÙĦØ®Ùħ ÙĬس +ĠÙĬ ÙĤÙĪÙħ +æĻĤ ãģ® +ĠsÅĤ ow +ĠØ£ ÙĩÙħ +Ø®ÙĦ ÙĤ +ĠØ£ صبØŃ +Ġchứ a +Ġth ác +Ùģ اÙĦ +Ġch á»Ŀ +ĠاÙĦØ® ار +ĠاÙĦخار ج +ĠاÙĦخارج ÙĬØ© +Ø· ائر +Ġt Ãł +ĠtÃł u +à¸ģล à¹īà¸Ńà¸ĩ +ĠاÙĦÙħر Ø£ +ĠاÙĦÙħرأ Ø© +åħ¨ ãģı +ĠÃĸ n +çļĦ ãģ«ãģ¯ +Ġpiè ce +×Ĵ ×Ļ×ij +ĠاÙĦ ÙĪاÙĤع +ä»Ĭ ãģ® +ĠاÙĦÙħ ÙĤ +cz nÄħ +Ùģع اÙĦ +ен ного +ĠÑĦак ÑĤ +ìĭł ì²Ń +ĠÐŀ ни +ĠاÙĦبÙĦ اد +ов иÑĩ +ëı Į +ÑĦ ÑĥнкÑĨи +Ġìĸ´ ëĬIJ +ãĥķãĤ© ãĥ¼ +d ÃŃ +ил оÑģÑĮ +Ùħ Ùī +ĠاÙĦØ£ÙħرÙĬ Ùĥ +ĠاÙĦØ£ÙħرÙĬÙĥ ÙĬØ© +×ĺ ×Ļפ×ķ׾ +íĶĦ ë¡ľê·¸ +íĶĦë¡ľê·¸ ëŀ¨ +Ġש ×ķ׳×ķת +Ø´ ÙħÙĦ +ĠпаÑĢ а +Ġ×Ķ×Ĺ ×ķק +ÙĪز ارة +ãģ¨ ãģĻãĤĭ +Ġqu ảng +ĠaÄŁ ır +ĠاÙĦÙĦ ج +ĠاÙĦÙĦج ÙĨØ© +ê¸ ´ +ĠT ân +ج ÙħÙĦ +д ол +à¹ģà¸ŀ à¸Ĺย +à¹ģà¸ŀà¸Ĺย à¹Į +Ġר×IJ ש×Ļ +Ñī ей +Ġçev re +Ġкомп лекÑģ +Ġ×ij ×ŀש×ļ +Ġalt ın +ĠØ£ عÙħاÙĦ +ĠÑģво его +ãĤĪ ãģĦ +×Ĺ׾ ×Ļ×ĺ +×ŀ׳ ×¢ +Ġר ×ij×Ķ +ĠØ£ÙĬضا Ùĭ +×ĸ ׾ +ĠاÙĦسÙĬ اسÙĬ +æĢĿ ãģĨ +קר ק +קרק ×¢ +ĠاÙĦÙģ رÙĬÙĤ +б иÑĤ +ק ׳×Ķ +ĠØ¥ ÙĨÙĩ +ĠÐĴ ам +Ðł Ðŀ +ãĥĪ ãĥª +å¿ħè¦ģ ãģª +Ġch âu +ç¶ļ ãģij +Ġçöz üm +gÅĤ ow +ع ÙĤÙĦ +売 ãĤĭ +i ết +à¸Ĭิ à¹īà¸Ļ +ĠØŃÙĤ ÙĪÙĤ +Ø·ÙĦ ع +ĠÄij en +ĠÙĥ اÙģØ© +ãģ® ãģĶ +Ġë ¬ +Ġë¬ ¼ +Ġ물 ë¡ł +Ġرس ÙĪÙĦ +з ам +зам ен +Ġkullan ıcı +×¢ ×ķ׾ +èī² ãĢħ +ÑĪи ÑĢ +Ġ׊ש +Ġwy gl +Ġwygl Äħda +ש ×Ļ×ŀ×ķש +å¿ĺ ãĤĮ +×¢ ×Ļצ×ķ×ij +ĠاÙĦس ÙĪرÙĬ +å°ij ãģªãģĦ +Ġпо иÑģк +สำ à¸Ļัà¸ģà¸ĩาà¸Ļ +Ġ×ŀצ ×ĵ +Ġmü ÅŁ +ĠmÃ¼ÅŁ ter +ĠmÃ¼ÅŁter i +ĠÙħÙĨ ÙĩÙħ +à¸ķำ à¹ģ +à¸ķำà¹ģ หà¸Ļ +à¸ķำà¹ģหà¸Ļ à¹Īà¸ĩ +ÅĽ mie +Ġש ×ł×ª +Ġ×Ķ פ×Ļ +פר ש +×¢×ijר ×Ļת +สà¸Ļ ัà¸ļ +สà¸Ļัà¸ļ สà¸Ļุ +สà¸Ļัà¸ļสà¸Ļุ à¸Ļ +è¨Ģ ãģ£ãģ¦ +à¸ģาร à¸Īัà¸Ķ +ĠMo że +из аÑĨии +ứ t +ĠÙĪب عد +ĠdeÄŁ ild +ĠdeÄŁild ir +Ġת ×ŀ +Ġ×ŀ×ŀ ׳×ķ +話 ãĤĴ +ĠÑĨ ена +Ġth úc +×Ļ×ŀ ×ķף +ĠB áo +ãĤĴ åıĸãĤĬ +å®ī ãģĦ +Ġ×¢×ķש ×Ļ×Ŀ +èĩªåĪĨ ãģĮ +l ée +ãĤĭ ãģ®ãģ§ +иÑĢÑĥ еÑĤ +ãģ¦ ãĤĭ +ست ر +ĠاÙĦØŃ ÙĬ +×Ļ׾ ×ķת +Ġ×Ĺ ×ij +ÙĤر Ø£ +تÙħ ÙĥÙĨ +س ائÙĦ +prü f +ãģĭ ãģijãģ¦ +ĠÑģоб ÑģÑĤвенно +ĠìľĦ íķĺìŬ +׾ ×Ļ×ĺ +ãģĮ å¤ļãģı +ÙĬت Ùĩا +ç«ĭ ãģ¦ +ม à¸Ńà¸ļ +ìĭľ ìŀ¥ +оÑĢ а +Ġs avaÅŁ +×ĺ×Ļ×ij ×Ļ +×ij ׳×ķ +Ùħا ذا +기 ê°Ħ +ãģªãģ© ãģ§ +Ġ×ŀ ת×Ĺ×Ļ׾ +Ġnhi á»ħ +Ġnhiá»ħ m +ка ÑĢ +каÑĢ ÑĤ +Ġ׾×Ķ שת×ŀש +׳ ×Ļ×Ĺ +اد ÙĬØ© +ราย à¸ĩาà¸Ļ +Ġprzy kÅĤad +Ñī ий +ØŃض ÙĪر +Ġh ôn +à Ŀ +ת ×ķצ×IJ×ķת +راب Ø· +Ġb ếp +ĠполÑĥÑĩ и +åĩºä¼ļãģĦ ç³» +à¸Ľà¸¥ à¹Īà¸Ńย +ĠاÙĦØ´ باب +اÙĩ ÙĦ +ä»Ĭ ãģ¾ãģ§ +رج ع +ãĤ¶ ãĥ¼ +ÙĤ Ùģ +ĠGro ÃŁ +ĠíļĮ ìĽIJ +اج ر +Ġ×ij×ŀ קר×Ķ +Ġseg urança +fü hl +ãģ¦ ãģĦãģı +หม à¸Ń +ĠкоÑĤоÑĢ ом +ĠN Äĥm +ĠdÅĤ ugo +ÙħÙĨ ØŃ +ש×ķ ×ķ×Ļ +ĠØ£ÙĬ اÙħ +ส à¸łà¸²à¸ŀ +r zÄħ +شر Ùĥات +ãĤĴ èĢĥãģĪ +д аÑĢ +à¸Ľà¸£à¸° à¸Ĭุม +Ġ×ķ×IJ ×ĸ +i á»ĩn +Ġt Æ°Æ¡i +ש ×Ļ×Ĺ +à¸Ń à¹Īà¸Ńà¸Ļ +æĽ¸ ãģĦãģ¦ +Ġng ữ +×ij×Ļ×ĺ ×Ĺ +×ij×Ļ×ĺ×Ĺ ×ķף +Ġs ẵ +Ġsẵ n +ì§Ģ ëıĦ +ĠпÑĢ еп +ĠпÑĢеп аÑĢаÑĤ +Ġна ÑĥÑĩ +ĠÃľ nivers +ĠÃľnivers ites +ĠÃľniversites i +Ġ×Ĵ×ĵ ×ķ׾×Ķ +Ġ×Ķ ×ł×ª +Ġ×Ķ×ł×ª ×ij×¢ +ãģ§ãģĤ ãģ£ãģŁ +Ġmies iÄħ +ĠmiesiÄħ c +г ÑĢам +гÑĢам м +Ġبش Ø£ÙĨ +ĠÑħ ÑĢ +ק ×Ļ×ĵ +ק×Ļ×ĵ ×ķ×Ŀ +Ø´ Ùĥر +Ġ á»ķ +Ġá»ķ n +ãģĮãģĤ ãģ£ãģ¦ +ãģķãĤĮ ãģ¾ãģĻ +Ġ×Ĺ ×ķ×ĵ +Ġ×Ĺ×ķ×ĵ ש×Ļ×Ŀ +ÙħÙĪا جÙĩ +ÙħÙĪاجÙĩ Ø© +أش خاص +ب غ +à¹Ģรียà¸Ļ รูà¹ī +ãģĹãģ¦ ãģĦãģı +Ġs ạn +å¿ħ ãģļ +׳ ×Ļ×Ĵ +׳×Ļ×Ĵ ×ķ×ĵ +باÙĦ غ +׊ש×ŀ +×Ĺש×ŀ ׾ +Ġnap raw +Ġnapraw dÄĻ +Ø´Ùĩ اد +×IJ ×ķ×Ķ +×IJ×ķ×Ķ ×ij +и ÑĨÑĭ +Ġ×Ķ ר׼×ij +ëŀ ij +Ġת ×¢ +Ġ×Ķ ×Ļש +Ġ×Ķ×Ļש ר×IJ +Ġ×Ķ×Ļשר×IJ ׾×Ļ +Ø£ ÙħÙĨ +ÑİÑī аÑı +sk ór +LER Ä° +Ġ×Ķ×IJ×Ĺר ×ķף +×¢ ׳ק +ĠÙĪ ÙĥÙĦ +ãģĵãģĵ ãģ§ +Ġqu án +liÄŁ in +à¸ģà¸İ หมาย +Ø· Ùħ +Ø£ جÙĩ +أجÙĩ زة +ĠEr doÄŁan +ãģ§ ãģĬ +Ġв ÑĢа +ĠвÑĢа Ñĩ +ĠPh ó +à¸Ĭั à¹Īว +à¸Ĭัà¹Īว à¹Ĥม +à¸Ĭัà¹Īวà¹Ĥม à¸ĩ +Ġph úc +×Ļפ ×ķת +×¢×Ļ ×ķף +Ġduż o +ãĥģ ãĥ¼ãĥł +ĠÙĬ Ùİ +Ġзад аÑĩ +Ġ×Ĵ×ij×ķ×Ķ ×Ķ +Ġ׼ ׼׾ +лож ен +ét at +Ġng Äĥn +èµ· ãģį +ĠTi ến +ص عب +Ġexperi ência +Ø® Ùħ +à¸ģาร à¸Ĺำà¸ĩาà¸Ļ +س ÙĬد +ĠD á»± +ĠкоÑĤоÑĢ ого +lad ıģı +Ġkh á»ķ +Ġê³Ħ ìĨį +Ñī ик +สà¹Īวà¸Ļ à¸ķัว +з оÑĢ +ÙĨ Ùı +Ġ à¸Ķัà¸ĩ +Ġà¸Ķัà¸ĩ à¸Ļัà¹īà¸Ļ +Ġc ấu +ĠÄij á»ijc +о ÑĦ +ĠاÙĦØ£ عÙħاÙĦ +ãģªãģı ãģ¦ãĤĤ +×ķ׼ ×Ļ×Ŀ +à¹ģ à¸Ľ +ĠB ên +ãĥ¯ ãĥ³ +Ġgi ám +ĠÅŀ u +Ġd áng +ع ÙĦÙĬ +à¹Ģà¸ģ ษ +à¹Ģà¸ģษ à¸ķร +ÙĪج ب +н нÑĭе +ÙĤ ضاء +à¸Ħว à¸ļ +à¸Ħวà¸ļ à¸Ħุ +à¸Ħวà¸ļà¸Ħุ ม +ãģ¤ ãģ¤ +ĠVi á»ĩc +×ŀ×ij ×ĺ +ש×Ļת ×ķ×£ +Ġв едÑĮ +k aza +kaza ÅĤ +à¸ķำ รวà¸Ī +ãĤ¿ ãĥ« +Ġпов Ñĭ +ĠповÑĭ ÑĪен +ĠS ợ +ĠìĦ¤ ëªħ +ĠÃĩ ünkü +ìĥĿ íĻľ +Ö ¾ +ãĤĮ ãģ¦ãģĦãĤĭ +Ġ×ij ר×IJש +ר ×ķ×Ĵ +Ġо ÑĦи +ĠоÑĦи ÑĨиалÑĮн +ĠÑĥ ÑģÑĤанов +ĠÑĥÑģÑĤанов лен +ĠاÙĦÙħ صر +ĠاÙĦÙħصر ÙĬØ© +ĠÐŁÐ¾ ÑįÑĤомÑĥ +ÙĨ صÙģ +ĠÙĪاÙĦ ÙĨ +Ġh Ãłi +à¸Ħ ิ +ĠApr ès +ì³ IJ +à¹Ģà¸ĭ ีย +×ĵ ×ŀ×Ķ +activ ité +à¸Ħิà¸Ķ วà¹Īา +ÑĤ ÑĢен +à¹Ģ ฮ +ãĥı ãĤ¤ +ãģĮ å¢ĹãģĪ +ен наÑı +Ġìĺ¤ ëĬĺ +ãĥ¢ ãĥ³ +Ġкон еÑĩно +ĠÙħÙĤ ابÙĦ +cl é +Ġh ü +Ġth ẳng +ìłģ ìĿ´ +ĠÐIJ лекÑģ +ĠÐIJлекÑģ ан +ĠÐIJлекÑģан дÑĢ +ãĥŀãĥ³ ãĤ·ãĥ§ãĥ³ +ãģ²ãģ¨ ãģ¤ +ãģª ãģĬ +à¹Ģà¸Īà¹īา à¸Ĥà¸Ńà¸ĩ +ëĵľ 리 +Ø´ اء +ĠsaÄŁ lık +ĠÅŁ imdi +×Ļ×IJ ׾ +تأ Ø«ÙĬر +Ø£ سب +أسب اب +ĠвÑĭполн ен +л ок +ש ×Ļ×ij×Ķ +Ġl ắm +ĠTr Æ°á»Ľc +Ġ×Ķ×¢ ׾ +리 를 +ĠÑĢ еж +ĠÑĢеж им +int é +inté gr +×Ĵ ׳×Ļ +ĠاÙĦØ´ عر +Ġmil hões +Ġpeque ño +ãĤ³ ãĥ¼ãĤ¹ +×ķ׼ ×Ĺ +à¹Ģà¸Ĭ à¹īา +شر ÙĤ +Ġh Æ°Æ¡ng +รัà¸IJ à¸ļาล +à¸ģล าย +à¸ģลาย à¹Ģà¸Ľà¹ĩà¸Ļ +Ġпод Ñħод +תש ×ķ×ij×Ķ +ãģıãģª ãģ£ãģ¦ +ĠاÙĦØ£Ùħ Ùħ +ĠH á»įc +ĠwspóÅĤ pr +ĠwspóÅĤpr ac +Ñĩ Ñĥв +ÑĩÑĥв ÑģÑĤв +ÃŃst ico +à¹Ģà¸ģ าะ +ìĽ Ģ +Ġназ ад +ãĤĭ ãĤĪãģĨãģ« +ĠС Ш +ĠСШ ÐIJ +м он +ĠAs ÃŃ +×ķר ×Ĵ +полн ен +×ŀס ׾ +×ŀ×¡×ľ ×ķ׾ +à¹Ģลืà¸Ń à¸Ķ +à¹Ģริà¹Īม à¸ķà¹īà¸Ļ +ĠاÙĦØ¥ Ùħ +ĠاÙĦØ¥Ùħ ارات +צ×Ķ ר +ãĥ¡ãĥª ãĥĥãĥĪ +ĠпоÑĤ ом +в из +ĠÙģ ترة +å¾Į ãģ® +ÐĿ ÐIJ +×ŀס ר +ÙĬر ÙĬ +pr é +Ġte ÅŁek +ĠteÅŁek kür +Ġöd eme +د اÙĨ +ãģ¾ ãģĹãģ¦ +缮 ãģ« +ĠÑĤ еÑĩение +l ard +lard ır +à¹Ģรา à¸Īะ +ס פ×Ļ +ĠÙĪÙĥ Ø°ÙĦÙĥ +Ġh át +Ġt á»Ļc +à¸Ħุ ย +Ġb ức +ØŃ ÙĬÙĨ +èģŀ ãģĦãģ¦ +Ùħؤ شر +ĠNh Æ° +Ġмен ее +ละ à¸Ħร +Ñģ ин +ĠÑĢ ек +ĠÑĢек л +ĠÑĢекл ам +ĠÙģ ÙĩÙĪ +Ġ׾ ×ĸ +×Ļ׳ ×ķת +ĠÅŁ art +ÑģÑĤав ка +Ġíı¬ íķ¨ +ãģ«è¡Į ãģı +ï¼ Ŀ +ĠпозволÑı еÑĤ +Ġת×ķ׼ ׾×ķ +ов ал +صÙĦ Ø© +Ġ׾ש ׳×ķת +ĠÐĺ гÑĢ +ÙħÙĨتج ات +Ġsat Ä±ÅŁ +Ñģ ко +ĠاÙĦØ«ÙĦاث اء +Ġ×Ķ×ĵ×ijר ×Ļ×Ŀ +ãģĹãģ¾ ãģĹãĤĩãģĨ +بÙĤ Ùī +åĬĽ ãĤĴ +ĠÃĩ ok +ãĥģ ãĥ¥ +à¹Ģà¸Ĭ ืà¹īà¸Ń +ยุ à¸Ħ +ศา ล +Ġק×ķ×ĵ ×Ŀ +×ĸר ×Ļ×Ŀ +ãģ® åł´åIJĪ +ĠìķĬ ìķĺ +ãģĤãĤĬãģ¾ãģĻ ãģĮ +×IJ שר +è¡Į ãģı +ãģ» ãģĭ +æ°Ĺ ãģ«ãģªãĤĭ +й деÑĤ +íķĺìĺĢ ëĭ¤ +ستÙħر ار +ĠÐŁÑĢ е +ĠÑģ боÑĢ +ĠìķĦ 무 +ç§ģ ãĤĤ +ع ص +Ġн иÑĩ +ĠниÑĩ его +ĠпÑĢи ем +ק ×ķ×ŀ +ĠìĪĺ ëıĦ +Ġì ¡´ +Ġì¡´ ìŀ¬ +ĠØ£ Ø«ÙĨ +ĠأثÙĨ اء +ĠÙĪاÙĦ ØŃ +ãģĮ ãģ§ãģįãĤĭ +Ġת ×Ķ +Ġת×Ķ ×Ļ×Ķ +ר ף +ĠÑģвÑıз и +×Ĵ שת +Ñģп екÑĤ +ס ×ij×Ļ×ij +ס×ij×Ļ×ij ×Ķ +ĠíķĦìļĶ íķľ +ت خصص +Ġж ив +Ġжив оÑĤ +ĠMay ıs +تع ا +تعا ÙĪÙĨ +ĠعÙĨ Ùĩا +ów ki +ĠاÙĦÙģÙĦسطÙĬÙĨ ÙĬ +ãģłãģijãģ§ ãģªãģı +ìĿ¸ ì§Ģ +ĠاÙĦس ÙĪد +ĠاÙĦسÙĪد اÙĨ +إجراء ات +Ġkö tü +Ġ×Ļ תר +×Ĵ ×Ļש×Ķ +Ġצ ×ķר×ļ +รà¸ĸ ย +รà¸ĸย à¸Ļà¸ķà¹Į +Ñħ оÑĤ +Ðł ÐIJ +ÙĪ Ø·ÙĨ +Ġsay ısı +ס ×Ĺר +Ùħ ÙĪÙĦ +ãĤĴæĮģ ãģ£ãģ¦ +ع اÙĨ +Ġt á»Ļi +ĠвÑĭ ÑĪе +Ġt ầm +ãĥĪ ãĥ¬ +×Ļצ ×ķ +ม ุม +س ÙĪد +ìłĦ ìŀIJ +ãĤµ ãĥŃãĥ³ +ìĤ° ìĹħ +ĠоÑģнов ан +Ø® Ùģض +רצ ×Ķ +بÙĬ ض +×ķÖ ¹ +ס×Ļ ×Ļ×¢ +Ġש ×IJ×Ļ +ĠاÙĦÙĤر Ø¢ÙĨ +ĠТак же +×ŀש ×ŀ×¢×ķת +س ÙĩÙĦ +Ġ×Ķ ׳×Ķ +ãĤĴ ãģĹãģ¦ãģĦãĤĭ +×Ļ ×Ļס +×Ķ ×ķ×IJ +ĠB ÃŃ +Ġмал о +ĠëĶ°ëĿ¼ ìĦľ +Ġר ×Ĺ×ij +ãģĮ é«ĺãģĦ +ÙĪ اس +ìĤ ¼ +׳ ×¢ +ãģ£ ãģ¡ãĤĥ +ĠT üm +à¸Ńีà¸ģ à¸Ķà¹īวย +ãģĹãģ¦ ãģıãģłãģķãģĦ +ÙĨØ´ اط +ãĥĹ ãĥ©ãĥ³ +али ÑģÑĮ +×ĵ ×ľ×ª +Ġwc zeÅĽ +ĠwczeÅĽ niej +ĠÑįÑĤ им +Ġthá»ĭ t +à¸ļ ัà¸į +à¸ļัà¸į à¸Ĭี +ãģļ ãģ£ãģ¨ +ÑĢ ин +Ġswo jÄħ +íķĺëĬĶ ëį° +Ġë§Įëĵ¤ ìĸ´ +تش Ùĥ +تشÙĥ ÙĬÙĦ +ائ Ùĩ +Ġ׾פ ×Ĺ×ķת +ãĥĭ ãĥ¥ +ãĥĭãĥ¥ ãĥ¼ãĤ¹ +׼×IJ ף +ãģ§ãģį ãģŁ +зв он +Ġsta ÅĤ +×Ĺ×ijר ת×Ļ +ĠØ£ عÙĦÙĨ +à¹ģà¸ļà¸ļ à¸Ļีà¹ī +بد Ø¡ +ãĤģ ãģŁ +Ġ×ŀש ×ŀ×¢×ķת +Ġ×ŀש×ŀ×¢×ķת ×Ļ +ör ü +Ġh ạnh +z ähl +ĠL ý +Ġ×ij ×Ķת +Ġ×ij×Ķת ×IJ×Ŀ +б аÑĢ +ì¦ Ī +ä»ĬåĽŀ ãģ® +Ġy ü +Ġyü ks +Ġyüks el +ãĤ½ ãĥ¼ +ãģĤ ãĤĮ +ת ׾×ŀ×Ļ×ĵ +ãģ¤ ãģª +×ij ׳×Ļ×Ŀ +Ġx ếp +ĠмÑĥж Ñĩин +ĠاÙĦÙĥ تاب +׼ ×ŀ×ķת +Ġç e +Ġçe ÅŁ +ĠçeÅŁ it +ĠçeÅŁit li +×ĵ ×Ļר×ķת +à¸ļุ à¸į +ĠاÙĦØ¥ ÙĦÙĥ +ĠاÙĦØ¥ÙĦÙĥ ترÙĪ +ĠاÙĦØ¥ÙĦÙĥترÙĪ ÙĨÙĬ +ĠباÙĦØ¥ ض +ĠباÙĦإض اÙģØ© +Ġyö nel +Ġyönel ik +mys ÅĤ +à¸Ķà¹īวย à¸ģาร +à¸ģาร à¸Ĺำ +ов Ñĭм +Ø£ زÙħØ© +æİ¢ ãģĹ +íļ ¨ +Ġ×ķ×IJ ×Ŀ +Ġnghi êm +ÑĪ ин +ка л +Ġcrian ças +èĩªåĪĨ ãģ§ +Ġн ай +Ġнай ÑĤи +ĠS á»ij +ĠÃ¶ÄŁrenc iler +ãĥ¶ æľĪ +Ñģ ан +ĠJ á +ĠkonuÅŁ ma +شر Ø· +ëĪ Ī +ar rière +ضر ÙĪرة +ãĥĶ ãĥ³ +×¢ שר +аÑĢ ÑĮ +جÙħ اع +Ġdé co +Ġ×Ļ×Ķ ×ķ×ĵ×Ļ +à¸ŀ ลาà¸Ķ +ĠÙĬ ÙĥÙĨ +Ġج اÙħعة +Ø· بÙĤ +Ġbo ÅŁ +×ķ ×ķ×IJ +×ŀ×ĵ ×¢ +ק×ij×ķצ ת +פ ×Ļר +jÄħc ym +ÙħØ´ ا +Ùħشا ÙĥÙĦ +צ פ×ķף +Ø¥ ست +×ŀ׼ ר +سÙħ ع +Ġкак ой +ÑĤ воÑĢ +ØŃ Ø¬ +Ùģر ض +пÑĢав лен +Ġник ак +Ġmi á»ĩ +Ġmiá»ĩ ng +ü ÃŁ +иÑĢов ал +׾ ×ŀ×ķת +次 ãģ® +ÙĦ Ø· +à¸ķ ัà¸Ļ +×Ķ ת×Ĺ×Ļ׾ +Ġfoto ÄŁ +ĠfotoÄŁ raf +طر ØŃ +à¸Ńà¸Ńà¸ģ à¹Ħà¸Ľ +Ġy ên +Ġп ок +Ġпок Ñĥп +ĠпокÑĥп а +ÑĨ Ñĥ +Ġкомп ÑĮÑİ +ĠкомпÑĮÑİ ÑĤеÑĢ +ĠاÙĦÙĥ رÙĬÙħ +تص Ùħ +تصÙħ ÙĬÙħ +Ġоказ а +Ġzar ówn +Ġzarówn o +ëĮĢ ì¶ľ +ãĤ»ãĥ³ ãĤ¿ãĥ¼ +Ġjako ÅĽci +æĤ © +æĤ© ãģ¿ +Ø£ÙĨ ÙĪ +Ø£ÙĨÙĪ اع +ë¹ ł +Ġìłķ ë§IJ +Ġk ẻ +ĠÑģай ÑĤа +Ġ×Ķ ער×ij +Ùĩ ز +pres ión +ĠÑģÑĤ ен +ãģ£ãģ¦ ãĤĭ +Ġhız lı +Ðļ ÐIJ +×ŀשפ ×Ĺת +ĠÙĨ Ùĩا +ĠÙĨÙĩا ÙĬØ© +ãģ¾ ãģĦ +о ÑħÑĢан +ร à¹īà¸Ńย +ล ึà¸ģ +ĠÙĪب اÙĦ +ãĤĤãģ® ãģĮ +ר׼ ×Ļ×ij +ãĤ¤ ãĥ¤ +س ؤ +سؤ اÙĦ +ĠÙĦØ£ÙĨ Ùĩ +ĠkonuÅŁ tu +Ðļ ÑĥпиÑĤÑĮ +Ġש×IJת ×Ķ +ĠÙĪاÙĦ س +Ġmożliwo ÅĽci +Ġpró b +ëĶ ° +ãģ© ãĤĮ +ĠÐľ ин +ĠоÑĢганиз м +ãģ«å¯¾ ãģĻãĤĭ +ĠPr é +Ġpriv é +ch è +ãģĦãģŁãģł ãģį +สà¸Ļุ à¸ģ +ajÄħ ce +ĠD zi +ĠDzi ÄĻki +ÅĤat w +r än +rän k +æĿ¥ ãģŁ +Ġ×Ķ×Ļ×Ķ ×ķ×ĵ×Ļ +ãĤ¬ ãĥ¼ +ĠÑĢаР´ +ĠÑĢад и +к ÑĤив +Ø£ Ùĩد +Ø£Ùĩد اÙģ +ש ×IJ×Ļר +ãģ¦ ãģĦãģªãģĦ +Ġfr üh +Ġок ол +Ġокол о +Ġreg ião +ĠÑĩиÑģ ле +Ġpon iew +Ġponiew aż +ìĦ¼ íĦ° +Ġb ầu +Ġê · +Ġê· ľ +Ġê·ľ ìłķ +ĠH òa +ĠÑĤ оÑĤ +ãĤĤ å¤ļãģĦ +ĠاÙĦإسÙĦاÙħ ÙĬØ© +ãģĭ ãģĦ +Ñį н +ĠÑĥказ ан +ĠÑĤак ое +ï¼ ³ +ëĮĢ íķĻ +Ġgen iÅŁ +ĠاÙĦØ® ÙĬ +ĠاÙĦØ®ÙĬ ارات +ãĤĴè¡Į ãģĨ +ש ×ŀ×Ķ +ĠLÃł m +ÙĪÙĨ ÙĬ +Ġ×IJ ׾×Ļ×ķ +Ä ĺ +à¹Ħมà¹Ī สามารà¸ĸ +人 ãģ¨ +بر ز +×Ļס ×ķ×ĵ +×Ĵ ׾×Ļ +ĠÙĬ ÙĨا +ĠÙĬÙĨا ÙĬر +ĠкаÑĢÑĤ ин +Ġt ôn +à¹Ģ à¸ģร +à¸Ħ à¸Ķี +Ġ׾×IJ ×ķר×ļ +ãĤĤãĤī ãģĨ +ãģĭ ãģĭãĤĭ +ани и +Ġara ÅŁtırma +ÙĦاØŃ Ø¸ +ãģĦ ãĤĦ +ĠT Ãłi +Ġ à¸Ļà¸Ńà¸ģà¸Īาà¸ģ +Ġà¸Ļà¸Ńà¸ģà¸Īาà¸ģ à¸Ļีà¹ī +ĠÄIJ ảng +ãģ£ãģ¦ ãģįãģŁ +Ġà¸ĭึà¹Īà¸ĩ à¹Ģà¸Ľà¹ĩà¸Ļ +Ġt ả +Ġmożliwo ÅĽÄĩ +ĠS ản +ĠÄ° ki +Ġc ắt +س Ø£ÙĦ +Ġbak ım +Ø´ ب +à¸ķ ีà¹ī +à¸ŀ ยาย +à¸ŀยาย าม +สั à¸Ľ +à¸ªà¸±à¸Ľ à¸Ķา +à¸ªà¸±à¸Ľà¸Ķา หà¹Į +ë° Ģ +еÑĢ Ñĭ +Ġc ánh +Ġthu ế +ت بع +ãģ«åħ¥ ãĤĮ +Ñİ ÑģÑĮ +íļĮ ìĿĺ +ç°¡ åį +ç°¡åį ĺ +ç°¡åįĺ ãģ« +Ġtr úc +ĠاÙĦÙĥ ÙĪÙĬ +ĠاÙĦÙĥÙĪÙĬ ت +ãĤıãģij ãģ§ãģĻ +ĠÑģв об +ĠÑģвоб од +ĠÑĥÑĩаÑģÑĤ ник +สิ à¹īà¸Ļ +ĠпÑĢо ÑĦеÑģÑģиона +ĠпÑĢоÑĦеÑģÑģиона лÑĮн +Ñģп оÑĢ +×Ĺ ×ķ×ij×Ķ +Ùħع ÙĨÙī +ĠاÙĦÙģ ترة +สูà¸ĩ สุà¸Ķ +ãĤı ãģļ +ĠÄij è +ĠÄijè n +æ¯Ķ ãģ¹ +า à¸ĺิ +Ġmoż emy +à¹ģ à¸ĭ +à¸Īะ à¹Ħมà¹Ī +Ġs ắp +Ðļ Ðŀ +Ġprá ctica +ÙĪÙĥ اÙĦØ© +è¾¼ ãĤĵãģ§ +ológ ica +Ġе Ñī +ĠеÑī Ñij +تع دÙĬÙĦ +ĠØ£ Ùĥد +Ġצר ×Ļ׼ +Ġצר×Ļ׼ ×Ļ×Ŀ +Ø« Ùħ +Ġк ÑĢÑĥ +ĠкÑĢÑĥ п +×ij×Ļק ×ķרת +Ġì¡° ê¸Ī +ãģ¨ãģį ãģ¯ +Ġb ạc +ĠÑĢаÑģ пол +ĠÑĢаÑģпол ож +ĠÑĢаÑģполож ен +ز ÙĬÙĨ +ĠÐļ ÑĢоме +ĠاÙĦÙĨ ظر +×Ķ ×ķ×ĵ +ĠاÙĦس بت +ãģ¨æĢĿ ãģĦ +Ġpa ÅĦst +ĠpaÅĦst w +ĠÙĦÙĬ ست +ĠбÑĥд Ñĥ +à¸Ĺัà¸Ļ à¸Ĺี +ร าม +ØŃ ØµÙĪÙĦ +ãģĹãģ¦ãģıãĤĮ ãĤĭ +ĠاÙĦØ¥ سرائÙĬÙĦ +ĠاÙĦإسرائÙĬÙĦ ÙĬ +ãģĵãĤĮ ãģ¾ãģ§ +ìĤ¬ 를 +Ġs ürü +à¹Ģว à¸Ńรà¹Į +à¹Ģà¸ĭ à¸Ńรà¹Į +Ġutilis é +ĠÑģиÑģÑĤем а +Ġdw ó +Ġdwó ch +Ġpróp rio +Ġëĵ± ìĿĦ +arr êt +ĠЧ а +×IJ×ŀ ׳×ķת +عار ض +à¹Ģà¸ģม สà¹Į +Ġ׾×Ķ ×ij×Ļף +Ġ׾ ×ij×Ĺ +Ġ׾×ij×Ĺ ×ķר +สา à¸Ĥา +ĠÐľÐ¾Ñģк ве +ب عد +ĠاÙĦÙĤر ار +ĠÄIJ á»ĭa +Ġ×Ĺ ×Ĵ +Ùģ تر +ÙĪÙĨ Ø© +Ġ×Ķ×ĸ ×IJת +å¸Ĥ ãģ® +ãģ» ãģĹãģĦ +Ġ×ij×¢ ×Ļר +ĠÑĤеп еÑĢÑĮ +ìĬµ ëĭĪê¹Į +à¹Ħม à¹Īว +à¹Ħมà¹Īว à¹Īา +à¹Ħมà¹Īวà¹Īา à¸Īะ +×ŀ ×IJ×Ķ +æĥħ åł± +æĥħåł± ãĤĴ +غ ÙĨ +Ġпо Ñı +ĠпоÑı ви +éģİ ãģĶ +تش غ +تشغ ÙĬÙĦ +в ел +Ġ×Ĺ ×ŀ +ãģ¨ãģªãĤĬ ãģ¾ãģĻ +Ġra ÄŁ +ĠraÄŁ men +ãģĭ ãģ©ãģĨ +ãģĭãģ©ãģĨ ãģĭ +ен ко +ì§Ģ ê³ł +Ġ×IJ׾ ×Ļ×Ķ +ĠØ£ ÙĦ +à¸Īำ หà¸Ļ +à¸Īำหà¸Ļ à¹Īาย +nız ı +Ġ׾ק ×Ĺת +Ø£ ÙĩÙħ +Ø£ÙĩÙħ ÙĬØ© +ت غÙĬر +ש ×Ĺר +ס×ķפ ר +×ĵ ×Ļר +èī¯ ãģĭãģ£ãģŁ +×ŀ׾×Ĺ ×ŀ×Ķ +ÑģÑĤв ие +ÑĤ ÑĢаÑĤ +ĠاÙĦØ£ Ø® +ĠاÙĦأخ ÙĬرة +ĠاÙĦØŃ ØµÙĪÙĦ +Ġcréd ito +צ ×Ļ×¢ +ãĥ¬ ãĥĻãĥ« +بر ÙĬ +ëIJ IJ +ãģł ãģ£ãģ¦ +Ġreal tÃł +س Ùģر +×ķ׳ ×ķ +×Ĵ ×ķ×ĵ +×Ĵ×ķ×ĵ ׾ +ฮ า +ãģĹãģ¦ ãģĬãĤĬãģ¾ãģĻ +Ġg Ãł +Ġ׾×ij צע +å¼ķ è¶ĬãģĹ +Ġ×ŀ ×Ļ׾×Ļ +Ġ×ŀ×Ļ׾×Ļ ×ķף +Ùħ در +Ùħدر سة +פ ×ķ×ĺ +à¸Ļà¹īำ มัà¸Ļ +ëģ Ŀ +ع Ùĥس +ĠÙĤ ض +ĠÑĢÑĭ б +خط Ø· +×ŀ×ķס ×ĵ +Ġ׼׾ ׾×Ļ +ĠкоÑĤоÑĢ ое +צ×Ļ ×ķף +ĠмеÑģÑĤ а +ãģĭ ãģ¤ +г ÑĢÑĥпп +׾ ×Ļ׾ +ת ×ķ×IJר +ë³µ ì§Ģ +à¹ģà¸ľ à¹Īà¸Ļ +Ġ×ij×¢ ת +æĻĤéĸĵ ãĤĴ +ï¼ £ +ãģ¨ãģĦãģĨãģĵãģ¨ ãģ§ +Ġ׾×Ķ ק +Ġ׾ ×ĸ×Ķ +ĠìłĢ ëĬĶ +ĠاÙĦØ¥ رÙĩاب +ĠìŀĪëĬĶ ëį° +ĠÑĤ огда +Ġ×Ķ צ×Ļ +×ķ׾ ×ĺ +Ġר פ×ķ×IJ×Ļ +ãģĵãģ¨ ãģ§ãģĻ +ĠÄij ÃŃch +ØŃ ÙĬا +Ġ×Ķ×ŀש ×Ĺק +ãģľ ãģ² +Ġ×ŀ×IJ פשר +ãģ¿ ãģ¾ãģĹãģŁ +ĠاÙĦØ£ÙħÙĬر ÙĥÙĬ +Ùħج تÙħع +Ġس اب +Ġساب ÙĤ +׼ ×Ļ׾ +Ạ¾ +ãĥª ãĤ¹ãĥĪ +Ġì ĥ +Ġìĥ Ī +ĠìĥĪ ë¡ľ +ĠìĥĪë¡ľ ìļ´ +ĠD á»ĭch +à¹Ģหมาะ สม +ĠاÙĦÙĨ بÙĬ +׾ ׾ +ÙĨ ع +Ðĵ лав +Ðĵлав наÑı +Ùħر ض +Ġ×ķ ×ĵ +ت ÙĤÙĬ +تÙĤÙĬ ÙĬÙħ +Ġb ảng +ĠÙģ ÙĤاÙĦ +×¢ ×ŀ×Ļ +д ÑĢа +Ġsu á»ijt +سر عة +Ġc á»Ń +Ġ×Ķ ×Ļ×Ĺ×Ļ×ĵ +سع ÙĬد +à¸Ńา à¸Ĭีà¸ŀ +Ġس ÙĪاء +ãĤ½ ãĥķãĥĪ +Ġл иÑĩно +ĠÐļ оÑĢ +اÙĩ تÙħ +اÙĩتÙħ اÙħ +à¸Ń à¸Ķี +à¸Ńà¸Ķี à¸ķ +ãģIJ ãĤīãģĦ +Ġiht iya +Ġihtiya ç +ãģ¾ãģ§ ãģ® +ìĭľ ìĬ¤ +ìĭľìĬ¤ íħľ +ÑĢÑĥ ÑĪ +ãĤĦ ãģ£ãģ± +ãĤĦãģ£ãģ± ãĤĬ +к еÑĢ +Ġ ży +Ġży w +кл он +Ġl ượt +à ¾ +да Ñĩи +tür k +غ ÙĪ +ĠигÑĢ ок +Ġph ê +Ġש ×¢×ľ +ĠاÙĦÙħ دÙĨÙĬ +ĠìŬ룬 ë¶Ħ +ער ×Ļ×Ŀ +Ñħод ÑıÑĤ +Ġx ứ +ÐĹ Ð° +ĠÙģ رص +à¸Īะ à¸Ĺำà¹ĥหà¹ī +íģ ´ +×¢ ×ij×ķר +à¹Ģหลà¹Īา à¸Ļีà¹ī +èĢĥãģĪ ãĤĭ +ÑĢ еÑģÑĤ +н нÑĭй +Ġc ầm +دا Ø®ÙĦ +ĠÙħÙĦÙĬ ار +ĠÐIJ л +ĠвÑĢем ен +à¸Ĭà¹Īวย à¹ĥหà¹ī +ר×Ļ ×ķת +ëĵ ¯ +飲 ãģ¿ +׳ ׾ +שת ×£ +ĠاÙĦسعÙĪد ÙĬ +u ÃŁ +ìĿ¸ ëį° +ĠìĿ¼ ë°ĺ +ÅĤ ÄĻ +Ġm á»iji +×ŀ ×Ļ׳ +ĠاÙĦØ£ Ø·ÙģاÙĦ +Ġçı kan +é cole +ק ×Ļש +ק×Ļש ×ķר +ĠоÑģ ÑĥÑīеÑģÑĤв +ĠоÑģÑĥÑīеÑģÑĤв лÑı +×ij ×IJר +à¹Ħà¸Ľ à¸Ķà¹īวย +Ġ×¢ ×ķ׾×Ķ +à¸ģà¹ĩ à¹Ħมà¹Ī +ãĥ¢ ãĥĩ +ãĥ¢ãĥĩ ãĥ« +تØŃ ÙĪÙĦ +Ġод ного +ת×Ĺ×Ļ׾ ת +Ġت Ø® +Ġch cia +Ġchcia ÅĤ +ãĥIJ ãĥ³ +èĢħ ãģ¯ +ĠÙħ ØŃÙĦ +Ñģл ож +Ñģлож н +Ġt ÄĻ +Ġçı kt +Ġçıkt ı +ĠC Æ¡ +à¹Ħà¸Ķà¹ī à¹Ģลย +ır ken +à¹Ģà¸Ĥà¹īา สูà¹Ī +ÙħØŃ Ùĥ +ÙħØŃÙĥ ÙħØ© +à¸Ħุ à¹īม +à¸Ļà¹Īา à¸Īะ +лÑİ Ð´ +де ÑģÑı +деÑģÑı ÑĤ +ĠлÑİб ой +تØŃر ÙĬر +צע ×ĵ +Ġе Ñij +ĠاÙĦØŃ ÙĥÙħ +Ġص باØŃ +à¹Ģà¸ļ à¸Ńรà¹Į +Ġróż nych +ги б +ĠÑģ оÑĤ +ĠÑģоÑĤ ÑĢÑĥд +ĠÑģоÑĤÑĢÑĥд ник +ĠобÑĬ ем +פ ×ĺר +ãģĻãģĶ ãģı +ãģ«éĸ¢ ãģĹãģ¦ +в ол +Ø« ÙħاÙĨ +Ġd ần +æĬ ľ +æĬľ ãģij +Ġ×¢ ש +Ġעש ×ķ×Ļ +ס ×ķף +ãģªãģ® ãģ§ãģĻ +ãģ¯ ãģ©ãģĨ +×ŀ×¢ ר×ij +ï¼ ° +Ùħ صر +ÙħÙĨ اسب +ÙħÙĨاسب Ø© +ä¸Ĭ ãģ® +×IJ×Ļש ×ķר +ĠìĦ¤ ì¹ĺ +×ŀ×ĵ×Ļ׳ ×ķת +×ŀר ת +ãĤĭ ãģ®ãģĮ +د Ùİ +ĠاÙĦشر Ùĥات +ìĭľ ê°Ħ +ĠÑĢеÑĪ ение +ãģĻãĤĭ ãģ®ãģ¯ +ĠìŀIJìĭł ìĿĺ +׾ ×ŀ×ķ +ãģ¨ãģĵãĤį ãģ§ +Ġק צר +Ġmã i +Ġkü ltür +ãĥ©ãĤ¤ ãĥĸ +à¸ľà¸¹à¹ī หà¸įิà¸ĩ +æĻĤéĸĵ ãģĮ +клÑİÑĩ и +diÄŁ iniz +มาà¸ģ à¹Ĩ +تØŃ ÙħÙĦ +Ġh ạt +ãĤ¦ ãĤ£ +п ле +×ŀ ׾×IJ +ÅĤ ó +Ġg á»ijc +Ġ×IJ ×ķ×ĵ×ķת +หว าà¸Ļ +ĠاÙĦ ÙĪز +ĠاÙĦÙĪز راء +ëĵ¤ ê³¼ +Ġص ØŃ +ĠصØŃ ÙĬÙģØ© +Ġм м +تد Ø®ÙĦ +Ġpersön lich +Ġز ÙĬ +ĠزÙĬ ادة +ãĤ· ãĤ¢ +Ġng ắn +à¸Ħล ิà¸ģ +Ġs ông +Ġtü ket +Ñį ÑĦÑĦ +ÑįÑĦÑĦ екÑĤ +ש ×Ļ×ij +Ġا عت +ت ض +تض ÙħÙĨ +ĠاÙĦÙħØ´ رÙĪع +Ġprodu ção +ĠпÑĢимен Ñı +ни ÑĨÑĭ +주 ëĬĶ +ر Ùı +Ġm Æ¡ +Ġhayat ı +ëŁ ½ +Ġü cret +Ġyan ında +Ġpr ática +×ij×Ļק ×ķר +Ãľ N +Ñģ оÑĤ +ãĤıãģij ãģ§ +Ġдол го +ת ׼×ķ +ĠìķĦ ëĭĮ +ë į°ìĿ´ +Ġç iz +Ġcho Äĩ +Ġ×Ķ ×Ļת +Ġ×Ķ×Ļת ר +Ġso át +׼ ×ij×ĵ +à¹Ģล à¹Īา +Ġд еÑĢ +ĠдеÑĢ ев +ãĤĴ åħ¥ãĤĮ +×Ĺ ×ķס +×Ĺ×ķס ר +ج ÙĬÙĨ +t ón +onn é +Ġпол ноÑģÑĤÑĮÑİ +人 ãģŁãģ¡ +Ġpr êt +ëł ¸ +Ġdéc embre +cı lar +Ġת ת +Ġê²½ìļ° ìĹIJëĬĶ +ÙĪ عد +è¦ĭ ãĤĭ +วิ à¸Īัย +ë ¶Ī +ز ÙĪا +زÙĪا ج +d ì +ãģ§ãģĻ ãĤĪ +Ġвод о +ĠÙĬ ÙĪجد +Ñģ оÑģÑĤоÑı +Ðŀ С +ĠÄIJ ó +׊פש +Ġצ ×Ļ×ij×ķר +ĠاÙĦÙĤ Ø· +ĠاÙĦÙĤØ· اع +Ġиме ÑİÑĤ +Ġph áºŃn +×Ľ×¡ פ×Ļ +полн иÑĤелÑĮ +éĻIJ ãĤĬ +ĠÑģ ÑĢав +ĠÑģÑĢав н +ÙħاÙĦ Ùĥ +×ĵר ×ķ×Ŀ +çļĨ ãģķãĤĵ +ØŃÙĤ ÙĤ +à¹ģหล à¹Īà¸ĩ +ĠاÙĦر سÙħÙĬ +оÑĩ ки +×ĺ ×ij×Ĺ +Ġcan lı +Ġ׾ ׾ +Ġ׾׾ ×ŀ×ķ×ĵ +×ŀ×ij ×ķ +ת ׼ +×ª×Ľ ׳×Ļת +ĠاÙĦÙħ شار +ĠاÙĦÙħشار ÙĥØ© +Ä° Åŀ +ĠسÙĬ اسÙĬ +в олÑĮ +ĠÑģ пÑĢав +æĿ¥ ãģ¦ +פ×ķר ×ķ×Ŀ +สำ à¹Ģรà¹ĩ +สำà¹Ģรà¹ĩ à¸Ī +ĠÅŁ öyle +Ġzosta ÅĤa +ĠH ü +ר ×ķש +د ÙĦÙĬÙĦ +ÑĢи д +ש ף +×ŀק ×ķר +ĠÑĥ Ñĩ +ĠÑĥÑĩ еб +ĠÑį ÑĤа +ков а +à¸ķà¸Ļ à¹Ģà¸Ńà¸ĩ +ÙĨ ÙIJ +à¸Ńีà¸ģ à¸Ħรัà¹īà¸ĩ +ระ à¸ļุ +Ġd ữ +ĠاÙĦØŃ Ø§ÙĦÙĬ +׼ ×ķ׼ +׼×ķ׼ ×ij +Ġ×ŀ×IJ שר +Ġtr ụ +ÑĤел ем +Ġв ли +Ġвли Ñı +Ġש×IJת ×Ŀ +Ġuw ag +Ġuwag ÄĻ +×ĺ ×Ļת +×IJ ×ĵ×Ŀ +à¸Ķ ุ +Ġ×Ķ×IJ ׾×Ķ +Ġkar Ä±ÅŁ +ĠÄIJ á»iji +да ÑİÑĤ +ãģªãģ® ãģ« +Äħ cych +à¹Ģà¸Ļ à¹īà¸Ļ +ãģĹãģ¦ ãģĹãģ¾ãģĨ +int érieur +ĠfÃŃs ica +ĠÐŁ ол +ãģĹãģ ķ +à¸Ĺำ à¹Ħม +ĠL âm +ĠاÙĦÙħ سÙĦÙħ +ĠاÙĦÙħسÙĦÙħ ÙĬÙĨ +ص ØŃØ© +ìĹ Ħ +à¹Ģà¸Ķà¹ĩ à¸Ķ +ĠÑĥ ÑĩеÑĤ +â Ìģ +Ġب ÙĦا +ĠاÙĦاجتÙħاع ÙĬ +פרס ×Ŀ +ãĥķ ãĥ© +ĠÐļ огда +mie ÅĽci +ĠبÙĬÙĨ Ùħا +Ġ×ŀ×IJ ×ŀר×Ļ×Ŀ +Ġ×ij×IJ ×ĸ×ķר +×ķש ×Ļ×Ŀ +ĠÑģдел а +entr ée +à¹Ģ à¸Ħà¹īา +Ñĥг л +ĠاÙĦÙģ ÙĨÙĬ +ĠÐĴ оÑĤ +à¸Ĺีà¹Ī มา +×ķצ ×Ĵ +ÙĤد رة +Ġëª © +Ġ목 ìłģ +íıī ê°Ģ +ĠاÙĦØ£ ربع +ĠاÙĦأربع اء +פס ×Ļק +ĠÑıвлÑı ÑİÑĤÑģÑı +ب ÙĪÙĨ +ì° ¾ +×ŀ×¢ ר׼ +×ŀ×¢×¨×Ľ ×ķת +ãĤ· ãĤ§ +ĠباÙĦ Ø£ +íĸĪ ëįĺ +ĠاÙĦبر ÙĨاÙħج +ĠاÙĦØ£ ØŃد +Ġm Å© +ĠmÅ© i +п аÑĤ +ب Ø« +ĠÑĨ енÑĭ +Ġ×ijת ׾ +è¨Ģ ãĤıãĤĮ +ĠاÙĦÙħ جاÙĦ +ĠìĦ¸ ìĥģ +Ġ×Ĵ ×ķפ +ĠнаÑĪ ей +Ġкомп аниÑı +б ин +öl ü +×Ļ ×Ļ×ĺ +Ġ×ŀס פ×Ļק +ยัà¸ĩ à¸Ħà¸ĩ +ĠЧ и +Ġан ÑĤи +ĠÑģÑĢед и +สà¹Īวà¸Ļ à¹ĥหà¸įà¹Ī +оÑĩ ка +íĬ¹ ë³Ħ +ว à¹Īาà¸ĩ +гоÑĢ од +با Ùĥ +à¹Ģส ีà¹Īย +à¹Ģสีà¹Īย à¸ĩ +ãĤĤãĤī ãģĦ +ק ×ķ×Ŀ +ãģĽ ãģļ +ĠاÙĦÙĤ اÙĩرة +Ġ×ij ׼×ļ +Ùħشار ÙĬع +باØŃ Ø« +Ġпо Ñĩ +ĠпоÑĩ ÑĤи +ĠÑĦоÑĢм а +S Ä° +Ġ×ŀצ ×Ļ×¢ +ล ื +ลื ม +ĠÑĤ еÑĢ +ĠÑĤеÑĢ ÑĢиÑĤоÑĢ +ĠÑĤеÑĢÑĢиÑĤоÑĢ ии +Ġв меÑģÑĤ +ĠвмеÑģÑĤ е +dıkl arı +op ération +à¹Ĥ ห +ص دÙĬ +صدÙĬ ÙĤ +íĸī ìłķ +تج ا +تجا ÙĪز +Ġsu ç +Ġar ty +Ġarty ku +Ġartyku ÅĤ +ãĤ·ãĥ§ ãĥĥãĥĹ +ש פ +שפ ×Ļ×¢ +Ġ×Ķש ×Ļר×ķת +à¹ģà¸ĸ ม +ë¸ Ķ +Ġuk ÅĤad +Ġ×ķ ׼×Ļ +หล าà¸ģ +หลาà¸ģ หลาย +æĸ¹ ãĤĤ +Ġpodr óż +ĠE ÄŁer +Ġком наÑĤ +ĠÑģам ÑĭÑħ +Ġв кÑĥÑģ +б еж +Ġ×ij ק×ķ +æİĽ ãģij +ãģ¿ ãĤĭãģ¨ +ĠiliÅŁ kin +ĠÙĬ عÙħÙĦ +Ġпод аÑĢ +Ġyaz ılı +ãĤĴ å¾Ĺ +Ġwyst ÄĻp +à¸Ĺีà¹Ī à¹ĥà¸Ĭà¹ī +ØŃاد Ø« +ÙĪ ÙĬد +кÑĥ лÑĮÑĤ +кÑĥлÑĮÑĤ ÑĥÑĢ +à¸ģาร à¹ģà¸Ĥà¹Īà¸ĩ +à¸ģารà¹ģà¸Ĥà¹Īà¸ĩ à¸Ĥ +à¸ģารà¹ģà¸Ĥà¹Īà¸ĩà¸Ĥ ัà¸Ļ +ÙħÙĪ ظ +ÙħÙĪظ Ùģ +ÙĬÙħ ÙĬ +ãĤĵãģ§ãģĻ ãģĮ +diÄŁ im +diÄŁim iz +ĠÐŁ еÑĢ +ĠÐŁÐµÑĢ в +Ġm ão +ĠÑģ ез +ĠÑģез он +Ġ×Ķ×ŀ ×¢ +Ùħ جÙħÙĪعة +ĠинÑĦоÑĢм аÑĨии +i ếc +ã ng +ĠÄij ấy +ãģĶ ç´ +ãģĶç´ ¹ +ãģĶç´¹ ä»ĭ +Ġad ım +à¹Ħ หล +Ġп ÑĢакÑĤи +ĠпÑĢакÑĤи Ñĩ +ĠпÑĢакÑĤиÑĩ еÑģ +ĠпÑĢакÑĤиÑĩеÑģ ки +ĠاÙĦÙĨ Ùģس +ĠÑĢабоÑĤ е +ÙĦÙĬ Ùģ +ĠاÙĦجÙĨ ÙĪب +Ġвод Ñĭ +ì¹ Ļ +Ġм иÑĢа +ĠÄij ừng +ĠпÑĢоÑĤив о +ĠÑģÑĤÑĢан Ñĭ +ล ู +ìĤ ¶ +kre ÅĽl +Ġbul und +Ġbulund uÄŁu +à¹ģ สà¸Ļ +ãĤ± ãĤ¢ +ת×Ĺ ×ķ×ŀ×Ļ +ר׼ ×Ķ +Ġ׾ק ×ķ×Ĺ +Ġ׾ק×ķ×Ĺ ×ķת +Ġ×Ľ×ª ×ķ×ijת +ĠÙĦ ÙĥÙħ +ب شر +Ġr Ãłng +Ġ×ŀ×Ķ ×ŀ +Ġ×IJ×Ĺר ×ķת +Ġб он +Ġбон ÑĥÑģ +ï½ Ĺ +à¹ģ ยà¸ģ +ãģĤãģªãģŁ ãģ® +ĠÑĥÑĩаÑģÑĤ ие +ĠE yl +ĠEyl ül +ĠçalÄ±ÅŁmalar ı +Ø® طر +ìĿ ½ +à¸ģาร à¹ĥà¸Ĭà¹īà¸ĩาà¸Ļ +Ġана лиз +תק ×ij׾ +ни ем +ĠÄ° ns +ĠÄ°ns an +ĠبÙĪ اس +ĠبÙĪاس طة +Ġ׳ ×Ľ×ł×¡ +Ġ×Ķ×ŀ ×Ļ×ĵ×¢ +Ġç o +Ġço ÄŁu +á» ĺ +ĠêµŃ 민 +ãĤĤ ãģĦãģĦ +Ġ׼ ׾×Ļ +ĠÑģÑĢед не +g ÅĤo +gÅĤo ÅĽ +Ġneg ó +Ġnegó cio +ĠÑĢ егиÑģÑĤ +ĠÑĢегиÑģÑĤ ÑĢа +ĠÑĢегиÑģÑĤÑĢа ÑĨии +Ġtr á»ĵng +ĠпÑĢ Ñı +ĠпÑĢÑı мо +ëłĪ ìĿ´ +Ġk ém +к ле +à¸Ļำ มา +ĠÑĦ ин +ĠÑĦин анÑģ +ĠÑĦинанÑģ ов +Ġki á»ĩm +ยัà¸ĩ à¹Ħ +ยัà¸ĩà¹Ħ à¸ĩ +ย ิà¸ĩ +à¹Ĥ à¸Ľ +ĠполÑĥÑĩ ил +×Ļ×ĸ ×Ŀ +à¹ģละ à¸Ħวาม +Ġво обÑīе +ص ÙĬر +ãĥı ãĥ³ +ĠاÙĦÙĤ اد +ĠاÙĦÙĤاد Ùħ +Ġب دÙĪÙĨ +ع ظÙħ +ת ׳×ķ×¢ +×ª×ł×ķ×¢ ×Ķ +Ø£ ÙħÙĦ +ãģķ ãģĪ +ÑĤ ем +ÑĤем пеÑĢ +ÑĤемпеÑĢ аÑĤÑĥÑĢ +Ġ׾ ×Ļצ×ķר +Ġr ÄĻk +ر سÙĦ +ìŀIJ 를 +Ġ×Ļצ ×Ļרת +ÙĨ بÙĬ +Ñĩ наÑı +تØŃ ÙĦÙĬÙĦ +Ġм ик +Ġмик ÑĢо +ĠS öz +Ġfor ça +Ñģ он +ĠاÙĦع را +ĠاÙĦعرا ÙĤÙĬ +ĠH á»ĵng +ãģĻãĤĭ ãģŁãĤģãģ« +à¸Ĺีà¹Ī à¸Ńยูà¹Ī +Ġ×ķ×IJ ×£ +ص ÙĬد +ĠìķĬ ê³ł +ร ัà¸ĩ +ĠاÙĦت ÙĪاصÙĦ +à¹Ģม à¸ķร +Ñĥ ÑģÑĤÑĢой +ÑĥÑģÑĤÑĢой ÑģÑĤв +m ıyor +Ġبا سÙħ +Ġ×ķ ׼×ķ +ĠG ül +á» IJ +Ãī tat +غ اÙĦ +Ø¥ ÙĨØ´ +Ø¥ÙĨØ´ اء +T Ä° +à¸Ĥà¹īา ม +Ġtro ch +Ġtroch ÄĻ +Ø¥ ص +إص ابة +ĠØ« اÙĨÙĬ +ĠاÙĦص ØŃØ© +Ġ×ĸ×Ķ ×ķ +jÄħ cej +ãĥĢ ãĥ³ +ìĿ¸ ìĿ´ +Ġв олоÑģ +ëIJĺ ë©´ +Ġzak ÅĤad +ãģĻ ãģĵãģ¨ +以ä¸Ĭ ãģ® +Ġ×Ķ×ŀק ×ķ×Ŀ +ÙħØ´ اÙĩ +ÙħشاÙĩ دة +Ñĩ ив +ب Ø´ +ย à¹īาย +Ġsür dür +ĠN ẵ +ĠNẵ ng +ĠигÑĢ аÑĤÑĮ +Ġê·¸ëŁ¬ ë©´ +ãĥķ ãĥ« +ล à¹Īะ +Ġtend rá +Ġb Ãły +à¹Ģà¸Ľà¹ĩà¸Ļ à¸ľà¸¹à¹ī +Ġok o +Ġoko ÅĤo +w ÅĤa +wÅĤa ÅĽci +wÅĤaÅĽci w +æĢĿ ãĤı +ĠYa ÅŁ +ĠB á»ĩnh +íı Ń +بÙĬ د +קר ף +à¹Ģศ ร +à¹Ģศร ษ +à¹Ģศรษ à¸IJ +à¹Ģศรษà¸IJ à¸ģิà¸Ī +ĠاÙĦØ£ ÙĪرÙĪ +ĠاÙĦØ£ÙĪرÙĪ بÙĬ +fl äche +ä¹Ĺ ãĤĬ +Ġb á»ģn +Ùĩ ب +æľĢ ãĤĤ +Ġsa ç +à¸Ńำ à¹Ģà¸ł +à¸Ńำà¹Ģà¸ł à¸Ń +ĠØ£ ج +ĠاÙĦد اخÙĦ +ĠاÙĦداخÙĦ ÙĬØ© +×ĺ ×ķ×ij +ãĤĤ ãģªãģı +Ġли ÑĨа +à¹ģลà¹īว à¸ģà¹ĩ +×ĸ׼ ×Ļר +Ġqu Ãł +ĠÙĥ Ø°ÙĦÙĥ +صØŃ Ùģ +ĠÃĤ u +ÙĪب ا +à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļ à¹ģà¸Ľà¸¥ +à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļà¹ģà¸Ľà¸¥ à¸ĩ +à¸ķัว à¸Ńยà¹Īาà¸ĩ +Ġráp ida +Ġtas ar +Ġtasar ım +ĠعÙĦÙĬ ÙĩÙħ +ס ×ķ׾ +c ılı +cılı k +Ġر غÙħ +ìĭľ íĤ¤ +Ġ×IJ׾ ק +Ġ×IJ׾ק ×ĺר +Ġ×IJ׾ק×ĺר ×ķ׳×Ļ +à¹ģà¸ļ à¹Īà¸ĩ +Ġh ạng +ãģ£ãģ¦ ãģıãĤĮ +ĠÙĨ تÙĬ +ĠÙĨتÙĬ جة +ıkl ı +غ اÙĨ +à¸Ĥà¹īà¸Ń à¸Ħวาม +à¸Ľà¸¥ าย +ĠØ£ Ùħس +à¸Ĺีà¹Ī à¹Ģà¸ģีà¹Īยว +à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยว à¸Ĥ +à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยวà¸Ĥ à¹īà¸Ńà¸ĩ +Ġdé fin +Ġdéfin i +ÙģÙĨ اد +ÙģÙĨاد ÙĤ +à¹Ħà¸Ķà¹ī วà¹Īา +ãģªãģĦ ãĤĪãģĨãģ« +Ġpróp ria +ĠPh át +ãĤĦãģĻ ãģı +สวย à¸ĩาม +ê³ł ìļĶ +Ñı еÑĤ +ãģĭãĤĤãģĹãĤĮãģ¾ãģĽãĤĵ ãģĮ +تر جÙħ +ĠкÑĢаÑģ ив +Ġ×ŀ ר×IJש +д еж +ĠÙĬ ÙĪÙĨ +ĠÙĬÙĪÙĨ ÙĬÙĪ +Ñģк оÑĢ +ĠKas ım +ê³Ħ ìķ½ +к оÑģ +Ġна ÑĢÑĥ +ĠнаÑĢÑĥ ÑĪен +Ġdu że +acc ès +Ġh á»ĵng +Ġv Å© +ãģĦãģŁ ãģĹãģ¾ãģĻ +Ġ×ĺ ×Ļ +Ġ×ĺ×Ļ ×ķ׾ +lıkl arı +Ġqu ê +ëħ¸ ëıĻ +ìķ Ķ +CI ÃĵN +Ġt ắc +press ão +ĠìŀĪ ìľ¼ +สิà¸Ĺà¸ĺิ à¹Į +íĥ Ħ +Ġ×Ķ×ŀ ×ŀש׾×Ķ +å¬ī ãģĹãģĦ +ĠÄIJ ặc +ÙĨ زÙĦ +ĠдÑĢÑĥг ой +д ÑĥÑĤ +ìĪ Ļ +Ġth ụ +à¹Ģส ร +à¹Ģสร à¹ĩ +à¹Ģสรà¹ĩ à¸Ī +Ġto plant +Ġtoplant ı +×IJ×ŀ ף +×ķ׾ ת +п омн +Ġyo ÄŁun +ÅĦsk iego +ì° © +ĠØ« ÙĦاث +ĠØ«ÙĦاث Ø© +Ġl ắng +ë¦ ´ +ราà¸Ĭ à¸ģาร +ĠÑģлов а +á» Ĩ +à¸Ķี à¸ģวà¹Īา +ãģĶãģĸ ãģĦãģ¾ãģĻ +Ġд из +Ġдиз айн +fé rence +lıkl ar +ãģªãĤĵ ãģ§ãģĻ +ajÄħ cy +Ġëĭ¤ ìĸij +Ġëĭ¤ìĸij íķľ +ק ×Ļר +ØŃ Ø§Ø± +ส ูà¹ī +Ġz ro +Ġzro bi +Ġzrobi Äĩ +×ŀ ×Ļ׼×Ķ +à¸Ĭà¹Īวย à¹Ģหลืà¸Ń +ĠÑįÑĤ Ñĥ +ë´ ī +楽 ãģĹãģĦ +س ÙĪر +íķĺ ê±°ëĤĺ +Ùħؤ تÙħر +Ġpoc zÄħ +ĠpoczÄħ tk +ĠpoczÄħtk u +Ġع ربÙĬ +اÙĦØ£ ر +اÙĦأر دÙĨ +à¸Ķ ร +Åĵ uvre +ĠÙĪÙĥ اÙĨت +ĠÅĽ redni +Ø® ضر +Ġch uyến +н ÑĤ +ĠìķĮ ê³ł +Ġv á»Ŀi +Ġ×ij ×Ļ×ĵ×Ļ +×ŀ×ĵ ×ķ×ijר +ÙĪ Ùģر +ÙĬ Ø¡ +׳ ×Ľ×¡ +ĠÐĽ а +л он +Ġx ấu +Ùģ ÙĬÙĨ +Ġfé vrier +ĠÐŀ на +ĠV á»ģ +ĠÅŁey ler +ĠполÑĥÑĩ ен +з ад +Ġn ét +à¹Ħà¸Ľ ยัà¸ĩ +×Ĺש×ij ×ķ +à¸ļัà¸Ļ à¸Ĺ +à¸ļัà¸Ļà¸Ĺ ึà¸ģ +Ġgerçek leÅŁ +иÑĩеÑģк ое +ìĪĺ ê°Ģ +Ø« بت +ãģ¤ ãģ¾ãĤĬ +ĠÑĥÑģловиÑı Ñħ +ëĭ¤ ê°Ģ +ราย à¹Ħà¸Ķà¹ī +׼×IJ ×ij +à¹Ĥà¸Ľà¸£ à¹Ĥม +à¹Ĥà¸Ľà¸£à¹Ĥม à¸Ĭัà¹Īà¸Ļ +j ähr +jähr ige +ק ׳×Ļ×Ŀ +×ŀ ×ķק +×ŀ×ķק ×ĵ +ãģ«è¡Į ãģ£ãģ¦ +Ø¢ ÙĦ +вед ение +Ġ׾ ×Ľ×ª×ķ×ij +جÙħ Ùĩ +جÙħÙĩ ÙĪرÙĬØ© +à¸ī à¸ļ +à¸īà¸ļ ัà¸ļ +ĠC òn +à¸ľ สม +ãģªãģ© ãģĮ +×IJ×Ķ ×ij +ĠдейÑģÑĤв иÑı +y ız +à¹Ħมà¹Ī à¹Ģà¸Ħย +ج ÙĪز +×Ķ×Ĺ׾×ĺ ×Ķ +f ällt +ãĥĵ ãĤ¸ +ãĥĵãĤ¸ ãĥį +ãĥĵãĤ¸ãĥį ãĤ¹ +Ġ×IJ ×Ļ׳×Ŀ +ĠнаÑħод иÑĤÑģÑı +Ġdzi ÅĽ +ست Ø·ÙĬع +׾ ×Ļף +Ø® ÙĦاÙģ +Ùĩ ÙIJ +Ġatr ás +íĺ ģ +ãĤĴ ãģĶ +Ġ×Ķ×ŀ ×ķצר +ĠBakan lıģı +ÑİÑī ее +ÙħÙĨ اط +ÙħÙĨاط ÙĤ +Ùģ د +à¸Ļำ à¹Ħà¸Ľ +Ġв аж +Ġваж но +Ġm ạch +׼ ׳×ķ +بع Ø« +lan ması +Ġa yr +Ġayr ıl +ìĤ¬ íļĮ +d ÃŃa +p ÅĤyw +اÙħ ÙĬØ© +íĺ ľ +×IJ׳ ×Ĵ׾ +×IJ׳×Ĵ׾ ×Ļת +ĠìŀĪëĭ¤ ëĬĶ +Ġس اعة +ĠëĤĺ íĥĢ +b ö +à¸Ħ ัà¸Ļ +ĠdziaÅĤ ania +Ø© Ùĭ +Ġng Å© +׳צ ×Ĺ +ãģ¯ ãģĤãĤĭ +ĠyaÅŁ ında +st ück +car acter +caracter ÃŃsticas +Ġr á»Ńa +ĠÙħختÙĦÙģ Ø© +ãģ«ãģĬ ãģijãĤĭ +à¹ģà¸ŀ à¸ĩ +วิ à¹Īà¸ĩ +ת פ×ķ +سا ÙĩÙħ +使 ãģĨ +Ùĥ رÙĬ +×IJ פ×Ļ +........ ....... +ĠÑĤак им +×Ļ׼ ×ķ×Ļ +Ø´ بÙĩ +ج ÙĬر +ãģĿãģ® ãģ¾ãģ¾ +ac jÄĻ +ĠاÙĦت رÙĥ +ĠاÙĦترÙĥ ÙĬ +ĠпÑĢав илÑĮно +Ġت عÙħÙĦ +à¸ģล à¹īา +Ġbi ên +Ġ×ij׳×Ļ ×Ļת +Ġкл Ñĥб +Ġ×ŀ ש×Ķ +в ÑĪий +ãģĵãģ¨ãģĮãģ§ãģį ãĤĭ +à¸ŀัà¸Ļà¸ĺ ุ +à¸ŀัà¸Ļà¸ĺุ à¹Į +ר ×ķ×Ŀ +ĠاÙĦÙģ رÙĨ +ĠاÙĦÙģرÙĨ سÙĬ +à¹Ģà¸Ľà¹ĩà¸Ļ à¸Ħà¸Ļ +ãģĹãģ¦ ãģĬãĤĬ +Ġth ầy +ãĤĵ ãģłãģijãģ© +ìĶ ¨ +Ùħ دÙĨ +ت ÙĪÙĨ +ĠмеÑĤ ал +ĠмеÑĤал л +Ġin ÃŃcio +à¸Ńà¸Ńà¸ģ à¸Īาà¸ģ +ëĴ ¤ +Ġcu á»ijn +Ġbu á»Ļc +ÙĨ سÙĬ +ä cht +×ŀ ×Ļ׳×Ļ×Ŀ +ãģķ ãģ¦ +ãģĮ ãģ§ãģį +ÑĬ ем +Ġtá i +ĠЧ ÑĤ +ĠЧÑĤ обÑĭ +à¸Ľà¸¥ ูà¸ģ +à¸Ĭุม à¸Ĭà¸Ļ +н Ñģкий +Ġv ững +Ġ×Ķ ׾×ij +ë le +Ġש ×¢×ijר +в аÑĤÑĮÑģÑı +б ой +ع ÙĪÙĨ +à¹ģà¸Ķ à¸Ļ +Ġספר ×Ļ×Ŀ +Ġt uyên +Ġnhi êu +ĠQu ý +Ġh uyết +ãĤı ãģĭãĤīãģªãģĦ +Ġ×ŀ ׼ף +Ġ×Ķ ק׾ +Ġ׾×IJ ×ķר +ĠÄIJi á»ĩn +Ø´ ؤ +شؤ ÙĪÙĨ +Ġ×ŀ׊פש +ĠпоÑģÑĤоÑıн но +×ŀ ×Ļר +ìħ Ķ +Ðŀ Ñģ +ÐŀÑģ нов +×ĸ ×Ļת +ĠH á +ĠÑĩаÑģ ов +×IJ ×ķ׾×Ļ +Ġm át +Ø® رÙĪ +خرÙĪ ج +ÙĤ ضا +ÙĤضا ÙĬا +à¹Ģà¸Ľ à¸Ńรà¹Į +ĠÙĬ ÙĪÙĦ +ĠÙĬÙĪÙĦ ÙĬÙĪ +à¹Ĥà¸Ĺ ษ +׳ פ׾ +ת ×ķש +ת×ķש ×ij×Ļ +Ġv ários +×ŀ ר×IJ×Ķ +ëĿ¼ ìĿ´ +ÙĨ غ +×ij צע +г он +ĠÄIJ ược +ع Ùı +пÑĥÑģ к +ĠÙĪاÙĦ Ùģ +üc ü +×Ļק ×Ļ×Ŀ +Ġس بÙĬÙĦ +׾×ij ף +ĠاÙĦÙĤ رÙĨ +ס ×ķת +ĠQu áºŃn +ãģĵãĤĮ ãģĮ +ãĥĸ ãĥ©ãĥ³ãĥī +×Ĵ ×ŀר +Ġwarto ÅĽci +ĠÙĪب ÙĬÙĨ +Ġd ạ +ÐIJ в +ÐIJв ÑĤо +Ġol acaktır +à¸Ļ à¸Ĺà¹Į +Ùħ طار +Ġ×¢ ק×ij +Ġת פ +ãģĹãģ¦ ãģĦãģ¦ +צ ×ŀ×Ĺ +à¸Ī à¸Ńà¸ĩ +Ġö de +ìį ¨ +ÙĨ اس +調 ãģ¹ +ĠогÑĢ омн +ë³´ íĹĺ +×ĺ ק +×ĺק ס×ĺ +ĠbaÅŁ v +ĠbaÅŁv uru +Ġpom ys +Ġpomys ÅĤ +ãģ« ä¹Ĺ +Ġש ׼ף +ĠاÙĦÙħس ؤÙĪÙĦ +Ġз ан +Ġзан ÑıÑĤ +Ġd Æ°Æ¡ng +ãĥĹãĥ¬ ãĤ¤ +ล à¸ļ +ÑĤи ка +ĠAr alık +Ġнед о +Ġm á»Ļ +Ġor an +Ġoran ı +Ġktó r +Ġktór Äħ +Ġ×Ķ×IJ×Ĺר ×ķ׳×ķת +ائ ÙĨ +ÅĦ s +ÅĦs ka +åĽ½ ãģ® +×ŀ ×ĺ×Ļ +ĠвопÑĢоÑģ Ñĭ +à¸Ńà¸ĩà¸Ħà¹Į à¸ģร +×ŀ ×ķצ×IJ +Ġpó ź +Ġpóź niej +ש×ŀ ×IJ׾ +Ġk aps +Ġkaps am +Ġkapsam ında +Ġmá quina +ĠÅĽwie cie +Ġho Ãłng +Ġöz gü +×Ĵ×ķר ×Ŀ +ãģĤ ãģŁãĤĬ +à¸ķัà¸Ķ สิà¸Ļ +à¸ķัà¸Ķสิà¸Ļ à¹ĥà¸Ī +б ÑĢи +ãģ«ãģªãĤĭ ãģ¨ +ت ÙĥÙĪÙĨ +Ġ×ķ×Ķ ×Ļ×IJ +Ġchi ếu +ÑģÑĤан ав +ÑģÑĤанав ли +ÑģÑĤанавли ва +×ŀ ×ķ×Ĵ +c ité +ĠK örper +Ġש ×Ĵ×Ŀ +ع ظ +عظ ÙĬÙħ +Ġ×Ķ×IJ ×Ļש×Ļ +Ġmat ière +ĠÙģ ÙĪÙĤ +Ġk to +Ġkto ÅĽ +à¸Ļ à¹Ĥย +à¸Ļà¹Ĥย à¸ļาย +å¾ħ ãģ¡ +à¹Ģม à¸Ļ +à¹Ģมà¸Ļ ู +A ÃĩÃĥO +Ġt ù +Ġtù y +ãĥĪ ãĥ³ +ĠоÑĤ каз +Ġ×ŀ ×ķצר +ül ü +ãģķãĤĵ ãģ« +Ġ×Ĺ ×ķ×ij +קר ×Ļ×IJ×Ķ +ĠاÙĦØ® دÙħات +ĠÙĦÙħ دة +ر ؤ +رؤ ÙĬØ© +ãĤĴè¦ĭ ãģ¤ãģij +à¸Ł า +Ġréuss i +à¸Ļัà¸ģ à¹Ģรียà¸Ļ +ĠÑĩиÑģ л +à¸ģาร à¹Ģลà¹Īà¸Ļ +Ġhaz ırl +Ġhazırl an +ĠпеÑĢв Ñĭй +ли м +ĠоÑĤзÑĭв Ñĭ +Ġwy jÄħ +ĠwyjÄħ tk +ĠØ£ ÙĤÙĦ +ס ×ļ +Ġê²° ìłķ +Ġ׾×ŀ×¢ ש×Ķ +Ġl ắp +à¹ģà¸ļ ร +à¹ģà¸ļร à¸Ļà¸Ķà¹Į +วà¹Īา à¹Ģà¸Ľà¹ĩà¸Ļ +Ġب دا +Ġبدا ÙĬØ© +ãģ¨ãģĦãģĨ ãģ®ãģĮ +иÑĩеÑģк им +à¸ģาร à¸ŀัà¸Ĵà¸Ļา +Ġb Ãło +Ġmia ÅĤa +y waÄĩ +ĠMär z +ĠÙĨ سبة +Ġéconom ique +×ĸ ×ŀ +×ĸ×ŀ ׳×Ļ×Ŀ +æŃ¢ ãĤģ +Ġt ủ +íķĺ ìĭł +Ġkażde go +stra ÃŁe +à¸Ĭ ีà¹ī +à¹Ģ à¸ļา +ÑĢеÑģ ÑĥÑĢÑģ +ев ой +Ø´ باب +à¸ķà¹Īาà¸ĩ à¸Ľà¸£à¸°à¹Ģà¸Ĺศ +Ġ×IJ ×Ļש +Ġ×IJ×Ļש ×Ļת +×Ļ ×ķפ +×Ļ×ķפ ×Ļ +ĠìļĶ 구 +ì¡° ìĤ¬ +ãģ£ãģŁ ãĤī +׾ ×Ļק +миниÑģÑĤ ÑĢ +ãĤĤãģ® ãģ¯ +Ġl Æ°Æ¡ng +Ġна и +Ġнаи бол +Ġнаибол ее +íİ ĺ +à¹ģà¸ŀ à¹ī +ãĤŃ ãĥ¥ +ĠкоÑĤоÑĢ Ñĭм +à¹ģà¸Ĺ à¸ĩ +à¹ģà¸Ĺà¸ĩ à¸ļà¸Ńล +Ġ׳ ×Ļ×Ķ +Ġ׳×Ļ×Ķ ×ķ׾ +âĤ ª +ĠGi ải +ĠиÑģполÑĮзов а +ëł¥ ìĿĦ +ãģĹãģĭ ãĤĤ +à¸ģà¹ĩ à¸ķà¹īà¸Ńà¸ĩ +ĠÑĢ еб +ĠÑĢеб ен +ĠÑĢебен ка +ت ÙĪاصÙĦ +ãĤ°ãĥ« ãĥ¼ãĥĹ +ãĤĦ ãĤī +à¹Ģà¸Ľà¸´à¸Ķ à¸ķัว +б ÑĢо +ë°ĸ ìĹIJ +ÙĨ Ùİا +×Ķ ×Ĵ +×Ķ×Ĵ ׳×Ķ +à¸Ĺ รั +à¸Ĺรั à¸ŀ +à¸Ĺรัà¸ŀ ยà¹Į +Ġkh á»iji +עצ ×ŀ×ķ +бол езн +Ġë°Ľ ìķĦ +ม à¸Ļ +มà¸Ļ ุ +มà¸Ļุ ษ +มà¸Ļุษ ยà¹Į +âĹ Ĩ +×ŀ צ׾×Ļ×Ĺ +Ñıв ление +Ùħ Ø·ÙĦ +ÙħØ·ÙĦ ÙĪب +Ø® اÙĦÙģ +ت ÙĪÙĤÙģ +ãģ§ãģį ãģ¾ãģĽãĤĵ +оÑģÑĤ ей +м еÑĩа +기 ëĬĶ +תש ×¢ +ص ÙĬب +Ġ×ij×¢ ×ķ×ĵ +à¸Ĥà¸Ńà¸ĩ à¹Ģà¸Ĥา +ÑĤÑı ж +ĠÑĥ пÑĢав +ĠÑĥпÑĢав лениÑı +Ġgén ér +Ġth ÃŃ +פ ×ļ +Ġر Ùħض +ĠرÙħض اÙĨ +Ġtr uyá»ĩn +Ø¥ عداد +ãĤµ ãĥĿãĥ¼ãĥĪ +Ġпол но +Ø® اÙħ +ÐŁ еÑĤ +ÐŁÐµÑĤ еÑĢ +ÐŁÐµÑĤеÑĢ бÑĥÑĢ +ÐŁÐµÑĤеÑĢбÑĥÑĢ г +ÙħÙĨت دÙī +ãģķãĤĮ ãģ¾ãģĹãģŁ +ĠëĮĢ íķĺìŬ +à¸ľà¸¹à¹ī à¸Ĺีà¹Ī +Ġ×ŀ×IJ ×ķ +׾ ׳×ĵ +оÑĩ нÑĭе +ĠнаÑĩ ала +Ġ׾ ×Ļ׾×ĵ×Ļ×Ŀ +ов ое +ãģĻãĤĭãģĵãģ¨ ãģ§ +ĠاÙĦÙĨ Ùģ +ĠاÙĦÙĨÙģ Ø· +ìŀĪ ëĬĶ +غ ÙĨÙĬ +פ ×ĵ +ãĤ ¾ +ĠCr é +ãģ© ãģ¡ãĤī +Ø« اÙĨ +ÑĢаб аÑĤ +ÑĢабаÑĤ Ñĭва +Ġê°Ļ ëĭ¤ +à¸Ī ั +à¸Īั à¸ģร +Ġch ụ +Ġchụ p +Ġм аÑģÑĤ +ĠмаÑģÑĤ еÑĢ +Ġn ắm +ĠÑģÑĤ али +Ġ×Ķ×IJ ×Ļר×ķ×¢ +ãĤ½ ãĥ³ +åĪĨ ãģĭãĤĬ +Ø· بع +بد ا +gr áfico +г еÑĢ +à¸Ķำà¹Ģà¸Ļิà¸Ļ à¸ģาร +Ġsal dır +Ġsaldır ı +в ÑĪиÑħ +ãģĭãģ£ãģŁ ãģ§ãģĻ +Ġyapı yor +ĠاÙĦÙģ ت +צר פת +з доÑĢов +×ij×¢ ׾ +Ġ×IJ ×ŀ×Ļת×Ļ +Ġоб Ñĭ +ĠобÑĭ Ñĩ +ĠобÑĭÑĩ но +Ġ׾ ×ķ×ŀר +ت ÙĥÙĨ +تÙĥÙĨ ÙĪÙĦÙĪج +تÙĥÙĨÙĪÙĦÙĪج ÙĬا +Ġhakk ı +ĠÑĢаР² +ĠÑĢав но +رÙĬ Ùĥ +Ġ×ij ×ŀ×Ļ×ĵ +Ġ×ij×ŀ×Ļ×ĵ ×Ķ +à¹ģà¸ģ à¹īว +Ġìĸ ĺ +Ġìĸĺ 기 +ãģĹãģ¦ ãģĦãģ¾ãģĹãģŁ +Ġkı sm +Ġkısm ı +ê± ¸ +åĨħ ãģ® +ì§ ķ +à¹Ģหมืà¸Ńà¸Ļ à¸ģัà¸Ļ +ĠÙģ ÙIJ +ĠÙģÙIJ ÙĬ +ÙĤ اعدة +Ġmoż esz +Ùħ صاÙĦ +ÙħصاÙĦ ØŃ +ãģ¾ãģŁ ãģ¯ +б ег +Ġs ıc +Ġsıc ak +Ñĩ иÑģ +ÑĩиÑģ лен +Ġн ог +ãĥģãĥ£ ãĥ³ +ãĥ« ãĥī +Ġgi ó +Ġs ını +Ġsını f +ив аÑĤÑĮ +Ġqu ên +Ġì łģ +Ġìłģ ìļ© +ĠJo ão +Ùģ اد +ĠGl ück +à¸Ĺ à¸Ńà¸Ķ +Ġg ói +ï¼ Ĭ +Ġdé tail +ĠدÙĬ سÙħ +ĠدÙĬسÙħ بر +ë¡ľ ìĦľ +×ŀ ×ķ×Ĺ +à¹Ħ ฮ +ĠоÑĤ д +ĠоÑĤд ÑĭÑħ +Ġkh uyến +à¸Ħ à¸Ńย +Ġج ÙĨÙĬ +ĠجÙĨÙĬ Ùĩ +ĠاÙĦد Ùģاع +à¸Ļà¹īำ หà¸Ļัà¸ģ +ĠìĤ¬ëŀĮ ëĵ¤ìĿ´ +Ġth ừa +ĠÃ¶ÄŁrenc i +ĠпомоÑī и +ĠczÄĻ ÅĽÄĩ +ש ×ĺר +ĠN hi +ĠNhi á»ģu +׳ צ×Ļ +ĠнаÑĪ ем +ĠkarÅŁÄ± laÅŁ +Ġ×Ķש ׳×Ļ×Ŀ +ĠÄIJ Æ°á»Ŀng +Ġtr ú +ĠÑĢазлиÑĩ нÑĭÑħ +ĠاÙĦØ´ Ùĩر +Ġ×ľ×¢ ×ķ׾×Ŀ +ØŃ Ø¬Ø± +ĠÄij á»ķ +ĠìĿĺ íķ´ +à¸ļ à¹Īà¸Ńย +Ġ×Ķ ×Ļ׾×ĵ +ãģ¨ãģª ãģ£ãģŁ +Ġ×Ĺ×ķ ×ķת +Ġש×Ļר×ķת ×Ļ +Äħ cy +س رÙĬ +K Ä° +פ ׳×ķ +ÑģÑĤÑĢÑĥк ÑĤÑĥÑĢ +ÑĤ ÑĢÑĥд +Ġ×Ķ קר +Ġ×Ķקר ×ķ×ij +Ġth áºŃm +èģŀ ãģį +ÙĤÙĪ ÙĬ +клÑİÑĩ ен +ÑĤе Ñħ +ÑĤеÑħ нолог +è¡Į ãģ£ãģŁ +Ġ×ķ×IJ ×Ļף +ĠÅŁek lin +ĠÅŁeklin de +r ô +ÑĢ ог +Ġнов Ñĭе +Ġס ×ij×Ļ×ij +Ġtecn ologÃŃa +ס ׼ +×¡×Ľ ×ķ×Ŀ +ĠÅŀ ub +ĠÅŀub at +Ġ×Ķ×ŀ ׾×IJ +Ġwy pos +Ġwypos aż +ãģ¯ ä½ķ +ãĤ¬ ãĥ³ +ê° ĸ +Ġкак ие +Ġçocuk lar +Ġ׾צ ×ĵ +Ġkay ıt +ĠмеÑģÑĤ е +Ùħ دÙĬÙĨØ© +Ġ׼ ×Ĵ +Ġ׼×Ĵ ×ķף +ãģĹãģ¦ ãĤĭ +ĠÙħا ÙĬÙĪ +ãģ£ãģ¦ãģĹãģ¾ ãģ£ãģŁ +ĠпÑĢогÑĢамм Ñĭ +à¹ģล à¸Ļà¸Ķà¹Į +ãĥ¯ ãĤ¤ +ער ×ķ×¥ +Ñģ ид +ĠB öyle +Ġì²ĺ ìĿĮ +Ġת פק×Ļ×ĵ +ĠTr ên +íĥ Ī +ĠÐłÐ¾ÑģÑģ ий +ĠÐłÐ¾ÑģÑģий Ñģкой +Ġs Ãłn +Ġrè gle +ĠyaklaÅŁ ık +à¹Ģล ิà¸ģ +Ġد ائÙħ +Ġ×ķ ×Ĵ +اب ر +Ġb è +ĠاÙĦ ÙĤدÙħ +ĠÑĢеÑĪ ениÑı +hi ên +ÑĤи к +Ä Ħ +à¸ļรร ยาà¸ģ +à¸ļรรยาà¸ģ าศ +רצ ×ķף +åĭķ ãģį +ĠGä ste +Ġ기 본 +ĠÙĬ عرÙģ +ĠS á»Ń +gÅĤ ÄĻb +à¹Ģà¸Ń ส +×IJ×ŀ ×Ļף +Ġп Ñĥнк +ĠпÑĥнк ÑĤ +Ġ×Ļ×ķ×ĵ ×¢×Ļ×Ŀ +ãĤ« ãĥ©ãĥ¼ +Ġ×ijס ×ĵר +Ġbu á»ĵn +й ÑĤ +йÑĤ еÑģÑĮ +ãĤĴ æ±ĤãĤģ +Ġ×IJת ׼×Ŀ +Ġ모 르 +ظ رÙĪÙģ +Ñĩ еÑģÑĤво +ìĸ´ ìĦľ +Ġод на +Ġkap ı +Ġëħ¸ ëł¥ +ĠKü che +ĠاÙĦت Ø´ +Ø· ÙĬب +ĠíĬ¹ íŀĪ +ĠвÑĭп ÑĥÑģ +ĠвÑĭпÑĥÑģ к +×ĵ ת×Ļ +Ġu ÄŁ +ĠuÄŁ ra +ائ Ùĩا +Ġtho át +ãģª ãĤĤãģ® +Ñij ÑĢ +기 ê°Ģ +ĠgeliÅŁ me +تØŃ ÙĤ +تØŃÙĤ ÙĤ +Ġоп аÑģ +б ÑĢоÑģ +ห ุ +หุ à¹īà¸Ļ +ì¼ Ģ +ãĤ¹ ãĥŀ +ãĤ¹ãĥŀ ãĥĽ +Ø£ Ùģر +Ø£Ùģر اد +ĠTh á»±c +Ġth ắ +ãĥªãĥ³ ãĤ¯ +Ġni á»ģm +ĠHö he +عÙħ ار +ÙĥÙĪر ÙĪÙĨ +ÙĥÙĪرÙĪÙĨ ا +ĠÄIJ ến +ĠÑģам ом +ĠÑĤ еле +ĠÄijo án +à¸Ħวามà¸Ħิà¸Ķ à¹Ģหà¹ĩà¸Ļ +Ġд иÑģк +Ø£ Ø·ÙģاÙĦ +ม ารà¹Į +à¸Ĺ หาร +à¸Ĺ à¸Ļ +Ġب عÙĬد +ĠاÙĦÙĩ ÙĨد +åĩº ãģĹãģ¦ +Ġkar de +Ġkarde ÅŁ +×Ķ×Ļס×ĺ ×ķר +×Ķ×Ļס×ĺ×ķר ×Ļ×Ķ +éģ¸ ãģ³ +ع اÙħÙĦ +à¸Ĥ ยาย +Ġtü rl +Ġtürl ü +ĠìĿ¼ ìĿ´ +Ġmaté ria +Ġ׼׾ ×ķ×ŀר +ãĥģãĥ£ ãĥ¼ +جÙħ اعة +ĠÑģво им +Ø¥ÙĤ اÙħØ© +ä¾ĭ ãģĪãģ° +س اب +Ø¢ خر +ÙĤ دÙĬر +×IJ×ŀ ×Ļ +ìĸ » +Ġ׳×ķס פת +ĠÐĴ лад +ĠÐĴлад им +ĠÐĴладим иÑĢ +Ġest ará +ãģĵãģĨ ãģĦãģĨ +ãĤĴ 使çĶ¨ +มา à¸ķร +มาà¸ķร à¸IJาà¸Ļ +ãģ£ãģ ½ +Ġn ú +Ġnú i +ย าà¸ĩ +ĠاÙĦج ÙĨس +Ġüst ün +ëľ » +ãĤ» ãĥ« +ãģ¦ãģĦ ãģįãģ¾ãģĻ +Ġ×Ĺ ×ķ×ĸ +Ġ×Ĺ×ķ×ĸ ר +ĠÐĵ лав +à¹Ĥà¸Ĭ à¸Ħ +íı IJ +ÙĨت ظر +Ġ×Ĵ ×ij×Ļ +ع ÙĤب +int ér +intér êt +×ŀ פ×Ĵ +×ŀפ×Ĵ ש +Ġth ù +اÙģ ت +Ġ×ŀש פ +Ġ×ŀשפ ×ĺ×Ļ +ĠÙħ ÙĪاÙĤع +è¦ ļ +è¦ļ ãģĪ +×ĵ ×Ļף +à¹Ģรืà¹Īà¸Ńà¸ĩ ราว +ãģ¾ ãģĤ +Ġgh ế +иÑĢÑĥ ÑİÑĤ +à¸ģ ว +à¸ģว à¹īาà¸ĩ +Ġпов еÑĢ +ĠповеÑĢ Ñħ +ĠповеÑĢÑħ ноÑģÑĤ +׳ ×ĵר +Ġкон ÑĨе +Ġдолж на +Ġ×Ļש ×Ļר +acaģı z +ìĹ Ķ +Ġn ÃŃvel +Ġö r +Ġör nek +Ùĥ Ùģ +ĠФедеÑĢ аÑĨии +Ġ구 ìĦ± +หัว à¹ĥà¸Ī +ĠV áºŃy +м ед +мед и +меди ÑĨин +медиÑĨин Ñģк +از ÙĬ +×Ĵ×ij ×ķ׾ +ÑĦ ÑĢ +Ġzus ätzlich +à¸ģ à¸ģ +ĠاÙĦاÙĤتصاد ÙĬØ© +Ġh è +lu ÄŁun +ج Ùİ +à¹Ħà¸Ł ลà¹Į +ÄIJ T +ãģĿãģ® ä»ĸ +à¸Ĺิ à¹īà¸ĩ +ĠاÙĦØ£ ÙĪ +ر سÙħ +æ°Ĺ ãģ¥ +ìĿ´ ë©° +ÑĮ ев +ص Ø· +ĠاÙĦاست Ø« +ĠاÙĦاستث Ùħار +à¸Ńา à¸Ħาร +ĠÑĤоÑĩ но +ĠV ân +à¸Ń ร +à¸Ńร à¹Īà¸Ńย +ĠاÙĦس ÙĨØ© +Ġc Æ°á»Ľi +×Ļ×Ķ ף +íį ¼ +話 ãģĹ +âĹ ĭ +ĠìķĬ ìĿĢ +ãĥ¡ ãĥ¼ãĤ +ãĥ¡ãĥ¼ãĤ « +ãĥ¡ãĥ¼ãĤ« ãĥ¼ +ĠÑĤеп ло +å½¼ ãĤī +ĠÄ° z +ĠÄ°z mir +íĻ į +Ġr ượ +Ġrượ u +æĢĿãģĦ åĩº +ĠPh ạm +Ġchá u +צ×Ļ ×ķת +ĠìĿ¼ 본 +ìĤ¬ ëĬĶ +ĠÑģозд ан +Ġar acı +Ġ×¢ ר +Ġער ×Ļ׼×Ķ +ĠíķĺëĤĺëĭĺ ìĿĺ +dzi ÅĤ +à¸Ľà¸£à¸° à¸ĺาà¸Ļ +Ġser ÃŃa +ĠìŀĪ ëıĦë¡Ŀ +در ج +íķľëĭ¤ ëĬĶ +à¸Ńา à¸Ĺ +à¸Ńาà¸Ĺ ิà¸ķ +à¸Ńาà¸Ĺิà¸ķ ยà¹Į +ÑĤелÑĮ нÑĭй +ĠØ® دÙħات +×ŀ׳ ×ĺ +Ġl ược +ĠS Ãłi +ĠÙĪ اض +ĠÙĪاض ØŃ +غ از +ĠdoÄŁ al +Ġ×ijש ×Ŀ +Ġд лин +ĠØ¥ طار +Ġ×ijס פר +ãĤĴ ä¸İ +ãĤĴä¸İ ãģĪ +Ġë²ķ ë¥ł +ĠÑĥ вели +ĠÑĥвели Ñĩи +ส à¹Ħà¸ķ +สà¹Ħà¸ķ ลà¹Į +à¹Ħ à¸ģล +×ij׊ף +ĠìĿ´ íĽĦ +Ġm unic +Ġmunic ÃŃpio +تÙħ Ø«ÙĦ +ĠÄij áo +H ôtel +Ġl á»Ńa +ĠÄij ẳng +Ñĩ ки +Ø´ رÙĪ +شرÙĪ Ø· +ĠìĿ´ 를 +ÙĬ Ùĭا +×ŀ׾ ×ļ +×ŀ×Ķ ×Ļר×ķת +ĠобÑıз аÑĤелÑĮ +ĠобÑıзаÑĤелÑĮ но +é nergie +Ġmud ança +Ġm ụ +Ġmụ n +Ġn º +ĠاÙĦت عا +ĠاÙĦتعا ÙĪÙĨ +ĠاÙĦاجتÙħاع ÙĬØ© +Ġп лаÑģÑĤ +Ġëĵ± ìĿĺ +ãĥIJãĤ¤ ãĤ¯ +Ùĩج ÙĪÙħ +ĠSa úde +Ġì¤ijìļĶ íķľ +Ġ×Ķצ ×Ļ×ij×ķר +תק ף +ĠاÙĦعاÙĦÙħ ÙĬ +ĠболÑĮÑĪ ой +ĠÙĥ ÙĦÙħ +ĠÙĥÙĦÙħ Ø© +ãģ®ãģ§ãģ¯ãģªãģĦ ãģ§ãģĹãĤĩãģĨãģĭ +ĠÙħ باراة +Ġש×IJ ׳ +Ġש×IJ׳ ×Ĺ׳×ķ +ãĤ¹ãĤ¿ ãĤ¤ãĥ« +ĠSa ÄŁ +ĠSaÄŁ lık +Ġh Æ° +׳ ×Ĺ×Ķ +Ġ×ij קר×ij +Ø· عÙħ +ห ิà¸Ļ +à¸Ĺุà¸ģ วัà¸Ļ +à¸Ħรัà¹īà¸ĩ à¸Ĺีà¹Ī +ĠlÃł nh +Ġdonn é +ãģĽ ãģĦ +جز ÙĬرة +доÑĢ ож +ì¼ ľ +تÙĨظ ÙĬÙģ +ãĥģ ãĥ§ +Ġald ıģı +ج اج +ĠÑĤ омÑĥ +à¸Ľ ิ +Ġ×ijר שת +ãģıãģªãĤĬ ãģ¾ãģĻ +ĠпÑĢин ÑĨип +Ġ׊׾×ķ +ëı ¼ +×ķ×Ĵ ש +س س +à¸Ľ ู +Ġh ầu +æĦŁãģĺ ãĤĭ +ï¼ ´ +د ÙĪا +ĠÑģм ог +scri ção +Ġth áºŃn +Ġר ×ķ×IJ×Ķ +обÑĢаж ен +ĠاÙĦتج ارÙĬØ© +Ø· بÙĬع +jÄħc Äħ +íĸī ìľĦ +Ġнов Ñĭй +Ġ×ŀ ×Ĺ×ĵש +æĮ¯ ãĤĬ +gu é +Ġ×IJ ×Ļר×ķ×¢ +Ġ×IJ×Ļר×ķ×¢ ×Ļ×Ŀ +ĠاÙĦ Ø°Ùĩب +×ĵ ×IJ +ت اÙĨ +ãģł ãģĹ +à¸Ńั à¸ķรา +à¹Ĥ à¸Ī +بÙĦ اد +×Ķ×Ļ ×Ļ׳×ķ +ĠÑģп е +ĠÑģпе ÑĨиалÑĮно +ĠÅĽwi ata +ãĤĵãģ§ãģĻ ãĤĪ +شر ÙĥØ© +ĠpÅĤ yt +Ġsitu é +Ġ׼×IJ ׾×Ķ +ס ×ijר +Ġkaż d +Ġkażd ym +ãĤĴæĮģ ãģ¤ +׾×Ķ ׾ +׾×Ķ׾ ף +ĠwÅĤ as +ĠwÅĤas ne +ĠsaÄŁ lan +×ŀ×¢ ׾×Ķ +ĠاÙĦا ÙĪÙĦ +ìĹIJìĦľ ëıĦ +×IJ×Ļר ×ķפ×Ķ +تÙĤ ÙĨÙĬØ© +Ùħ ائ +Ùħائ Ø© +Ġcompañ ÃŃa +Ġsü rek +Ġsürek li +ĠиÑģ кÑĥÑģ +ĠиÑģкÑĥÑģ ÑģÑĤв +ĠB ürger +ת ×Ĺר +ת×Ĺר ×ķת +à¸ŀรà¹īà¸Ńม à¸ģัà¸ļ +Ø´ Ùħ +à¸ĸืà¸Ń วà¹Īา +è¾¼ ãĤĢ +ä¼ij ãģ¿ +ĠاÙĦØ£ ب +ĠÑģÑĤоим оÑģÑĤÑĮ +ĠпÑĢав а +may ın +ห วย +ĠاÙĦØ· بÙĬعÙĬ +à¸Ĺีà¹Ī à¸ŀัà¸ģ +ĠEst á +Ñĭва ÑİÑĤ +ب سÙĬ +بسÙĬ Ø· +Ġ×ij×¢ ×ijר +åı¯èĥ½ ãģ§ãģĻ +Ġ×ĵ ×ķ׾ +Ġ×ĵ×ķ׾ ר +Ùĩ Ùİا +воÑĢ оÑĤ +ãģ¦ ãģĦãģ¾ãģĹãģŁ +à¹Ĥà¸Ĺร ศ +à¹Ĥà¸Ĺรศ ั +à¹Ĥà¸Ĺรศั à¸ŀ +à¹Ĥà¸Ĺรศัà¸ŀ à¸Ĺà¹Į +Ġק ׳ +ĠاÙĦØ« ÙĨ +ĠاÙĦØ«ÙĨ ائÙĬØ© +Ġco ût +à¸ķิà¸Ķ à¸ķัà¹īà¸ĩ +Ġö rg +Ġörg üt +ĠاÙĦØ® ÙĦÙĬ +ĠاÙĦØ®ÙĦÙĬ ج +Ġb á»įn +×ķ׾×ķ×Ĵ ×Ļ +ëŀ ľ +ĠÐij олÑĮ +ĠÐijолÑĮ ÑĪ +×Ĵ ×ijר×Ļ×Ŀ +ÙĤ ÙĬد +×ij×Ļ×ĺ ×ķ×Ļ +æīĵ ãģ¡ +Ġol muÅŁ +f äh +fäh ig +ล าà¸Ļ +ĠÙĤ طر +ש פ×Ķ +èªŃ ãĤĵãģ§ +à¸Ĥ วา +Ġchi ếm +ãĤ¤ãĥ³ ãĤ¿ +ãĤ¤ãĥ³ãĤ¿ ãĥ¼ãĥ +ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥ į +ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥį ãĥĥãĥĪ +Ġ׾ש×ŀ ×ķר +Ġت رÙĥ +ĠترÙĥ ÙĬا +ר ×ķ×ĺ +ãģ¨æĢĿ ãģĦãģ¾ãģĹãģŁ +ĠاÙĦت ÙĤ +Ġd Æ° +ãģ¦ãģıãĤĮ ãĤĭ +ãģĹãģŁ ãģĵãģ¨ +Ġróż ne +ĠاÙĦØ· ÙģÙĦ +ĠPost é +Ġ×ŀש ×ķ×Ŀ +Ñį ÑĢ +ĠÑĢабоÑĤ аеÑĤ +ãĤ· ãĥª +ãĤ·ãĥª ãĥ¼ãĤº +Ġ×ij×Ķ ×Ĺ׾×ĺ +ק×Ķ ×Ļ׾×Ķ +ãĤ« ãĥ¡ +ãĤ«ãĥ¡ ãĥ© +ï¼ ¯ +ĠìĤ¬ ìĿ´ +Ġk ì +Ġth Æ°á»Ľc +ض بط +ÙĤب ÙĪÙĦ +åĪ¥ ãģ® +Ġparticul ière +ĠÑģво ем +Ġ×¢ סק +Ġעסק ×Ļ×Ŀ +×ij×Ĺ ×Ļר×ķת +×ij ×Ļ׳×ķ +à¸ĭ à¸Ń +Ġ×¢ ×ķ×ijר +ãģłãģ£ãģŁ ãģ®ãģ§ +ıld ıģı +Ùħ دار +Ùħدار س +주 ìĭľ +à¸Ńา ศ +à¸Ńาศ ัย +Ġt ấm +à¸ŀิ à¸Ī +à¸ŀิà¸Ī าร +à¸ŀิà¸Īาร à¸ĵา +ÑĤелÑĮ нÑĭе +Ñģк ÑĥÑİ +Ðľ Ðĺ +à¹Ģà¸ģ า +à¹Ģà¸ģา หล +à¹Ģà¸ģาหล ี +×ĵ ×Ĺ +à¹Ģà¸Ĭ ิà¸ĩ +Ġد ÙĤÙĬÙĤØ© +íķĻ ìĥĿ +Ġש×IJ ׾×Ķ +Ġcontr ôle +Ġsit uação +à¸Ĥà¸Ńà¸ĩ à¸ľà¸¹à¹ī +ÙĨ Ø·ÙĤ +ê³¼ íķĻ +หลาย à¸Ħà¸Ļ +Ġn ắng +ÙĤ Ùı +ì¡° ê±´ +Ñ ķ +ãĥĥ ãģ¨ +×ŀ ×Ļ׾×Ķ +Gr ün +×Ļ ×Ļ×¢ +×Ļ×Ļ×¢ ×ķ×¥ +×ŀ׳ ׼ +ë ŃIJ +×ŀ×¢ ×ŀ×ĵ +สำ à¸Ļัà¸ģ +ج دد +à¸Ħ ัà¸Ķ +Ġ×Ķ×ŀש פ +Ġ×Ķ×ŀשפ ×Ĺ×Ķ +×ŀש ק׾ +ÙĦ Ùı +Ġty tu +Ġtytu ÅĤ +ÑĪ ей +ĠìĿ¼ ë¶Ģ +ÑĪ ение +Ġph óng +ĠìĹŃ ìĤ¬ +ãĤ« ãĥ³ +Ġtú i +ĠÙĨ ÙĪÙģ +ĠÙĨÙĪÙģ Ùħبر +gr ün +ĠاÙĦØ´ ÙħاÙĦ +ÅĽwi adc +ÅĽwiadc zenie +ער ×Ķ +Ġ×¢ ×ķ×ij +Ġ×¢×ķ×ij ×ĵ×Ļ×Ŀ +×ĵ×ķ×Ĵ ×ŀ×IJ +ä»Ĭ ãģ¯ +Ġv ão +ĠТ ем +Ñģ илÑĮ +Ġch ợ +Ùħ را +Ùħرا ÙĤب +à¹Ħมà¹Ī รูà¹ī +Ġر ائع +×IJ׳ ×Ĺ׳×ķ +สà¹Īà¸ĩ à¹Ģสริม +צ ×Ĺ +ĠìŀĪìĸ´ ìĦľ +Ġkur ulu +Ġkurulu ÅŁ +ĠÃĸ zellik +ĠÃĸzellik le +Ġת ×Ļק +Ġgh é +Ġspr zÄĻ +ĠsprzÄĻ t +ער ×ķת +را ØŃØ© +ãģ£ ãģį +ãģ£ãģį ãĤĬ +ĠìķĦ ëŀĺ +stit uição +Ġдолж но +×Ķ רש +×Ķרש ×ŀ×Ķ +×Ķ׾ ×ļ +ãģ¡ ãģª +ãģ¡ãģª ãģ¿ +ãģ¡ãģªãģ¿ ãģ« +פ ×Ĺ×ĵ +ĠاÙĦج ÙħÙĬع +×ij×¢ ׾×Ļ +Ġtr ùng +Ġפ ת×Ĺ +×ŀ׾×Ĺ ×ŀת +ãĥĨ ãĥ¼ãĥ +ãĥĨãĥ¼ãĥ ŀ +Ùħ تاب +Ùħتاب عة +Ġ모 ìĬµ +ÙĬ ص +åIJĪ ãģĨ +ĠY ap +ĠYap ı +ĠÑģ казаÑĤÑĮ +ëª ° +à¸Ĺีà¹Ī สำà¸Ħัà¸į +ĠìĹĨ ìĬµëĭĪëĭ¤ +Ġnh ắc +Ġülk eler +Ġмног ие +íķĺ ìħ¨ +มาà¸ģ à¸Ĺีà¹Īสุà¸Ķ +à¸ģ à¹īา +à¸ģà¹īา ว +ĠÄ° yi +л еж +леж а +ãĤ¸ ãĥ§ +à¸Ĺั à¸ŀ +ا ÙĪر +Ġ×Ĺ×ijר ×Ļ +Ġ׾ ש×Ŀ +ì² « +ĠT á»Ń +×ŀ ×ķ׳×Ļ +ÙĤ ÙĪد +à¸ģระ à¹Ģà¸Ľ +à¸ģระà¹Ģà¸Ľ à¹ĭ +à¸ģระà¹Ģà¸Ľà¹ĭ า +ĠпÑĢоблем Ñĭ +Ġaç ıs +Ġaçıs ından +Ġ×Ķ×ŀ ׼ +ĠÙħع ظÙħ +ÙĤÙĬ اس +ĠпÑĢод олж +ĠпÑĢодолж а +Ġver diÄŁi +ĠпÑĢед меÑĤ +ãģĦãģ¾ãģĻ ãģĮ +ĠëĶ° 른 +ĠاÙĦ ÙĤÙĬاÙħ +ĠØ¥ÙĦÙĬ Ùĩا +Т ÐIJ +п оз +ãĤ· ãĥ¥ +ä¸ĬãģĮ ãĤĬ +à¹Ģà¸Ķิม à¸ŀัà¸Ļ +à¸ģุ ล +ØŃر ÙĬØ© +ק×ij×ķצ ×ķת +ë¯ ¿ +ĠاÙĦÙħ ÙĨا +ĠاÙĦÙħÙĨا Ø·ÙĤ +ĠвÑĭп ол +ĠвÑĭпол нÑı +ãĥĭ ãĤ¢ +Ġê²° êµŃ +×Ĺ ×ķ×ŀ +×Ĺ×ķ×ŀ ר×Ļ×Ŀ +ĠУкÑĢа инÑĭ +ห à¸Ńม +ר ×Ļס +ĠÑħоÑĤ ел +ĠобÑĢаз ованиÑı +Ġkh ẳng +Ġm Æ°a +Ġgör me +Ġgüç lü +سع Ùī +มัà¹Īà¸Ļ à¹ĥà¸Ī +íķĺ ê²łìĬµëĭĪëĭ¤ +Ġпол Ñĥ +Ġfün f +ãģ¨æĢĿ ãģ£ãģ¦ãģĦãģ¾ãģĻ +Ġê·¸ê²ĥ ìĿĢ +ĠdÃ¼ÅŁÃ¼n ce +ìŀ ł +ĠH Æ°á»Ľng +ĠTi á»ĥu +Ġç ift +ãģij ãģ° +à¸Īà¸Ļ à¸ĸึà¸ĩ +à¸Ĺำ à¹Ħà¸Ķà¹ī +ĠìŀIJ ì²´ +Ġd õ +Ġdõ i +à¸Ī ัà¸Ļ +à¸Īัà¸Ļ à¸Ĺ +à¸Īัà¸Ļà¸Ĺ รà¹Į +ece ÄŁini +׳×ķ×¢ ר +غ ار +ĠاÙĦØ£ÙħرÙĬ ÙĥÙĬ +داع Ø´ +ĠбезопаÑģ ноÑģÑĤи +Ġб Ñİ +ĠбÑİ Ð´Ð¶ +ĠбÑİдж еÑĤ +ãĥĬ ãĤ¤ +à¸ŀà¸ļ วà¹Īา +da ÄŁ +×IJ ×ķפף +íĹ Į +ãĥĢãĤ¤ ãĤ¨ +ãĥĢãĤ¤ãĤ¨ ãĥĥãĥĪ +ĠëĮĢ íĨµ +ĠëĮĢíĨµ ëł¹ +D Ä° +Ø£ ØŃداث +ĠA ÄŁ +ĠAÄŁ ust +ĠAÄŁust os +ØŃÙĦ ÙĪÙĦ +Ġw ÅĽ +ĠwÅĽ ród +ĠÑģо оÑĤвеÑĤ +ĠÑģооÑĤвеÑĤ ÑģÑĤв +ĠÑģооÑĤвеÑĤÑģÑĤв ии +ĠLu áºŃt +Ġ׼׾ פ×Ļ +Ġв еÑī +ĠвеÑī еÑģÑĤв +ק ×Ļ×¥ +ĠبÙĩ ذا +عا Ø´ +à¹Ģà¸Ľà¹ĩà¸Ļ à¹Ģรืà¹Īà¸Ńà¸ĩ +Т Ðķ +Ġ×ij×IJ ×Ļ׳×ĺר׳×ĺ +س عد +Ġ×Ķ×ĺ ×Ļפ×ķ׾ +פ ×Ļס +à¸ĩà¹Īาย à¹Ĩ +ĠGer ät +׾ ×Ļ×ĵ×Ķ +ĠÑĢ иÑģк +׾ק ×Ĺ +н наÑı +ר ×Ļ×ĵ +п ÑĢакÑĤи +пÑĢакÑĤи к +à¸Ĥัà¹īà¸Ļ à¸ķà¸Ńà¸Ļ +à¸Ļà¹Īา รัà¸ģ +larınız ı +à¸Ńà¸Ļุ à¸įา +à¸Ńà¸Ļุà¸įา à¸ķ +ĠzdjÄĻ cia +Ġb ây +Ñģ ÑĢ +ÑģÑĢ оÑĩ +ãĥĭ ãĥ³ãĤ° +Ġö ner +Ġöner i +Ġнов ÑĭÑħ +دع ÙĪØ© +Ġg ắn +ĠاÙĦÙĦ بÙĨ +ĠاÙĦÙĦبÙĨ اÙĨÙĬ +ãĥĨãĤ£ ãĥ¼ +Ġص ØŃÙĬØŃ +ем ÑĭÑħ +çĸ² ãĤĮ +ĠпÑĢо иÑģ +ĠпÑĢоиÑģ ÑħодиÑĤ +ส à¸ķิ +ĠT ết +Ġ×Ķ׾ ׾×ķ +à¹Ģรืà¹Īà¸Ńà¸ĩ à¸Ļีà¹ī +×ŀ×ij ׳×Ķ +Ġconte údo +Ġا خت +Ġاخت ÙĬار +Ùħ سÙĦ +ÙħسÙĦ سÙĦ +ëı Ī +Ġ׾ ×Ļ×ĵ +à¸ŀิ à¸ĺี +ĠÑģов Ñģ +ĠÑģовÑģ ем +ãģĮãģĤãĤĬ ãģ¾ãģĹãģŁ +Ġsó ng +Ø¥ صÙĦاØŃ +ë§ ģ +Ùģ ÙĬر +ĠJe żeli +ìłľ ëıĦ +d ÅĤug +ìĥģ ìĿĦ +Ġc áºŃn +Ġhá»į p +Ø£ ست +أست اذ +Ġ×ŀ ×Ļש×Ķ +Ġ×ŀ×Ļש×Ķ ×ķ +Ġd Ãły +Ġch Ãłng +ãģ¡ãĤĥãĤĵ ãģ¨ +ĠÄij ám +Ġsw ój +Ġpoder á +ĠоÑĤлиÑĩ а +Ġpéri ode +ünd ig +×ĺ×¢ ף +ÑģÑĤÑĢо иÑĤелÑĮ +ר ת×Ļ +Ġ×Ļ×Ķ ×Ļ×ķ +׾ ס +ĠاÙĦÙħÙĨ زÙĦ +à¸Ļิ à¹īว +иÑĦ ика +иÑĦика ÑĨи +ðŁĺ ī +Ġad ına +ãĢĤãĢĤ ãĢĤ +×IJ ×Ļף +ס ×Ļר +ĠÙĬ عد +çŃĶ ãģĪ +اÙĦ جز +اÙĦجز ائر +енÑĮ к +ร ห +รห ัส +ĠTürk çe +ê¾ ¸ +Ġ×Ļ ×ķ׼׾ +Ġש ×ķ׳×Ķ +Ġ×ij×ŀ צ×ij +ĠдейÑģÑĤв иÑĤелÑĮно +ĠبأÙĨ Ùĩ +×ŀק ×ĵ +Ġ×Ķש ק +Ø®ÙĬ ارات +Ġf ı +Ġfı rs +Ġfırs at +ëij ĺ +ĠìĦľ ìļ¸ +Ġ×Ķ×Ĵ ×ķ×£ +ر عا +رعا ÙĬØ© +ĠK ết +к Ñģи +ĠÑĥÑģлÑĥг и +ноÑģÑĤ ей +ìļ´ ëıĻ +ĠобÑĬ Ñı +ĠобÑĬÑı вл +н еж +×Ķפ ×ļ +Ġ×ij×¢ ×Ļ׳×Ļ +ëĨ Ĵ +ĠпÑĢоÑĨ ед +ĠпÑĢоÑĨед ÑĥÑĢ +Ġiht iy +Ġihtiy acı +Ġë°Ķ ëŀį +Ġë°Ķëŀį ëĭĪëĭ¤ +à¸ģล ัว +ĠÑģл ожно +ק×Ļ ×Ļ×ŀת +ĠÄIJ ình +ĠÙħ ÙĦÙģ +Ġà¹Ĥà¸Ķย มี +Ġkat kı +تØŃ ÙĪÙĬÙĦ +à¹Ħ à¸ŀ +ĠH á»į +ñ e +Ġдо Ñħод +Ġtho ải +íķĺìŬ ìķ¼ +ãĤ¹ãĥĿ ãĥ¼ãĥ +ãĤ¹ãĥĿãĥ¼ãĥ Ħ +ĠG òn +Ġk è +Ġkè m +éĢ² ãĤģ +ãĤ¹ ãĥ¼ãĥ +ãĤ¹ãĥ¼ãĥ ij +ãĤ¹ãĥ¼ãĥij ãĥ¼ +ĠgiÃł u +ĠØ¥ عادة +Ġ׾ ×ķק +Ġ׾×ķק ×Ĺ +ĠÑħоÑĩ еÑĤ +×ĺ ׾×ķ×ķ +×ĺ׾×ķ×ķ ×Ļ×ĸ +×ĺ׾×ķ×ķ×Ļ×ĸ ×Ļ×Ķ +Ġth uyết +ãģĿãĤĮ ãģ§ +Ġvard ı +à¹Ħร à¹ī +ع بد +ĠRep ública +ãĥ¼ãĤ¿ ãĥ¼ +Ġ×ŀ×IJ ×ķת +à¹Ħà¸Ľ à¹ģลà¹īว +Ġyapıl acak +ãĤ¹ãĤ¿ ãĥ¼ãĥĪ +ãģ» ãģ¼ +Ġko ÅŁ +ĠмаÑĤ еÑĢи +Ġsiè cle +ĠاÙĦÙħ ختÙĦÙģ +ĠاÙĦÙħختÙĦÙģ Ø© +Ġ׾ק ר×IJ +Ġ׾קר×IJ ת +Ġ×Ķפ ×ķ×¢×ľ +Ġt òa +Ġr Æ¡i +åij¨ ãĤĬ +à¸Ŀ à¸Ļ +j ÅĽÄĩ +ĠìķĬ ìĿĦ +اÙĨت ÙĤاÙĦ +ëĸ ł +ив аеÑĤ +ãĥĪ ãĥ« +ĠاÙĦÙģÙĦسطÙĬÙĨ ÙĬØ© +à¸ģลà¹Īาว วà¹Īา +ا Ùĥت +ĠÃĸ l +ĠÑĢе ÑĪи +ĠÑĢеÑĪи л +Ġ׳×ķס פ×ķת +Ġìłķ ì¹ĺ +вл еÑĩен +Ùħر ØŃÙĦØ© +Ġcome ça +Ġy ık +ìĤ ´ +à¸ĺ à¸Ļา +à¸ĺà¸Ļา à¸Ħาร +à¸Ńà¸Ļ า +à¸Ńà¸Ļา à¸Ħ +à¸Ńà¸Ļาà¸Ħ à¸ķ +Ġpeque ña +ä»ķ äºĭãĤĴ +Ġب Ø°ÙĦÙĥ +Ġнов ого +ãģĹãģ¦ ãģĦãģªãģĦ +ĠاÙĦÙħ ÙĬاÙĩ +à¸ģà¹ĩ à¹Ģà¸Ľà¹ĩà¸Ļ +Ġж ÑĥÑĢ +ĠжÑĥÑĢ нал +в еÑģ +خت ار +Ġ매 ìļ° +ĠM ã +ĠавÑĤомаÑĤ Ñĭ +ضع Ùģ +ĠاÙĦÙģ Ùĥر +ãģ§ãģĻ ãģ®ãģ§ +ãĥ¡ãĥ³ ãĥIJãĥ¼ +Ġк ÑĢÑĥг +ĠاÙĦسÙĦ طة +à¸Ħรัà¹īà¸ĩ à¹ģรà¸ģ +à¸ģระà¸Ĺ รว +à¸ģระà¸Ĺรว à¸ĩ +ÑĨ ов +éķ· ãģĦ +大ãģį ãģĦ +Ġgeç miÅŁ +ìĦ± ìĿ´ +Ġצר ×Ļ׼×Ķ +Ġм оÑī +ĠмоÑī н +Ġק ×Ļש +Ġק×Ļש ×ķר×Ļ×Ŀ +ĠNas ıl +г ÑĢан +Ġ×ŀ ×ķצר×Ļ×Ŀ +Ġ×ŀס ×ķ×Ĵ +Ġy ür +Ġyür üt +Ġ׾׊צ×ķ +×ķÖ ¼ +ĠìŀĪ ìĹĪëĭ¤ +Ġter ör +ĠTh Æ°Æ¡ng +ĠÙĪ ÙĬÙħ +ĠÙĪÙĬÙħ ÙĥÙĨ +ج ÙĪÙĨ +ĠÙĪغÙĬر Ùĩا +×ŀ פ×ķ +×Ĵ×ķר ×ŀ×Ļ×Ŀ +׼×ij ×Ļש +ĠاÙĦÙĦ غ +ĠاÙĦÙĦغ Ø© +شر Ùĥ +ĠاÙĦر اب +ĠاÙĦراب ع +ĠпÑĢ ек +ĠпÑĢек ÑĢаÑģ +ĠпÑĢекÑĢаÑģ н +Ġenerg ÃŃa +ק×ĵ ×ŀ×Ļ +ãģıãģª ãģ£ãģŁ +ĠÄij ứ +ĠÄijứ a +Serv i +Servi ço +Ġkald ır +åĥį ãģį +Ġод еж +Ġодеж д +물 ìĿĦ +ãģĿãģĨ ãģ§ +ãģĮãģĤ ãĤĮãģ° +ìĻ ķ +צ×ĵ ק +Ġart ır +Ġile ti +Ġileti ÅŁim +ãĤĪãģĨ ãģ§ +ãĥĪ ãĥ¼ +ãĤ¢ ãĥĭ +ãĤ¢ãĥĭ ãĥ¡ +×ĺ×Ļ ×Ļ׾ +ãĥķ ãĥªãĥ¼ +ãĥĿ ãĥ³ +ÐŁÑĢ о +Ġع اÙĦÙĬØ© +ĠÃ¶ÄŁ ret +ĠÃ¶ÄŁret men +ĠкаÑĩеÑģÑĤв а +Ġ×Ķ×ĺ ×ij×¢ +Ġзна Ñİ +ãģ¦ ãģıãĤĭ +Ġm ừng +ÙħÙĪ ت +ש ×ķ×ŀר +×Ĺ׾ ×ij +Ġwzgl ÄĻ +ĠwzglÄĻ du +ë²Ī 째 +Ġtá» ĵ +Ġtá»ĵ n +ãĥ¯ãĥ¼ ãĤ¯ +Ġpo życz +Ġpożycz k +×Ļ ×ķצר×Ļ×Ŀ +Ùĥر Ùħ +Ġг аÑĢ +ĠгаÑĢ ан +ĠгаÑĢан ÑĤи +ล à¹īาà¸ĩ +Ġìĺģ íĻĶ +×ĺ ×Ļס +Ġth ẻ +ĠìŀĪëĭ¤ ê³ł +اÙĦت ز +اÙĦتز اÙħ +Ġна ÑĪи +is ée +ãģĵãĤĮ ãĤĴ +Ġm ẽ +ض ÙĦ +بÙĪ ت +Ġ׼ ׼×Ķ +h ợ +ĠاÙĦس ÙĪرÙĬØ© +Ġ×ľ×¢ ×ķ×ŀ +Ġ×ľ×¢×ķ×ŀ ת +ĠbaÅŁ ar +ĠbaÅŁar ılı +е ÑģÑĤÑĮ +à¸Ħร ี +à¸Ħรี ม +ĠìłĦ ì²´ +ĠسÙĬ ÙĥÙĪÙĨ +Ġ×ŀ×ĵ ×ķ×¢ +ĠëķĮ문 ìĿ´ëĭ¤ +Ġc ứng +ger ät +Ġм иÑĢ +ĠмиÑĢ е +ĠÙĥÙĬÙģ ÙĬØ© +Ġפר ×ĺ×Ļ×Ŀ +Ġgo ÅĽci +иÑĤ еÑģÑĮ +ÑĥÑĪ ки +ؤ ÙħÙĨ +Ġ×IJ ׼ף +ĠاÙĦر جÙĦ +Ġl á»įc +à¹Ģรีย à¸ģวà¹Īา +ãģĵãģ® ãĤĪãģĨãģª +ë§Į íģ¼ +Ġп еÑĩ +ÙĪÙĦ ات +ĠÃľ ye +liÄŁ inde +à¸Ħะ à¹ģà¸Ļ +à¸Ħะà¹ģà¸Ļ à¸Ļ +ãĤĭãģĵãģ¨ ãģ¯ +วิ à¹Ģà¸Ħร +วิà¹Ģà¸Ħร าะ +วิà¹Ģà¸Ħราะ หà¹Į +Ġвозмож ноÑģÑĤи +ĠاÙĦÙĨ ساء +ãĥīãĥ© ãĥŀ +Ġgü c +Ġgüc ü +Ġt Æ°á»Ŀng +Ġacomp aña +ãĤ¤ ãĥ© +ק צ×ij +ĠY ö +ĠYö net +ĠYönet im +สัม à¸ľ +à¸ªà¸±à¸¡à¸ľ ัส +à¸Ļ าม +ĠÄij ợi +à¹ģหà¹Īà¸ĩ à¸Ĭาà¸ķิ +ãģĿãĤĮ ãģ§ãĤĤ +ät ig +ת ×ķ×Ŀ +ĠbaÅŁ lat +ĠвÑģ ей +ת ×Ļק +ת×Ļק ×ķף +ĠNg ô +ĠGesch ä +ĠGeschä fts +Ø£ Ùħ +Ø£Ùħ راض +à¹Ģà¸Ĺ à¸Ħà¸Ļ +à¹Ģà¸Ĺà¸Ħà¸Ļ ิ +à¹Ģà¸Ĺà¸Ħà¸Ļิ à¸Ħ +Ġм енÑĮ +ĠменÑĮ ÑĪе +Ġöl ç +Ġölç ü +ĠÙĬ جعÙĦ +ĠÄij ỡ +ש ×Ļ׾ +ש×Ļ׾ ×ķ×ij +ĠGr Ã¶ÃŁe +ĠÙĩ اتÙģ +รà¹īาà¸Ļ à¸Ńาหาร +×Ķ׾ ×Ļ׼ +×Ķ׾×Ļ׼ ×Ļ +иÑĢÑĥ ÑİÑī +èĭ¥ ãģĦ +ĠÃĸ zel +ãģĦãģŁ ãĤī +à¸Ħำ à¸ĸาม +Ġzosta ÅĤy +Ġ×Ķס ×Ļפ×ķר +×Ķ ×ķ׾ +×Ķ×ķ׾ ×ļ +à¹Ģà¸Ĭà¹Īà¸Ļ à¸ģัà¸Ļ +à¹Ĥ à¸Ĩ +à¹Ĥà¸Ĩ ษ +à¹Ĥà¸Ĩษ à¸ĵา +×IJר צ×ķת +×Ĵר פ×Ļ +Ġao ût +ĠÙĬ رÙĬد +ت ÙĪج +تÙĪج ÙĬÙĩ +ĠÑįÑĤ ап +ãĤ¹ãĤ¿ ãĥ³ +Ġkr ó +Ġkró tk +ãĤĴ使 ãģĨ +ì ·¨ +éĸ¢ ãĤı +à¸Ķà¹īวย à¸Ħวาม +à¸Ļำ à¹Ģสà¸Ļà¸Ń +Ġa yrıca +à¸Ī à¹īาà¸ĩ +ĠÑĦоÑĤ огÑĢаÑĦ +Ġв еÑĩ +ĠвеÑĩ еÑĢ +åĩº ãģĹãģŁ +ĠÐ¥ о +Ġ×ŀ ר×Ĵ×Ļש +à¹ĥหà¹ī à¹Ģà¸Ľà¹ĩà¸Ļ +ãĤĴ 缮 +ãĤĴ缮 æĮĩ +׾ ×ŀ×Ļ×Ŀ +nÄħ ÅĤ +ĠÑģÑĤ анд +ĠÑģÑĤанд аÑĢÑĤ +ĠSü d +ĠT âm +اخت بار +à¹Ģà¸ģ à¸Ńรà¹Į +Ùħس رØŃ +Ġbi á»ĩn +ب Ùı +Ġص اÙĦ +ĠصاÙĦ ØŃ +ĠPh ụ +íľ ´ +ãĥ¬ãĥĵ ãĥ¥ãĥ¼ +Ġbụ ng +Ġrég ime +ĠØ£ Ø´Ùĩر +ĠÑĢабоÑĤ ник +à¸Ŀ ัà¸Ļ +اع تÙħ +اعتÙħ اد +Ġзам еÑĤ +ãģ¾ ãģ£ãģ¦ +Ġch ặt +æĿ¥ ãĤĭ +ĠاÙĦÙĤ ÙĪات +ãģ«åħ¥ ãģ£ãģ¦ +تØŃ Ø§ÙĦÙģ +Ùħ زÙĬد +ĠÙĬ صÙĦ +ìĹ ¼ +à¹Ģà¸Ĭ à¹ĩ +à¹Ģà¸Ĭà¹ĩ à¸Ħ +Ġk á»ĭ +Ġká»ĭ p +ĠìķĦ ì§ģ +×IJ׳ ×Ĵ +Ġобла ÑģÑĤÑĮ +Ġpomoc Äħ +Ġ×ķ ש׾ +ëĵł ì§Ģ +ĠGi ám +ĠSt ück +Ġchá y +ĠëĤĺ ìĺ¤ +ש ×Ļ×ĺת +×ŀ×ĵ ר +×ŀ×ĵר ×Ļ×ļ +Ġsüre ç +к ва +×ij׾ ×Ļ×Ŀ +×Ķ ת×Ļ +×Ķת×Ļ ×Ļ×Ĺס +ÙĤب اÙĦ +Ġס ×ķ×Ĵ +Ġס×ķ×Ĵ ×Ļ +ÑģÑĤ олÑĮ +ä½ķ ãĤĤ +×ĸ׼ ×ķר +è²· ãģĨ +å®ī ãģı +à¸Ħรัà¹īà¸ĩ à¸Ļีà¹ī +kö p +ĠÑģеÑĢ виÑģ +оÑĩ нÑĭÑħ +ê±° ëŀĺ +تأ Ùĥ +تأÙĥ ÙĬد +×ĵ ׾ק +Ġпо Ñĩем +ĠпоÑĩем Ñĥ +пиÑģ аÑĤÑĮ +×ij שר +ĠH Ãłng +ĠT ìm +Ġtr ừ +ãĤ» ãĥĥãĤ¯ãĤ¹ +×ķ׳ ×Ĵ +mız da +п Ñģи +ĠìŀĪ 기 +Ġr út +ز اÙĨ +تÙĨ ÙĪع +ÙħÙĤ ا +ÙħÙĤا ÙĪÙħØ© +Ġ׾צ ×ķר×ļ +Ġ×ij ×Ļר×ķש׾×Ļ×Ŀ +ãĥ´ ãĤ£ +eb ile +ebile ceÄŁi +ãĥ¦ ãĥ¼ãĤ +ãĥ¦ãĥ¼ãĤ ¶ +ãĥ¦ãĥ¼ãĤ¶ ãĥ¼ +ãĤĴä½ľ ãĤĭ +Ñģ меÑĢ +ÑģмеÑĢ ÑĤ +Ġì§ ģ +Ġì§ģ ìłij +ĠÐŁ аÑĢ +ØŃ Ø§Ø¶ +ØŃاض ر +Ùħ ÙĥاÙģ +ÙħÙĥاÙģ ØŃØ© +ล ิà¸Ļ +ãģ¦ ãģįãģ¦ +ÑĢоÑģ л +ĠÄ°ÅŁ te +ÙĤص ÙĬر +Ġ×ij×Ĵ ×Ļ׾ +Ġ×ŀת ×IJ×Ļ×Ŀ +Ġ×Ķ ×Ĺ×ĵ +Ġ×Ķ×Ĺ×ĵ ש×Ķ +ר ×ķ×¢ +Ġprodukt ów +ĠÙħ صدر +не ÑĨ +ĠاÙĦعÙħÙĦ ات +Ġçık ma +Ġد بÙĬ +ק ×Ļף +ת ×IJר +ת×IJר ×Ļ×ļ +׳×Ļ ×Ļ×ĵ +صر اع +l ève +צ ×Ļר +à¸Ķ ัà¸Ļ +à¹ĥหà¹ī à¹Ħà¸Ķà¹ī +ãĤ¿ãĤ¤ ãĥł +Ġgi ảng +С ÐŁ +ĠاÙĦÙħ ØŃÙĦ +ĠاÙĦÙħØŃÙĦ ÙĬØ© +ĠT ất +׾ ×ķ×ĺ +h á»ķ +Ġam éric +Ġaméric ain +Ġ×ijש׾ ×ij +Ġ׾×IJ ×ķ×ŀ×Ļ +Ġpe ça +ĠÑĢаз нÑĭÑħ +ãģĦãĤĭ ãģ¨ +ãĥĩ ãĥ³ +ס קר +Ġ×Ķ×ŀ×Ĺ ×Ļר +ãģ¨ãģĦãģĨ ãĤĤãģ® +رت بط +ĠиÑģÑĤ оÑĩ +ĠиÑģÑĤоÑĩ ник +สมัà¸Ħร สมาà¸Ĭิà¸ģ +Ġ à¸Ĺัà¹īà¸ĩ +Ġà¸Ĺัà¹īà¸ĩ à¸Ļีà¹ī +ĠT áºŃp +ãģ£ãģ¦ ãģĦãģĨ +ĠاÙĦÙĪ صÙĪÙĦ +Ġdéc ada +Ġо ÑĦоÑĢм +ĠоÑĦоÑĢм лен +สำหรัà¸ļ à¸ģาร +Ġog óln +ãģĨãģ¡ ãģ« +Ġvá rias +ãģĻãģİ ãĤĭ +ÙĪ Ùĩا +à¹Ĥà¸Ľà¸£ à¸Ķ +ĠÐłÐ¾ÑģÑģ иÑı +人 ãĢħ +ãģĹãģ¦ ãģįãģŁ +Ġsı rasında +Ġng ôn +س ÙĨØ© +تÙħ تع +×ŀ׼ ×ij×Ļ +Ġnh ấn +×¢ ×ŀ×Ļ×ĵ +á» ¨ +ж иÑĤÑĮ +ãĤī ãģĽ +gr áf +gráf ica +ĠÙĤ ÙĪÙĦ +ĠÙĤÙĪÙĦ Ùĩ +ëĭ¨ ì²´ +ห à¹īา +หà¹īา ม +使 ãģ£ãģ¦ +ת ×Ļ×ij +ת×Ļ×ij ת +i á»ĥu +à¹ģ à¸Ĭม +à¹ģà¸Ĭม à¸Ľ +à¹ģà¸Ĭà¸¡à¸Ľ à¹Į +Ạ¬ +ĠëĤĺ ëĿ¼ +ĠÙħباشر Ø© +Ġtr Äĥm +سÙĥ ÙĪ +ĠاÙĦØ° Ùī +Ġbi ç +Ġbiç im +ت راجع +Ġоб еÑģп +ĠобеÑģп еÑĩ +ĠобеÑģпеÑĩ ива +Ġвозд ÑĥÑħ +Ñĭв аÑĤÑĮ +ÙĦ ØŃÙĤ +ĠMü dü +ĠMüdü rl +ĠMüdürl Ã¼ÄŁÃ¼ +Ġyapt ır +Ġפר ס +Ġפרס ×ķ×Ŀ +Ø· ÙĪر +ÑģÑĤв оваÑĤÑĮ +ìŀ¥ ìĿĦ +à¸Ĺีà¹Īà¸Ķี à¸Ĺีà¹Īสุà¸Ķ +à¸Ńั ล +ÑĢ Ñİ +Ùħست ÙĤبÙĦ +Ñģл ÑĥÑĪ +ÑģлÑĥÑĪ а +èªį ãĤģ +Ġ׾ ×Ļ×ŀ +Ġ׾×Ļ×ŀ ×ķ×ĵ×Ļ +ת ש×ķ×ij +תש×ķ×ij ×ķת +ĠgerçekleÅŁtir il +ĠاÙĦ اتÙģاÙĤ +ĠÑĥÑĢов не +ĠÑĤ ÑĢав +Ġ×Ķ×ŀ ×ķף +ØŃÙģ اظ +ĠÙħ ÙIJ +ĠÙħÙIJ ÙĨ +ĠÙħÙIJÙĨ ÙĴ +Ġdem ás +×ŀ×ķ×ĸ ×Ļק×Ķ +ש ×Ļ×Ĺ×Ķ +Ġb ú +алÑĮ нÑĭм +ãĤı ãģŁ +ãĤıãģŁ ãģĹ +ĠاÙĦÙħÙĪ اد +ת ׼׳ +×ª×Ľ×ł ×ķף +ãĥŃ ãĥĥãĤ¯ +hi ếu +ĠÑĥ ме +ÙħØŃا ÙĪÙĦØ© +×IJ ×ķשר +Ġкон кÑĥÑĢ +ĠконкÑĥÑĢ Ñģ +Ġ×ŀ ×ij×Ĺ +Ġ×ŀ×ij×Ĺ ×Ļ×ł×ª +Ġan lam +Ġanlam ı +Ġli á»ĩt +Ġв Ñħод +ĠH ình +ĠÙĨ ÙĬ +ĠÙĨÙĬ ÙĪز +ãĤ¸ãĥ£ ãĥ¼ +×ij ×Ļ×¥ +ÑĤелÑĮ нÑĭÑħ +à¸Ĺุà¸ģ à¸Ńยà¹Īาà¸ĩ +ĠkiÅŁ inin +Ø£ Ùĥثر +ĠиÑģÑĤоÑĢ ии +Ġë³Ģ íĻĶ +פ׾ ס×ĺ +×¤×ľ×¡×ĺ ×Ļ׳×Ļ +ĠÑģ еÑĤ +ĠÑģеÑĤ и +dıģ ımız +íķĺ ëıĦë¡Ŀ +×Ķ ר +×Ķר ×ij×Ķ +ãģĻãĤĭãģĵãģ¨ ãģ¯ +Ġphi ếu +تØŃ Ø³ÙĬÙĨ +ĠÅĽ rod +ĠÅĽrod ow +ĠÅĽrodow isk +ĠÑĢаÑģ Ñħод +بر ÙĬد +Ġر ÙĬ +ĠرÙĬ اÙĦ +Ġ×ķ ׼×ļ +ì§Ģ ìļĶ +׼ ×ŀ×ķ +Ġ×¢×ľ ×Ļ×Ķ×Ŀ +f ÃŃcio +Ġkar arı +tıģ ını +ĠС ов +ĠСов еÑĤ +ãģĬéĩij ãĤĴ +м еждÑĥ +междÑĥ на +междÑĥна ÑĢод +междÑĥнаÑĢод н +Ġm á»Ŀi +ĠاÙĦØ¥ ÙĬر +ĠاÙĦØ¥ÙĬر اÙĨÙĬ +ĠاÙĦرÙĪ سÙĬ +ص ÙĨد +صÙĨد ÙĪÙĤ +ĠاÙĦØ¥ÙĨ ترÙĨت +Ġt ắm +ĠÑĤак ого +Ġ×ij ׾×ķ×Ĵ +Ġü crets +Ġücrets iz +×Ĺ×ĸ ×Ļר +ìĸ´ ìķ¼ +ĠPh ần +ï¼ ľ +Ġ×ĺ ×ij×¢ +Ġ×ĺ×ij×¢ ×Ļ +×IJ×ŀ ×IJ +اÙĤ ÙĦ +Ġcondi ções +ÙĤات ÙĦ +ĠÑĢезÑĥлÑĮÑĤаÑĤ е +ĠÑģво ими +צ×ij ×Ļ×¢ +gé ni +Ġz es +Ġzes po +Ġzespo ÅĤ +ÑĪ ив +Ġפר×ĺ×Ļ ×ķת +Ùħست Ø´Ùģ +ÙħستشÙģ Ùī +شر ع +Ġko ÅĽci +Ġ×Ķ×IJ ×Ļ׳×ĺר׳×ĺ +ĠЧ еÑĢ +поÑĩ ÑĤ +Ġactiv ités +çŁ¥ ãģ£ãģ¦ +Ġ×ij ×ĸ×Ķ +Ġyüz den +ãģªãĤĬ ãģ¾ãģĽãĤĵ +Ġíĺ ¹ +Ġíĺ¹ ìĿĢ +Ġ×ŀש ׳×Ķ +ĠÐĴ еÑĢ +Ġ×ij×IJ×ķת ×ķ +éĿ¢ çĻ½ +éĿ¢çĻ½ ãģĦ +شر ØŃ +gr ünde +Ùģ Ø´ +ÙģØ´ ÙĦ +Ġsé jour +ë´ IJ +Ġr ôle +Ø´ عار +ем Ñĭе +ĠاÙĦج سÙħ +алÑĮ ное +Ġìĥģ íĥľ +ï¼ ¤ +ë¯Ģ ë¡ľ +ĠÙĨ ÙĤØ· +ĠÙĨÙĤØ· Ø© +ãģĿãģĨ ãģł +ãģĻãĤĭ ãģ®ãģĮ +ห ู +Ġnh á»ĭ +Ġeconóm ica +ס×ĺ ×ķ×ĵ +ס×ĺ×ķ×ĵ ׳×ĺ +มี à¹Ĥà¸Ńà¸ģาส +Ġgest ão +รูà¹ī วà¹Īา +Ġlo ạt +ĠاÙĦÙħ Ùı +ĠاÙĦØŃ ÙħÙĦ +ĠاÙĦعÙħÙĦ ÙĬØ© +Ġê²ĥ ëıĦ +ĠÐľÐ¾Ñģк ва +ק×ĺ ×ķר +Ġпод ÑĢоб +ĠподÑĢоб н +Ġl Æ°ng +ت Ùģس +تÙģس ÙĬر +ĠاÙĦ بع +ĠاÙĦبع ض +ئ ت +Ðķ ÐĿ +ìĹ° 구 +à¹ĥหà¹ī à¸Ħุà¸ĵ +ãģĤãĤĬ ãģ¾ãģĹãģŁ +Ġbir ka +Ġbirka ç +ĠÄ° sl +ĠÄ°sl am +çĹĽ ãģ¿ +Ġh ảo +Ġм аÑı +ĠiÅŁ çi +ש × +×©× ģ +à¸ģาร à¹Ģมืà¸Ńà¸ĩ +×ķ×Ķ ר +Ġch ó +ëĨ Ģ +Ġyan lı +Ġyanlı ÅŁ +幸 ãģĽ +×IJר×Ĵ ×ķ׳×Ļ +à¸Ńาà¸Ī าร +à¸Ńาà¸Īาร ยà¹Į +ĠинÑĦоÑĢм аÑĨиÑİ +Ðĵ Ðŀ +׳ ×Ĺש +ĠìķĮ ìķĦ +ĠÑħаÑĢакÑĤеÑĢ иÑģÑĤ +ĠÑħаÑĢакÑĤеÑĢиÑģÑĤ ик +à¸Ħุà¸ĵ สามารà¸ĸ +è¦ĭ ãģĪãĤĭ +à¸Ĭัà¸Ķ à¹Ģà¸Ī +à¸Ĭัà¸Ķà¹Ģà¸Ī à¸Ļ +ĠdziaÅĤ al +ĠdziaÅĤal noÅĽci +à¹Ĥà¸ŀ สà¸ķà¹Į +ĠÐļ ол +ĠÙģ ÙĩÙĬ +Ġ×ŀ פ׳×Ļ +Ġ×Ķק שר +Ùħر Ùĥ +ÙħرÙĥ ز +Ġho á +Ġа пп +Ġапп аÑĢаÑĤ +Ġp ami +Ġpami ÄĻ +ĠpamiÄĻ ta +Ġç ünkü +×ĵ ×ķף +ãģ¯ ãģĵãģ¡ãĤī +ĠM Ãł +ĠÙĬ ÙĤدÙħ +ĠпÑĢ ез +ĠпÑĢез иденÑĤ +à¸Ńุ à¸ķ +à¸Ńุà¸ķ สา +à¸Ńุà¸ķสา ห +à¸Ńุà¸ķสาห à¸ģรรม +ì§Ģ ìĽIJ +Ġ×IJפשר ×ķת +sch üt +schüt z +ĠTi ên +Ġsay ılı +ĠгÑĢÑĥпп Ñĭ +оÑĩ нÑĭй +Ġ×ľ×¢ ×ŀ×ķ×ĵ +Ġwr zeÅĽ +ĠwrzeÅĽ nia +ĠÄIJ ầu +à¹Ģà¸Ĥà¹īา รà¹Īวม +nız da +Ø®ÙĬ ص +Ġgü nc +Ġgünc el +ĠÙĦÙĩ Ø°Ùĩ +ĠÙĬ عتبر +lé gi +ãĤı ãģĭãĤĭ +Ġr ừng +ظ Ùĩ +ظÙĩ ÙĪر +Ġ×ŀ×ij ×Ļף +Ġ기 íĥĢ +åĪĩ ãĤĮ +lan mÄ±ÅŁ +à¸Ĺีà¹Ī มีà¸Ħวาม +Ġh á»ģ +ت ÙĪجÙĩ +ĠاÙĦØ¥ دارة +Ġú til +ס פ×ķ +à¸Ħวาม รัà¸ģ +à¹Ĥ ฮ +Ġпол иÑĤ +ĠполиÑĤ ик +Ġsat ın +ĠÅŀ imdi +×ŀ ×ķר×Ļ×Ŀ +ìķĺ ëĭ¤ +×Ĺ ×ķ×ķ +×Ĺ×ķ×ķ ×Ļ×Ķ +à¸Ħà¸Ńม à¸ŀิ +à¸Ħà¸Ńมà¸ŀิ ว +à¸Ħà¸Ńมà¸ŀิว à¹Ģà¸ķà¸Ńรà¹Į +Ġا ذا +تخ اذ +ãĤ¨ ãĥ« +Ġpossibilit é +ยืà¸Ļ ยัà¸Ļ +Ġü nivers +Ġünivers ite +ĠاÙĦد ÙĪرÙĬ +ĠìķĬëĬĶ ëĭ¤ +ĠìĦľ ë¡ľ +ØŃ Ø§ÙĦ +Ġë ¨ +Ġë¨ ¼ +Ġ먼 ìłĢ +à¸Ĺีà¹Ī à¸ĸูà¸ģ +ì§ ľ +Ġsk óry +лÑĮ ÑĨ +à¹ĥà¸Ĭà¹ī à¹Ģวลา +×ijק שת +ĠØ° ÙĪ +æĹ¥ ãĢħ +ĠкоÑĤоÑĢ ÑĥÑİ +ĠÑĥÑĢов енÑĮ +ê¹ ¨ +à¹Ħ à¸Ĺ +ãĤµ ãĥĹãĥª +ãĤ¸ ãĥ§ãĥ³ +ãģĻ ãģ¹ãģį +ĠG ór +ãĥĪ ãĤ¤ +ãĥĪãĤ¤ ãĥ¬ +ĠyaÅŁ ama +Ġdá»ĭ p +Ġb ữa +à¸ĭ ุ +Ġöl üm +ãģ£ãģ¦ ãģıãĤĭ +à¸ģาร à¸Ħà¹īา +ש ער +ĠÑĤип а +Ġг еÑĢ +ĠгеÑĢ о +רק ×¢ +Ġu waż +Ġuważ a +ש×ŀ ף +Ġhast alık +ãĤıãĤĮ ãĤĭ +ba ÅŁÄ± +Ñĩ ÑĤо +Ġ×ij ×ŀר׼×ĸ +Ġìļ°ë¦¬ ìĿĺ +ĠÙĥاÙĨ ÙĪا +ĠØ£ بر +Ġأبر ÙĬÙĦ +ì¸ µ +à¹Ħà¸Ĥ à¹Ī +ĠÙĪ ÙĦÙĪ +à¸Ĺ ัว +à¸Ĺัว รà¹Į +ĠÙĪØ£ Ùĥد +à¸Ĭ วà¸Ļ +׾ ×ķק +æį ¨ +æį¨ ãģ¦ +Ġİç in +p éri +Ġy al +Ġyal nız +ÑĮÑı н +Ġg ắng +à¸ģà¹ĩ ยัà¸ĩ +ĠУкÑĢа ин +ĠÑģ ами +ĠпÑĢовед ен +à¸ķà¸ģ à¹ģà¸ķà¹Īà¸ĩ +ĠQu ân +é paration +ĠbaÅŁ ında +Ġzn ale +Ġznale ź +Ġznaleź Äĩ +ãĤ± ãĥ¼ +ãĥİ ãĥ¼ +à¸ĸูà¸ģ à¸ķà¹īà¸Ńà¸ĩ +ëª ¸ +Ġëı Į +ĠëıĮ ìķĦ +ĠSch üler +Ġпод гоÑĤов +ĠподгоÑĤов к +ع رÙĪ +عرÙĪ ض +la ÅŁtır +ĠÑģоÑģÑĤав лÑıеÑĤ +ĠпÑĢоиз вод +ĠпÑĢоизвод ÑģÑĤва +ĠоÑģнов е +ĠØ´ ÙħاÙĦ +à¸ģร ี +ĠgörÃ¼ÅŁ me +оÑĩ ек +Ġ×Ĺ×ijר ×Ļ×Ŀ +ÙħØ® اط +Ùħخاط ر +ï¼ Ń +ר פ×IJ +ĠM ẹ +ยà¸Ńม รัà¸ļ +Ġv ết +Ø® Ø° +ĠاÙĦت Ø· +ĠاÙĦتط بÙĬÙĤ +à¸Ļ ึà¸ģ +Ġ×Ķ ×Ľ×ł×¡×ª +ĠогÑĢ ани +ĠогÑĢани Ñĩен +ĠÃĩ alÄ±ÅŁ +ĠاÙĦÙħÙĨت دÙī +à¸Īำà¸Ļวà¸Ļ มาà¸ģ +ĠÑĤоÑĢ ÑĢ +ĠÑĤоÑĢÑĢ енÑĤ +ĠìĤ´ ìķĦ +à¸ŀลัà¸ĩ à¸ĩาà¸Ļ +à¸Ĭ ัà¸Ļ +ĠÐIJн дÑĢ +Ġréalis é +×ŀש ×IJ +à¹ģ à¸Ĭ +à¹ģà¸Ĭ รà¹Į +Ġб ог +มา à¹ģลà¹īว +ĠاÙĦÙĨ ار +Ġolmad ıģı +×ĵ ×¢×Ķ +ĠÑĥ веÑĢ +ĠÑĥвеÑĢ ен +ãĤĭ ãĤĤãģ® +Ø£ د +أد ÙĪات +Ġ×Ķ×ĸ ×ķ×Ĵ +Ø¥ عÙĦاÙħ +h á»ı +ĠNä he +ĠÑĤ еÑģÑĤ +Ġ×ŀ ×ķ׼ר +Ġë¬¸ìłľ ê°Ģ +ת ×ķצ×IJ×Ķ +m ó +mó vel +ĠاÙĦتج ارة +Ġмног иÑħ +обÑī а +Ġ×¢ סק×Ļ +ĠEdu cação +ק ש×Ļ×Ŀ +é tabl +établ issement +Ġд еле +иÑĢÑĥ еÑĤÑģÑı +Ø¢ ثار +Ġ×Ķ×ŀ ר׼×ĸ×Ļ +ãĥIJ ãĥ« +ĠвÑģÑĤÑĢ еÑĩ +ãģĴ ãĤĭ +Ġci Äħ +ĠciÄħ gu +ÙĬ ست +à¸łà¸² ว +à¸łà¸²à¸§ ะ +Ø£ Ùħر +Ġо жи +Ġожи да +Ġ ủy +ãĥŀ ãĥ« +ر اس +оÑĩ ной +ת ×Ĵ×ķ×ij×ķת +تع رÙĬÙģ +ĠÑģо ÑĨиалÑĮно +ãĤĴ éĸĭ +ĠиÑģÑģлед ова +Ġd ú +Ġdú vida +Ġsk ÅĤ +ĠskÅĤ ada +Ġhä ufig +ĠвÑĭб ÑĢ +ĠвÑĭбÑĢ аÑĤÑĮ +ãģ®ãģ§ãģ¯ãģªãģĦ ãģĭ +ĠÑģ илÑĮно +ÑĤвеÑĢж ден +ר פ +רפ ×ķ×IJ×Ķ +æĢĿ ãģĦãģ¾ãģĻ +ØŃر ص +ש×ķת ×£ +Ùħس جد +à¹Ĥà¸Ĭ วà¹Į +ем ÑģÑı +в ÑĪие +Ġм л +Ġмл н +Ġ׾×Ķ ×ij×Ļ×IJ +ĠÙĬ تعÙĦÙĤ +à¸ķ ูà¹ī +Ġп ÑĢаз +ĠпÑĢаз д +ĠпÑĢазд ник +Ġн ем +Ġнем ного +Ġs Ãłng +تÙĨ سÙĬ +تÙĨسÙĬ ÙĤ +Ġtá» Ŀ +Ġмед и +ãģ« æĪ +ãģ«æĪ » +à¸Ħว à¹īา +ãģĭ ãģijãĤĭ +×ij׾ ×ķת +ĠÑįк Ñģп +ĠÑįкÑģп еÑĢÑĤ +Ġдев ÑĥÑĪ +ĠдевÑĥÑĪ к +ĠØŃ Øµ +ÙĨØ´ Ø£ +ãģĮãģĤãĤĭ ãģ®ãģ§ +Ġت راÙħ +ĠتراÙħ ب +أس ÙĪاÙĤ +Ġ׾פ ׳×ķת +Ġا ï»· +ãģ« ãģı +ãģ«ãģı ãģĦ +ĠØ£ عÙĦÙī +Ġ׾×Ķ ×ŀש×Ļ×ļ +rä u +ש×ŀ ×Ļ×Ŀ +åĪĨ ãģij +ãģĻ ãģ§ +ãģĻãģ§ ãģ« +×Ķ׾ ׼×Ķ +×Ĺ׾ ×Ļ×£ +Ġì ±ħ +Ġì±ħ ìŀĦ +à¹Ģà¸Ī ริ +à¹Ģà¸Īริ à¸į +éģĬ ãģ³ +ج سد +สา à¸ĺ +สาà¸ĺ าร +สาà¸ĺาร à¸ĵ +Ġbas ın +ÑĢаР³ +г ад +Ġho ÅŁ +íķ µ +×ij×Ĺ ×Ļר×Ķ +×ŀס ×ļ +Ġìłľ íĴĪ +تÙħ ÙĪÙĬÙĦ +ĠL Æ°u +ë¡ľ ë¶ĢíĦ° +Ġп об +Ġпоб ед +ÙħÙĨ Ø° +常 ãģ« +ÙĤ س +ĠاÙĦÙħ صدر +ĠÙĪاÙĦ است +Ġkh ắp +ĠاÙĦج اÙĨب +Ġng uyá»ĩn +éĸĵ éģķãģĦ +ĠÑģÑĤ ÑĢа +ĠÑģÑĤÑĢа Ñħ +ĠÑģÑĤÑĢаÑħ ов +รี à¸ļ +Ġx Æ°Æ¡ng +Ġì° ¾ +Ġì°¾ ìķĦ +Ġng ại +г ал +à¸ĭ ีà¹Ī +Ġ×ij פ×Ļ×Ļס×ij×ķק +Ц енÑĤÑĢ +Ġaval iação +Ġeconóm ico +×ĸ ף +ĠÐľ ак +Ġinter és +à¸ģล ิà¹Īà¸Ļ +ÑģÑĤÑĮ Ñİ +ĠÄij Æ°Æ¡ng +å¼· ãģı +ĠKh ách +à¹Ģà¸Ļืà¹īà¸Ń หา +ĠYaz ı +è²· ãģ£ãģ¦ +Ðł Ðķ +à¹Ģà¸ŀิà¹Īม à¸Ĥึà¹īà¸Ļ +สม à¸ļู +สมà¸ļู รà¸ĵà¹Į +Ġм иÑĢов +×Ĵ ׳×Ļ×Ŀ +ĠÄij ức +à¸Ń ารà¹Į +ص اص +ãģĬ ãĤĪ +ãģĬãĤĪ ãģ³ +ÃªÌ ī +ĠاÙĦÙħؤ تÙħر +ĠاÙĦÙħر ØŃÙĦØ© +สà¸Ńà¸ļ à¸ĸาม +Ġà¸Īาà¸ģ à¸Ļัà¹īà¸Ļ +Ġت عد +ãģĿãģ® ãģŁãĤģ +Ġkh áng +à¸Ļ ิà¸Ķ +ãĥĬ ãĥ³ +ëĦ¤ ìļĶ +ĠاÙĦ اØŃت +ĠاÙĦاØŃت ÙĦاÙĦ +ìļ ķ +Ġмод ели +ĠпÑĢоÑĨ енÑĤ +à¸ŀวà¸ģ à¹Ģรา +Ġ×Ķצ ×ĵ +Ġ×Ķצ×ĵ ×ĵ×Ļ×Ŀ +ständ e +׳ ×Ĵר +Ġdot yc +Ġdotyc zÄħ +ĠdotyczÄħ ce +ĠÅĽ wiÄĻt +×ŀר ×Ķ +ãģĻãģĶ ãģĦ +ãĥĩãĤ£ ãĥ³ãĤ° +à¸ģาร สรà¹īาà¸ĩ +ë Ĥ¬ +Ġì°¸ ìŬ +Ñģ Ñħ +ÑģÑħ ем +ÙħÙĪ س +Ġn ấu +Ġ׾×ŀ×¢ ׾×Ķ +à¹Ģà¸Ľ à¹īา +à¹Ģà¸Ľà¹īา หมาย +Ġmù i +ائ ز +íĽ Ī +×Ĺ×ij ×ķר×Ķ +à¸ľà¸¹à¹ī à¹ĥà¸Ĭà¹ī +Ġpa ź +Ġpaź dzi +Ġpaździ ern +Ġpaździern ika +ลà¸ĩ à¹Ħà¸Ľ +ÙĤ اع +Ġch áºŃm +Ġözellik leri +ĠÄIJ o +ĠÄIJo Ãłn +ж ение +Ġh ẳ +Ġhẳ n +ĠaÅŁ k +ï½ į +ãĥij ãĤ¹ +×Ķ×ķר ×IJ×ķת +ĠÅ » +ĠÅ» y +×ŀ×ĸ ׾ +ĠÑĥ кÑĢа +ĠÑĥкÑĢа ин +à¹Ģà¸Ĭ ิ +à¹Ģà¸Ĭิ à¸į +Ðł Ðĺ +ĠzwiÄħz ku +×Ķ×Ĺ׾×ĺ ת +ãĤĵãģ§ãģĻ ãĤĪãģŃ +ãģ¦ ãģĬãĤĬ +лож иÑĤÑĮ +×ŀ ×ķ׳×Ļ×Ŀ +ฮ ิ +ì° ¬ +ĠاÙĦÙħØ´ ترÙĥ +ĠdÃ¼ÅŁ ük +аг енÑĤ +ĠاÙĦØ£ سبÙĪع +ĠÙĤ رÙĬب +ин д +инд ив +индив ид +индивид Ñĥ +индивидÑĥ алÑĮн +för der +Ġseç en +Ġseçen ek +Ġét ant +ĠлÑİб им +каз ÑĭваеÑĤ +ว ิà¸Ļ +Ġ×Ķ×ij ×IJ×Ļ×Ŀ +Ġд ов +Ġдов олÑĮ +ĠдоволÑĮ но +×¢×ĵ ×Ļ×£ +Ġok re +Ġokre ÅĽ +ĠokreÅĽ lon +Ġت رÙĬد +à¹Ģมืà¹Īà¸Ń วัà¸Ļà¸Ĺีà¹Ī +ãĤĪ ãģĭãģ£ãģŁ +Cum h +Cumh ur +Cumhur ba +Cumhurba ÅŁ +CumhurbaÅŁ kan +CumhurbaÅŁkan ı +Ġn ợ +à¸ľà¸¹à¹ī à¹Ģลà¹Īà¸Ļ +Ġcompl ète +à¹Ģà¸ŀ ศ +د ÙIJ +Ġdü z +Ġdüz ey +ãģ§ãģĤãĤĭ ãģĵãģ¨ +ext érieur +× ³ +Ġinform ação +ãĤ¯ãĥª ãĥĭãĥĥãĤ¯ +ĠPub li +ĠPubli é +ר ×ķ×ĵ +à¸Ħวาม à¸Ľà¸¥à¸Ńà¸Ķà¸łà¸±à¸¢ +ĠØ£ÙĬ ض +ĠØ£ÙĬض Ùĭا +ت سبب +ãģ¤ ãĤĤãĤĬ +из ма +à¸Ĥึà¹īà¸Ļ à¹Ħà¸Ľ +Ùĥ ÙIJ +ÙĦ ÙĪÙħ +Ġש צר +Ġשצר ×Ļ×ļ +ãģ¯ ãĤĤãģ¡ãĤįãĤĵ +Ġк ан +Ġкан ал +ãģ«ãģª ãģ£ãģ¦ãģĦãģ¾ãģĻ +ĠاÙĦØ£ Ùĥثر +ت اØŃ +ÙĨت Ùĩ +ÙĨتÙĩ اء +ا ÙĪÙĬØ© +ĠBug ün +н Ñģкого +à¸Ķ à¹Īวà¸Ļ +é volution +ãģ£ãģ¦ ãģĦãģ¾ãģĹãģŁ +ãĤ ħ +ĠV Æ°Æ¡ng +à¸łà¸²à¸ŀ ย +à¸łà¸²à¸ŀย à¸Ļ +à¸łà¸²à¸ŀยà¸Ļ à¸ķรà¹Į +Ġ×Ķ צ׾×Ļ×Ĺ +ĠاÙĦإسÙĦاÙħ ÙĬ +ÙĦÙĬ ب +Ġed ição +ÑģÑĤÑĢ ел +Ġkh úc +ÙĨÙħÙĪ Ø° +ÙĨÙħÙĪØ° ج +׾ צ×Ķ +ÑģÑĤав ил +à¸ĸ า +สรà¹īาà¸ĩ à¸Ħวาม +ãģĦ ãģ£ãģ± +ãģĦãģ£ãģ± ãģĦ +ÑģÑĤав лен +ĠاÙĦ ÙĤدس +Ġng ược +ب Ø® +ส หร +สหร ั +สหรั à¸IJ +ĠØ£ غ +Ġأغ سط +Ġأغسط س +ãģĨ ãģ¾ +ãģĨãģ¾ ãģı +ĠêµŃ ìłľ +ØŃض ار +Ġd ừng +æĬ¼ ãģĹ +ت ÙĪا +تÙĪا جد +ש×ŀ ×Ĺ×Ķ +ãģı ãĤĵ +Ġ×ij×¢ צ +Ġ×ijעצ ×Ŀ +×ŀ ׳×Ļ×ķת +×ķ ×Ļ×ĵ +×ķ×Ļ×ĵ ×IJ×ķ +à¸Ĭ ิà¸ĩ +Ġprac ÄĻ +Ġз аÑĤ +ĠзаÑĤ ем +ĠìŀIJ ìľł +Ġì¤ Ģ +Ġì¤Ģ ë¹Ħ +Ġb áºŃ +ĠbáºŃ c +Ġ×Ķ×ŀ צ×ij +ĠÙĤ ÙĬÙħØ© +à¹Ģà¸Ń à¹Ģà¸Ĭ +à¹Ģà¸Ńà¹Ģà¸Ĭ ีย +Ġperch è +ĠاÙĦع سÙĥر +ĠاÙĦعسÙĥر ÙĬØ© +ج ÙĬب +ëŀ µ +Ùħ Ùĩر +ÙħÙĩر جاÙĨ +Ùħ راÙĥ +ÙħراÙĥ ز +Ġод нако +à¸Ķี à¹Ĩ +Ġצ פ×ķ +Ġkullan ılan +Ġк ино +ãĥĨãĤ£ ãĥ³ãĤ° +ĠGi Ỽi +ت ÙĪز +تÙĪز ÙĬع +ย ิà¸Ļ +ยิà¸Ļ à¸Ķี +Ġc Åĵur +ĠiÅŁ aret +Ġ×ij×¢ ×ĸר +Ġ×ij×¢×ĸר ת +Ġп аÑĨи +ĠпаÑĨи енÑĤ +ãģ¿ãģŁãģĦ ãģ§ãģĻ +в ез +ли на +од е +Ġ×IJ×ķת ף +dıģ ınız +ĠÐIJ в +ĠÐIJв ÑĤоÑĢ +ï¼ ® +ĠC ần +ĠاÙĦا Ø® +ĠاÙĦاخ بار +Ġê±° ìĿĺ +Ġat enção +Ġgeld iÄŁi +ãĤª ãĤ¹ +ãĤªãĤ¹ ãĤ¹ +ãĤªãĤ¹ãĤ¹ ãĥ¡ +ев Ñĭе +кÑĢÑĭ л +à¹Ģà¸Ĭ ียà¸ĩ +à¹Ģà¸Ĭียà¸ĩ à¹ĥหมà¹Ī +Ġmar ço +ĠاÙĦÙħ ادة +Ġг ол +Ġsprzeda ży +Ġíķ´ ê²° +ĠÐķ го +ê¹ Ģ +Ġ׾ק×ij׾ ת +ĠاÙĦÙģ ÙĨاÙĨ +Ġcomunic ación +à¹Ģสà¹īà¸Ļ à¸Ĺาà¸ĩ +íĺ ¹ +à¸Ĭ ำ +à¸Ĭำ ระ +Ġ׼ ×IJ×ŀ +Ġ׼×IJ×ŀ ×ķר +à¸Ĭ à¹Īาà¸ĩ +ز Ùĩر +Ġklient ów +ива ÑİÑĤ +ан г +׳ ×ļ +Ġg á»įn +Ãľ R +ìĺģ ìĥģ +Ġغ زة +ìĿĮ ìĿĦ +Ġbez po +Ġbezpo ÅĽ +ĠbezpoÅĽ redni +ĠاÙĦÙħ ÙĪا +ĠاÙĦÙħÙĪا Ø·ÙĨ +ĠاÙĦÙħÙĪاطÙĨ ÙĬÙĨ +ãĤĮ ãģ¾ãģĻ +ĠмаÑĤ Ñĩ +×IJ ×ķף +Ġر سÙħÙĬ +ĠÑįк он +ĠÑįкон ом +ĠÑįконом иÑĩеÑģк +ãĥľ ãĥ¼ +Ġд иÑĢ +ĠдиÑĢ екÑĤоÑĢ +ĠÑģк оÑĢо +à¸ļ ำ +à¸ļำ ร +à¸ļำร ุà¸ĩ +ĠÑĦ ÑĥÑĤ +ĠÑĦÑĥÑĤ бол +Ġ×IJ ×Ļ׾ +Ġì¤ij êµŃ +ìľ ¤ +eÄŁ e +à¹Ħ à¸ģà¹Ī +tra î +traî n +ĠÑĤ ÑĢÑĥб +à¹Ģà¸ļ ื +à¹Ģà¸ļื à¹īà¸Ńà¸ĩ +à¹ģม à¸Ļ +ĠتØŃ Ø¯ÙĬØ« +Ġ׼ עת +ØŃ Ø§Ø³Ø¨ +lı ÄŁa +ק×Ļ ×Ļ×ŀ×Ļ×Ŀ +оÑģÑĤ ÑĮÑİ +à¸Ŀ ั +à¸Ŀั à¹Īà¸ĩ +Ø´ غÙĦ +ìĽ ¹ +Ġкажд ого +Ġbölüm ü +หà¸Ļ ี +Ġistedi ÄŁi +Ġtr Æ°ng +ãĥ Į +ฮ à¸Ń +Ø£ÙĨ Ø´ +Ø£ÙĨØ´ طة +ĠاÙĦÙħ سÙĬ +ĠاÙĦÙħسÙĬ ØŃ +ลัà¸ģษ à¸ĵà¹Į +Ġn á»Ńa +à¸Ĺีà¹Ī à¸ķà¹īà¸Ńà¸ĩà¸ģาร +ÑĪ ек +л Ñij +Ġש ×Ļ×Ķ +Ġש×Ļ×Ķ ×Ļ×Ķ +Ġkhu ôn +ĠÑĤÑĢеб ованиÑı +Ġ×ľ×¢ ×ĸ×ķר +ĠاÙĦع Ùħر +ราà¸Ħา à¸ĸูà¸ģ +ÙĩÙı ÙħÙĴ +ü st +üst ü +Ġден ег +Ġn ạ +à¸Ĥà¸Ļ ม +Ġбл аг +Ġблаг од +Ġблагод аÑĢ +ĠблагодаÑĢ Ñı +Ø¥ سÙĦاÙħ +à¸Ļิ ว +çŁ¥ ãĤīãģªãģĦ +Ø« ÙĤØ© +Ġг олоÑģ +×IJ×ķר ×Ĺ +Ġtr ứng +Ġод ном +ĠkoÅĦ cu +Ġ×ķ רק +Wi ÄĻ +WiÄĻ cej +Ġ×IJ ×Ļ׼×ķת +Ġ×IJ×Ļ׼×ķת ×Ļ +Ñģ оÑģ +Ġje żeli +以ä¸ĭ ãģ® +å°ı ãģķ +å°ıãģķ ãģª +олог ии +Ġоб ÑģлÑĥж +ĠобÑģлÑĥж ива +Ùĥت ابة +Ġê´Ģ ìĭ¬ +×¢ ש×Ļר +Ġaras ındaki +ĠÑĢай она +ÙĪا جب +Ġ×ij×Ĺ×Ļ ×Ļ +íķ´ 주 +Ġg óc +ай л +ĠT ình +æļ® ãĤī +æļ®ãĤī ãģĹ +æĻĤ ãģ«ãģ¯ +ĠгоÑĢод е +Ġ׼×IJ ×Ļ׾ +Ġ׼×IJ×Ļ׾ ×ķ +ĠC á»Ļng +ãģ©ãģĨ ãģĹãģ¦ãĤĤ +×Ĺ ×ķ×£ +تØŃ Ø±Ùĥ +ĠÑģлов ам +à¸Īะ à¸Ĭà¹Īวย +ĠاÙĦÙħست ÙĤبÙĦ +ÙĤ ض +ÙĤض ÙĬ +×ijס ×ķפ +×ijס×ķפ ×ķ +iÄĻ Äĩ +ĠY ıl +Ø´ ÙĬØ® +à¸Ħุà¸ĵ à¸Īะ +ש×ŀ ×ķת +Ġت عرض +Ġanál ise +ĠÑģоб иÑĢа +à¹Ģà¸ŀ à¸Ĭ +à¹Ģà¸ŀà¸Ĭ ร +Ġв ели +Ġвели к +สั à¹īà¸Ļ +Ġpop ulação +รà¹Īวม à¸ģัà¸Ļ +×Ĺ ×ŀ +×Ĺ×ŀ ×Ļש×Ļ +ס ×Ļס +åĨħ ãģ§ +Ġsob Äħ +ĠY ay +ĠYay ın +ãĥ¡ ãĥĭãĥ¥ãĥ¼ +ĠпÑĢедоÑģÑĤав лÑı +ãģł ãģ¨æĢĿãģĨ +Ġê³ł ê°Ŀ +Ġод ним +à¹ĥà¸Ļ à¹Ģรืà¹Īà¸Ńà¸ĩ +Ġs á»ķ +ĠÐĹ Ð´ÐµÑģÑĮ +Ġизмен ениÑı +ĠìĿ¼ ìĿĦ +ãģªãģ® ãģł +клад Ñĭва +ÑĢ ма +Ġ×ķ×ij ׼׾ +تأ ÙħÙĬÙĨ +ĠпÑĢи ÑıÑĤ +ĠпÑĢиÑıÑĤ н +Ùħ Ùħار +ÙħÙħار سة +ãģ¨ãģª ãģ£ãģ¦ +Ġج ÙħÙĬÙĦ +Ġì§ Ī +Ġì§Ī 문 +Ġquest ão +i é +ié ndo +หà¹īà¸Ńà¸ĩ à¸ŀัà¸ģ +ãĥij ãĥ¼ãĥĪ +ÑĤвеÑĢж да +н Ñģкой +з ал +มุ à¹Īà¸ĩ +á» Ĭ +Ġ×Ķ×IJ×Ĺר ×ķ׳×Ķ +ĠTh Æ° +주 민 +ĠاÙĦع ب +év én +évén ement +ÙĤÙĪ اعد +د Ùı +ĠìķĬ ìĬµëĭĪëĭ¤ +Ġë³´ 기 +Ġyapıl ması +à¹Ģร าà¸ģ +à¹Ģราà¸ģ à¹ĩ +ØŃ Ø°Ø± +ÙĤ صر +ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĹãģŁ +Ġà¹Ģà¸Ľà¹ĩà¸Ļ à¸ķà¹īà¸Ļ +ãģ¨ ãģ« +ãģ¨ãģ« ãģĭ +ãģ¨ãģ«ãģĭ ãģı +н ÑĨе +зв Ñĥк +ãģĹãĤĪãģĨ ãģ¨ +ĠاÙĦصØŃ ÙĬØ© +Ġש×Ķ ×Ļ×ķ +ĠDi ÄŁer +ÙĤÙĦ ÙĤ +ãĤ¸ãĥ£ ãĥ³ +Ġr á»Ŀi +Ġл еÑĩ +ĠлеÑĩ ениÑı +تب اد +تباد ÙĦ +צ פ×Ķ +à¸Ħวาม à¹Ģหà¹ĩà¸Ļ +ĠØ´ ب +Ġشب ÙĥØ© +ר ×Ļק +Ùħ عد +Ùħعد ات +dıģ ında +Ġ×ijש ׳×Ļ×Ŀ +Ġ×Ķ ×Ļשר×IJ׾ +Ġ×Ķ×Ļשר×IJ׾ ×Ļת +Ġsı nav +׳צ ×Ļ×Ĵ +วัà¸ķ à¸ĸุ +ĠاÙĦبر ÙĦÙħ +ĠاÙĦبرÙĦÙħ اÙĨ +t ivitÃł +ãĤĵãģł ãĤįãģĨ +ק×Ļ ×Ļ×ŀ +ÙĦÙĬ Ùĥ +ĠÄij ò +ĠÄijò i +ĠÐĺн ÑĤеÑĢ +ĠÐĺнÑĤеÑĢ неÑĤ +ãģ«ãģ¨ãģ£ãģ¦ ãģ¯ +ãģ£ ãģĵ +ק ×ķס +ست ØŃÙĤ +æķĻ ãģĪãģ¦ +ãĥĢ ãĥ¡ +ĠÙħÙĨ زÙĦ +à¹Ģà¸ĭ à¹ĩà¸Ļ +使 ãģĪãĤĭ +è¦ĭ ç©į +è¦ĭç©į ãĤĤãĤĬ +Ø£ Ùģ +Ø£Ùģ Ùĥار +Ġиг ÑĢов +ĠигÑĢов Ñĭе +Ġm ÄĻż +ĠmÄĻż czy +ĠmÄĻżczy zn +ĠاÙĦØŃ ÙĤÙĬÙĤÙĬ +ع بر +׼×ķ׾ ׳×ķ +íĿ ¥ +×ŀ×IJ ×ķ×Ĺר +خت ص +ãĥŀ ãĥŀ +Ġ×IJ×Ĺ ×ķ×ĸ +í ĮĢ +Ġr á»iji +Ġв ÑĤоÑĢ +ĠвÑĤоÑĢ ой +Ġl ẫn +пÑĢ ом +пÑĢом ÑĭÑĪ +пÑĢомÑĭÑĪ лен +пÑĢомÑĭÑĪлен н +ĠоÑĤноÑĪ ениÑı +Ġs ứ +Ġм обилÑĮ +ĠмобилÑĮ н +ĠÑįÑĤ омÑĥ +Ġt ạp +ĠìĤ¬ ê±´ +ĠìķĮ 볤 +Ùĥ Ùı +ÙĥÙı ÙħÙĴ +Ġק ×ķר×Ķ +ĠÑĦ иÑĢ +ĠÑĦиÑĢ м +Ġsık ıntı +׳ ׼ +׳׼ ×ķף +ÙĪÙĦÙĪج ÙĬ +ØŃ Ø§ÙĨ +Ġlo ạn +Ġ×IJ׾ ×£ +Ġm ắn +abh äng +abhäng ig +ĠÑĥÑĢов нÑı +Ġ׾×ij×ĵ ×ķק +ÙĬ ÙħÙĨ +lay ın +Ġh ải +Ġзав од +ĠìķĦ 주 +สà¸ĸ า +สà¸ĸา à¸ļัà¸Ļ +Ġgüven lik +à¹Ģà¸Ķ à¹Īà¸Ļ +×ij×ĵ ק +Ġë Ī +ĠëĪ Ħ +ĠëĪĦ 구 +éĩįè¦ģ ãģª +รà¸Ńà¸ĩ รัà¸ļ +sch lie +schlie ÃŁen +Ġìĸ ¼ +Ġìĸ¼ ë§Ī +Ġìĸ¼ë§Ī ëĤĺ +ÑĤи ки +íķľëĭ¤ ê³ł +ãģłãģ£ãģŁ ãĤī +Ġ×Ķ ×Ļ×ĺ×ij +ãģªãģijãĤĮãģ° ãģªãĤīãģªãģĦ +â Ì +Ã¢Ì £ +Ġph ạt +ak Ä±ÅŁ +ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĻ +à¹Ģà¸ĭ à¹ĩ +ĠС егоднÑı +Ġinsan ların +Ġdévelop pe +ת פר +תפר ×Ļ×ĺ +اÙĨت شار +ê° ij +Fran çois +Ø£ÙĦ ع +Ø£ÙĦع اب +ãĤĴ è¶ħ +ãĤĴè¶ħ ãģĪ +Ġê°Ļ ìĬµëĭĪëĭ¤ +ãĤ³ ãĥ¬ +ĠмеÑģÑı ÑĨев +íĮ ħ +ĠاÙĦج اÙħعة +ìĿ¸ íĦ° +ìĿ¸íĦ° ëĦ· +×ĵר ×ķש +ĠÙĪØ£ شار +ĠпÑĢав ила +ãģĿãģĵ ãģ« +×Ĺ ×ŀ×ĵ +à¹Ģหà¸ķุ à¸ģารà¸ĵà¹Į +Ġê²½ íĹĺ +ãģ¶ ãĤĬ +׾ ש +׾ש ×ķף +à¹Ģ à¸ĸ +ĠDo ÄŁu +ĠиÑģполÑĮзов ание +Ġçoc uÄŁu +магазин е +ĠÄiji á»ĥn +Ġas lı +Ġaslı nda +Ġdoen ça +Ġس اع +Ġساع ات +ĠиÑģполÑĮзов аниÑı +ר ×ķצ×Ļ×Ŀ +ĠзнаÑĩ иÑĤ +ĠÑĢаР¼ +ĠÑĢам каÑħ +ê±° 리 +Ġп ÑĭÑĤа +ãĥģ ãĥ³ +Ġпо Ñģк +ĠпоÑģк олÑĮ +ĠпоÑģколÑĮ кÑĥ +Ø¥ بر +إبر اÙĩ +إبراÙĩ ÙĬÙħ +ĠÑĤÑĢ еÑħ +ĠGen ç +س ÙĪÙģ +Ġve ÃŃculo +ĠNg ân +ĠоÑĩеÑĢ едÑĮ +à¸Ħร ึà¹Īà¸ĩ +×IJ ×ij×Ļ +à¸ķ à¹īม +ãĤĴè¡Į ãģĦ +ĠاÙĦساب ÙĤØ© +на ÑĨи +наÑĨи она +наÑĨиона лÑĮн +Ġgest ión +ت ÙĤد +ĠاÙĦبÙĬ اÙĨ +ĠاÙĦبÙĬاÙĨ ات +ĠاÙĦ اÙĨتخاب +ĠاÙĦاÙĨتخاب ات +à¹Ģà¸Ĭ à¹Īา +×ĵ ×IJ×Ĵ +Ġ׾×Ĵ ×ŀר×Ļ +Ġت ØŃتاج +Ġth ôn +à¸ķ à¹īà¸Ńà¸Ļ +à¸ķà¹īà¸Ńà¸Ļ รัà¸ļ +女 ãģ® +女ãģ® åŃIJ +Ġth ợ +Ø· ØŃÙĨ +ารà¹Į à¸Ķ +ת ×ŀ×Ļ×ĵ +ĠÑģам Ñĭм +Ġìĭľ íĸī +Ø¥ صد +إصد ار +ĠNgh á»ĩ +ìķ ķ +س ئ +سئ ÙĦ +à¸Ń าร +à¸Ńาร ม +à¸Ńารม à¸ĵà¹Į +à¹ģ ฮ +׳×ĺ ׾ +Ġì¢ĭ ìķĦ +×ķ׾ ׾ +Ġ×ij ×Ľ×ª×ij +ãĤ« ãĥ© +צע ×Ļר×Ļ×Ŀ +تعب ÙĬر +Ġ×ŀ קר×Ķ +ĠÑĦак ÑĤоÑĢ +Ġت ÙħاÙħ +ĠتÙħاÙħ ا +ëį ķ +Ġv Æ°á»Ŀ +ĠvÆ°á»Ŀ n +Ġd Ä±ÅŁÄ± +ãģĦ ãģ¡ +Ġ׾ק ׳×ķת +ĠاÙĦع ÙĦاÙĤات +п Ñĥб +пÑĥб ли +Ø¥ ÙĬÙħ +Ø¥ÙĬÙħ اÙĨ +à¸Ńำ à¸Ļา +à¸Ńำà¸Ļา à¸Ī +åIJ« ãģ¾ãĤĮ +ãĤĭ ãģŁãĤģãģ« +ס ×Ĵ +ס×Ĵ ׳×ķף +تØŃ Ø¯ÙĬ +Ġaup rès +ĠاÙĦج Ùĩا +ĠاÙĦجÙĩا ز +Ġ×ŀ ת×Ĺת +ен нÑĥÑİ +Ġз им +à¸ģา à¹ģà¸Ł +Ġ×ijת ×ķר +Ġngh è +Ġnghè o +ĠÐĽ Ñİ +ĠÐĽÑİ Ð± +תק צ×Ļ×ij +×ŀ×¢ ש×Ķ +ĠاÙĦبÙĬ ت +צ ×Ļפ +ĠобÑıз ан +ĠM á»Ĺi +ĠТ ÑĥÑĢ +ĠÙĪب اÙĦت +ĠÙĪباÙĦت اÙĦÙĬ +Ġdéc ision +Ġب د +Ġبد أت +Ġc ục +Ġb ask +Ġbask ı +Ġhat ırl +Ġhatırl a +å°ı ãģķãģĦ +Ġgerçek ten +à¸ľ ัà¸ģ +åı¯èĥ½ ãģª +×ŀ×IJ ס +Ġcr ÃŃtica +ĠìĿĺ ìĽIJ +عÙĤ ÙĪد +×ĺ ׼׳ +×ĺ׼׳ ×ķ׾×ķ×Ĵ×Ļ×Ķ +è¨Ģ ãģĪãģ° +ĠÙĤ ÙĨا +ĠÙĤÙĨا Ø© +ĠìĿ´ê²ĥ ìĿĢ +ت صر +à¸Ł ัà¸Ļ +ĠÑĢе ÑĨеп +ĠÑĢеÑĨеп ÑĤ +ĠبÙĨ Ùģس +ÑĢо ÑĪ +ĠмаÑĢ ÑĤа +Ġson ras +Ġsonras ı +×ķ×ij ש +ãĥª ãĤ¹ãĤ¯ +ĠFranç ais +á» ļ +ê° Ķ +Ġ×Ķ×ijר ×Ļת +פ ×Ļצ +פ×Ļצ ×ķ×Ļ +ĠÙĦÙħا ذا +ĠÐļи ев +ĠÑģ мÑĭÑģл +ê¸Ī ìľµ +ãĤ·ãĥ£ ãĥ« +ãĥ© ãĤ¤ãĥĪ +ìĽ ĥ +×ŀ ×Ĺר +ãĨ į +Ġkullan ım +Ġ×IJצ׾ ׳×ķ +Ġt Ãłn +ãĥı ãĥ¼ +ãģ¨ ãģ¨ãĤĤ +ãģ¨ãģ¨ãĤĤ ãģ« +ÑĢ ег +ÑĢег и +ÑĢеги он +ãģªãģı ãģªãĤĭ +Ġch ảy +Ġج ÙĩØ© +ÅĦsk iej +à¸Ńี à¹Ģม +à¸Ńีà¹Ģม ล +ãģį ãģ£ãģ¨ +ĠìĺĪ ìĤ° +Ġkit abı +Ġedu cação +Ġbul uÅŁ +олог иÑı +Ġкон кÑĢ +ĠконкÑĢ еÑĤ +×Ĵ ×Ļר +ĠпÑĢед лаг +ĠпÑĢедлаг аеÑĤ +ĠY ên +Ġíķľ ë²Ī +Ġ×ŀ ר׼×ĸ×Ļ +à¹Ģà¸Ľà¸´à¸Ķ à¹Ģà¸ľà¸¢ +ÑĤвеÑĢ д +ĠH á»ĩ +ĠÐĵ ÑĢ +à¸Ŀ à¹īา +×Ķ שק +×Ķשק ×¢×Ķ +Ġна Ñĥк +ìłIJ ìĿĦ +Ġн елÑĮ +ĠнелÑĮ з +ĠнелÑĮз Ñı +г ин +ĠB öl +ĠBöl ge +Ġв ла +Ġвла ÑģÑĤи +à¹Ģà¸Ļ à¹ĩ +à¹Ģà¸Ļà¹ĩ à¸ķ +ê³ ¨ +Ġö ld +Ġöld ür +׼׳ ×¢ +ĠاÙĦÙĩ ÙĬئة +ت ارÙĬØ® +ĠÐij ÑĢ +ĠÑģ мож +ĠÑģмож еÑĤе +ĠL úc +à¹Ħà¸Ľ à¸ĸึà¸ĩ +ĠBakan ı +Ġerklä rt +ĠÐIJ на +Ġsc ène +åķı ãģĦ +åķıãģĦ åIJĪãĤıãģĽ +ÙħÙĩ ÙĨد +ÙħÙĩÙĨد س +Ġн азвание +ив аниÑı +ãĤĴ å¤īãģĪ +ä»ĺãģį åIJĪ +ãĥij ãĤ½ +ãĥijãĤ½ ãĤ³ãĥ³ +æĺİ ãĤī +æĺİãĤī ãģĭ +à¹Ģà¸Ńà¸ģ สาร +à¹Ģà¸ģิà¸Ļ à¹Ħà¸Ľ +л еп +ãģĹãģŁ ãĤĤãģ® +ĠC âm +ĠCâm ara +ק×ķ׾ ׳×ķ×¢ +Ġ×ij×Ĵ ×Ļף +Ġoc zy +Ġoczy wiÅĽcie +att ivitÃł +ãĥĵ ãĥ¥ãĥ¼ +Ġeduc ación +Ä° YE +ê¹Į ìļĶ +ãĤ¨ ãĥªãĤ¢ +н еÑģÑĤи +Ġm óg +Ġmóg ÅĤ +Ġק×ĺ ׳×Ļ×Ŀ +ĠPr ä +Ġ×ľ×¢ ×ij×ķר +بÙĨ Ùī +з ол +зол оÑĤ +Ġwn ÄĻtr +ĠwnÄĻtr z +Ġconstr ução +รัà¸ļ รà¸Ńà¸ĩ +س جÙĨ +Ġק ×ķ׳ +ס ×Ļפ×ķר +ĠÙħ دÙī +رض Ùī +п лав +ï¼ ¥ +Ġil a +Ġila ç +ãĤĭ ãģ¹ãģį +ĠÙħ ÙĪÙĤÙģ +à¸ģร ุ +à¸ģรุ à¸ĵา +chodzÄħ c +ĠÑĤÑĭ Ñģ +Ðķ вÑĢо +ĠÙĬ ØŃدث +ãĥ¡ ãĤ¤ãĥ³ +ĠاÙĦص ØŃÙĬ +ĠÐĶ ан +دع اء +ãĤ´ ãĥ¼ãĥ« +ש ×ł×ª×Ļ +×©×ł×ª×Ļ ×Ļ×Ŀ +à¸Ķà¹īวย à¸ģัà¸Ļ +Ġol acaģı +Ġ×ij ×ŀ×Ĺ×Ļר +×Ķ ק +×Ķק ×ŀת +ãĥ¢ ãĥİ +ĠçalÄ±ÅŁ tı +Ġjó venes +ãģĦãģı ãĤī +ĠÙħ عدÙĦ +ĠC Å©ng +ĠSeg ún +Ġdönem de +Ġ׾ ×Ļ×ĵ×Ļ +ãģį ãģ¡ +ãģįãģ¡ ãĤĵ +ãģįãģ¡ãĤĵ ãģ¨ +Ùģر ÙĨس +ÙģرÙĨس ا +åIJij ãģį +Ġcamp aña +ĠÑģам оÑģÑĤоÑı +ĠÑģамоÑģÑĤоÑı ÑĤелÑĮно +á» Ģ +ÙĤ ÙĪا +س ÙĦاØŃ +à¸ģระ à¹ģ +à¸ģระà¹ģ ส +ĠполÑĮз Ñĥ +n qu +nqu ête +รà¹Īวม à¸ģัà¸ļ +ëĬIJ ëĥIJ +à¸Ĺีม à¸Ĭาà¸ķิ +Ġyıll ık +ìĬ ¬ +ĠØ£ صØŃاب +ill é +Ġdó la +Ġdóla res +Ġк ож +Ġкож и +ล à¹īà¸Ń +à¹Ģรีย à¸ļร +à¹Ģรียà¸ļร à¹īà¸Ńย +à¹Ģà¸ŀ ิ +à¹Ģà¸ŀิ à¹Īà¸ĩ +ÑĢиÑĤоÑĢ и +Ġí ijľ +Ġíijľ íĺĦ +ĠпеÑĢ ев +ĠпеÑĢев од +פ×Ĵ ×Ļ×¢×Ķ +ĠdeÄŁerlendir me +Ùģ ائ +ĠвÑĭ год +ınız ı +×ķ׼ ×Ļ×Ĺ +ĠдоÑģÑĤ иг +Ġng Ãłn +æĢĿ ãģ£ãģŁ +ĠÐķ ÑģÑĤÑĮ +ĠاÙĦر غÙħ +ĠzwiÄħz ane +رب Ø· +à¸Ļ ึà¸ĩ +Ġ׾×Ĺ ×ķק +Ġszczeg óln +Ġszczególn ie +Ġبا ستخداÙħ +ĠfÃŃs ico +×¢ ס +עס ×ķק +سÙĦ ÙĪÙĥ +Ġا ØŃد +Ñĩ ÑijÑĤ +×ĸ׼ ×Ķ +Ġl á»ĩnh +ĠÙĪ ØŃت +ĠÙĪØŃت Ùī +à¸Ħวาม สามารà¸ĸ +à¸Ńยูà¹Ī à¹ģลà¹īว +à¸ģาร à¹Ģà¸Ķิà¸Ļà¸Ĺาà¸ĩ +تخ Ø° +צ×Ļ ×ķ×ĵ +ĠاÙĦØ£ س +ĠاÙĦأس ÙĩÙħ +Ġt á»ĩ +ãģ£ãģ¦ ãģĦãģ¦ +สร ุ +สรุ à¸Ľ +Ġком ÑĦ +ĠкомÑĦ оÑĢÑĤ +ìĺ¤ ëĬĶ +ĠÑĢаз в +ĠÑĢазв ива +л анд +h änge +ĠبÙĨ سبة +à¹Ģà¸Ĥ ียว +עצ ×Ŀ +Ġ׾ ×ľ×Ľ×ª +Ñģо ÑĨиалÑĮн +Ġëĭ¤ìĿĮ ê³¼ +Ġרש ×ķ×ŀ +×ŀר ×Ĺ×ij +س ÙĤØ· +Ġalan ı +ĠÄij á»ĩ +é£Łãģ¹ ãĤĭ +à¸Ķ ึà¸ĩ +Ġgegen über +ĠبÙĩ Ø°Ùĩ +à¸ĸืà¸Ń à¹Ģà¸Ľà¹ĩà¸Ļ +ëķ ħ +à¸Ħà¸Ļ à¹Ħà¸Ĺย +ãĤ¢ ãĤ¦ +ãĤ¢ãĤ¦ ãĥĪ +ศ ัà¸ģ +ศัà¸ģ à¸Ķิ +ศัà¸ģà¸Ķิ à¹Į +ÙĤÙĪ اÙĨ +ÙĤÙĪاÙĨ ÙĬÙĨ +Ġhá»Ļ p +ãģªãģıãģª ãģ£ãģ¦ +Ġ×IJ ×ŀ׳ +Ġ×IJ×ŀ׳ ×Ŀ +à¹Ģà¸ķ ืà¸Ńà¸Ļ +ĠзавиÑģ им +ĠзавиÑģим оÑģÑĤи +ת ×Ļ×IJ +ת×Ļ×IJ ×ķר +å§ĭãĤģ ãģŁ +Ġng á»į +Ġngá»į t +íĴ į +ê³¼ ìŀ¥ +Ġb ại +ãģ§ãģį ãģ¦ +Ġcomeç ar +à¸Ľà¸£ าà¸ģ +à¸Ľà¸£à¸²à¸ģ à¸ı +Ġгод Ñĭ +м еÑģ +ĠاÙĦÙħست ÙĪÙī +ĠÑģам Ñĭе +л леÑĢ +ãģ£ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĻ +ãģ¨ãģ® ãģĵãģ¨ +bi ó +à¸ģล à¹Īà¸Ńà¸ĩ +ĠاÙĦز ÙĪج +ãģ«è¡Į ãģ£ãģŁ +à¸Ħà¹Ī à¸Ńà¸Ļ +à¸Ħà¹Īà¸Ńà¸Ļ à¸Ĥà¹īาà¸ĩ +ĠbaÄŁ l +ĠbaÄŁl ant +ĠbaÄŁlant ı +確 ãģĭ +確ãģĭ ãģ« +ãĥľ ãĥ¼ãĥ« +çµĤ ãĤıãĤĬ +ש ×ŀר +à¸Ĺีà¹Ī สามารà¸ĸ +ÙĦ زÙħ +д аеÑĤÑģÑı +รัà¸ļ à¸Ľà¸£à¸° +รัà¸ļà¸Ľà¸£à¸° à¸Ĺาà¸Ļ +å¤ī ãĤıãĤĬ +ï¼ ¢ +ĠìĺĪìĪĺ ëĭĺ +ãĤĪãģĨ ãģ¨ +มัà¸ģ à¸Īะ +ĠH Æ°Æ¡ng +ÙĨ ÙģØ° +×ŀ×ĵ ×ĵ +ĠìĿ¸ ìłķ +Ñħод иÑĤÑĮ +ĠзавиÑģ иÑĤ +×ķ×ĵ ×Ļ×¢ +ãģĵãģ¨ãģĮ ãģĤãĤĬãģ¾ãģĻ +ع راÙĤ +سط ØŃ +à¸ģำ à¹Ħร +ëĵ¤ ëıĦ +×Ļצ ×Ļר×Ķ +ãģĨ ãģĵãģ¨ +ÙĦا ØŃÙĤ +ãģĦ ãĤĮãģ° +ĠиÑģполÑĮз ÑĥÑİÑĤ +ĠB ợi +Ġשק׾ ×Ļ×Ŀ +ÑĨи кл +ÐIJ Ðŀ +Ġ×ijש ׳×Ķ +ÙĨØ´ Ø· +Ġש ×Ļ׳×ķ×Ļ +Ġש×Ļ׳×ķ×Ļ ×Ļ×Ŀ +Ġpobl ación +ĠH Æ°ng +ระ ว +ระว ัà¸ĩ +رÙĬاض Ø© +ر صد +تÙĤ ÙĦÙĬ +تÙĤÙĦÙĬ د +Ġülk em +Ġülkem iz +à¸Ĭ ะ +ãĤ¯ãĥª ãĥ¼ãĥł +èģŀ ãģĦãģŁ +Ġwa ż +Ġważ ne +ê±° ëĵł +ê±°ëĵł ìļĶ +×ŀ×IJ ×ijק +×Ĺ×ĵ ש×ķת +ĠW roc +ĠWroc ÅĤaw +ĠKü ltür +s ist +sist ência +×¢×ĸר ×Ķ +Ġg Æ°Æ¡ng +รà¹īาà¸Ļ à¸Ħà¹īา +ĠÙĪØ£ ÙĪضØŃ +ánd ose +ãĤ· ãĥ¼ãĥ³ +×IJ׳ ר×Ĵ +×IJ׳ר×Ĵ ×Ļ×Ķ +ãģªãģĦ ãģ§ãģĻ +Ġkh ủng +Ġ문 ìĦľ +Ġ×ij ×ĵ×ijר +×ĵ ×Ļ×ķ +×ĵ×Ļ×ķ ×ķ×Ĺ +Ġré gl +ÙħÙĪ اد +об оÑĢ +обоÑĢ оÑĤ +Ġ×Ķ ×ij׾ +Ġ×Ķ×ij׾ ×ķ×Ĵ +ØŃ Ø§Ùħ +ĠاÙĦع اص +ĠاÙĦعاص ÙħØ© +пеÑĢ аÑĤоÑĢ +ت Ø®ÙĦ +تخÙĦ ص +ãģŁãģł ãģĹ +ت سÙħ +à¹Ĥรà¸ĩ à¸ŀ +à¹Ĥรà¸ĩà¸ŀ ยา +à¹Ĥรà¸ĩà¸ŀยา à¸ļาล +ĠY ük +ĠYük sek +Ġש ׳×Ļת +Ġש׳×Ļת ף +liÄŁ e +Ġפ ת +Ġפת ×ķ×Ĺ +Ġbe ÄŁ +ĠbeÄŁ en +Ġ×ŀ ×ķר +Ġ×ŀ×ķר ׼×ij +Ġرس اÙĦØ© +íĨµ ìĭł +Ġaval ia +Ġavalia ções +Ġman h +Ġmanh ã +Ġìķ ŀ +Ġìķŀ ìľ¼ë¡ľ +ÙĤ تر +ÙĤتر ØŃ +à¹Ģà¸ģ ืà¸Ń +à¹Ģà¸ģืà¸Ń à¸ļ +Ġpropos é +Ø£ Ùħا +Ø£Ùħا ÙĥÙĨ +ĠÐŀ Ðŀ +ĠÐŀÐŀ Ðŀ +ÙħÙĤ ار +ÙħÙĤار ÙĨØ© +ëĦ IJ +ãģĦãģŁãģł ãģı +ÙĤ ÙĬÙĦ +Ġна ÑĪиÑħ +ãĤ« ãĥĥãĥĹ +×Ĺ׾ ת +Ġëĭ¤ ë§Į +à¸Ĺัà¹Īว à¹Ĥลà¸ģ +ãĥį ãĤ¿ +ØŃس اس +ãģ«ãģª ãĤĮ +ج ائ +جائ زة +é change +é conom +économ ie +Т Ðĺ +סת ׼׾ +à¸Ĺัà¹īà¸ĩ สà¸Ńà¸ĩ +ĠاÙĦØ® اÙħ +ĠاÙĦخاÙħ س +ק ×ĺ×¢ +au waż +à¸ľà¸¹à¹ī à¸Ĭาย +à¹ģà¸Ľà¸¥ à¸ģ +åIJĮæĻĤ ãģ« +зн аниÑı +ãģĦãģŁãģł ãģįãģ¾ãģĹãģŁ +Ġ×ŀ×ij ׾×Ļ +à¸Ĥà¸Ń à¹ĥหà¹ī +ĠاÙĦت ربÙĬØ© +Ġdécou vert +Ġżyc iu +apr ès +Ġy ab +Ġyab anc +Ġyabanc ı +ĠbaÅŁ layan +ìĹĪ ëįĺ +Ġhes abı +Ġë§Į ìķ½ +ë§ Īëĭ¤ +ĠTh ánh +ãĥ´ ãĤ¡ +à¸Ľà¸£à¸±à¸ļ à¸Ľà¸£ +à¸Ľà¸£à¸±à¸ļà¸Ľà¸£ ุà¸ĩ +ĠM ặc +à¹Ģหà¸ķุ à¸ľà¸¥ +ĠÐij ез +Ġcapac itÃł +ÅĤe ÅĽ +ĠпÑĢе им +ĠпÑĢеим ÑĥÑīеÑģÑĤв +ĠÅļ wiÄĻt +Ġpubli é +×ŀ×¢ צ×ij +Ùħشار Ùĥات +à¸łà¸² ษ +à¸łà¸²à¸© ี +Ġdeux ième +ĠÙħØŃ Ø§Ùģظ +ĠÙħØŃاÙģظ Ø© +ĠSch ön +ï½ ¤ +Ġ×Ķ ×ij×¢ +Ġ×Ķ×ij×¢ ×Ļ×Ķ +ĠÙĪاÙĦ ÙĦÙĩ +è¨Ģ ãģ£ãģŁ +à¸ķ à¹īาà¸Ļ +วร รà¸ĵ +à¸Ĺิ ศ +ĠbaÅŁ ına +Ġmog ÄĻ +ש ×Ļפ×ķר +ĠÙĪ عد +ĠÙĪعد Ùħ +Ġhistó rico +Ġk ısı +ĠìĿ´ ê²Į +ĠPol ÃŃtica +ĠÑģиÑĤÑĥ аÑĨии +ĠkoÅĦ ca +×ij×ĵ ×Ļק×Ķ +ĠاÙĦسÙĬ ارات +ãģªãĤī ãģ° +ãĤµ ãĥ© +ãĤĭãģĵãģ¨ãģĮãģ§ãģį ãĤĭ +Ġdecis ão +×ķ ×ķ×ĵ +lä ss +läss ig +Ġ׾ ×Ļשר×IJ׾ +ĠÙĬ أتÙĬ +ר ×ķ×ĸ +ö ÄŁ +Ã¶ÄŁ ret +Ã¶ÄŁret im +Ġд ек +Ġдек аб +Ġдекаб ÑĢÑı +Ġש ×Ĺ×ķר +ãģ¦ãģıãĤĮ ãģŁ +عب ارة +Ġélect rique +ĠاÙĦتÙĨ ÙħÙĬØ© +جر Ùī +ĠìĪĺ íĸī +à¸Ĺ ู +ĠÑĢе алÑĮно +Ñģп оÑģоб +à¸Ħล à¹īาย +Ġس عÙĪد +ön ü +ĠÙģ ÙħÙĨ +تÙĥ ÙĪ +تÙĥÙĪ ÙĬÙĨ +ĠкаÑĩ еÑģÑĤво +ĠконÑĤ ак +ĠконÑĤак ÑĤ +Ġsöz leÅŁme +à¸Ń à¹īาà¸ĩ +Ġت ÙĪÙģ +ĠتÙĪÙģ ÙĬر +×Ķ×ĸ ×ĵ +×Ķ×ĸ×ĵ ×ŀ׳×ķת +ĠØ·ÙĪÙĬÙĦ Ø© +Ġtér mino +Ġ×IJ ×Ļפ×Ķ +ãĥĵ ãĥ« +ส à¹Ĥม +สà¹Ĥม สร +ĠاÙĦ اث +ĠاÙĦاث ÙĨÙĬÙĨ +ев иÑĩ +Ġopin ión +à¸Ľ วà¸Ķ +åı¤ ãģĦ +ร à¹Īา +ĠB iaÅĤ +ĠÑģÑĤ ал +ĠÑģÑĤал о +ó logo +ĠìķĦ ëĭĪëĭ¤ +Ġ×IJ ×Ļת +Ġ×IJ×Ļת ×ķ +à¹Ģหà¹ĩà¸Ļ วà¹Īา +à¸ļ ารà¹Į +çĦ ¼ +çĦ¼ ãģį +ĠìĿ´ìļ© ìŀIJ +ĠнекоÑĤоÑĢ Ñĭе +ks z +ksz taÅĤ +ksztaÅĤ c +ãĤŃãĥ£ ãĥĥãĤ· +ãĤŃãĥ£ãĥĥãĤ· ãĥ³ãĤ° +Ġro ÅĽ +ĠroÅĽ lin +ÑĢаж а +×ij׳×Ļ ×Ļ×Ķ +à¸Ľà¸£ สิ +à¸Ľà¸£à¸ªà¸´ à¸ķ +Ġgörd ü +×ŀ׳×Ķ ×Ļ×Ĵ +å¤īãĤı ãģ£ãģ¦ +Ġ×IJ ×Ķ +Ġ×IJ×Ķ ×ijת×Ļ +à¹Ģร à¹Īà¸ĩ +Ġön ünde +Ġê·¸ ëĥ¥ +пол иÑĤ +полиÑĤ иÑĩеÑģк +ãĥ¡ ãĥĩãĤ£ +ãĥ¡ãĥĩãĤ£ ãĤ¢ +ĠDet ay +ĠDetay lı +ĠاÙĦصÙģ ØŃØ© +à¸ģาร à¹Ģà¸ĩิà¸Ļ +Ġìµľ ê·¼ +׼ ש׾ +ï¼ © +вÑĪ его +íķĺ ìĭ¤ +ĠÐŃ ÑĤ +ĠÐŃÑĤ оÑĤ +ส ื +สื à¸ļ +Ġng ừng +ĠдокÑĥменÑĤ ов +дав аÑĤÑĮ +ĠاÙĦشخص ÙĬØ© +Ġצ ×¢×Ļר +در Ùĥ +س ØŃب +à¹Ħมà¹Ī à¸Ħà¹Īà¸Ńย +Ġ×Ķ×ŀק ×ķ×ŀ×Ļ +สัà¹Īà¸ĩ à¸ĭืà¹īà¸Ń +Ġê·¸ê²ĥ ìĿĦ +ãģĤãĤĭ ãģĦ +ãģĤãĤĭãģĦ ãģ¯ +×IJ×ķ×ĺ ×ķ×ij +×IJ×ķ×ĺ×ķ×ij ×ķס +к ÑĨион +ĠÐľ ожно +ãģı ãģł +ãģıãģł ãģķ +ĠинÑĦоÑĢм аÑĨиÑı +ï» Ł +Ġìŀij ìĹħ +Ġ×Ļ ×ķסף +Ø¥ دارة +ĠاÙĦØŃ Ø§Ø¬ +×ł×¡ ×Ļ×¢×Ķ +из аÑĨиÑı +×IJ׾ ×ij +×IJ׾×ij ×ķ×Ŀ +п ед +Ġק×ĺ ׳×Ķ +ĠÙĨÙģس Ùĩا +ĠMinist ério +Ġп ен +Ġпен Ñģи +ãĥIJ ãĥ©ãĥ³ãĤ¹ +Ġ×Ķת ×ķר×Ķ +Ġt ạm +ĠìĹŃ ìĭľ +ï½ ¡ +Ġth á»± +Ġ ısı +ì» ¨ +ãģĹãģ£ãģĭãĤĬ ãģ¨ +Ġx Æ°a +Ġc ặp +×Ĺ ×Ļ×ij×ķר +วัà¸Ĵà¸Ļ à¸ĺรรม +st är +stär ke +ĠÑģам Ñĭй +p isa +pisa Äĩ +ĠoluÅŁ an +ĠاÙĦØ¥ ÙħاÙħ +ĠcÄĥ ng +Ġgü nl +Ġgünl ük +Ġ׳ש ×IJר +Ġkhi á»ĥn +ç¶ļ ãģijãĤĭ +stit ución +Ġcapac ité +Ġj aki +Ġjaki ÅĽ +вÑĪ иÑģ +вÑĪиÑģ ÑĮ +פע×ķ׾ ×ķת +ĠØŃ ÙĬات +ĠØŃÙĬات Ùĩ +Ġник огда +ÐĽ Ь +Ġ×Ķ×¢ ×ķ×ij +Ġ×Ķ×¢×ķ×ij ×ĵ×Ķ +Ġch Ãło +หลาย à¹Ĩ +ĠÑı н +ĠÑıн ваÑĢ +ĠÑıнваÑĢ Ñı +à¸Īำà¹Ģà¸Ľà¹ĩà¸Ļ à¸ķà¹īà¸Ńà¸ĩ +Ġhö her +ãģķãĤĮãģ¦ ãģĦãģŁ +สà¸ĩ สั +สà¸ĩสั ย +ĠاÙĦ اس +ĠاÙĦاس ÙĦاÙħ +ĠاÙĦØ´ Ùħس +สà¸ĸาà¸Ļ ี +ãĤ¯ãĥ© ãĤ¹ +à¸ŀร ร +à¸ŀรร à¸Ħ +p õ +põ e +Ġpor ém +à¸Ľà¸£à¸° สà¸ĩ +à¸Ľà¸£à¸°à¸ªà¸ĩ à¸Ħà¹Į +powied zie +powiedzie Äĩ +Ġмог Ñĥ +Ġж ел +Ġжел ез +ĠاÙĦØ« ÙĤ +ĠاÙĦØ«ÙĤ اÙģÙĬ +ĠпÑĢав ило +Ġgdy ż +פש ×ķ×ĺ +ÑĢабоÑĤ ка +ĠÙĥ رة +Ø´ دد +Ùħار Ùĥ +Ùħ ÙĥØ© +Ġпод пиÑģ +×ĺ×ķ ×ķ×Ĺ +ĠÅĽ c +ĠÅĽc ian +Ġر جاÙĦ +Ġ×ª×ľ ×ķ×Ļ +и ÑĪ +иÑĪ ÑĮ +Ġmé dec +Ġmédec in +ëįĶ ëĿ¼ëıĦ +ĠÑĤеб Ñı +Ġ׾×Ķ ×ķס×Ļ×£ +ãģĬ 話 +Ġà¹ģà¸ķà¹Ī à¸ģà¹ĩ +د اÙģ +داÙģ ع +ĠC ùng +ãĥ»ãĥ» ãĥ»ãĥ» +ê¶ ģ +Ġdeber ÃŃa +หà¸Ļà¹Īวย à¸ĩาà¸Ļ +Ġva ÌĢ +Ġעצ ×ŀ +Ġעצ×ŀ ×Ŀ +à¹Ģà¸Ĭืà¹Īà¸Ń วà¹Īา +שק ×¢ +Ġ×Ķ ׼×ķ׾ +Ġ×Ķ׼×ķ׾ ׾ +ни бÑĥд +нибÑĥд ÑĮ +ĠëĦĪ íĿ¬ +Ġоб ÑĢаÑī +ĠобÑĢаÑī а +Ġ×¢×ij×ķ×ĵ ת +ĠاÙĦÙħÙĨت خب +ıy ord +ıyord u +ÙĪ Ø° +×Ĺש ×Ļ×ij×ķת +Ġ×Ķ×¢ ×Ļק +Ġ×Ķ×¢×Ļק ר×Ļ +ì¢ Į +ยุ à¹Ĥร +ยุà¹Ĥร à¸Ľ +Ġа пÑĢ +ĠапÑĢ елÑı +sz ed +szed ÅĤ +д он +à¹Ģà¸ķิ à¸ļ +à¹Ģà¸ķิà¸ļ à¹Ĥà¸ķ +кол о +Ġkażde j +å¸ ° +帰 ãĤĬ +Ġмил ли +Ġмилли он +ç¾İåij³ ãģĹãģĦ +ت ÙĤار +تÙĤار ÙĬر +ĠìĿ´ 루 +ĠìĿ´ë£¨ ìĸ´ +Ġsprzeda ż +×Ķ ×ķצ×IJ×ķת +ãĤ¢ãĤ¯ ãĤ» +ãĤ¢ãĤ¯ãĤ» ãĤ¹ +ר ×ķ×¥ +ĠгоÑģÑĥдаÑĢÑģÑĤв енн +Ø£ ØŃÙĥ +Ø£ØŃÙĥ اÙħ +ĠoluÅŁ u +ĠA ç +ĠAç ık +ãĤ¸ ãĥ¼ +ç´ł æĻ´ +ç´łæĻ´ ãĤīãģĹãģĦ +Ġ×ijש×ij ×ķ×¢ +ب Ø° +بذ ÙĦ +สา à¹Ģหà¸ķุ +Ġpoz osta +Ġpozosta ÅĤ +ØŃر Ùħ +Ġimport ância +leÅŁtir me +Ġд ÑĢев +Ġmó vil +ĠA ynı +Ġна лог +Ġналог ов +Ġ×Ĺ ×Ļפ×Ķ +ĠÑĦоÑĢм Ñĥ +à¸Ĺà¸Ķ สà¸Ńà¸ļ +ĠksiÄħż ki +Ġma ÅĤe +Ùħس Ø£ÙĦ +ÙħسأÙĦ Ø© +ï¼¾ ï¼¾ +ç ãeste +év iter +Ġкон ÑģÑĤÑĢÑĥк +ĠконÑģÑĤÑĢÑĥк ÑĨи +ï¾ ŀ +Ġת×ķ׼ ׳ +ãĤ¹ãĥĪ ãĥ¬ãĤ¹ +ĠاÙĦاÙĤتصاد ÙĬ +×ŀ×ĵ ×Ļ +Ġw ÅĤad +ĠwÅĤad z +Ø® ÙĪÙģ +ĠмаÑĤеÑĢиал ов +ãģ¨ãģ£ãģ¦ ãĤĤ +Ġznaj du +Ġznajdu jÄħ +Ùģ ئة +ãģ©ãģ® ãĤĪãģĨãģª +æĬij ãģĪ +׳ ×Ĺ׾ +Ġdü ny +Ġdüny an +Ġdünyan ın +гÑĢ ани +гÑĢани Ñĩ +Ġ×Ķש׾ ×Ļש×Ļ +Ġ×Ķ×IJ ש +åıĬ ãģ³ +ìĭŃ ìĭľ +ìĭŃìĭľ ìĺ¤ +Ġдол л +Ġдолл аÑĢ +Ġпов ÑĤоÑĢ +Ġ×Ĺ ×Ļ׳×Ŀ +ת פת×Ĺ +Ñĥв ели +Ñĥвели Ñĩен +ãĤ« ãĥª +raw id +rawid ÅĤow +×ķ ×ķ׾ +ãĥŁ ãĥ¥ +ì½ ĺ +ĠBy ÅĤ +Ðľ ÐIJ +ع ÙIJ +ĠÑģовеÑĢ ÑĪ +ĠÑģовеÑĢÑĪ енно +Ġм ой +Ġ×ķ׾×IJ ×Ĺר +æħ £ +æħ£ ãĤĮ +ØŃ Ø§Ùģظ +Ġ무 ë£Į +à¸Ħà¸ĵะ à¸ģรรม +à¸Ħà¸ĵะà¸ģรรม à¸ģาร +Ġìĸ´ ëĶĶ +Ġdif eren +Ġdiferen ça +ĠاÙĦØ£ ساس +ĠاÙĦأساس ÙĬØ© +Ġ׾×IJ×Ĺר ×ķ׳×Ķ +ê· ł +Ġ×Ķש׳×Ļ ×Ļ×Ķ +ìľĦìĽIJ ìŀ¥ +ลุ à¸ģ +ç iler +Ġ×Ķ×IJ ׾×ķ +èģŀ ãģı +Ġ×ķ×IJ פ×Ļ׾×ķ +ĠÑĢе ализ +ĠÑĢеализ аÑĨи +ระยะ à¹Ģวลา +Ġجدا Ùĭ +تب اع +Ġveh ÃŃculo +Ġдол г +à¸Ľà¸£à¸´ มาà¸ĵ +ì¦ IJ +Ġ׾ ×ŀק×ķ×Ŀ +ĠìĤ¬ ì§Ħ +à¸Ĭ à¹īา +Ġ×ŀ×¢ ×ķ׾×Ķ +Ġgö rm +Ġgörm ek +ĠÙĪÙĩ Ø°Ùĩ +пеÑĢ в +пеÑĢв ÑĭÑħ +ê·¸ ëŀĺ +ĠاÙĦبر ÙĬØ· +ĠاÙĦبرÙĬØ· اÙĨÙĬ +ĠиÑİ Ð½Ñı +ĠÐĵ оÑĢ +Ġ׾ ש׾×Ŀ +ÐIJ ÐĿ +Ġназ наÑĩен +о оÑĢ +ооÑĢ Ñĥж +Ġöz elli +Ġözelli ÄŁi +Ġни же +ç¶ļ ãģijãģ¦ +Ġа ÑĢенд +Ġkat ılı +Ġkatılı m +ĠØ¥ Ø·ÙĦاÙĤ +ĠÙĪØ¥ ذا +Ġок ÑĤÑı +ĠокÑĤÑı бÑĢÑı +à¹Ĥà¸ķ ๠+à¹Ĥà¸ķ๠Ĭ +à¹Ĥà¸ķà¹Ĭ ะ +Ġolduk ları +Ùħ ÙĪÙĤع +ëĤ © +ãģ¨æĢĿ ãģ£ãģ¦ãģĦãĤĭ +Ġש ×Ļ׼×ķ׾ +วา à¸Ķ +س ÙĬÙĦ +à¸Ĥ วั +à¸Ĥวั à¸į +تØŃ ÙĥÙħ +ì ĤŃ +Ġconna ît +׳ פת×Ĺ +Ġch ặ +Ġchặ n +ĠÙħ ØŃÙħ +ĠÙħØŃÙħ ÙĪد +ãģ ´ +ĠпÑĢодÑĥк ÑĨии +зд ÑĢав +ãģĶ è¦ +ãģĶè¦ § +×IJ×ij ×IJ +Ġvé ritable +ĠØ· ÙģÙĦ +ãĥĪãĥ© ãĥĸãĥ« +ê³ ¡ +Ġת ×ŀ×ķ׳×Ķ +Ġki ên +ĠÙĤ ادر +Ø¥ÙĤ ÙĦÙĬÙħ +ĠпÑĢед пÑĢи +ĠпÑĢедпÑĢи ÑıÑĤиÑı +Ġb Äĥng +Ġay ında +Ġg ấp +еÑħ ал +Ġgi Ãłnh +Ġд ав +Ġдав но +ìĺĢ ëĭ¤ +à¸Ļัà¸ģ à¹Ģà¸ķ +à¸Ļัà¸ģà¹Ģà¸ķ ะ +Ùħست شار +ست راتÙĬج +ستراتÙĬج ÙĬ +رÙħ ز +Ġt Ä©nh +ë¡ Ń +ĠÑĩ еÑĤ +ĠÑĩеÑĤ Ñĭ +ĠÑĩеÑĤÑĭ ÑĢе +ĠEnt ão +Ġص غ +Ġصغ ÙĬرة +×ij×Ļ×ĺ ×ķ׾ +خط ÙĪØ· +ĠÑĢазвиÑĤ ие +Ġamacı yla +à¸Ĺี วี +Ġо ÑģÑĤ +ĠоÑģÑĤ алÑĮн +ש×ķ׾׊ף +Ġ׼ ׳×Ļס +Ġ׼׳×Ļס ×Ķ +Ġd áºŃy +ĠyaÅŁ ayan +Ġ×ŀ×Ķ ×ķ×ķ×Ķ +ĠÑĥ Ñģи +ĠÑĥÑģи ли +×ŀ פ×Ļ +ĠпÑĢовед ениÑı +Ġر ب +Ġرب Ùħا +ĠاÙĦØ£ ÙĪسط +Ġìľł ì§Ģ +Ġprac ownik +Ġpracownik ów +×ŀס ×ķרת +ÙĤار ب +à¸Ħวาม รูà¹īสึà¸ģ +à¹ģหล ะ +ĠاÙĦÙĨ ÙĤد +Ġ×IJ׾ פ×Ļ +Ùħس ئ +Ùħسئ ÙĪÙĦ +ев ÑĭÑħ +клÑİÑĩ ениÑı +×ij ×Ļ׳ +×ij×Ļ׳ ×Ļ×Ķ×Ŀ +ש ×ķ×IJ×Ķ +ĠÅŁ ark +ĠÅŁark ı +Ġsü rec +Ġsürec in +à¹Ģà¸Ħร à¸Ķ +à¹Ģà¸Ħรà¸Ķ ิà¸ķ +ãĥIJ ãĥ¬ +ĠØ´ Ø£ÙĨ +à¹Ģà¸Ńา à¹Ħวà¹ī +niÄĻ cie +רצ ×Ĺ +ĠaÅŁ ama +׳ פ×Ĵ×¢ +Ġth á»Ŀ +Ġkhu ẩn +diÄŁ inde +ÑıÑī иÑħ +ãĥĺ ãĥ« +Ġüber h +Ġüberh aupt +ĠÑĤÑĢеб ова +ĠdÅĤ ugi +×ĺ ×Ļף +à¸Ĥà¸Ļาà¸Ķ à¹ĥหà¸įà¹Ī +ĠاÙĦØ£ Ùĩ +ĠاÙĦØ£Ùĩ ÙĦÙĬ +ĠMü d +ĠMüd ürü +Ġ×Ļ×Ķ ×ķ×ĵ×Ķ +Ñĭв аеÑĤÑģÑı +س اط +×Ķת ׳×Ķ×Ĵ +×Ķ×ª×ł×Ķ×Ĵ ×ķת +à¸ģาร à¸ľà¸¥à¸´à¸ķ +íĴ Ģ +สà¸ĸาà¸Ļ à¸ģารà¸ĵà¹Į +Ġо ÑĦ +ĠоÑĦ иÑģ +ĠÙĦ عبة +Ġstron ÄĻ +Ġר×IJ ×ķ×Ļ +×Ĺ ×ij׾ +ĠÑĢÑĭ н +ĠÑĢÑĭн ке +Ġ׾×ŀ×¢ ף +اس ÙĦ +ห ัà¸Ļ +Ġ×IJ ×Ĺ×Ļ +ĠпÑĢод ол +ê°Ģ ìŀħ +Ġ×ijר ×Ĺ +Ġ×ijר×Ĺ ×ij×Ļ +дж еÑĢ +Ġ׾ ×Ĺ׾ +Ġ׾×Ĺ׾ ×ķ×ĺ +Ġ׾×Ĺ׾×ķ×ĺ ×Ļף +ศาส à¸Ļา +ãĤ¢ãĤ¤ ãĥĨ +ãĤ¢ãĤ¤ãĥĨ ãĥł +Ġפר ×ķפ +جز اء +ล à¸Ńย +Ġc iaÅĤa +Ġgi ết +ĠзнаÑĩ иÑĤелÑĮно +Ġolmad ıģ +Ġolmadıģ ını +н д +нд екÑģ +تأ Ùĥد +Ġìĸ ¸ +Ġìĸ¸ ìłľ +ay dın +ãĥī ãĥ¬ãĤ¹ +Ġs ắt +Ġíĺ¸ íħĶ +Ġë¶ ģ +Ġë¶ģ íķľ +ãĥij ãĤ¤ +Ġ×ŀש×Ĺק ×Ļ +à¸Ħà¸Ļ à¸Ńืà¹Īà¸Ļ +Ġиз гоÑĤов +ĠизгоÑĤов лен +à¹Ģà¸ģีย ร +à¹Ģà¸ģียร à¸ķิ +תק שר +ĠÑĢаÑģ ÑĩеÑĤ +ส à¹Ģà¸ķ +Ġl änger +ĠiÅŁ let +ĠiÅŁlet me +Ġع ÙĦÙĬÙĨ +ĠعÙĦÙĬÙĨ ا +é lection +ĠاÙĦغ ربÙĬØ© +íĭ Ģ +ãĤĤãĤī ãģĪ +Ġкни ги +Ø£ سÙħ +أسÙħ اء +Ġth á»ı +Ġthá»ı a +หà¸Ļ ู +Ġ×ł×¢ ש×Ķ +à¸łà¸²à¸¢ à¹ĥà¸ķà¹ī +à¸ŀื à¸Ĭ +رÙĬ Ø· +Ùģ ÙĪض +ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸ ãģĦãģ¾ãģĹãģŁ +ש ×ĵ×Ķ +Ġng á»±c +ĠÑģеÑĢ ÑĮ +ĠÑģеÑĢÑĮ езн +T ôi +Ġfiyat ları +ĠвÑģ Ñİ +ĠC ódigo +Ġ×Ķש ×IJ +Ġ×Ķש×IJ ׾×Ķ +ĠP ública +Ø¥ Ø® +إخ ÙĪاÙĨ +ĠзаÑıв ил +ãĥ¦ ãĥ¼ +ר×IJ ×Ļת +vol ución +Ġsz ko +Ġszko ÅĤy +جرÙĬ دة +Ġpens é +ìī ¬ +ĠBüyük ÅŁehir +ĠØ£Ùħ رÙĬ +ĠØ£ÙħرÙĬ ÙĥÙĬ +à¸Ļัà¸ģ ศึà¸ģษา +Ġtod av +Ġtodav ÃŃa +ĠС ан +ĠСан кÑĤ +íķĺ ìŀIJ +ØŃÙĪ اÙĦ +׼ ×ķשר +à¹Ģลย à¸Ħรัà¸ļ +Ġal gu +Ġalgu ém +Ùģ ز +Ġçek il +Ġ×ĵ ר׼×Ļ×Ŀ +ãĥIJ ãĥ© +à¸ģà¹ĩ สามารà¸ĸ +สà¹Īวà¸Ļ ลà¸Ķ +íı ° +ĠP úb +ĠPúb lico +à¹ģà¸Ļว à¸Ĺาà¸ĩ +×IJת ×Ĵר +Ø´ اش +شاش Ø© +ci ÅĽni +ĠÃľ rün +ÙĦÙĪ ØŃ +ĠاÙĦ بÙĨ +ĠاÙĦبÙĨ Ùĥ +ì¡° ì¹ĺ +Ġorganiz ación +ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸ ãģĦãģ¾ãģĻ +s ätze +ĠÑģем ей +ÙĤ صد +ÑģÑĤв еннÑĭе +Ġpréc éd +Ġprécéd ent +à¸ģรุà¸ĩà¹Ģà¸Ĺà¸ŀ ฯ +ãģ¨è¨Ģ ãģĦ +×ij׳×Ļ ×Ļף +ĠØŃ ÙĪ +ĠØŃÙĪ اÙĦÙĬ +סק ס +ĠsaÄŁlam ak +Ġ׾ צ×Ļ×Ļף +ק×ĵ ש +Ġ×Ķ×ŀ ×¢×¨×Ľ×ª +Ġ׾×Ķ ×¢×ij×Ļר +Ġg ünd +Ġgünd em +ĠнаÑĪ его +à¹ĥà¸Ļ à¸ŀืà¹īà¸Ļà¸Ĺีà¹Ī +à¹Ģà¸Ħร ืà¸Ń +à¹Ģà¸Ħรืà¸Ń à¸Ĥ +à¹Ģà¸Ħรืà¸Ńà¸Ĥ à¹Īาย +ظ اÙĩرة +ÙħÙĨ ظÙħ +ÙħÙĨظÙħ ات +Ùħت از +追 ãģĦ +dı kt +dıkt an +ĠëįĶ ìļ± +ĠÐĿ апÑĢимеÑĢ +tw ór +×ŀ×ķ×¢ צ×Ķ +Ùĥ ÙĪÙĥ +Ð © +×ŀ×ĺ פ׾ +ó lica +訪 ãĤĮ +ĠëĮĢ ë¶Ģ +ĠëĮĢë¶Ģ ë¶Ħ +ãĤ¯ãĥª ãĥĥãĤ¯ +ãĤĴ éģ¸ +ãĤĴéģ¸ ãģ¶ +Ġpow sta +Ġpowsta ÅĤ +Ġraz ón +×ij ×ķ×Ĺר +ĠÑģообÑī ил +Ġק ×ij×ķ×¢ +r êt +à¸Ķี à¸Ĥึà¹īà¸Ļ +×ŀס ×¢×ĵ +×ŀסע×ĵ ×ķת +ĠÃĸ sterreich +Ġ׳ ×Ĺש×ij +Ùħباد رة +ì´ ī +×Ĵ ׳×ĺ×Ļ +ä¿¡ ãģĺ +du ÄŁ +duÄŁ unu +Ġph ú +ĠاÙĦØ£ Ø®ÙĬر +Ġت عتبر +landır ıl +ãģ¨ãģ¯ ãģĦ +ãģ¨ãģ¯ãģĦ ãģĪ +ĠاÙĦ Ø·ÙĦ +ĠاÙĦØ·ÙĦ اب +ĠN º +éģ¿ ãģij +اÙĦ Ùħع +اÙĦÙħع رÙĪÙģ +ส à¸łà¸² +éĽ¢ ãĤĮ +ĠпомоÑī ÑĮ +Ġзна еÑĤ +ãĥĹãĥ¬ ãĤ¼ +ãĥĹãĥ¬ãĤ¼ ãĥ³ãĥĪ +Ġsup érieur +Ġש׾ ×Ļש×Ļ +ĠاÙĦÙĨ ÙĪع +ãĤĵãģ§ãģĻ ãģŃ +à¸Ńà¸ļ รม +Ġgi á»įng +Ġwzgl ÄĻd +ĠاÙĦÙģ ÙĤر +è rent +Ġ×ŀ×IJ ×Ĺ +Ġ×ŀ×IJ×Ĺ ×ķר×Ļ +×Ĵ ×Ĵ +×Ļ ×Ļ×ij +ÙħÙĦ اب +ÙħÙĦاب س +Ġhük ü +Ġhükü met +Ġ×ŀ×Ĵ ×Ļ×ij +ĠÐŀ Ñĩ +ĠÐŀÑĩ енÑĮ +æĹ© ãģĦ +Ġconstr ucción +Ġth ượng +ï¼ ĭ +Ġcor ação +à¹Ģหล à¹ĩà¸ģ +ĠBaÅŁ b +ĠBaÅŁb akan +éĢ£ ãĤĮ +ãģĻãĤĭ ãģĵãģ¨ãģĮãģ§ãģįãģ¾ãģĻ +ĠÙĤ اÙħت +Ġا Ùĥثر +Ùģاع ÙĦ +ĠÑĦ оÑĢ +ĠÑĦоÑĢ Ñĥм +غ Ø°ÙĬ +ĠiÅŁ le +ĠiÅŁle ml +ĠiÅŁleml eri +ĠìĤ¬ëŀĮ ìĿĢ +Ġìŀij ìĦ± +Ġë§Ī 볨 +Ùħ جÙĦس +หม ู +д в +дв иг +двиг а +à¹Ģสีย à¸Ĭีวิà¸ķ +×Ķת פת×Ĺ +×Ķתפת×Ĺ ×ķת +ĠмеÑĤ ÑĢо +ĠÑģ енÑĤ +ĠÑģенÑĤ Ñı +ĠÑģенÑĤÑı бÑĢÑı +ê³ § +Ġ׾ פע +Ġ×ľ×¤×¢ ×ŀ×Ļ×Ŀ +à¹Ģà¸ļ ีย +詳 ãģĹãģı +çķ° ãģªãĤĭ +ĠÄ°l çe +ĠAt at +ĠAtat ür +ĠAtatür k +รุ à¹Īà¸ĩ +Ġkald ı +Ġ주 ìŀ¥ +Ġprés ence +Ġн аб +Ġнаб лÑİ +ĠнаблÑİ Ð´Ð° +ĠÑģам ого +×Ĵ ×ķש +×ŀ×ĺ ×ķפ +×ŀ×ĺ×ķפ ׾ +ĠвÑĭб иÑĢа +ĠìŀIJ 리 +åĪĨ ãģĭãĤīãģªãģĦ +Ġз Ñĥб +Ġש׼ ×ijר +Ġد ائ +Ġدائ Ùħا +ĠпаÑĢ ÑĤи +ï¼ ² +ĠاÙĬ ضا +ĠÑħ оз +ĠÑħоз Ñı +ĠÑħозÑı й +ĠÑħозÑıй ÑģÑĤв +ĠاÙĦØ£ ج +ĠاÙĦأج ÙĨب +ĠاÙĦأجÙĨب ÙĬØ© +ĠÐĹ Ð½Ð° +ĠAp ós +ĠÑį неÑĢ +ĠÑįнеÑĢ ги +Ġy ans +Ġyans ı +ĠJust i +ĠJusti ça +Ġpré vu +ม วล +ìŀ¥ ëĭĺ +à¸ģระ à¸ļ +à¸ģระà¸ļ วà¸Ļ +à¸ģระà¸ļวà¸Ļ à¸ģาร +×ŀ ×ŀ +×ŀ×ŀ ×ķצע +Ġh ẹ +Ġhẹ n +зд ание +Ġak ÅŁ +ĠakÅŁ am +×ĺ ×ķפ +Ġgere kt +Ġgerekt i +Ġgerekti ÄŁini +Ġnar z +Ġnarz ÄĻdzi +é po +épo que +ĠTh ần +Ġwys oko +Ġwysoko ÅĽci +à¸ľà¸¹à¹ī à¸Ľ +à¸ľà¸¹à¹īà¸Ľ à¹Īวย +ĠÙĬ بدÙĪ +ÑĤелÑĮ ного +Ġвз глÑıд +Ġjed nÄħ +ĠìĿĺ 견 +Ġ à¸Ĥà¸ĵะà¸Ĺีà¹Ī +פ ×Ļ×ĵ +ìĥģ ëĭ´ +Ġm ỡ +×Ķ ×ŀ׾ +×Ķ×ŀ׾ צ×ķת +ĠÑģоÑģÑĤ о +ĠÑģоÑģÑĤо иÑĤ +Ġав и +Ġави а +ĠL änder +تص ÙĪÙĬر +×ŀ×ĵ ×Ļ×Ķ +ìłĪ ì°¨ +ãģ¨ ãĤĬ +ãģ¨ãĤĬ ãģĤ +ãģ¨ãĤĬãģĤ ãģĪ +ãģ¨ãĤĬãģĤãģĪ ãģļ +ĠÑĢ Ñıд +ĠÑĢÑıд ом +ĠNh ất +ĠاÙĦÙĥ اÙħÙĦ +×Ĺ׾ ׾ +ĠGi ấy +צ ×ĺר +צ×ĺר ×£ +Ġ׾×ij ×ĺ׾ +Ġим еÑĤÑĮ +ס×ŀ ×ķ×ļ +Ġparticip ação +íķľëĭ¤ ë©´ +ÙħÙĨت دÙĬ +ÙħÙĨتدÙĬ ات +ĠeÄŁ len +g änge +رب ØŃ +ãĤ® ãĥ£ +ĠاÙĦر ÙĤÙħ +à¸ĭ à¹īำ +ĠH óa +×ŀר ×Ĺק +ØŃÙħ اÙħ +بÙĪ Ùĥ +ĠArt ÃŃculo +ãĥĦ ãĤ¢ãĥ¼ +×Ķפ ׼×Ķ +×Ĺ׾ ×ķף +ĠпеÑĢе Ñħод +len miÅŁ +زر اعة +Ġseñ or +ãģ£ãģ¦ ãģįãģ¦ +Ø¥ Ø´ +إش ارة +Ġpod ÃŃa +ĠÃľ lke +н ÑģкаÑı +Ġadapt é +Ġdüzen len +Ġdüzenlen en +ĠÑģÑĤ ала +ĠÙĬ ØŃتاج +Ġn ier +Ġnier uch +Ġnieruch omo +Ġnieruchomo ÅĽci +ãģĵãģ¨ãģĮ ãģĤãĤĭ +ยà¸Ńà¸Ķ à¹Ģยีà¹Īยม +ĠÙħ ج +ĠÙħج اÙĨÙĬ +Ġз аб +Ġзаб ол +Ġзабол ев +Ġзаболев аниÑı +ĠÅĽ ro +ĠÅĽro dk +ĠÅĽrodk ów +Ġ×Ķ ׾×IJ×ķ×ŀ×Ļ +Ġdok ÅĤad +ĠdokÅĤad nie +ãģŁãģı ãģªãģĦ +ãģ¯ãģļ ãģ§ãģĻ +ãģ¨æĢĿ ãģ£ãģ¦ãģĦãģŁ +é cran +ìĹħ ì²´ +trzym aÅĤ +ÑģÑĤв еннÑĭй +ĠNot ÃŃc +ĠNotÃŃc ias +Ùħ رÙĬ +ÙħرÙĬ ض +æ°Ĺ è» +æ°Ĺè» ½ +æ°Ĺ軽 ãģ« +ëĵ £ +Ġ×ĵ ×ķ×IJר +Ġ׾ ×ŀ׳ +Ġ׾×ŀ׳ ×ķ×¢ +ĠçalÄ±ÅŁ ıyor +ĠÅŁ idd +ĠÅŁidd et +ĠM ặt +Ġate ÅŁ +ĠполÑĥÑĩ ениÑı +à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩ มืà¸Ń +Ġgrö ÃŁer +د ائ +دائ رة +Ġbul un +Ġbulun maktadır +à¹Ģห ร +à¹Ģหร ีย +à¹Ģหรีย à¸į +à¸Ļัà¸ģ à¸Ĺà¹Īà¸Ńà¸ĩà¹Ģà¸Ĺีà¹Īยว +Ġalan ında +ĠÑĥ зна +Ġл еÑĩение +売 ãĤĮ +Ġçev ir +Ġdeste ÄŁi +ĠheiÃŁ t +âĸ ² +ØŃ Ø· +à¸Ħำ à¸ķà¸Ńà¸ļ +ãĤªãĥ³ ãĥ©ãĤ¤ãĥ³ +Ġ×ij×Ĺ×Ļ ×Ļ×Ŀ +ãĥ¦ ãĥĭ +Ġdüzenle me +Ġmodal itÃł +سر Ø· +سرط اÙĨ +×ŀ׼ ×ķף +ĠданнÑĭ й +تر ت +ترت ÙĬب +à¸ļาà¸ĩ à¸Ħà¸Ļ +ĠÄIJ á»ĭnh +ม ูล +มูล à¸Ħà¹Īา +ÙĨ ÙĤص +à¸ģาร รัà¸ģษา +ĠÑĦ он +ĠÑĦон д +ãĤĪãģĨ ãģ«ãģªãģ£ãģŁ +Ùħع اÙĦ +ÙħعاÙĦ جة +ĠOs man +ĠOsman lı +иÑĩеÑģк ом +à¸Ńยาà¸ģ à¸Īะ +ãģķãģ¾ ãģĸ +ãģķãģ¾ãģĸ ãģ¾ +ãģķãģ¾ãģĸãģ¾ ãģª +Ġת ×ķ׼׾ +×¢ צ×ij +ĠاÙĦع سÙĥ +ĠاÙĦعسÙĥ رÙĬ +Ġvé hic +Ġvéhic ule +Ġ×Ļצ ×Ĺק +ĠاÙĦÙĪ ØŃ +ĠاÙĦÙĪØŃ ÙĬد +ĠاÙĦع دÙĪ +ĠQu ản +Ġê³µ ëıĻ +بد ÙĦ +ĠÄij ảng +Ġm á»ĩnh +Ġnie zb +Ġniezb ÄĻ +ĠniezbÄĻ dn +Ġyayın lan +обÑī и +Ġgö tür +צ פ +צפ ×ķ×Ļ +ĠÙĦÙĬ بÙĬ +ĠÙĦÙĬبÙĬ ا +ØŃ ÙĪا +Ġд об +Ġдоб ÑĢо +иÑĢÑĥ ем +ĠاÙĦØŃÙĥÙĪÙħ ÙĬØ© +m Ã¤ÃŁig +Ġed ición +влек аÑĤелÑĮ +влекаÑĤелÑĮ н +Ġת ש׾×ķ×Ŀ +Ġ×Ķש ×ķ׳×Ļ×Ŀ +มิ à¸ĸุ +มิà¸ĸุ à¸Ļ +มิà¸ĸุà¸Ļ ายà¸Ļ +é£Łãģ¹ ãģ¦ +ĠìĪĺ ì§ij +ס ×ij×Ļ +ĠиÑİ Ð»Ñı +Ġà¹Ħà¸Ķà¹ī à¹ģà¸ģà¹Ī +׾×Ĺ ×Ŀ +tr ä +trä gt +ãģĿãĤĤ ãģĿãĤĤ +ÐĿ Ðķ +Ġв нÑĥÑĤ +ĠвнÑĥÑĤ ÑĢи +ãģ¨ ä¸Ģç·Ĵãģ« +ãĤ« ãĥķãĤ§ +Ġ×ij×Ĺ ×ĵר +×Ĺ ×ŀש +ãĤ¨ ãĥį +ãĤ¨ãĥį ãĥ« +ãĤ¨ãĥįãĥ« ãĤ® +ãĤ¨ãĥįãĥ«ãĤ® ãĥ¼ +à¸Ĥà¸Ńà¸ĩ à¸ķัวà¹Ģà¸Ńà¸ĩ +بÙĤ اء +פס ×Ļ׼ +פס×Ļ׼ ×ķ׾×ķ×Ĵ +ãĥ¡ ãĥĥ +ãĥ¡ãĥĥ ãĤ» +ãĥ¡ãĥĥãĤ» ãĥ¼ãĤ¸ +ÙĦ ÙĤب +A Äŀ +שק ×Ļ×¢ +ÙĤ ساÙħ +×ĵ×ķ×Ĵ ×ŀ×Ķ +æ·± ãģĦ +íĸĪ ëĬĶëį° +ĠrozwiÄħz anie +à¸Ļัà¹Īà¸Ļ à¹Ģà¸Ńà¸ĩ +×Ļצ ×ij +Ġtr ông +à¹ĥà¸Ĭà¹ī à¸ļริà¸ģาร +ĠاÙĦÙħÙĪ سÙħ +ĠдеÑĤ и +ãģĹãģĭ ãģªãģĦ +ס ×Ļף +Ġréfé rence +à¹ģห à¹īà¸ĩ +ãĤĤãĤī ãģ£ãģŁ +Ġ׾ ר׼ +Ġ׾ר׼ ×ķש +شع ÙĪر +ĠÐij ог +Ġlaz ım +Ġ×Ļש ׳×Ŀ +Ġп аÑĢÑĤ +ĠпаÑĢÑĤ неÑĢ +ĠÑĥ ника +ĠÑĥника лÑĮн +Ġmaté riel +×ŀר ק +Ġph Æ°á»Ŀng +Ġз ай +Ġзай м +Ùģ ÙĤد +Univers itÃł +×¢ ר׼×Ļ×Ŀ +Ġba ño +Ġн оÑı +ĠноÑı бÑĢÑı +à¸Ľ à¹īาย +Ġt ats +Ġtats äch +Ġtatsäch lich +ĠÑĤÑĢ еÑĤÑĮ +Ñį м +ãĥĻ ãĥ¼ãĤ¹ +Ġnh á»±a +ìĬ¤ íģ¬ +ĠعبداÙĦ ÙĦÙĩ +Ġת ×ķר×Ķ +أش ÙĬ +أشÙĬ اء +ĠÙĦÙĦ غا +ĠÙĦÙĦغا ÙĬØ© +Ùħ ÙĪاÙĤ +ÙħÙĪاÙĤ Ùģ +ĠgÅĤówn a +Ġart Ä±ÅŁ +Ġ×ŀק ×ķ×ŀ×Ļ +ãĤ¯ãĥ© ãĥĸ +Ġس ÙĪÙī +ĠìŬ ìĦ± +اس ر +اسر ائÙĬÙĦ +Ġ׳ ×Ľ×ª×ij +ย à¹īà¸Ńà¸Ļ +Ġdeber á +Ġph ẫu +ÑİÑī ем +ĠÙĦدÙĬ ÙĨا +×ŀ×ĺ ×Ķ +Ġ׳ ×ķ׾×ĵ +ĠвÑģÑĤÑĢ еÑĩа +ãĤīãĤĮ ãģ¦ãģĦãģ¾ãģĻ +ĠcaÅĤ ej +ย ึ +ยึ à¸Ķ +поÑĤ ен +поÑĤен ÑĨи +Ġл иÑĤ +ĠлиÑĤ еÑĢ +ĠлиÑĤеÑĢ аÑĤÑĥÑĢ +Ġкажд ом +ĠíĮ IJ +ĠíĮIJ ëĭ¨ +à¸Ī ู +Ġpres ença +ãģªãĤĵ ãģ§ +Ùħ ÙĬاÙĩ +ин ÑĦоÑĢм +инÑĦоÑĢм аÑĨион +инÑĦоÑĢмаÑĨион н +ĠìŀIJ ìĹ° +ר׼ ש +Ġöd ül +ç¶ļ ãģı +Ġп Ñģ +ĠпÑģ иÑħ +ĠпÑģиÑħ олог +ت Ø°Ùĥر +Ġìŀħ ìŀ¥ +ล à¸Ķà¹Į +ìĦł ê±° +ãģ£ãģ¦ ãģĬãĤĬãģ¾ãģĻ +Ġ×Ļ ×¢ +Ġ×Ļ×¢ ק×ij +ĠاÙĦØ· عاÙħ +ãĥĨ ãĤ¹ãĥĪ +ĠTu ấn +Ġparticip ación +×ŀ×ķ×ŀ ×Ĺ×Ķ +×Ĵר ס×Ķ +ĠاÙĦتÙĨ ÙģÙĬ +ĠاÙĦتÙĨÙģÙĬ Ø°ÙĬ +ĠбезопаÑģ н +ge f +gef ähr +Ø´ ÙĪر +Ġmy ÅĽli +ÙĪا Ø´ÙĨ +ÙĪاشÙĨ Ø·ÙĨ +׳×ķס ×¢ +Ùĥ Ùĩ +ÙĥÙĩ رب +ÙĥÙĩرب اء +Ġmus iaÅĤ +ìĭ ¸ +ãĥĸãĥ© ãĥĥãĤ¯ +Ġcré é +ÙĨÙĩ ار +owo ÅĽÄĩ +ÙħØŃا ÙĥÙħ +ĠwÅĤa ÅĽ +ĠwÅĤaÅĽ c +ĠwÅĤaÅĽc iciel +ĠÙĬ ؤ +ĠÙĬؤ دÙĬ +×ŀ×¢ ×ķ׳ +×IJ ×ij׾ +خط Ø£ +ĠÑħ олод +×ĸ ×ķ׾ +ãģĵãĤĮ ãĤī +ãģĵãĤĮãĤī ãģ® +Ġbás ica +ฤ à¸Ķ +ฤà¸Ķ ูà¸ģ +ฤà¸Ķูà¸ģ า +ฤà¸Ķูà¸ģา ล +èIJ½ãģ¡ çĿĢ +ãģªãģĦ ãģĵãģ¨ +ص ÙĪÙħ +ÙĨج ØŃ +׳ק ×ķ×ĵ +׳ק×ķ×ĵ ת +кл аÑģÑģ +íķĺìĭľ ëĬĶ +ëĦ ĺ +Ġש×IJ ×Ļ׳×ķ +ĠС ейÑĩаÑģ +may acaģı +Ġyap ılır +Ġcategor ÃŃa +عب اد +ĠТ еп +ĠТеп еÑĢÑĮ +×Ķ×Ļס×ĺ ×ķר×Ļ +h ế +ãĤ³ ãĥ¼ãĥī +Ġcabe ça +ج Ùħا +جÙħا Ùĩ +جÙħاÙĩ ÙĬر +ä½İ ãģĦ +ĠÑĤоваÑĢ ов +à¸Ĭาว à¸ļà¹īาà¸Ļ +ĠÑģÑĤан ов +ĠÑģÑĤанов иÑĤÑģÑı +ĠавÑĤом обилÑĮ +ĠÑģлÑĥÑĩ ай +à¸Ńั à¸ŀ +ĠG iriÅŁ +ĠìĿ¼ ëĭ¨ +ĠпÑĢ оÑģ +ĠпÑĢоÑģ моÑĤÑĢ +ãģªãģıãģª ãģ£ãģŁ +มี à¸Ľà¸±à¸įหา +ïº İ +éc oute +ĠÙħ ÙĪجÙĪد +Ġس رÙĬع +ĠÙĪÙĩ ÙĨا +ĠÙĪÙĩÙĨا Ùĥ +à¸Ħุà¸ĵ สม +à¸Ħุà¸ĵสม à¸ļัà¸ķิ +Ġìļ° ìĦł +à¸ŀระ à¸ŀุà¸Ĺà¸ĺ +好 ãģ¿ +ظ ÙĦÙħ +Ġм акÑģ +ĠмакÑģ ималÑĮ +ĠмакÑģималÑĮ но +ãĥª ãĤ¢ãĥ« +à¹ģมà¹ī วà¹Īา +ĠاÙĦØŃ ÙĪار +ãĥĹãĥ© ãĤ¹ +Ġع ÙĦاÙĤØ© +Ġíĸī ëıĻ +Ġgönder il +Ġl ãi +ĠsaÄŁ lıkl +ĠsaÄŁlıkl ı +ĠÑĪ аг +Ġ×ij×IJר ×Ķ +prowadzi Äĩ +ãģĦãģı ãģ¤ãģĭ +Ġبت ارÙĬØ® +Ġ×ij×IJ×ķת ×Ķ +Ġmó c +ĠÐľ не +ãĥĹãĥ¬ ãĥ¼ +×IJ ×ĸר×Ĺ +åł´åIJĪ ãģ«ãģ¯ +使 ãģĪ +à¹Ģร ืà¸Ńà¸Ļ +ĠÐŁ еÑĤ +ĠÐŁÐµÑĤ ÑĢ +ãģ«åħ¥ ãĤĭ +Ùħ ادة +à¹Ģà¸ĩ ืà¹Īà¸Ńà¸Ļ +à¹Ģà¸ĩืà¹Īà¸Ńà¸Ļ à¹Ħà¸Ĥ +ĠÑģоÑģÑĤоÑı ние +ôn ica +ĠÑĦ ев +ĠÑĦев ÑĢа +ĠÑĦевÑĢа лÑı +Ġ×ķ ×ĸ +Ġ×ķ×ĸ ×IJת +à¸Ħร ิ +à¸Ħริ ส +ĠÐķ Ñīе +ãģ£ãģ¦ãģĹãģ¾ ãģĦãģ¾ãģĹãģŁ +ĠпÑĢав иÑĤелÑĮ +ĠпÑĢавиÑĤелÑĮ ÑģÑĤв +Ġtä glich +Ġëĭ¹ ìĭľ +×ŀ×ķ×¢ ×ŀ×ĵ +Ġдв оÑĢ +æī ķ +æīķ ãģĦ +ĠÑģÑĤан еÑĤ +Ġвозд ейÑģÑĤв +ĠвоздейÑģÑĤв и +Ġf ête +à¹Ģส า +תק ×ķ×ķ×Ķ +Ġu yar +Ġuyar ı +à¸ģลัà¸ļ à¹Ħà¸Ľ +Ġgi Æ°á»Ŀng +Ġв а +Ġва ÑĪи +ĠÄij áºŃu +ĠSpa ÃŁ +ĠìķĦ ë§Ī +à¹Ħà¸Ķà¹ī à¸ĩà¹Īาย +Ġ×Ķ×ŀ ×ijקש +æĸ° ãģŁ +æĸ°ãģŁ ãģª +ılı yor +пл ан +Ġ×Ķ×ijר ×Ļ×IJ×ķת +ĠaÄŁ rı +Ġsay gı +建 ãģ¦ +Ġnaj wyż +Ġnajwyż sz +سÙĬاس ات +ãģĬ å¾Ĺ +ĠاÙĦع ÙĦÙĬ +ĠاÙĦعÙĦÙĬ ا +Ġcoraz ón +ì¹ĺ ë£Į +หัว à¸Ĥà¹īà¸Ń +Ġب ØŃÙĬ +ĠبØŃÙĬ Ø« +зв езд +بÙĪ ابة +ÐĽ Ðĺ +ÙĦا زÙħ +Ġroz p +Ġrozp oc +Ġrozpoc zÄĻ +触 ãĤĮ +ĠاÙĦج ÙħÙĩ +ĠاÙĦجÙħÙĩ ÙĪر +Ġsp ÄĻd +ĠspÄĻd z +วิà¸Ĺยา ศาสà¸ķรà¹Į +ив аеÑĤÑģÑı +Ġдан ной +Ġreprés ente +ĠÄij á»ĭch +Ġ×¢×ŀ ×ķק +à¸Ńัà¸Ļ à¸ķร +à¸Ńัà¸Ļà¸ķร าย +Ġestr atég +Ġestratég ia +pad ÅĤ +Ġв полн +Ġвполн е +ĠпÑĢедоÑģÑĤав лен +×Ĺ׾ ×ķק +×Ĺ׾×ķק ת +ãĤ¢ ãĥĬ +ĠاÙĦغ Ø° +ĠاÙĦغذ ائÙĬ +ĠÑĥ зн +ĠÑĥзн аÑĤÑĮ +à¸ĭ à¹īาย +å½ĵ ãģ¦ +ØŃÙĬ اء +Ġbás ico +ק×ķ×ij ×¢ +ĠاÙĦÙħ باراة +ĠاÙĦÙĩ اتÙģ +Ġ׼ ׳×Ĵ×ĵ +à¸Ľà¸£à¸° หย +à¸Ľà¸£à¸°à¸«à¸¢ ัà¸Ķ +Ðļ ак +à¸Ĺีà¹Ī à¸Ļà¹Īา +à¸Ĺีà¹Īà¸Ļà¹Īา สà¸Ļà¹ĥà¸Ī +ãģ¾ ãģģ +ï½ ¢ +Ñģк оп +Ġson rasında +Ġur zÄħd +ĠurzÄħd zenia +׼×ķ ×ķ׳ +׼×ķ×ķ׳ ת +Ġ׾×Ķת ×ŀ×ķ×ĵ +Ġ׾×Ķת×ŀ×ķ×ĵ ×ĵ +ĠÑģ ли +ĠÑģли ÑĪ +ĠÑģлиÑĪ ком +ĠÑģÑĤ Ñĥд +ĠÑģÑĤÑĥд енÑĤ +Ġ×Ķ ×ķ×ĵ +Ġ×Ķ×ķ×ĵ ×¢×Ķ +ë¹Ħ ìļ© +à¸Ńยาà¸ģ à¹ĥหà¹ī +Ġb á»ģ +ยุ à¸Ĺà¸ĺ +Ðĺ ÐĿ +س ائر +Ø£ صÙĪÙĦ +ĠاÙĦغ رÙģ +ãģĵãģ¨ãĤĤ ãģĤãĤĬãģ¾ãģĻ +è¾¼ ãģ¾ãĤĮ +ĠاÙĦساب ع +Ġc ủ +ãģĦãģŁãģł ãģĦãģŁ +ì§ ĵ +ìĤ¬ 무 +powied ź +تÙģ Ùĥ +تÙģÙĥ ÙĬر +иÑĢов ки +ĠíĨµ íķ´ìĦľ +ãĤ¨ ãĤ¹ãĥĨ +ĠдеÑıÑĤелÑĮ ноÑģÑĤÑĮ +ĠданнÑĭ м +Ġ×¢ ×ķר +Ġ×¢×ķר ׼×Ļ +×ķ×ĵ עת +Ġhayat ını +Ġb Äħd +ĠbÄħd ź +obs ÅĤug +à¹Ģà¸ŀียà¸ĩ à¹ģà¸Ħà¹Ī +à¸ĭ à¹Īา +è²ł ãģij +ĠÑģÑĤÑĢ ем +ĠÄij á»īnh +ĠÐł ÑĥÑģ +ĠN ữ +Ġ׾×Ķש ×Ļ×Ĵ +Ġjed noc +Ġjednoc ze +Ġjednocze ÅĽnie +Ġ×Ķ×Ĵ ×ij×ķ×Ķ +أخ ÙĦاÙĤ +ĠнаÑģ ел +ĠнаÑģел ениÑı +ĠÙĬ ÙĨب +ĠÙĬÙĨب غÙĬ +ãģĮ ãģĭ +ãģĮãģĭ ãģĭ +×Ĵ עת +Ðŀ Ðł +ĠналиÑĩ ии +Ġë§Ī ì§Ģ +Ġë§Īì§Ģ ë§ī +Ġíĸī ìĤ¬ +Ġtre ÅĽci +Ġê°Ģ ì¹ĺ +ì¦ ĺ +Ġана лог +×Ķצע ת +в лад +влад е +ĠÑģдел ал +Ġ׳ ×Ĵ×Ļש +Ġ׳×Ĵ×Ļש ×ķת +полн ение +à¸Ĩ à¹Īา +ĠD ön +׼׾׼ ׾×Ķ +×ŀ×ĸ ×Ĵ +Ùħ Ùģ +ÙħÙģ Ùĩ +ÙħÙģÙĩ ÙĪÙħ +×Ķ ×ĵ +×Ķ×ĵ פס +×Ķ×ĵפס ×Ķ +ãģĻãģİ ãģ¦ +Ġг ÑĢ +ĠгÑĢ н +×ŀ×ĺ ×ķס +Ġ기 ìĸµ +ï¾ Ł +ĠpÅĤ yn +ĠGr ünde +ĠBü cher +Ġwed ÅĤug +ãģ¾ãģł ãģ¾ãģł +Ġ׳×Ķ ×ĵר +ĠÙĬست Ø·ÙĬع +ĠHi á»ĩp +ãĤŃãĥ£ãĥ³ ãĥļ +ãĤŃãĥ£ãĥ³ãĥļ ãĥ¼ãĥ³ +Ġth á»ķ +Ġeuropé enne +à¸ļ ัà¸ĩ +à¸ļัà¸ĩ à¸Ħัà¸ļ +ĠszczegóÅĤ owo +׳ שק +ãĥķ ãĥ©ãĥ³ãĤ¹ +×ŀ×ķ×ŀ ×Ĺ×Ļ +Ġcom ún +Ġç arp +ØŃت ÙĬا +ØŃتÙĬا ج +ØŃتÙĬاج ات +ëĭ´ ëĭ¹ +ä½ķ 度 +ä½ķ度 ãĤĤ +×ĵ ×ijק +ãģį ãĤĮ +ãģįãĤĮ ãģĦ +Ġк ам +Ġкам еÑĢ +ĠespecÃŃf ico +Ġtel éfono +à¸ķัà¹īà¸ĩ à¸Ńยูà¹Ī +I Åŀ +ãģ© ãĤĵãģ© +ãģ©ãĤĵãģ© ãĤĵ +עצ ×ŀ×IJ×Ļ +à¸Ķัà¸ĩ à¸Ļีà¹ī +ĠÑĦоÑĢм иÑĢов +ĠÑĦоÑĢмиÑĢов а +×ķ×ŀ ×ij +Ġkullan ımı +Ðľ Ðŀ +×¢ ש×Ļ +עש×Ļ ×Ļ×Ķ +Ġön lem +à¹Ģà¸Ń à¹ĩ +à¹Ģà¸Ńà¹ĩ ม +×ŀשק ×Ļ×¢ +ר ×Ļ×Ĺ +à¸Ĥ ัà¸Ķ +ĠíĻ ľ +ĠíĻľ ìļ© +à¸ĭ ะ +ãĤĪãģĨ ãģ«ãģªãĤĬãģ¾ãģĹãģŁ +ĠÑĢаÑģ пÑĢ +ĠÑĢаÑģпÑĢ оÑģÑĤ +ĠÑĢаÑģпÑĢоÑģÑĤ ÑĢан +ĠÑĢаÑģпÑĢоÑģÑĤÑĢан ен +׼×Ļ ×ķף +ÙĤب ض +تص رÙĬØŃ +تصرÙĬØŃ Ø§Øª +Ġо ÑĢи +ĠоÑĢи г +ĠоÑĢиг ина +ĠоÑĢигина л +ĠاÙĦع اÙĦÙĬ +à¹ģหà¹Īà¸ĩ à¸Ļีà¹ī +ãĥķãĤ¡ ãĥ¼ +ãģ¦ãģĦ ãģį +ãģ¦ãģĦãģį ãģŁãģĦ +פ תר +פתר ×ķ׳×ķת +Ġ×ij ×Ļ×Ĺ +Ġ×ij×Ļ×Ĺ ×ĵ +Ġod by +Ġodby ÅĤ +ĠоÑĩеÑĢ ед +Ġtr Æ°Æ¡ng +ãĤŃ ãĥ³ +×ŀ ×ķפ +×ŀ×ķפ ×¢ +ëĵľ 립 +ëĵľë¦½ ëĭĪëĭ¤ +à¸ŀืà¹īà¸Ļ à¸IJาà¸Ļ +ìŀIJ 격 +ĠVi á»ĩn +ĠDes pués +Ġ×IJ׾ ×Ļ׳×ķ +Ġdur ée +íĩ ´ +Ġmü zik +i ếu +ĠÑĢаз меÑīен +Ġк Ñĥд +ĠкÑĥд а +غ ض +غض ب +ĠTamb ém +à¸Īัà¸Ķ สà¹Īà¸ĩ +à¸ģาร à¹ģสà¸Ķà¸ĩ +onom ÃŃa +Ġан г +Ġанг ли +Ġангли й +Ġанглий Ñģк +Ġzn al +Ġznal az +Ġznalaz ÅĤ +תר ×Ĵ +תר×Ĵ ×ķ×Ŀ +ĠÑģ нов +ĠÑģнов а +ĠÑĩаÑģ а +Ġcommun auté +ĠespecÃŃf ica +ĠL á»ĭch +Ġli é +Ùģ جر +à¹Ģà¸ģ à¹Īà¸ĩ +ع اÙĦ +عاÙĦ ج +Ø£ÙĨ ظ +Ø£ÙĨظ ÙħØ© +ES Ä° +ĠاÙĦØŃ Ø¯ÙĬد +à¸ŀระ à¸Ńà¸ĩà¸Ħà¹Į +Ġפר שת +Ġдв иж +Ġдвиж ениÑı +ĠاÙĦج ارÙĬ +à¸ĺาà¸Ļ ี +неÑģ ен +ĠاÙĦÙĨ ÙĩائÙĬ +Ġб еÑĢ +ĠбеÑĢ ем +ĠбеÑĢем енн +Ġdépart ement +à¹Ģà¸Ĺ ีย +à¹Ģà¸Ĺีย à¸ļ +ĠÐľ аÑĢи +ĠнекоÑĤоÑĢ ÑĭÑħ +об еÑģп +обеÑģп еÑĩен +×Ĺ ×ķ×ĸ +×Ĺ×ķ×ĸ ×Ķ +ÙĨت ج +à¸Īะ à¹Ħà¸Ķà¹īรัà¸ļ +á» ° +Ġél éments +ع Ø· +عط اء +Ġt ắt +i á»ĩm +ÑİÑīиÑħ ÑģÑı +ãģĹãģ ° +ãģĹãģ° ãĤīãģı +Ġпом ожеÑĤ +à¸Ĥà¸ĵะ à¸Ļีà¹ī +Ġ×¢ שר×ķת +éģķ ãģ£ãģ¦ +ĠпÑĢ ог +ĠпÑĢог н +ĠпÑĢогн оз +Ġt ÅĤ +ĠtÅĤ um +ĠtÅĤum acz +T ür +Tür kiye +ãģį ãģ£ +ãģįãģ£ ãģĭãģij +Ġ×Ķ׳ ×ķ׼ +Ġ×Ķ׳×ķ׼ ×Ĺ×Ļ +ĠìĥĿ ìĤ° +ĠÑĦоÑĢм Ñĭ +ç¾İ ãģĹãģĦ +à¸Ľà¸£ ึà¸ģ +à¸Ľà¸£à¸¶à¸ģ ษา +Ġlum ière +ãĤª ãĥ¼ãĥĹ +ãĤªãĥ¼ãĥĹ ãĥ³ +à¸Ľ ืà¸Ļ +วั สà¸Ķ +วัสà¸Ķ ุ +еÑĢÑĤ в +ÙĥÙĦ Ùģ +ï½ £ +à¸ĺรรม à¸Ķา +׳ ×ĺר +ĠпÑĢедÑģÑĤав лÑıеÑĤ +Ġanál isis +Ġb ãi +با ÙĤÙĬ +à¸Ľà¸£à¸° à¹Ģà¸Ķ +à¸Ľà¸£à¸°à¹Ģà¸Ķ à¹ĩà¸Ļ +ĠÑģлÑĥÑĩ аÑı +ĠÑģлÑĥÑĩаÑı Ñħ +ÐĽ ÐIJ +สัà¸ĩ à¹Ģà¸ģ +สัà¸ĩà¹Ģà¸ģ à¸ķ +Ġprz ec +Ġprzec ież +Ùħ صÙĦ +ÙħصÙĦ ØŃØ© +ש×ķק ×ķ׾×ĵ +ĠобоÑĢÑĥд ованиÑı +Ġtr waÅĤ +رÙĪ Ùħ +ìķĪ ëĤ´ +ĠNgh á»ĭ +Ø® Ø´ +à¸ļา à¸Ħาร +à¸ļาà¸Ħาร à¹Īา +Ġоп ÑĨион +ĠÑģозд аниÑı +ãĤ³ ãĤ¹ãĥĪ +Ġ×Ķ×¢ ׾×Ļ +Ġ×Ķ×¢×ľ×Ļ ×ķף +lä uft +ãĥĻ ãĤ¹ãĥĪ +Ġr ê +Ġrê ve +×IJ ×ij×Ļ×ij +×Ļ ×Ļ×ļ +ë¶ Ļ +ãĤ¤ãĥ³ ãĥī +ÅĤo ży +ÅĤoży Äĩ +ع ائÙĦ +عائÙĦ Ø© +Ø£ ÙĪر +Ø£ÙĪر اÙĤ +à¸Ĺà¹īà¸Ńà¸ĩ à¸ĸ +à¸Ĺà¹īà¸Ńà¸ĩà¸ĸ ิà¹Īà¸Ļ +Ġä hn +Ġähn lich +ãĥŁ ãĥĭ +à¸ľ ู +à¸ľà¸¹ à¹īà¸Ļ +à¸ľà¸¹à¹īà¸Ļ ำ +ĠмаÑĤеÑĢиал Ñĭ +Ġкап иÑĤ +ĠкапиÑĤ ал +ï¼ ¦ +Ġseç il +Ġh ứng +Ġintéress ant +ãģ£ãģ¦ ãģĦãģı +Ġe ÄŁer +ëIJĺ ìĹĪìĬµëĭĪëĭ¤ +Ġan laÅŁma +ãģĶ åĪ©çĶ¨ +Ġ×ij ×ĸ׼ +Ġ×ij×ĸ׼ ×ķת +ëĿ¼ ë©´ +ĠÙĬ ÙĪس +ĠÙĬÙĪس Ùģ +أسÙĦ ØŃØ© +ĠGef ühl +ĠноÑĢм алÑĮн +ãĥĻ ãĥ³ +ãģķãĤĮ ãĤĭãģĵãģ¨ +ĠÐij еÑģ +ãģ¨ãģĦ ãģĪãģ° +ĠÙħ ÙĩÙħ +ĠÙħÙĩÙħ Ø© +ãģ§ãģĹãĤĩãģĨ ãģŃ +ĠêµŃ ëĤ´ +à¹Ģม à¹ĩà¸Ķ +×ŀ×ij קר +ĠاÙĦد ÙĨÙĬ +ĠاÙĦدÙĨÙĬ ا +à¸Ĭ ู +к ÑĢÑĥÑĤ +Ġtho áng +Ġ׳ ×ĵר +Ġ׳×ĵר ש +ĠÑĢаÑģÑģ казал +ĠAu ÃŁerdem +פ ×IJר +פ×IJר ק +Ġ×ŀש×Ĺק ×Ļ×Ŀ +צ ר׼×Ļ×Ŀ +×ŀ×ĵ ×ķ +×ŀ×ĵ×ķ ×Ļק +èĭ¦ ãģĹ +ĠÑģ иг +ĠÑģиг нал +ĠM á»įi +Ġtr ữ +Ġnast ÄĻp +ĠnastÄĻp nie +Ġì¶Ķ ì§Ħ +ĠاÙĦÙģ ÙĨد +ĠاÙĦÙģÙĨد ÙĤ +koÅĦ czyÅĤ +ส ีà¹Ī +ק ×Ļ×ij +ק×Ļ×ij ×ķ×¥ +ĠнÑĥж нÑĭ +大 åĪĩ +大åĪĩ ãģª +æıĽ ãģĪ +ת ×ķס +ת×ķס פת +ãģ£ãģ¦ ãģĦãģªãģĦ +Ġм Ñı +ĠмÑı г +ĠмÑıг к +Ġjak ie +Ġjakie ÅĽ +à¸ķำ à¸ļ +à¸ķำà¸ļ ล +ĠìŀĪ ì§Ģ +×ij×ĺ ×IJ +ĠоÑĤлиÑĩ но +ÙĤ ÙIJ +ĠавÑĤом об +ĠавÑĤомоб и +ĠавÑĤомоби лÑı +دÙĬÙħÙĤرا Ø·ÙĬ +ĠاÙĦ ÙĪا +ĠاÙĦÙĪا ØŃد +Ġس ÙĪرÙĬØ© +Ø£ غÙĦ +أغÙĦ ب +ĠÑįк ÑĢан +ãĥĹ ãĥ©ãĤ¤ +Ġjeste ÅĽ +ãĥIJ ãĥª +Ġ×Ķ×IJ ×ķ×ķ×Ļר +ائ Ùĥ +à¸Ńยà¹Īาà¸ĩ ยิà¹Īà¸ĩ +ÑĢ екÑĤ +Ġum o +Ġumo ż +Ġumoż li +Ġumożli w +Ġumożliw ia +Ġnäch ste +ĠìŀĪ ì§Ģë§Į +ĠпÑĢед н +ĠпÑĢедн аз +ĠпÑĢедназ наÑĩен +Ġma çı +Ġp omi +Ġpomi ÄĻd +ĠpomiÄĻd zy +ĠاÙĦÙĦ ÙĤاء +à¹Ģà¸Ķ à¸Ńะ +Ġнов оÑģÑĤи +×ŀ׊׾×Ķ +رÙĬاض ÙĬ +à¸Ķ à¸Ļ +à¸Ķà¸Ļ à¸ķรี +ب صر +ìĬ¤ íĥĢ +scri pción +Ġnap isa +Ġnapisa ÅĤ +Ġ׳ש ×ŀ×¢ +ĠاÙĦÙħØŃ ÙĦÙĬ +Ġhi á»ĥn +×IJ ×Ĺ +×IJ׊ר×IJ×Ļ +Ġг ÑĢаниÑĨ +æīĭ ç¶ļãģį +Ùĥ سب +Ġà¹ģà¸ķà¹Ī à¸ĸà¹īา +à¸Ķาว à¸Ļà¹Į +à¸Ķาวà¸Ļà¹Į à¹Ĥหลà¸Ķ +ãĤĭãģĵãģ¨ãģĮãģ§ãģį ãģ¾ãģĻ +åŁºæľ¬ çļĦãģ« +ÙĪÙĦ اد +rä ume +د Ùģاع +×Ļצ ×¢ +ĠO czy +ĠOczy wiÅĽcie +ĠÅ ģ +ĠÅģ a +اÙĦÙĬ اب +اÙĦÙĬاب اÙĨ +áºł I +ĠBir liÄŁi +×Ķ ×ķצ +×Ķ×ķצ ×IJת +ĠÄij ua +Ġê·¸ëŁ¬ ëĭĪê¹Į +Ġréal ité +ع ÙĦاÙĤات +J este +Jeste ÅĽ +Ġмн ож +Ġмнож еÑģÑĤво +ï¼ « +ãĥĹãĥŃ ãĤ¸ãĤ§ +ãĥĹãĥŃãĤ¸ãĤ§ ãĤ¯ãĥĪ +ĠÑĦ л +ظ ÙĨ +×Ĵ׾ ×Ĵ׾ +ĠmÅĤod zie +ĠmÅĤodzie ż +à¸Ļà¹īำ à¸ķา +à¸Ļà¹īำà¸ķา ล +ÐĽ Ðķ +×ij ×ķ×ĺ +Ġ׾×Ķ ×Ĵ×Ļ×ĵ +ãģĵãģ¨ãĤĤ ãģĤãĤĭ +ز اد +×ŀ×Ļ×ĵ ×¢ +ĠgÅĤówn ie +ãĥı ãĤ¦ +ãĥıãĤ¦ ãĤ¹ +б ел +Ġét ape +ðŁĺ Ģ +Ġмод елÑĮ +a ģını +ש ×Ĺק +ש×Ĺק ף +Ġni ño +à¸Ĭ à¹īาà¸ĩ +à¹Ģล ีย +ĠÑĦоÑĢм е +ĠاÙĦØ´ رÙĬÙģ +ĠÑĥд аÑĢ +arr iv +arriv ée +Ġmies iÄĻ +ĠmiesiÄĻ cy +ØŃ Ø±Ùĥ +ØŃرÙĥ ات +ĠDi á»ħn +ÐĿ Ы +ãģ¾ãģ£ãģŁ ãģı +Ġ×Ļ ר×ķק +еÑģÑĤ еÑģÑĤв +еÑģÑĤеÑģÑĤв енн +Ġê·¸ ëŁ¼ +ĠاÙĦÙħ تÙĪ +ĠاÙĦÙħتÙĪ سط +Ġbéné fic +Ġbénéfic ie +Ġwy bra +Ġwybra Äĩ +ĠاÙĦز ÙħÙĨ +ĠпÑĢин Ñı +ĠпÑĢинÑı л +Ùģر ØŃ +Ġk sz +Ġksz taÅĤ +ĠksztaÅĤ t +ק׾ ×ĺ +×ij×ĵ×Ļק ת +Ġgi ấ +Ġgiấ c +Ġpropriet Ãł +деÑĢж ан +ĠKö ln +ĠGü zel +×Ļפ ×ķ×Ļ +ĠCu á»Ļc +ÑįÑĤ аж +تر ÙĥÙĬ +ترÙĥÙĬ ز +лож ений +Ġп Ñĥ +ĠпÑĥ ÑĤи +اخت ÙĦاÙģ +åĩºãģ¦ ãģıãĤĭ +à¸ļุ à¸ģ +âĿ ¤ +ÑĦ ан +פש ×ĺ +à¸ļัà¸Ļ à¹Ģà¸Ĺ +à¸ļัà¸Ļà¹Ģà¸Ĺ ิà¸ĩ +ĠاÙĦس اد +ĠاÙĦساد س +ĠاÙĦÙĤ ÙĪÙħ +ĠاÙĦÙĤÙĪÙħ ÙĬ +Ġyönet ici +Ùĩ ÙĪات +ÙĩÙĪات Ùģ +Ġrespons ável +Ġпод деÑĢжива +ĠاÙĦسÙĦ Ø· +ĠاÙĦسÙĦØ· ات +ãģĹãģ¦ ãģĬãģı +ãĥļ ãĥĥãĥĪ +à¸Ľ ุà¹Īม +Ġogl Äħda +ÙĨا ÙĤ +ÙĨاÙĤ Ø´ +à¸Ħà¸Ńà¸Ļ à¹Ĥà¸Ķ +ĠMü sl +ĠMüsl ü +ĠMüslü man +ĠMo ż +ĠMoż na +Ġnum érique +Ġv á»ı +ĠسÙĬ تÙħ +Ġyer leÅŁ +монÑĤ аж +Ġgo ût +ãģ¦ ãģĬãĤĬãģ¾ãģĻ +ĠKh ánh +Ġе дин +Ġедин ÑģÑĤв +اÙĨ Ø®Ùģ +اÙĨØ®Ùģ اض +ìĭľ íĹĺ +Ġl ặng +ĠÑĢ олÑĮ +à¸ķัว à¹ģà¸Ĺà¸Ļ +à¸Ħà¹Īา à¹ĥà¸Ĭà¹ī +à¸Ħà¹Īาà¹ĥà¸Ĭà¹ī à¸Īà¹Īาย +Ġver füg +Ġverfüg bar +ìĻĶ ëĭ¤ +ãģĦ ãģļ +ãģĦãģļ ãĤĮ +ĠиÑģÑģлед ованиÑı +меÑī а +×Ķ ×Ĺ +×Ķ×Ĺ ×ĸר +à¹ģà¸Ł à¸Ĭัà¹Īà¸Ļ +ت صرÙģ +Ø¥ رÙĩاب +Ġexerc ÃŃcio +Ġé lev +Ġélev é +สัà¸įà¸įา à¸ĵ +Ãĸ Z +ãĥĹ ãĥŃãĤ° +ãĥĹãĥŃãĤ° ãĥ© +ãĥĹãĥŃãĤ°ãĥ© ãĥł +Ġw ewnÄĻtrzn +Ġhen üz +é£Ľ ãģ³ +à¹Ģà¸Ķ à¸Ńรà¹Į +Ñģ Ñĥж +ÑģÑĥж ден +شع ÙĪب +ãģ²ãģ¨ ãĤĬ +Ġwy ÅĤÄħ +ĠwyÅĤÄħ cznie +Ġпло Ñħо +ÐĶ Ðķ +Ạ¦ +Ùģع اÙĦÙĬ +ÙģعاÙĦÙĬ ات +ĠاÙĦع شر +ÑģÑĤÑĥп ил +Ġy arg +Ġyarg ı +нÑİ Ñİ +×ķ×IJ ×ij +Ġu ç +Ġuç ak +ë² ½ +تÙĪ ÙĤÙĬ +تÙĪÙĤÙĬ ع +Ġì¤ij ìĭ¬ +׳×Ļ×ķ ×ķ×ĺ +Ø£ ÙĥÙĦ +ç½® ãģĦãģ¦ +éłĤ ãģį +Ġ×Ķת ×ij +Ġ×Ķת×ij ×Ļ×¢×Ķ +Ġdür fen +Ùħ ÙĤاÙĦ +ÙħÙĤاÙĦ ات +Ġز ÙħÙĨ +à¸ŀฤ ศ +à¸ŀฤศ à¸Ī +à¸ŀฤศà¸Ī ิà¸ģ +à¸ŀฤศà¸Īิà¸ģ ายà¸Ļ +ĠнеÑģк олÑĮ +ĠнеÑģколÑĮ ки +ĠнеÑģколÑĮки Ñħ +Ġcrian ça +มิ à¸ķร +×ŀ׼ ×Ļר×ķת +à¸ģาร à¸ļริหาร +Ġtélé charg +Ġ×IJ×ķ×Ķ ×ijת +ĠBü ro +ä½ľ ãģ£ãģŁ +ĠKi ÅŁi +ç¾İåij³ ãģĹ +à¹Ģลย à¸Ħà¹Īะ +à¸ŀà¸ļ à¸ģัà¸ļ +à¸Ī à¹īา +Ġç er +Ġçer ç +Ġçerç eve +ãĤĴä½ľ ãģ£ãģ¦ +ĠпеÑĢв ÑĥÑİ +×ŀצ ר×Ļ×Ŀ +×IJ׾ ×ķ×Ķ +×IJ׾×ķ×Ķ ×Ļ×Ŀ +Ġagr é +Ġagré able +Ġay ır +Ä°L Ä° +ãĤ ¥ +Ġíĺ Ħ +ĠíĺĦ ìĭ¤ +ثاÙĦ Ø« +ת ×ĸ +ת×ĸ ×ķ׳×Ķ +ãģ¨ãģĦ ãģ£ãģ¦ +ãģ¨ãģĦãģ£ãģ¦ ãĤĤ +Ġا بÙĪ +ĠÑģоб ак +é£Łãģ¹ ãģŁ +Ġдан ном +à¹Ģล ิ +à¹Ģลิ ศ +Ġí ļ +Ġíļ ¨ +Ġíļ¨ ê³¼ +ãĤĤãĤī ãģĪãĤĭ +׳ צ׾ +ÑĦ ик +ÑĦик Ñģ +Ġjeste ÅĽmy +ת×Ĺ×ķש ×Ķ +à¹Ħมà¹Ī à¸Ħวร +ĠØŃ Ø³ÙĬÙĨ +à¸ģาร ลà¸ĩà¸Ĺุà¸Ļ +ë´ ¤ +ĠÐĺ менно +à¸ļ à¸Ńรà¹Į +à¸ļà¸Ńรà¹Į à¸Ķ +ĠC ảnh +ìĦľ ë¹ĦìĬ¤ +Ġпол ов +Ġполов ин +Ġзам еÑĩа +ãģĦãĤį ãĤĵãģª +Ġ×ij ×Ļק +Ġ×ij×Ļק ש +л ÑĥÑĪ +ãĤĴ è¿İ +ãĤĴè¿İ ãģĪ +جرÙĬ ÙħØ© +Ġt ây +ĠاÙĦÙĨ ÙĪ +ĠاÙĦÙĨÙĪ ÙĪÙĬ +ÃĤ N +ì¿ ł +หà¸Ļ าว +Ġ×ij׊ש×ij×ķף +ز ار +à¸Ķ าร +à¸Ķาร า +ĠÅĽ l +ĠÅĽl ub +มีà¸Ħวาม สุà¸Ĥ +Ġn hu +Ġnhu áºŃn +ÙħØŃ Ø·Ø© +à¹Ģสืà¹īà¸Ń à¸ľà¹īา +ĠТ олÑĮко +ĠÙĥ س +ĠÙĥس ارة +ÙħØ´ رÙĪع +niÄĻ cia +×¢ ׼ש×Ļ×ķ +ت ÙĦÙģ +تÙĦÙģ زÙĬ +تÙĦÙģزÙĬ ÙĪÙĨ +Ġl Æ°á»Ľi +ĠÐľÐ¾Ñģк вÑĭ +Ġré serve +Ġan laÅŁ +ĠanlaÅŁ ıl +Ġed eceÄŁi +รà¸Ńà¸ĩ à¹Ģà¸Ĺà¹īา +Ġب Ø· +Ġبط رÙĬ +ĠبطرÙĬ ÙĤØ© +ãģ¦ãģĹãģ¾ ãģ£ãģ¦ +ãĤĤãĤī ãģ£ãģ¦ +بر ج +æ± ļ +æ±ļ ãĤĮ +Ġch oc +Ġchoc ia +Ġchocia ż +Ġzob ac +Ġzobac zyÄĩ +пÑĢ Ñı +пÑĢÑı жен +ĠÑĨ иÑĦ +ĠÑĨиÑĦ ÑĢ +Ġм ам +Ġвз ÑıÑĤÑĮ +Ġch ạm +ج سÙħ +ØŃÙħ اس +à¹Ģล à¹Īม +à¸ŀิ ษ +×Ķפ ׼×ķ +à¸Ĭà¹Īà¸Ńà¸ĩ à¸Ĺาà¸ĩ +Ġв ек +Ġвек а +Æ¡ Ìģ +Æ¡Ìģ i +ĠTi á»ģn +Ġtr ầm +мÑĭ ÑĪ +мÑĭÑĪ л +ĠÑĤ Ñĥ +ĠÑĤÑĥ ÑĢиÑģÑĤ +Ġch c +Ġchc Äħ +Ġав г +Ġавг ÑĥÑģÑĤ +ĠавгÑĥÑģÑĤ а +ס ×IJ×ķת +Ġר ×Ĵ׾ +à¸ľà¸¥ à¸ģระà¸Ĺ +à¸ľà¸¥à¸ģระà¸Ĺ à¸ļ +å¤īãĤı ãĤĭ +Ġ×Ķ×IJ×Ĺר ×ķ׳×Ļ×Ŀ +سÙģ ÙĬر +ĠÑĩа Ñīе +ãģĦ ãĤī +ãģĦãĤī ãģ£ +ãģĦãĤīãģ£ ãģĹãĤĥ +×ķ×ŀ ׳×Ļ×Ŀ +Ġart tır +ĠCh á»ĭ +Ġì¡° ì§ģ +ĠÑĥÑģп еÑħ +Ġ×¢ ×ķס +Ġ×¢×ķס ק +ĠìĥĿ ëªħ +ÑĨ иÑĤ +Ġreg ión +Ðŀ ÐĿ +ĠdoÄŁ um +ĠyaÅŁ ad +ĠyaÅŁad ıģı +à¸Ĺà¸Ķ ลà¸Ńà¸ĩ +Ġgöz ü +ש ×Ļר×Ķ +дÑĥм ал +Ġda ģı +Ġdaģı t +à¸Ĺีม à¸ĩาà¸Ļ +Ġti á»ģm +ĠاÙĦÙĥ بر +ĠاÙĦÙĥبر Ùī +ì¹ Ń +ĠGü nc +ĠGünc elle +ĠGüncelle me +ê¹ Ĭ +ĠобоÑĢÑĥд ование +ĠÑĢеÑĪ а +á» ¤ +Ġп иÑĤ +ĠпиÑĤ аниÑı +à¹Ģรีย à¸ļ +×Ľ×ª ×Ļ×ij×Ķ +Ġп он +Ġпон ÑĢав +ĠпонÑĢав и +Ġ×Ķ ×ķ׾×ĵ +Ġ×Ķ×ķ׾×ĵ ת +Ġê² ģ +Ġê²ģ ëĭĪëĭ¤ +ĠпеÑĢв ой +ãĥ©ãĤ¤ ãĥķ +ĠÅŁi ir +kr ÄĻ +krÄĻ c +Ġthi á»ĥu +à¹Ģลย à¸Ĺี +à¹Ģลยà¸Ĺี à¹Ģà¸Ķียว +×ĺ×¢ ׳×ķת +ائ ÙĩÙħ +Ġ×IJ ס×ķר +ĠплаÑĤ еж +تر دد +Ġmożli we +Ġkh Ỽ +ĠkhỼ p +تÙģاع ÙĦ +ĠÑĪ колÑĮ +ĠÑĪколÑĮ н +ĠÙĤ صة +Ġmét ier +nÄĻ ÅĤa +หล à¹Īà¸Ń +Ġ ủng +Ġprz egl +Ġprzegl Äħd +ĠاÙĦÙħ تعÙĦ +ĠاÙĦÙħتعÙĦ ÙĤØ© +ĠÑģÑĭ н +Ġв олн +ãĥĩ ãĥ¼ãĥĪ +ĠÐŃ ÑĤи +Ġк ÑĢоме +à¸Ħ ารà¹Į +׳ק ×ķ×ĵ×Ķ +Ġ׾ש×ŀ ×ķ×¢ +Ġ×ĸ ×ķ׼ר +ï¼ § +ÙĬ Ùİا +Ġgi á»ıi +åĥį ãģı +ĠÑģ ни +ĠÑģни жен +à¹ģà¸Ķ à¸Ķ +รุ à¸Ļ +รุà¸Ļ à¹ģรà¸ĩ +Ġhi á»ĩp +ograf ÃŃa +à¹Ģà¸Ī à¸Ńรà¹Į +Ġдв иг +Ġдвиг аÑĤ +ĠдвигаÑĤ ел +Ġü y +Ġüy eler +Ġüyeler i +Ġб Ñĥк +ĠбÑĥк в +ãĤĤ å¤ļãģı +Ġthi á»ĩt +ĠPa ÃŃs +ĠØ· بÙĬعÙĬ +à¹ģà¸Ī à¸ģ +ĠاÙĦص ØŃÙĬØŃ +Ġapp ré +Ġappré ci +Ġdecis ión +Ġë°ĺ ëĵľ +Ġë°ĺëĵľ ìĭľ +ĠÑĤеб е +ãĤ· ãĥ¼ãĤº +ãĤ·ãĥ¼ãĤº ãĥ³ +Ġд алÑĮн +ĠìĬ ¤ +ĠìĬ¤ ìĬ¤ +ĠìĬ¤ìĬ¤ ë¡ľ +ĠTh á»ĥ +Ġkar ÅŁ +ĠkarÅŁ ıs +ĠkarÅŁÄ±s ında +ĠK ön +ĠKön ig +ив ание +×ij ×ķצע +г лаÑģ +Ġtw ó +Ġtwó rc +à¸Ľà¸ģ à¸Ħร +à¸Ľà¸ģà¸Ħร à¸Ńà¸ĩ +ĠG ÅĤ +ĠGÅĤ ówn +ĠUnter stüt +ĠUnterstüt zung +Ġд ÑĥÑħ +ĠдÑĥÑħ ов +Ø£ ÙħاÙĨ +×Ĺש ש +ت ظ +تظ اÙĩر +ĠлÑİб ом +à¸ķ าร +à¸ķาร าà¸ĩ +Ġkr ól +Ø£ ØŃدث +ì¡Į ëĭ¤ +Ðļ ÑĥÑĢÑģ +ãĥĥ ãĥĦ +×ŀק ×ķ×ij׾ +ĠÑģимв ол +Ġdés orm +Ġdésorm ais +w üns +wüns che +Ñĥ ни +Ñĥни ÑĨип +ÑĥниÑĨип алÑĮн +หลัà¸ģ สูà¸ķร +ÙĨت شر +Ġа л +Ġал к +Ġалк ог +Ġалког ол +ĠÑĥ ÑĩиÑĤÑĭва +à¸ģำ à¸ģัà¸ļ +Ġ׾ פע×ķ׾ +ĠìĹ° ê²° +s Äħd +ĠاÙĦØ£ ÙĬ +ĠاÙĦØ£ÙĬ اÙħ +غÙĬ اب +Ġна ÑĢ +ĠнаÑĢ ко +×ŀ×ķ×ĵ ×¢ +ĠÑģеÑĢ ии +пиÑģ Ñĭва +สิ ว +ç¶ļ ãģĦãģ¦ +çĶ³ãģĹ è¾¼ãģ¿ +Ġ׾ ×Ĵר +Ġ׾×Ĵר ×ķ×Ŀ +Ġд ем +Ġдем о +Ġë³´ ëĤ´ +تÙĩ دÙĬد +ĠÙħØ´ ÙĬرا +Ġdu y +Ġduy á»ĩt +ĠwiÄĻks ze +Ùħع اÙĬ +ÙħعاÙĬ ÙĬر +ĠG da +ĠGda ÅĦsk +Ġr ah +Ġrah ats +Ġrahats ız +ר ×ķצ×Ķ +l ös +lös ung +ĠТак им +ÑĪ ед +ÑĪед ÑĪ +ع زÙĦ +Ġרש ×Ļ×ŀת +Ġ׾×Ķ ×Ļ׼ +Ġ׾×Ķ×Ļ׼ ×ł×¡ +Ġп ÑĥÑĤ +ĠпÑĥÑĤ еÑĪ +ĠпÑĥÑĤеÑĪ еÑģÑĤв +Ġnot ÃŃcia +Ġal Ä±ÅŁ +ĠalÄ±ÅŁ ver +ĠalÄ±ÅŁver iÅŁ +ĠwÅĤ os +ĠwÅĤos ów +Ġب غ +Ġبغ داد +Ġver öffent +Ġveröffent licht +ĠKh á +Ġt án +ëIJĺ 기 +Ġë°© 문 +Ùģ ÙĬÙĦ +à¹Ģà¸ģิà¸Ķ à¸Īาà¸ģ +åı¯ æĦĽ +åı¯æĦĽ ãģĦ +à¸ĸ ุà¸ĩ +Ġz ewnÄĻtrzn +à¸łà¸²à¸©à¸² à¸Ńัà¸ĩà¸ģฤษ +Ġmá xima +Ġul us +Ġulus lararası +Ġ׳×Ķ ׳ +à¸Ĥà¹Īาว สาร +ĠìĿĺ ìĤ¬ +à¹Ģหล ืà¸Ńà¸ĩ +Ġد ÙĤ +ĠدÙĤ ائÙĤ +สืà¹Īà¸Ń สาร +ë¨ ¼ +ĠÑģоÑģÑĤоÑı нии +สมา à¸Ħม +á» Ĥ +ĠÐľÐ¾Ñģ ков +ĠÐľÐ¾Ñģков Ñģк +×ŀס ×ķ×Ĵ׾ +ãģĭ ãģĭãĤĬ +ĠTr uyá»ģn +à¹ģà¸Ĥà¹ĩà¸ĩ à¹ģรà¸ĩ +×ŀ×Ĺ ×ĸ×Ļק +à¹Ĥà¸ģ à¹ī +ÙĬس ر +ìĶ © +×IJ ×ķק +×IJ×ķק ×ĺ +×IJ×ķק×ĺ ×ķ×ijר +Ġprox imité +ÙħÙĨ Ùĩج +ĠاÙĦج ز +ĠاÙĦجز ائ +ĠاÙĦجزائ رÙĬ +ĠÄIJi á»ĥm +Ġден еж +Ġденеж н +ÙģØŃ Øµ +Ùģ ئ +ĠÐij Ñĥд +×Ĵ×Ļ×ĵ ×ķ׾ +ĠÐĴ едÑĮ +عÙĦ اÙħØ© +Ġ×IJ×Ĺר ×ķ׳×ķת +ãģĦãģŁãģł ãģĦãģ¦ +سÙĦ ØŃ +ØŃ ÙĦÙħ +ز ÙĪار +Ùĥ سر +×ĺ קס +Ġб ан +Ġбан ков +ĠпÑĢ ож +ĠпÑĢож ива +li wo +liwo ÅĽci +ĠTi ếp +ĠاÙĦÙħÙĨ اسب +ĠاÙĦØ® ÙĬار +ãģĬ ãģĭ +ãģĬãģĭ ãģĴ +à¸Ķà¸Ńà¸ģ à¹Ħมà¹ī +ä mp +ämp fe +à¸ķัà¹īà¸ĩ à¹ĥà¸Ī +Ġза ÑīиÑĤ +ĠзаÑīиÑĤ Ñĭ +ĠTh Æ°á»Ŀng +Ġص Ùģ +ĠصÙģ ØŃØ© +×Ĺ×ķר ×£ +ãĥIJ ãĥĥãĤ° +Ġ×ĵ ×Ļ×Ĵ +Ġ×ĵ×Ļ×Ĵ ×Ļ×ĺ +Ġ×ĵ×Ļ×Ĵ×Ļ×ĺ ׾×Ļ +Ġ×Ķ×Ĺ ×ķ׾×Ļ×Ŀ +в еÑī +веÑī а +Ġк ÑĥлÑĮÑĤ +ĠкÑĥлÑĮÑĤ Ñĥ +ĠкÑĥлÑĮÑĤÑĥ ÑĢÑĭ +ĠاÙĦاÙĨ ترÙĨت +Ġhö ch +Ġhöch st +Ġíĺ ķ +Ġíĺķ íĥľ +Ġв ой +Ġвой нÑĭ +ÐĽ Ðŀ +ìĭł ìļ© +Ġ×ŀ×ij ×ķס +Ġ×ŀ×ij×ķס ס +×ŀ׳ ×Ļ×¢ +Ġfiyat ı +ĠÑģл Ñĥж +ĠÑģлÑĥж бÑĭ +à¸Ĺั ศ +à¸Ĺัศ à¸Ļ +ãģĵãģ¨ãģĮ å¤ļãģĦ +Ġ×Ķ×ŀש ת +Ġ×Ķ×ŀשת ×ŀש +å¯Ħ ãģĽ +×ŀש׾ ×ķ×Ĺ +æĻĤ çĤ¹ +æĻĤçĤ¹ ãģ§ +à¸ŀร ี +à¸ŀรี à¹Ģมีย +à¸ŀรีà¹Ģมีย รà¹Į +à¸ŀรีà¹Ģมียรà¹Į ลีà¸ģ +Ġdiffic olt +Ġdifficolt Ãł +ãĥ¬ ãĤ¹ãĥĪ +ãĥ¬ãĤ¹ãĥĪ ãĥ©ãĥ³ +สม à¹Ģà¸Ķà¹ĩ +สมà¹Ģà¸Ķà¹ĩ à¸Ī +Ġж ид +Ġжид к +Ġzu peÅĤ +ĠzupeÅĤ nie +ĠÙħ جر +ĠÙħجر د +ãģĮ å§ĭ +ãģĮå§ĭ ãģ¾ +ãĤŃãĥ£ ãĥ© +Ġ×IJ ×ķ×ķ×Ļר +ãģĬ äºĴ +ãģĬäºĴ ãģĦ +Ġpot rÃł +ĠPa ÅĦst +ĠPaÅĦst wo +Ġب ÙĬاÙĨ +ĠبÙĬاÙĨ ات +Ġин огда +ĠÑĢ а +ĠÑĢа ÑģÑĤв +ĠÑĢаÑģÑĤв оÑĢ +Ġ×ĸ ×ŀ׳ +ยิ à¹īม +Ä Ĩ +ãģ¾ ãģķ +ãģ¾ãģķ ãģ« +ãĥķãĤ¡ ãĤ¤ãĥ« +Ġgörd Ã¼ÄŁÃ¼ +สà¸ĩ à¸Ħร +สà¸ĩà¸Ħร าม +ĠArk adaÅŁ +ĠrozwiÄħz ania +×ŀ ×ķ×ĺ +pi ÄĻ +piÄĻ t +ص غر +ส ย +สย าม +ãĤĨ ãģ£ãģıãĤĬ +Ġtr ần +Ġeconom ÃŃa +Ġgeh ören +ãĤ·ãĥ§ ãĥ¼ +ĠsÅĤ ucha +à¸ŀà¸Ń à¹ĥà¸Ī +ĠоÑĤмеÑĤ ил +ÙĨت ÙĤÙĦ +Ġprop ósito +ĠваÑĪ его +Ġnh ắn +à¹ģà¸ĸ ว +Ġком иÑģ +ĠкомиÑģ Ñģи +waż nie +Ġy avaÅŁ +×ŀ ×Ļק +×ŀ×Ļק ×ķ×Ŀ +ש×IJ׾ ת +Ġyıll arda +ĠÐ ® +ĠЮ ÑĢ +×ł×¡ ×Ļ×ij×ķת +ת צ +תצ ×ķ×Ĵ +Ġод нÑĥ +Ġ à¸Ńยà¹Īาà¸ĩà¹Ħร +Ġà¸Ńยà¹Īาà¸ĩà¹Ħร à¸ģà¹ĩà¸ķาม +ëģ ¼ +à¹Ħล à¹Ī +تس ÙĦÙĬÙħ +بÙĦ اغ +Ġì ī +Ġìī ½ +Ġìī½ ê²Į +ãĥļ ãĥ³ +зв ÑĥÑĩ +ĠW äh +ĠWäh rend +Ġ×Ļ ×Ļת +Ġ×Ļ×Ļת ׼ף +Ġkh uyên +Ġv ẽ +Ġа меÑĢ +ĠамеÑĢ ик +ĠамеÑĢик ан +ĠамеÑĢикан Ñģк +ع جب +ãĥĽãĥ¼ãĥł ãĥļãĥ¼ãĤ¸ +Ġник ÑĤо +ĠÙĤ Ùİ +ĠÙĤÙİ Ø§ÙĦ +ĠÙĤÙİاÙĦ Ùİ +ÐIJ ÐĹ +Ùħ جÙħÙĪع +ÙħجÙħÙĪع ات +Ġnecess itÃł +Ġpob li +Ġpobli żu +Ġph ấn +ĠСо обÑī +ÙħÙĤ اط +ÙħÙĤاط ع +Ġ×Ķצ ×ķר×ļ +la ÅŁtırma +ว ิà¸Ķ +วิà¸Ķ ี +วิà¸Ķี à¹Ĥà¸Ń +Ġ그리 ìĬ¤ +Ġ그리ìĬ¤ ëıĦ +ãĤ¿ãĤ¤ ãĥŁ +ãĤ¿ãĤ¤ãĥŁ ãĥ³ãĤ° +ק×ĺ ×Ĵ×ķר +ק×ĺ×Ĵ×ķר ×Ļ×Ķ +Ġ×Ĺ ×ķפ +Ġ×Ĺ×ķפ ש×Ļ +Ø£ جر +Ġим ени +ĠÑĢан ее +à¹Ģà¸ŀืà¹Īà¸Ńà¸Ļ à¹Ĩ +ĠJes ús +Ñģо един +Ñģоедин ен +Ġר ×Ĺ×ķק +à¹Ĥà¸ļ รา +à¹Ĥà¸ļรา à¸ĵ +ĠH Æ¡n +Ġth áºŃp +تع ÙĬÙĬÙĨ +Ġtart Ä±ÅŁ +ĠtartÄ±ÅŁ ma +ĠGes pr +ĠGespr äch +תר ×ķפ +תר×ķפ ×ķת +Ġcat égorie +Ġоказ Ñĭва +ĠналиÑĩ ие +Ġprésent é +Ġk ull +Ġkull and +Ġkulland ı +Ġü nl +Ġünl ü +ĠÙģ Ùĥرة +из аÑĤоÑĢ +×IJ ×ķ׳ +×IJ×ķ׳ ×Ļ×ij +×IJ×ķ׳×Ļ×ij רס +×IJ×ķ׳×Ļ×ijרס ×Ļ×ĺת +ĠÑĢаÑģÑģ маÑĤ +ĠÑĢаÑģÑģмаÑĤ ÑĢ +ĠÑĢаÑģÑģмаÑĤÑĢ ива +تÙĥÙĦ Ùħ +Ùĥت رÙĪ +ÙĥترÙĪ ÙĨÙĬ +ĠÑģо ÑĩеÑĤ +ĠÑģоÑĩеÑĤ а +ãĤĴè¦ĭ ãģĽ +Ġng ừa +ĠÐł еÑģп +ĠÐłÐµÑģп Ñĥб +ĠÐłÐµÑģпÑĥб лик +ãĤ¦ ãĤ© +ãĤ¦ãĤ© ãĥ¼ +ĠÐľ еждÑĥ +ĠìŀĪ ê²Į +Ġm â +ĠìļĶ ì²Ń +ض ار +ลุ à¹īà¸Ļ +ëĮĢ íķĻêµIJ +×ĸ ×Ļ׼ +×ĸ×Ļ׼ ר×ķף +ãĤ¹ ãĥļ +ãĤ¹ãĥļ ãĥ¼ãĤ¹ +ĠкÑĢаÑģ оÑĤ +ï¼ ¨ +ê¼ Ń +ãĤĴ éĽĨ +ãĤĴéĽĨ ãĤģ +ë° Ŀ +Ġ×Ķ׳ ×IJ +Ġ×Ķ׳×IJ ש×Ŀ +Ġê°Ģ ìļ´ +Ġê°Ģìļ´ ëį° +تÙĥÙĦ ÙģØ© +ĠØŃ ÙĤÙĬÙĤÙĬ +Ġh alk +Ġhalk ın +ÑİÑī ÑĥÑİ +ĠÑģп ин +סר×ĺ ף +ĠпеÑĢв ого +Ġпол ож +Ġполож иÑĤелÑĮн +Ġд л +Ġдл иÑĤелÑĮн +ĠV Ä©nh +ê´ ´ +ĠÑģÑĭ ÑĢ +ĠíĨµ íķĺìŬ +ë³ij ìĽIJ +à¹Ĥรà¸ĩ à¸ĩาà¸Ļ +รัà¸ļ à¸ľà¸´à¸Ķ +รัà¸ļà¸ľà¸´à¸Ķ à¸Ĭà¸Ńà¸ļ +تج ÙĨب +s ÅĤ +sÅĤ uch +ãĤ¢ãĥ« ãĥIJ +ãĤ¢ãĥ«ãĥIJ ãĥł +ëī´ ìĬ¤ +Ġpat ië +Ġpatië nt +Ġìĺ ¤í +Ġìĺ¤í ŀ +Ġìĺ¤íŀ Ī +Ġìĺ¤íŀĪ 볤 +ĠDer ne +ĠDerne ÄŁi +wró ci +wróci Äĩ +Ġоб Ñī +ĠобÑī еÑģÑĤв +ĠобÑīеÑģÑĤв енно +ĠêµIJ ìĪĺ +tıģ ımız +Ġ×Ķ×ŀש ×Ļ×ij +k örper +Ġпозв ол +Ġпозвол иÑĤ +ĠChi ến +أخ ÙĪ +ĠAy dın +à¸Ķà¹īาà¸Ļ ล +à¸Ķà¹īาà¸Ļล à¹Īาà¸ĩ +Ġdr u +Ġdru ż +Ġdruż yn +Ġë°ľ íijľ +ĠTh ảo +جÙĩ اد +à¸ģระà¸Ĺ ูà¹ī +Ġк ÑĢов +ĠкÑĢов и +Ġiçer ik +Ġnad zie +Ġnadzie jÄĻ +ĠС моÑĤÑĢ +Ġph ức +ج تÙħاع +جتÙħاع ÙĬØ© +ком пон +компон енÑĤ +Ġб ил +Ġбил еÑĤ +ãĥIJ ãĥ³ãĥī +ĠPol ÃŃcia +اÙĦ تÙĩ +اÙĦتÙĩ اب +ØŃر Ùģ +ت خط +تخط ÙĬØ· +ãĤ³ ãĥ¼ãĥ +ãĤ³ãĥ¼ãĥ Ĵ +ãĤ³ãĥ¼ãĥĴ ãĥ¼ +･･ ï½¥ +à¸ĭ à¸Ńย +Ġcréd it +è²· ãģ£ãģŁ +ĠпоÑĢ Ñıд +ĠпоÑĢÑıд ке +Ġph ó +Ġw ida +Ġwida Äĩ +جر ائÙħ +à¸ľ ี +ĠbÄĻd ÄĻ +Ġ×ŀ פת×Ĺ +ãĥij ãĥ¼ãĥ +ãĥijãĥ¼ãĥ Ĩ +ãĥijãĥ¼ãĥĨ ãĤ£ +ãĥijãĥ¼ãĥĨãĤ£ ãĥ¼ +ĠKa ż +ĠKaż dy +ĠнеобÑħодим оÑģÑĤи +à¸Ł à¸Ńรà¹Į +à¸Łà¸Ńรà¹Į ม +Ġмал ÑĭÑĪ +Ġпл оÑĤ +ĠÑĥ ÑģÑĤÑĢой +ĠÑĥÑģÑĤÑĢой ÑģÑĤва +à¸ĸ à¸Ńà¸Ļ +ĠoluÅŁtur ul +ĠÅĽwi ad +ĠÅĽwiad om +Ùħع Ùĩد +ĠпÑĢоиз веден +Æ ł +ר ×Ļש +Ùħست Ø« +Ùħستث Ùħر +׳×Ļ ×Ļר +pa ñ +Ġ; -) +Ġë°ľ 견 +Ġgör üyor +Ùħؤ ÙĦÙģ +ĠÄIJ á»ģ +ĠاÙĦÙĨ ÙĪاب +×Ĺק ×Ļר×Ķ +Ġm á»ıi +è¿° ãģ¹ +ÐĿ ик +ìŀĸ ìķĦ +ìŀĸìķĦ ìļĶ +prowadzi ÅĤ +l óg +lóg ica +פס ×ĺ +פס×ĺ ×Ļ×ij׾ +Ġ×ŀ ×ĵ×Ķ +Ġ×ŀ×ĵ×Ķ ×Ļ×Ŀ +ãģĵãģĵ ãģ¾ãģ§ +×Ķ ת×Ĺ +×Ķת׊׾×Ķ +Ġפ ×ķס +Ġפ×ķס ×ĺ×Ļ×Ŀ +Ġн ев +Ġнев оз +Ġневоз можно +ĠdostÄĻp ny +Ġغ اÙĦ +ĠغاÙĦ ب +Ġbez pieczeÅĦst +ĠbezpieczeÅĦst wa +åĪĨ ãģĭãĤĭ +ĠF ührung +à¸ģ ีà¹ī +gem Ã¤ÃŁ +à¸Ĭà¹Īวà¸ĩ à¹Ģวลา +Ġìļ°ë¦¬ ëĤĺ +Ġìļ°ë¦¬ëĤĺ ëĿ¼ +ãģ¥ ãģıãĤĬ +ĠاÙĦÙħ سÙĦ +ĠاÙĦÙħسÙĦ ØŃØ© +Ġlibert é +клÑİÑĩ ение +Ġzam ów +Ġzamów ienia +รà¸ĸ à¹Ħà¸Ł +Ø£ ÙģÙĦ +Ø£ÙģÙĦ اÙħ +Ùħ راج +Ùħراج عة +Ġë¹Ħ êµIJ +ĠاÙĦت اب +ĠاÙĦتاب عة +Ġë§Į ëĤĺ +Ġб Ñĥм +ĠбÑĥм аг +Ġgé nero +Ġìŀĺ 못 +×ŀ פ×ķר×ĺ +è²·ãģĦ çī© +ĠÙĦدÙĬ Ùĥ +Ġ×ľ×¢ ×Ļת +Ġ×ľ×¢×Ļת ×Ļ×Ŀ +ĠsÅĤ ab +ĠпÑĢедÑģÑĤав лÑı +ãĤ¿ ãĤ¤ãĥĪ +ãĤ¿ãĤ¤ãĥĪ ãĥ« +Ùħ ص +Ùħص Ø·Ùģ +ÙħصطÙģ Ùī +Ġdifficult é +ãĥĨãĤ£ ãĥĸ +Ġpew noÅĽci +ĠpewnoÅĽci Äħ +Ġ무 ìĬ¨ +Ø¥ رس +إرس اÙĦ +Ġд алÑĮ +ĠдалÑĮ ÑĪе +Ġ׾ ×ł×¡ +Ġ×ľ×ł×¡ ×ķת +หมูà¹Ī à¸ļà¹īาà¸Ļ +×ŀס×ŀ ׼×Ļ +أسÙĦ ÙĪب +Ġzw ÅĤ +ĠzwÅĤ as +ĠzwÅĤas zc +ĠzwÅĤaszc za +ĠпÑĢ еж +ĠпÑĢеж де +ĠоÑĢганиз аÑĨиÑı +Ġdön emin +Ġdönemin de +Ġ Ủ +ĠỦ y +ä¸ĭ ãģĴ +ĠпоÑģлед ние +Ġgü ne +Ġgüne ÅŁ +Ġ×IJ ×ĸר +Ġ×IJ×ĸר ×Ĺ×Ļ +ãģ§ãģĤ ãĤįãģĨ +ĠÙĨ ÙĤ +ĠÙĨÙĤ اط +æŃ£ ãģĹãģĦ +ĠÑĢ ег +ĠÑĢег иона +ĠFör der +ê²½ ìĺģ +dıkl ar +dıklar ını +trzym aÄĩ +أش Ùĥ +أشÙĥ اÙĦ +×Ķת ×IJ +×Ķת×IJ ×ŀ×Ķ +à¸Ĺำà¹ĥหà¹ī à¹Ģà¸ģิà¸Ķ +ĠGeb ä +ĠGebä ude +ĠСеÑĢ г +ĠСеÑĢг ей +Ġз доÑĢов +ĠздоÑĢов ÑĮÑı +Ġr ãi +ĠпÑĢед ÑĥÑģ +ĠпÑĢедÑĥÑģ моÑĤÑĢ +ĠпÑĢедÑĥÑģмоÑĤÑĢ ен +Ġ×Ķצ ×Ļ×ij +Ġ×Ķצ×Ļ×ij ×ķר×Ļ +Ġdés ir +Ġн оÑĩ +ĠноÑĩ ÑĮ +möglich keiten +Ġ×IJ×Ĺר ×ķ׳×Ļ×Ŀ +Ġsoir ée +ĠNh áºŃn +Ù ª +à¸Ľà¸£à¸°à¸§à¸±à¸ķิ ศาสà¸ķรà¹Į +êµIJ íĨµ +ĠØ£ Ø®ÙĬ +Ġdé cid +Ġdécid é +Ġwy ja +Ġwyja ÅĽni +Ġ สิ +Ġสิ à¸ĩ +Ġสิà¸ĩ หา +Ġสิà¸ĩหา à¸Ħม +à¹ģ à¸Ńรà¹Į +หà¸Ļà¹īา à¸Īà¸Ń +ס תר +Ġê ¶ +Ġê¶ Į +Ġê¶Į 리 +pl ätze +ب Ø·ÙĦ +ê±´ ìĦ¤ +Ġ×IJ ×Ļ×ŀ×Ļ +Ġ×IJ×Ļ×ŀ×Ļ ×Ļ׾ +ãģ ½ +تر اث +×IJ׾ ×Ļ×ŀ×ķת +Ġdispon ÃŃveis +Ġz ale +Ġzale ży +à¸Ľà¸£à¸°à¸Ĭา สัมà¸ŀัà¸Ļà¸ĺà¹Į +ĠÅļw iat +Ġpor ówn +Ġporówn a +Ġ׾×ĺ ×ķ×ijת +×Ķ×ĸ ×ŀ׳×Ķ +Ġ×Ľ×ª ×ķצ×IJ×Ķ +Ġ×ij ק׾ +Ġ×ijק׾ ×ķת +ĠоÑĤ кÑĢ +ĠоÑĤкÑĢ Ñĭва +ãĥij ãĥ¯ãĥ¼ +ë¿IJ ë§Į +Ġв ÑģÑı +ĠвÑģÑı к +ãģ¨ãģª ãģ£ãģ¦ãģĦãĤĭ +Ġgi áºŃn +Ġок ÑĢÑĥ +ĠокÑĢÑĥ жа +ĠокÑĢÑĥжа ÑİÑī +ĠUnivers ität +ĠÑĢ ож +ĠÑĢож д +ĠÑĢожд ениÑı +Ø® ÙĬÙĦ +Ġкомпани й +ĠÑĢазлиÑĩ нÑĭе +ĠЦ ена +׳×Ļ ×ķ×ĸ +׳×Ļ×ķ×ĸ ׾ +׳×Ļ×ķ×ĸ׾ ×ĺר +Ġê³µ ê°Ħ +Ġê°ľ ëħIJ +landır ma +ĠÑĥдал ен +à¸ŀัà¸ģ à¸ľ +à¸ŀัà¸ģà¸ľ à¹Īà¸Ńà¸Ļ +Ġprote cción +Ġb ÅĤ +ĠbÅĤ ÄĻd +à Ī +Ġíĸī ë³µ +ĠÅŁ ü +ĠÅŁÃ¼ phe +Ġí Ķ +ĠíĶ ¼ +ĠíĶ¼ íķ´ +Ġëĭ¤ 르 +à¹Ħมà¹Ī à¹Ģà¸ģิà¸Ļ +ãģ¿ ãģª +ãģ¿ãģª ãģķãĤĵ +ĠпоÑĤ ÑĢеб +ĠпоÑĤÑĢеб иÑĤел +ĠاÙĦÙĥÙĦ اÙħ +ìķĦ ë²Ħ +ìķĦë²Ħ ì§Ģ +ãĤĴ使 ãģ£ãģŁ +Ġbụ i +ĠпоÑĤ еÑĢ +ĠпоÑĤеÑĢ Ñı +ĠØ¢ ÙĦاÙģ +ĠнаÑģÑĤоÑıÑī ее +ãģıãģªãĤĬ ãģ¾ãģĹãģŁ +clus ão +ãĤ³ ãĥĶãĥ¼ +צ פ×Ļ +צפ×Ļ ×Ļ×Ķ +Ø® ÙĦا +Ø®ÙĦا ص +ล à¹īำ +ãĥ¯ ãĤ¤ãĥ³ +Ġมี à¸Ļา +Ġมีà¸Ļา à¸Ħม +Ø´ خص +شخص ÙĬات +Ġ×ĸ ק +Ġ×ĸק ×ķק +×Ļ ×Ļצ +×Ļ×Ļצ ×Ĵ +èĢĥãģĪ æĸ¹ +Ġürün ü +ĠиÑģп ол +ĠиÑģпол ни +Ġcompañ ero +ק צ×Ķ +×ŀ×¢ ׳×Ļק +Ùħ ØŃÙħد +Ġc ámara +Ġп ед +Ġпед аг +Ġпедаг ог +м аÑĢ +маÑĢ к +×Ķת ׳×Ĵ×ĵ +ĠìĨĮ ê°ľ +Ġcom unitÃł +ê³ ¤ +ĠNg Ãłi +สà¸ĩ à¸ļ +ĠmieszkaÅĦ ców +ĠÙĨ ÙĩائÙĬ +iv ité +Ġи де +Ġиде алÑĮн +ĠØ£ سبÙĪع +Ġ×Ļ ×¢×ľ +Ġ׾ ר×IJש +Ġ׾ר×IJש ×ķ׳×Ķ +ĠзапиÑģ и +ĠкоÑĢ пÑĥÑģ +วà¸ĩ ศ +วà¸ĩศ à¹Į +ĠÐĶ м +ĠÐĶм иÑĤ +ĠÐĶмиÑĤ ÑĢ +Ġkön nt +Ġböl ges +Ġbölges inde +׼ ×Ļ׼ +׼×Ļ׼ ר +ĠاÙĦØ¥ Ø«ÙĨ +ĠاÙĦإثÙĨ ÙĬÙĨ +Ġng á»Ļ +ì¹ ł +د راج +Ġu da +Ġuda ÅĤo +ìº IJ +بر ÙĨاÙħج +ĠÑģÑĥд еб +ĠÑģÑĥдеб н +Ġzun ächst +ĠEduc ación +ãģ¨ãģª ãģ£ãģ¦ãģĦãģ¾ãģĻ +Ġ×Ķ×IJ ×ŀ×Ļת×Ļ +ĠÄ° nt +ĠÄ°nt ernet +ĠcaÅĤ ego +ãĥĹãĥª ãĥ³ +Ø¥ بد +إبد اع +ĠпоÑĢ ÑĤал +à¹Ĥà¸ķ à¹ī +Ġ×Ķק ש×ķר +пл од +ĠÙħ د +ĠÙħد رÙĬد +×ŀסע ×ĵ×Ķ +ĠØ´ÙĬ ئ +ĠØ´ÙĬئ ا +à¸ģà¹Īà¸Ń สรà¹īาà¸ĩ +Ġì°¸ ê³ł +à¹Ģà¸Ĺ ร +à¹Ģà¸Ĺร à¸Ķ +Ġ×ij×ŀ קר×Ļ×Ŀ +Ġb ât +Ġbât iment +åij¼ ãģ³ +ç´ł æķµ +ç´łæķµ ãģª +przedsiÄĻbior st +przedsiÄĻbiorst w +Ġ×ł×ª ×ķ׳×Ļ×Ŀ +×Ĺ׾ ×ķ×Ŀ +ร วย +Ùħ ÙĪضÙĪع +ĠÑģоб ÑĢан +вед ÑĥÑī +ĠÑĤе аÑĤ +ĠÑĤеаÑĤ ÑĢ +m eye +meye ceÄŁi +Ġpien iÄħ +ĠpieniÄħ d +ĠpieniÄħd ze +ÑĢез иденÑĤ +ØŃ ØµØ± +ìĺ ¥ +à¹Ģย ืà¸Ńà¸Ļ +ĠÑĥ ни +ĠÑĥни веÑĢ +ĠÑĥнивеÑĢ Ñģ +ĠÑĥнивеÑĢÑģ иÑĤеÑĤ +ĠاÙĦر ØŃ +ĠاÙĦرØŃ ÙħÙĨ +ĠÑĤеÑħ нолог +ĠÑĤеÑħнолог ии +ìĹIJ ëĦĪ +ìĹIJëĦĪ ì§Ģ +Ġíķ Ń +ĠíķŃ ìĥģ +à¸ĺ า +à¸ĺา à¸ķุ +ĠEspañ ol +×ĵ×Ĵ ש +Ġêµ ī +Ġêµī ìŀ¥ +Ġêµīìŀ¥ íŀĪ +ĠÅĤ at +ĠÅĤat wo +Ġk á»ĭch +Ø¥ ز +إز اÙĦØ© +ĠдейÑģÑĤв ие +ĠsaÄŁ layan +สุà¸Ķ ยà¸Ńà¸Ķ +Ġzosta Äĩ +Ġdispon ÃŃvel +ïº į +ver ständ +verständ lich +tw or +twor zyÄĩ +ع جز +à¹Ģà¸Ĥ à¹īม +ยà¹Ī à¸Ńม +Ġstrat ég +Ġstratég ie +à¸ľà¸¥ à¹Ħมà¹ī +Ġê°ģ ì¢ħ +ĠÙħ ÙĪا +ĠÙħÙĪا ض +ĠÙħÙĪاض ÙĬع +اØŃ ØªØ¬ +اØŃتج اج +Ġ Ấ +ĠẤ n +×ŀ ×ŀש׾×Ķ +ĠÅŁek il +×ŀ ×Ĺ׾ +×ŀ×Ĺ׾ ×ķת +Ġ à¸ĺ +Ġà¸ĺ ัà¸Ļ +Ġà¸ĺัà¸Ļ วา +Ġà¸ĺัà¸Ļวา à¸Ħม +Ġìĭ¤ ìłľ +Ġìĭ¤ìłľ ë¡ľ +ì¤ij ìķĻ +ëįĶ ëĿ¼ +ĠÑĪ иÑĢ +ĠÑĪиÑĢ око +Ġsol ución +วาà¸ĩ à¹ģà¸ľà¸Ļ +×IJ×ķ×ĺ ×ķ×ŀ +×IJ×ķ×ĺ×ķ×ŀ ×ĺ×Ļ +ĠÑĢ еÑģÑĤ +ĠÑĢеÑģÑĤ оÑĢ +ĠÑĢеÑģÑĤоÑĢ ан +ëį ¸ +ÑĤ ÑĢад +ÑĤÑĢад и +ÑĤÑĢади ÑĨион +ÑĤÑĢадиÑĨион н +มะ à¹Ģรà¹ĩ +มะà¹Ģรà¹ĩ à¸ĩ +à¹Ĥ ส +Ġol masını +×ŀ×ķס ר +ĠоÑĤноÑĪ ении +Ġê°ĢëĬ¥ ìĦ± +Ġy uk +Ġyuk arı +ìĨ Ķ +ĠÑģ ÑĦ +ĠÑģÑĦ еÑĢе +Ġק ×ķפ +ãĤ± ãĥ¼ãĤ +ãĤ±ãĥ¼ãĤ Ń +âĢķ âĢķ +ĠاÙĦØ£ ÙĦÙħ +ĠاÙĦØ£ÙĦÙħ اÙĨÙĬ +Ả N +ת×ķ׼ ׳×Ļ×ķת +ĠÑģÑĥÑīеÑģÑĤв ÑĥеÑĤ +æĪij ãĢħ +ĠاÙĦص ادر +ĠTr á»įng +Ġа д +Ġад миниÑģÑĤ +ĠадминиÑģÑĤ ÑĢа +ĠадминиÑģÑĤÑĢа ÑĨи +ĠдÑĢÑĥг ими +Ñģп еÑĪ +عÙĦاÙħ ات +Ġа б +Ġаб Ñģол +ĠабÑģол ÑİÑĤ +ĠабÑģолÑİÑĤ но +ฤ à¸Ķู +é tr +étr anger +нÑı ÑĤи +нÑıÑĤи е +×¢ ×ķ׳ +×¢×ķ׳ ש +ĠÙĤ ائ +ĠÙĤائ ÙĦا +Ġм аÑģ +ĠмаÑģ ло +ãĥī ãĤ¤ +ãĥīãĤ¤ ãĥĦ +å¿ħè¦ģ ãģĮãģĤãĤĬãģ¾ãģĻ +×ŀ×ķ×ĸ ×Ļ×IJ +×ŀ×ķ×ĸ×Ļ×IJ ×ķף +ĠNgo ại +Ġkê nh +à¸ģาร à¸Ńà¸Ńà¸ģà¹ģà¸ļà¸ļ +×ŀ פק +×ŀפק ×ĵ +ÙħÙĨ از +ÙħÙĨاز ÙĦ +ë· ° +íĹ ¤ +ÙħÙĩ ارات +Ġpropri été +פ×Ĵ ×Ļש×Ķ +Ñĩ ÑĢ +ÑĩÑĢ еж +ÑĩÑĢеж ден +×Ķ ×ķצ×IJ×Ķ +ØŃÙĥ ÙĬÙħ +ĠíĻ Ī +ĠíĻĪ íİĺìĿ´ì§Ģ +åİ ³ +åݳ ãģĹãģĦ +×¢ ×ŀ×ĵ×Ķ +ĠAu ÃŁen +سÙĪ Ø¡ +ë¹ Ī +ĠÙĪ Ø® +ĠÙĪØ® اصة +ин ÑĤеÑĢ +инÑĤеÑĢ еÑģ +èĩ´ ãģĹãģ¾ãģĻ +Ġhük üm +à¹Ħà¸Ĥ มัà¸Ļ +Ġdav ran +Ġdavran Ä±ÅŁ +à¹Ģà¸ķ ียà¸ĩ +в ÑĢем +вÑĢем енно +à¹Ģà¸Ĺศ à¸ģา +à¹Ģà¸Ĺศà¸ģา ล +å¼ķ ãģ£ +å¼ķãģ£ è¶ĬãģĹ +×IJר ×ķ×Ĺ +×IJר×ķ×Ĺ ×ª +à¹Ģ วิ +à¹Ģวิ รà¹Į +à¸Ńยà¹Īาà¸ĩ รวà¸Ķà¹Ģรà¹ĩว +ĠìŬ íĸī +ĠÑĢан ÑĮ +ĠÑĢанÑĮ ÑĪе +Ġzob ow +Ġzobow iÄħ +ĠzobowiÄħ z +Ġ×ķ׼ ×ŀ×ķ×ijף +ĠاÙĦÙħ Ùĩ +ĠاÙĦÙħÙĩ ÙĨÙĬ +ãĤ¢ ãĤ¸ +ãĤ¢ãĤ¸ ãĤ¢ +ë°© ìĨ¡ +à¸Ńà¸Ńà¸ģ à¸ģำลัà¸ĩ +à¸Ńà¸Ńà¸ģà¸ģำลัà¸ĩ à¸ģาย +am éli +améli orer +å½ĵãģŁãĤĬ åīį +Ġreg elm +Ġregelm Ã¤ÃŁig +ãģĬ åĭ +ãģĬåĭ § +ãģĬåĭ§ ãĤģ +Ġm Æ°á»Ŀi +بر Ùħج +ĠNat ürlich +ĠD Å©ng +ĠاÙĦر جاÙĦ +Ġthé p +Ġol muÅŁtur +×ŀ×ķס ×Ļק×Ķ +f älle +주 íĥĿ +ĠاÙĦÙģ رص +Ġnaj wiÄĻks +ĠnajwiÄĻks zy +Ġça ÄŁ +ĠçaÄŁ rı +ì¸ ł +ĠvÃŃ ct +ĠvÃŃct ima +ĠÑģовеÑĢ ÑĪен +×Ķ×Ļ ×Ļת×Ļ +à¹Ģà¸Ķ ี +à¹Ģà¸Ķี à¹ĭ +à¹Ģà¸Ķีà¹ĭ ยว +ü yü +Ġд оп +Ġдоп олн +Ġдополн иÑĤелÑĮно +à¹ģà¸ķà¸ģà¸ķà¹Īาà¸ĩ à¸ģัà¸Ļ +Ġá l +Ġál bum +à¸Ľà¸£à¸°à¸Īำ à¸Ľà¸µ +ĠÑĦ едеÑĢ +ĠÑĦедеÑĢ алÑĮн +Ġobs ÅĤ +ĠobsÅĤ ugi +à¹Ģร ืà¹Ī +à¹Ģรืà¹Ī à¸Ńย +à¹Ģรืà¹Īà¸Ńย à¹Ĩ +ëģ Į +Ġngh ìn +ĠBaÅŁkan lıģı +تأ سÙĬ +تأسÙĬ س +Ġ×ij×ij ×ķקר +Ġ×¢×ij×ķ×ĵ ×ķת +Ġبص ÙĪرة +ãĤıãģij ãģ§ãģ¯ãģªãģĦ +führ er +ãĤ¹ ãĤŃ +ãĤ¹ãĤŃ ãĥ« +ĠاÙĦÙĤ ض +ĠاÙĦÙĤض ÙĬØ© +Ġдолж ноÑģÑĤ +Ùģار ÙĤ +Ġcomeç ou +Ġorganis é +Ġxu ân +ĠÑģообÑī аеÑĤ +ĠпÑĢи д +ĠпÑĢид еÑĤÑģÑı +TÃľ RK +ãĥ¬ ãĥ¼ãĤ·ãĥ§ãĥ³ +Kh ông +است Ùģ +استÙģ ادة +ä¸ĬãģĮ ãģ£ãģ¦ +Ġum ie +Ġumie jÄĻ +ĠumiejÄĻ tn +ĠumiejÄĻtn oÅĽci +ëĤ ¸ +à¹Ģà¸Ļ à¸Ńรà¹Į +×ĵ×ķ ×ķ×Ĺ +ÃŃs imo +I ÃĬ +IÃĬ N +Ġalcan ç +Ġ à¸ķุ +Ġà¸ķุ ลา +Ġà¸ķุลา à¸Ħม +ש׾ ×ĺ×ķף +Ġél è +Ġélè ves +ĠÄij u +ĠÄiju á»ķi +ĠØ£ Ùģ +ĠØ£Ùģ رÙĬ +ĠØ£ÙģرÙĬ ÙĤÙĬ +ĠØ£ÙģرÙĬÙĤÙĬ ا +ãĤĴæİ¢ ãģĻ +ĠпÑĢед ложениÑı +ج اد +ĠÑħоÑĤ ÑĮ +Ñģ ал +Ñģал он +à¸Ľà¸£à¸° à¹Ģม +à¸Ľà¸£à¸°à¹Ģม ิà¸Ļ +ãĤŃ ãĥĥãĥģ +ãĤŃãĥĥãĥģ ãĥ³ +×ij×ĵ×Ļק ×ķת +Ġch ù +Ġchù a +ÐĴ иде +ÐĴиде о +иÑĢов ка +ĠÑħоÑĤ иÑĤе +Ġspéc ifique +รส à¸Ĭาà¸ķิ +è¾¼ ãĤĵãģł +伸 ãģ³ +×Ķצ׾ ×Ĺת +ãģ©ãģ® ãĤĪãģĨãģ« +سع ادة +Ġл ид +Ġлид еÑĢ +ม à¸ĩ +มà¸ĩ à¸Ħล +ØŃ Ø§ÙħÙĦ +หล ุà¸Ķ +à¸Ńยà¹Īาà¸ĩ à¸ķà¹Īà¸Ń +à¸Ńยà¹Īาà¸ĩà¸ķà¹Īà¸Ń à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ +ãģķãģĽãģ¦ éłĤ +تس ÙĪÙĬ +تسÙĪÙĬ ÙĤ +ĠaÅŁaģı d +ĠaÅŁaģıd aki +ĠÑĨ елÑĮ +ĠÑĨелÑĮ Ñİ +ĠAra ÅŁtırma +à¸Ĥัà¸ļ รà¸ĸ +Ùĩ Ø°Ùĩ +ลà¸ĩ à¸Ĺะ +ลà¸ĩà¸Ĺะ à¹Ģà¸ļ +ลà¸ĩà¸Ĺะà¹Ģà¸ļ ียà¸Ļ +تÙĥ اÙħÙĦ +Ġc io +Ġcio è +ãģ¦ ãģĬãģı +ĠاÙĦصØŃ ÙģÙĬ +ĠíĬ¹ ìłķ +полн иÑĤÑĮ +ãĤĵ ãģĺãĤĥãģªãģĦ +ãĤĵãģĺãĤĥãģªãģĦ ãģĭ +ĠاÙĦج Ùĩ +ĠاÙĦجÙĩ ات +ĠÑĥÑģпеÑĪ но +Ġв ок +Ġвок ÑĢÑĥг +ĠÑģиÑĤÑĥ аÑĨиÑı +Ġ×Ķ×IJ ×ŀר +Ġ×Ķ×IJ×ŀר ×Ļק +Ġ×Ķ×IJ×ŀר×Ļק ×IJ×Ļ +×ŀ ×Ĵ×ĸ +×ŀ×Ĵ×ĸ ×Ļף +Ġак ÑĤÑĥ +ĠакÑĤÑĥ алÑĮн +é ta +éta is +Ġmog ÅĤa +ĠÑĤоÑĩ ки +Ġ×ŀ×Ķ ×ŀ×¢ +Ġ×ŀ×Ķ×ŀ×¢ ×¨×Ľ×ª +มี à¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺà¸´à¸łà¸²à¸ŀ +×Ļר ×Ļ×ĵ×Ķ +×Ĵר ×ŀ׳ +×Ĵר×ŀ׳ ×Ļ×Ķ +Ġг лав +Ġглав ное +Ġ미 ëŀĺ +Ġ׳׼ ×ķ׳×Ķ +ĠÙĪ Ø·ÙĨÙĬ +op port +opport unitÃł +Ġh ủy +ĠÙĦ تØŃ +ĠÙĦتØŃ ÙĤÙĬÙĤ +Ġó rg +Ġórg ão +ãĤ¹ ãĥĶ +ãĤ¹ãĥĶ ãĥ¼ãĥī +Ġön ü +Ġönü ne +Ùħع اÙħÙĦ +ש×ŀ ×Ļר×Ķ +ĠвеÑģÑĮ ма +ĠwiÄĻks zo +ĠwiÄĻkszo ÅĽÄĩ +Ġاست راتÙĬج +ĠاستراتÙĬج ÙĬØ© +ĠÙģ Ø¥ +ĠÙģØ¥ ذا +à¹Ģà¸Ĭืà¹Īà¸Ń ม +à¹Ģà¸Ĭืà¹Īà¸Ńม à¸ķà¹Īà¸Ń +Ġ׾ פר +Ġ׾פר ×ĺ×Ļ×Ŀ +Ùħض ÙĬ +ĠGer çek +Ġçocuk ların +ÙĪØ« ائÙĤ +ĠÙħساء Ùĭ +Ġunterstüt zt +Ġpré st +Ġprést amo +ĠÐłÐ°Ð· меÑĢ +ĠÅŁ eker +Ġsé culo +×ij×Ķ ×Ļר +Ø´Ùĩ ÙĪر +Ġ à¸Ńีà¸ģ +Ġà¸Ńีà¸ģ à¸Ĺัà¹īà¸ĩ +Ġlleg ó +à¸¨à¸´à¸¥à¸Ľ ะ +æĪij ãģĮ +æĪijãģĮ 家 +ع ÙĤÙĪ +عÙĤÙĪ بات +ĠF älle +Ġs ÅĤuż +ĠsÅĤuż b +ĠاÙĦØŃÙĤ ÙĪÙĤ +Ġпл иÑĤ +Ġи ноÑģÑĤ +ĠиноÑģÑĤ ÑĢан +ĠиноÑģÑĤÑĢан н +à¹ĥà¸Ļ à¸Ĥà¸ĵะà¸Ĺีà¹Ī +ãĤ« ãĥĨ +ãĤ«ãĥĨ ãĤ´ +ãĤ«ãĥĨãĤ´ ãĥª +à¸Ńิ ส +à¸Ńิส ระ +à¹Ģà¸ľà¸¢ à¹ģ +à¹Ģà¸ľà¸¢à¹ģ à¸ŀร +à¹Ģà¸ľà¸¢à¹ģà¸ŀร à¹Ī +ãģĬ ãģĦ +ãģĬãģĦ ãģĹãģĦ +است ÙĤÙĦ +استÙĤÙĦ اÙĦ +تØŃ Ø¶ +تØŃض ÙĬر +åĬ© ãģij +Ùħر اÙģÙĤ +Ġ×ĵ ×ķר +Ġ×ĵ×ķר ש +×ŀת×Ļ ×Ļ×Ĺס +ס ×Ļ׼ +ס×Ļ׼ ×ķ×Ŀ +íĮĮ íĬ¸ +Ġwy ÅĽ +ĠwyÅĽ w +ĠwyÅĽw iet +ĠwyÅĽwiet l +ĠاÙĦاÙĨ ساÙĨ +ĠStra ÃŁen +ï¼ ¬ +ãģ« åŁº +ãģ«åŁº ãģ¥ +Ġcap ÃŃtulo +ลุ ย +Ġ×Ķ×ŀק צ×ķ×¢×Ļ +ãģĤãĤĭ ç¨ĭ度 +á» ¢ +ĠاÙĦ ÙĦا +ĠاÙĦÙĦا زÙħØ© +æķĻ ãģĪ +Ġרש ×IJ×Ļ +з ав +зав иÑģ +завиÑģ им +à¸Ľà¸±à¸Ī à¸Īัย +à¹Ģà¸ĭ ล +à¹Ģà¸ĭล ลà¹Į +Ġdiffé rence +ĠAlt ın +Ġк ÑĢай +ĠкÑĢай не +Ġз ло +Ġgün ümüz +Ġн аÑĤÑĥÑĢ +ĠнаÑĤÑĥÑĢ алÑĮн +×Ĵ×ķ׾ ש×Ļ×Ŀ +Ġк аÑĤегоÑĢ +ĠкаÑĤегоÑĢ ии +Ġз нак +à¸ģà¹Īà¸Ńà¸Ļ หà¸Ļà¹īา +à¸ģà¹Īà¸Ńà¸Ļหà¸Ļà¹īา à¸Ļีà¹ī +ĠÙħÙĨ ت +ĠÙħÙĨت خب +ãĥĽ ãĥ¼ãĥ« +Ġе вÑĢо +ส ว +สว ม +ĠìľĦ ìĽIJ +ĠìľĦìĽIJ ëĭĺ +ĠاÙĦØŃ ÙĪØ« +ĠاÙĦØŃÙĪØ« ÙĬ +ĠÑģодеÑĢж иÑĤ +ãĥķãĤ¡ ãĥĥãĤ·ãĥ§ãĥ³ +Ġ à¸ģัà¸Ļ +Ġà¸ģัà¸Ļ ย +Ġà¸ģัà¸Ļย ายà¸Ļ +ãĤª ãĥª +ãĤªãĥª ãĤ¸ +ãĤªãĥªãĤ¸ ãĥĬãĥ« +Ġб ÑĢенд +ãĤĴæĮģ ãģ£ãģ¦ãģĦãĤĭ +Ġinvers ión +Ġê° ĸ +Ġê°ĸ ê³ł +Ġnov itÃł +ê´Ģ ê´ij +Ġà¸ŀ ฤษ +Ġà¸ŀฤษ à¸łà¸² +Ġà¸ŀà¸¤à¸©à¸łà¸² à¸Ħม +×ķר ×Ĺ×Ļ×Ŀ +׼׾ ×ķ׾ +Ġng ạc +×Ļ ×Ļש +×Ļ×Ļש ×ķ×ij +f äll +fäll ig +ĠÑĤÑĢеб ÑĥеÑĤÑģÑı +Ġcar á +Ġcará cter +Ġprinc ÃŃpio +ĠÅĤ az +ĠÅĤaz ien +ĠÅĤazien k +Ġgi ãn +ÑģÑĤÑĢа ива +Ùħس اب +Ùħساب ÙĤØ© +à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩ à¸Ķืà¹Īม +ترÙĥ ÙĬب +vol ução +ĠÐŁ оÑĩ +ĠÐŁÐ¾Ñĩ ем +ĠÐŁÐ¾Ñĩем Ñĥ +казал оÑģÑĮ +ĠпÑĢимен ениÑı +à¹Ģà¸Ĺ ียม +íĮ Ķ +à¸Ĥà¹īà¸Ń à¹Ģสà¸Ļà¸Ń +à¸Ľà¸±à¸į à¸įา +Ġоб ÑĥÑĩ +ĠобÑĥÑĩ ениÑı +ĠÑģеÑĢ и +ĠÑģеÑĢи ал +Ġingl és +ĠÙĦ Ùĥرة +Ġ×ĺ ׾ +Ġ×ĺ׾ פ×ķף +Ġìł ij +Ġìłij ê·¼ +×IJ ×ķ×Ĵ +×IJ×ķ×Ĵ ×ķס +×IJ×ķ×Ĵ×ķס ×ĺ +ĠболÑĮÑĪ ое +ĠÐļон еÑĩно +×¢×Ļת ×ķ׳ +×¢×Ļת×ķ׳ ×IJ×Ļ +Ġкноп к +Ġз н +Ġзн аÑĤÑĮ +ĠÄij á»± +ĠÄijá»± ng +вл аж +влаж н +×ŀ ×Ļ×ĺ×ij +ãĤ¬ ãĤ¤ +ãĤ¬ãĤ¤ ãĥī +........ .. +Ġà¸ģ ุม +Ġà¸ģุม à¸łà¸²à¸ŀ +Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀ ัà¸Ļ +Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļ à¸ĺ +Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļà¸ĺ à¹Į +be z +bez pieczeÅĦst +bezpieczeÅĦst w +ãĥijãĥij æ´» +ع اط +عاط Ùģ +ĠÄij áºŃm +Ġз ÑĢ +ĠзÑĢ ениÑı +Ġbor ç +Ġнед ел +Ġнедел Ñİ +Ġh á»ı +Ġhá»ı ng +ìŀ¥ ìķł +ìŀ¥ìķł ìĿ¸ +ĠاÙĦع ÙĦاÙĤØ© +Ġíģ ¬ +Ġíģ¬ ê²Į +à¹Ħร à¹Ī +à¸ļา à¸Ķ +à¸ļาà¸Ķ à¹Ģà¸Īà¹ĩà¸ļ +à¸Ŀ รั +à¸Ŀรั à¹Īà¸ĩ +à¸Ŀรัà¹Īà¸ĩ à¹Ģศ +à¸Ŀรัà¹Īà¸ĩà¹Ģศ ส +ר ×¢×Ļ +רע×Ļ ×ķ׳×ķת +Ġë Į +ĠëĮ ĵ +ĠëĮĵ ê¸Ģ +Ġnaj b +Ġnajb li +Ġnajbli ż +Ġnajbliż sz +ĠиÑģполÑĮз ÑĥеÑĤÑģÑı +Ġcient ÃŃf +ĠcientÃŃf ico +×¢ ×ŀק +Ġg ợi +Ø´ ØŃÙĨ +ĠÅĽ m +ĠÅĽm ier +ĠÅĽmier ci +à¸Ħาสิà¹Ĥà¸Ļ à¸Ńà¸Ńà¸Ļà¹Ħลà¸Ļà¹Į +×Ĺש×ij ת×Ļ +Ġn ingu +Ġningu ém +è¾¼ ãĤģ +ãģ · +ĠÑĥ г +ĠÑĥг ол +ï½ ° +פת ×Ļ×Ĺ +פת×Ļ×Ĺ ×ª +Ġ×Ķר×IJש ×ķ׳×Ļ×Ŀ +p ósito +ãĤŃ ãĥ¬ãĤ¤ +ãģ© ãģĵãĤį +à¹Ģà¸Ĺà¹Īา à¹Ħ +à¹Ģà¸Ĺà¹Īาà¹Ħ หร +à¹Ģà¸Ĺà¹Īาà¹Ħหร à¹Ī +ĠинÑĤеÑĢ ÑĮеÑĢ +ĠØŃ Ø§Ø¬ +ĠØŃاج Ø© +สี à¸Ĥาว +ìĸ ¼ +Ġn á»Ļ +Ġná»Ļ p +ĠÃŃ nd +ĠÃŃnd ice +สำ รวà¸Ī +Ġкажд ой +Ġhot éis +Ġnast ÄĻ +ĠnastÄĻ pn +Ġ×Ķק ×ķ×ĵ +Ġ×Ķק×ķ×ĵ ×Ŀ +פ ×ķפ +פ×ķפ ×ķ׾ +פ×ķפ×ķ׾ ר×Ļ +вÑĪ ей +ãĤ·ãĥ³ ãĥĹ +ãĤ·ãĥ³ãĥĹ ãĥ« +ĠzdjÄĻ Äĩ +ĠгÑĢÑĥпп а +Ġпом еÑī +ĠпомеÑī ениÑı +ãģ©ãģĨ ãģĦãģĨ +ĠиÑģп ÑĭÑĤа +Ġog ÅĤ +ĠogÅĤ os +ĠogÅĤos zen +ĠogÅĤoszen i +สรà¹īาà¸ĩ สรร +สรà¹īาà¸ĩสรร à¸Ħà¹Į +à¸ŀร รà¸ĵ +Ġçık Ä±ÅŁ +ĠÑĩаÑģÑĤ ноÑģÑĤи +Ġ×ķ ×Ļ×ķתר +ç¶ļãģį ãĤĴ +ç¶ļãģįãĤĴ èªŃ +ç¶ļãģįãĤĴèªŃ ãĤĢ +à¸ģร ั +à¸ģรั ม +г ÑĢаÑĦ +Ġв лад +Ġвлад елÑĮ +ĠвладелÑĮ ÑĨ +Ġistedi ÄŁ +ĠistediÄŁ iniz +×ij׾ ×¢ +×ij×ľ×¢ ×ĵ×Ļ +ÙħÙĪ اÙģ +ÙħÙĪاÙģ ÙĤØ© +Ġ×Ļ ×ķר +Ġ×Ļ×ķר ק +ãĤ«ãĥ¼ãĥī ãĥŃãĥ¼ãĥ³ +ĠاÙĦÙħØ´ ÙĥÙĦ +ĠاÙĦÙħØ´ÙĥÙĦ Ø© +ĠêµŃ íļĮ +ס פ×ĺ +ספ×ĺ ×ŀ +ספ×ĺ×ŀ ×ijר +Ġìĸ´ ëłµ +Ùĥ اÙħ +ÙĥاÙħ ÙĬرا +sch lü +schlü sse +ĠØ« ÙĨ +ĠØ«ÙĨ ائÙĬ +ìī ½ +ĠÐŀ Ñģоб +ĠÐŀÑģоб енно +Ġин веÑģÑĤи +ĠинвеÑģÑĤи ÑĨи +اØŃ ØªÙħ +اØŃتÙħ اÙĦ +E Äŀ +EÄŀ Ä° +íķĺ ê²łëĭ¤ +Ġ×IJ ×ijר×Ķ +Ġ×IJ×ijר×Ķ ×Ŀ +Ġ×ij×Ĺ ×Ļ׳×Ŀ +Ø£ ÙĪض +Ø£ÙĪض اع +Ġdé l +Ġdél ai +Ġ×IJ×ķ×Ķ ×ij×Ļ×Ŀ +ĠÑģо Ñħ +ĠÑģоÑħ ÑĢ +ĠÑģоÑħÑĢ ани +ĠдоÑģÑĤ иж +ĠдоÑģÑĤиж ени +สิà¹Īà¸ĩ à¹ģ +สิà¹Īà¸ĩà¹ģ วà¸Ķ +สิà¹Īà¸ĩà¹ģวà¸Ķ ล +สิà¹Īà¸ĩà¹ģวà¸Ķล à¹īà¸Ńม +ĠاÙĦÙħ باشر +ĠÑĦ иг +ĠÑĦиг ÑĥÑĢ +мож ем +׾×ŀ×Ļ×ĵ ×Ķ +Ġcin é +Ġciné ma +Ġb ada +Ġbada ÅĦ +جب ÙĩØ© +Ġд еп +Ġдеп ÑĥÑĤ +ĠдепÑĥÑĤ аÑĤ +Ġdist ância +ĠاÙĦÙħ عار +ĠاÙĦÙħعار ضة +thè se +ü nc +ünc ü +Ġдан ного +ĠBel gi +ĠBelgi ë +Ġ×ij ×ijק +Ġ×ij×ijק ש×Ķ +ย à¹Īาà¸Ļ +Ġsol ução +Ġ×Ķצ ×ĺר +Ġ×Ķצ×ĺר פ×ķ +ĠØ£ÙĨ ØŃ +ĠØ£ÙĨØŃ Ø§Ø¡ +Ġد ÙħØ´ +ĠدÙħØ´ ÙĤ +มั à¹ī +มัà¹ī ย +Ùħ غرب +است عÙħاÙĦ +ĠS ÅĤow +ĠëıĻ ìĭľ +ĠëıĻìĭľ ìĹIJ +ĠÑģ оÑģ +ĠÑģоÑģ ед +ì²Ń ìĨĮ +ì²ŃìĨĮ ëħĦ +Ġг ÑĢаÑĦ +ĠгÑĢаÑĦ ик +Ġìŀij ìĿĢ +Ġyet i +Ġyeti ÅŁtir +ĠìĿ´ê²ĥ ìĿ´ +ห à¹Īาà¸ĩ +Ø¥ ÙħÙĥاÙĨ +Ø¥ÙħÙĥاÙĨ ÙĬØ© +است عراض +ÙħØ® در +ĠÑĩ ÑĥÑĤÑĮ +Ùħ دÙĬر +ÙħدÙĬر ÙĬØ© +Ġà¹Ģม ษ +Ġà¹Ģมษ ายà¸Ļ +Ġм еÑħ +ĠмеÑħ аниз +ĠмеÑħаниз м +ĠÑģ Ñĥм +ĠÑģÑĥм мÑĥ +Ġv ö +Ġvö ll +Ġvöll ig +Ġд ÑĢÑĥз +ĠдÑĢÑĥз ÑĮÑı +ãĤĴåĪ©çĶ¨ ãģĹãģ¦ +à¸ļรร à¸Īุ +po życz +×ŀש ׼ +×ŀש׼ ×ł×ª +×ŀ×©×Ľ×ł×ª ×IJ +Ġeuropé en +Ġpropri é +Ġproprié taire +Ġkh ấu +ãģĦãģŁãģł ãģijãĤĭ +Ġtec rü +Ġtecrü be +×Ķ ×ij +×Ķ×ij ׳×Ķ +Ġcu Ì +ĠcuÌ ī +ĠcuÌī a +×IJ ×ķ×ķ +×IJ×ķ×ķ ×Ļר×Ķ +Ġ׼×ķ׾ ×ķ +U lus +Ulus lararası +Ġ׳ ×ķת +Ġ׳×ķת ף +ãģ« åIJij +ãģ«åIJij ãģijãģ¦ +ë¹ Ľ +à¸Ĺ ัà¸ģษ +à¸Ĺัà¸ģษ ะ +س ÙĤÙĪ +سÙĤÙĪ Ø· +Ġв н +Ġвн еÑĪ +ĠвнеÑĪ не +Ġur z +Ġurz ÄĻd +Ġá mb +Ġámb ito +à¸Ń à¸ĺิ +à¸Ńà¸ĺิ à¸ļาย +Ġ ÅĤad +ĠÅĤad n +ê±´ ì¶ķ +wód zt +wództ w +Ġquest ões +Ġש ק +Ġשק ×Ļ×ij׾ +Ġmiejsc owoÅĽci +Ġв ал +Ġвал ÑİÑĤ +hä user +หà¸Ļ à¸Ńà¸ĩ +ãģ¨ åħ± +ãģ¨åħ± ãģ« +ãĥı ãĥ¼ãĥī +Ġê°ľ ìµľ +ĠоÑģнов ном +Ġм ÑıÑģ +اع ت +اعت ÙĤاÙĦ +สà¸ĸ ิ +สà¸ĸิ à¸ķิ +N gu +Ngu á»ĵn +ĠÙħ جÙĦ +ĠÙħجÙĦ Ø© +à¹ģà¸Ĥ à¸Ļ +ĠاÙĦÙĦÙĬ بÙĬ +פע×Ļ׾ ×ķ×Ļ×ķת +Ġ×Ķר פ×ķ×IJ×Ļ +פר ×ķפ +פר×ķפ ×Ļ׾ +ק ׾×IJ +ק׾×IJ ס×Ļ +Ùĥت Ø´Ùģ +ãģ«ãģª ãģ£ãģ¦ãģĹãģ¾ãģĨ +à¹Ģà¸Ħล à¹ĩà¸Ķ +à¹Ģà¸Ħลà¹ĩà¸Ķ ลัà¸ļ +Ġì» ´ +Ġì»´ íĵ¨ +Ġì»´íĵ¨ íĦ° +Ġ×Ĺ×Ļ ×ķ×ij×Ļ +Ġnä m +Ġnäm lich +åij¼ ãģ° +åij¼ãģ° ãĤĮ +ĠÑĢ ол +ĠÑĢол и +Ġspécial isé +à¸Ļ วัà¸ķ +à¸Ļวัà¸ķ à¸ģรรม +ÙĨص ÙĪص +пеÑĢ ед +пеÑĢед аÑĩ +thè que +Ġר×IJ ×Ļת×Ļ +ãĥĢ ãĤ¦ãĥ³ +ãĤı ãģĭ +ãĤıãģĭ ãģ£ãģ¦ +беÑĢ еж +ĠÑģ ек +ĠÑģек ÑĢ +ĠÑģекÑĢ еÑĤ +ĠпоÑģÑĤоÑıн н +à¸Ĥà¸Ļ สà¹Īà¸ĩ +Ġm ük +Ġmük em +Ġmükem mel +еÑĤ еÑģÑĮ +ĠاÙĦسÙĨ ÙĪات +ĠìłĦ íĺĢ +Ġ×Ķ×ŀק ×ķר×Ļ +Ġmü d +Ġmüd ah +Ġmüdah ale +Ġwy b +Ġwyb ór +Ġtend ência +Ø¥ دار +إدار ÙĬØ© +Ġunterstüt zen +ת ×ijר +ת×ijר ר +Ġdi á +Ġdiá logo +ĠÃĸ nce +ĠÃĸnce ki +ãĤ¹ãĥĿ ãĥĥãĥĪ +ëĦ £ +ĠG eli +ĠGeli ÅŁ +ãĤĴ éĢļ +ãĤĴéĢļ ãģĹãģ¦ +ĠFuÃŁ ball +Ġsal ari +Ġsalari é +ĠпÑĢодÑĥк ÑĤов +صÙģ ÙĤØ© +รว à¸ļ +รวà¸ļ รวม +à¹ĥà¸Ļ à¸IJาà¸Ļ +à¹ĥà¸Ļà¸IJาà¸Ļ ะ +Ġkay na +Ġkayna ģı +Ġìŀij íĴĪ +ĠвÑĭ ÑĢаж +ĠвÑĭÑĢаж ен +ĠÑģÑĤ еп +ĠÑģÑĤеп ени +ĠاÙĦÙħ ÙĪجÙĪد +ĠاÙĦÙħÙĪجÙĪد Ø© +ล à¹īม +Ġnaj czÄĻ +ĠnajczÄĻ ÅĽcie +ĠnajczÄĻÅĽcie j +Ġz wy +Ġzwy k +Ġzwyk ÅĤ +Ġê·¸ëłĩ ì§Ģ +à¸ģระ à¸Ī +à¸ģระà¸Ī าย +Ġëĭ µ +Ġëĭµ ë³Ģ +ĠÑĢе ак +ĠÑĢеак ÑĨи +ĠÅĽwie ż +ĠÑģÑĤоим оÑģÑĤи +ÙħÙĨ اÙĤ +ÙħÙĨاÙĤ Ø´ +ÙħÙĨاÙĤØ´ Ø© +ĠÑħоÑĩ Ñĥ +ãĥľ ãĥ¼ãĥī +Ġróż nic +Ġк ÑĢÑĭ +ĠкÑĢÑĭ ÑĪ +âľ ĵ +ãĤ³ãĥ³ ãĥĨãĥ³ +ãĤ³ãĥ³ãĥĨãĥ³ ãĥĦ +ĠпÑĢед поÑĩ +×ŀר ×ij×Ļת +ĠØ´ Ùĥ +ĠØ´Ùĥ را +Ġд ал +Ġдал ек +Ġдалек о +بر ÙĬØ· +برÙĬØ· اÙĨÙĬا +ع ÙĨا +عÙĨا ÙĬØ© +ĠÑĢаÑģÑģ каз +ĠÑĢаÑģÑģказ Ñĭва +Ø£ ÙĦÙĪ +Ø£ÙĦÙĪ اÙĨ +æĮģ ãģ£ãģ¦ +æĮģãģ£ãģ¦ ãģĦ +Ùħباد ئ +×Ķ ×¢×ijר +×Ķ×¢×ijר ת +Ġyay ı +Ġyayı ml +Ġyayıml a +m át +mát icos +à¸ģ ัà¸ĩ +à¸ģัà¸ĩ วล +Ġ׾ פת +Ġ×ľ×¤×ª ×ķ×Ĺ +à¸ŀฤ à¸ķิ +à¸ŀฤà¸ķิ à¸ģรรม +í Ĥ¬ +Ġок ÑĢÑĥг +Ġ×ŀצ ×ķ×ķ×Ķ +ÐĽ ени +ÐĽÐµÐ½Ð¸ н +ĠTri á»ģu +ãĤ³ãĥŁ ãĥ¥ +ãĤ³ãĥŁãĥ¥ ãĥĭ +ãĤ³ãĥŁãĥ¥ãĥĭ ãĤ± +ãĤ³ãĥŁãĥ¥ãĥĭãĤ± ãĥ¼ãĤ·ãĥ§ãĥ³ +Ùĥ ÙĨÙĬ +ÙĥÙĨÙĬ سة +ãĤĴ ä¸Ńå¿ĥ +ãĤĴä¸Ńå¿ĥ ãģ« +ĠmiÄĻd z +ĠmiÄĻdz yn +ĠmiÄĻdzyn ar +ĠmiÄĻdzynar od +ĠmiÄĻdzynarod ow +ÙĦ ÙĨ +ÙĦÙĨ دا +بر Ø´ +برش ÙĦÙĪÙĨ +برشÙĦÙĪÙĨ Ø© +à¸ģระ à¸ķุ +à¸ģระà¸ķุ à¹īà¸Ļ +Ġg ı +Ġgı da +à¸Ľà¸£à¸° à¸Ĺัà¸ļ +à¸Ľà¸£à¸°à¸Ĺัà¸ļ à¹ĥà¸Ī +Ġë¶Ī 구 +Ġë¶Ī구 íķĺê³ł +ĠÙĨ Ø· +ĠÙĨØ· اÙĤ +ĠÐľ ожеÑĤ +Pr äs +Präs ident +ĠÑģк оÑĢ +ĠÑģкоÑĢ оÑģÑĤÑĮ +Ġ×Ķ×ij ×ķקר +еÑħ аÑĤÑĮ +Ġg ạo +Ġש×IJ ×Ļ׳×Ŀ +Ġ×ij׳ ×ķ×Ĵ +Ġ×ij׳×ķ×Ĵ ×¢ +Ġо пиÑģание +Ġucz ni +Ġuczni ów +à¹Ģà¸Ń à¹ĩà¸Ļ +Ġت Ø´ +Ġتش رÙĬÙĨ +Ġnh ãn +ë¹ ¨ +Ġcaract ère +×¢ ׾×Ļ +×¢×ľ×Ļ ×Ļ×Ķ +楽ãģĹ ãĤģãĤĭ +ĠÑģ аÑħ +ĠÑģаÑħ аÑĢ +дÑĥм аÑĤÑĮ +ĠÐĴоз можно +ص ÙĬاÙĨ +صÙĬاÙĨ Ø© +öm ür +ส ล +สล à¹ĩ +สลà¹ĩ à¸Ń +สลà¹ĩà¸Ń à¸ķ +ë¡ ¯ +Ġth ói +gr Ã¶ÃŁe +Ġksi ÄĻ +ĠksiÄĻ g +ĠÑĢ ом +ĠÑĢом ан +ÙĤ اسÙħ +×ŀ×ij ×ķ×Ĵ +×ŀ×ij×ķ×Ĵ ר×Ļ×Ŀ +bes ch +besch äft +beschäft ig +×Ķצע ×Ķ +ĠÃģ rea +ĠзаÑıв к +Ä ¹ +ĠлÑİб ого +Ġ ม +Ġม à¸ģร +Ġมà¸ģร าà¸Ħม +ÑĦ из +ÑĦиз иÑĩеÑģк +ин ÑĦ +инÑĦ ек +инÑĦек ÑĨи +اÙĦ Ø· +اÙĦØ· ائÙģ +Ġкол л +Ġколл екÑĤив +ез жа +Ġس بØŃ +ĠسبØŃ Ø§ÙĨ +ĠسبØŃاÙĨ Ùĩ +sch lä +schlä ge +Ġд и +Ġди аг +Ġдиаг ноÑģÑĤ +ĠоÑĤмеÑĤ иÑĤÑĮ +Т Ь +ĠاÙĦ در +ĠاÙĦدر اسÙĬ +עצ ×ŀ +עצ×ŀ ×IJ×ķת +Ġdém arch +Ġdémarch e +Ġ×ĺ ×ķ×¢ +Ġ×ĺ×ķ×¢ ף +Ġfuncion ários +á» µ +׾ ׼×IJ +׾׼×IJ ×ķר×Ķ +à¸ĭ à¹Ī +à¸ĭà¹Ī à¸Ńม +ĠÑĩ Ñĥв +ĠÑĩÑĥв ÑģÑĤво +âĸ ¼ +п ÑĥÑī +пÑĥÑī ен +Ġм еÑĢ +ĠмеÑĢ оп +ĠмеÑĢоп ÑĢи +ĠмеÑĢопÑĢи ÑıÑĤиÑı +Ġu çu +Ġuçu ÅŁ +ãĤĴåĪ©çĶ¨ ãģĻãĤĭ +a ÄŁ +aÄŁ lı +ìĺĪ ìĪł +à¹ģ ยà¹Ī +ĠاÙĦÙĥ Ùħ +ĠاÙĦÙĥÙħ بÙĬ +ĠاÙĦÙĥÙħبÙĬ ÙĪتر +ت ÙĪÙĬ +تÙĪÙĬ تر +à¹Ģà¸Ĭ ีà¹Īยว +à¹Ģà¸Ĭีà¹Īยว à¸Ĭา +à¹Ģà¸Ĭีà¹Īยวà¸Ĭา à¸į +á» Ķ +Ġhi ếm +ذا Ùĥرة +Ġ×Ķ×ŀ×Ļ ×ķ×Ĺ×ĵ +ĠìĪ ľ +ĠìĪľ ê°Ħ +ĠK ı +ĠKı sa +Ġgele ceÄŁi +пÑĢо ÑĦеÑģÑģиона +пÑĢоÑĦеÑģÑģиона л +Ġog ó +Ġogó le +ĠgÅĤ ów +ĠgÅĤów ne +ĠÑģÑĤ илÑĮ +×IJ פ׾ +×IJפ׾ ×Ļק +×IJפ׾×Ļק צ×Ļ×Ķ +สม ารà¹Į +สมารà¹Į à¸Ĺ +สมารà¹Įà¸Ĺ à¹Ĥà¸Ł +สมารà¹Įà¸Ĺà¹Ĥà¸Ł à¸Ļ +Ġth ánh +ÐŁ од +ÐŁÐ¾Ð´ ÑĢоб +ÐŁÐ¾Ð´ÑĢоб нее +ĠاÙĦت ÙĪÙĨ +ĠاÙĦتÙĪÙĨ سÙĬ +Ġbah çe +à¹ģà¸ģà¹ī à¸Ľà¸±à¸įหา +é ducation +eu rop +europ ä +europä ische +ĠK si +ĠKsi ÄĻ +ĠëĦ ĺ +ĠëĦĺ ìĸ´ +Ġv üc +Ġvüc ud +Ġyay g +Ġyayg ın +Ġnie kt +Ġniekt óry +Ġniektóry ch +ãģŃ ãģĩ +Ġк аж +Ġкаж еÑĤÑģÑı +к аж +каж еÑĤ +ĠاÙĦ دÙĬÙħÙĤرا +ĠاÙĦدÙĬÙħÙĤرا Ø· +ĠاÙĦدÙĬÙħÙĤراط ÙĬØ© +æŃ © +æŃ© ãģĦãģ¦ +Ġv az +Ġvaz ge +Ġvazge ç +Ġмин ималÑĮ +ĠминималÑĮ н +ãĥij ãĤ¿ +ãĥijãĤ¿ ãĥ¼ãĥ³ +Ġë Ĭ +ĠëĬ IJ +ĠëĬIJ ëĤĮ +ãģ¡ ãĤĩãģĨ +ãģ¡ãĤĩãģĨ ãģ© +Ġ à¸ģร +Ġà¸ģร à¸ģà¸İ +Ġà¸ģรà¸ģà¸İ าà¸Ħม +تج دÙĬد +ĠØ´ اÙħÙĦ +หลัà¸ģ à¸IJาà¸Ļ +ĠмаÑĢ ÑĪ +ĠмаÑĢÑĪ ÑĢÑĥÑĤ +Ġv ÃŃt +ĠvÃŃt ima +Ġquiz á +ay gı +×ĵ×ijר ×Ļ×ķ +Ġиз д +Ġизд ели +Ġиздели Ñı +п ла +пла Ñĩ +плаÑĩ ива +ä»» ãģĽ +Ġéquip é +ä¹ħ ãģĹãģ +ä¹ħãģĹãģ ¶ +ä¹ħãģĹãģ¶ ãĤĬ +Ġк аÑĤ +ĠкаÑĤ ал +ĠкаÑĤал ог +ส à¹īม +ĠÑĢ ей +ĠÑĢей ÑĤ +ĠÑĢейÑĤ инг +Ġth uyá»ģn +ĠاÙĦÙħ ÙĤدس +esp ère +ãģ«åħ¥ ãģ£ãģŁ +หมาย à¹Ģลà¸Ĥ +ת×Ĺ×ķש ת +à¸Ļ à¹Īะ +Ġpe ÅĤ +ĠpeÅĤ ne +Ġpé rd +Ġpérd ida +หม วà¸Ķ +หมวà¸Ķ หมูà¹Ī +иÑĩеÑģк ÑĥÑİ +çµĤ ãĤı +çµĤãĤı ãģ£ãģŁ +Ġ×Ĵ ×ķ×Ĵ׾ +à¸Ĺำ à¸Ħวาม +à¸Ĺำà¸Ħวาม สะà¸Ńาà¸Ķ +Hot éis +Ġз аÑĢ +ĠзаÑĢ егиÑģÑĤ +ĠзаÑĢегиÑģÑĤ ÑĢи +ĠзаÑĢегиÑģÑĤÑĢи ÑĢова +ĠÑģ обÑĭÑĤи +ĠÑģобÑĭÑĤи Ñı +Ġ×ĸ ׼×IJ +ÙħÙĨظ ÙĪÙħØ© +Ġ×Ķ×ŀ צ +Ġ×Ķ×ŀצ ×Ļ×IJ×ķת +Ùħ ÙĥÙĪÙĨ +ÙħÙĥÙĪÙĨ ات +ä¸ĬãģĮ ãĤĭ +Ġm ÄĻ +ĠmÄĻ sk +หรืà¸Ń à¹Ģà¸Ľà¸¥à¹Īา +ëĤ ® +Ġnok tas +Ġnoktas ı +ĠболÑĮÑĪ им +ĠлÑĥÑĩ ÑĪиÑħ +Ø´Ùĩ ÙĬد +à¸Ńำ à¸Ļ +à¸Ńำà¸Ļ วย +à¸Ńำà¸Ļวย à¸Ħวาม +à¸Ńำà¸Ļวยà¸Ħวาม สะà¸Ķวà¸ģ +Ġе в +Ġев ÑĢ +ĠевÑĢ оп +ĠевÑĢоп ей +à¸ī าย +ìĦ Ń +Ùħ Ùģا +ÙħÙģا ÙĪض +ÙħÙģاÙĪض ات +ë¹ Į +赤 ãģ¡ãĤĥãĤĵ +ĠÑĥдал оÑģÑĮ +ĠÐ¥ оÑĤ +ĠХоÑĤ Ñı +przedsiÄĻbior c +ĠH ôm +íķĺìĺĢ ìĬµëĭĪëĭ¤ +Ġн аг +Ġнаг ÑĢÑĥз +ĠнагÑĢÑĥз к +Ġ×ij×Ļ׳ ׾×IJ×ķ×ŀ×Ļ +Ġê°ĢëĬ¥ íķľ +ĠH ữu +à¸Ń ุà¸Ķ +à¸Ńุà¸Ķ ม +ת ×ķפ +ת×ķפ ×¢×Ķ +Ġmi ÅĤo +ĠmiÅĤo ÅĽci +ksi Äħż +ksiÄħż ka +ĠاÙĦÙĦ عبة +à¸ī าà¸ģ +สะ สม +×ŀ תר +×ŀתר ×Ĺש +Ġlég ère +Ġ׾צ פ +Ġ׾צפ ×Ļ×Ķ +ĠиÑģÑĤоÑĢ иÑı +Ġ ãĥĪãĥ© +ĠãĥĪãĥ© ãĥĥãĤ¯ +ĠãĥĪãĥ©ãĥĥãĤ¯ ãĥIJãĥĥãĤ¯ +Ġк а +Ġка ÑĦе +×ŀס×ŀ ×ļ +Ġc üm +Ġcüm le +à¹Ģà¸Ħลืà¹Īà¸Ńà¸Ļ à¹Ħหว +ãģĬ ãģĿ +ãģĬãģĿ ãĤīãģı +ìŀIJ ëıĻ +ìŀIJëıĻ ì°¨ +à¸Ńั à¸ķ +à¸Ńัà¸ķ à¹Ĥà¸Ļ +à¸Ńัà¸ķà¹Ĥà¸Ļ มั +à¸Ńัà¸ķà¹Ĥà¸Ļมั à¸ķิ +ĠÅŁ ik +ĠÅŁik ay +ĠÅŁikay et +extr ême +kr ä +krä fte +ëĤ Ļ +íķ ij +ì² Ļ +íĺ Ī +ì° į +âĻ ¡ +ìŀ Ķ +ë¢ ° +íĿ Ķ +íĿ IJ +âĩ Ĵ +ë§ Ľ +ìĬ Ī +á» Ĵ +ìĺ µ +âĹ İ +í Ĥ¨ +ê¿ Ī +ìĪ ¨ +ìĽ ¨ +ë§ ¥ +ï½ Ģ +ï¼ ª +Ạ¨ +ãħ İ +Ñ Ĺ +ìĦ ¬ +ì¹ ¼ +ï¼ ¶ +ìĽ ł +ëŁ ´ +Å ĥ +ëĤ ¼ +ëĭ IJ +âĢ ¹ +ë¦ Ń +ì§ IJ +âĢ ¤ +à ħ +ëľ ¨ +íĦ ¸ +íľ ĺ +ê² ģ +ë´ ħ +à ĺ +ëŃ Ķ +ëĺ ij +âĹ ĩ +ìĹ ĺ +ï» ´ +ë§ ¹ +ï¾ Ŀ +ìĬ · +íĥ ķ +ï¼ ł +ì» ´ +ëł Į +ì½ ľ +ï» ¹ +ãħ ł +ì¡ ¸ +ëħ ¹ +âĤ º +âĸ ¶ +íĥ IJ +êµ ´ +íij ¸ +Ñ Ķ +íĶ ½ +Ð ħ +ë° ¤ +Ô ģ +ì² ¨ +ì¶ ĺ +ë² Ĺ +ë© ¸ +ï¼ » +ï¼ ½ +ï¼ · +ì° Į +à Ĵ +íı ´ +ìĵ ¸ +ì´ Į +ëģ Ķ +ëĶ © +ëĩ Į +ë© Ģ +ë² ¨ +ï¼ µ +ë§ ¡ +ëĭ « +ภ¿ +ãģ ± +ìĩ ¼ +ìº ł +ë® ¤ +ê± ± +ì» ¬ +âĦ ĥ +ëĶ ± +ëĥ Ī +ìĭ ± +íĻ Ī +ëŀ IJ +ìħ Ģ +ìł ł +Ð Ĩ +ëł ī +ï½ ħ +ï½ ı +íĻ Ģ +ëĽ ° +á» ® +í Ĥ¹ +ê½ ĥ +ï» ¤ +ïº Ķ +êº ¼ +ìķ ī +âĻ ¦ +ï½ ģ +ìĵ ´ +ãĢ ī +ì° ® +ì¤ ĺ +á» ª +ëģ Ħ +ëIJ ¨ +ìķ Į +íĿ ĺ +íħ IJ +ãĢ Ī +ê² ª +ëĭ ¥ +ê² ¼ +á» Į +ë§ ¨ +ëģ Ĭ +ë² ¤ +ëij Ķ +íĿ ¡ +á» ¬ +ë¬ ĺ +ãģ ī +ëŀ « +íĶ Ī +í ħį +ìŀ ĥ +ï½ ī +ìģ ľ +âĸ ½ +ë¬ » +âĸ ³ +ï¼ ¸ +ìģ ĺ +ì¶ ° +ìĬ ´ +ìķ ± +ìĩ Ħ +Ạ® +ï´ ¿ +ï´ ¾ +âĤ ½ +ëĦ ĵ +ë£ © +ì³ ¤ +ê´ ľ +à Ļ +á» ľ +ï¿ £ +ëĵ Ń +ë© ĺ +ê» ´ +ëł ´ +Ð ĥ +ë¬ µ +ì§ Ŀ +ãģ º +ðŁĺ Ĥ +ëŀ ¬ +ìł Ĭ +ê´ Ħ +ìŀ Ĭ +íŀ Į +ìĦ ¯ +âĪ Ģ +âĸ ¡ +ëĢ Į +ëŀ Ļ +ï½ ĥ +Ạ¶ +ï¾ Ħ +ïº ĺ +ë¹ ¼ +à Į +âĸ · +ê¸ į +ë© ĭ +ãģ ĥ +ìĺ Ĩ +ìĺ ® +ëª ¬ +ë¡ ¤ +ëł ¬ +ëĬ ¦ +âĸ ª +ì¼ ĵ +ìľ Ī +ì§ § +ï½ ½ +ëĥ ī +ï¾ Į +ëĺ IJ +ï¼ ĥ +á» Ħ +ì´ ¬ +ì¶ ¤ +ï¼ ¹ +ï» Ń +âĤ « +ï½ ĩ +ìĺ · +ëĸ ¨ +âī « +ë¦ ¿ +âľ ¨ +Ù ± +ì¯ ¤ +ê¹ Ķ +ðŁĺ Ĭ +ìĪ « +ê³ ± +êµ ³ +ï½ ĭ +ภĮ +Ä ł +ëĶ ¸ +ë° ij +ìħ ĭ +íİ ´ +âľ ħ +íĥ ij +ëĪ ĩ +íı ¼ +ðŁĺ į +ìĺ Ľ +ï» £ +Ñ ĺ +ì© Į +ë¦ ħ +ìĿ į +ï½ ¸ +ëį ľ +ãģ ħ +íİ ¼ +ëĭ Ŀ +ë¿ Į +ì¼ ° +ìĭ « +ë° ¥ +íĽ Į +ì¨ Į +ë¹ Ļ +ï½ İ +ë´ Ħ +ìĦ ¹ +ï½ ² +ìĮ ĵ +Ò ij +ë° į +ëł Ģ +íĨ ¤ +ï½ ¯ +ë¤ Ħ +ê½ ¤ +ï½ Ĵ +ìķ ¨ +ï½ ¼ +ê¹ IJ +íģ IJ +âĦ ĸ +ë§ º +ïº ® +ëħ ģ +ê² ¸ +ï» ł +íĬ ľ +Å ¹ +ë¥ Ń +ëĪ ī +ï½ Ķ +íĮ ¬ +ìŀ ĩ +ï ¬ģ +ï» ¨ +ëij ¥ +ëŀ Ħ +Ù ¬ +íĭ ´ +ìŀ ī +Ú ¾ +ìĽ ħ +ï» ® +ëĭ ī +âī ª +âĹ Ħ +ëĪ Į +íĽ ¼ +ì¤ į +Å ¸ +ì¤ ¬ +ì¾ Į +ï½ ĵ +ï¾ Ĭ +ðŁı » +ï¾ ī +Ð ģ +íĺ IJ +ï¾ Ļ +ê¼ ¬ +íŀ IJ +âĢ ¥ +ëŁ Ń +ë§ ŀ +ìĥ ¤ +ïº Ĵ +íĭ ± +ë½ ij +à ķ +âĪ ļ +ëĤ Ħ +ê¹ Ŀ +ëĨ Ī +Ạº +ìħ Ī +ìĮ į +âĢ ¡ +ï¼ ± +ìģ ¨ +âĺ º +ëĴ · +ìĺ ³ +ðŁij į +ëª ½ +ëĤ Ń +ïº Ń +ë© Ī +á» Ī +íķ Ģ +ëĭ Ļ +ë¦ ĩ +ìķ ¤ +ìį ¼ +ãĥ µ +Ñ £ +ìľ Ĺ +â ŃIJ +ï¾ ĺ +íĹ ¬ +ê¾ ¼ +ìķ Ĺ +ï» Į +ê± · +ëħ ķ +ë¡ ± +ìķ Ĭ +ï¾ Ģ +ìĩ ł +íĮ © +ïº ª +ë§ Ļ +ï¼ ¿ +ê¿ Ķ +íİ ľ +ë£ ¸ +íĶ Ķ +ï» ³ +ëı ķ +ìĭ ¼ +á» İ +ë§ ĺ +ì¢ ĭ +íĨ ¡ +ï½ ± +íĿ ij +á» ¸ +ì¦ Į +ì¹ ¸ +ëŃ ĺ +ï¾ Ĺ +ï» ĭ +íĬ Ģ +ë¥ Ļ +ì½ © +ëģ Ĺ +ëį ´ +ìħ ľ + ¸ +ë» IJ +ìĥ µ +ê² IJ +ëĵ ¬ +ë£ ° +ãħ ĭ +ìĹ ī +á» ĸ +ëĦ Į +ï½ ¶ +ë´ ĩ +ëĤ ³ +ãĤ ľ +ëĸ » +íİ Ģ +ëį © +íķ ¸ +à · +ê¼ ¼ +ëĶ ľ +ë° ´ +ë© į +âĹ ¯ +ìĹ ij +ìĻ ¼ +ïº ij +ë¶ ķ +ë¡ ¬ +ï½ Į +íĨ ¨ +ïº ´ +ëł ĺ +ê° ¤ +ìĪ ² +Ñ ĵ +ìħ ī +ï» ĵ +ëĪ Ķ +ëį § +âĢ ¼ +ï» ² +ê° ± +ê¿ Ģ +ëĭ · +Ạ¸ +Ạª +Æ Ĵ +ëį ¤ +ìĪ Ń +ï½ Ĥ +ï½ Ī +Å ł +ë£ ¬ +Ñ µ +ëĸ ¡ +ëĥ Ħ +ìĦ ° +ëĵ Ī +ï¾ ĥ +ëĩ ¨ +ï½ IJ +êµ ½ +ìĹ ½ +ëĤ Ģ +ë¬ ¶ +ï½ · +ìı Ł +íĺ Ķ +ê¼ Ī +ëģ Ī +ì¥ IJ +ïº Ĺ +Ä Į +ëĪ ł +ëĸ ¼ +íĢ ´ +âī ¥ +ëĭ Ń +ì± Ļ +ê» ı +ë© ¤ +ìĥ ĺ +ëį ® +ë£ ¡ +ìĤ ½ +ãĪ ľ +Ä ¨ +âĢ § +ï½ º +Ä £ +ì¦ ī +ï¼ ¼ +Û © +âĪ Ļ +ë° ı +ë¹ ħ +ðŁĺ Ľ +íĪ ´ +ðŁĴ ķ +ãĢ Ĵ +ìŀ ĺ +ïº ¤ +ï½ ĸ +ë© ľ +ë² ¼ +ëĿ Ħ +ëļ ľ +ï» ĺ +ìĥ Į +ï½ Ħ +ì© Ķ +ï½ Ļ +ïº © +Û ŀ +âĺ İ +ìł ¤ +ëIJ © +Å Ŀ +âŀ ¡ +ï» § +Ð ı +ì« ĵ +ê³ ½ +É ij +ãĥ ² +ëĤ « +ë¦ ī +ì¢ ģ +ë° Ń +ðŁĺ ģ +ë¹ µ +ì² © +ì» µ +ðŁĺ ĺ +ë± ħ +âī Ī +ë¹ ļ +ï» ľ +ðŁĻ ı +íģ ° +ìĦ ŀ +ï¾ ļ +ìĺ ¹ +ë¼ Ī +ëĤ ¯ +ëŀ © +íļ ¡ +ï½ ķ +íĥ ĵ +ëĿ ł +ê³ ģ +ëĵ Ģ +ìĹ ł +ï¼ º +ë§ ij +ëĭ ¿ +ì¿ ¨ +ãİ ¡ +Ð Ĭ +íĦ ± +Å ¨ +ïº ³ +ï¾ ı +âĭ ħ +ê¼ ´ +âī ¤ +íĮ ģ +Î © +ê¶ ¤ +ìĪ į +âľ ¿ +ì½ ¤ +ëĪ ħ +íĨ ± +ãħ ľ +áIJ ħ +Å Ĵ +ðŁij ī +ï» ¦ +Ð ª +ë¥ ľ +íķ « +ï¾ ĭ +âĻ « +ê¹ ľ +ë° ¸ +ëĶ ĺ +íĿ ī +ï¾ ģ +ï¾ Ľ +ëł Ľ +ê² ¹ +ì¿ ¼ +ï» ¬ +âŀ ¤ +ðŁĻ ģ +ïº ł +ëĨ ¨ +ë¯ ¹ +ê¸ ĭ +ë» Ķ +ê¹ ĥ +ëij ij +íĭ ¸ +íİ Ļ +âŀ ĸ +ãĥ ½ +ì§ ļ +ï½ ¬ +ï» ¥ +íĮ ½ +âĢ Ĵ +ì ĮĢ +ìŃ ī +ëļ ± +ãĤ ŀ +íĭ Ī +ãĤ IJ +ëī ĺ +Î £ +ê³ ° +ë¹ Ĺ +ï¾ İ +ðŁĺ Ń +íĿ ł +ìĹ ¿ +ê° ļ +ì¤ Į +ë§ µ +ï½ ³ +ãģ ¢ +ï» Ĺ +âī ¦ +Ú ¤ +ë łģ +ê¼ ½ +ï» « +âī § +ì´ Ľ +ìł Ŀ +Ạ° +âĻ £ +ìº ĺ +âĪ ĩ +ê² ī +ë° Ł +ï» Ķ +íĸ ĩ +âĸ Ĵ +ðŁij ı +à ŀ +ðŁĺ Ĩ +ïº ¼ +âĿ Ĺ +ìº Ķ +ì¹ © +ëĸ ¤ +ëĥ ħ +âĶ ľ +ï½ » +Î Ķ +áĥ ¦ +ìŀ İ +âĺ Ģ +âĪ ¼ +ðŁĶ ¥ +ë° Į +ìł ĸ +íĹ Ľ +Î ķ +ïº ĥ +ë¶ ī +âĪ ŀ +íĥ Ń +à ĭ +âģ Ħ +ãħ ĩ +ëĦ ¥ +ëĭ ® +ëł · +íĮ Ŀ +ìº ¡ +ë· Ķ +ì© į +íĤ ´ +ëļ « +âĵ Ĵ +íķ į +âĻ Ĥ +ï¾ Ĩ +âĨ © +ìį © +ïº ķ +íĿ Ļ +Ñ ľ +íĤ · +íĿ ° +íĥ ± +ëķ IJ +ï¾ Ĵ +× ĥ +ëĮ Ħ +ìĺ ´ +ìķ µ +ê¹ ¥ +ëŀ Ń +ìª ¼ +ãİ Ŀ +ðŁĺ ħ +ëı ĭ +ëª « +ïº ¸ +ë® ¬ +ë² ħ +ëij ł +ìħ ° +ì» · +ëĶ ª +ëħ Ķ +ãħ ¡ +ìĶ » +íķ ı +ëį ± +ïº ¨ +ï¾ į +ï½ µ +ì¢ Ģ +íİ Į +ï» ° +ïº £ +Æ £ +ðŁ¤ £ +ï· º +ëĤ ļ +âĭ Ĩ +ë³ į +ðŁĺ Ħ +ìĸ Ģ +ìĻ ł +ëĨ Ķ +íĹ ¨ +ï» Ľ +ï» Ŀ +á» ¶ +ìĸ ĺ +ìİ Ħ +Ú Ĩ +ï» ŀ +ëĢ IJ +ê² Ķ +ï» µ +âĹ ¦ +íļ Ł +ê¹ ģ +ê° ĵ +ëĶ ´ +ìı ĺ +ëļ Ŀ +á» ł +ëŀ ´ +ëĦ ī +âĺ ŀ +ï½ ĺ +Å ½ +ë¦ İ +âĸ ¬ +ëŃ ī +âĩ Ľ +ìį ¬ +ïº Ł +Ë ľ +ë¶ ĵ +ìĽ ° +Å ľ +ëŃ ĩ +á» ² +Ë ļ +ëķ Ģ +âĺ ij +ðŁı ¼ +ìĸ ½ +âĮ Ĵ +Ð İ +É ¾ +íĮ ¡ +ï¾ ħ +ìŀ Ń +ï½ ¨ +ì¹ « +ìľ Į +Ò Ľ +êµ ¿ +ëĭ ¦ +âĶ Ķ +ï¾ ij +ì§ ĸ +ìº Ħ +ãĢ ĥ +Ê ¼ +ê² Ł +ï½ § +Ä ¢ +íİ ł +ë§ · +ê° ĩ +ìĭ ¹ +ðŁĴ ¦ +ï¾ ľ +ëĬ Ļ +ë² ¡ +Å ¿ +ðŁĺ ĭ +ðŁĴ ª +ì¿ Ħ +ë© ķ +ìŃ ¤ +ëĬ Ħ +ðŁĮ ¸ +ãĤ Ŀ +Ç İ +ï½ ļ +Ä Ĺ +ëģ ĵ +ê¶ IJ +áµ ī +ãĥ Ĥ +ê» į +ðŁĺ ¦ +ãĢ Ŀ +ðŁ¤ Ĺ +Ñ Ł +ìĹ İ +âľ Į +ìī IJ +à Ĩ +íĹ IJ +ðŁİ ī +Î ij +ï½ Ń +ðŁĴ Ļ +ìĽ ¬ +íĢ ĺ +ï» ¢ +ðŁĺ İ +íij ¼ +íĿ © +ï» Ħ +íħ Ģ +ëł IJ +ì¥ ¬ +Ð ĭ +ìĥ · +ëľ ¬ +ðŁĺ ĥ +ëĦ ¬ +ë¥ ¨ +ìĽ į +ï½ Ĩ +ï½ ´ +ãĥ ħ +à ı +ï» ª +âĻ ł +ëĬ ¬ +ë± Ģ +ë° ĭ +ìĥ Ģ +ï½ ¾ +ëĤ ± +ì» ¸ +ðŁĴ ĸ +ðŁij Į +Ñ ŀ +ì§ ± +Ë Ĩ +ðŁĵ ļ +âŃ ķ +ï¬ Ĥ +ï» ¡ +ëij ¬ +íĪ ¼ +âĸ ¸ +ê° ¯ +ê¹ ħ +ï½ ® +ëĺ ¥ +Ä ¡ +íĮ Ł +Ð Į +ìĨ Ł +ïº ĵ +ï» ¼ +à Ľ +ãĥ ¾ +ëĮ ĵ +íĴ ĭ +ìķ ĵ +ï½ ¹ +ëĤ ¡ +ðŁij ĩ +Ạ¼ +ãĢ Ł +ðŁĮ Ł +íĥ ł +ãĢ Ĩ +âĢ Ł +ë¸ IJ +ðŁĮ ¹ +ìł ¼ +ðŁĵ Į +ìĶ ¬ +âĹ Ģ +ðŁĴ ĵ +ê¹ İ +ìĤ IJ +ìĶ Į +Ñ Ľ +âĶ Ī +ë² ³ +ãİ ŀ +Õ ¡ +íĤ µ +ðŁ¤ Ķ +ëĢ Ķ +ìĬ IJ +íĻ ī +âľ ¦ +ëľ ¯ +ìł ¯ +ëĶ § +Î ¦ +Ë Ī +ìī ¼ +âĹ Ĭ +ëľ © +ëľ ° +ï¾ IJ +ë¿ Ķ +ìĹ ® +ì· Į +ïº § +Î Ĵ +ëµ Ļ +ï» Ĭ +ì° Ķ +íİ Ħ +ðŁĴ Ĺ +Ạ´ +ì° ¢ +íľ ¼ +ê½ Ĥ +ì± Ķ +ìī ´ +âĸ ¾ +íĪ ° +ëĭ Ľ +âĿ £ +ï½ ª +ðŁĴ ľ +Ë ĺ +ãħ ¤ +âĨ Ĺ +íĸ Ħ +âĻ ¬ +ìķ ° +ïº ľ +âī ¡ +ãĢ ĵ +ìij ¥ +íĮ į +íī ģ +ë» Ĺ +íľ ł +íľ © +âľ Ī +íĢ Ħ +ìĸ ĩ +ì¢ ĩ +íŀ Ļ +ëª ¹ +ãĤ Ľ +ðŁĺ ± +ëį Ł +๠ħ +êµ ¶ +Ù « +ìĶ ģ +âľ ª +ï¾ Ī +ðŁĻ Į +âļ ¡ +Î ļ +ì¼ Ī +ï¾ Ķ +ï¾ Ĥ +êµ ī +ïº » +ðŁĴ ĭ +á¹ £ +Ó Ļ +ìĨ ľ +ìĹ £ +âľ © +ìľ Ļ +ïº ° +Ạ² +ìŀ £ +âĿ Į +âĺ ģ +ìķ İ +Ä ½ +Û ģ +ãĦ ± +ëŁ ¿ +íĮ ¸ +ê½ ī +ìı ł +ðŁį Ģ +âĨ Ķ +ëŃ ¡ +ï» ģ +ï¼ Ħ +ðŁĴ ¥ +âĺ Ľ +íĹ · +ëij ¡ +Î ł +Î ¤ +âĦ ĵ +ïº · +Î Ļ +ëı Ķ +ì§ ¤ +âĶ ĥ +ãĦ · +Ç Ĵ +ðŁ¥ ° +ëĶ ķ +ìļ ¥ +ì¸ Ħ +íĽ Ķ +ïº ĩ +ïº ¬ +ðŁĺ ¢ +ë¹ ¡ +ìĶ ¹ +Å ³ +Ë Ŀ +íİ ij +ï¾ ĵ +ðŁĴ ļ +ëĬ ij +êº ¾ +íĨ ° +à ¿ +Ð Ħ +ëĮ IJ +ë½ Ģ +ì· Ħ +ðŁ ĵį +ðŁĻ Ī +âĹ Ī +ê¿ ĩ +ì¼ Ħ +íİ « +ðŁĩ · +âĶ ĭ +âļ ł +ë± ī +ì į° +ìĻ Ī +É ª +ïº ĭ +ðŁĺ ľ +Î Ł +ðŁ ĻĤ +âļ ½ +Å Ī +ë¹ Ķ +íĮ ľ +๠ı +ìĸ ¹ +íĪ Ń +ðŁ¥ ĩ +ãĦ ´ +ëĶ ¥ +ìŃ Ī +âĪ Ĩ +ëĸ ³ +ë± ĥ +ìŀ ¦ +ï» IJ +Î ľ +âľ § +Ï į +ìł ĵ +âĹ ķ +ëĴ Ģ +ï» Ģ +ðŁĶ ´ +ê½ ģ +ëĮ Ī +ëİ Į +ãĤ İ +⦠ģ +ì½ § +ï¯ ¾ +âĿ ¯ +ภħ +ðŁĻ Ħ +âĿ Ģ +ðŁĶ ¹ +âĩ IJ +êµ µ +âĩ Ķ +ë¶ IJ +ðŁĴ Ľ +Î ¾ +íĥ ¬ +âĿ Ħ +Ò £ +ãĢ ° +âĪ ij +âĺ ¼ +âī ł +Ò ¯ +ïº ¯ +ê¿ ¨ +âľ ĸ +Ê ĸ +íĢ Ģ +ê¾ Ģ +íĹ Ŀ +âĶ £ +ãİ ľ +ëĶ Ľ +ëľ ¸ +ï º« +ê¿ ° +ðŁĩ ¹ +Ç IJ +Û Ĵ +ë£ » +ïº ĸ +Ñ ļ +ëĬ ł +Û ķ +ê¹ ¡ +ë¿ ľ +ì² ¼ +ï¨ ij +ë¥ µ +ìį ¸ +íħ ħ +íij ¹ +Ö Ģ +ï³ Į +ãħ £ +ìij ¤ +ì½ ķ +ëķ ł +ðŁĮ ¿ +íĥ Ķ +ìĽ ģ +Î ¶ +âŀ ľ +ìĬ ĺ +íĽ Ĺ +ë© § +ìī ĺ +Õ ¶ +á¹ ĩ +ðŁİ ģ +ï½ ¿ +ï¼ Ĥ +á¼ IJ +âľ ķ +âŀ ¢ +ëĦ ¨ +ì» « +ì¯ Ķ +ì° ľ +ðŁĴ ° +íħ Ŀ +ãİ ı +ë³ ¶ +Ò ĵ +âĨ ³ +ìĥ ´ +íģ ĺ +âĸ Ģ +ë² Ļ +ภĥ +á½ ¶ +Ä ķ +⬠ĩ +ë¤ ĺ +ðŁİ µ +âľ ļ +ïº ı +Î ¡ +âĹ ī +ðŁĴ « +Ð Ī +ìĸ Ħ +ì§ Ļ +ï» ĥ +ðĿij Ĵ +ëŃ Ħ +âĿ ¥ +âĿ ĸ +âĺ Ŀ +Ê ¹ +Ḡ¥ +âĢ ¿ +ãħ ħ +ê¸ ģ +ëķ ¡ +ëį ¥ +âĪ © +ê» Ħ +ë® Į +Ò ± +âĪ Ĺ +ëł Ļ +ïº Į +Ë IJ +ðŁĺ ³ +ðŁij © +ðŁİ ¶ +ì¿ µ +ðŁ¤ © +ê· ¤ +ëĮ Ķ +ïº IJ +Ï İ +ì¶ ¥ +ï½ Ĭ +á¹ Ń +ë¤ ¼ +âĸ « +ì§ ł +á¼ Ģ +ê» ij +ëĮ ģ +íĢ ¸ +âĻ Ľ +ðŁĴ ŀ +âĸ ° +ðĿij ĸ +ëĿ ¤ +ठ¦ +ì´ ĺ +ðŁĺ ĩ +ëĶ ¤ +Î Ĺ +ðŁĻ ĩ +Ë Ľ +ì© ¡ +âĪ § +Õ ¥ +Ñ Ļ +ëIJ ¬ +ëĸ Ħ +ðŁĮ · +ìĹ Į +ðŁĺ ¥ +ëĪ ´ +ï» ļ +É Ľ +ïº Ħ +ï» ı +Å Į +ë² ļ +ìĭ £ +ïº Ģ +Î ĵ +ðŁĺ Į +Ë Ļ +ëŀ ı +ðŁĶ ¸ +ðŁĵ · +ëģ ½ +íģ ½ +ðŁĴ ¡ +ðŁĮ ± +ëº ı +ìģ ł +ìĥ IJ +ëı Ĺ +ì¸ ° +ëĪ ķ +Î Ŀ +âģ ī +ðŁĮ ¼ +íĮ ł +âĭ ¯ +áĥ ĺ +âľ ¤ +ê± Ķ +íĮ İ +ðŁĴ ¯ +ìı Ļ +íĹ ī +Ù Ń +ì½ ° +ïº ¿ +ï» ± +ì± Į +âĺ ķ +ðŁİ Ģ +Ä Ŀ +ë° § +ìĤ ¿ +áij ķ +ðŁį ĥ +âĩ ¨ +Î Ľ +ë§ ´ +ë³ ķ +á ijIJ +âĸ ĵ +ðĿ ijľ +âĻ » +íĤ ¥ +Õ ¸ +ãĪ ± +ëº Ģ +ì² ¸ +ïº Ľ +ðŁı Ĩ +ðŁĩ ª +âĿ ĵ +Ä Ģ +ì½ ¥ +ðŁĩ § +á½ · +âľ Ĥ +ìŀ ¼ +ï§ ¡ +ðŁĵ ¸ +âĻ ¯ +É Ķ +á½ ¸ +âĮ ª +ï» ĸ +ï¥ § +âļ « +âĶ Ĺ +ðŁĮ Ī +ï» © +ðŁĵ ² +Ï Ī +ðŁĺ ¡ +ðĿij İ +ìľ ½ +ì§ ¬ +ì§ Ĭ +á½ ³ +ìĮ ¤ +ëĤ į +âī Ĵ +ðŁij ¨ +âĺ ĺ +Ó © +âĤ ĵ +âĪ Ĥ +ï¹ ģ +ðŁĴ IJ +íħ ĥ +ðŁı ½ +ê· Ħ +ðŁĺ ı +ðŁĮ º +ðŁĺ Ķ +ï½ « +âľ İ +ëµ Ī +ðŁĩ ¸ +âĢ £ +âŀ Ķ +ëĺ ĺ +ìĥ ¬ +Ê ĥ +⬠ħ +ì© IJ +ðŁĻ Ĩ +ðŁİ Ħ +Ä ¾ +⣠¶ +áĥ IJ +âĺ » +ì± ķ +ìģ © +ë½ ķ +ìº £ +ðŁij Ī +ðŁĻ ĭ +ï¾ ĸ +Ò ļ +Õ « +ìĮ Ī +ë² § +ðŁĩ ® +ï½ Ŀ +ðŁį ģ +ìĹ ¥ +Ä ³ +ë½ IJ +íį ½ +íĽ ij +âĤ ¹ +ãħ ģ +ìĶ ½ +ðŁĶ ģ +ठ¯ +ê¾ ¹ +ëī ľ +âĹ ¡ +íķ Į +Î ĺ +ë£ ¹ +ìĻ ĵ +ðŁĩ ¦ +ðŁij Ģ +âĶ Į +á¿ ¦ +ëĦ Ľ +ìĦ £ +ìŃ Ļ +ï± ł +Î ŀ +Ê » +á¿ ¶ +âĿ Ŀ +ê± Ģ +ëĸ ´ +ãĦ ¹ +ðŁĴ İ +Ï ¹ +⼠ħ +ï» ķ +ãĥ ± +ï½ Ľ +ëĮ ķ +ë¹ ½ +ì¥ Ķ +ì¿ ¤ +ðŁĸ ¤ +Ñ Ĵ +ê¹ į +ëİ Ģ +ìĭ ¯ +ë» ¤ +ðŁĵ ŀ +ðŁĵ £ +ðŁĺ Ŀ +ìį ¹ +ìĹ ¡ +ì° IJ +á½ IJ +ï» Ī +âľ į +Ä ı +ðŁĮ ŀ +âĦ ¦ +ê½ Ŀ +ë» ĺ +ìĪ ± +âĶ ĺ +ðŁĮ » +âĤ ´ +âŀ ¨ +íIJ ģ +ê ¶Ī +âĺ ¢ +ðŁĺ Ī +ï½ © +âĦ Ĺ +ê° Ń +ê° ¸ +ë» ij +ì¥ ´ +ì» ¥ +ï¤ Ĭ +ï» Ĵ +ðŁĺ ķ +âĺ Ķ +ìĺ IJ +ðŁļ Ĺ +ëĹ Ħ +ë§ ı +Õ ½ +âĸ » +⣠µ +ìī ° +ï» ij +âĻ © +Î ¥ +ðŁĺ £ +âĬ Ĥ +ãħ Ĥ +ìħ ¸ +íı Ħ +âľ ½ +ì¦ Ļ +âĸ £ +ê± į +ê¿ ĭ +ì« Ħ +ìº ĩ +ðŁĩ µ +ðŁij ij +âľ ĺ +ðĿij Ľ +ìį ½ +ìº ī +ï¬ µ +ðŁĶ º +âĦ ® +íĥ ¤ +ðŁĩ º +ðŁĴ µ +íħ ¨ +ï½ ij +Î ¨ +ìĥ ¹ +ìĸ ķ +ì¹ µ +ðŁĵ ± +ठµ +ðŁij Ĭ +ðŁĴ Ħ +ðŁĴ Ŀ +ãĮ Ķ +ìĻ ģ +Ð ĩ +à® IJ +âĸ ¹ +á´ Ľ +âĹ ĺ +ëº ¨ +íĥ ī +ìĸ Į +ðŁIJ ¶ +ãĤ ij +Ë ĩ +Å ı +á½ ¹ +ìħ § +ï¹ ° +ðĿij ¡ +ðŁĶ Ŀ +ðŁĺ » +ðŁĴ ĥ +ðŁ¤ ¦ +ðŁį Ĵ +íĢ µ +âľ Ĩ +ë¹ ´ +ï§ ¤ +ï» Ļ +á´ Ĺ +ðŁĮ ´ +Í ¾ +ëĮ ij +ì¨ ĭ +ìµ ¸ +ðŁİ Ī +ðŁı ł +á½ ± +Û Ĩ +á¿ ĸ +âĢ Ľ +ì° ¼ +íķ ¥ +íĹ ´ +ðŁĩ ¬ +ì° Ŀ +âĪ ł +ï¼ ĩ +âĬ Ļ +âĿ ij +ëĦ ĭ +ëŀ Ĺ +ë° ī +ìĹ Ĭ +ì¢ Ĩ +íĮ ¥ +ï° ² +ðŁĵ ĸ +ðŁĺ ® +âļ ª +ðŁĺ ļ +âĿ ŀ +ðĿij Ł +ðŁİ Ĥ +Å ķ +áIJ Ī +êº ½ +ì± ł +ïº Ŀ +ê¿ ī +áĥ ł +ðŁı ĥ +ðŁĴ ¸ +âĿ ģ +âĹ ¾ +Ú ª +á¹ ĥ +íĬ ¬ +ðŁĩ ± +íİ Ń +ðŁĺ ŀ +ë¾ ° +á¹ Ľ +ëĽ ¸ +âĿ Ĥ +êĴ ³ +âĶ IJ +íĵ ° +âŀ ł +ê´ ĺ +ëħ ĺ +ë» ¥ +ì¾ ħ +ðŁĺ IJ +âĪ ª +ðŁij ģ +âĪ ´ +âĹ ģ +ëº IJ +ìŀ ¤ +ì± Ĺ +ðŁı ¾ +Î § +á½ » +âŀ ¥ +ìŁ Ī +ï» ī +âĸ Į +ãĥ ® +ðŁ¤ ¤ +âĩ ĵ +ì¼ ł +á´ ı +ë§ ¬ +ë» £ +ðŁĴ ¬ +ðŁį ĵ +Ä ¸ +Ù ¹ +Ê ¿ +á½ ° +ëķ ľ +ì° ¡ +ì° » +íİ į +ðŁİ ¯ +ðŁį Ĥ +ðŁij § +âĻ ¢ +áĨ ŀ +âĻ § +âļ ľ +âľ ī +ëĵ ¦ +ëŃ £ +ìĪ ı +ìĵ ± +Å Ń +Ê Ĭ +âĴ ¸ +âĩ © +ðŁĴ Ķ +Õ µ +Ð ī +Ò » +ë§ £ +ìĽ ľ +ì¿ ¡ +íĽ ħ +íĽ ¤ +ïº ¢ +âľ ĭ +âĪ Ī +ðŁĮ į +Ê ľ +ëĬ ª +ëĴ ¹ +ïº ² +âĸ Ħ +ãħ Ī +ëļ ¤ +íİ © +âĪ ¨ +ðŁ¤ ª +áĥ ļ +ê³ ¶ +íĬ ķ +ðŁĺ ¬ +âĪ « +ðŁij ĭ +Ò IJ +íĬ ¿ +ðŁĶ µ +ðŁĴ ¨ +ðŁĮ Ļ +ëĩ © +âľ ³ +ë¨ ģ +ëº Ħ +ìĻ ij +ìº ħ +íı Ī +ðĿij Ļ +ðŁĴ ĺ +ãİ ¥ +âĿ ı +âľ ° +ï¯ ¿ +ëµ IJ +ì¼ IJ +ïº ± +Õ ´ +ï¬ Ģ +âľ ´ +ðŁ¤ Ń +ðŁij Ĩ +⼠Ķ +ê· ĵ +ìĮ Į +ðŁ¤ · +Û Ķ +ðŁ§ ¡ +ðŁĺ ĵ +Î ĸ +âı ° +ê² ľ +ëĭ ³ +ëİ ħ +ë° Ī +ï® IJ +ðŁı ¡ +âĨ ª +âĵ Ķ +âľ Ĭ +Ï ² +Ü IJ +ðŁĩ ³ +Ö Ĥ +âľ ı +ìĸ Ĺ +ì« Ļ +ðŁĺ ² +Ä Ń +âĻ Ń +âĶ ı +âĹ Į +ðŁĺ ¯ +áµ Ĵ +íĬ ł +Ä · +Ê ģ +ठŁ +á¹ ģ +á¼ ° +á¿ Ĩ +â « +â« ¸ +ëį « +ì³ ĩ +ì¼ ¤ +íĽ ¨ +ðŁĴ Ł +Ê Ģ +Ê ³ +ëĵ IJ +âķ ° +âĿ ĩ +Ç Ģ +Ç Ķ +É ´ +âĺ ļ +âĺ ľ +ê¶ Ĥ +ì« Ĵ +ì± Ī +ðŁĩ ¨ +ðŁİ ¥ +ðŁĵ Ŀ +Ä § +ðĿ ijIJ +Û Ī +ठ¬ +ì¬ IJ +íĹ ¥ +âĻ ¨ +ðŁį ´ +ï¹ ı +Ë ĭ +ðŁ¥ º +âĸ ¨ +íĻ ĭ +âĪ ħ +ëģ Ļ +ëŀ ł +ìĨ ¥ +âĢ ĸ +ðŁ¤ ĺ +ðŁIJ » +áµ ķ +Ç Ŀ +âĺ ı +ïº ļ +ï» Ĥ +ðŁļ © +ìĪ Ł +Ë Ĭ +⤠µ +ðŁĴ § +ã ħį +ë© © +Æ ¬ +Î ĩ +âĩ § +âĵ ļ +ìĤ ¯ +ìĪ ¯ +ëĨ ĭ +âľ ¯ +ðŁļ Ģ +Ú ĺ +Ú ¨ +âľ Ń +ê² ħ +íĮ ° +íľ Ļ +ðŁĮ Ĭ +ðŁİ ĵ +ðŁĺ Ļ +Ë ĥ +ðŁĴ ģ +ðŁij İ +âĺ ¹ +ðŁĺ « +ðŁĴ » +ëĤ µ +ìĿ Ĭ +íĮ » +Ò ³ +á½ ² +âŀ ŀ +ëĤ ij +ëĿ Ī +ì£ ¤ +ï» ¯ +ðŁĩ © +ðŁ¥ ³ +âĴ ¼ +ðŁ¦ ĭ +âĺ Ĥ +ðŁĺ ° +ðŁĻ ĥ +ðŁĺ Ĵ +Û İ +Ï ķ +Ḡ¤ +ë£ ½ +ìĬ ¥ +ðĿij ī +É IJ +ðŁį İ +âķ ¯ +âķ ¹ +ຠ² +ï¾ ł +ë¹ ķ +ïº Ĩ +Ê º +Ó § +âĨ ł +ëĥ ĩ +ìİ Ī +ìŁ ¤ +ï± ¢ +âķ ¬ +âĺ ł +ðŁİ Ĭ +ãį į +ãİ İ +âĺ ° +âľ ĥ +ãħ ī +ë¯ Ī +ë¹ ¤ +ìı Ń +ðĿij ¢ +ðŁIJ ¾ +Å ĭ +ðŁij ¶ +âĶ Ľ +ï¿ ¢ +áĥ ¡ +Ä ¼ +Å Ĩ +Ñ IJ +ìĥ Ľ +ìĺ Į +ì± ¤ +íħ ģ +íļ ĥ +ï³ Ĭ +ðĿij Ķ +ðŁĩ « +âĭ ° +ðŁĺ ¨ +âĤ © +Õ ¬ +Ḡį +á» ´ +âĨ ĺ +âĺ ¯ +ãħ ı +ìł ¬ +âĻ Ķ +ðŁĶ Ķ +ðŁĺ ł +ðŁĻ Ĭ +à® ľ +á¹ ħ +âĹ IJ +âĿ Ī +âŀ ½ +ìĥ ħ +ðĿij ł +Æ ¢ +âĭ Ļ +ê° Ľ +ëĿ µ +ë£ Ł +ìı ľ +ïº ģ +ðŁĴ Ń +âĬ ĥ +ðŁIJ ° +ãħ Į +Ü ĵ +âŀ ķ +á½ ģ +ìķ ³ +ðĿij Ŀ +ðŁİ ¬ +É ¡ +à¤ Ĺ +áIJ ī +ì© ľ +ì¶ § +ï³ ī +ï» ħ +ðĿIJ ŀ +ठ¶ +ðŁĵ ¢ +ðŁį ĭ +ðŁĴ ħ +ï¾ ķ +⬠Ĩ +âĪ µ +ðŁ¤ ij +áĥ £ +Æ Ħ +Ñ ¹ +á¼ Ķ +ê° ł +ê´ Į +ê· IJ +ëĽ ´ +ì± ĺ +ï® Ń +ïº ¹ +ïº ¾ +âľ Ĺ +âĿ ¦ +ðŁij ¦ +áĥ Ĺ +Ù ² +á½ ´ +âĪ ı +âľ ® +ê¹ ° +ë² µ +ìĦ Ģ +ì© Ŀ +ïº ŀ +ïº ½ +ðŁĩ Ń +Ë Ĥ +ðŁį ij +ðŁį Į +ðŁĶ » +ê¹ ¬ +ìĬ Ń +ìľ · +ðŁĽ ij +Ç § +ë¼ Ľ +ïº ¡ +ïº º +ðĿij ļ +ðŁĵ ¦ +ðŁĶ İ +ðŁĹ ĵ +áĥ Ķ +âľ Ĵ +âľ ¡ +ðŁĮ µ +âĶ ķ +ëĢ Ŀ +ðŁį Ĭ +âĺ ĥ +ìĺ ħ +ঠ¬ +ðŁ¦ ģ +âİ ¯ +ðŁIJ ķ +Ñ ¿ +ॠ¤ +༠ĭ +ê· Ī +ì« Į +ðŁĩ ° +âĿ ī +ì« Ģ +íĿ Ħ +ðĿIJ ¢ +ðŁļ ¨ +âĻ ¤ +ðŁĺ © +ðŁį į +ðŁĺ ij +ðŁļ ļ +Ö Ħ +ë « +ë« ¼ +ठı +á¿ · +âĮ © +âĺ IJ +âŀ £ +ê¸ ± +ê¼ ¿ +ëĦ Ŀ +ìı ´ +ìļ ¤ +ì¿ ± +íİ IJ +ðŁĴ ¢ +ì´ IJ +âĩ ij +âĶ ĵ +âģ ¾ +Ü Ŀ +ðŁ į° +â´ ° +Æ ı +Ï Ł +Ú º +Û ĥ +áĦ Ĵ +âĪ Ł +âĿ į +ãĦ ² +ìľ ħ +ì¤ ı +ðŁĩ ² +êº Ħ +ðŁİ ¤ +âľ £ +⸠Ŀ +ï¸ µ +ຠ§ +áĢ Ļ +âķ ł +Õ ¯ +âı © +ðĿij £ +ðŁĴ £ +Å ĺ +ॠIJ +âģ ĥ +âĮ ĺ +ê» Į +ìĮ Ķ +ðĿij ĺ +ðŁ¤ ĵ +Õ ¿ +à¤ Ń +âĮ ļ +âľ Ŀ +ðŁIJ ¼ +Ë Į +âķ ļ +ï¦ Ĺ +âĿ ķ +âķ £ +ðŁIJ ± +à® ¤ +Ñ ¾ +ठļ +ठľ +ìĪ Ħ +ìļ ľ +ðŁİ ® +É Ĵ +Ú · +ຠį +âĨ µ +â Īĺ +âĿ Ĭ +ë¿ į +ìIJ Ī +ìļ ĺ +ì¯ § +íĥ ¯ +ìĸ ı +ï¸ ° +ðŁĩ ¯ +ðŁ§ ļ +ðŁĺ µ +ðŁĺ · +ðŁĮ ³ +ຠ¥ +Ä ī +Ä ¥ +âľ ¶ +á¿ ¾ +âĬ ± +âĺ ¾ +ê° ī +ê¼ ° +ëº ij +ðŁĶ Ĭ +ðŁĸ IJ +Å ¤ +Ò « +à® ® +âĮ Ī +âĹ Ĺ +ëĦ µ +ëħ ľ +ëľ ¹ +ðĿij ¥ +ðŁĴ ¿ +ðŁĽ Ĵ +Ê Ĵ +áŀ ĵ +ðŁIJ Ŀ +ðŁ¦ Ħ +ðŁį · +âĺ Ł +ï¸ ¶ +ðŁ¤ Ł +Ô ± +âĨ ² +âĪ İ +âľ « +ëĩ ½ +ëı IJ +ëķ Ħ +ï¦ ³ +ï§ Ŀ +ïº Ļ +ðŁij » +ðŁĵ º +êµ ¼ +ìĮ © +ðŁĮ ² +È ± +íĶ ķ +ðŁĺ ¤ +ãĮ ¢ +Ê Ķ +ठ¡ +á¼ Ī +ëİ ĥ +ë© ± +ë® Ī +ðĿIJ « +âĬ ķ +ëĥ ł +ë» ¬ +íĭ Ķ +Õ ¤ +á¼ ± +âľ ¥ +âĺ Ħ +âĪ ¥ +âļ ķ +ðŁij Ħ +ðŁİ ħ +ຠĻ +âĶ ¬ +á½ µ +Õ ¾ +Ö ģ +âĹ Ķ +ê¿ į +ëĸ µ +ë© İ +ë® ´ +ìķ ´ +áĥ ľ +á¼ ¡ +âĶ Ĭ +âķ ® +âĹ ¼ +ðŁį ¾ +ðŁĽ į +ðŁij Ĺ +ðŁ¤ ŀ +âľ Ħ +Õ Ģ +ঠ² +Ë ī +⣠¨ +Ä ¯ +Ï Ĭ +á´ ľ +ë¹ ³ +ï³ ĭ +ï¿ ł +Ä ª +âĤ ¸ +âľ ± +ê» IJ +ëĭ » +ë§ ¸ +ìŀ ¿ +ì© ¨ +ì ŃIJ +ì° ¿ +íħ Ł +ðĿIJ § +ðĿij ij +ðŁĮ İ +ðŁĵ ® +ðŁķ Ķ +âĹ Ļ +âĹ » +âŀ § +ìŁ Ŀ +âľ ¬ +ãĥ ° +âģ Ī +â ĵĺ +ðŁ ĴĮ +ï¬ ĥ +ຠĶ +ìĶ ° +ðŁĺ ª +× Ģ +ìĥ ¨ +ïŃ ĭ +ðŁį ķ +ðŁĺ ´ +Ï ³ +á¼ Ħ +á½ ħ +âĩ ¢ +âķ Ń +ìĺ » +íĬ ¤ +Ü ĺ +⤠´ +âĹ į +áŀ Ł +ðŁį º +áŀ ļ +ðŁı Ĭ +ðŁIJ · +Ê Į +á½ º +âģ » +ê½ Į +ëĪ Ĺ +ë Ĺı +ì¿ ° +íĢ ¼ +íį ħ +ï· ² +ðŁĮ ı +ðŁį « +ðŁį ³ +ðŁİ ° +ðŁij ° +ðŁĴ ² +ᥠĻ +ðŁIJ Ł +ï¿ ¡ +ðŁĹ £ +ðŁį ľ +âľ ² +ãİ ¢ +ðŁĶ ° +á¼ ¸ +á½ ij +Ä İ +áĦ Ģ +âĻ ķ +ëł Ŀ +ìĪ ´ +ïŃ Ń +Ó ľ +Ô Ģ +ëĢ ľ +ëĥ Ķ +ìĬ Ľ +ì« ij +ìº ¥ +ìº ¬ +ðĿij ¦ +ðŁĶ ¶ +ì¾ ¨ +ðĿIJ ļ +ðŁį » +ðŁĴ į +ðŁ¤ ¡ +ðŁķ Ĭ +â½ ĩ +âĵ IJ +ðŁį Ń +ðŁį ª +ðŁĶ Ĩ +Ò ¡ +á´ ĩ +É Ĺ +Ü Ķ +âĦ İ +âĿ ĥ +ëĹ Ģ +ï² Ķ +ïº Ī +ðĿIJ » +ðŁĴ Ĭ +ðŁļ « +Ñ ° +Ñ ³ +ठ· +âĹ ł +ðŁij ¤ +ï¾ ĩ +âĺ ĵ +ðŁį µ +ðŁ¤ ¨ +âĸ Ń +à® ´ +Ü ¢ +Ü ¬ +à´ ® +ðŁķ º +Ô ¹ +Õ £ +à´ ¯ +á ´Ģ +âĮ ī +âľ IJ +âŀ ¦ +ê¹ ½ +ëĮ ľ +ðŁı ¥ +ðŁĵ © +Ò ¹ +Ó ĺ +ठħ +âĿ § +Æ Ĺ +âĹ ½ +ðŁij « +ðŁİ § +ðŁij £ +âľ » +ðŁĻ ħ +ðŁĺ ĸ +ðŁĴ ® +ຠ° +ðŁĶ ľ +ðŁį Ħ +ðŁ¤ Ŀ +á ĥĿ +áŀ Ģ +âĩ ¦ +Ê ¾ +Ò ® +Õ ¼ +ठĨ +âĹ ħ +âļ ĵ +âļ ĸ +ê¿ © +ë¯ Ħ +ìIJ IJ +ìŀ ° +ì§ Ń +íĭ ĭ +íİ ¨ +íĻ § +ï² ij +ðŁİ Ĺ +Ù ³ +ðŁij ¸ +ঠ® +ðŁij ķ +Ú µ +âĢ ¾ +âŀ ° +ðŁij ¯ +ðŁİ ¼ +ðŁı ģ +Ä º +Ê ı +Ú ³ +âı ± +ê½ Ī +ëĿ Į +ìĮ ī +ìĹ · +ìŀ ´ +íĹ ¹ +íľ ¨ +ðĿĹ ² +ðŁĮ IJ +ðŁİ Ļ +ðŁı µ +íĽ Ļ +ðĿij ħ +ðŁĺ ¶ +âĵ ħ +âķ ¥ +ðŁį ı +ï¦ İ +Õ © +ðĿIJ Ħ +Ó £ +Ú ¿ +âĻ ļ +ðŁĶ Ĺ +Ḡ« +âĭ ® +âĸ ¦ +⼠½ +âľ µ +ãħ Ĩ +ãħ Ĭ +ëĦ Ļ +ëĿ ¨ +ë¥ Ħ +ìĦ ¦ +ì§ ° +ì§ ¹ +íī Ī +ï§ ij +ï» ĩ +ðŁĮ ¾ +ðŁı ĸ +ðŁIJ ij +ðŁĴ ³ +ðŁĵ Ĩ +Û ĩ +Ü ķ +á½ ½ +ëĦ ľ +à´ ² +à´ ³ +àº Ń +áĥ Ľ +âĿ Ķ +âij ħ +áĥ ¥ +ðŁĵ ħ +âŀ ³ +á´ µ +ï¹ ¡ +ï¹ ¶ +Î Ĩ +ठ¥ +áī µ +âĿ Ļ +âĿ ± +ëī ł +ëİ ł +ëı Ľ +ë¿ ħ +ìĶ ¸ +íij ¯ +íŀ ī +íŀ Ľ +ï§ Ħ +ïŃ ĺ +ïº ¦ +ï» ¸ +ðĿij Ĥ +ðĿij ı +Ï ij +Ú ł +áĢ Ķ +áŀ Ķ +á¹ ¢ +ëĦ ¸ +ðĿIJ ¨ +ðŁĩ ´ +Õ ° +ðŁij ł +ðŁį Ĩ +ðŁı Ģ +ðŁ ijIJ +ðŁį ĩ +ðŁIJ £ +áĪ Ń +Ü ª +ðŁ ĮĢ +áŀ ĺ +âĩ Ħ +ðĿIJ Ģ +Ê Ļ +âĶ ¼ +ðŁı ¿ +Æ · +È ł +Ñ ½ +âĤ ¨ +ê´ Ń +ê¹ » +ëĶ ¨ +ìĪ Ģ +ì¾ ° +íĨ Ī +ï® § +ï¯ ½ +ðŁĶ ħ +ðŁĶ ® +Å ¢ +Ê ° +Ñ ¸ +ठ£ +âĬ Ĺ +ëª Ħ +ï¹ · +ïº ħ +ðĿIJ µ +ðŁĮ ¶ +ðŁĵ ° +ðŁĶ · +ðŁĸ Ĵ +ðŁ¤ ² +ëī © +ðŁİ Ĩ +ðŁ§ IJ +ðŁį ® +âĨ º +âĿ ¢ +ðŁij ª +ðŁij ± +âĨ ¡ +áŀ ı +Ú ķ +ðŁį ¹ +ðŁĴ Ģ +Ë ® +Ó ¨ +Ö ħ +ठĩ +âĤ ¡ +âĪ ķ +âĺ ī +ê¹ ¼ +ê¼ IJ +ì½ ¸ +ðĿIJ ¬ +ðŁı ħ +ðŁij Ļ +ðŁĴ ī +ðŁ¤ Ļ +È ĺ +É ³ +É ¹ +Ù º +áĢ Ħ +á¿ ³ +âļ ĺ +âĿ Ĩ +ëĨ ī +ìĸ į +ìĺ ĩ +ì¥ ĺ +íĸ ħ +íĻ ij +ï® Ĭ +ï¿ Ń +ðĿĴ IJ +ðĿĹ ¢ +ðŁĶ ĸ +ðŁĶ ¨ +ðŁļ ij +ðŁļ ² +Æ ¸ +âĹ ¥ +ðĿIJ Ń +ðŁį ½ +âĹ ij +âĵ ĩ +ðŁĶ ± +âľ ¼ +ï¹ ĥ +âķ ± +ãĢ Ĺ +ðŁı ĭ +ðŁļ ´ +ðĿIJ ® +Ä ļ +Õ ı +Ä ¶ +áĥ ij +á¹ ¬ +Ä Ī +Ä Ĵ +Ò ° +Ó ķ +â IJ +âIJ £ +âĹ ¢ +âļ Ļ +ãħ Ĺ +ê° ¬ +ê³ ª +ê» Ģ +ëĦ ´ +ëİ ģ +ëĿ Ķ +ë¬ ½ +ëŃ į +ìĩ ³ +ì° ¹ +íĮ ¹ +íŀ Ŀ +ï® ĭ +ï ¶Ī +ðĿĴ Ĥ +ðŁ¥ Ģ +ðŁ¦ ħ +Ê ĺ +á¼ ij +âģ İ +ðŁį ŀ +âĨ ĸ +âĨ Ļ +ðŁİ ĥ +âĦ ¡ +âĭ ± +ðŁĶ į +ಠ¨ +áµ ĥ +âĶ « +⦠¿ +ðŁĩ » +Æ ¤ +Ò ı +Ò · +Û ī +à® ķ +Ḡ³ +ï¬ ± +ðŁĨ Ķ +Ú Ń +Û ¦ +áħ ¡ +âĦ ¹ +ê¿ İ +ëķ Ķ +ë¼ ī +ìļ § +ì² µ +ì´ ¨ +íĬ Ī +íĸ IJ +ðĿĹ ĺ +ðŁĩ ¿ +ðŁİ ĸ +ðŁij ħ +ðŁ ĵĺ +ðŁļ Ļ +ðŁĽ µ +ච½ +⼠µ +ðĿIJ ³ +ðĿIJ ¸ +âļ Ķ +ðŁij Ń +Ó ij +âĶ ¯ +ðŁħ ¿ +ðŁĺ ¹ +ï¿ « +â¼ ¤ +ðŁĴ ĩ +ðŁĵ İ +ðŁĸ ĭ +ঠ¸ +ðĿIJ į +Ä ² +Ï ĭ +Ñ ¬ +Ú ¬ +Ü Ĵ +á´ ¬ +ï¨ Ħ +É £ +Ë ij +Ï µ +Ò Ŀ +Û ¥ +Ü ł +๠Ľ +áĥ ķ +áĬ ķ +á¾ ¶ +âĤ · +âĩ ¾ +âķ © +âĸ IJ +âĺ ª +âĺ ® +âĿ ļ +âĿ Ń +âŀ ± +âµ İ +ãı Ĭ +ë© ĵ +ìĹ ¾ +ìª Ħ +íĵ Į +íķ ¼ +ïŃ ¬ +ðĿij Ĩ +ðĿij ŀ +ðĿĸ Ĭ +ðŁİ ¸ +ðŁı Ħ +ðŁij µ +ðŁĴ ł +ðŁĶ ĺ +ðŁ¥ Ĥ +Å ª +à· ĥ +á´ ¼ +âĬ ° +ë³ ı +ë´ £ +ï¥ ľ +ðŁĵ Ī +ðŁķ ¯ +ðŁ§ Ģ +âĻ IJ +ðŁĨ Ĺ +ðŁĵ ķ +ðŁ§ ģ +Ü « +âĿ IJ +Õ ķ +འķ +âŀ Ŀ +ঠķ +ðĿIJ ¶ +É ¢ +Î Ħ +áĨ ¢ +âĤ ± +Õ į +à¡ ķ +á´ ° +Ḡ© +⼠· +âĿ ® +ê¡ ĵ +ëı ¤ +ëĹ IJ +ëµ Į +ìij Ī +íı ¿ +íĹ µ +ðĿIJ İ +ðŁĨ ĺ +ðŁı Ł +É ¥ +Õ » +à¡ Ķ +ठĸ +á´ ¸ +âİ Ļ +âİ ¥ +âı ³ +ëģ ķ +ëĬ ī +ì¡ į +ì¹ ¡ +ï¦ ¶ +ï¬ Ł +ï® « +ï® ¯ +ï± ĥ +ï ·» +ïº µ +ðĿĹ Ķ +ðĿĹ ¡ +ðŁİ ¨ +ðŁĶ Ĵ +Ú Ľ +ठ§ +âŀ ¹ +áĢ Ģ +ðŁį ħ +âĹ ¤ +ठł +ðŁIJ ¥ +áĥ Ĵ +ðŁı Ŀ +ðŁį ¼ +ãĮ § +âĿ Ľ +ðŁIJ Ī +ঠ¯ +áĢ ŀ +ãĢ ĸ +áŀ Ļ +ঠª +Õ Ĩ +âĬ Ĩ +âľ ¾ +ðŁIJ Ĺ +ï¹ ¿ +Ä ¦ +Ü Ł +ಠł +ಠ¥ +áŀ ī +á´ ¥ +á´ © +á½ Ģ +á½ ¡ +âĨ ķ +âŀ ¯ +ê¡ ij +ëij £ +ë± Į +ìĪ ij +ìľ Ķ +ìŀ ½ +ì¨ į +ðĿij Ģ +ðŁĮ Į +ðŁį ¦ +ðŁį © +ðŁIJ ļ +ðŁĵ Ĵ +ðŁĵ ¹ +ðŁ¥ ij +Ä ĭ +Ë Ĺ +Ñ « +Õ ¢ +Ú ° +â ĮĢ +âĹ Ĥ +âĹ £ +âľ Ľ +âĿ Ĵ +âĿ ĺ +âŀ Ļ +âŀ ² +ãİ į +ê¡ IJ +ëŀ ĸ +ìĬ Ŀ +ìĽ ¤ +ì¡ ĭ +ì¨ ° +íĹ Ļ +ï¥ ¸ +ï³ į +ï» İ +ðĿij ĵ +ðŁĵ Ĭ +ðŁļ ¼ +ï¦ ģ +ðĿķ Ĵ +ðŁ ijľ +ðŁij ¿ +ðŁĩ ½ +à· Ħ +âĸ ´ +ãį ī +âĬ ĩ +ðŁ§ ¸ +Ú ¡ +â¾ ĥ +ðŁĹ » +âĵ ij +ðŁ¤ ¸ +ðŁ¤ ¯ +êĴ ° +ðĿIJ ĵ +âĶ ´ +êĴ ± +áĢ ĺ +â ĽĦ +ï¹ ¹ +Ó Ķ +áĥ ± +Ü ¡ +ß ŀ +âĻ ı +âľ ¸ +ìij ¨ +ðĿIJ Ŀ +ðĿIJ ¥ +ðŁį ī +ðŁij ¼ +ðŁ¥ Ŀ +Æ Ķ +Ý ¬ +ठ« +ຠļ +á´ ´ +á½ ĸ +âĤ ¶ +âİ ¢ +âĿ ħ +⣠« +ãİ Ľ +ë® ¨ +ëº Į +ë¼ ĺ +ìĨ Ŀ +ìľ ³ +ìŀ Į +ì£ Ĺ +ìª ĺ +ì» ¹ +ï· ¼ +ïº Ĥ +ðĿIJ ´ +ðĿIJ ¼ +ðŁĮ ļ +ðŁı « +ðŁĴ ¤ +ðŁĴ ¶ +ðŁĴ ¼ +Ê ķ +Ê ½ +â² Ł +ãī ł +ê¡ Ĵ +ëľ Ģ +ìĥ ¾ +ì¸ ¤ +ï¥ ģ +ðĿļ Ĭ +ðŁļ ĥ +âŀ Ľ +ìħ ´ +áĦ ĭ +âĩ Ĺ +ï§ · +âĺ ĸ +ðŁIJ ¦ +⸠ľ +ðŁĴ ´ +ðŁ¤ ļ +ãĬ Ĺ +âĮ Ľ +áĪ Ľ +༠º +â½ ī +ðŁı ¢ +âĵ ŀ +âĺ ½ +ãĢ Ļ +ðŁ¤ ® +Å IJ +áĥ ¬ +ðĿĹ » +ðŁį ĸ +Æ Ĭ +Ê Ł +ß ĭ +ठĭ +áµ Ķ +á¿ ĥ +âĦ ī +âĮ ĭ +âı ² +âĵ Ī +âĵ ¢ +âķ Ķ +âļ ij +âĿ ĭ +âĿ İ +â µľ +âµ £ +ëĴ Ī +ëľ ģ +ë¶ ĩ +ìį » +ìĺ Ń +ì§ ¢ +íĹ Ģ +ï§ Ĭ +ï ¬¸ +ï± ¡ +ðĿIJ º +ðĿij § +ðĿĺ ¦ +ðŁĵ ¥ +ðŁĺ Ł +ðŁ¥ IJ +Ä ĸ +É ¨ +áĢ IJ +áĥ ĵ +Ạĵ +á¼ ¶ +á½ Ħ +âĤ ¤ +âĮ ľ +âĮ Ł +âİ ł +⼠¸ +âµ į +âµ ı +âµ ĵ +ãĢ ĺ +ë ·¸ +íħ ¼ +ï¦ Į +ïŃ Ħ +ïŃ İ +ðĿĻ ļ +ðĿļ ĺ +༠ĵ +ëŃ ħ +áIJ Ľ +ãİ ¾ +ï¨ Ģ +ðŁĹ ½ +âĻ ŀ +Ë ĸ +âĹ ŀ +ðŁ¤ « +ðŁĺ Ĺ +ï½ ¦ +ðŁ¤ ¢ +âģ ĩ +ãĢ µ +ðŁį Ķ +áĬ ł +ðŁĺ ¼ +ðĿĹ ® +ðŁIJ ³ +ðĿIJ ĭ +ðŁĨ ļ +ðŁĶ Ľ +Ñ » +Ü ¨ +à® ² +âľ ŀ +âµ Ļ +êµ £ +ì¸ ¨ +ðĿ IJľ +ðĿĺ ° +ðŁĶ ½ +Ç » +Ç ¿ +Ê ĩ +Î IJ +Ð Ģ +Ñ ¡ +Ñ ² +Ò Ĵ +Ù ¶ +ß ķ +ච± +áIJ ģ +âģ ŀ +âĸ § +⼠Ī +âľ ľ +âľ ¹ +⣠¹ +⤠ĩ +ê² Ĭ +ê¾ ľ +ë¯ IJ +ë³ IJ +ìħ © +ìIJ ¬ +ìij ¹ +ï¤ Ķ +ï¦ ļ +ï¬ ł +ïŃ Ķ +ïº ¶ +ðĿĴ ı +ðĿĸ Ĩ +ðĿĹ ¶ +ðŁı Ĥ +ðŁIJ ½ +ðŁĴ © +ðŁĵ ½ +ðŁĹ ¨ +ðŁĹ º +ðŁĺ ¸ +ðŁ¥ § +Å Ĺ +Ê İ +Ò Ļ +× ² +ठĪ +á¼ ´ +á¿ ij +âµ ī +ãħ ĵ +ì½ ´ +ðĿĸ ĵ +ðŁĵ Ĺ +ðŁĶ ª +ðŁĸ į +Ï Ĵ +ðŁij ¬ +áĥ Ļ +âĨ ¬ +âĶ ¤ +⼠¹ +âĻ Ł +ðŁļ ¶ +ðŁij ¾ +âĪ ĭ +ðŁIJ ¯ +à¼ İ +âľ · +ï¨ Ļ +âĶ » +ðŁij ¹ +áĦ ī +ຠª +â¾ ı +â½ ħ +ãİ ĸ +Ñ ´ +Õ ® +Ú ¼ +áĢ ķ +áĨ ¼ +ëŃ ı +ðŁIJ ¸ +ðŁļ £ +Æ Ŀ +Ô » +áĥ ¢ +ðŁį ¯ +É ¦ +Õ ¦ +âĻ ĭ +ï¬ « +ðĿĹ ¦ +Ç ļ +É ± +ठī +á´ Ħ +âĻ ĵ +⼠° +⣠ª +ëĥ ĺ +ë¢ ¸ +ìĤ ij +ï® Ķ +ðĿķ ĸ +ðĿĹ § +ðŁĩ ¼ +ðŁĵ ĭ +ðŁļ ľ +ðŁ¥ ¤ +Ä ® +Å · +ß Ĭ +ॠ¥ +à® ª +áŀ Ħ +áµ Ģ +Ḡħ +á¼ ¢ +âĪ Ŀ +âĬ ¹ +âĴ ¶ +âķ ´ +⼠± +⼠³ +⼠º +âŀ Ł +ãı Ħ +ê¸ Ķ +ê¹ Ł +ëĩ ° +ë¹ » +ìĤ ¥ +ìĽ » +ì° Ł +íĥ ° +íĨ º +íļ ½ +ï¤ ´ +ï¥ ¾ +ï³ Ŀ +ðĿIJ ¦ +ðĿĴ ľ +ðĿĴ Ł +ðĿļ Ĺ +ðŁİ Ń +ðŁı ĵ +ðŁı ³ +ðŁı º +ðŁIJ į +ðŁij ĥ +ðŁĴ ı +ðŁ¤ ĸ +ðŁ¤ µ +Õ ² +âµ Ķ +ëĺ ¬ +ï¦ £ +Ê Ĥ +áĨ « +áŀ ij +ðĿĸ İ +ðĿĹ ĸ +áĦ ĥ +âĩ ł +áĢ ¡ +འĦ +âŀ ¸ +ï¦ Ļ +âĩ ļ +ðŁIJ ¬ +ðŁIJ ¢ +â¾ Ĵ +ðŁIJ ¤ +ðŁĶ « +ãĢ ŀ +ï¸ º +ðŁĺ º +â½ ´ +ðŁĨ ķ +âģ ¿ +ðŁį ¨ +ಠķ +ðŁļ ĺ +áŀ ħ +ঠħ +áŀ ¢ +ਠľ +â ļĮ +ãĢ ½ +à· ´ +âĵ Ľ +áĢ ľ +ìĨ ¨ +Ë © +Ü Ĺ +âĭ ¼ +ðŁĻ ī +Å Ĭ +É ĵ +Ê ² +Î ° +Ñ ¼ +Ô ¿ +à¡ IJ +༠ľ +འ¦ +ᶠľ +âĤ ² +âĨ ¨ +âĬ ¥ +âķ § +âĻ ľ +ãĭ ¡ +ë´ ¬ +ë¶ ij +ìī ¿ +ìİ ħ +ìł ± +ì° § +ï² ¡ +ðĿĴ Ľ +ðĿķ £ +ðĿĹ ľ +ðŁį ² +ðŁİ © +ðŁIJ IJ +ðŁIJ ł +ðŁij ½ +ðŁĴ ij +ðŁĵ ľ +ðŁķ µ +ðŁ ļĮ +ðŁĽ £ +Ê ĭ +Ó ¯ +Ù ¸ +ß Ķ +ß Ļ +à¡ ĵ +á´ į +Ḡ¿ +âı º +âĸ ¥ +ë¤ ½ +íľ ij +ðĿIJ ¹ +ðĿĸ Ķ +ðĿļ İ +ðŁĵ Ħ +ðŁ¦ · +Æ ĥ +ঠŁ +âĮ Ĥ +âĺ Ń +â² ļ +ëĿ ķ +ðŁİ £ +à® ĩ +འĨ +áħ µ +áĹ ľ +âĢ ½ +âĮ £ +âģ ½ +ðŁĵ ¬ +ðŁ¤ § +âĩ ª +â½ £ +âĹ Ł +ï¨ Ĺ +êĴ ª +ðŁĽ Ģ +Ç Ĥ +ðŁ¥ ¶ +ðŁİ į +ï¿ © +ðŁij Ĵ +áµ Ī +ï¸ ¿ +áħ © +â¾ ¦ +à° ¤ +á´ ĸ +ਠ¬ +àº Ĺ +༠» +Ñ º +ਠª +á´ ³ +ðĿIJ Ī +à» Ģ +á´ ¿ +âĤ į +âĩ ¡ +⼠ª +ðĿIJ Ĥ +ðĿĴ ķ +ðŁ IJľ +Ê į +Ñ ± +འĥ +ë® IJ +ìĽ ¡ +ìľ ģ +ðĿIJ ¿ +ðĿķ ł +ðŁij Ľ +Æ ª +Ï º +Ó ¬ +Ù ¿ +Ý £ +ઠī +à® ¹ +འij +áĨ ¯ +áµ ĩ +âĩ ¥ +âı ª +âĻ ° +âļ Ń +âļ ¾ +ãħ Ħ +êĢ ° +ê° Ĺ +ê² ĭ +ê² » +ê¶ ľ +ê¼ ĩ +ê½ ¹ +ëĤ Ł +ëħ Ī +ëĭ ¢ +ë§ Ł +ëª Ĩ +ëµ Ģ +ì½ ± +íĩ ĺ +íľ ľ +ï§ ¾ +ï± µ +ï² ¢ +ï² ¤ +ðĿĴ Ĭ +ðĿĺ ¯ +ðŁį Ĺ +ðŁı į +ðŁIJ ĺ +ðŁĵ ¡ +ðŁĶ ŀ +ðŁ¤ ³ +ðŁ¥ ģ +ðŁ¥ Ĺ +ðŁ¦ Ĭ +Ä µ +Æ ¦ +Ç µ +É ¯ +Î ı +Õ Ħ +Ü ¥ +འģ +ᨠł +âķ « +ãİ ī +ë· ´ +ìĨ İ +ìİ Į +ì£ µ +íĽ ł +ï§ ª +ï³ ı +ï» º +ðĿij ģ +ðĿij ĩ +ðĿĴ Ĩ +ðŁİ ł +ðŁIJ Ķ +ðŁij Ł +Å ĸ +ठĮ +á¾ ½ +ê¦ Ĵ +à® Ł +á´ ± +ðŁı ° +ðŁIJ ŀ +འĢ +áĢ ħ +âĬ ¿ +ðŁIJ § +ἠģ +â¼ Ī +âĶ ¿ +ðŁ¥ ´ +â¼ ¿ +ðŁ§ ľ +ãħ ¿ +âĦ « +ãĢ ³ +ãĬ Ļ +â¼ Ģ +ï ¦¬ +ðŁı ¬ +ðŁĵ » +áĬ Ľ +áĦ ħ +ຠĬ +ຠĽ +áħ ³ +ðŁij ® +à® ± +âĺ ĩ +ðĿIJ ı +à´ µ +à» ģ +འı +འ¢ +ᥠ± +âĤ £ +ï¥ ¦ +ïŃ Ļ +ï´ © +ï¹ Ĥ +ðŁį £ +ðŁķ ¹ +Ï ĸ +ච¸ +ຠ¢ +áĭ Ń +âİ Ŀ +âĹ Ŀ +âĻ Ī +âĻ İ +ê½ ¥ +ì³ Ķ +ì¼ ij +ï± ° +ðĿij ĥ +ðŁĮ ª +ðŁį ¡ +Å İ +Ê ¦ +Ñ § +Ó İ +Ô ´ +Ú Ī +ß ĵ +ß § +ठĶ +áĪ « +áĪ µ +áĹ © +á´ ł +á¼ ł +âĢ Ĺ +âģ ij +âĦ ı +âĸ ĩ +â² £ +ãĦ ³ +ãī ® +ê³ Ĺ +ëĦ Ĵ +ëĸ « +ë¡ Ħ +ë¹ ° +ë½ ģ +ìĦ ģ +ìĮ ĺ +ìŁ Į +ì³ ī +ì¼ ķ +ï¬ » +ï³ İ +ï¹ ¸ +ï¹ ¾ +ðĿIJ Ĩ +ðĿij · +ðĿĽ ¼ +ðŁİ ı +ðŁİ ŀ +ðŁIJ Ļ +ðŁij Ĥ +ðŁĵ ģ +ðŁĸ ± +ðŁļ į +ðŁļ § +ðŁĽ ¡ +ðŁ¤ Ĵ +ðŁ¥ ŀ +ðŁ¥ © +ðŁ¦ Ģ +ðŁ¦ ĸ +Ë ¢ +Ü ļ +à® µ +áĢ ģ +áī ° +âı Ń +âĻ ¿ +ê³ ĺ +ëı Ŀ +ëķ ĥ +ìħ Į +ìĴ ¸ +ìĽ Ł +íħ Ħ +íľ « +ï§ ĺ +ï¿ ¬ +ðŁı · +ðŁĶ § +ðŁ¥ Ī +Æ ĸ +áŀ ĩ +áŀ ĸ +âģ º +âĹ ľ +âŀ © +ê¦ Ń +ëĻ ¤ +ïŃ ¼ +ðĿĻ ĸ +ðĿĻ £ +ðĿĻ ¤ +ðŁĮ Ŀ +ðŁĶ ij +ðŁĽ ł +ຠĩ +âĺ £ +ãĦ ¨ +ðĿĸ Ĺ +Ó ĵ +âĨ £ +ðŁ¥ ī +ðŁĮ ł +ðŁĺ ½ +ãİ ł +Å § +ðŁIJ Ĵ +ï§ IJ +ðŁĺ ¿ +âĪ ¬ +ðŁIJ ® +⣠± +ಠ¡ +â¾ ¼ +à° ² +Ë ¶ +âĸ ¿ +Õ Ī +áŀ İ +áħ ¥ +áŀ Ĺ +Õ § +ðŁ¤ IJ +ðŁį ł +ঠ¤ +චº +âĻ į +ìĺ Ļ +íĺ ĵ +ï¹ º +ðŁĽ ³ +Å ī +á´ İ +âı ľ +âĶ ³ +ê¸ · +ì¡ Ķ +ðĿĴ Ī +ðĿĴ į +ðĿĴ ¹ +ðĿĵ ĩ +ðĿķ Ł +ðĿĹ ¹ +ðŁĮ ħ +ðŁı ´ +Ä Ķ +Ä ¤ +Å µ +Ç ¾ +Ï ŀ +Ï ¶ +Ô ³ +Ü Ĩ +ß © +à¡ Ĵ +ठĺ +චļ +འĸ +áģ Ĭ +áĥ ŀ +áĦ Ĥ +áĭ « +á´ º +Ḡ£ +Ḡª +á¹ Ĥ +á¼ · +á¿ ĩ +âĩ Į +âı ¬ +âĻ Į +â® Ł +â´ » +âµ Ł +ê¦ ķ +ê¦ ª +ê¦ ® +ê² Ħ +ê¾ IJ +ëĥ ij +ëķ ĭ +ë¡ ¸ +ë¬ Ģ +ìĩ ¤ +ìĪ © +ìľ ķ +ìŃ ĺ +ì· ° +ì ·¸ +íľ Ģ +ï¤ £ +ï§ į +ï± Ħ +ï³ ij +ðĿIJ ¤ +ðĿĴ ĵ +ðĿĴ ¶ +ðĿĹ ¼ +ðĿĻ Ĭ +ðŁĩ ¾ +ðŁĮ Ľ +ðŁĮ ® +ðŁİ ĩ +ðŁİ ² +ðŁı Ľ +ðŁij ¥ +ðŁij ´ +ðŁĴ Ĩ +ðŁĵ Ĥ +ðŁĵ § +ðŁķ IJ +ðŁĸ ķ +ðŁĺ § +ðŁĻ Ģ +ðŁļ Ĵ +ðŁĽ « +ðŁ¤ ł +ðŁ¥ ļ +ðŁ¥ Ľ +ðŁ¥ £ +Ç ¯ +È § +Î Ĭ +Ò ² +× ° +Û ij +áĥ © +áĦ Į +áĪ į +áī ¥ +áı Ĥ +âģ ± +âĬ ¢ +âĹ ĵ +âĿ ° +ë¿ ¡ +ìĽ © +íģ Ń +íĨ ³ +íĬ Ħ +íĵ ¸ +ï¥ £ +ï¥ ´ +ï± IJ +ï± ¯ +ï³ ļ +ðĿĸ ĺ +ðĿĺ Ģ +ðŁIJ Ĭ +ðŁIJ Į +ðŁij ļ +ðŁĵ ĥ +ðŁļ Ľ +ðŁļ ª +ðŁ¤ ° +Ä ´ +áĥ ® +áĹ ¨ +âĻ ® +â² ŀ +ãĪ Ķ +ì ħį +ãħ ĥ +ï¥ ¡ +ຠ¡ +Õ İ +Õ º +⬠Ľ +â½ ¤ +ðĿIJ ² +âŀ µ +áĢ Ľ +âĶ ħ +âĨ Ł +â¼ Ĭ +ðŁĮ ½ +ðŁļ ¿ +ï¦ Ĭ +ãĦ £ +⼠© +ï© Ľ +ðŁį ± +â¾ ¨ +à´ ¤ +áŀ ģ +ຠŀ +Ê ļ +ðĿIJ Ĵ +à´ ± +áŀ ľ +à® © +à° Ĺ +à´ ļ +âĩ £ +ï¦ ķ +Õ ħ +Æ ĺ +âĤ ¦ +âĶ Ħ +ï¦ Ł +ï¦ « +ðĿIJ ģ +ðĿIJ ĥ +ðŁį ¸ +ðŁIJ ² +Å ¶ +É ĸ +ß ĺ +ภ¦ +འĶ +áĨ · +âģ ķ +âĵ Ĥ +âĿ ľ +ï¥ ¥ +ï¬ ® +ðĿĹ Ŀ +ðĿĹ ¿ +ðŁİ ¾ +ðŁĹ Ŀ +ðŁ¦ Į +Æ ħ +Ç ª +Ò Ĺ +Ü Ľ +ß ł +à¡ ij +áī £ +áĬ Ń +á¹ ¡ +âŀ ¼ +âŀ ¾ +â´ ± +ãī ¡ +ê³ ¯ +ë½ Ī +ìĤ ĺ +ìī ij +ì «ĺ +íĮ ĥ +íĻ ° +ï¤ Ĺ +ðŁĮ ¬ +ðŁĮ ° +ðŁį ¤ +Ä » +Å ĩ +Æ ¨ +É ķ +Ò ¢ +Ò º +Ö į +× ± +Ú ± +Ú ½ +Û IJ +ठĽ +à· Ģ +๠ļ +ຠ« +á´ ¹ +á ½Ķ +á¾ ³ +âĤ Ĵ +âĨ ´ +âĩ Ŀ +âī ħ +â Į¨ +âĵ ĵ +âĸ ¢ +âļ ¬ +âŀ Ń +â² Ĵ +ãİ ¿ +ê¿ ´ +ëĪ ± +ëį ¬ +ëİ IJ +ëIJ « +ëĶ « +ë± ģ +ìĥ ¥ +íĮ ¼ +ïŃ ĵ +ï® ¥ +ï² ° +ðĿIJ ĩ +ðĿIJ ij +ðĿij Į +ðĿĵ ª +ðĿķ ļ +ðĿĺ ª +ðĿĺ ¼ +ðĿļ Ľ +ðŁĩ ¶ +ðŁĮ Ħ +ðŁĮ ķ +ðŁĮ ¤ +ðŁĮ § +ðŁį ¬ +ðŁİ ĭ +ðŁİ » +ðŁı ¨ +ðŁIJ ĩ +ðŁij ĵ +ðŁĵ IJ +ðŁĵ Ļ +ðŁĶ ¼ +ðŁķ Ĵ +ðŁĸ ı +ðŁĸ ¥ +ðŁ¤ ¬ +ðŁ¥ Ĭ +ðŁ¥ Ĵ +ß Į +ຠĦ +á¼ µ +âķ ¡ +â² ¤ +â´ ¼ +âµ ¢ +ãĪ ¯ +ëĵ ¸ +ëŁ ĩ +ëº į +ðĿĻ § +ðŁį Ī +ðŁĶ ¬ +ðŁĸ Ĭ +ðŁ¤ ¾ +Ë ¡ +Ü © +âĮ ¡ +âŃ ij +â² ¦ +ë© ī +ì¼ Ń +ï¿ ¤ +ðĿĴ İ +ðĿĹ ¥ +ðŁIJ µ +ðŁķ ¶ +ðŁķ ¸ +ðŁ¤ ľ +Õ ª +áĪ ĭ +ðŁ¥ µ +ï° ģ +áµ IJ +âķ ĵ +áĢ ĸ +âĭ Ī +É ŀ +âŀ ® +ॠ° +ãĨ ģ +ðŁĴ ± +ðŁı Ń +áĨ ¨ +ðŁį ļ +ðŁ¦ IJ +á´ » +âĺ Į +à´ ķ +Õ ± +áħ ® +ðĿIJ Į +Å ¦ +ຠķ +âľ Ļ +Ë ³ +Ô µ +âķ Ĵ +ðĿĹ Ĺ +ðĿĹ ł +Ú ļ +ঠ§ +âĨ Ŀ +âĻ ī +ãĮ » +ì¹ Ĭ +ðĿĹ º +ðŁ§ ĺ +ì³ £ +ï¬ Ŀ +ðŁij º +Ç Ł +Î Ī +Î « +Ñ ¥ +Ô ² +Õ ¨ +Ü ¦ +ঠĨ +ঠ¥ +áIJ ¢ +á¼ ģ +á¼ ĺ +á¼ ¦ +âĵ Ŀ +ãĪ ° +ãİ Ĺ +ê² ¡ +ë¨ Ģ +ì£ Ķ +ì´ ¤ +ìµ Ŀ +ï§ ´ +ïŃ Ĭ +ï² Ł +ðĿIJ · +ðĿij ĭ +ðĿĵ ī +ðĿĺ µ +ðŁĴ · +ðŁĽ © +ðŁ§ ¹ +Å Ķ +Ê ŀ +Ë ¥ +Î Į +Ñ © +Ó IJ +Ó ł +Ú ij +Ú Ĵ +ß ¨ +ઠĪ +áIJ ĥ +á¹ ¯ +âĤ ĭ +âĤ µ +âĦ ħ +âĦ ł +âĪ £ +âī º +âī » +âĬ Ľ +âĮ IJ +âİ ĵ +âĺ ¸ +âĻ Ĵ +âļ Ĵ +âľ ĩ +âľ ł +â´ · +âµ ĸ +ãĦ ¸ +ãī ¢ +ãī ° +êĩ ´ +ê´ ¸ +êº ł +ëĤ ı +ëĤ ¢ +ëIJ Ģ +ëº ´ +ìĥ ľ +ìį ħ +ì¤ « +ì± ¦ +ìº ij +ì¼ ģ +ì¿ ³ +íĤ ģ +íħ ¡ +íĴ Ĥ +íĴ ī +íľ Ħ +ïŃ ª +ï® ¬ +ï¯ ¦ +ï± ª +ï² ı +ï ´Ģ +ï» Ĩ +ï¿ ¦ +ðĿij Ĺ +ðĿĸ Ļ +ðŁĮ ¡ +ðŁį Ŀ +ðŁį § +ðŁİ « +ðŁı ĺ +ðŁı ª +ðŁIJ ĭ +ðŁIJ Ľ +ðŁIJ º +ðŁij ĸ +ðŁij ŀ +ðŁij · +ðŁĵ Ģ +ðŁ ĶĦ +ðŁĶ Į +ðŁķ Ļ +ðŁĻ į +ðŁĻ İ +ðŁ¦ į +Ç ° +É Ł +Ê Ĩ +Ô ¼ +Ú ľ +ঠ¡ +ঠ¶ +áĴ ĥ +á¼ © +âĵ ķ +â² Ī +ê° ° +ê¹ ł +êº ħ +ëĦ ¹ +ë¯ ĵ +íIJ Ī +ï§ ¶ +ï® ij +ï² ¨ +ðĿĴ ī +ðĿĴ Ķ +ðĿĹ ¨ +ðĿĻ ŀ +ðĿļ Ĵ +ðĿļ ķ +ðŁIJ İ +ðŁ¤ ķ +ðŁ§ Ķ +Ï ° +Ô Ŀ +âĮ Ĭ +âĴ ¾ +ãī £ +ïŃ © +ðĿļ ŀ +Ê ij +ঠ¦ +áĦ ĩ +âī ĥ +â² Ģ +ìŁ İ +ðĿij ¶ +ðĿĵ ² +ðŁ İ· +ðŁļ ¹ +ຠģ +áł ł +ãĦ ļ +ðŁIJ ¿ +ἠļ +âķ ³ +ðŁIJ Ń +âĴ ¹ +ðĿĸ ļ +âĻ ĸ +ãĪ ² +âĨ ¾ +áĦ Ĩ +âķ Ľ +ðŁ¤ į +â½ ¥ +ðŁ Į¨ +âĪ ® +ãĮ ĺ +ãį ij +ï¹ Ģ +âĵ Ĺ +âĬ Ħ +ðŁı ¹ +Ë Ĵ +ðŁ¤ ± +ãı ľ +ðŁİ Į +ï¥ Ń +ঠ£ +ðŁİ ¹ +ãĬ Ł +à´ ° +ðĿIJ Ķ +à´ ¨ +འļ +âľ º +Õ · +ðŁij ³ +ঠľ +âĺ ĭ +âĻ Ĭ +ãĢ Ľ +È ĭ +à® ° +áĥ ¨ +âĦ ķ +íij Ģ +ðĿĵ ĥ +ðŁ¦ Ķ +Ä ¿ +Å Ģ +Æ ³ +É ļ +Ö ĥ +Ü £ +ß Ł +à¦ Ń +ৠ¡ +ච» +ຠ£ +འĩ +Ḡ¨ +á½ Ī +â½ ¬ +ê¡ Ķ +ì³ Ħ +ï¨ ī +ðĿIJ ¡ +ðĿĺ ¢ +ðŁį ¿ +ðŁİ Ł +ðŁı ī +ðŁĶ IJ +ðŁļ ħ +ðŁ¤ ½ +Æ į +Ç « +Ç ½ +È ļ +Î ī +Ó ¤ +Ó ª +Õ Ĭ +Ù ¼ +Ú ´ +ß Ŀ +චľ +á¼ ķ +á¿ ¥ +âİ ŀ +ãĢ ļ +ãī ¤ +ê³ ¸ +ê· ģ +ëĵ Ħ +ëĵ ķ +ì¨ Ķ +ì± ¨ +ðĿIJ ¾ +ðĿij » +ðĿĶ ¼ +ðĿķ Ŀ +ðĿĺ Ń +ðŁĨ Ļ +ðŁĵ ¤ +ðŁĶ Ł +ðŁĹ ¼ +Ä ľ +Æ ģ +Æ ¿ +Ç ³ +Ç · +É ĥ +É ł +Ê ī +Ê § +Ë ² +Ï ´ +Õ ģ +Õ ŀ +Ö ĩ +Û Ĥ +Û ĵ +ß Ĺ +ß ¦ +ঠ¹ +à® ³ +à´ ¸ +à» Ĥ +áĪ Ŀ +áĪ ª +áĭ µ +áIJ Ĭ +áĴ ª +áļ ĸ +áŀ Ľ +á´ ¢ +áµ ı +áµ Ń +ᶠ« +Ḡı +ẠĴ +á¼ ¥ +á½ ķ +á½ ¼ +âĤ Ĭ +âĦ Ĥ +âĦ © +âĩ ī +âī £ +âĮ ł +âİ Ł +âı ® +âķ ĺ +âĹ ĸ +âĺ © +âĻ ij +âĻ ² +âļ Ľ +ãĦ Ł +ãī ± +ãİ ļ +ê¡ ķ +êª ĸ +ê° ¹ +ê² Ĩ +êµ Ħ +ëĩ ¬ +ëĭ ¯ +ëı ł +ëĴ ¬ +ëĸ Ī +ëĸ ½ +ëĺ Ķ +ëŀ ¸ +ë¸ ħ +ë» ł +ë¿ Ł +ìĤ µ +ìĬ ī +ìľ ° +ìł ĭ +ìł Ķ +ì¥ ¡ +ìŃ Ŀ +ì¼ ¬ +íĪ ĩ +íī ľ +íį Ħ +íĽ ¾ +íĿ £ +ï¤ © +ï¤ ¯ +ï¦ ľ +ï¦ § +ï§ ľ +ï¨ Ī +ï¬ ª +ï ¬´ +ïŃ ½ +ï® ī +ï¯ ŀ +ï° Ĵ +ï± ĩ +ï¿ Ħ +ðĿIJ ħ +ðĿij Ħ +ðĿij º +ðĿĴ Ĺ +ðĿĵ ® +ðĿķ Ľ +ðĿķ ŀ +ðĿĸ ij +ðĿĺ ģ +ðĿĺ Ĩ +ðĿĺ ¶ +ðĿĻ ¢ +ðĿļ ľ +ðŁĮ ĥ +ðŁĮ ¦ +ðŁį Ł +ðŁİ İ +ðŁı Ļ +ðŁIJ © +ðŁIJ « +ðŁIJ ´ +ðŁij Ķ +ðŁĵ ī +ðŁĵ Ľ +ðŁĶ ī +ðŁĸ ¼ +ðŁĹ ĥ +ðŁĹ ¯ +ðŁļ ĩ +ðŁļ IJ +ðŁļ µ +ðŁ¤ ¶ +ðŁ¥ ĭ +ðŁ¥ ĵ +ðŁ¥ ® +ðŁ¦ İ +ðŁ¦ ł +ðŁ§ Ĵ +ðŁ§ ¨ +Æ IJ +Ç į +Ó Ģ +Ô Ľ +ಠ° +à´ Ļ +áĢ Ĵ +ê² Ŀ +ê¹ ¹ +ë© ¥ +ìĸ Ķ +ï¤ ģ +ï¤ ı +ï¦ ī +ï¦ ĵ +ï§ ī +ï² Ŀ +ðĿĹ ŀ +ðĿĹ ± +ðŁĮ ĭ +ðŁį ¶ +ঠļ +ìķ ľ +ðĿIJ ¯ +ðĿļ Ŀ +à° ¨ +འĺ +འł +á¡ ¥ +á¾ ° +âģ į +âĶ ° +⬠ľ +ðĿIJ ł +ðĿij ¯ +ðĿĹ Ľ +ðĿĵ » +ðĿĸ Ī +âŀ » +áŀ ł +â¡ ± +â» ij +ðŁ§ µ +ï¦ ¢ +ðŁij ĺ +ãĤ Ķ +â¼ Ł +ãĬ ¤ +ï¦ Ŀ +ãĮ ¦ +âĢ ¸ +ðŁĶ Ļ +ã ¹ +ã¹ ¦ +ï¹ ħ +ï© Į +ãī ¨ +ï¸ ½ +âį ¥ +ðŁļ ī +ðŁ¥ ľ +âĵ ľ +â» Ŀ +ï¨ ľ +ðŁĴ Ĵ +áĦ ij +â¾ ŀ +ï¨ ģ +à´ ª +áĦ İ +âŀ ´ +ঠ· +áħ ¬ +áŀ § +âĨ ¢ +âķ ¦ +âľ ij +Ë ¬ +Õ IJ +༠Ķ +Ê ¤ +Ë ¨ +ठŀ +à» ĥ +༠ļ +âĵ ¥ +âķ ľ +ðŁIJ ĸ +á¼ Ļ +á¼ ¤ +ìĨ ° +È Ĥ +Ê ± +à® ļ +áĥ § +á´ ĭ +á´ ® +âĿ ¡ +âŀ · +ëĿ ¡ +ï§ ¢ +ï¯ ¡ +ðĿķ ķ +ðŁħ ° +ðŁ¦ ¸ +Ç ¸ +Ó ŀ +Ô ¶ +Ö Ĩ +Ú ģ +Û ĭ +áİ ¥ +á¾ ¿ +âĶ Ń +âĶ ® +êĢ Ģ +ê± ĺ +ëIJ Ń +ë½ Ħ +ìĶ IJ +ì¸ Į +íģ ł +íĻ ± +ï¥ ī +ï¨ ĸ +ðĿij ´ +ðĿĸ Ĵ +ðĿĺ ¨ +ðĿ ļĮ +ðŁIJ ¡ +ðŁij ¢ +ðŁĵ Ķ +Å ħ +Æ İ +È © +Ò ª +Ô ĥ +áĥ « +Ḡĩ +⼠Ł +ê» Ń +ë¨ Ħ +ìŁ Ģ +ì¤ ´ +íļ IJ +ï¤ ³ +ðŁŁ ¢ +Æ § +È ¼ +Ê Ŀ +Ë Ħ +Ë ħ +Ë į +Ë § +Ò ¥ +Õ Ķ +Ø ı +Ø ¼ +ß IJ +ß ľ +ठĵ +ঠĻ +à® ĵ +ච´ +༠į +༠Ĵ +འ£ +áĢ Ĥ +áĢ Ĭ +áĦ Ħ +á Īĺ +áĭ Ĭ +áĮ į +áij ĭ +áŀ Ĥ +áł ¢ +á¡ Ŀ +á´ ¦ +áµ į +áµ ¨ +Ḡ¡ +Ḡ¯ +á¼ £ +âģ Ĥ +âĦ ĺ +âĦ ľ +âĦ ³ +âĦ µ +âĨ ¦ +âĩ Ĩ +âĪ · +âĬ ļ +âĮ « +âĮ ¯ +âİ Ľ +âİ ľ +âİ ¤ +âİ ¦ +âİ ® +âij ī +âĶ ī +âķ Ļ +âĸ Ĥ +âĹ Ń +âĺ Ĭ +âĺ į +âĺ Ĵ +âļ Ĩ +⼠§ +⼠² +âŀ ĺ +⥠Ħ +â´ ³ +â´ ½ +âµ Ī +ãī ¯ +ãİ ij +㧠¬ +êĻ ¬ +ê§ ģ +ê³ ¬ +ê´ ŀ +ê» ľ +ëħ ĵ +ëĭ ¼ +ëį ĸ +ëĸ ± +ëĿ ° +ë¡ ¹ +ë¢ ´ +ë£ Ģ +ë¤ ł +ë¨ ķ +ëŃ ¥ +ìĦ ¶ +ìħ ¤ +ìĮ ķ +ìį ª +ìı © +ìĴ Ģ +ìĶ ¯ +ìĿ Ķ +ìĿ ľ +ìł Ń +ì§ ¦ +ì¨ © +ì² ¬ +ì³ ¥ +ì¼ ¯ +íĢ « +íĢ Ń +íĥ ¸ +íĵ ģ +íķ ¬ +íĹ ¸ +íĽ ķ +íľ Ń +íĿ Ĺ +ï¤ Į +ï¤ ª +ï§ ¿ +ï¬ Ħ +ï¬ ħ +ïŃ ij +ïŃ « +ïŃ º +ï® Ĥ +ï® ¢ +ï® ¨ +ï° İ +ï° ł +ï² £ +ï³ IJ +ï³ Ĵ +ï³ ĺ +ï³ ľ +ï¹ ¼ +ï¿ ¨ +ðĿIJ © +ðĿĴ ļ +ðĿķ Ķ +ðĿķ ¤ +ðĿĸ Į +ðĿĹ £ +ðĿĹ ° +ðĿĹ ´ +ðĿĺ Ĥ +ðĿĺ ¥ +ðĿĺ ® +ðĿĺ ¸ +ðĿĻ Ģ +ðĿĽ ¾ +ðĿľ ı +ðŁĮ ģ +ðŁĮ ľ +ðŁĮ ¥ +ðŁĮ ¯ +ðŁį IJ +ðŁİ Ĵ +ðŁı Ķ +ðŁı ķ +ðŁı ® +ðŁIJ Ĥ +ðŁIJ ī +ðŁIJ ¹ +ðŁĶ ķ +ðŁĶ ļ +ðŁķ ij +ðŁķ £ +ðŁĹ ŀ +ðŁĹ ¡ +ðŁĹ ¿ +ðŁļ Ĩ +ðŁļ Ĭ +ðŁļ ĵ +ðŁļ ķ +ðŁļ ¾ +ðŁĽ ģ +ðŁĽ İ +ðŁĽ ı +ðŁ¤ ´ +ðŁ¥ ķ +ðŁ¥ ĸ +ðŁ¥ ł +ðŁ¥ ¥ +ðŁ¦ Ĩ +ðŁ¦ ī +ðŁ¦ ļ +ðŁ§ ij +ðŁ§ ¥ +ðŁ§ ¿ +Å ° +Æ º +É § +ઠĩ +à® £ +áĪ Ī +áĬ ¤ +áĭ ® +áĮ Ī +áĮ µ +ᥠ² +âĵ Ł +êĻ ³ +ê° Ĭ +ëķ ģ +ëķ ¨ +ìĬ ģ +ï¦ µ +ï¬ ² +ðĿĸ į +ðĿĺ Į +ðĿĺ ³ +ðĿĻ © +ðŁį Ļ +ðŁĸ ĸ +áī ³ +áĭ ¨ +áĸ ĩ +áŀ Į +á¹ § +âķ ª +âŀ ļ +â² ĺ +ê ķ +êķ ¥ +ï¤ · +ï® £ +ï¯ ł +ðĿĴ ĸ +ðĿķ ĺ +ðĿĸ ĩ +ðĿĹ Ł +ðĿĹ ª +ðĿĹ ¯ +ðĿĻ ł +ðŁĵ ı +à¦ Ĺ +âĴ » +â² ł +ðĿĵ µ +Ê £ +à° ľ +áĬ ¢ +áŀ IJ +Ḡ· +âĦ Ľ +âĩ Ģ +âĩ Ĭ +êĴ ¦ +ê¦ ł +ï® ¤ +ðŁį Ľ +ðŁ¤ Ľ +ᨠ¾ +âŀ º +áķ ¯ +ἠı +âĩ Ĥ +âĶ ¹ +âĻ Ĺ +ðŁĸ ¨ +ê¦ ı +ઠ° +áļ ¨ +ðŁ¤ ¥ +ðŁ§ ¢ +ãIJ Ĥ +ãĦ ¥ +ðŁĸ Į +â¼ Ĵ +ãĬ § +âį © +ðŁ¦ ij +âĶ · +ï© IJ +ï© ¡ +ðĵ Ī +ðĵĪ Ĵ +â» Ħ +ï¨ Ĵ +âĦ ª +Ò § +Ú Į +âĢ ¶ +⺠ł +â» ģ +âĨ ¸ +áĦ IJ +ãħ IJ +à» Ħ +áĹ ª +âĨ ¼ +âĩ ĭ +âĩ ĺ +âĮ ij +âĸ © +ðĿIJ Ĺ +Ä Ĭ +ঠī +ìī ł +É ¤ +ß į +ß ı +áµ Ĺ +âĤ ¥ +âĵ ī +âĶ ł +âĶ ¨ +âķ Ħ +ä ¤ +ä¤ Ģ +ê» ¸ +ï® ģ +ðĵ Ĥ +ðĵĤ ĥ +ðŁ¦ ķ +Æ Ľ +ঠĩ +ãı ĺ +ï® ¼ +Ú ĵ +Ú Ŀ +ঠĵ +ච¯ +á´ ħ +á½ Ļ +âģ ¼ +âĸ İ +â¼ © +ä Ķ +äĶ Ģ +ë» ¡ +ìĽ ½ +íģ Ħ +ï¥ ¼ +ï± ī +ï¹ » +ðĿĸ ĭ +ðĿĻ Ī +ðĿĻ ª +ðĿ Ļ¶ +ðŁIJ Ħ +ðŁIJ Ĩ +áİ ¢ +ḠĮ +âĿ ´ +ðŁı ¸ +È Ŀ +É ¸ +Î ħ +Ï ľ +Ó ¢ +Õ ¹ +à´ ħ +ຠĪ +áĭ ° +áij İ +áł µ +á¡ ł +á´ ī +Ḡµ +á¿ ´ +âĵ £ +âĶ ¶ +â½ ¯ +ê² ¥ +ê¿ ĺ +ëģ İ +ëİ Ī +ëĶ ¯ +ë² ° +ìĺ ¯ +ìĽ ¸ +ìŀ Ĺ +ì§ ĺ +ì¬ ¬ +ì· ¬ +íģ ħ +íĵ Ķ +íĽ Ŀ +ï¤ ® +ï¤ ¹ +ï¥ ² +ï¯ ĸ +ðĿĵ ħ +ðĿĻ Ħ +ðŁĵ ¶ +ðŁĹ Ĵ +ðŁ¥ Ķ +ðŁ¥ Ń +Å ® +Å ´ +Æ ī +Æ « +Ç ģ +Ç £ +Ç º +Ç ¼ +È į +È ¯ +É ľ +Ê ¬ +Ë ģ +Ë ¤ +Ë µ +Ï Ľ +Ò ¤ +Ò ¬ +Ó ı +Ó Ľ +Ó ¡ +Ó ³ +Ô Į +Ô ¬ +Õ ³ +Ù » +Ú ī +Ú § +Ü ľ +ß ª +ठĿ +ঠĽ +ਠĨ +ઠķ +ઠ¡ +à® İ +à° ¬ +ൠ» +ൠ¼ +චł +à¶ Ń +ච¶ +à· Ĩ +༠½ +áĢ ļ +áħ ¢ +áĨ ¸ +áĪ Ģ +áĪ ķ +áĪ ° +áī ¡ +áī ¤ +áĬ ¦ +áĬ « +áĭ ĭ +áĭ į +áİ ¯ +áij Ń +áķ Ĺ +ᣠĽ +ᥠĴ +á© ī +áŃ º +á´ ¡ +áµ ĺ +áµ Ľ +ᶠł +Ḡģ +Ḡĭ +á¹ Ļ +á¹ Ŀ +á¹ ¦ +Ạħ +á¼ Ĥ +á½ ĥ +á½ į +á½ § +á¾ · +âĢ µ +âĤ İ +âĦ Ŀ +âħ Ģ +âĨ ŀ +âĨ § +âĩ ħ +âĪ ĥ +âī ı +âī ½ +âĬ ŀ +âĬ ¡ +âĬ § +â Ĭ¶ +âĭ Ħ +âİ Ĵ +âİ ¡ +âİ £ +âİ ª +âı İ +âĵ ĥ +âĵ ĸ +âĵ ¨ +âķ ĭ +âķ ĸ +âķ ¢ +âķ ² +âĸ Ĩ +âĸ Ĭ +âĸ į +âĸ ® +âĺ ¡ +âĺ ¦ +âĺ ± +âĺ ¿ +âĻ ĺ +âĻ Ŀ +âļ ° +⼠ij +âŀ ª +⤠Ŀ +⤠¢ +⤠· +⧠« +â¨ Ń +⨠¯ +â± £ +â² İ +âµ Ľ +ãħ Ķ +ãĪ ı +ãī ² +ãī ³ +ãĬ ij +ãĭ Ľ +ãİ IJ +ê² ¤ +ê· ¿ +ê¹ ŀ +ê» ¨ +ê¼ į +ê¿ ¸ +ëĥ ¬ +ëĩ IJ +ëĭ ł +ëį ¯ +ëĹ Į +ëĹ ij +ë¥ Ģ +ëª ĥ +ëª ¯ +ë± ¡ +ë³ ĵ +ë³ ½ +ë µľ +ìĤ ³ +ìħ ¥ +ìĩ ½ +ìı ¨ +ìı ¸ +ìķ į +ìĸ ĸ +ìŁ ¨ +ì¢ ĥ +ì¢ į +ì¥ ij +ì§ ¼ +ì© ĥ +ì® ľ +ì® ¸ +ì³ ij +ì´ ¥ +ì¾ ĥ +íħ ¦ +íĪ ¿ +íĵ ½ +íķ ³ +íĸ ı +íĹ ł +íĿ « +ï¤ ĵ +ï¤ ĺ +ï¥ İ +ï¥ ¶ +ï¦ ħ +ï¦ ½ +ï§ ĩ +ï¬ Ĩ +ï¬ ³ +ï® ĩ +ï® Ī +ï® Ŀ +ï® © +ï® ± +ï¯ ĺ +ï¯ Ļ +ï¯ ¢ +ï¯ £ +ï¯ ¤ +ï¯ ¥ +ï± Ĥ +ï² Ĩ +ï² ª +ï´ ¼ +ïº ī +ïº Ĭ +ïº ¥ +ðĿij ¨ +ðĿij © +ðĿij ² +ðĿ ĴĮ +ðĿĴ ª +ðĿĴ ® +ðĿĵ Ĥ +ðĿĵ Ī +ðĿĵ ¯ +ðĿĶ ¨ +ðĿķ Ģ +ðĿķ Ĩ +ðĿķ ¦ +ðĿķ § +ðĿķ « +ðĿķ · +ðĿĹ µ +ðĿĹ ¸ +ðĿĺ Ħ +ðĿĺ Ļ +ðĿĺ ł +ðĿĺ ¬ +ðĿĻ į +ðĿĻ ij +ðĿĻ ¡ +ðĿ Ļ¨ +ðĿĻ · +ðĿļ į +ðĿĽ ¿ +ðŁ ĥ +ðŁĥ ı +ðŁħ ĺ +ðŁ ī +ðŁī ij +ðŁİ ¡ +ðŁİ ª +ðŁİ ± +ðŁİ ³ +ðŁİ º +ðŁı İ +ðŁı Ĺ +ðŁı ļ +ðŁı ŀ +ðŁı ¦ +ðŁı § +ðŁIJ ģ +ðŁIJ ħ +ðŁIJ ĵ +ðŁĴ Ĥ +ðŁĵ ij +ðŁĵ ĵ +ðŁĵ ¨ +ðŁĵ « +ðŁĶ ĭ +ðŁĶ Ń +ðŁĶ ¯ +ðŁķ Ĺ +ðŁļ Ĥ +ðŁļ ¢ +ðŁļ ¦ +ðŁļ ¬ +ðŁĽ ĭ +ðŁĽ Į +ðŁĽ ¬ +ðŁĽ ¶ +ðŁŁ ¡ +ðŁ¥ ĺ +ðŁ¥ Ł +ðŁ¥ ¦ +ðŁ¦ ĩ +ðŁ¦ Ī +ðŁ§ Ĭ +ðŁ§ Ĺ +ðŁ§ ¤ +Ê · +Ë ¹ +á¹ ļ +á½ ¥ +âĦ Ł +ê² ¯ +ê» « +ë° · +ìĥ Ĩ +ìĽ Ŀ +ì¨ ī +ì« ı +ï¯ ķ +ðĿľ ĭ +É ² +Ò Ń +Ó Ī +འĽ +áĭ ĵ +áĻ Ń +áł © +á¹ ® +âĦ Ĵ +âĨ » +âµ ĥ +ëĢ ¨ +ëł § +ìī ¥ +ìĮ ľ +ìĹ ¶ +ì¨ Ī +ìª ¾ +íı ½ +íļ Ķ +íĽ µ +ï¤ ¸ +ï¦ IJ +ï§ Ĺ +ï§ ļ +ï¬ ¯ +ðĿIJ Ĭ +ðĿķ Ĺ +ðĿĹ ļ +ðĿļ ĸ +ðŁħ ´ +È ĥ +É Ŀ +Ï ± +Ó Ĺ +ठ¢ +áħ ł +áī ¦ +áij Į +áĴ ¼ +áŀ ¡ +áł ¨ +áł Ń +ᨠħ +ᨠĶ +á´ ĺ +ᶠ¦ +á¸ İ +á¼ ħ +á¼ ¹ +âĨ ¯ +âĵ İ +ãı Į +ê ī +êī Ĥ +ëĨ § +ëĿ ± +ì¢ ¡ +íĪ ½ +ï¤ ĩ +ï¤ Ľ +ðĿIJ ķ +ðĿĵ ¸ +ðĿĵ ¼ +ðĿĹ ķ +ðĿĺ Ī +ðŁı £ +ðŁı ¤ +ðŁĹ Ħ +Ñ · +Ò ł +áµ ĸ +á¼ ¨ +ë¬ Ħ +ï° ´ +âĪ ½ +Õ Ń +Ú ¹ +ॠŁ +áĢ Ĩ +áŀ Ĵ +ãĢ ¶ +ê¦ « +ï¸ ĵ +ðĿIJ Ľ +ðĿĺ Ĺ +ðŁı ľ +ì« Ń +ðŁ§ ŀ +འĤ +âĨ ¿ +âĩ ı +âĵ ģ +âĶ § +âķ ģ +âķ ¤ +ê¦ Ĺ +ê¦ ¤ +ðŁı Ī +áŀ ķ +Ô ½ +àª Ĺ +ଠĨ +âķ ķ +ï½ ł +â¼ ¦ +â¼ ¯ +â¾ · +âĶ ĸ +ଠĵ +âĺ Ĺ +âį ĭ +ï¨ Ŀ +â¼ ¥ +ï¦ ª +âĦ Ĭ +ãĢ ´ +âį ¢ +ð¡ Ī +ð¡Ī ½ +ï© ¨ +ãĢ » +ãı ĥ +ï¦ ¡ +ï¨ ĺ +ðŁIJ ĥ +ðŁĨ ĸ +ðŁĹ ¾ +ãĦ ĩ +Þ ĭ +â¼ ¼ +ï¨ Ń +Þ Ģ +Þ Ħ +Þ Ī +Þ IJ +âĮ Ħ +â» ĺ +ãŁ ¢ +á ħ§ +ðIJĮ ¿ +Ë » +à² Ĺ +áĢ ĩ +áŀ Ĭ +âķ ĩ +ãĩ ¼ +ãİ ° +Õ Ĵ +Ü Ī +ß ¥ +à¿ IJ +áĢ Ł +âĨ ¥ +âķ Į +â½ Ģ +â½ ° +â¾ Ĭ +ä Ħ +äĦ Ģ +ðĵ IJ +ðĵIJ į +ðŁİ ¦ +âĤ ¯ +âĬ ĺ +âĦ į +Ê µ +Ñ ¶ +Ú ĥ +ঠĶ +à´ ¦ +áİ ¶ +áĵ ķ +á¹ ¨ +âĤ ł +âĩ ° +âĹ Ĵ +â¿ Ĭ +ê· ± +ì¹ ķ +íĪ © +ïŃ Ģ +ðĿĴ ¸ +ðĿĵ Ĭ +ðĿĺ © +Ç ¦ +É « +áĬ ¨ +È ¹ +Ê ¯ +Î ª +Ú Ģ +áĮ ¸ +áİ » +áı ķ +áı ´ +á² Ĥ +á½ ¨ +âı Ŀ +âĺ Ļ +ëĥ ¨ +ëĦ ¼ +ëĪ Ļ +ë£ ħ +ìĶ ¼ +ìķ Ŀ +ìļ ¬ +ìľ ± +ï¥ Ĥ +ï¦ ¹ +ï¬ ¹ +ïŃ ģ +ï³ Ī +ðĿĶ ħ +ðĿĺ ¤ +ðĿĻ ı +ðĿĻ Ļ +ðŁķ ī +ðŁ§ Ļ +Ḡij +ê´ ¼ +ëģ į +ëĹ ´ +ëĿ ³ +ë° ŀ +ë° ¢ +ëµ ĺ +ìĤ Ķ +ìĦ Ħ +ì¼ ļ +íĢ ł +íĬ ± +íĮ ĸ +ï¤ ij +ï¦ ´ +ï¦ ¸ +ï´ į +ðĿĺ · +Ä ¬ +Å ¬ +Æ Ģ +Æ ĭ +Æ ľ +Ç ij +Ç ĺ +Ç ŀ +Ç ¥ +Ç ® +É ° +É ¶ +É · +É ½ +Ê Ī +Ê IJ +Ë İ +Ë Ł +Ë ¦ +Ë ¯ +Ï IJ +Ï ĵ +Ï ¢ +Ï ¤ +Ï ª +Ï Ń +Ï ® +Ï » +Ñ ł +Ñ Ń +Ò ¨ +Ó Ŀ +Ô ¡ +Ô · +Õ ī +Õ ĵ +Õ ĸ +Õ ļ +Õ Ŀ +Ö İ +Ø ¿ +Ú ħ +Ú į +Ú Ķ +Û Ĭ +Û ¾ +Ü Ļ +Ý Ĵ +Ý ĺ +ß Ĵ +ß ĸ +ठĬ +ठIJ +ঠı +ঠĸ +ৠŁ +ઠ® +ઠ¹ +à® ħ +à® Ĩ +à° ¡ +à° ° +ಠļ +ಠ® +ಠ¯ +à´ Ł +à´ · +ൠ¾ +චij +චŀ +༠¼ +འĵ +áĢ ĵ +áĤ ¦ +áĥ ĸ +áĥ Ń +áĥ ¯ +áħ ¨ +áħ ª +áĨ ° +áĪ ģ +áĪ İ +áĪ ĵ +áĪ ¥ +áĪ ² +áĪ ´ +áĪ » +áī ł +áī ² +áī ¶ +áĬ £ +áĬ ¥ +áĬ ª +áĭ ĺ +áĭ ² +áĭ ¶ +áĮ £ +áį ¡ +áį £ +áİ ¬ +áİ ¾ +áIJ ¡ +áķ ķ +áĸ ± +áĹ IJ +áĹ Ń +áĺ ī +áļ ± +ἠŁ +áŀ ¥ +ᣠĶ +áł £ +áł ª +áł ° +áł ´ +ᤠĸ +ᥠ£ +á ® +á® ł +á ¯ +ᯠĻ +á ° +á° į +á´ Ĭ +á´ ¾ +áµ ģ +áµ İ +áµ ŀ +áµ ¤ +ᶠħ +ᶠĺ +ᶠŁ +ᶠ¢ +ᶠ¤ +ᶠ± +ᶠ» +Ḡī +Ḡŀ +Ḡº +á¹ ĵ +á¹ Ĺ +á¹ ª +ẠĬ +Ạı +ẠĽ +á¼ ĥ +á¼ Į +á¼ ¿ +á½ Ĥ +á½ ĵ +á½ Ĺ +á½ ¦ +á¾ ± +á¾ ´ +á¿ ĺ +á¿ Ł +á¿ ¸ +âģ ĺ +âĤ ij +âĤ Ľ +âĤ ¿ +âĦ ĩ +âĦ ŀ +âĦ ± +âĩ Ł +âĩ ² +âĪ ¤ +âĪ ¶ +âī Ĥ +âī ¾ +âĬ ¨ +âĬ ³ +âĬ · +âĭ Į +âĭ ĺ +âĮ ķ +âĮ ¥ +âĮ µ +âĮ º +âį £ +âį ² +âį µ +âİ ĩ +âı ĥ +âı IJ +âı ł +âı ¤ +âı ¶ +âı ¸ +âı ¹ +âij Ĥ +âĴ · +âĴ º +âĵ ¡ +âĵ ¤ +âĶ ¾ +âĸ ĺ +âĸ µ +âĹ ª +âĹ · +âĺ ¨ +âĺ « +âĺ ² +âĺ ³ +âĻ Ĩ +âļ ¤ +âļ ¥ +⼠ĵ +⼠´ +⼠¾ +âŀ « +âŀ ¿ +⣠· +⤠ij +⤠« +⤠¶ +⤠½ +⧠ª +⨠Ģ +â ©½ +⬠¡ +⬠¢ +⬠¤ +â² ĸ +â² ª +âµ Ģ +⸠® +⸠½ +ãĢ ł +ãĢ · +ãĦ Į +ãĦ ĺ +ãħ ij +ãĪ İ +ãĪ IJ +ãĬ ľ +ãĮ ĵ +ãĮ ł +ãİ Ł +ãİ ¤ +ãİ § +㬠® +ä Ī +äĪ Ģ +ä ° +ä° Ģ +ê ħ +êħ ī +êĩ Ĺ +ê Ī +êĪ į +ê§ Ĥ +ê§ Ĭ +êª Ģ +ê² Ī +ê² į +ê³ Ģ +êµ ł +ê½ IJ +ê¾ Ī +ê¿ ± +ëĥ ı +ëĦ ij +ëħ ¤ +ëĩ ¸ +ëĪ ¼ +ëī ħ +ëĬ £ +ëĭ º +ëį ŀ +ëIJ Į +ëķ ¸ +ëĺ ł +ëĻ ĩ +ëĻ Ī +ëľ ½ +ëŀ Ķ +ëł ľ +ë£ IJ +ë§ Ģ +ë§ Ĭ +ëª Ģ +ë¬ Ń +ë¯ ¾ +ë³ ľ +ë´ Ĭ +ëµ ī +ë· ľ +ë¸ Ģ +ë¹ ĭ +ìģ Ħ +ìĤ £ +ìĤ » +ìĦ µ +ìħ Ĵ +ìī Ī +ìī Ķ +ìĬ Į +ìĬ Ļ +ìIJ ´ +ìĵ º +ìķ ļ +ìķ º +ìĸ ľ +ìĹ ª +ìĺ ľ +ìĻ ¤ +ìļ Ľ +ìļ º +ìĿ ħ +ìĿ ı +ìĿ Ń +ìĿ ¶ +ìł Ľ +ì¡ Ī +ì¢ ī +ì¢ Ķ +ì© ł +ìŃ Į +ì¯ © +ì´ £ +ì¸ ķ +ì¹ Ł +ì¾ ¡ +ì¿ Ļ +íģ ĩ +íģ ī +íĩ Ģ +íĪ ¶ +íĸ ij +íĸ ¤ +íĹ ħ +íľ ı +íĿ Ŀ +ï¤ Ĵ +ï¤ ķ +ï¤ ¬ +ï¥ ħ +ï¥ ĩ +ï¥ ı +ï¥ ļ +ï¥ Ł +ï¦ Ħ +ï¦ Ī +ï¦ ¨ +ï¦ © +ï¦ ² +ï§ ģ +ï§ ĥ +ï§ Ķ +ï§ ł +ï§ £ +ï§ ® +ï ŃIJ +ïŃ ĸ +ïŃ ¦ +ïŃ ´ +ïŃ µ +ïŃ ¶ +ïŃ ¸ +ï® Į +ï® İ +ï® ŀ +ï® Ł +ï® ¡ +ï® ª +ï¯ Ķ +ï¯ Ĺ +ï¯ ļ +ï¯ Ľ +ï¯ Ŀ +ï¯ Ł +ï¯ § +ï¯ ¨ +ï¯ « +ï¯ ¯ +ï¯ ° +ï¯ ± +ï¯ ² +ï¯ ³ +ï¯ ´ +ï¯ µ +ï¯ ¶ +ï° Ģ +ï± ħ +ï± Ķ +ï± ´ +ï² ģ +ï³ ķ +ï· ½ +ï¸ ķ +ï¸ ± +ï¹ £ +ï¹ ½ +ï» į +ï¾ ± +ðĿIJ Ļ +ðĿIJ ½ +ðĿij ¤ +ðĿij ® +ðĿij µ +ðĿĴ ĥ +ðĿĴ Ħ +ðĿĵ Ń +ðĿĵ · +ðĿĶ ĸ +ðĿĶ ŀ +ðĿĶ ¢ +ðĿĶ ¦ +ðĿĶ ¬ +ðĿķ Ħ +ðĿķ Ĭ +ðĿķ İ +ðĿķ Ļ +ðĿķ ľ +ðĿķ Ń +ðĿķ ³ +ðĿķ ¸ +ðĿķ ¾ +ðĿ ĸī +ðĿĸ ı +ðĿĺ ĩ +ðĿĺ ī +ðĿĺ ĸ +ðĿĺ Ľ +ðĿĺ ŀ +ðĿĺ « +ðĿĺ ¾ +ðĿĻ ĩ +ðĿĻ ī +ðĿĻ ĭ +ðĿĻ İ +ðĿĻ ĺ +ðĿĻ ¥ +ðĿļ ĥ +ðĿļ IJ +ðĿļ Ķ +ðĿľ ĥ +ðŁĦ · +ðŁħ Ŀ +ðŁħ ¾ +ðŁĨ Ĥ +ðŁĨ ĵ +ðŁĮ Ĥ +ðŁĮ Ĩ +ðŁĮ ī +ðŁĮ ij +ðŁĮ ĺ +ðŁĮ © +ðŁĮ « +ðŁį ¢ +ðŁį ¥ +ðŁİ Ľ +ðŁİ ¢ +ðŁİ ´ +ðŁij ¡ +ðŁĴ ¾ +ðŁĵ Ń +ðŁĶ Ī +ðŁĶ ¦ +ðŁĶ ² +ðŁĶ ³ +ðŁķ ĵ +ðŁķ ķ +ðŁķ ĺ +ðŁķ Ł +ðŁķ · +ðŁĹ ³ +ðŁļ Ħ +ðŁļ Ķ +ðŁļ ĸ +ðŁĽ IJ +ðŁĽ ¤ +ðŁĽ ¸ +ðŁ ł +ðŁł ³ +ðŁ¤ ¹ +ðŁ¥ ĥ +ðŁ¥ ¨ +ðŁ¥ ª +ðŁ¥ ¾ +ðŁ¦ ĥ +ðŁ¦ Ĵ +ðŁ¦ Ļ +ðŁ¦ ¶ +ðŁ§ ł +ðŁ§ ª +ðŁ§ Ń +ðŁ§ ² +𣠷 +ð£· Ń +ð¦ ĺ +ð¦ĺ Ĵ +Æ ij +Ç Ļ +È ® +Ø ł +Ú Ħ +Ü Ģ +ß ¢ +áī Ģ +áĬ IJ +áİ ł +Ạŀ +ëĪ ŀ +ëķ Ł +ë£ ģ +ë¤ Ĺ +ìĦ ¥ +ìħ ij +ìĸ IJ +ìĽ Ľ +ì£ ķ +íİ ı +íĽ ĵ +ï¥ º +ï³ Ľ +ï´ « +ðĸ § +ðĸ§ · +ðĿķ ģ +ðŁIJ ª +ðŁĴ Ī +ðŁĵ ł +ðŁķ Ľ +ðŁķ ´ +Ñ Ŀ +Ó Ĭ +ॠ² +ઠª +áĥ ¤ +áį IJ +ᶠ° +á¼ Ŀ +á½ © +âĭ ĭ +âĴ ½ +âĻ ¾ +â ½Ķ +â¾ ¯ +ãĦ Ĵ +ãħ ļ +ëIJ į +ë· ģ +ìĭ Ģ +ìļ Ŀ +ì¥ ° +ìº ´ +íĭ ī +íĿ ½ +ï¦ Ģ +ï¦ ¿ +ï§ ħ +ï§ ĵ +ïŃ ¯ +ï® Ĩ +ðIJ¤ ķ +ðĿIJ Ł +ðĿĴ ħ +ðĿĵ ľ +ðĿĶ ° +ðĿĶ » +ðĿĺ į +ðĿĻ ¯ +ðŁĦ ½ +ðŁħ Ĥ +ðŁħ Ķ +ðŁħ ½ +ðŁĵ ´ +ðŁ§ ĸ +Ó Ĵ +Ḡ² +ëī ¼ +Ç ı +È ĵ +Ê ¸ +Õ Ĥ +Û ħ +ß ¡ +ß £ +à® ¯ +à° Ī +ಠ¸ +ຠ® +༠ķ +áĢ İ +áĨ ¡ +áIJ ĭ +áIJ ķ +áij ¯ +áŀ Ĩ +ᨠķ +á© Ī +âģ ħ +âĨ ļ +âĶ İ +âł © +â² Ĥ +â² Ķ +â² ¨ +ãĬ ļ +íĵ ² +ðĿij Ī +ðĿij ¬ +ðĿij ¹ +ðĿĴ ¾ +ðĿĵ ± +ðĿĵ ½ +ðĿķ ¯ +ðĿķ » +ðĿĺ ½ +ðĿļ Ĩ +ðŁĦ ° +ðŁIJ ¨ +Ò ķ +ಠħ +ï¨ Ĩ +ðĿij ° +ðŁĦ ¸ +Ô İ +Ø į +Ù µ +ಠ¶ +áĢ Ī +áĺ Ĺ +áł ¸ +á¡ ¡ +ᨠ² +á© ģ +á´ · +áµ § +âķ ¨ +âļ ģ +â¾ Ŀ +ãĢ ¼ +ãĦ ı +êĴ « +ê¦ ¥ +ê¦ © +ê¦ ² +ìĺ ¼ +íĵ IJ +ðĵ ĩ +ðĵĩ ¼ +ðĿķ ¿ +ðŁĽ ´ +ë¨ ľ +ಠµ +à´ İ +༠Ģ +âĩ ĸ +ãĪ « +âĵ Ģ +áħ ´ +áļ ¾ +ἠŀ +ἠ« +ᥠ´ +âĨ Ľ +âĨ ¶ +âĩ ¤ +âķ Ł +âĺ · +âļ IJ +ðŁ§ ´ +á¹ ³ +âĶ į +âĶ Ĵ +âĶ © +âĶ ¦ +â¾ µ +ઠľ +ઠ¤ +âĩ Ļ +âĶ ± +âķ Ģ +â½ Ĭ +ï½ Ł +ଠ¡ +ðł ® +ðł® · +âķ ĥ +â° Ķ +ãĬ ¦ +ðŁİ IJ +ãĩ ° +â¼ Ŀ +â¾ Ķ +â½ Ĵ +âł Ĵ +ï¨ ¦ +ï© Ĵ +ï¨ ² +ï© ĸ +ðĵı ¸ +ãĮ ĥ +ðĸ ¤ +ðĸ¤ IJ +ï¦ Ń +âĬ ħ +â¾ ³ +ä´ ¥ +ï© ķ +ðŁĮ Ķ +áŀ ĭ +âļ į +â¼ ĭ +ãİ ĺ +ðIJĮ ² +É © +áİ ij +âĨ ® +âĩ ĥ +âļ İ +ãĩ ± +ãĭ © +ãĮ ¶ +êĻ ª +ëİ ¬ +ï¨ IJ +ï¨ Ľ +ï© Ĭ +ï© į +ðĵ ħ +ðĵħ º +Ï ¡ +È ij +É Ĥ +Ô ĵ +ß İ +à´ § +áĢ ī +áĢ ĭ +áĢ ij +áĢ ł +áļ Ļ +ᨠĦ +ᨠ© +ᨠ¹ +á© ĵ +ᬠľ +á´ Ļ +áµ ij +âĤ Ń +âĨ ° +âľ ģ +â½ IJ +ãĭ ¯ +ãĮ ½ +íĨ ¢ +ï¤ ¿ +ðŁ Ĥ +ðŁĤ » +È Ĵ +Í º +Ô ¥ +Õ ij +Ú ¶ +à§ İ +ච® +ຠĸ +ຠľ +ຠ½ +áĥ » +áħ ¯ +áĭ ŀ +áĸ ķ +á ´Ī +ᶠĨ +Ḡľ +á¹ ¼ +á¿ ¨ +âĦ ĭ +âĦ Ń +âĪ ± +âĮ ĵ +âĶ ĩ +âĶ ¢ +â± ® +â² Ħ +ãĩ ¾ +ãĪ ¬ +ë¸ ¡ +ìIJ ī +íĻ Ľ +ðĿķ ª +Æ ¹ +Í ² +Ó ģ +Û ¼ +ঠ« +áħ Ł +áī Ĩ +áį Ī +Ạĸ +á½ ī +âĶ ¸ +â½ © +ê ľ +êľ ¥ +êµ ħ +ëĤ Ķ +ëĦ ł +ëĩ Ĺ +ëĻ Ŀ +ìļ ¯ +ìļ · +ìŁ Ľ +ì· IJ +íŁ ¬ +íŁ ® +íŁ ° +ï¦ Ĩ +ï¦ ± +ï² ŀ +ï³ ¤ +ï³ ¥ +ðIJĮ ¸ +ðĿĶ ı +ðĿķ ® +ðĿĺ £ +ঠĪ +âı ı +ãĦ ĸ +ê² ĩ +ëĸ ĺ +ëľ · +ëŀ Ĵ +ë¡ ĵ +ë¢ ī +ë£ ĥ +ë§ ĭ +ë² ĭ +ìĤ · +ìĪ ķ +ì Į¨ +ìĵ » +ìĸ Ĭ +ìĻ ¬ +ìĿ » +ì¦ ģ +ìµ ¤ +ì· ĥ +íĢ ľ +íħ ī +íį ł +íı ħ +íij ± +íķ ķ +íĸ ł +íĿ ķ +Æ Ļ +Æ ļ +Æ ŀ +Ç ĥ +Ç Ĭ +Ç ľ +Ç ¤ +Ç Ń +Ç ¹ +È Ģ +È ģ +È ħ +È ī +È Ĺ +È Ł +È ¤ +È ¥ +È ¨ +È µ +È º +È » +É Į +É ® +Ê ħ +Ê ¥ +Ê ¨ +Ë ĵ +Ë Ķ +Ë ł +Ë £ +Ë ¸ +Í ´ +Ï Ĺ +Ï ĺ +Ï Ļ +Ï ļ +Ï Ŀ +Ï ¨ +Ï ¬ +Ï ¾ +Ï ¿ +Ñ ª +Ò Ģ +Ò ľ +Ò ¼ +Ò ½ +Ó Ĥ +Ó ħ +Ó ĩ +Ó į +Ó ĸ +Ó Ł +Ó « +Ó ± +Ô Ĩ +Ô ĩ +Ô º +Õ ĭ +Ö ī +Ø Ī +Ø Ĭ +Ø ½ +Ø ¾ +Ù · +Ú Ĥ +Ú Ĭ +Ú ĸ +Ú Ĺ +Ú £ +Ú « +Ú ¸ +Û Ģ +Û į +Û ½ +Ü ī +Ü ¤ +Ý § +Ý ´ +Þ ĥ +Þ ¤ +Þ ¥ +ß ļ +ß Ľ +ß ¤ +àł į +àł ĵ +àł ³ +à¡ ¢ +ॠł +ৠł +ৠº +ਠĬ +ਠIJ +ਠ® +ਠ¯ +ਠ° +ਠ¸ +ઠĨ +ઠ³ +ઠµ +ઠ½ +ଠĮ +ଠĺ +ଠ½ +à® ĥ +à® ¸ +à° Ĩ +à° ķ +à° ¦ +ಠĨ +ಠĬ +ಠĮ +ಠIJ +ಠĽ +ಠ¤ +ಠ¦ +ಠª +ಠ² +ಠ¹ +à´ Ĩ +à´ ı +à´ Ĺ +à´ « +à´ ¹ +ൠº +ൠ½ +චħ +චĬ +චĶ +ච§ +ච« +ච° +༠Ħ +༠ħ +༠Ĭ +འĻ +འ¡ +འ§ +à¿ Ģ +à¿ Ļ +áĢ Ŀ +áĢ § +áĢ © +áĢ ¿ +áģ µ +áĤ ģ +áĤ ½ +áĥ Ĥ +áĥ ª +áĦ Ĭ +áĦ ¢ +áħ ¦ +áħ Ń +áĨ ® +áĨ ± +áĨ » +á ĩ +áĩ Ĥ +áĪ ħ +áĪ ī +áĪ Į +áĪ IJ +áĪ Ĵ +áĪ Ļ +áĪ ļ +áĪ ľ +áĪ ŀ +áĪ © +áĪ ³ +áĪ º +áĪ ½ +áī ħ +áī ¢ +áī ± +áī ´ +áĬ ĥ +áĬ į +áĬ ĸ +áĬ ® +áĬ ¸ +áĭ Ľ +áĭ Ŀ +áĭ ³ +áĮ ģ +áĮ ħ +áĮ ¥ +áĮ ¦ +á Į¨ +áį Ĭ +áį į +áį ķ +áį ĸ +áį ¢ +áį ¤ +áİ Ĵ +áİ ª +áı ģ +áı IJ +áı Ł +áIJ Ĥ +áIJ ĸ +áIJ Ŀ +áIJ ŀ +áIJ Ł +áIJ ł +áij ĸ +áĴ ĭ +áĴ į +áĴ ¡ +áĵ « +áĶ ķ +áķ ĭ +áķ ij +áķ Ļ +áķ ļ +áķ Ľ +áķ ¤ +áķ ¦ +áķ ® +áķ ¼ +áĸ ĵ +áĹ Ĺ +áĹ ¢ +áĹ ¯ +áĹ · +áĺ Ħ +áĺ ij +ἠĤ +ἠĻ +áŀ į +áł Ĩ +áł ¡ +áł ¦ +áł ® +áł ¯ +áł ² +áł · +á¡ į +á¡ ŀ +á¡ ¤ +á ¡´ +á¡ µ +ᤠĵ +ᥠĸ +ᥠ° +ᨠ¦ +ᨠ§ +ᨠ¨ +ᨠª +ᨠ¬ +ᨠ¯ +ᨠ³ +ᨠµ +á© ĥ +ᬠķ +áŃ £ +á ± +á± ļ +á² ł +á´ ĵ +á´ ¶ +áµ Ĥ +áµ Į +áµ ¥ +áµ ´ +ᶠĩ +ḠĪ +Ḡł +Ḡ§ +Ḡ´ +Ḡ¾ +á¹ Ģ +á¹ ĸ +á¹ Ł +á¹ ł +á¹ « +á¹ ± +á¹ · +á¹ ¿ +ẠĦ +Ạį +Ạij +áº Ĺ +á¼ ī +á¼ ĵ +á¼ Ń +á½ ĭ +á½ Ĵ +á½ ł +á½ £ +á¾ Ħ +á¾ ı +á¾ ij +á¾ Ĺ +á¾ ¦ +á¾ § +á¾ ¾ +á¿ Ħ +á¿ ĵ +á¿ ¡ +á¿ ¬ +âģ ļ +âĤ Į +âĦ ģ +âĦ Ķ +âĦ £ +âĦ § +âĦ ¯ +âĦ ° +âĦ ´ +âħ ħ +âĨ ľ +âĨ « +âĨ Ń +âĨ ± +âĨ ¹ +âĨ ½ +âĩ ĩ +âĩ ľ +âĩ µ +âĪ ī +âĪ Ĭ +âĪ ĸ +âĪ ľ +âĪ ¾ +âī Ģ +âī ĭ +âī Į +âī ĵ +âī ľ +âī ´ +âī ¿ +âĬ Ĭ +âĬ ĭ +âĬ Ķ +âĬ ĸ +âĬ £ +âĬ ¦ +âĭ İ +âĭ ª +âĭ ² +âĮ ¦ +âĮ § +âį º +âİ Ī +âİ ¨ +âİ ¬ +âİ ³ +âİ ¼ +âİ ¾ +âı Į +âı ļ +âı « +âı ¯ +âı µ +âĴ ľ +âĴ Ŀ +âĴ « +âĵ Ħ +âĵ Ĭ +âĵ Ļ +âĵ © +âĶ ij +âĶ Ļ +âĶ ļ +âĶ ¥ +âķ ħ +âķ ī +âķ į +âķ ı +âķ ŀ +âĸ ļ +âĸ ¯ +âĹ ĥ +âĹ ļ +âĹ ¬ +âĹ ´ +âĺ Ī +âĺ ¤ +âĺ ¥ +âĺ § +âĺ ¬ +âĻ ģ +âĻ ± +âļ ĥ +âļ Ħ +âļ ħ +âļ ı +âļ ļ +âļ ŀ +âļ Ł +âļ ± +âļ ² +âľ Ģ +âľ Ł +âľ ¢ +âĿ µ +⣠¡ +⣠¦ +⣠§ +⣠³ +⣠¾ +⣠¿ +âł ĩ +⤠Ħ +⤠º +⥠Ĥ +⥠¹ +⧠ī +⧠¼ +⧠½ +⨠į +⬠Ĭ +⬠Ł +âŃ ŀ +â® ŀ +â® ³ +⯠Ī +⯠ij +â± ł +â± ± +â² Ń +â´ ¹ +âµ ķ +⸠¾ +â º« +â¼ Ĩ +â¼ ł +â½ Ł +â½ ¼ +â¾ Ľ +â¾ § +â¿ ĥ +â¿ » +ãĤ ķ +ãĤ Ł +ãĦ Ľ +ãĦ ¡ +ãĦ ¶ +ãĦ º +ãħ Ĵ +ãħ Ł +ãĨ Ģ +ãĩ » +ãĪ ij +ãĪ Ń +ãĪ ® +ãĪ ³ +ãĪ ¹ +ãī ¥ +ãī ¦ +ãī ¹ +ãī ¿ +ãĬ ŀ +ãĬ ¨ +ãĭ ij +ãĭ ¥ +ãĭ ´ +ãĭ º +ãİ Ħ +ãİ ķ +ãİ ¯ +ãı Ĥ +ãı Ī +ãı ĵ +ãı ĸ +ãı ± +ãIJ ± +ãŁ ģ +ã ¢ +㢠¨ +ã ¨ +㨠³ +ã« ª +ã« ´ +㶠³ +㺠¾ +ä Ģ +äĢ Ģ +ä ĭ +äĭ Į +ä ĮĢ +äIJ Ģ +ä łĢ +ä ł +äł ¼ +ä § +ä§ ŀ +ä¨ ° +ä¨ º +ä ´Ģ +ä · +ä· ħ +ä ·¸ +ê Ĥ +êĤ « +ê Į +êĮ ¼ +ê į +êį ² +êĴ µ +ê ĵ +êĵ ½ +êĻ Ń +êĿ Ľ +êĿ ¥ +ê ŀ +êŀ Ĭ +ê¦ Ĩ +ê¦ ĩ +ê¦ Ł +ê¦ ¨ +ê§ Ī +ê © +ê© Ł +êª ĭ +êª ij +êª ķ +êª Ĺ +êª ľ +êª ® +êª ± +êª » +êª ¼ +ê« Ģ +ê« Ŀ +ê° ĥ +ê° ĺ +ê± ľ +ê² ĵ +ê² ļ +ê³ Ļ +ê³ ¾ +ê´ Ĺ +ê´ Ļ +êµ Ľ +ê¶ ĥ +ê¶ ķ +ê¶ ¨ +ê¸ © +ê¸ ¿ +ê ¹Ħ +ê¹ Ĩ +ê¹ ī +ê¹ ĵ +ê¹ ¢ +ê¹ £ +ê¹ ¸ +êº ³ +ê¿ ı +ê¿ ķ +ê¿ § +ëĢ © +ëģ ħ +ëĥ µ +ëĦ ĸ +ëĦ Ĺ +ëĦ ¢ +ëħ Ĥ +ëĨ IJ +ëĩ ľ +ëĪ ĭ +ëĪ ļ +ëī į +ëī ¨ +ëĬ ļ +ëĬ ¡ +ëĭ ľ +ëĭ ª +ëĮ ĺ +ëĮ ¤ +ëĮ ¸ +ëİ Ł +ëı ¨ +ëIJ Ħ +ëIJ ı +ëIJ ´ +ëIJ ¸ +ëij ģ +ëij ¿ +ëĴ ¨ +ëĵ · +ëĶ ® +ëĶ ² +ëķ § +ëĸ Ķ +ëĸ ª +ëĺ Ń +ëļ Ģ +ëļ ł +ëĽ Ķ +ëĽ © +ëľ ħ +ëŀ ķ +ëŀ ° +ëŁ IJ +ëł ¡ +ë¡ ŀ +ë¡ £ +ë¡ µ +ë£ Ħ +ë£ į +ë¤ ³ +ë¦ į +ë¦ ı +ë¦ ³ +ë§ Ħ +ë§ Ĩ +ë§ į +ë§ ľ +ë§ « +ë§ » +ë¨ ® +ë© Ĥ +ë© Ń +ëª ´ +ë¬ ľ +ë¬ ł +ë¬ « +ë¬ ¾ +ëŃ ¬ +ë® ĺ +ë® ¹ +ë¯ ķ +ë¯ ľ +ë° ¨ +ë° ª +ë± Ķ +ë² ĺ +ë² Ľ +ë² ± +ë² ´ +ë´ ½ +ëµ ¤ +ëµ ¨ +ë· Ĺ +ë· ĺ +ë¸ ĵ +ë¸ ľ +ë¹ ª +ëº ĥ +ëº ĺ +ëº µ +ë» ´ +ë¼ IJ +ë¾ Ķ +ìģ Ń +ìĤ ł +ìĤ ® +ìĥ ı +ìĥ Ļ +ìĦ º +ìħ ¢ +ìĨ Ģ +ìĨ ħ +ìĨ ¤ +ìĨ ¦ +ìĨ ¬ +ìĩ ± +ìĪ µ +ìĭ ¨ +ìĭ ´ +ìĮ ° +ìį ľ +ìİ Ĺ +ìİ ĺ +ìİ ¼ +ìij ī +ìij Ŀ +ìij » +ìĴ Ķ +ìĴ ¯ +ìĵ © +ìķ IJ +ìķ ĸ +ìĸ ł +ìĸ ¾ +ìĹ ĥ +ìĹ Ĺ +ìĹ ľ +ìĹ ¨ +ìĺ Ĥ +ìĺ Ħ +ìĺ ı +ìĺ ¾ +ìĺ ¿ +ìľ § +ìĿ IJ +ìĿ ĸ +ìĿ · +ìŀ į +ìŀ ı +ìŀ ¨ +ìŀ ª +ìŀ ³ +ìł ¡ +ìł ´ +ìł ¹ +ì¡ Ģ +ì¡ ª +ì¡ µ +ì¢ IJ +ì¢ ¨ +ì£ Į +ì£ Ļ +ì£ ³ +ì¦ ij +ì§ ¥ +ì§ ´ +ì§ ¾ +ì¨ ĵ +ì¨ ķ +ì© ° +ì© » +ì© ¼ +ìª Ĺ +ì¬ Ķ +ì¬ ĺ +ì® ® +ì¯ ķ +ì¯ ĺ +ì° İ +ì° ¯ +ì± ĥ +ì± µ +ì² § +ì² ® +ì² ¯ +ì³ ¬ +ì´ ĭ +ì´ ¢ +ìµ ¥ +ì¶ £ +ì¸ Ī +ì¸ Ļ +ìº ¤ +ìº Ń +ì» ½ +ì¼ Ļ +ì½ ¬ +ì¾ Ģ +ì¿ ħ +ì¿ ½ +íĢ ħ +íģ ¦ +íĤ ħ +íĥ ¶ +íĥ ¹ +íĦ Ķ +íħ £ +íĨ Ħ +íĨ § +íĨ ¹ +íĩ ¼ +íī ¤ +íĬ ½ +íĭ Ĥ +íĭ ij +íį Ī +íį Ļ +íį ¿ +íİ ¶ +íIJ Ŀ +íĴ ľ +íĵ Ŀ +íĵ ª +íĵ ± +íĵ · +íĵ ¼ +íĶ Ļ +íĶ ł +íķ ļ +íķ Ľ +íķ ŀ +íķ Ł +íķ § +íķ ¶ +íĸ Ĭ +íĸ ĭ +íĸ į +íĸ Ķ +íĸ ĺ +íĸ ¡ +íĸ ¬ +íĹ £ +íĹ ¿ +íĺ ĸ +íĺ Ń +íļ ° +íĽ į +íĽ ½ +íĿ Ł +íĿ Ń +íĿ ´ +íŀ ľ +ï¤ ī +ï¤ Ń +ï¤ ² +ï¤ µ +ï¤ ¼ +ï¥ Ģ +ï¥ ij +ï¥ Ĵ +ï¥ ķ +ï¥ ĺ +ï¥ Ļ +ï¥ « +ï¥ ¬ +ï¥ ° +ï ¥¿ +ï¦ ĭ +ï¦ ı +ï¦ Ķ +ï¦ ĸ +ï¦ ĺ +ï¦ Ľ +ï¦ ł +ï¦ ® +ï¦ ¯ +ï¦ º +ï¦ » +ï¦ ¾ +ï§ Ĩ +ï§ ĸ +ï§ Ľ +ï§ ŀ +ï§ Ł +ï§ § +ï§ ³ +ï§ º +ï§ ½ +ï¨ ĥ +ï¨ ļ +ï¨ ¢ +ï© Ł +ï¬ ¤ +ï¬ ¬ +ï¬ ¼ +ïŃ Ĵ +ïŃ ķ +ïŃ Ľ +ïŃ Ŀ +ïŃ ŀ +ïŃ Ł +ïŃ ¤ +ïŃ § +ïŃ ¨ +ïŃ ® +ïŃ ° +ïŃ ± +ïŃ · +ïŃ ¹ +ïŃ » +ï® Ģ +ï® ĥ +ï® Ħ +ï® ħ +ï® į +ï® Ĵ +ï® ĵ +ï® ķ +ï® ¦ +ï® ® +ï® ° +ï¯ ĵ +ï¯ ľ +ï¯ © +ï¯ ª +ï¯ ¬ +ï¯ Ń +ï¯ ® +ï¯ · +ï¯ ¹ +ï¯ » +ï¯ ¼ +ï° ĥ +ï° Į +ï° IJ +ï° ĺ +ï° Ļ +ï° ľ +ï° ŀ +ï° ¢ +ï° ® +ï° ° +ï° ¼ +ï° ¿ +ï± Ģ +ï± ģ +ï± Ī +ï± ĭ +ï± ı +ï± Ń +ï² Ģ +ï² ĩ +ï² Ī +ï² ĭ +ï² İ +ï² Ĵ +ï² ľ +ï² ł +ï² ¬ +ï² » +ï³ ĩ +ï³ Ķ +ï³ £ +ï³ « +ï´ ĺ +ï´ ° +ï´ ½ +ï ¶ +ï¶ ° +ï¸ ĸ +ï¸ ´ +ï¸ ¹ +ï¹ į +ï¹ Ĺ +ï¹ ¢ +ï¹ ¤ +ï¹ © +ï¹ ± +ï¾ ° +ï¿ Ĥ +ï¿ ® +ðIJĮ ° +ðIJĮ ¹ +ðIJĮ º +ðIJĮ ½ +ðIJį Ĥ +ðIJį ĥ +ðIJį Ħ +ðIJ İ +ðIJİ ¹ +ðIJ¤ Ĥ +ðIJ¤ į +ðIJ¤ ı +ðIJ¤ ĵ +ðIJŃ ī +ðIJŃ į +ðIJ° ĩ +ðIJ° ° +ðij Ĥ +ðijĤ Ħ +ðij ĺ +ðijĺ ģ +ðĴ Ģ +ðĴĢ ¸ +ðĴ ģ +ðĴģ º +ðĴ Ħ +ðĴĦ · +ðĴ Ĭ +ðĴĬ ij +ðĴ ĭ +ðĴĭ Ĺ +ð ĴĮ +ðĴĮ ¨ +ðĵĥ ¢ +ðĵĥ ° +ðĸ ł +ðĸł ļ +ðĿĦ ĥ +ðĿĦ ħ +ðĿĦ ķ +ðĿĦ Ļ +ðĿĦ ± +ðĿĦ ´ +ðĿĦ ¹ +ðĿħ İ +ðĿħ ª +ðĿĨ £ +ðĿĨ ³ +ðĿĨ ¹ +ðĿĩ Ĭ +ðĿĩ Ĺ +ðĿĩ ļ +ðĿĩ ľ +ðĿĩ ł +ðĿIJ ī +ðĿIJ ĸ +ðĿIJ ĺ +ðĿIJ £ +ðĿIJ ± +ðĿij Ĭ +ðĿij Ń +ðĿij ¼ +ðĿij ½ +ðĿĴ ° +ðĿĴ · +ðĿĴ ¿ +ðĿĵ ģ +ðĿĵ ĭ +ðĿĵ İ +ðĿĵ Ĵ +ðĿ ĵĺ +ðĿĵ ¢ +ðĿĵ ¦ +ðĿĵ « +ðĿĵ ¿ +ðĿĶ İ +ðĿĶ ± +ðĿĶ ´ +ðĿĶ · +ðĿĶ ¸ +ðĿĶ ½ +ðĿķ Ĥ +ðĿķ ĥ +ðĿķ ĭ +ðĿķ ı +ðĿķ IJ +ðĿķ ¥ +ðĿķ ´ +ðĿķ º +ðĿĸ IJ +ðĿĸ Ľ +ðĿĸ Ŀ +ðĿĸ ŀ +ðĿĹ © +ðĿĹ ³ +ðĿĹ ½ +ðĿĺ Ĭ +ðĿĺ ĭ +ðĿĺ Ķ +ðĿĺ ± +ðĿĺ ´ +ðĿĺ ¿ +ðĿĻ Ĵ +ðĿĻ Ŀ +ðĿĻ Ł +ðĿĻ ¬ +ðĿĻ Ń +ðĿĻ » +ðĿĻ ¾ +ðĿļ Ī +ðĿļ ĭ +ðĿļ ij +ðĿļ Ł +ðĿļ ł +ðĿļ £ +ðĿĽ ½ +ðĿľ Ĥ +ðĿľ Ķ +ðĿľ Ļ +ðŁ Ģ +ðŁĢ Ħ +ðŁĦ ² +ðŁĦ ¶ +ðŁħ IJ +ðŁħ ĸ +ðŁħ ļ +ðŁħ Ľ +ðŁħ ¦ +ðŁħ ¶ +ðŁħ » +ðŁħ ¼ +ðŁĨ ĥ +ðŁĨ Ĩ +ðŁĨ İ +ðŁĪ ¯ +ðŁĪ ² +ðŁĪ ¹ +ðŁĮ ĩ +ðŁĮ ĵ +ðŁį ĺ +ðŁİ ij +ðŁİ ¿ +ðŁı ı +ðŁı Ĵ +ðŁı © +ðŁı ¯ +ðŁIJ Ģ +ðŁij Ŀ +ðŁĴ ¹ +ðŁĴ º +ðŁĵ Ł +ðŁĵ ª +ðŁĵ ¼ +ðŁĶ Ģ +ðŁĶ Ĥ +ðŁĶ ĥ +ðŁĶ ĩ +ðŁĶ ĵ +ðŁĶ ¢ +ðŁĶ ¤ +ðŁĶ © +ðŁķ ĸ +ðŁķ ļ +ðŁķ ľ +ðŁķ Ŀ +ðŁķ ŀ +ðŁķ ł +ðŁķ ¢ +ðŁķ ³ +ðŁĸ ĩ +ðŁĸ ij +ðŁĸ ¶ +ðŁĹ ģ +Ñ ¨ +Ú İ +á¡ Į +Ḡ° +ẠĢ +á¼ ® +á½ Ŀ +âĦ ¬ +âļ § +⼠¤ +ã³ ¬ +êĻ ĭ +ê¸ ij +ëĶ ī +ëĹ į +ë¡ ij +ë¯ ij +ë» ħ +ë¼ Ŀ +ìĦ IJ +ìī ¡ +ìĭ ² +ìı ± +ìĹ ¤ +ìĿ © +ìĿ ¿ +ìŁ Ļ +ìł ° +ì¥ ī +íĬ Ń +íķ ® +ï® ı +ðŁħ ± +ðŁĨ Ĵ +ðŁķ ĭ +É ĺ +Ê ĵ +Õ ĥ +à´ ´ +འħ +áĨ º +áĪ Ĭ +áĪ ¨ +áĪ ¾ +áī IJ +áĮ ĥ +áĮ ½ +áĶ Ń +áł Ĥ +áł ¬ +ᨠ¸ +á© ĭ +ᶠı +á¾ Ķ +á¿ IJ +á¿ ļ +âĻ Ļ +âļ Ĥ +âļ Ĺ +â¡ ¢ +⤠¦ +ëĸ ° +ë¤ Ĥ +ë§ ł +ë± ĭ +ë± IJ +ìĽ ¢ +ìľ ¾ +ì³ ħ +ì» ģ +íģ » +íĥ Ļ +íĵ ĸ +íĵ Ń +íķ ± +íĽ ľ +ï¤ ħ +ï¤ Ĩ +ï¦ ĥ +ï§ © +ï¨ Ĥ +ðIJ¤ Ķ +ðIJŃ ĵ +ðIJ° ¼ +ðĿĵ ŀ +ðĿĵ ° +ðĿĻ ľ +ðĿļ ģ +ðŁħ ¢ +ðŁı ĩ +È ² +Ê ¶ +Ô Ī +Ô ij +Ý ĵ +Ý ¥ +ठij +ॠ± +ଠī +à° ³ +à° µ +ಠŁ +áĢ ı +áģ ¼ +áī ¨ +áĬ Ĵ +áĭ © +áĮ Ħ +áĮ Ķ +áIJ § +á ĴĮ +áĶ ħ +áĶ Ĭ +áł Ħ +ᨠģ +Ḡĥ +Ḡ» +âĶ ŀ +âĺ µ +âļ £ +â² ¢ +ãĪ ª +ä¶ µ +ê² Ļ +ê² ´ +ê³ Ĥ +ë¡ ¼ +ìĨ Ĭ +ì¼ ĩ +íĭ į +íĵ ¬ +íĵ ® +íĵ ¶ +íĵ » +ï¤ ¦ +ï¥ ł +ï¥ ± +ïŃ ² +ðIJŃ Ĭ +ðIJ ±ħ +ðĸ ¥ +ðĸ¥ ¨ +ðĿij ³ +ðĿĵ ķ +ðĿĵ ¬ +ðĿĵ ¹ +ðĿĵ ¾ +ðĿĶ ĵ +ðĿķ į +ðĿķ ¡ +ðĿķ ± +ðĿĸ ĸ +ðĿĺ ı +ðĿĺ IJ +ðĿĺ ļ +ðĿĻ ® +ðĿĻ ° +ðĿĻ ¸ +ðĿĻ º +ðĿĻ ¼ +ðĿĻ ½ +ðĿĻ ¿ +ðĿļ Ħ +ðĿļ ı +ðŁħ ħ +ðŁħ ĵ +Æ Ī +àł Į +áĻ ³ +á ļĮ +ἠħ +ἠIJ +ᤠĬ +ḠĬ +âĶ ½ +âķ Ĭ +⼠ĩ +⼠ı +âĿ ª +âĿ « +⣠° +ãĦ į +ãĦ ĵ +ãĦ § +ãħ ĸ +ãī « +ê¦ Ķ +ï± Ĭ +ຠĤ +áħ £ +ᥠĶ +ᥠ¤ +âĨ ¤ +âĨ · +âĩ ŀ +âĸ ¤ +âŀ ¶ +ãĪ ¼ +ï¨ · +ðĵı § +âĶ ² +âĢ ´ +âĴ Ł +âĴ ¡ +â° Ĥ +â° į +â° İ +â° IJ +â° ij +â° Ł +â° ł +â° ¡ +â¼ Ń +ãĬ ¥ +âĴ ł +â½ º +ãĩ º +ãĩ ½ +ï¨ Ĭ +áķ · +âį ¨ +⺠Ł +â½ Ĺ diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/tokenizer.json b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/tokenizer.json new file mode 100644 index 0000000000000000000000000000000000000000..cd71f61a15a522601badb3dc960d800d9cb3766c --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/tokenizer.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:aeb13307a71acd8fe81861d94ad54ab689df773318809eed3cbe794b4492dae4 +size 11422654 diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/tokenizer_config.json b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..417d038a63fa3de29cfde265caedae14d1a58d92 --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/tokenizer_config.json @@ -0,0 +1,239 @@ +{ + "add_bos_token": false, + "add_prefix_space": false, + "added_tokens_decoder": { + "151643": { + "content": "<|endoftext|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151644": { + "content": "<|im_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151645": { + "content": "<|im_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151646": { + "content": "<|object_ref_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151647": { + "content": "<|object_ref_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151648": { + "content": "<|box_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151649": { + "content": "<|box_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151650": { + "content": "<|quad_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151651": { + "content": "<|quad_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151652": { + "content": "<|vision_start|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151653": { + "content": "<|vision_end|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151654": { + "content": "<|vision_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151655": { + "content": "<|image_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151656": { + "content": "<|video_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": true + }, + "151657": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151658": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151659": { + "content": "<|fim_prefix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151660": { + "content": "<|fim_middle|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151661": { + "content": "<|fim_suffix|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151662": { + "content": "<|fim_pad|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151663": { + "content": "<|repo_name|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151664": { + "content": "<|file_sep|>", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151665": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151666": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151667": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + }, + "151668": { + "content": "", + "lstrip": false, + "normalized": false, + "rstrip": false, + "single_word": false, + "special": false + } + }, + "additional_special_tokens": [ + "<|im_start|>", + "<|im_end|>", + "<|object_ref_start|>", + "<|object_ref_end|>", + "<|box_start|>", + "<|box_end|>", + "<|quad_start|>", + "<|quad_end|>", + "<|vision_start|>", + "<|vision_end|>", + "<|vision_pad|>", + "<|image_pad|>", + "<|video_pad|>" + ], + "bos_token": null, + "chat_template": "{%- if tools %}\n {{- '<|im_start|>system\\n' }}\n {%- if messages[0].role == 'system' %}\n {{- messages[0].content + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou may call one or more functions to assist with the user query.\\n\\nYou are provided with function signatures within XML tags:\\n\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n\\n\\nFor each function call, return a json object with function name and arguments within XML tags:\\n\\n{\\\"name\\\": , \\\"arguments\\\": }\\n<|im_end|>\\n\" }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {{- '<|im_start|>system\\n' + messages[0].content + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" and message.content is string and not(message.content.startswith('') and message.content.endswith('')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if message.content is string %}\n {%- set content = message.content %}\n {%- else %}\n {%- set content = '' %}\n {%- endif %}\n {%- if (message.role == \"user\") or (message.role == \"system\" and not loop.first) %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- else %}\n {%- if '' in content %}\n {%- set reasoning_content = content.split('')[0].rstrip('\\n').split('')[-1].lstrip('\\n') %}\n {%- set content = content.split('')[-1].lstrip('\\n') %}\n {%- endif %}\n {%- endif %}\n {%- if loop.index0 > ns.last_query_index %}\n {%- if loop.last or (not loop.last and reasoning_content) %}\n {{- '<|im_start|>' + message.role + '\\n\\n' + reasoning_content.strip('\\n') + '\\n\\n\\n' + content.lstrip('\\n') }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls %}\n {%- for tool_call in message.tool_calls %}\n {%- if (loop.first and content) or (not loop.first) %}\n {{- '\\n' }}\n {%- endif %}\n {%- if tool_call.function %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {{- '\\n{\"name\": \"' }}\n {{- tool_call.name }}\n {{- '\", \"arguments\": ' }}\n {%- if tool_call.arguments is string %}\n {{- tool_call.arguments }}\n {%- else %}\n {{- tool_call.arguments | tojson }}\n {%- endif %}\n {{- '}\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.first or (messages[loop.index0 - 1].role != \"tool\") %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n\\n' }}\n {{- content }}\n {{- '\\n' }}\n {%- if loop.last or (messages[loop.index0 + 1].role != \"tool\") %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '\\n\\n\\n\\n' }}\n {%- endif %}\n{%- endif %}", + "clean_up_tokenization_spaces": false, + "eos_token": "<|im_end|>", + "errors": "replace", + "model_max_length": 131072, + "pad_token": "<|endoftext|>", + "split_special_tokens": false, + "tokenizer_class": "Qwen2Tokenizer", + "unk_token": null +} diff --git a/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/vocab.json b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/vocab.json new file mode 100644 index 0000000000000000000000000000000000000000..4783fe10ac3adce15ac8f358ef5462739852c569 --- /dev/null +++ b/VideoX-Fun/models/Diffusion_Transformer/Z-Image-Turbo/tokenizer/vocab.json @@ -0,0 +1 @@ +{"!":0,"\"":1,"#":2,"$":3,"%":4,"&":5,"'":6,"(":7,")":8,"*":9,"+":10,",":11,"-":12,".":13,"/":14,"0":15,"1":16,"2":17,"3":18,"4":19,"5":20,"6":21,"7":22,"8":23,"9":24,":":25,";":26,"<":27,"=":28,">":29,"?":30,"@":31,"A":32,"B":33,"C":34,"D":35,"E":36,"F":37,"G":38,"H":39,"I":40,"J":41,"K":42,"L":43,"M":44,"N":45,"O":46,"P":47,"Q":48,"R":49,"S":50,"T":51,"U":52,"V":53,"W":54,"X":55,"Y":56,"Z":57,"[":58,"\\":59,"]":60,"^":61,"_":62,"`":63,"a":64,"b":65,"c":66,"d":67,"e":68,"f":69,"g":70,"h":71,"i":72,"j":73,"k":74,"l":75,"m":76,"n":77,"o":78,"p":79,"q":80,"r":81,"s":82,"t":83,"u":84,"v":85,"w":86,"x":87,"y":88,"z":89,"{":90,"|":91,"}":92,"~":93,"¡":94,"¢":95,"£":96,"¤":97,"¥":98,"¦":99,"§":100,"¨":101,"©":102,"ª":103,"«":104,"¬":105,"®":106,"¯":107,"°":108,"±":109,"²":110,"³":111,"´":112,"µ":113,"¶":114,"·":115,"¸":116,"¹":117,"º":118,"»":119,"¼":120,"½":121,"¾":122,"¿":123,"À":124,"Á":125,"Â":126,"Ã":127,"Ä":128,"Å":129,"Æ":130,"Ç":131,"È":132,"É":133,"Ê":134,"Ë":135,"Ì":136,"Í":137,"Î":138,"Ï":139,"Ð":140,"Ñ":141,"Ò":142,"Ó":143,"Ô":144,"Õ":145,"Ö":146,"×":147,"Ø":148,"Ù":149,"Ú":150,"Û":151,"Ü":152,"Ý":153,"Þ":154,"ß":155,"à":156,"á":157,"â":158,"ã":159,"ä":160,"å":161,"æ":162,"ç":163,"è":164,"é":165,"ê":166,"ë":167,"ì":168,"í":169,"î":170,"ï":171,"ð":172,"ñ":173,"ò":174,"ó":175,"ô":176,"õ":177,"ö":178,"÷":179,"ø":180,"ù":181,"ú":182,"û":183,"ü":184,"ý":185,"þ":186,"ÿ":187,"Ā":188,"ā":189,"Ă":190,"ă":191,"Ą":192,"ą":193,"Ć":194,"ć":195,"Ĉ":196,"ĉ":197,"Ċ":198,"ċ":199,"Č":200,"č":201,"Ď":202,"ď":203,"Đ":204,"đ":205,"Ē":206,"ē":207,"Ĕ":208,"ĕ":209,"Ė":210,"ė":211,"Ę":212,"ę":213,"Ě":214,"ě":215,"Ĝ":216,"ĝ":217,"Ğ":218,"ğ":219,"Ġ":220,"ġ":221,"Ģ":222,"ģ":223,"Ĥ":224,"ĥ":225,"Ħ":226,"ħ":227,"Ĩ":228,"ĩ":229,"Ī":230,"ī":231,"Ĭ":232,"ĭ":233,"Į":234,"į":235,"İ":236,"ı":237,"IJ":238,"ij":239,"Ĵ":240,"ĵ":241,"Ķ":242,"ķ":243,"ĸ":244,"Ĺ":245,"ĺ":246,"Ļ":247,"ļ":248,"Ľ":249,"ľ":250,"Ŀ":251,"ŀ":252,"Ł":253,"ł":254,"Ń":255,"ĠĠ":256,"ĠĠĠĠ":257,"in":258,"Ġt":259,"ĠĠĠĠĠĠĠĠ":260,"er":261,"ĠĠĠ":262,"on":263,"Ġa":264,"re":265,"at":266,"st":267,"en":268,"or":269,"Ġth":270,"ĊĊ":271,"Ġc":272,"le":273,"Ġs":274,"it":275,"an":276,"ar":277,"al":278,"Ġthe":279,";Ċ":280,"Ġp":281,"Ġf":282,"ou":283,"Ġ=":284,"is":285,"ĠĠĠĠĠĠĠ":286,"ing":287,"es":288,"Ġw":289,"ion":290,"ed":291,"ic":292,"Ġb":293,"Ġd":294,"et":295,"Ġm":296,"Ġo":297,"ĉĉ":298,"ro":299,"as":300,"el":301,"ct":302,"nd":303,"Ġin":304,"Ġh":305,"ent":306,"id":307,"Ġn":308,"am":309,"ĠĠĠĠĠĠĠĠĠĠĠ":310,"Ġto":311,"Ġre":312,"--":313,"Ġ{":314,"Ġof":315,"om":316,");Ċ":317,"im":318,"čĊ":319,"Ġ(":320,"il":321,"//":322,"Ġand":323,"ur":324,"se":325,"Ġl":326,"ex":327,"ĠS":328,"ad":329,"Ġ\"":330,"ch":331,"ut":332,"if":333,"**":334,"Ġ}":335,"em":336,"ol":337,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":338,"th":339,")Ċ":340,"Ġ{Ċ":341,"Ġg":342,"ig":343,"iv":344,",Ċ":345,"ce":346,"od":347,"Ġv":348,"ate":349,"ĠT":350,"ag":351,"ay":352,"Ġ*":353,"ot":354,"us":355,"ĠC":356,"Ġst":357,"ĠI":358,"un":359,"ul":360,"ue":361,"ĠA":362,"ow":363,"Ġ'":364,"ew":365,"Ġ<":366,"ation":367,"()":368,"Ġfor":369,"ab":370,"ort":371,"um":372,"ame":373,"Ġis":374,"pe":375,"tr":376,"ck":377,"âĢ":378,"Ġy":379,"ist":380,"----":381,".ĊĊ":382,"he":383,"Ġe":384,"lo":385,"ĠM":386,"Ġbe":387,"ers":388,"Ġon":389,"Ġcon":390,"ap":391,"ub":392,"ĠP":393,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":394,"ass":395,"int":396,">Ċ":397,"ly":398,"urn":399,"Ġ$":400,";ĊĊ":401,"av":402,"port":403,"ir":404,"->":405,"nt":406,"ction":407,"end":408,"Ġde":409,"ith":410,"out":411,"turn":412,"our":413,"ĠĠĠĠĠ":414,"lic":415,"res":416,"pt":417,"==":418,"Ġthis":419,"Ġwh":420,"Ġif":421,"ĠD":422,"ver":423,"age":424,"ĠB":425,"ht":426,"ext":427,"=\"":428,"Ġthat":429,"****":430,"ĠR":431,"Ġit":432,"ess":433,"ĠF":434,"Ġr":435,"os":436,"and":437,"Ġas":438,"ect":439,"ke":440,"rom":441,"Ġ//":442,"con":443,"ĠL":444,"(\"":445,"qu":446,"lass":447,"Ġwith":448,"iz":449,"de":450,"ĠN":451,"Ġal":452,"op":453,"up":454,"get":455,"Ġ}Ċ":456,"ile":457,"Ġan":458,"ata":459,"ore":460,"ri":461,"Ġpro":462,";čĊ":463,"ĉĉĉĉ":464,"ter":465,"ain":466,"ĠW":467,"ĠE":468,"Ġcom":469,"Ġreturn":470,"art":471,"ĠH":472,"ack":473,"import":474,"ublic":475,"Ġor":476,"est":477,"ment":478,"ĠG":479,"able":480,"Ġ-":481,"ine":482,"ill":483,"ind":484,"ere":485,"::":486,"ity":487,"Ġ+":488,"Ġtr":489,"elf":490,"ight":491,"('":492,"orm":493,"ult":494,"str":495,"..":496,"\",":497,"Ġyou":498,"ype":499,"pl":500,"Ġnew":501,"Ġj":502,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":503,"Ġfrom":504,"Ġex":505,"ĠO":506,"ld":507,"Ġ[":508,"oc":509,":Ċ":510,"Ġse":511,"Ġle":512,"--------":513,".s":514,"{Ċ":515,"',":516,"ant":517,"Ġat":518,"ase":519,".c":520,"Ġch":521,"":589,"ust":590,"que":591,"Ġres":592,"))":593,"'s":594,"Ġk":595,"ans":596,"yst":597,"unction":598,"********":599,"Ġi":600,"Ġus":601,"pp":602,"one":603,"ail":604,"====":605,"name":606,"Ġstr":607,"Ġ/":608,"Ġ&":609,"ach":610,"div":611,"ystem":612,"ell":613,"Ġhave":614,"err":615,"ould":616,"ull":617,"pon":618,"ĠJ":619,"_p":620,"Ġ==":621,"ign":622,"St":623,".Ċ":624,"Ġpl":625,");ĊĊ":626,"form":627,"put":628,"ount":629,"}ĊĊ":630,"dd":631,"ite":632,"Ġget":633,"rr":634,"ome":635,"ĠâĢ":636,"aram":637,"cc":638,"Ġ*/":639,"ER":640,"In":641,"les":642,"_s":643,"ong":644,"ie":645,"Ġcan":646,"ĠV":647,"erv":648,"pr":649,"Ġun":650,"row":651,"ber":652,"Ġdo":653,"ll":654,"Ġel":655,"Ġself":656,"ated":657,"ary":658,"Ġ.":659,"']":660,"ud":661,"Ġen":662,"ĠTh":663,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":664,"te":665,"_c":666,"uct":667,"Ġab":668,"ork":669,".get":670,"Ġ#":671,"aw":672,"ress":673,"ob":674,"Name":675,"app":676,"['":677,"Ġall":678,"ory":679,"ition":680,"ance":681,"ear":682,"Ġcont":683,"vent":684,"ia":685,"Ġwill":686,"IN":687,"ĠĠĠĠĠĠĠĠĠ":688,"return":689,"Ġ":755,"\",Ċ":756,"ec":757,"ĠIn":758,"ph":759,"Ġ|":760,"_f":761,"Ġvar":762,"ence":763,"Id":764,"ree":765,"ink":766,"lect":767,"ug":768,"eth":769,"Ġelse":770,"----------------":771,"cont":772,"Ġso":773,"atic":774,"Ġlo":775,"pro":776,"ton":777,"ss":778,"own":779,"abel":780,"oint":781,"ous":782,"eld":783,"ST":784,"The":785,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":786,"RE":787,"\":":788,"olor":789,"tp":790,"eg":791,"key":792,"ude":793,"ĠSt":794,"ound":795,"Ġar":796,"\");Ċ":797,"ener":798,"ser":799,"bject":800,"essage":801,"fer":802,"Ġmore":803,"ations":804,"ents":805,"Ġhis":806,"Ġthey":807,".S":808,"ĠY":809,"use":810,"ne":811,"ish":812,"old":813,"_d":814,"io":815,"ield":816,"Ġper":817,"Cont":818,"ings":819,"####":820,"Ġdata":821,"Ġsa":822,"ef":823,"fo":824,"Ġone":825,"eng":826,"Ġdis":827,"AT":828,"Ġname":829,"Ġtrue":830,"val":831,"led":832,".f":833,"Ġne":834,"Ġend":835,".T":836,"cre":837,"ark":838,"log":839,"Ex":840,"error":841,"_id":842,"urre":843,"ange":844,"Ġnull":845,"rray":846,"Ġmy":847,"pan":848,"ict":849,"ator":850,"View":851,"List":852,"ĉreturn":853,"âĢĿ":854,"Ġpre":855,"Ġx":856,"clude":857,"arg":858,"ov":859,".h":860,"Ġ>":861,"Ġtheir":862,"')":863,"irst":864,"ick":865,"gh":866,"LE":867,"OR":868,"Ġprivate":869,"tem":870,"čĊčĊ":871,"user":872,"Ġ)":873,"com":874,".A":875,"\";Ċ":876,"Ġid":877,"read":878,"Ġwho":879,"_b":880,"\">Ċ":881,"Ġtime":882,"Ġman":883,"ry":884,"========":885,"roup":886,"rop":887,"public":888,"vel":889,"umber":890,"ble":891,"Ġwhich":892,"****************":893,"Ġany":894,"Ġfalse":895,"we":896,"Ġvalue":897,"Ġli":898,"\")":899,"nder":900,"gr":901,"Ġno":902,"param":903,"fig":904,".com":905,"Ġapp":906,"_l":907,"ions":908,".D":909,"ĠCh":910,"Ġabout":911,"Ġadd":912,"Ġsu":913,"Ġstring":914,"ID":915,"Ġover":916,"string":917,".l":918,"ource":919,"_C":920,"]Ċ":921,"Ġqu":922,"ĠString":923,"ca":924,"SE":925,"Ġro":926,"sh":927,"ual":928,"Type":929,"son":930,"new":931,"ern":932,"Ġag":933,"AR":934,"];Ċ":935,"].":936,"Ġ?":937,"ical":938,"Ġdes":939,"uth":940,"ix":941,"ays":942,"Ġtype":943,"'t":944,"ault":945,"Ġinter":946,"var":947,".b":948,"Ġpart":949,".d":950,"urrent":951,"IT":952,"EN":953,"enc":954,"(f":955,"ra":956,"value":957,"cho":958,"utton":959,"ose":960,"Ġ!=":961,"ater":962,"é":963,"reate":964,"oll":965,"pos":966,"yle":967,"ng":968,"AL":969,"using":970,"ames":971,"Ġ{čĊ":972,"ates":973,"ely":974,"Ġwork":975,"Ġem":976,"inal":977,"Ġsp":978,"Ġwhen":979,".set":980,"ĠĠĠĠĠĠ":981,"):Ċ":982,"to":983,"quire":984,"indow":985,"lement":986,"pect":987,"ash":988,"[i":989,"Ġuse":990,".F":991,"pec":992,"Ġad":993,"ove":994,"ception":995,"ength":996,"include":997,"ader":998,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":999,"atus":1000,"Th":1001,"itle":1002,"rit":1003,"void":1004,"().":1005,"(Ċ":1006,"Ġoff":1007,"Ġother":1008,"Ġ&&":1009,"';Ċ":1010,"ms":1011,"Ġbeen":1012,"Ġte":1013,"ml":1014,"co":1015,"nc":1016,"ervice":1017,"Ġ%":1018,"**Ċ":1019,"ann":1020,"ade":1021,"ĊĊĊĊ":1022,"lock":1023,"const":1024,"ponse":1025,"Ġsup":1026,"++":1027,"date":1028,"Ġacc":1029,"Ġhad":1030,"Ġbu":1031,"ĠRe":1032,"Ġwere":1033,"Ġfile":1034,"Ġwould":1035,"ĠâĢľ":1036,"ven":1037,"iss":1038,"Ġour":1039,"class":1040,"raw":1041,"Ġyear":1042,"Data":1043,"Ġval":1044,"Ġsome":1045,"fter":1046,"ys":1047,"Ġ///":1048,"round":1049,"view":1050,"Ġpe":1051,"Ġthere":1052,"Ġsaid":1053,"du":1054,"of":1055,"line":1056,"/*":1057,"duct":1058,"Ġher":1059,"ĠĠĠĠĠĠĠĠĠĠĠĠĠ":1060,"Res":1061,"Ġco":1062,"Ġcomm":1063,"ise":1064,"min":1065,"ĠĠĠĠĊ":1066,"#include":1067,"ethod":1068,".P":1069,"ute":1070,"Ġass":1071,"Int":1072,"ask":1073,"loc":1074,"Ġlike":1075,"ody":1076,"Ġlet":1077,"load":1078,"Ġam":1079,"rol":1080,"Ġgr":1081,"yp":1082,"Ġalso":1083,"ĠIt":1084,"url":1085,"ific":1086,"ors":1087,"_P":1088,"_n":1089,"igh":1090,"Ġthan":1091,"Com":1092,"AN":1093,"UL":1094,"ating":1095,"ĠThis":1096,"ref":1097,"_S":1098,"Ġstatic":1099,"roll":1100,"Ġjust":1101,"Ġresult":1102,"ian":1103,"idth":1104,"Ġthem":1105,"));Ċ":1106,"der":1107,"reak":1108,"Con":1109,"://":1110,"ule":1111,"...":1112,"arch":1113,"ement":1114,"Ġ<<":1115,"ush":1116,"ense":1117,"arr":1118,"Ġinto":1119,"cess":1120,"amp":1121,"ied":1122,"ument":1123,"Ġ\\":1124,"],":1125,"wo":1126,"als":1127,"Ġwhat":1128,"anc":1129,"Value":1130,"='":1131,"olum":1132,"Ġpos":1133,"ages":1134,"ayer":1135,"Ġsc":1136,"ues":1137,"\")Ċ":1138,"_T":1139,"Ġlist":1140,"(s":1141,"Ġcase":1142,"Ch":1143,"ĉĉĉĉĉ":1144,"////////":1145,"ponent":1146,"Ġz":1147,"Ġkn":1148,"let":1149,"DE":1150,"red":1151,"Ġfe":1152,"Ġ},Ċ":1153,"Ġ,":1154,"(t":1155,"Ġfirst":1156,"');Ċ":1157,"word":1158,"Ġimport":1159,"Ġact":1160,"Ġchar":1161,"CT":1162,"ĠTr":1163,"ople":1164,"={":1165,"ĉf":1166,"ient":1167,"cent":1168,".j":1169,"lection":1170,"))Ċ":1171,"Ġonly":1172,"Ġprint":1173,"mer":1174,".W":1175,"ock":1176,"Ġ--":1177,"Text":1178,"Ġop":1179,"ank":1180,"Ġits":1181,"Ġback":1182,"[\"":1183,"Ġneed":1184,"Ġcl":1185,"Ġsub":1186,"Ġla":1187,"((":1188,".\"":1189,"Object":1190,"Ġstart":1191,"file":1192,"(self":1193,"ner":1194,"ey":1195,"Ġuser":1196,"Ġent":1197,"ĠCom":1198,"its":1199,"ĠCon":1200,"ouble":1201,"ower":1202,"item":1203,"very":1204,"ĠWe":1205,"lick":1206,"ĠQ":1207,"php":1208,"ttp":1209,"':":1210,"ics":1211,"Ġunder":1212,"Ġ*Ċ":1213,".L":1214,");":1215,"ices":1216,"Ġreg":1217,")čĊ":1218,"ĉpublic":1219,"SS":1220,"Ġthen":1221,"reat":1222,"ious":1223,".G":1224,"ek":1225,"irect":1226,"heck":1227,"cript":1228,"ning":1229,"ĠUn":1230,"Ġmay":1231,"ĠWh":1232,"Bo":1233,"Item":1234,"struct":1235,".st":1236,"ream":1237,"ible":1238,"loat":1239,"Ġorg":1240,"und":1241,"sum":1242,"_in":1243,"../":1244,"_M":1245,"Ġhow":1246,"rite":1247,"'Ċ":1248,"To":1249,"ww":1250,"Ġpeople":1251,"index":1252,".n":1253,"http":1254,"(m":1255,"ector":1256,"Ġind":1257,"Ġjav":1258,"],Ċ":1259,"ĠHe":1260,"_st":1261,"ful":1262,"ole":1263,"){Ċ":1264,"Ġshould":1265,"opy":1266,"elp":1267,"ier":1268,"_name":1269,"erson":1270,"ION":1271,"ote":1272,"Ġtest":1273,"Ġbet":1274,"rror":1275,"ular":1276,"ãĢ":1277,"ĠÐ":1278,"bs":1279,"ting":1280,"Ġmake":1281,"Tr":1282,"Ġafter":1283,"arget":1284,"RO":1285,"olumn":1286,"rc":1287,"_re":1288,"define":1289,"Ġright":1290,"right":1291,"day":1292,"Ġlong":1293,"[]":1294,"(p":1295,"td":1296,"cond":1297,"ĠPro":1298,"Ġrem":1299,"ptions":1300,"vid":1301,".g":1302,"Ġext":1303,"Ġ__":1304,"')Ċ":1305,"pace":1306,"mp":1307,"Ġmin":1308,"stance":1309,"air":1310,"action":1311,"wh":1312,"type":1313,"util":1314,"ait":1315,"ĊĊ":1339,"Ġshe":1340,"\"]":1341,"aph":1342,"Ġexp":1343,"erty":1344,"ĠSe":1345,"Ġpar":1346,"unc":1347,"ET":1348,"Ġread":1349,"print":1350,"Ġrel":1351,"Ġform":1352,"Ġdr":1353,"Exception":1354,"input":1355,"Ġtrans":1356,"########":1357,"order":1358,"By":1359,"Ġaw":1360,"ities":1361,"uff":1362,"play":1363,".add":1364,"ĠâĢĵ":1365,"Ġwant":1366,"Ġcomp":1367,"ments":1368,"Ġ||":1369,"az":1370,"be":1371,"Ġnumber":1372,"Ġrequire":1373,"ĠEx":1374,"Ġcol":1375,"Ġkey":1376,"ember":1377,"Ġtwo":1378,"Ġsize":1379,"Ġwhere":1380,"UT":1381,"result":1382,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":1383,"ough":1384,"orld":1385,"ood":1386,"uch":1387,"ative":1388,"ger":1389,"arent":1390,"Ġ/*":1391,"Ġarg":1392,"Ġwhile":1393,"(this":1394,"Ġrec":1395,"Ġdif":1396,"State":1397,"Ġspec":1398,"ride":1399,"_F":1400,"Ġlook":1401,"AM":1402,"ility":1403,"eter":1404,"âĢĻt":1405,"ĊĊĊ":1406,"ayout":1407,"--------------------------------":1408,"ager":1409,"Ġcould":1410,"Ġbr":1411,"ends":1412,"ures":1413,"Ġknow":1414,"ets":1415,"ĠIf":1416,"ĠSh":1417,".w":1418,"back":1419,"Ġser":1420,"Ġ+=":1421,"Ġfr":1422,"());Ċ":1423,"Ġhand":1424,"Ind":1425,"ULL":1426,"Im":1427,"();ĊĊ":1428,"Ġmost":1429,"Ġtry":1430,"Ġnow":1431,"rough":1432,">čĊ":1433,"ackage":1434,"Ġhim":1435,"._":1436,"ify":1437,"Ġbreak":1438,"Ġ);Ċ":1439,"ren":1440,"#define":1441,"itt":1442,"Ġap":1443,"ĉc":1444,"(n":1445,"ĠYou":1446,":ĊĊ":1447,"-m":1448,"Ġevery":1449,"ustom":1450,"lient":1451,"ocument":1452,"cription":1453,"Error":1454,"-b":1455,"о":1456,"][":1457,"trans":1458,"Ġpoint":1459,"Ġstd":1460,"Ġfil":1461,"Time":1462,"Ġmod":1463,"Ġ->":1464,"Ġerror":1465,"ah":1466,"Ġtext":1467,"roller":1468,"lose":1469,"ql":1470,"Ġpol":1471,"><":1784,".B":1785,"-c":1786,"Ġopen":1787,"Ġest":1788,"ĠĠĠĠĠĠĠĠĊ":1789,"Ġnext":1790,"IM":1791,"ÑĤ":1792,"OT":1793,"ó":1794,"Ġfollow":1795,"content":1796,"ĠĠĠĠĠĠĠĠĠĠĠĠ":1797,"Ġinclud":1798,"HE":1799,"ĠRes":1800,"Ġhref":1801,"и":1802,"Ġcar":1803,"ypes":1804,"image":1805,"Un":1806,"Ġbool":1807,"AD":1808,"Ġgame":1809,".Form":1810,"rows":1811,"*/":1812,"velop":1813,".Drawing":1814,"Ġpath":1815,"ision":1816,"Ġeach":1817,"ĠPl":1818,"_type":1819,"Path":1820,"nection":1821,"Ġav":1822,"').":1823,"Ġsupport":1824,"ENT":1825,"rem":1826,"\").":1827,"Ġown":1828,"Ġcor":1829,"count":1830,"miss":1831,"ually":1832,"Ġmem":1833,"std":1834,"ience":1835,"search":1836,"\"ĊĊ":1837,"Form":1838,"Ġsex":1839,"ename":1840,"Ġsign":1841,"Ġet":1842,"ĠĠĠĠĠĠĠĠĠĠ":1843,"','":1844,"ĠApp":1845,"Ġthose":1846,"off":1847,"Ġerr":1848,"Ġsystem":1849,"Ġbest":1850,"code":1851,"Ġsame":1852,"Ġdi":1853,"uss":1854,"Ġcreate":1855,"ather":1856,"Array":1857,".in":1858,"fe":1859,"Service":1860,"UN":1861,"ats":1862,"ĠZ":1863,"alth":1864,"Ġmade":1865,"true":1866,"AB":1867,"Ġmark":1868,"rid":1869,"ified":1870,",čĊ":1871,"yn":1872,"press":1873,"Ġgroup":1874,"Ġfin":1875,"ĠLicense":1876,"Field":1877,"eger":1878,"Ġworld":1879,"iness":1880,"ty":1881,"Ġprocess":1882,"(b":1883,"Ġcre":1884,"arn":1885,"ives":1886,"Ġmain":1887,"ideo":1888,"_g":1889,"AG":1890,"valid":1891,"img":1892,"PI":1893,"Ġcolor":1894,"Ġreport":1895,"Ġtake":1896,"rib":1897,"OM":1898,"Ġday":1899,"Request":1900,"Ġsk":1901,"bers":1902,"ĉs":1903,".Add":1904,"oot":1905,"Image":1906,"Ġcomple":1907,"ollection":1908,"Ġtop":1909,"Ġfree":1910,"AS":1911,"De":1912,"ĠOn":1913,"IG":1914,"eta":1915,"Date":1916,"Ġaction":1917,"Over":1918,"itor":1919,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":1920,"not":1921,"Ġindex":1922,"her":1923,"icon":1924,"On":1925,";čĊčĊ":1926,"ivity":1927,"mand":1928,".Windows":1929,"OL":1930,"Ġreal":1931,"Ġmax":1932,"land":1933,"....":1934,"raph":1935,"Ġbuild":1936,"leg":1937,"assword":1938,"?ĊĊ":1939,"âĢ¦":1940,"ook":1941,"uck":1942,"Ġmessage":1943,"test":1944,"ivers":1945,"Ġinput":1946,"Ġart":1947,"Ġbetween":1948,"Get":1949,"enter":1950,"ground":1951,"ene":1952,"á":1953,".length":1954,"Node":1955,"(i":1956,"Class":1957,"for":1958,"ĠâĢĶ":1959,"ten":1960,"oin":1961,"Ġke":1962,"ui":1963,"ĠIN":1964,"Ġtable":1965,"sub":1966,"ĠLe":1967,"Ġhead":1968,"Ġmust":1969,"////////////////":1970,".util":1971,"Context":1972,"Ġorder":1973,"Ġmov":1974,"over":1975,"Ġcontin":1976,"Ġsay":1977,"static":1978,".Text":1979,"ĠclassName":1980,"pany":1981,"Ġter":1982,"head":1983,"rg":1984,"Ġproduct":1985,"This":1986,".âĢĿ":1987,"ĠBut":1988,"loy":1989,"Ġdouble":1990,"sg":1991,"Ġplace":1992,".x":1993,"message":1994,"Ġinformation":1995,"private":1996,"Ġoper":1997,"ced":1998,"db":1999,"\">":2179,"aterial":2180,"iled":2181,"Ġput":2182,"Qu":2183,"ÑĢ":2184,"ung":2185,"map":2186,"ĉĉĉĉĉĉĉĉ":2187,"Ġlevel":2188,"Component":2189,"book":2190,"creen":2191,"_RE":2192,"Ġconfig":2193,"ãģ":2194,"Or":2195,".data":2196,"Ġdocument":2197,"\",\"":2198,"tribute":2199,"ux":2200,"Log":2201,"ference":2202,"post":2203,"_e":2204,"Ġlocal":2205,"andom":2206,"assert":2207,"Val":2208,"lected":2209,"ina":2210,"atabase":2211,"Add":2212,"Ġcontent":2213,".print":2214,"signed":2215,"ric":2216,".\"ĊĊ":2217,"Ġfa":2218,"!ĊĊ":2219,"-f":2220,"ived":2221,"Ġquest":2222,".ex":2223,"Ġfloat":2224,"Ġdevelop":2225,"оÐ":2226,"Map":2227,"ading":2228,"Ġposs":2229,"UE":2230,"namespace":2231,"_O":2232,"ĉb":2233,".Get":2234,">(":2235,"json":2236,"etails":2237,"Ġtoo":2238,"Ġextends":2239,"ĠNone":2240,"Ġfore":2241,"(String":2242,"format":2243,"Ġgreat":2244,"inter":2245,"cale":2246,"Ñģ":2247,"ron":2248,"iving":2249,"Ent":2250,"ency":2251,"xt":2252,"oy":2253,"Ġmonth":2254,"Ġhapp":2255,"Ġsuper":2256,"bar":2257,"default":2258,"_de":2259,"ords":2260,"ln":2261,"({Ċ":2262,"ĠInd":2263,"ases":2264,"Ġtitle":2265,"Ġcontext":2266,"oh":2267,"-p":2268,"Em":2269,"Ġmet":2270,"Test":2271,"Ġlife":2272,"_v":2273,"ĠUS":2274,"UI":2275,"ocation":2276,"md":2277,"Ġ[Ċ":2278,"Ġ]":2279,"sw":2280,"Ġincre":2281,"script":2282,"ential":2283,"ways":2284,".de":2285,"Ġsrc":2286,"Ġcatch":2287,"ĠAmeric":2288,"//Ċ":2289,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":2290,"Ġpay":2291,"plit":2292,"âĢĶ":2293,"Ġcoun":2294,"obj":2295,".php":2296,"Ġchange":2297,"ething":2298,"'re":2299,"aster":2300,"los":2301,"lation":2302,"ĠĠĊ":2303,"Le":2304,"ä":2305,"({":2306,"ready":2307,"ĠNo":2308,"Ġposition":2309,"Ġold":2310,"Ġbook":2311,"abled":2312,"bug":2313,"Hand":2314,"};ĊĊ":2315,"isplay":2316,"aving":2317,"Ġgover":2318,"Ġversion":2319,"System":2320,"nect":2321,"response":2322,"Style":2323,"Up":2324,"angu":2325,"Ġthree":2326,"init":2327,"ero":2328,"Ġlaw":2329,"endif":2330,"Ġbase":2331,"email":2332,"(l":2333,"_V":2334,"Ġconf":2335,"ATE":2336,"Ġduring":2337,"tes":2338,"Ġconsole":2339,"ĠPr":2340,"Ġspe":2341,"ves":2342,"path":2343,"ialog":2344,"dition":2345,"_to":2346,"ards":2347,"Ġagainst":2348,"etwork":2349,"ĠPh":2350,"_L":2351,"cur":2352,"imit":2353,"With":2354,"Ġpower":2355,"ium":2356,"';ĊĊ":2357,"Ġwom":2358,"left":2359,"ources":2360,"atri":2361,"ĠIm":2362,"ĠMan":2363,"orth":2364,"${":2365,"quals":2366,"ese":2367,"_size":2368,"Ġiss":2369,"otal":2370,"-g":2371,"ique":2372,"rame":2373,"Ġwidth":2374,"erg":2375,")(":2376,"ittle":2377,"TR":2378,"ĠThey":2379,"ences":2380,"rl":2381,"ons":2382,"Ġlabel":2383,".y":2384,"-t":2385,"update":2386,"anel":2387,"sc":2388,".to":2389,"Ġproject":2390,"ü":2391,"Ġelement":2392,"Ġsuccess":2393,"ĉĉĊ":2394,".sh":2395,"ram":2396,"ched":2397,"())Ċ":2398,"Ġ(Ċ":2399,"Ġdate":2400,"Ġtot":2401,"_ST":2402,"All":2403,"ification":2404,"ĉvar":2405,"Ġtri":2406,"chem":2407,"my":2408,"Ġbig":2409,"ĠAd":2410,"ĠAt":2411,"ots":2412,"num":2413,"Act":2414,"Ġmap":2415,"era":2416,"cope":2417,".$":2418,",âĢĿ":2419,"Ġpop":2420,"Ġfew":2421,"Ġlen":2422,"uid":2423,"eters":2424,"ules":2425,"ÃŃ":2426,"source":2427,"https":2428,"Ġdem":2429,"Ġear":2430,"################":2431,"Ġmatch":2432,"ories":2433,"aces":2434,"ĠCl":2435,"Ġnode":2436,"irc":2437,"local":2438,"unity":2439,"};Ċ":2440,"Ġanother":2441,"<<":2442,"ogle":2443,"Ġsit":2444,"ework":2445,"TE":2446,".I":2447,"NS":2448,"ology":2449,"ought":2450,".Cont":2451,">>":2452,"Ġcare":2453,"state":2454,"ĉprivate":2455,"Ġeffect":2456,"++)":2457,"_file":2458,"ending":2459,"Line":2460,"For":2461,"ior":2462,"ĠSc":2463,"Ġfun":2464,".Size":2465,"ĉelse":2466,"])":2467,"start":2468,"vious":2469,"Ġ},":2470,"ours":2471,"Ġleg":2472,"Ġservice":2473,"Ġsince":2474,"iron":2475,"Label":2476,"Ġnon":2477,"Ġlos":2478,"iction":2479,"Ġfull":2480,"acter":2481,"board":2482,"gress":2483,"Ġturn":2484,"ither":2485,".size":2486,"Ġbody":2487,"resh":2488,"eturn":2489,"(_":2490,"yles":2491,"ormal":2492,"pi":2493,"Ġsomething":2494,"!--":2495,"uint":2496,"Ġprodu":2497,"Ġstand":2498,"Ġproble":2499,"Ġavailable":2500,"mt":2501,"ĠBl":2502,"Ġ...":2503,"Ġblock":2504,"Input":2505,"Ġkeep":2506,"Count":2507,"open":2508,"Ġ['":2509,"Ġthrow":2510,"uilder":2511,"Action":2512,"Ġthings":2513,"True":2514,"Ġurl":2515,"ĠBo":2516,"printf":2517,"Ġred":2518,"js":2519,".create":2520,"ĠOr":2521,"Status":2522,"Instance":2523,"Ġcontrol":2524,"Ġcome":2525,"Ġcustom":2526,"location":2527,"model":2528,"ĠčĊ":2529,"Ġsource":2530,"Ġeas":2531,".out":2532,"]ĊĊ":2533,"oney":2534,"Ġawait":2535,"Ġpartic":2536,"AP":2537,"ublish":2538,"odes":2539,"_pro":2540,"ply":2541,"riter":2542,"Ġprov":2543,"Ġmill":2544,"HT":2545,"])Ċ":2546,"Ġchang":2547,"Ġask":2548,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":2549,"Ġoutput":2550,"Ġemail":2551,".push":2552,"Ġ}čĊčĊ":2553,"ination":2554,"atrix":2555,"Table":2556,"uccess":2557,"]);Ċ":2558,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":2559,"Ġdisc":2560,"([":2561,"Ġbusiness":2562,"height":2563,".html":2564,"ta":2565,"field":2566,"Ġrequired":2567,"_R":2568,"Ġgovern":2569,"}čĊčĊ":2570,"lex":2571,".,":2572,"ĠSet":2573,"urch":2574,"///":2575,"ts":2576,"af":2577,"Ġmight":2578,"istory":2579,"Str":2580,"Ġnever":2581,"Response":2582,"arse":2583,"ada":2584,"ĠHow":2585,"Ġ*)":2586,"Ġ;":2587,"Ġhard":2588,"Ad":2589,"Ġintern":2590,"used":2591,"(data":2592,"mod":2593,"annel":2594,"Ġnp":2595,"ugg":2596,"Ġ/>Ċ":2597,"Ġcalled":2598,"body":2599,"Ġcho":2600,"(r":2601,"_set":2602,"ird":2603,"Ġ>=":2604,"Ġ};Ċ":2605,"Ġoptions":2606,"ĠGener":2607,"Ġheight":2608,"Point":2609,"You":2610,"ety":2611,"Click":2612,"Ġsmall":2613,"Ġide":2614,"Ġaccess":2615,"anguage":2616,"Ġprotected":2617,"Ġjob":2618,"ĠThere":2619,"Def":2620,"Ġaddress":2621,"Ġuint":2622,"Not":2623,"oo":2624,"aps":2625,"":2759,"ĉĠĠĠ":2760,"\"))":2761,"Content":2762,"_W":2763,"plement":2764,"Ġwon":2765,"Ġvideo":2766,"adi":2767,"point":2768,"%%":2769,"Ġgl":2770,"erved":2771,"viron":2772,"IF":2773,"uted":2774,"ãĥ":2775,"'m":2776,"Ġcert":2777,"Ġprof":2778,"Ġcell":2779,"ari":2780,"Ġplayer":2781,"ais":2782,"Ġcost":2783,"Ġhum":2784,"(R":2785,"Ġoffic":2786,"ks":2787,".text":2788,"atures":2789,"Ġtotal":2790,"Ġ*/ĊĊ":2791,"ope":2792,"Ġstat":2793,"UM":2794,"Ġload":2795,"ights":2796,"Ġclear":2797,"uro":2798,"Ġtechn":2799,"upport":2800,"IR":2801,"Ġrow":2802,"Ġseem":2803,"Ġq":2804,"Ġshort":2805,"ĠNot":2806,"ipp":2807,"Group":2808,"section":2809,"max":2810,"irl":2811,"Ġoverride":2812,"Ġcompany":2813,"Ġdone":2814,"\");čĊ":2815,"Ġgre":2816,".Re":2817,"Ġbelie":2818,"rist":2819,"Ġhealth":2820,"ANT":2821,"()ĊĊ":2822,"ĠBe":2823,".value":2824,"ĠGr":2825,"ottom":2826,"Ġargs":2827,"PT":2828,"status":2829,"func":2830,"uments":2831,"-h":2832,"Number":2833,":čĊ":2834,"ĠLog":2835,"erver":2836,"Ġ),Ċ":2837,"ament":2838,"Ġobj":2839,"inc":2840,"Ġchildren":2841,"icy":2842,"IZ":2843,"ands":2844,"ably":2845,"Ġdistrib":2846,"Ġcur":2847,"erial":2848,"Ġdays":2849,"reated":2850,"rect":2851,"-l":2852,"irm":2853,"idden":2854,"omb":2855,"Ġinitial":2856,".js":2857,"Ġâ":2858,"Query":2859,"Ġonline":2860,"imal":2861,".con":2862,"au":2863,"Url":2864,"control":2865,"irection":2866,"Ġinstance":2867,"ORT":2868,"ĠFr":2869,"where":2870,"Ġjavax":2871,"Ġorgan":2872,"apter":2873,"Ġreason":2874,"options":2875,"ĠMar":2876,"(a":2877,"Ġwithin":2878,".âĢĿĊĊ":2879,"ODE":2880,"_DE":2881,"admin":2882,"ended":2883,"Ġdesign":2884,"ĠData":2885,"une":2886,"ĠFile":2887,"root":2888,"Ġcent":2889,"Ġarr":2890,"_add":2891,"len":2892,"page":2893,",'":2894,"_str":2895,"Ġbro":2896,"ability":2897,"outh":2898,"/c":2899,"pose":2900,"irtual":2901,"earch":2902,"_url":2903,"argin":2904,"Http":2905,"Ġschool":2906,"ava":2907,"Ġconsider":2908,".label":2909,"ĠArray":2910,"web":2911,"opt":2912,".println":2913,"ulation":2914,"Ġfunc":2915,"PL":2916,"Ġ\"\\":2917,"ĠText":2918,"actory":2919,"(function":2920,"null":2921,"Ġeng":2922,"down":2923,"Ġinclude":2924,"ĠEn":2925,"ĠDr":2926,"Ġdb":2927,"!!":2928,"side":2929,"Ġinit":2930,"quired":2931,"ĠShe":2932,"Column":2933,"react":2934,"Ġann":2935,"Ġstop":2936,"Ġlater":2937,"ĠThat":2938,"ention":2939,"df":2940,"UG":2941,"ILE":2942,"Ġclient":2943,"raft":2944,"ffer":2945,"POST":2946,"elper":2947,"Ġlove":2948,"quote":2949,"oud":2950,"Ġjson":2951,"Ġable":2952,"Ġmen":2953,"AX":2954,"ĠCopyright":2955,"ö":2956,"avig":2957,"req":2958,"Client":2959,"});Ċ":2960,".Com":2961,"erc":2962,"ilt":2963,"pecial":2964,"_com":2965,"room":2966,".Name":2967,"Ġgive":2968,"amb":2969,"ike":2970,"Ġcondition":2971,"client":2972,"ators":2973,":\"":2974,"Ġcopy":2975,"uture":2976,"iversity":2977,"ernal":2978,"{{":2979,"ĠCan":2980,"ounc":2981,"do":2982,"Ġocc":2983,"Ġappro":2984,"thers":2985,"ze":2986,"Ġeither":2987,"ĠFl":2988,"Ġimportant":2989,"Ġlead":2990,"attr":2991,"ART":2992,"Equal":2993,"Ġda":2994,"etch":2995,"entity":2996,"Ġfamily":2997,"adding":2998,"Ġoption":2999,"Ġexist":3000,"ica":3001,"ĠObject":3002,"'ve":3003,"vers":3004,"itional":3005,"output":3006,"ĠTrue":3007,"ĠOF":3008,"_time":3009,"Ġoffer":3010,"Ġ});ĊĊ":3011,"HER":3012,"egin":3013,"\"\"":3014,"Ġwater":3015,"Ġche":3016,"ĠMy":3017,"ored":3018,"Ġstep":3019,"ances":3020,"CK":3021,"AY":3022,"à¸":3023,"struction":3024,"(C":3025,"ouch":3026,"Stream":3027,"active":3028,"ama":3029,"Entity":3030,"product":3031,"(){Ċ":3032,"Ġgovernment":3033,"ĠID":3034,"ajor":3035,"And":3036,"Ġdisplay":3037,"л":3038,"Ġtimes":3039,"Ġfour":3040,"Ġfar":3041,"Ġpresent":3042,"ĠNS":3043,"Ġ\\Ċ":3044,"uest":3045,"Ġbas":3046,"echo":3047,"child":3048,"ifier":3049,"Handler":3050,"Ġlib":3051,"Property":3052,"translation":3053,"Ġroom":3054,"Ġonce":3055,"Ġ[]":3056,"center":3057,"================================":3058,"Ġresults":3059,"Ġcontinue":3060,"Ġtalk":3061,"_get":3062,"Ġgrow":3063,".sw":3064,"eb":3065,"ĠPublic":3066,"OP":3067,"ecute":3068,"ols":3069,"Ġ**":3070,"\");ĊĊ":3071,"Ġmass":3072,"ured":3073,".class":3074,"omic":3075,"Ġmean":3076,"ips":3077,"Ġaut":3078,");čĊčĊ":3079,"Ġuntil":3080,"Ġmarket":3081,"Ġarea":3082,"uit":3083,"Ġlength":3084,"ĠWith":3085,"structor":3086,"event":3087,"\"><":3088,"ĠSp":3089,"IV":3090,"Ġmus":3091,"iff":3092,"Ġkind":3093,"author":3094,"ounds":3095,"mb":3096,"_key":3097,"width":3098,"pository":3099,"Ġlight":3100,"uk":3101,"Row":3102,"ohn":3103,"alf":3104,"vironment":3105,"apper":3106,"ollections":3107,"Ġside":3108,"_info":3109,"Ġexample":3110,"imary":3111,"Ġwr":3112,"Ġcamp":3113,"cribe":3114,"\"/":3115,"Ġmiss":3116,"way":3117,"Ġbased":3118,"Ġplan":3119,"Vis":3120,"omain":3121,"unk":3122,"Ġaway":3123,"UP":3124,"":3370,"Ġden":3371,"obile":3372,"change":3373,"ĠĠĠĠĠĠĠĠĠĠĠĠĊ":3374,"ici":3375,"na":3376,"ĠForm":3377,"Ġsort":3378,"Select":3379,"pare":3380,"Ġthought":3381,"_con":3382,"Ġtask":3383,"ocus":3384,"ĠDE":3385,"ĠMin":3386,"Ġopt":3387,"ĉbreak":3388,"umer":3389,"KE":3390,"then":3391,"Ġdet":3392,"ĠTest":3393,"ports":3394,"Ġreview":3395,"('/":3396,"move":3397,"Ġswitch":3398,"ERT":3399,"patch":3400,"annot":3401,"ãĤ":3402,"Ġabove":3403,"itive":3404,"Ġquestion":3405,"ĠQu":3406,"ãĢĤĊĊ":3407,"gle":3408,"Ġword":3409,"Ġprovide":3410,"ĠReturn":3411,"Ġresearch":3412,"ão":3413,"ustr":3414,"Ġpublish":3415,"chema":3416,"}}":3417,"ĠCON":3418,"-in":3419,"allback":3420,"Ġcover":3421,"\\\\":3422,"color":3423,"ĠIS":3424,"Ġwhether":3425,"imate":3426,"isc":3427,"Bar":3428,"Ġdiv":3429,"Be":3430,"ourn":3431,"Ġhaving":3432,"lem":3433,"player":3434,"abs":3435,"amera":3436,"ney":3437,"Ġexc":3438,"gether":3439,"plied":3440,"ao":3441,"[$":3442,"Ġ++":3443,"ipe":3444,"show":3445,"/d":3446,"[:":3447,"agement":3448,"lev":3449,"_ID":3450,"rary":3451,"ades":3452,"_se":3453,"ause":3454,"Ġemploy":3455,"Ġ*/čĊ":3456,"Ġfre":3457,"Ġ'@":3458,"Ġcomplet":3459,"Ġlarge":3460,"ral":3461,"\\x":3462,"Ġfac":3463,">":3578,"Ġface":3579,"CTION":3580,"Ġsave":3581,"Ġtyp":3582,"dev":3583,"(\"#":3584,"AGE":3585,"container":3586,"edit":3587,"QL":3588,"Ġitems":3589,"Ġsocial":3590,"ien":3591,"ĠReact":3592,").ĊĊ":3593,"Ġmar":3594,"Ġredu":3595,"ĠRE":3596,".put":3597,"Ġmajor":3598,"Cell":3599,"next":3600,"Ġexpected":3601,"Ġyet":3602,"Ġindiv":3603,"tributes":3604,"atis":3605,"amed":3606,"Ġfood":3607,"Source":3608,"(string":3609,"Ġ+Ċ":3610,"ites":3611,"dr":3612,"Ġmembers":3613,"Ġcomb":3614,"items":3615,"ĠPer":3616,"TH":3617,"=True":3618,"Ġbar":3619,"_SE":3620,"comm":3621,"(w":3622,")ĊĊĊ":3623,"Ġsend":3624,"Ġinc":3625,"unsigned":3626,"FA":3627,"Ġparams":3628,"apping":3629,"ros":3630,"ugin":3631,"fa":3632,"Ġconnection":3633,"Ġ};ĊĊ":3634,"Ġbecome":3635,"Mode":3636,"Ġev":3637,"Ġdiff":3638,"ĠUnited":3639,"Height":3640,"fully":3641,"images":3642,"Ġmakes":3643,"Ġglobal":3644,"Ġcontact":3645,"':Ċ":3646,"Ġabs":3647,"аÐ":3648,"float":3649,"Ġexcept":3650,"ĠPol":3651,"Child":3652,"typ":3653,"Ġcertain":3654,"ión":3655,"OUT":3656,"Ġimpro":3657,"iles":3658,"Ġ-->Ċ":3659,"ĠPart":3660,"values":3661,"oss":3662,"/**":3663,"ilit":3664,"ĠEvent":3665,"curity":3666,"ster":3667,"Ġcharacter":3668,"Ġnews":3669,"Ġ\",":3670,"Ġdevice":3671,"cel":3672,"login":3673,"heet":3674,"Default":3675,"@\"":3676,"ĉĠ":3677,"click":3678,"(value":3679,"ĠAb":3680,"Ġprevious":3681,"ERROR":3682,"ocal":3683,"Ġmaterial":3684,"Ġbelow":3685,"ĠChrist":3686,"Ġmedia":3687,"cover":3688,"ĠUI":3689,"Ġfail":3690,"Ġblack":3691,"Ġcomponent":3692,"ĠAmerican":3693,"Ġadded":3694,"Ġbuy":3695,"stit":3696,"Ġcame":3697,"Ġdelete":3698,"property":3699,"oding":3700,"Ġcard":3701,"rops":3702,"Ġhttps":3703,"Ġroot":3704,"Ġhandle":3705,"CC":3706,"Back":3707,"emplate":3708,"Ġgetting":3709,"_by":3710,"mail":3711,"_sh":3712,".assert":3713,"ĠDec":3714,"(true":3715,"Ġcomput":3716,"Ġclaim":3717,"'=>":3718,"ĠSub":3719,"Ġair":3720,"ops":3721,"nav":3722,"ements":3723,"(id":3724,"Ġenter":3725,"anged":3726,"End":3727,"Ġlocation":3728,"Ġnight":3729,"Ġdoing":3730,"ĠRed":3731,"lin":3732,"}ĊĊĊ":3733,"vider":3734,"Ġpick":3735,"Ġwatch":3736,"essages":3737,"Ġhuman":3738,"Ġdam":3739,"pend":3740,"dir":3741,"Ġtax":3742,"Ġgirl":3743,"reet":3744,"Ġbox":3745,"Ġstrong":3746,"(v":3747,"rel":3748,"Ġinterface":3749,"Ġmsg":3750,"fect":3751,"_at":3752,"Ġhouse":3753,"Ġtrack":3754,"');ĊĊ":3755,"je":3756,"ĠJohn":3757,"istr":3758,"(S":3759,"ube":3760,"Ġce":3761,"itted":3762,"VER":3763,"*)":3764,"parent":3765,"Ġapplication":3766,"any":3767,".swing":3768,"Ġpack":3769,"\\u":3770,"Ġpract":3771,"Ġsection":3772,"ctx":3773,"Ġunsigned":3774,".Point":3775,"ĠOne":3776,"ı":3777,"iple":3778,"aid":3779,"Ñĥ":3780,"Vector":3781,"byte":3782,"Ġwait":3783,"ĠÃł":3784,"Ã¥":3785,"Ġtogether":3786,"Ġthrows":3787,"FO":3788,"'))":3789,"host":3790,"ising":3791,".view":3792,"Ġterms":3793,"framework":3794,"-r":3795,"Ġapply":3796,"Ġsession":3797,"Options":3798,"uggest":3799,"Ġothers":3800,"witter":3801,"Ġfund":3802,"Init":3803,"__(":3804,"ensor":3805,"GET":3806,"Ġseveral":3807,"ii":3808,"[j":3809,"IO":3810,"Ġtemplate":3811,"Position":3812,"Ġecon":3813,"achine":3814,"Ġil":3815,".spring":3816,"main":3817,"elt":3818,"iment":3819,"Rec":3820,"mm":3821,"ĠUniversity":3822,"ursor":3823,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":3824,"GL":3825,"icture":3826,"ithub":3827,"cer":3828,"cast":3829,"From":3830,"ales":3831,"Ġsubject":3832,"password":3833,"ny":3834,"Ġesc":3835,".write":3836,"ï¼Į":3837,"What":3838,".H":3839,"Ġhistory":3840,"ĠFe":3841,"Ġindividual":3842,"unit":3843,"Ġ-->":3844,"Ġdu":3845,"IST":3846,"Ġusers":3847,"fs":3848,"false":3849,"unt":3850,"Title":3851,"Ġmot":3852,"Ġfuture":3853,"ached":3854,"Ġstarted":3855,"Ġmode":3856,"Ġ'<":3857,"_array":3858,"Ġax":3859,"'];Ċ":3860,"ires":3861,"There":3862,"ught":3863,"tml":3864,"posed":3865,"icult":3866,"Ġtook":3867,"Ġgames":3868,"Ġ}}":3869,"Ġ?>Ċ":3870,"Ġproducts":3871,"Is":3872,"Ġbad":3873,"ĠDes":3874,".path":3875,"'ĊĊ":3876,"ĠPost":3877,"avel":3878,"(:":3879,"Ġneeds":3880,"Ġknown":3881,"Fl":3882,"Ġexec":3883,"Ġseen":3884,"ume":3885,"Ġborder":3886,"Ġlive":3887,"temp":3888,"Per":3889,"Ġvariable":3890,"iet":3891,"ĠDef":3892,"Ġge":3893,"eme":3894,"_back":3895,"first":3896,"Ġprovided":3897,"////////////////////////////////":3898,"Ġfilename":3899,"Ġhope":3900,"uly":3901,"auto":3902,"find":3903,"_string":3904,"btn":3905,"itude":3906,"Attribute":3907,"Ġyoung":3908,".txt":3909,"Ġwebsite":3910,"ĠProp":3911,"Ġey":3912,">();Ċ":3913,"ional":3914,"ARR":3915,"ictionary":3916,"urther":3917,".":3997,"tx":3998,"Ġpur":3999,"uel":4000,"ymbol":4001,"uation":4002,"anger":4003,"Ġbackground":4004,"ecess":4005,"efined":4006,"........":4007,"Ġdescription":4008,"Ġrepresent":4009,"\"));Ċ":4010,"pression":4011,"rowser":4012,"Ġseries":4013,"wards":4014,"($_":4015,"aise":4016,"Ġhot":4017,"acity":4018,"ries":4019,"actions":4020,"Create":4021,"adio":4022,"amples":4023,"Ġoriginal":4024,"ensive":4025,"font":4026,"stream":4027,"using":4028,".springframework":4029,"server":4030,"Ġbill":4031,"ACK":4032,"ilename":4033,"Ġframe":4034,"Ġ=Ċ":4035,"Edit":4036,"adius":4037,"Ġdraw":4038,"anks":4039,"Ġdeter":4040,"Ġcomes":4041,"_int":4042,"Ġforeach":4043,"angle":4044,"Ġelect":4045,"pected":4046,"Header":4047,"istration":4048,"False":4049,"ĠGame":4050,"Ġfilter":4051,"Activity":4052,"Ġlarg":4053,"inition":4054,"Ġ\"<":4055,"ised":4056,"Ġremove":4057,"ĠTrans":4058,"met":4059,"see":4060,"Format":4061,"Command":4062,"ĠEX":4063,"None":4064,"Ġfront":4065,"ASE":4066,"ĠRec":4067,"oundation":4068,"Ġvo":4069,"=\\\"":4070,"(*":4071,"Change":4072,".Write":4073,"group":4074,"ients":4075,"uy":4076,"****************************************************************":4077,"Ġdig":4078,"hr":4079,"(-":4080,"Ġgen":4081,"number":4082,"vec":4083,"urope":4084,"entry":4085,"LL":4086,"Ġste":4087,"Valid":4088,"'],":4089,"_param":4090,"Ġselected":4091,"Ġaccording":4092,"ĠDis":4093,"Ġutil":4094,"Buffer":4095,"_error":4096,"Ġassoci":4097,"_SIZE":4098,"Ġwor":4099,"Ġprintf":4100,"rag":4101,"Âł":4102,"DD":4103,"ĠVal":4104,"Ġactiv":4105,"Eng":4106,"etime":4107,"Ġvirtual":4108,"aign":4109,"aur":4110,"ĠPres":4111,"ĠException":4112,"Ġanything":4113,"ĠOff":4114,"Ġhours":4115,"Ġwar":4116,"Args":4117,"aging":4118,"Ġmodels":4119,"ĠTime":4120,"Ob":4121,"ams":4122,"joy":4123,"Ġearly":4124,".read":4125,"Ġcenter":4126,"ĠInitial":4127,"Ġlanguage":4128,"length":4129,"xy":4130,"Ġsn":4131,"Ġinf":4132,"Post":4133,"Ġago":4134,"Ġeasy":4135,"_code":4136,"ĠANY":4137,"_ch":4138,"Ġdownload":4139,"(T":4140,"aved":4141,"âĢĵ":4142,"Ġstudents":4143,"Ġfig":4144,"light":4145,"xx":4146,"Ġbuffer":4147,"ĠDep":4148,"ĠMath":4149,"ITH":4150,"Ġvari":4151,"Ġdue":4152,"Factory":4153,"Ġpor":4154,"Ġep":4155,"otype":4156,"Ġcannot":4157,"Ġwhite":4158,"čĊ":4424,".annot":4425,"Ġcollection":4426,"'.":4427,"Ġsimilar":4428,"Ġtaken":4429,"(\"%":4430,"Order":4431,"']Ċ":4432,"-md":4433,"ĠTH":4434,"aced":4435,"Ġisn":4436,"/j":4437,"Ġson":4438,"graph":4439,"ĠInteger":4440,"Ġnecess":4441,"reen":4442,"Ġum":4443,"Ġ\\<":4444,"Ġmoment":4445,"Ġbring":4446,"Ġindic":4447,"ysis":4448,"Level":4449,"verse":4450,"urrenc":4451,"_test":4452,"Ġentire":4453,"Down":4454,"Ġ}ĊĊĊ":4455,"(result":4456,"ĠRead":4457,"è":4458,"Mod":4459,"Ġtrying":4460,"\"),Ċ":4461,"Ġmember":4462,"ĠCor":4463,"ODO":4464,"-control":4465,"untime":4466,"ĠSim":4467,"Dialog":4468,"plot":4469,"_on":4470,"Ġphys":4471,"}/":4472,"Ġnamespace":4473,"ĉčĊ":4474,"acc":4475,"Player":4476,"ARE":4477,"Ġfoot":4478,"Ġboard":4479,"part":4480,"Ġsus":4481,"wise":4482,"ĠMc":4483,"Ġpush":4484,"ATA":4485,"Ġplease":4486,"ried":4487,"weet":4488,"bit":4489,"ided":4490,"VE":4491,"ĠSw":4492,"UB":4493,"Ġtypes":4494,"edia":4495,"Ġclos":4496,"acebook":4497,"When":4498,"Ġedit":4499,"igger":4500,"Ġenerg":4501,"Container":4502,"Ġphot":4503,"ĠCount":4504,"ĠEurope":4505,".Is":4506,"ĠRuss":4507,"peed":4508,"ĠStr":4509,"Ġpy":4510,"Ġcult":4511,"Ġdefined":4512,"ccount":4513,"Ġobt":4514,".Location":4515,"Ġthread":4516,"ille":4517,"Ġinstead":4518,"strong":4519,"ĠSec":4520,"URE":4521,"Ġidea":4522,".se":4523,"emy":4524,"selected":4525,"Connection":4526,"acing":4527,"thread":4528,".next":4529,"Ġcoll":4530,"Ġfilm":4531,"istic":4532,"Ġcompet":4533,"Ġconn":4534,"though":4535,"Ġcompan":4536,"ocket":4537,"Ġteach":4538,"=(":4539,"Ġphone":4540,"Ġactive":4541,"delete":4542,"tries":4543,"Ġmo":4544,"Ġdeath":4545,"});ĊĊ":4546,"ocol":4547,"Widget":4548,"Ġarticle":4549,"rodu":4550,"andid":4551,"Ñĭ":4552,"ĠCr":4553,"ka":4554,"():":4555,"lood":4556,"ĉĉĉĊ":4557,"Ġalmost":4558,"Ġsell":4559,"ervlet":4560,"rip":4561,"Unit":4562,"Ġapplic":4563,"Ġconnect":4564,"Ġfeature":4565,"Ġvia":4566,"'),":4567,"Ġlim":4568,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":4569,"ĠGu":4570,"Engine":4571,"Ġens":4572,"Ġenvironment":4573,"block":4574,"HERE":4575,"NULL":4576,"gy":4577,"tag":4578,")).":4579,"exp":4580,"Ġcompl":4581,"Ġinstall":4582,"Ġcomplete":4583,"queue":4584,"atural":4585,"Ġgeneral":4586,"thon":4587,"Ġasked":4588,"ores":4589,"(res":4590,"Ġreserved":4591,"SP":4592,"ĠâĢ¦":4593,"ÅĤ":4594,"Ġsignific":4595,"Off":4596,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":4597,"ĠAg":4598,"ĠJust":4599,"ĠError":4600,"Ġinfl":4601,"adata":4602,"Ġicon":4603,"asks":4604,"''":4605,"_LO":4606,"?.":4607,"account":4608,"Ġ(*":4609,"')ĊĊ":4610,"rap":4611,"_var":4612,"ĠFOR":4613,"Ġparty":4614,"ĠYour":4615,"cat":4616,"stry":4617,".new":4618,"boot":4619,"ĠNov":4620,"Ġvector":4621,"Ġnormal":4622,"Ġfurther":4623,"Repository":4624,"Ġdatabase":4625,"attle":4626,"Ġmusic":4627,"Ġspeed":4628,"Ġdoc":4629,"process":4630,"IGHT":4631,".parse":4632,"Ġtaking":4633,"Ġviol":4634,"ceed":4635,"ĠAfter":4636,"Ġforward":4637,"Ġcrit":4638,"\"/>Ċ":4639,"rot":4640,"Ġfailed":4641,"efore":4642,"Ġconcern":4643,"oe":4644,"ba":4645,"Ġsender":4646,"Ġterm":4647,"has":4648,"=\"#":4649,"Ġpotential":4650,"Num":4651,"Ġpublished":4652,".close":4653,"ĠImage":4654,"straint":4655,"UD":4656,"ĠOb":4657,"Ġprobably":4658,"lim":4659,"\":Ċ":4660,"olume":4661,"Ġconsum":4662,"ague":4663,"ensions":4664,"Ġinvestig":4665,"-year":4666,"');":4667,"-sm":4668,"Ġenjoy":4669,"orig":4670,"ering":4671,"cp":4672,"leased":4673,"plements":4674,"Ġreturns":4675,"pat":4676,"BO":4677,"ĠHouse":4678,".Label":4679,"Ġweight":4680,"ighb":4681,"Ġconditions":4682,"Ġexception":4683,"description":4684,"Ġtrad":4685,"-to":4686,"Ġ{}":4687,"Ġmodule":4688,"END":4689,".ap":4690,".props":4691,"Ġconstructor":4692,"aves":4693,"Ġfavor":4694,"ĠNow":4695,";i":4696,"ĠMain":4697,"_k":4698,"eries":4699,"âĢĻll":4700,"transform":4701,"imestamp":4702,"Pre":4703,"Ġmer":4704,".res":4705,"stant":4706,"Location":4707,"_NAME":4708,"Ġloss":4709,"ĠĊĊ":4710,"net":4711,"Ġengine":4712,"Block":4713,"Ġissues":4714,"Ġparse":4715,"ĠBar":4716,"Ġstay":4717,"ĠJSON":4718,"Ġdom":4719,"airs":4720,"wner":4721,"Ġlower":4722,"\",čĊ":4723,"ĠDem":4724,"ufact":4725,"Ġps":4726,"Ġperfect":4727,"RL":4728,"Ġeduc":4729,"ls":4730,"emory":4731,"ARRANT":4732,"uge":4733,"Ġexact":4734,".key":4735,"alled":4736,"ech":4737,"ief":4738,"\\/":4739,"oke":4740,"Ġformer":4741,"alloc":4742,"Ġsix":4743,"ida":4744,"Ġmargin":4745,"Ġheart":4746,"ald":4747,"pack":4748,".getElementById":4749,"ĠWARRANT":4750,"Ġrather":4751,"Ġbuilding":4752,"erman":4753,"lice":4754,"Ġquestions":4755,"izes":4756,"lege":4757,"irectory":4758,"Ġje":4759,"Ġcas":4760,"props":4761,"utf":4762,"Ġsecurity":4763,"Ġhowever":4764,"weight":4765,"Ġinside":4766,"Ġpresident":4767,"Char":4768,"ĠWITH":4769,".map":4770,"Ġgraph":4771,"Ġtag":4772,"_status":4773,"Ġattempt":4774,"opp":4775,"uses":4776,"ĉconst":4777,"Ġround":4778,",$":4779,"Ġfriends":4780,"Email":4781,"?>":4782,"Resource":4783,"KEY":4784,"osp":4785,".query":4786,"ĠNorth":4787,"ables":4788,"istrib":4789,"_class":4790,"ello":4791,"That":4792,"к":4793,"pecially":4794,"ĠPresident":4795,"Ġcampaign":4796,"Ġalt":4797,"area":4798,"Ġchall":4799,"Ġopport":4800,".Con":4801,"Ġenergy":4802,"like":4803,".string":4804,"ington":4805,")*":4806,"yy":4807,"Ġprofession":4808,"irth":4809,"Ġseg":4810,"æľ":4811,"Ġhor":4812,"iers":4813,"can":4814,"Ġbehind":4815,"Product":4816,"fg":4817,"ĠSk":4818,".jpg":4819,"?:":4820,"];ĊĊ":4821,"Ġcallback":4822,"ĠHttp":4823,"ÑĮ":4824,"long":4825,"MS":4826,"ATH":4827,"Ġraise":4828,"Ġwanted":4829,"rown":4830,"utor":4831,"lt":4832,"]=":4833,"eline":4834,"MA":4835,"Ġsepar":4836,"cs":4837,"semb":4838,"Dis":4839,"bserv":4840,"ĠWill":4841,"Ġpolicy":4842,"Ġthird":4843,"phone":4844,"Ġbed":4845,"/g":4846,".__":4847,"ĠInc":4848,"izing":4849,".remove":4850,"instance":4851,".type":4852,"Ġserv":4853,"Each":4854,"Ġhar":4855,"ĠMessage":4856,"(key":4857,"SELECT":4858,"Pos":4859,"));čĊ":4860,"Ġrecomm":4861,"Ġtraining":4862,"ĠEnt":4863,"ĠChar":4864,"icht":4865,"(file":4866,"Ġprior":4867,"Game":4868,"Ġexit":4869,"Params":4870,".core":4871,"PC":4872,"nes":4873,"anced":4874,"(request":4875,"Password":4876,"}>Ċ":4877,"Ġmag":4878,"Ġrelease":4879,"Ġshall":4880,"udent":4881,"ĠSouth":4882,"ando":4883,":'":4884,".TabIndex":4885,"sk":4886,"anner":4887,"isset":4888,"Ġoutside":4889,"ledge":4890,"Ġå":4891,"ĠRob":4892,"Ġimm":4893,"!Ċ":4894,"ĠWeb":4895,"Des":4896,"BC":4897,"ancial":4898,"Route":4899,"Dec":4900,"ferences":4901,"Ġpurch":4902,"ĠModel":4903,"ctor":4904,"gn":4905,"_start":4906,"_un":4907,".*":4908,"ises":4909,"Ġground":4910,"Ġunique":4911,"Ġbeaut":4912,"{\"":4913,"Ġpour":4914,"ĠOct":4915,"Ġtree":4916,"sets":4917,"_res":4918,"')->":4919,"_reg":4920,"(\"\\":4921,"Ġbyte":4922,"Bl":4923,"Ġdating":4924,"Ġmatter":4925,"ĠRem":4926,"Ġ'../":4927,"ĠAug":4928,"ĠLa":4929,"Ġ$(":4930,"ournal":4931,"iam":4932,"Ġshows":4933,"write":4934,"Ġball":4935,"Ġsimply":4936,"Ġfast":4937,"Ġmemory":4938,"ASS":4939,"ĠOf":4940,"oved":4941,"ante":4942,"aul":4943,"istry":4944,")));Ċ":4945,"Ġfit":4946,"_":5129,"\")ĊĊ":5130,"ox":5131,"application":5132,"Ġ]Ċ":5133,"ĊĊĊĊĊĊ":5134,"Ġsoon":5135,"ctions":5136,"inger":5137,"Ġjoin":5138,"ĠPe":5139,"Ġë":5140,"Ġlas":5141,".E":5142,"css":5143,"/or":5144,"ĠStart":5145,"ĠTO":5146,"Ġsubs":5147,"conn":5148,"components":5149,"DEBUG":5150,"quare":5151,"Function":5152,"endar":5153,".index":5154,"Ġfill":5155,"ÄĻ":5156,"Ġchoose":5157,"how":5158,"ĠAmerica":5159,"assets":5160,"------------":5161,"ĠValue":5162,"Ġoffice":5163,"Ġveh":5164,"Ġtransform":5165,"ĠArt":5166,"Ġinde":5167,"Ġfn":5168,"Ġimplements":5169,"ango":5170,"plete":5171,"+\"":5172,"tmp":5173,"amily":5174,"Ġhash":5175,"missions":5176,"EST":5177,"gt":5178,"Provider":5179,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":5180,"Ġflag":5181,"Ġparticip":5182,"den":5183,"ĠReturns":5184,"Ġnote":5185,"ür":5186,"pm":5187,"ideos":5188,"Ġspecified":5189,"ĠEN":5190,"ester":5191,"olid":5192,"Ġupon":5193,"(std":5194,"ĉv":5195,"Ġ'\\":5196,"uz":5197,"Ġvert":5198,"Ġvict":5199,"ĉself":5200,"Ġ\"$":5201,".k":5202,"Ġgroups":5203,"github":5204,"lang":5205,"Ġmut":5206,"TO":5207,"Ġve":5208,"ĠPlease":5209,";ĊĊĊ":5210,"access":5211,"Ġ{\"":5212,"rea":5213,"Ġrisk":5214,"icker":5215,"oggle":5216,"ĉwhile":5217,"ANG":5218,".send":5219,"Ġwoman":5220,"Ġgets":5221,"Ġign":5222,"ĠId":5223,"_log":5224,"ONE":5225,"Ġevid":5226,"ĠHar":5227,"_sub":5228,"Ġendl":5229,"Ġincluded":5230,"());ĊĊ":5231,"ĠAp":5232,"igr":5233,"Ġsem":5234,"ĠBlack":5235,"doc":5236,"_table":5237,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":5238,"-up":5239,"Ġcause":5240,"Ġ..":5241,"Ġvan":5242,"_dict":5243,"Ġfocus":5244,"IND":5245,"CESS":5246,".Log":5247,"Ġmultiple":5248,"ido":5249,"Ġregard":5250,"-M":5251,"andler":5252,"ourse":5253,"Ġdeg":5254,".U":5255,"Ġaddition":5256,"Ġvarious":5257,"Ġreceive":5258,"ен":5259,"ĠHT":5260,"Obj":5261,"DF":5262,"Ġincrease":5263,"ĠOpen":5264,"];":5265,"Ġcommit":5266,"?Ċ":5267,"ategories":5268,"atory":5269,"ship":5270,"ĠMich":5271,"Ġhtml":5272,"romise":5273,"Ġleave":5274,"Ġstrateg":5275,"aven":5276,"ĠConsole":5277,"known":5278,"-n":5279,"_LE":5280,".component":5281,"Ġbre":5282,"Session":5283,"iance":5284,"Ġalign":5285,"typedef":5286,"_result":5287,"ĠWHERE":5288,".split":5289,"Ġreading":5290,"FAULT":5291,"Ġclo":5292,"Ġnotice":5293,"_pr":5294,"arter":5295,"Ġlock":5296,"Ġstandard":5297,"etic":5298,"ellow":5299,"Ġpadding":5300,"ĠHis":5301,"Ġstates":5302,"_cast":5303,"(P":5304,"aa":5305,"Ġinternal":5306,"ean":5307,"ĠPRO":5308,"ĠKey":5309,"Ġespecially":5310,"ming":5311,"Ġcross":5312,"Ġnational":5313,"_object":5314,"filter":5315,"Ġscript":5316,".update":5317,"_i":5318,"ĠAssert":5319,"/core":5320,"%%%%":5321,"Ġproblems":5322,"istor":5323,"Ġ.=":5324,"Ġarch":5325,"Ġwritten":5326,"Ġmilit":5327,"MENT":5328,".ch":5329,"cape":5330,"ĠMus":5331,"_config":5332,"ĠAPI":5333,"foot":5334,"Ġimages":5335,"endl":5336,".In":5337,"First":5338,"Ġplatform":5339,".prot":5340,"Option":5341,"ste":5342,"ĠTODO":5343,"Ġforce":5344,".cont":5345,"ĉecho":5346,"ĠDav":5347,"Ptr":5348,"(B":5349,"RT":5350,"ĠBase":5351,"]['":5352,"Ġannounc":5353,"console":5354,"ĠPy":5355,"ds":5356,".as":5357,"Ġprevent":5358,"apan":5359,"Ġ{'":5360,"}'":5592,"Ġdead":5593,"VAL":5594,"QUE":5595,"************************************************************************":5596,"Ġcharg":5597,"Return":5598,"Ġful":5599,"dom":5600,"Ġrules":5601,"Ġmodify":5602,"Ġeval":5603,"ham":5604,"atement":5605,"\\<":5606,"ula":5607,"=False":5608,"RA":5609,"Ġcontains":5610,"Ġstack":5611,"mar":5612,"Ġ{}Ċ":5613,"Ġundefined":5614,"Ass":5615,"ĠChina":5616,"vey":5617,"*Ċ":5618,"Ġplaying":5619,")/":5620,"actor":5621,"Ġbottom":5622,"lier":5623,"ĠNumber":5624,"Ġcouple":5625,"DC":5626,"ĠSO":5627,"gor":5628,".setText":5629,"success":5630,"command":5631,"Filter":5632,"ĠOur":5633,"_item":5634,"Ġctx":5635,"Ġroad":5636,"Version":5637,"case":5638,"urt":5639,"avior":5640,"ych":5641,"sembly":5642,"ĠProduct":5643,"Ġheld":5644,"afe":5645,"Ġincludes":5646,"&":5789,"CON":5790,"Ġrepl":5791,"Ġregular":5792,"Storage":5793,"ramework":5794,"Ġgoal":5795,"Ġtouch":5796,".widget":5797,"Ġbuilt":5798,"des":5799,"Part":5800,"(re":5801,"Ġworth":5802,"hib":5803,"game":5804,"Ġв":5805,"acion":5806,"ĠWhite":5807,"(type":5808,"(`":5809,"Ġnatural":5810,"Ġinj":5811,"Ġcalcul":5812,"ĠApril":5813,".List":5814,"Ġassociated":5815,"ĉSystem":5816,"~~":5817,"=[":5818,"Ġstorage":5819,"Ġbytes":5820,"Ġtravel":5821,"Ġsou":5822,"Ġpassed":5823,"!=":5824,"ascript":5825,".open":5826,"Ġgrid":5827,"Ġbus":5828,"Ġrecogn":5829,"Ab":5830,"Ġhon":5831,"ĠCenter":5832,"Ġprec":5833,"build":5834,"HTML":5835,"ĠSan":5836,"Ġcountries":5837,"aled":5838,"token":5839,"kt":5840,"Ġqual":5841,"Last":5842,"adow":5843,"Ġmanufact":5844,"idad":5845,"jango":5846,"Next":5847,"xf":5848,".a":5849,"Ġporno":5850,"ĠPM":5851,"erve":5852,"iting":5853,"_th":5854,"ci":5855,"=None":5856,"gs":5857,"Ġlogin":5858,"atives":5859,"']);Ċ":5860,"Äħ":5861,"Ġill":5862,"IA":5863,"children":5864,"DO":5865,"Ġlevels":5866,"Ġ{{":5867,"Ġlooks":5868,"Ġ\"#":5869,"ToString":5870,"Ġnecessary":5871,"ĠĠĠĊ":5872,"cell":5873,"Entry":5874,"Ġ'#":5875,"Ġextrem":5876,"Selector":5877,"Ġplaceholder":5878,"Load":5879,"Ġreleased":5880,"ORE":5881,"Enumer":5882,"ĠTV":5883,"SET":5884,"inq":5885,"Press":5886,"ĠDepartment":5887,"Ġproperties":5888,"Ġrespond":5889,"Search":5890,"ael":5891,"Ġrequ":5892,"ĠBook":5893,"/Ċ":5894,"(st":5895,"Ġfinancial":5896,"icket":5897,"_input":5898,"Ġthreat":5899,"(in":5900,"Strip":5901,"ìĿ":5902,"ção":5903,"Ġevidence":5904,"));":5905,"ĠBro":5906,"Ġ[];Ċ":5907,"Ġou":5908,"buf":5909,"Script":5910,"dat":5911,"Ġrule":5912,"#import":5913,"=\"/":5914,"Serial":5915,"Ġstarting":5916,"[index":5917,"ae":5918,"Ġcontrib":5919,"session":5920,"_new":5921,"utable":5922,"ober":5923,"Ġ\"./":5924,"Ġlogger":5925,"Ġrecently":5926,"Ġreturned":5927,"ččĊ":5928,")))Ċ":5929,"itions":5930,"Ġseek":5931,"Ġcommunic":5932,"Ġ\".":5933,"Ġusername":5934,"ECT":5935,"DS":5936,"Ġotherwise":5937,"ĠGerman":5938,".aw":5939,"Adapter":5940,"ixel":5941,"Ġsystems":5942,"Ġdrop":5943,"Ġstructure":5944,"Ġ$(\"#":5945,"encies":5946,"anning":5947,"ĠLink":5948,"ĠResponse":5949,"Ġstri":5950,"ż":5951,"ĠDB":5952,"æĹ":5953,"android":5954,"submit":5955,"otion":5956,"(@":5957,".test":5958,"ĊĊĊĊĊĊĊĊ":5959,"];čĊ":5960,"Ġdirectly":5961,"Ġ\"%":5962,"ris":5963,"elta":5964,"AIL":5965,"){čĊ":5966,"mine":5967,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":5968,"(k":5969,"bon":5970,"asic":5971,"pite":5972,"___":5973,"Max":5974,"Ġerrors":5975,"ĠWhile":5976,"Ġarguments":5977,"Ġensure":5978,"Right":5979,"-based":5980,"Web":5981,"Ġ-=":5982,"Ġintrodu":5983,"ĠInst":5984,"ĠWash":5985,"ordin":5986,"join":5987,"Database":5988,"Ġgrad":5989,"Ġusually":5990,"ITE":5991,"Props":5992,"?>Ċ":5993,"ĠGo":5994,"@Override":5995,"REF":5996,"Ġip":5997,"ĠAustral":5998,"Ġist":5999,"ViewById":6000,"Ġserious":6001,"Ġcustomer":6002,".prototype":6003,"odo":6004,"cor":6005,"Ġdoor":6006,"ĠWITHOUT":6007,"Ġplant":6008,"Ġbegan":6009,"Ġdistance":6010,"()).":6011,"Ġchance":6012,"Ġord":6013,"came":6014,"pragma":6015,"Ġprotect":6016,"ragment":6017,"ĠNode":6018,"ening":6019,"Ñĩ":6020,"Ġroute":6021,"ĠSchool":6022,"hi":6023,"Ġneighb":6024,"After":6025,"licit":6026,"Ġcontr":6027,"Ġprimary":6028,"AA":6029,".WriteLine":6030,"utils":6031,"Ġbi":6032,"Red":6033,".Linq":6034,".object":6035,"Ġleaders":6036,"unities":6037,"Ġgun":6038,"onth":6039,"ĠDev":6040,"FILE":6041,"Ġcomments":6042,"_len":6043,"arrow":6044,"amount":6045,"Range":6046,"sert":6047,"GridView":6048,"Ġupdated":6049,"ĠMo":6050,"Ġinform":6051,"ociety":6052,"ala":6053,"Access":6054,"Ġhab":6055,"Ġcreat":6056,"_arg":6057,"ĠJanuary":6058,"ĠDay":6059,"\")čĊ":6060,"uple":6061,"document":6062,"gorith":6063,"menu":6064,"ĠOver":6065,"bb":6066,".title":6067,"_out":6068,"Ġled":6069,"uri":6070,"Ġ?>Ċ":6107,"run":6108,"Ġscene":6109,"(array":6110,"device":6111,"_title":6112,"agon":6113,"]čĊ":6114,"aby":6115,"Ġbecame":6116,"boolean":6117,"Ġpark":6118,"ĠCode":6119,"upload":6120,"riday":6121,"ĠSeptember":6122,"Fe":6123,"Ġsen":6124,"cing":6125,"FL":6126,"Col":6127,"uts":6128,"_page":6129,"inn":6130,"Ġimplied":6131,"aling":6132,"Ġyourself":6133,".Count":6134,"conf":6135,"Ġaud":6136,"_init":6137,".)":6138,"Ġwrote":6139,"NG":6140,".Error":6141,"ä»":6142,".for":6143,"Ġequal":6144,"ĠRequest":6145,"Ġserial":6146,"Ġallows":6147,"XX":6148,"Ġmiddle":6149,"chor":6150,"ø":6151,"erval":6152,".Column":6153,"reading":6154,"Ġescort":6155,"ĠAugust":6156,"Ġquickly":6157,"Ġweap":6158,"ĠCG":6159,"ropri":6160,"ho":6161,"Ġcop":6162,"(struct":6163,"ĠBig":6164,"Ġvs":6165,"Ġfrequ":6166,".Value":6167,"Ġactions":6168,"Ġproper":6169,"Ġinn":6170,"Ġobjects":6171,"Ġmatrix":6172,"avascript":6173,"Ġones":6174,".group":6175,"Ġgreen":6176,"Ġpaint":6177,"ools":6178,"ycl":6179,"encode":6180,"olt":6181,"comment":6182,".api":6183,"Dir":6184,"Ġune":6185,"izont":6186,".position":6187,"Ġdesigned":6188,"_val":6189,"avi":6190,"iring":6191,"tab":6192,"Ġlayer":6193,"Ġviews":6194,"Ġreve":6195,"rael":6196,"ĠON":6197,"rics":6198,"np":6199,"Ġcore":6200,"());čĊ":6201,"Main":6202,"Ġexpert":6203,"ĉĉčĊ":6204,"_en":6205,"Ġ/>":6206,"utter":6207,"IAL":6208,"ails":6209,"ĠKing":6210,"*/ĊĊ":6211,"ĠMet":6212,"_end":6213,"addr":6214,"ora":6215,"Ġir":6216,"Min":6217,"Ġsurpr":6218,"Ġrepe":6219,"Ġdirectory":6220,"PUT":6221,"-S":6222,"Ġelection":6223,"haps":6224,".pre":6225,"cm":6226,"Values":6227,"Ġ\"Ċ":6228,"column":6229,"ivil":6230,"Login":6231,"inue":6232,"Ġbeautiful":6233,"Ġsecret":6234,"(event":6235,"Ġchat":6236,"ums":6237,"Ġorigin":6238,"Ġeffects":6239,"Ġmanagement":6240,"illa":6241,"tk":6242,"Ġsetting":6243,"ĠCour":6244,"Ġmassage":6245,"ĉend":6246,"Ġhappy":6247,"Ġfinish":6248,"Ġcamera":6249,"ĠVer":6250,"ĠDemocr":6251,"ĠHer":6252,"(Q":6253,"cons":6254,"ita":6255,"Ġ'.":6256,"{}":6257,"ĉC":6258,"Ġstuff":6259,"Ġ:Ċ":6260,"ĠAR":6261,"Task":6262,"hidden":6263,"eros":6264,"IGN":6265,"atio":6266,"ĠHealth":6267,"olute":6268,"Enter":6269,"'>":6270,"ĠTwitter":6271,"ĠCounty":6272,"scribe":6273,"Ġ=>Ċ":6274,"Ġhy":6275,"fit":6276,"Ġmilitary":6277,"Ġsale":6278,"required":6279,"non":6280,"bootstrap":6281,"hold":6282,"rim":6283,"-old":6284,"ĠDown":6285,"Ġmention":6286,"contact":6287,"_group":6288,"oday":6289,"Ġtown":6290,"Ġsolution":6291,"uate":6292,"elling":6293,"]->":6294,"otes":6295,"ental":6296,"omen":6297,"ospital":6298,"ĠSup":6299,"_EN":6300,"Ġslow":6301,"SESSION":6302,"Ġblue":6303,"ago":6304,"Ġlives":6305,"Ġ^":6306,".un":6307,"inst":6308,"enge":6309,"Ġcustomers":6310,"Ġcast":6311,"udget":6312,"ï¼ģ":6313,"icens":6314,"Ġdetermin":6315,"Selected":6316,"_pl":6317,"ueue":6318,"Ġdark":6319,"//ĊĊ":6320,"si":6321,"thern":6322,"ĠJapan":6323,"/w":6324,"PU":6325,"ĠEast":6326,"ovie":6327,"Ġpackage":6328,"Ġnor":6329,"Ġapi":6330,"bot":6331,"\"];Ċ":6332,"_post":6333,"ulate":6334,"Ġclub":6335,"'));Ċ":6336,"Ġloop":6337,"PIO":6338,"ione":6339,"shot":6340,"Initial":6341,"Ġplayed":6342,"register":6343,"rought":6344,"_max":6345,"acement":6346,"match":6347,"raphics":6348,"AST":6349,"Ġexisting":6350,"Ġcomplex":6351,"DA":6352,".Ch":6353,".common":6354,"mo":6355,"Ġ'../../":6356,"ito":6357,"Ġanalysis":6358,"Ġdeliver":6359,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":6360,"idx":6361,"Ãł":6362,"ongo":6363,"ĠEnglish":6364,"Ċ":9992,"_default":9993,"ĠDatabase":9994,"rep":9995,"ESS":9996,"nergy":9997,".Find":9998,"_mask":9999,"Ġrise":10000,"Ġkernel":10001,"::$":10002,".Q":10003,"Ġoffering":10004,"decl":10005,"ĠCS":10006,"Ġlisted":10007,"Ġmostly":10008,"enger":10009,"Ġblocks":10010,"olo":10011,"Ġgoverning":10012,"\\F":10013,"Ġconcent":10014,".getText":10015,"Ġmb":10016,"Ġoccurred":10017,"Ġchanging":10018,"Scene":10019,"_CODE":10020,"Beh":10021,"\"The":10022,"Ġtile":10023,"ĠAssociation":10024,"ĉP":10025,"alty":10026,"_ad":10027,"odies":10028,"iated":10029,"Ġprepared":10030,"possible":10031,"Ġmort":10032,"TEST":10033,"Ġignore":10034,"Ġcalc":10035,"Ġrs":10036,"ĠassertEquals":10037,"Ġsz":10038,"ĠTHIS":10039,".\"Ċ":10040,"Ġcanvas":10041,"java":10042,"Ġdut":10043,"VALID":10044,".sql":10045,".input":10046,"Ġaux":10047,"Sup":10048,"Ġartist":10049,"Vec":10050,"_TIME":10051,".stringify":10052,"etween":10053,"ĠCategory":10054,"Ġ[-":10055,"ĠDevExpress":10056,"ĠJul":10057,"Ġring":10058,".ed":10059,"YY":10060,"Let":10061,"TextField":10062,"Ġflat":10063,"_print":10064,"ĠOTHER":10065,"adian":10066,"Ġchecked":10067,"ele":10068,"Align":10069,"standing":10070,"Ġ[],":10071,"Ġlab":10072,"ucky":10073,"ĠChristmas":10074,"(image":10075,".module":10076,"Ġlots":10077,"Ġslightly":10078,"(final":10079,"erge":10080,"è¿":10081,"ĠPolice":10082,"ĠRight":10083,"Ġaward":10084,"ĠOS":10085,"Ġ{}ĊĊ":10086,"Ġptr":10087,"oves":10088,"icated":10089,"ем":10090,"Ġmanage":10091,"oliday":10092,"Amount":10093,"oolStrip":10094,"tbody":10095,"Nav":10096,"wrap":10097,"BB":10098,"Ġwatching":10099,"arios":10100,"Ġoptional":10101,"_K":10102,"ĠLicensed":10103,".Map":10104,"Timer":10105,"ĠAP":10106,"ĠRev":10107,"(o":10108,",c":10109,"umin":10110,"etailed":10111,"ĠHy":10112,"Ġblank":10113,"agger":10114,"ĠSelf":10115,"()[":10116,".make":10117,"earn":10118,"channel":10119,";Ċ":10133,"World":10134,"Ġpython":10135,"Ġlif":10136,"Ġtrav":10137,"Ġconven":10138,"company":10139,"ĠClub":10140,"Ver":10141,"Btn":10142,"Ġzone":10143,"products":10144,"ĠEduc":10145,"Ġverify":10146,"ĠMil":10147,"ono":10148,"]);ĊĊ":10149,"ENCE":10150,"Ġpacket":10151,"Ġcer":10152,"Ġenumer":10153,"Ġpars":10154,"formed":10155,"Ġoccup":10156,"tre":10157,"Ġexercise":10158,"Day":10159,"_sum":10160,"Ġasking":10161,"aption":10162,"Ġorders":10163,"Ġspending":10164,"ĠERR":10165,".Dis":10166,"ĠUtil":10167,"âĢľI":10168,"\\'":10169,"?)":10170,"/>Ċ":10171,"Ġemot":10172,"Ġinfluence":10173,"ĠAfrica":10174,"atters":10175,"Ùħ":10176,".session":10177,"Ġchief":10178,"ĉĉĉĉĉĉĉĉĉĉĉ":10179,"Ġtom":10180,"cluded":10181,"serial":10182,"_handler":10183,".Type":10184,"aped":10185,"Ġpolicies":10186,"-ex":10187,"-tr":10188,"blank":10189,"merce":10190,"Ġcoverage":10191,"Ġrc":10192,"_matrix":10193,"_box":10194,"Ġcharges":10195,"ĠBoston":10196,"Pe":10197,"Ġcircum":10198,"Ġfilled":10199,"Ġnorth":10200,"ictureBox":10201,"ĉres":10202,"è®":10203,"Ġtermin":10204,"Ġ[âĢ¦":10205,"IRECT":10206,"Ġber":10207,"Ġ\"../../":10208,"retch":10209,".code":10210,"_col":10211,"ĠGovernment":10212,"Ġargv":10213,"ĠLord":10214,"asi":10215,"Exec":10216,"ĉlet":10217,"vertis":10218,"Ġdiscussion":10219,"enance":10220,"outube":10221,"typeof":10222,"Ġserved":10223,"ĠPut":10224,"ĉx":10225,"Ġsweet":10226,"Before":10227,"ategy":10228,".of":10229,"ĠMaterial":10230,"Sort":10231,"ONT":10232,"igital":10233,"Why":10234,"Ġsust":10235,"Ġç":10236,"abet":10237,"Ġsegment":10238,"Ġ[],Ċ":10239,"ĠMuslim":10240,"ĠfindViewById":10241,"cut":10242,"_TEXT":10243,"ĠMary":10244,"Ġloved":10245,"Ġlie":10246,"ĠJO":10247,"Ġisset":10248,"month":10249,"Ġprime":10250,"ti":10251,"ĠCarol":10252,"Use":10253,"ĠPop":10254,"ĠSave":10255,"Interval":10256,"execute":10257,"dy":10258,"ĠIran":10259,"_cont":10260,"ĉT":10261,"Ġphase":10262,"checkbox":10263,"week":10264,"Ġhide":10265,"Ġtil":10266,"Ġju":10267,"Custom":10268,"burg":10269,"/M":10270,"TON":10271,"Ġquant":10272,"Ġrub":10273,"ixels":10274,"Ġinstalled":10275,"Ġdump":10276,"Ġproperly":10277,"(List":10278,"Ġdecide":10279,"apply":10280,"Has":10281,"Ġkeeping":10282,"Ġcitizens":10283,"Ġjoint":10284,"pool":10285,"Socket":10286,"_op":10287,"Ġweapon":10288,"gnore":10289,"ĠExec":10290,"otten":10291,"ĠMS":10292,"Ġ(-":10293,"ĠReview":10294,"Ġexamples":10295,"Ġtight":10296,"!(":10297,"DP":10298,"ĠMessageBox":10299,"Ġphotograph":10300,"URI":10301,"ét":10302,"low":10303,"ĠGrand":10304,".persistence":10305,"Ġmaintain":10306,"Ġnums":10307,"Ġzip":10308,"ials":10309,"ĠGets":10310,"peg":10311,"ĠBuffer":10312,"~~~~":10313,"rastructure":10314,"ĠPL":10315,"uen":10316,"obby":10317,"sizeof":10318,"Ġpic":10319,"Ġseed":10320,"Ġexperienced":10321,"Ġodd":10322,"Ġkick":10323,"Ġprocedure":10324,"avigator":10325,"-on":10326,",j":10327,"ĠAlthough":10328,"ĠuserId":10329,"accept":10330,"Blue":10331,"IColor":10332,"layer":10333,"available":10334,"Ġends":10335,".table":10336,"Ġdataset":10337,"bus":10338,"Ġexplain":10339,"(pro":10340,"ĠCommittee":10341,"Ġnoted":10342,"]:Ċ":10343,"Dim":10344,"stdio":10345,".\",Ċ":10346,"_source":10347,"ĠWeek":10348,"ĠEdge":10349,"Ġoperating":10350,"Ġeste":10351,"ipl":10352,"agination":10353,"Ġproceed":10354,"Ġanimation":10355,".Models":10356,"ĠWatch":10357,"iat":10358,"Ġoppon":10359,"/A":10360,"Report":10361,"Ġsounds":10362,"_buf":10363,"IELD":10364,"Ġbund":10365,"ĉget":10366,".pr":10367,"(tmp":10368,"Ġkid":10369,">ĊĊĊ":10370,"Ġyang":10371,"NotFound":10372,"ÑĨ":10373,"math":10374,"@gmail":10375,"ĠLIMIT":10376,"redients":10377,"Ġvent":10378,"avigate":10379,"Look":10380,"Ġreligious":10381,"Ġrand":10382,"rio":10383,"(GL":10384,"_ip":10385,"uan":10386,"iciency":10387,"ĠChange":10388,">čĊčĊ":10389,"ĠEntity":10390,"Ġrencontre":10391,"ĠRet":10392,"plan":10393,"én":10394,"BOOL":10395,"uries":10396,"train":10397,"Definition":10398,"============":10399,"zz":10400,"Animation":10401,"ĠOK":10402,"_menu":10403,".bl":10404,"_score":10405,"Ġacad":10406,"(System":10407,"Ġrefresh":10408,"'=>$":10409,".Graphics":10410,"amento":10411,"pid":10412,"tc":10413,"Ġtips":10414,"Ġhomes":10415,"Ġfuel":10416,"âĸ":10417,"_helper":10418,"ĠĠčĊ":10419,"ĠRoom":10420,".Close":10421,"_attr":10422,"ĠMount":10423,"ĠEv":10424,"arser":10425,"_top":10426,"eah":10427,"ĠDelete":10428,"ãĢį":10429,"uke":10430,"Ġusage":10431,"aria":10432,"_dev":10433,"Ġtexture":10434,"Ġconversation":10435,"eper":10436,"Bean":10437,"done":10438,"nonatomic":10439,"ĠSecond":10440,"Ġshooting":10441,"_pre":10442,"Components":10443,"Ġ]ĊĊ":10444,"__,":10445,"stitution":10446,".Char":10447,">();ĊĊ":10448,"Ġpresented":10449,"Ġwa":10450,"oker":10451,"-ĊĊ":10452,"iner":10453,"Ġbecoming":10454,"Ġincident":10455,"Att":10456,"Ġrevealed":10457,"forc":10458,"Ġboot":10459,".page":10460,"Enumerator":10461,"_->":10462,"Photo":10463,"Ġspring":10464,".\",":10465,"ĠDictionary":10466,"BJECT":10467,"Ġlocations":10468,"Ġsamples":10469,"InputStream":10470,"ĠBrown":10471,"Ġstats":10472,"quality":10473,"Ñħ":10474,"-dis":10475,"Ġhelping":10476,"Ġped":10477,"(se":10478,"ĠWho":10479,"alian":10480,"internal":10481,"Ġft":10482,">().":10483,"->{":10484,"Ġmine":10485,"Ġsector":10486,"Ġgro":10487,"Ġopportunities":10488,"Ġü":10489,"Ġmp":10490,"Ġalleged":10491,"Ġdoubt":10492,"Mouse":10493,"About":10494,"_part":10495,"Ġchair":10496,"Ġstopped":10497,"loop":10498,"entities":10499,"Ġapps":10500,"ansion":10501,"Ġmental":10502,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":10503,"FR":10504,"Ġdefend":10505,"care":10506,"Ġideal":10507,"/api":10508,"urface":10509,"Ġele":10510,"ulator":10511,"ĠRights":10512,"anguages":10513,"Ġfunds":10514,"Ġadapt":10515,"Attributes":10516,"Ġdeploy":10517,"opts":10518,"Ġvalidation":10519,"Ġconcerns":10520,"uce":10521,".num":10522,"ulture":10523,"ila":10524,"Ġcup":10525,"Ġpure":10526,".Fore":10527,"ĠHashMap":10528,".valueOf":10529,"asm":10530,"MO":10531,"Ġcs":10532,"Ġstores":10533,"Ġ************************************************************************":10534,"Ġcommunication":10535,"mem":10536,".EventHandler":10537,".Status":10538,"_right":10539,".setOn":10540,"Sheet":10541,"Ġidentify":10542,"enerated":10543,"ordered":10544,"Ġ\"[":10545,"Ġswe":10546,"Condition":10547,"ĠAccording":10548,"Ġprepare":10549,"Ġrob":10550,"Pool":10551,"Ġsport":10552,"rv":10553,"ĠRouter":10554,"Ġalternative":10555,"([]":10556,"ĠChicago":10557,"ipher":10558,"ische":10559,"ĠDirector":10560,"kl":10561,"ĠWil":10562,"keys":10563,"Ġmysql":10564,"Ġwelcome":10565,"king":10566,"ĠManager":10567,"Ġcaught":10568,")}Ċ":10569,"Score":10570,"_PR":10571,"Ġsurvey":10572,"hab":10573,"Headers":10574,"ADER":10575,"Ġdecor":10576,"Ġturns":10577,"Ġradius":10578,"errupt":10579,"Cor":10580,"Ġmel":10581,"Ġintr":10582,"(q":10583,"ĠAC":10584,"amos":10585,"MAX":10586,"ĠGrid":10587,"ĠJesus":10588,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":10589,".DE":10590,"Ġts":10591,"Ġlinked":10592,"free":10593,"ĠQt":10594,"Ġ/**čĊ":10595,"Ġfaster":10596,"ctr":10597,"_J":10598,"DT":10599,".Check":10600,"Ġcombination":10601,"Ġintended":10602,"-the":10603,"-type":10604,"ectors":10605,"ami":10606,"uting":10607,"Ġuma":10608,"XML":10609,"UCT":10610,"Ap":10611,"ĠRandom":10612,"Ġran":10613,".sort":10614,"Ġsorted":10615,".Un":10616,"_PER":10617,"itory":10618,"Ġpriority":10619,"ĠGal":10620,"ĠOld":10621,"hot":10622,"ĠDisplay":10623,"(sub":10624,"_TH":10625,"_Y":10626,"ĠCare":10627,"loading":10628,"Kind":10629,"_handle":10630,",,":10631,"rase":10632,"_replace":10633,".addEventListener":10634,"ĠRT":10635,"Ġentered":10636,"gers":10637,"Ġich":10638,"(start":10639,"/app":10640,"Ġbrother":10641,"Memory":10642,"Outlet":10643,"Ġutf":10644,"prec":10645,"Ġnavigation":10646,"ORK":10647,"Ġdst":10648,"Detail":10649,"Ġaudience":10650,"Ġdur":10651,"Ġcluster":10652,"unched":10653,"Ġ],":10654,"Ġcomfortable":10655,".values":10656,"ĠTotal":10657,"Ġsnap":10658,"Ġstandards":10659,"Ġperformed":10660,"hand":10661,"(\"@":10662,"åŃ":10663,"Ġphil":10664,"ibr":10665,"trim":10666,"Ġforget":10667,"Ġdoctor":10668,".TextBox":10669,"icons":10670,",s":10671,"ĠOp":10672,"Sm":10673,"Stop":10674,"ĉList":10675,"ĉu":10676,"Comment":10677,"_VERSION":10678,".Xtra":10679,"Person":10680,"rb":10681,"LOB":10682,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":10683,"ĠCentral":10684,"ICK":10685,"raq":10686,"Ġputting":10687,"Ġmd":10688,"ĠLove":10689,"Program":10690,"Border":10691,"oor":10692,"Ġallowing":10693,"after":10694,"Ġentries":10695,"ĠMaybe":10696,"]).":10697,"ĠShort":10698,")\\":10699,".now":10700,"friend":10701,"Ġprefer":10702,"ĠGPIO":10703,"osis":10704,"ĠGameObject":10705,"Ġskip":10706,"Ġcompetition":10707,"_match":10708,"lications":10709,"_CONT":10710,".groupBox":10711,"Ġals":10712,"\"We":10713,"_eq":10714,"lan":10715,"_search":10716,"ĠMusic":10717,"asis":10718,"Ġbind":10719,"ĠIsland":10720,"rum":10721,"(E":10722,"Ġseat":10723,"Video":10724,"Ġack":10725,"reek":10726,"={()":10727,"Ġrating":10728,"Ġrestaurant":10729,"DEX":10730,"(buf":10731,"pping":10732,"uality":10733,"Ġleague":10734,"Ġfocused":10735,"apon":10736,"$data":10737,"CLUD":10738,"CLUDING":10739,"Ġabsolute":10740,"(query":10741,"Ġtells":10742,"Ang":10743,"Ġcommunities":10744,"Ġhonest":10745,"oking":10746,"Ġapart":10747,"arity":10748,"/$":10749,"_module":10750,"ĠEnc":10751,".an":10752,".Config":10753,"Cre":10754,"Ġshock":10755,"ĠArab":10756,"IENT":10757,"/re":10758,"Ġretrie":10759,"ycler":10760,"isa":10761,"ĠOrgan":10762,".graph":10763,"Ġí":10764,"ĠBAS":10765,"Enum":10766,"Ġpossibly":10767,"ÑĢаÐ":10768,"ĠJapanese":10769,"Ġcraft":10770,"ĠPlace":10771,"Ġtalent":10772,"Ġfunding":10773,"Ġconfirmed":10774,"Ġcycle":10775,"/x":10776,"GE":10777,"Ġhearing":10778,"Ġplants":10779,"Ġmouth":10780,"pages":10781,"oria":10782,"ĠRemove":10783,"_total":10784,"Ġod":10785,"ollapse":10786,"door":10787,"Ġbought":10788,"Ġaddr":10789,"ARCH":10790,"_dim":10791,"dden":10792,"Ġdecades":10793,"REQUEST":10794,"Ġversions":10795,"fire":10796,"Ġmoves":10797,"fb":10798,"Ġcoffee":10799,".connect":10800,"ĠRow":10801,"Ġschema":10802,"Scope":10803,"-Type":10804,"Ġfighting":10805,"Ġretail":10806,"Ġmodified":10807,"TF":10808,"Files":10809,"nie":10810,"_command":10811,"stone":10812,"ĠÑĤ":10813,"_thread":10814,"Ġbond":10815,"ĠDevelopment":10816,"Ġpt":10817,"FORM":10818,"plet":10819,"Ġidentified":10820,"cpp":10821,"Ġcoding":10822,"oked":10823,"ĠMaster":10824,"IDTH":10825,"Ġresidents":10826,"redit":10827,"ĠPhoto":10828,"=-":10829,"unte":10830,"ateur":10831,"_STATE":10832,"ĠSing":10833,"Ġsheet":10834,".val":10835,"orse":10836,"Ġhers":10837,"Ġdetermined":10838,"Common":10839,"Ġwed":10840,"_queue":10841,"PH":10842,"ĠAtl":10843,"cred":10844,"/LICENSE":10845,"Ġmes":10846,"Ġadvanced":10847,".java":10848,".Sh":10849,"Go":10850,"kill":10851,"fp":10852,"_settings":10853,"Ġpal":10854,"Ġtruck":10855,"Ġcombined":10856,"Ġ\"${":10857,"ĠCorpor":10858,"Ġjoined":10859,"ĠJose":10860,"ĠCup":10861,"uns":10862,"estival":10863,"levision":10864,"Ġbroken":10865,"Ġmarriage":10866,"ĠWestern":10867,"Ġrepresents":10868,"ĠTitle":10869,"Ġss":10870,".Ass":10871,"ongoose":10872,"iento":10873,"<>();Ċ":10874,"Ġabsolutely":10875,"Ġsmooth":10876,"TERN":10877,"ĠUnless":10878,"Word":10879,"Ġmerge":10880,"igan":10881,"ĠVol":10882,"Ġnn":10883,".getId":10884,"Ġз":10885,"Ġsexy":10886,"Ġseeking":10887,"Single":10888,".this":10889,"Ġkom":10890,"bound":10891,";\"":10892,"ĠfontSize":10893,"_df":10894,"Ġinjury":10895,"(H":10896,"Ġissued":10897,"_END":10898,":self":10899,"Ġpatch":10900,"Ġleaves":10901,"Ġadopt":10902,"FileName":10903,"ãĢIJ":10904,"Ġexecutive":10905,"ĠByte":10906,"]))Ċ":10907,"Ġnu":10908,"outing":10909,"cluding":10910,"-R":10911,".options":10912,"Ġsubstant":10913,"avax":10914,"ĠBUT":10915,"Ġtechnical":10916,"Ġtwice":10917,"Ġmás":10918,"Ġunivers":10919,"yr":10920,"Ġdrag":10921,"ĠDC":10922,"Ġsed":10923,"Ġbot":10924,"ĠPal":10925,"ĠHall":10926,"forcement":10927,"Ġauch":10928,".mod":10929,"notation":10930,"_files":10931,".line":10932,"_flag":10933,"[name":10934,"Ġresolution":10935,"Ġbott":10936,"(\"[":10937,"ende":10938,"(arr":10939,"Free":10940,"(@\"":10941,"ĠDistrict":10942,"PEC":10943,":-":10944,"Picker":10945,"ĠJo":10946,"ĠĠĠĠĠĊ":10947,"ĠRiver":10948,"_rows":10949,"Ġhelpful":10950,"Ġmassive":10951,"---Ċ":10952,"Ġmeasures":10953,"ĠRuntime":10954,"Ġworry":10955,"ĠSpec":10956,"ĉD":10957,"ãĢij":10958,"Ġ){Ċ":10959,"Ġworse":10960,"(filename":10961,"Ġlay":10962,"Ġmagic":10963,"ĠTheir":10964,"oul":10965,"stroy":10966,"ĠWhere":10967,"Ġsudden":10968,"Ġdefe":10969,"Ġbinding":10970,"Ġflight":10971,"ĠOnInit":10972,"ĠWomen":10973,"ĠPolicy":10974,"Ġdrugs":10975,"ishing":10976,"('../":10977,"ĠMel":10978,"peat":10979,"tor":10980,"Ġproposed":10981,"Ġstated":10982,"_RES":10983,"Ġeast":10984,"ĠCONDITION":10985,"_desc":10986,"Ġwinning":10987,"folio":10988,"Mapper":10989,"ĠPan":10990,"ĠAnge":10991,".servlet":10992,"Ġcopies":10993,"LM":10994,"Ġvm":10995,"åį":10996,"Ġdictionary":10997,"Seg":10998,"elines":10999,"ĠSend":11000,"Ġiron":11001,"ĠFort":11002,".domain":11003,"Ġdebate":11004,"NotNull":11005,"eq":11006,"acher":11007,"lf":11008,"ĉfmt":11009,"Ġlawy":11010,"ÄŁ":11011,"ĠMen":11012,"Ġtrim":11013,"(NULL":11014,"Ġ!!":11015,"Ġpad":11016,"Ġfollows":11017,"\"][\"":11018,"requ":11019,"ĠEp":11020,".github":11021,"(img":11022,"eto":11023,"('\\":11024,"Services":11025,"umbnail":11026,"_main":11027,"pleted":11028,"fortunately":11029,"Ġwindows":11030,"Ġplane":11031,"ĠConnection":11032,".local":11033,"uard":11034,"}\\":11035,"==\"":11036,"andon":11037,"ĠRoy":11038,"west":11039,"iginal":11040,"emies":11041,"itz":11042,"'):Ċ":11043,"ĠPeter":11044,"Ġtough":11045,"Ġreduced":11046,"Ġcalculate":11047,"Ġrapid":11048,"customer":11049,"Ġefficient":11050,"Ġmedium":11051,"Ġfell":11052,".ref":11053,"ĠCas":11054,"Ġfeedback":11055,"Speed":11056,"(output":11057,"aje":11058,"Ġcategories":11059,"Ġfee":11060,"};":11061,"Ġdeleted":11062,"reh":11063,"Ġproof":11064,"Desc":11065,"Build":11066,"Ġsides":11067,".ArrayList":11068,"-%":11069,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":11070,"ر":11071,".match":11072,"ли":11073,"Ġfeels":11074,"Ġachieve":11075,"Ġclim":11076,"_ON":11077,"ĠCD":11078,"Ġteacher":11079,"_current":11080,"bn":11081,"_PL":11082,"isting":11083,"Enable":11084,"GEN":11085,"Ġtv":11086,"Ġsock":11087,"Ġplays":11088,"Ġdiscount":11089,"ĠKE":11090,"ĠDebug":11091,"Fore":11092,"ĠIraq":11093,"Ġappearance":11094,"Mon":11095,"Ġstyled":11096,"ĠHuman":11097,"iot":11098,"ĠHistory":11099,"Ġsac":11100,"ĠCollection":11101,"Ġrecommended":11102,".Selected":11103,"Ġorganizations":11104,"Ġdiscovered":11105,"cohol":11106,"adas":11107,"ĠThomas":11108,"May":11109,"Ġconserv":11110,"Ġdomin":11111,"ĠFollow":11112,"ĠSection":11113,"ĠThanks":11114,"Username":11115,"Ġrecipe":11116,"Ġwonderful":11117,".sleep":11118,"_if":11119,"ĉĊĉĊ":11120,"orno":11121,"Ġru":11122,"_target":11123,".\"\"":11124,"à¦":11125,"EventArgs":11126,"Ġinputs":11127,"Ġfif":11128,"Ġvision":11129,"cy":11130,"ĠSeries":11131,")(((":11132,"Ġtrading":11133,"Ġmarker":11134,"Begin":11135,"Ġtypically":11136,"Ġcauses":11137,"dropdown":11138,"_DEBUG":11139,"Ġdetect":11140,"country":11141,"!\");Ċ":11142,"ĉR":11143,"appy":11144,"Ġcref":11145,"('<":11146,"\"=>":11147,"ĠLE":11148,"reader":11149,"Ġadministr":11150,"õ":11151,"ucket":11152,"Ġfashion":11153,".char":11154,"izar":11155,"Ġdisable":11156,"Ġsuc":11157,"ĠLive":11158,"issue":11159,"Ġmetadata":11160,"flags":11161,"ĠðŁ":11162,"Ġcommitted":11163,"Ġva":11164,"Ġrough":11165,"Ġ'''Ċ":11166,"Ġhighlight":11167,"_vars":11168,"VO":11169,"Ġencoding":11170,"-Z":11171,"_sign":11172,"$(\"#":11173,"Ġrain":11174,"reatest":11175,"ĠEND":11176,"Selection":11177,"Ġcandidates":11178,"Ġsav":11179,".Empty":11180,"Ġdecisions":11181,"Ġcollabor":11182,"ridge":11183,"feed":11184,"ression":11185,"Ġpersons":11186,"VM":11187,"ega":11188,"_BIT":11189,"According":11190,"acked":11191,"Ġdollars":11192,"_loss":11193,"ĠCost":11194,"}\"Ċ":11195,"Notification":11196,"Ġprostit":11197,"Ġauthority":11198,".rec":11199,"Ġspokes":11200,"ĠToday":11201,"istant":11202,"ĠHead":11203,"âĢĿ.":11204,"ertainment":11205,"cean":11206,"culate":11207,"Ġven":11208,"However":11209,"_arr":11210,"Ġtokens":11211,"Graph":11212,"ĠJud":11213,"ĠVirgin":11214,"ĠSerial":11215,"unning":11216,"Mutable":11217,"agers":11218,".csv":11219,"Ġdeveloping":11220,"Ġinstructions":11221,"Ġpromise":11222,"Ġrequested":11223,"_encode":11224,"/\"":11225,"ĠIcon":11226,"uilt":11227,"-day":11228,"Ġintelligence":11229,".IS":11230,"ĠObservable":11231,"ĠHard":11232,"Bool":11233,"idential":11234,".Anchor":11235,"Ġselling":11236,"CI":11237,"AGES":11238,"tle":11239,"bur":11240,"UFFER":11241,"RY":11242,"Ġbigger":11243,"Ġrat":11244,"Ġfamous":11245,"Ġtypename":11246,"Ġexplained":11247,"}}Ċ":11248,"Ġnuclear":11249,"-N":11250,"Ġcrisis":11251,"ĠEnter":11252,"Ġanswers":11253,"/${":11254,"/pl":11255,"Ġsequ":11256,"_next":11257,"mask":11258,"Ġstanding":11259,"Ġplenty":11260,"ĠCross":11261,"ĉret":11262,"dro":11263,"ĠCast":11264,"=true":11265,"ĠChris":11266,"icio":11267,"ĠMike":11268,"Decimal":11269,"addComponent":11270,"Len":11271,"Ġcock":11272,"Ġ#{":11273,"URN":11274,"":11403,"Ġ*=":11404,"ĠPS":11405,"Ġdangerous":11406,"[p":11407,"OME":11408,"Other":11409,"ĠStringBuilder":11410,"Points":11411,"heading":11412,"Ġcurrency":11413,"Ġpercentage":11414,"_API":11415,"Ġclassic":11416,"thead":11417,"ĠMO":11418,"FE":11419,"Idx":11420,"await":11421,"Ġè":11422,"Ġaccident":11423,"Ġvariant":11424,"Ġmyst":11425,"ĠLand":11426,"ĠBre":11427,"Ġharm":11428,"ĠAcc":11429,"Ġcharged":11430,"iones":11431,"Visibility":11432,"arry":11433,"ĠLanguage":11434,"Ġwalking":11435,"\".ĊĊ":11436,"ifer":11437,"Ġleadership":11438,".From":11439,"ynam":11440,"Ġtimestamp":11441,"ipt":11442,"ĠHas":11443,"REFER":11444,"ĠIts":11445,"Ġlistener":11446,"UTE":11447,"_description":11448,"Ġexperiences":11449,"Ġcreates":11450,"RS":11451,"cart":11452,"black":11453,"Ġchoices":11454,"war":11455,"Ġ'''":11456,"Ġordered":11457,"Ġevening":11458,"Ġpil":11459,"Ġtun":11460,"ĠBad":11461,"(app":11462,"random":11463,"Ġexplicit":11464,"Ġarrived":11465,"Ġfly":11466,"Ġeconom":11467,"-mail":11468,"Ġlists":11469,"Ġarchitect":11470,"ĠPay":11471,"Ġds":11472,"ĠSol":11473,"Ġvehicles":11474,"Hz":11475,"-com":11476,"Ġking":11477,"_equal":11478,"ĠHelp":11479,"Ġabuse":11480,"--;Ċ":11481,"Ġextr":11482,"Ġchemical":11483,"ä¿":11484,"Ġorient":11485,"Ġbreath":11486,"ĠSpace":11487,"(element":11488,"wait":11489,"DED":11490,"igma":11491,"Ġentr":11492,"Ġsob":11493,"-name":11494,"Ġaffected":11495,"ika":11496,"Ġcoal":11497,"_work":11498,"Ġhundreds":11499,"Ġpolitics":11500,"subject":11501,"Ġconsumer":11502,"ANGE":11503,"Ġrepeated":11504,"Send":11505,"Ġ#[":11506,"Ġprotocol":11507,"Ġleads":11508,"useum":11509,"Every":11510,"Import":11511,"(count":11512,"Ġchallenges":11513,"Ġnovel":11514,"Ġdepart":11515,"bits":11516,".Current":11517,"Ġ`${":11518,"oting":11519,"(\\":11520,"Ġcreative":11521,"Ġbuff":11522,"Ġintroduced":11523,"usic":11524,"modules":11525,"Are":11526,"-doc":11527,"language":11528,"_cache":11529,"Ġtod":11530,"?>{{":11764,"ĠResource":11765,"ĠStandard":11766,"ĠPrem":11767,"updated":11768,"ivalent":11769,"Ġassets":11770,"_temp":11771,"Ġinterests":11772,"Ġhardware":11773,"ĠRom":11774,"ĠShare":11775,"Ġ''Ċ":11776,"Ġ*,":11777,"ĠTake":11778,"ĠImages":11779,"_CHECK":11780,"(typeof":11781,"ĠJun":11782,"\\<^":11783,"Ġliqu":11784,"Ġworst":11785,"ymbols":11786,"ĉĉĉĠĠĠ":11787,"Ġdrivers":11788,"ĠDocument":11789,"eno":11790,"ĠTechnology":11791,"Ġapproved":11792,"umps":11793,"Ġsnow":11794,"formance":11795,"_ASSERT":11796,"uits":11797,"ÙĨ":11798,"Ġdifferences":11799,".Visible":11800,"ĉĉĉčĊ":11801,"ĠPs":11802,"_fetch":11803,"Ġtodo":11804,".',Ċ":11805,"Ġsel":11806,"urers":11807,"invalid":11808,"Ġtweet":11809,"VEL":11810,"Ġresearchers":11811,"Ġsprintf":11812,"ĠRO":11813,"Ġpel":11814,".Trans":11815,"Ġillegal":11816,"dialog":11817,"smarty":11818,"lg":11819,"_MIN":11820,"Ġhero":11821,"final":11822,"Ġpp":11823,".Le":11824,"Ġci":11825,"ĉRT":11826,"Ġsuggested":11827,"pdf":11828,"aching":11829,"ĠRo":11830,"ĠProperties":11831,"ĠSi":11832,"Ġbuying":11833,"Ġmu":11834,"Ġlands":11835,"ifiers":11836,"ĠFILE":11837,"ROUP":11838,"Ġholder":11839,"ĠSon":11840,"Ġsympt":11841,".route":11842,")?":11843,"Ġargc":11844,"Ġfort":11845,"Ġcasino":11846,"_category":11847,"Ġforum":11848,"prefix":11849,"apture":11850,"Tube":11851,"ems":11852,"imize":11853,"Ġnue":11854,"aus":11855,"course":11856,"ATOR":11857,"()),":11858,"Advertis":11859,"INGS":11860,"Ġacknow":11861,"ĠKorea":11862,"pling":11863,"Ġworker":11864,"PLIED":11865,"hal":11866,"ĠRichard":11867,"Elements":11868,"ĉĉĉĠ":11869,"star":11870,"Ġrelationships":11871,"Ġcheap":11872,"ACH":11873,"ĠXML":11874,",&":11875,"ĠLouis":11876,"Ġride":11877,"_FAIL":11878,"Ġchunk":11879,"[s":11880,"_OUT":11881,"Ġchosen":11882,"_[":11883,"/(":11884,"ĠJeff":11885,"_sl":11886,"priv":11887,"ĠCanadian":11888,"Ġunable":11889,"_FLAG":11890,"Ġnos":11891,"high":11892,"Ġlift":11893,"fun":11894,"(){":11895,"elly":11896,"yclerView":11897,"_as":11898,"_LIST":11899,"Ġradi":11900,".getValue":11901,"ĠAngeles":11902,"ĠSpan":11903,"_instance":11904,"itors":11905,"Ġmigration":11906,"AK":11907,"Oh":11908,"®":11909,".selected":11910,"ĠGT":11911,"Ġadvance":11912,"ĠStyle":11913,".DataGridView":11914,"ection":11915,"Ñİ":11916,"pio":11917,"rog":11918,"Ġshopping":11919,"ĠRect":11920,"Illuminate":11921,"OU":11922,"ĉarray":11923,"Ġsubstantial":11924,"Ġpregn":11925,"Ġpromote":11926,"IEW":11927,".Layout":11928,"Ġsigns":11929,"/.":11930,"Ġletters":11931,"Board":11932,"ctrl":11933,"\"\\":11934,"ĠJones":11935,"Ġvertex":11936,"Ġja":11937,"Ġaffili":11938,"Ġwealth":11939,"ĉdefault":11940,"Ġsignificantly":11941,"Ġec":11942,"Ġxs":11943,"actual":11944,".per":11945,"_step":11946,"anvas":11947,"mac":11948,"Ġtransl":11949,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":11950,"Iterator":11951,"Ġoch":11952,"agnostic":11953,"ĠDuring":11954,"ĠDEFAULT":11955,"Ġtill":11956,"Ġsignature":11957,"Ġbird":11958,"ĠOl":11959,"ĠIr":11960,"HS":11961,"avatar":11962,"ESSAGE":11963,"Ġelev":11964,"Ġmt":11965,"ĠNav":11966,"Ġrelax":11967,"Ġplate":11968,"ITEM":11969,"(date":11970,".not":11971,"Ġgrade":11972,"Ġ}),Ċ":11973,"?\"ĊĊ":11974,"iences":11975,"High":11976,"ĠDIS":11977,"disabled":11978,"QUI":11979,"Ġnoise":11980,"aux":11981,"ĠUP":11982,"osa":11983,"Ġvoc":11984,"Ġ))":11985,"ocom":11986,"_OFF":11987,"ĠDb":11988,"Lock":11989,".eclipse":11990,",d":11991,"ĠDraw":11992,"Ġ\"(":11993,"Ġvisited":11994,"ĠâĪ":11995,"Ġsucceed":11996,"Ġimpossible":11997,"aire":11998,"ĠTurn":11999,"Ġdish":12000,"FG":12001,"Ġsensor":12002,"ANN":12003,"aba":12004,"Ġsurg":12005,"]);čĊ":12006,"Ġfp":12007,"_an":12008,"-J":12009,"-G":12010,"ĠJob":12011,"Convert":12012,"ĠKEY":12013,"Ġauthors":12014,"_server":12015,"\\r":12016,"Ġ-*-":12017,"flex":12018,"Ġsoc":12019,"Ret":12020,"Ġsalt":12021,"ĠâĢ¦ĊĊ":12022,"ĠClear":12023,"(page":12024,"-danger":12025,"Ġrooms":12026,"conv":12027,"#{":12028,".op":12029,"ĠArea":12030,"_SC":12031,"hen":12032,"Ġbegins":12033,"-y":12034,"Ġexcited":12035,"Ġignored":12036,"Ġbonus":12037,"student":12038,"ĠMember":12039,"Ġrelatively":12040,"ĠLow":12041,"ĠProdu":12042,"ateway":12043,"posure":12044,"Ġthick":12045,"aniel":12046,"(view":12047,"ĠCrush":12048,"Extension":12049,"Il":12050,"eed":12051,"LOC":12052,".im":12053,".Items":12054,"Ġconflict":12055,".prevent":12056,"ĠonCreate":12057,"uv":12058,"iser":12059,"Ġwave":12060,"Mar":12061,"ĠCommunity":12062,"iche":12063,"ĠNothing":12064,"[m":12065,"ĠLee":12066,"riends":12067,"ère":12068,"!!!":12069,"anz":12070,".result":12071,"ĠSK":12072,"_PARAM":12073,"Ġdemocr":12074,"BackColor":12075,".exists":12076,"\"It":12077,"(options":12078,"razy":12079,"aser":12080,"\\Database":12081,"alendar":12082,"_ass":12083,";}Ċ":12084,"vertex":12085,"inecraft":12086,"Warning":12087,"argo":12088,"Ġactor":12089,"ĠInstead":12090,"ĠUsing":12091,"Self":12092,"@interface":12093,"Ġspeaking":12094,"ĠParis":12095,"ĠLICENSE":12096,".node":12097,"ĠFood":12098,"EIF":12099,"ĠBi":12100,".Start":12101,"ĠIB":12102,"Ġuniversity":12103,"ĠHeader":12104,".product":12105,"Copy":12106,"etc":12107,"rical":12108,"Ġ>>>":12109,"books":12110,"Ġalgorithm":12111,"Ġ'__":12112,"(javax":12113,"Ġnumerous":12114,"Share":12115,"Have":12116,"Ġrecru":12117,"Ġprove":12118,".substring":12119,"health":12120,"ел":12121,"Ġdecimal":12122,"Ġcommission":12123,"scription":12124,"xC":12125,"Ġsummary":12126,"atted":12127,"Ġcloser":12128,"finished":12129,"()){Ċ":12130,"ĠWood":12131,"_fields":12132,"ku":12133,"_items":12134,"Flag":12135,"Ġconfidence":12136,"ĠFederal":12137,"dux":12138,"Ġcompat":12139,"Ġvertical":12140,"й":12141,"ès":12142,";\">Ċ":12143,"_manager":12144,"()))Ċ":12145,"IDE":12146,":\",":12147,"__Ċ":12148,"ĠWay":12149,"ÑĪ":12150,"Temp":12151,"ĠSTR":12152,"ritten":12153,"Sync":12154,"ĠAV":12155,"ĠCEO":12156,"ĠGuid":12157,"Ġenvironmental":12158,"Ġcorresponding":12159,"ĉconsole":12160,"Ġjustice":12161,"ĠJS":12162,"Ġlived":12163,"gar":12164,"ĠGraph":12165,"ĠStat":12166,"ĠiPhone":12167,".al":12168,"ĠHD":12169,"Ġoccur":12170,"Ġthreshold":12171,"Ġonclick":12172,"REG":12173,".GraphicsUnit":12174,"Meta":12175,"ž":12176,"Ġcum":12177,".gnu":12178,"ë":12179,"Ġobtained":12180,"Ġcomplaint":12181,"Ġeating":12182,"Ġtar":12183,"_task":12184,"Ġopts":12185,"(to":12186,"Pass":12187,"Ġplastic":12188,"tility":12189,"ĠWin":12190,".preventDefault":12191,"pile":12192,"ĠGar":12193,"Ġquantity":12194,"_last":12195,"Ġgreatest":12196,"Dao":12197,"_DIS":12198,"ĠUsed":12199,"ĠHP":12200,"riting":12201,"SION":12202,"blue":12203,"domain":12204,"Ġscores":12205,"Normal":12206,"_admin":12207,"ĠASSERT":12208,"Then":12209,"***":12210,"dist":12211,"lon":12212,"Ġhate":12213,"shal":12214,"ImageView":12215,"database":12216,"Ġpand":12217,"Ġlogic":12218,"=false":12219,"bg":12220,"ĠConfiguration":12221,"Ġnur":12222,"OG":12223,"Ġmarried":12224,":+":12225,"Ġdropped":12226,"Ġregistration":12227,"ом":12228,"ultiple":12229,"izers":12230,"shape":12231,".copy":12232,"Ġwearing":12233,"ĠCath":12234,"Ġdedicated":12235,"Ġ...Ċ":12236,"Ġadvoc":12237,"ĠFamily":12238,"Ġstatements":12239,"ematic":12240,"ampionship":12241,"Ġmotiv":12242,"ĠHave":12243,"Ġblow":12244,"Job":12245,"cert":12246,"_vector":12247,"install":12248,"ĠCOPY":12249,"embed":12250,"DIR":12251,"ĠSpring":12252,"Ġexhib":12253,"cdn":12254,"ĠComment":12255,"ĠOptional":12256,".player":12257,"ĠDark":12258,"(pos":12259,"ĠShould":12260,"Ġcentre":12261,"ĠGuard":12262,"ów":12263,"Ġtrouble":12264,"ENER":12265,"(unsigned":12266,"_service":12267,"Ġns":12268,"uling":12269,"ĠMexico":12270,"ĠNY":12271,"mysql":12272,"Ġlic":12273,"åľ":12274,"Mr":12275,"-fl":12276,"ĠCustomer":12277,"idi":12278,"Ġ?>ĊĊ":12279,"rible":12280,"ĠпÑĢ":12281,"Ġsizes":12282,"_STRING":12283,"validation":12284,"ĠJon":12285,"(Http":12286,"addClass":12287,"Nodes":12288,"Ġfragment":12289,"Ġspoke":12290,"Ġwaste":12291,"Join":12292,"Ġillustr":12293,"eli":12294,"cient":12295,"Ġaid":12296,"Ġprosec":12297,"'){Ċ":12298,"Ġpassing":12299,"Ġfaces":12300,"Shape":12301,"_Z":12302,"iti":12303,"Ġalle":12304,"Ġrobot":12305,"ĠĠĠĠĠĠĠĊ":12306,"ĠSpe":12307,"Ġreceiving":12308,"ĠDetails":12309,"Ġ\")":12310,"mg":12311,"_REF":12312,"Ġcomparison":12313,"*,":12314,"ĠFound":12315,"_session":12316,"(U":12317,"/F":12318,"Ġxxx":12319,"Network":12320,"ders":12321,"Ġcapture":12322,"Ġcorre":12323,"ĠLtd":12324,"ĠAdv":12325,"[@":12326,"Ġclip":12327,"Mill":12328,"ĠProfile":12329,"Ġendif":12330,"Ġoblig":12331,"describe":12332,".element":12333,"riterion":12334,"LD":12335,"ered":12336,"Ġfavour":12337,"score":12338,"ĠFilter":12339,"attributes":12340,"Ġchecks":12341,"Inflater":12342,"ĠPlus":12343,"Ġscientific":12344,"Ġprivacy":12345,"Head":12346,"Ġfeat":12347,"Ġdegrees":12348,"ĠPale":12349,";\">":12350,"Ġfilms":12351,"ĠAudio":12352,"ĠTag":12353,"ĠEnergy":12354,"itar":12355,"parator":12356,"Ġfellow":12357,"Ġevt":12358,"ĠTri":12359,"ĠDAM":12360,"cloud":12361,"ĠPassword":12362,"ĠDemocrats":12363,"ĠAcad":12364,"$lang":12365,"Ġreb":12366,"())ĊĊ":12367,"нÑĭ":12368,"ĠBur":12369,"readcr":12370,"Ġhex":12371,"Console":12372,"ctl":12373,"ousel":12374,"ĠWilliam":12375,"Ġaz":12376,"_PORT":12377,"Ġpractices":12378,"Ġanywhere":12379,"ĠPosition":12380,"Ġ->Ċ":12381,"iams":12382,".username":12383,"placeholder":12384,"Ġoder":12385,"ĠSecretary":12386,"ĠiT":12387,"mond":12388,"events":12389,"?âĢĿ":12390,".Sub":12391,"Ġattached":12392,"Ġnão":12393,"Ġestate":12394,".action":12395,"Ġfigures":12396,"Ġ});čĊ":12397,"Ġsubscri":12398,".tag":12399,"nam":12400,".plot":12401,"noon":12402,"liament":12403,"Character":12404,".tab":12405,"Ġwinter":12406,"ĠVariable":12407,"Ġtrees":12408,"Ġproud":12409,"(V":12410,"_load":12411,"Ġhier":12412,"ĠEcon":12413,"Ġfd":12414,"Ġvictims":12415,"Rest":12416,"iana":12417,"Ġfake":12418,".Println":12419,"Ġstrlen":12420,"Ġsad":12421,"Ġble":12422,"Prot":12423,"Ġbuttons":12424,"Ġtelevision":12425,"Ġlogo":12426,"extension":12427,"ĉj":12428,"stein":12429,"aciones":12430,"Ġ\"\"\"ĊĊ":12431,"Ġsimp":12432,"Ġrecorded":12433,"Ġbrings":12434,"Ġprincipal":12435,"Ġfees":12436,"(source":12437,"kdir":12438,"Ġutils":12439,"Ġcorrectly":12440,"fil":12441,"Ġwel":12442,"Pair":12443,"-button":12444,"scale":12445,"verify":12446,"[c":12447,"Ġ---":12448,"Ġescape":12449,"ikes":12450,"LowerCase":12451,"ician":12452,"Ġchapter":12453,"ĠTYPE":12454,"Ġshadow":12455,"Ġawesome":12456,"WE":12457,"elif":12458,"Ġlambda":12459,"Ġdistinct":12460,"Ġbare":12461,"-off":12462,"Ġcolour":12463,".appendChild":12464,"olec":12465,"aga":12466,".fill":12467,"ĉsuper":12468,"Ġadj":12469,"(position":12470,".getItem":12471,"Short":12472,"Ġtotally":12473,"VD":12474,"ĠTre":12475,"_ep":12476,"vements":12477,"ĠSolution":12478,"Ġfundament":12479,"Follow":12480,"Ġfacility":12481,"Ġhappening":12482,"OF":12483,".textBox":12484,"Span":12485,"Ġ«":12486,"iden":12487,"Ġexceed":12488,"(parent":12489,"Ġcp":12490,"ç»":12491,"Ġhasn":12492,"Ġpri":12493,"Ġconsequ":12494,"nen":12495,"ĠINTO":12496,"Ignore":12497,"ĠFuture":12498,"Ġcarbon":12499,"ĠSteel":12500,"fmt":12501,"okie":12502,"Ġspl":12503,"(title":12504,"-info":12505,"Ġdeals":12506,"Ġfixture":12507,"ea":12508,"Div":12509,"Ġtested":12510,"_return":12511,")ĊĊĊĊ":12512,"upported":12513,"ĠCook":12514,"Ġpaying":12515,"ĠIll":12516,"Ġarrested":12517,"ĠPrime":12518,"_callback":12519,">,Ċ":12520,"driver":12521,"Once":12522,"abb":12523,"_bytes":12524,"ĠSets":12525,"(Object":12526,"Ġcc":12527,"Ġshell":12528,"alo":12529,");//":12530,"(log":12531,"ctors":12532,")":13004,"Ġ$(\".":13005,".pos":13006,"Ġboys":13007,"Ġwedding":13008,"Ġagents":13009,"=\"_":13010,"ĠArmy":13011,"Ġhint":13012,"vision":13013,"Ġtech":13014,"ĠConnect":13015,"Ġlegend":13016,"ĠBet":13017,".Base":13018,"Subject":13019,"Ġlit":13020,"Remove":13021,"Ġ\":":13022,"ĠFinal":13023,"pearance":13024,"ĠiTunes":13025,"Ġparticipants":13026,"ĠPython":13027,"Ġbusy":13028,"iel":13029,"vertices":13030,"ĠtemplateUrl":13031,"ĠClose":13032,"Img":13033,"ĠCorporation":13034,"timestamp":13035,"Ġextend":13036,"Ġwebsites":13037,"Ġpossibility":13038,"оÑĤ":13039,"Ġkö":13040,"Ġmeat":13041,"Ġrepresentation":13042,"Ġĉĉ":13043,"_START":13044,".apply":13045,"ĠValley":13046,"ĠSuccess":13047,"Hi":13048,"Ġnob":13049,"ĠIEnumerable":13050,"_select":13051,"geo":13052,".\")Ċ":13053,"Ġturning":13054,"Ġfabric":13055,"(\"\");Ċ":13056,"Ġperspective":13057,"éĹ":13058,"ĠSn":13059,"Thank":13060,";j":13061,".Parameters":13062,"ĉĠĠĠĠĠĠĠĠĠĠĠ":13063,"Ġfacts":13064,"Ġunt":13065,".instance":13066,"################################################################":13067,"-end":13068,"ĠJOIN":13069,"ĠHen":13070,"Ġuri":13071,"åIJį":13072,"Ġна":13073,"ĠInfo":13074,"Ġconducted":13075,"ĠÃ¥":13076,"OURCE":13077,"Ġwine":13078,"John":13079,".Errorf":13080,"ĠAge":13081,"ounded":13082,"Ġrealize":13083,"Ġ];":13084,"Ġsubsequ":13085,",m":13086,"(User":13087,"iano":13088,"Ġaccompl":13089,"isp":13090,".std":13091,"éĩ":13092,"ĠBed":13093,".setAttribute":13094,"BR":13095,"keep":13096,"ĠALL":13097,"Ġisol":13098,"amma":13099,"Package":13100,"Ġoccasion":13101,"-success":13102,"ед":13103,"ĠLIMITED":13104,"strip":13105,"()ĊĊĊ":13106,"istribution":13107,"Colors":13108,"Ġ+:+":13109,"DidLoad":13110,"aler":13111,"Ġtid":13112,"ĠLED":13113,"ĠLinked":13114,"ĠCart":13115,"())čĊ":13116,"_READ":13117,"Ġkilling":13118,"ĠPHP":13119,"fection":13120,"Ġinstances":13121,"cv":13122,"\"/>":13123,"Ġsf":13124,"Ġtaxes":13125,"_location":13126,"ĠBitcoin":13127,"uable":13128,"rank":13129,"ignore":13130,"track":13131,"ка":13132,"Ġshouldn":13133,"ĠOP":13134,"=>{Ċ":13135,"Ġkm":13136,"Ġhelper":13137,"_head":13138,"ĠWhether":13139,"oco":13140,"_bl":13141,"Ġstatistics":13142,"Ġbeauty":13143,"Ġtog":13144,"tip":13145,"ëĭ¤":13146,"Ġcsv":13147,"(sql":13148,"stdlib":13149,"weak":13150,"Ġlikes":13151,"Äį":13152,"Ġrepeat":13153,"Ġapartment":13154,"Ġemph":13155,"_edit":13156,"Ġvit":13157,"ĉtype":13158,"Even":13159,"uten":13160,"Ġcircumstances":13161,"bian":13162,"Ġsugar":13163,"Windows":13164,"ìŀ":13165,"Ġobserved":13166,"/data":13167,"Ġcalendar":13168,"Ġstrike":13169,"ĠRES":13170,"_sc":13171,"fony":13172,"orem":13173,"(z":13174,"power":13175,"etect":13176,"ĠSat":13177,".description":13178,"Ġgang":13179,"ĠSports":13180,"ongs":13181,"ĠBundle":13182,".sum":13183,"once":13184,"Ġaccused":13185,"Ġexplore":13186,"Ġapproximately":13187,"Ġlosing":13188,"thesis":13189,"ĠFund":13190,"Ġdiagn":13191,"Autowired":13192,"properties":13193,"Ġ_.":13194,"Ġcnt":13195,"cedure":13196,"Ġyy":13197,"Ġgrant":13198,"sock":13199,".innerHTML":13200,"Ġ]);Ċ":13201,"ĠCONFIG":13202,"='$":13203,"]];Ċ":13204,"UND":13205,"Ġglob":13206,"Ġdire":13207,"uffle":13208,"_MEM":13209,"Ġauthentic":13210,">(\"":13211,"Ġdecade":13212,"ĠImport":13213,"Ġoriginally":13214,"ĠjQuery":13215,"Ġindicate":13216,"Ġourselves":13217,"Sw":13218,".lbl":13219,"enerate":13220,"Ġbasically":13221,"ĠHom":13222,"Ġ+#+":13223,"ĠBritain":13224,"ĠKar":13225,"toEqual":13226,".stop":13227,"Ġmodal":13228,"isi":13229,"Ġsuggests":13230,"Ġdtype":13231,"Ġtur":13232,"bf":13233,"Ġconnections":13234,"ĠBefore":13235,"isted":13236,"mouse":13237,"Ġpulled":13238,".build":13239,"Ġlegislation":13240,"Ġforth":13241,"pad":13242,"ego":13243,".Now":13244,"Ġexciting":13245,"}ĊĊĊĊ":13246,"Ġcompr":13247,"Ġshares":13248,"Ġrig":13249,"green":13250,"_vec":13251,"Ġenumerate":13252,"Auto":13253,"icator":13254,"ĠRay":13255,"asse":13256,"Ġholiday":13257,"Ġnullable":13258,"gun":13259,"_details":13260,"Ġwrapper":13261,"seq":13262,"ĠYoung":13263,"juana":13264,"Ġ\"__":13265,"license":13266,"serve":13267,"^(":13268,"iders":13269,".Remove":13270,"ropdown":13271,"'S":13272,"pin":13273,"(token":13274,".Default":13275,"Ġreasonable":13276,"ampion":13277,"ĠSociety":13278,"Ġbei":13279,"erves":13280,"rad":13281,"ĠFox":13282,"_images":13283,"Ġwheel":13284,"')[":13285,"Ġcfg":13286,"(By":13287,"Constructor":13288,"Ġvary":13289,".swift":13290,"Ġproxy":13291,"ĉH":13292,"ĠAnother":13293,"ĠPen":13294,"Ġchecking":13295,"Ġjest":13296,"manager":13297,"Origin":13298,"ugs":13299,"oir":13300,">čĊ":15956,"Ġrelief":15957,"lap":15958,"quer":15959,"_parent":15960,"heap":15961,"LOSE":15962,"Ġcombine":15963,"ĠRose":15964,"owers":15965,"Ġprocedures":15966,"ĠSort":15967,"anim":15968,"variant":15969,"ehicle":15970,"Ġsigning":15971,"Primary":15972,"currency":15973,"Ġsexe":15974,"oen":15975,"theta":15976,"eman":15977,"Ġimpressive":15978,"('_":15979,"ĉU":15980,"ĠTextStyle":15981,"_cnt":15982,"Ġslice":15983,"(':":15984,"Ġunderstood":15985,"His":15986,"Ġinformed":15987,"Ġnick":15988,"(TAG":15989,"hd":15990,"Ġelections":15991,"esture":15992,"ĠSanta":15993,"ĠCoast":15994,".pdf":15995,"inciple":15996,".clone":15997,"born":15998,"uta":15999,"Ġlicensed":16000,"Cr":16001,"Ġbread":16002,"ĠHouston":16003,"Ġnod":16004,"Ġhopes":16005,"ĠCGRect":16006,"Ġguilty":16007,".gif":16008,"Ġrose":16009,".Common":16010,"Tip":16011,"ANK":16012,"ĠFC":16013,"During":16014,"ĠSymfony":16015,"Ġdefensive":16016,"km":16017,")>":16018,"archive":16019,"ĠURI":16020,"ycling":16021,"-o":16022,"ĠWebsite":16023,"AMP":16024,"ishment":16025,"Ġdoctors":16026,"Direct":16027,"ARI":16028,"ĠRedirect":16029,"ieren":16030,"_dist":16031,"yo":16032,"ĠProgress":16033,"Ġzum":16034,"Ġmemor":16035,"ĠED":16036,"Ġjur":16037,"æį®":16038,"_TABLE":16039,"Ġuuid":16040,"Expr":16041,".head":16042,"('%":16043,"pointer":16044,"Ġestimate":16045,"ĠGreg":16046,"Ġloader":16047,"ĠiOS":16048,"Ġmens":16049,"[y":16050,"Ġrefused":16051,"Ġprecision":16052,"isch":16053,"ĠACTION":16054,"Cloud":16055,"sWith":16056,"(ret":16057,"_ADDR":16058,"_conf":16059,"(df":16060,"Ġlocked":16061,"Ġrising":16062,"ãĥ»ãĥ»":16063,"ĠMs":16064,"Ġscenes":16065,"_EXT":16066,"_raw":16067,"_the":16068,"people":16069,"Ġrecon":16070,"ĠFun":16071,"Ġbless":16072,"ĠUpdated":16073,"ün":16074,"ĠĠĠĠĠĠĠĠĠĠĠĠčĊ":16075,"pection":16076,"Release":16077,".logger":16078,"ĠSY":16079,"Ġcounsel":16080,"urd":16081,"_true":16082,"Ġeverybody":16083,"ivot":16084,"Ġhence":16085,"ĠNAS":16086,"Ġopposed":16087,"unknown":16088,"ĠDESC":16089,"ĠChair":16090,"failed":16091,"ĠINCLUDING":16092,"Ġwriters":16093,"{}Ċ":16094,"ÃŃt":16095,"_copy":16096,"}:":16097,"ĠBat":16098,"Ġconverted":16099,"eding":16100,"placement":16101,"ĠHost":16102,"Sound":16103,"им":16104,"Ġsought":16105,"mid":16106,"Ġsalary":16107,"ogg":16108,"âĦ¢":16109,"bul":16110,"Ġwir":16111,"validator":16112,"_STAT":16113,".store":16114,"ĠBattle":16115,"ın":16116,"Ġ-->ĊĊ":16117,"Trump":16118,"dot":16119,"ĠCONT":16120,".fetch":16121,"Ġcontinu":16122,"was":16123,"Ġfraud":16124,"_tmp":16125,"mitter":16126,".pictureBox":16127,"GA":16128,"Ġtournament":16129,".Input":16130,"[r":16131,"exion":16132,"centage":16133,"ĠKorean":16134,"undef":16135,"ĠAvailable":16136,"reshape":16137,"Ġkit":16138,"ĠStruct":16139,"ĠSUB":16140,"Answer":16141,"_lib":16142,".twitter":16143,"Ġore":16144,"ĠDragon":16145,".Ext":16146,",k":16147,"Ġexplanation":16148,"refs":16149,"ĠDrive":16150,"ĠTraining":16151,".Has":16152,"intage":16153,"big":16154,"ologist":16155,"ennis":16156,"Ùĩ":16157,"Ġchicken":16158,"ĠĠĠĠĠĠĠĠĠĠĊ":16159,"çĽ":16160,"ãģ§":16161,"Ġpeak":16162,"Ġdrinking":16163,"Ġencode":16164,"ĠNEW":16165,"malloc":16166,"ĉfprintf":16167,"Ġ=================================================================":16168,"including":16169,"Ġprinciples":16170,"ĠMah":16171,"storage":16172,"-key":16173,"Ġkeyword":16174,"%;":16175,"Ġtrained":16176,".contrib":16177,"Ġkv":16178,"__':Ċ":16179,"ĠBoy":16180,"parameter":16181,"Ġsuite":16182,"Ġthousand":16183,"Ġcoordinate":16184,"-generated":16185,"íķĺ":16186,"generated":16187,"Ġadmitted":16188,"Ġpussy":16189,"#w":16190,"Ġswim":16191,"union":16192,"Na":16193,"ĠRoyal":16194,".channel":16195,"Updated":16196,"_ROOT":16197,"Ġvital":16198,"raction":16199,"ĠCrusher":16200,"Ġpreced":16201,"Ġhorizontal":16202,"Blueprint":16203,"Ġattrs":16204,"Ġsmoke":16205,"ÐĴ":16206,".Equals":16207,"FB":16208,"ĠResources":16209,"rolling":16210,"Ġpasses":16211,"ĠNum":16212,"rotate":16213,"etype":16214,"\\\",":16215,"Ġsensitive":16216,"Ġtall":16217,"?âĢĿĊĊ":16218,"Proxy":16219,"iy":16220,"_section":16221,"âĢĶâĢĶâĢĶâĢĶ":16222,"brid":16223,"Ġcircuit":16224,"atan":16225,"ENC":16226,"Ġdriven":16227,"Ġvoted":16228,"Ġeducational":16229,"Ġinteraction":16230,"abetes":16231,"Ġtone":16232,"ĠInitializeComponent":16233,"Ġmerely":16234,"Ġìŀ":16235,"cookie":16236,"_div":16237,"ĠUILabel":16238,"vely":16239,"});čĊ":16240,"_ENT":16241,"#+#+":16242,"articles":16243,"ĠSouthern":16244,"Ġstronger":16245,"ĠGiven":16246,"ĠEric":16247,"ĠIR":16248,"abstract":16249,"Under":16250,"nable":16251,"Ġincrement":16252,"oven":16253,"Ġcoin":16254,"_timer":16255,"Ġsuffered":16256,"ĠFREE":16257,"'].\"":16258,"ĠQueen":16259,"stats":16260,"Ġmeetings":16261,"Ġentering":16262,"Ġalongside":16263,"(session":16264,"itals":16265,"Ġfoundation":16266,"ĠCredit":16267,".div":16268,"_ALL":16269,"pcion":16270,"_stat":16271,"icking":16272,"Defaults":16273,"_src":16274,"Ġoutputs":16275,"/B":16276,"Ġenthus":16277,"-bl":16278,".ForeColor":16279,"ĉtemp":16280,"Face":16281,"Ġinteract":16282,"Ġweird":16283,"Mount":16284,"rell":16285,"udents":16286,"Ġrequirement":16287,"ĠSus":16288,"IER":16289,"Ġelected":16290,"reference":16291,"ĠME":16292,"Ġservers":16293,".wait":16294,"Ġsnapshot":16295,"ilton":16296,"Ġtries":16297,"Ġtipo":16298,".Time":16299,">w":16300,"Ġmountain":16301,"Ġpounds":16302,"Ġ[...":16303,"exists":16304,"ĠngOn":16305,"_MAP":16306,"Ġflying":16307,"xiety":16308,"ĉvalue":16309,"_DB":16310,"uno":16311,"Ġseats":16312,"TURN":16313,".author":16314,"!)":16315,"orce":16316,"Ġindicated":16317,".sin":16318,"Ġassignment":16319,"imiento":16320,"ĠFrame":16321,"_gen":16322,"inery":16323,"_)":16324,"messages":16325,".settings":16326,"ĠMean":16327,"ĠMuseum":16328,"irq":16329,"attach":16330,"ĠPalestin":16331,"_QU":16332,"_tags":16333,"Ġcasual":16334,"emen":16335,"ASSWORD":16336,"$s":16337,"ĠCirc":16338,"ой":16339,"etric":16340,"/P":16341,"Ġepoch":16342,"The":16357,"ĠAk":16358,"Ġgrass":16359,"/*čĊ":16360,"(dis":16361,"Ġguns":16362,"Ġtb":16363,"ĠKevin":16364,".args":16365,"ĠAh":16366,"oped":16367,"(J":16368,"columns":16369,"arguments":16370,"ĠWithEvents":16371,"_full":16372,"ĠDefense":16373,"Simple":16374,"Ġdeaths":16375,"Ġextensive":16376,"ĠStill":16377,"ĠExpression":16378,"ĠAgency":16379,"Ġperforming":16380,"FX":16381,"Ġusuario":16382,"UAL":16383,"Side":16384,"odos":16385,"aptop":16386,"Ġcredentials":16387,"_cap":16388,"atient":16389,"ĠDisney":16390,"Ġai":16391,"Ġchip":16392,"Ġvolt":16393,".makeText":16394,"%%%%%%%%%%%%%%%%":16395,"Ġbelief":16396,"_LOC":16397,"ĠCivil":16398,"Navigation":16399,"Ġreveal":16400,"Ġviolent":16401,"ĠFil":16402,"Ġcatalog":16403,"emed":16404,"scan":16405,".control":16406,"Ġconstitution":16407,"Country":16408,"Separator":16409,"_APP":16410,"topic":16411,"uetooth":16412,"MIN":16413,"Ġdescriptor":16414,"yt":16415,"ETHER":16416,"Ġdistribute":16417,"'}Ċ":16418,".trim":16419,".Line":16420,"Ġlbl":16421,"assertEquals":16422,"ĠDet":16423,"ombok":16424,"(width":16425,"Ġtort":16426,"ĠEXPRESS":16427,"aco":16428,"Using":16429,"ĠBrand":16430,"wall":16431,"EMENT":16432,"ĠCommunic":16433,"(Ċ":17055,"?>\"":17056,"Ġ///Ċ":17057,"Ġeiner":17058,"Ġweekly":17059,"ĉlogger":17060,"_pop":17061,"_man":17062,"Ġmigrations":17063,"Ġasks":17064,"Ġbs":17065,"Ġfalls":17066,".Where":17067,"-height":17068,"_feature":17069,".Min":17070,"Ġhyper":17071,"Ġvolatile":17072,"Ġtwenty":17073,"Typography":17074,"Unable":17075,"Det":17076,",f":17077,"-mod":17078,"Ġsettlement":17079,"Ġcontracts":17080,"nome":17081,"Bad":17082,"ĠBrian":17083,"(username":17084,"!!!!":17085,"Ġhack":17086,".Field":17087,"HR":17088,"ĠJordan":17089,"iza":17090,"ĠÂł":17091,"ĠSher":17092,".header":17093,"(other":17094,"ĠDub":17095,"(op":17096,"ĠRound":17097,"Ġvie":17098,"Ġappl":17099,"ĉJ":17100,"ĠInsert":17101,"ĠLP":17102,"regon":17103,"ĠMPI":17104,"Ġanchor":17105,"aca":17106,"ør":17107,"Ġade":17108,"anchor":17109,"quee":17110,"ĠTreeNode":17111,"Ġtargeted":17112,"Ġlaid":17113,"ABEL":17114,"vet":17115,"ĠOrigin":17116,"Ant":17117,".');Ċ":17118,"expect":17119,"edReader":17120,"ĠMajor":17121,"Ġinch":17122,"Compar":17123,"Ġpreview":17124,"Ġillness":17125,"ĠCONTRACT":17126,"ĠIndepend":17127,"uuid":17128,"Ġnome":17129,"Ġtc":17130,"ĠAvenue":17131,"isan":17132,"Ġphrase":17133,"_move":17134,"\")[":17135,"Ġprovision":17136,"Ġconcentr":17137,"_IR":17138,"ĠUt":17139,"()+":17140,"Ġnas":17141,"!,":17142,"ĠRobin":17143,"iations":17144,"atitude":17145,"Ġpx":17146,"ĠWithout":17147,"/bash":17148,"ekt":17149,"reement":17150,"Observer":17151,"ĠRegion":17152,"UBLIC":17153,"Ġ{//":17154,"KN":17155,"å·":17156,"GameObject":17157,"å¾":17158,"encoding":17159,"Ġ***":17160,"projects":17161,"Ġtk":17162,"Ġcheese":17163,"EMPL":17164,"aro":17165,"ĠاÙĦ":17166,"Ġconsists":17167,"refresh":17168,"ureau":17169,"ĠScanner":17170,"Ġsoil":17171,"Ġflavor":17172,"DataSource":17173,"Execute":17174,"ение":17175,"Ġshit":17176,"åĪĨ":17177,"Ċ":17419,"Ġsubsequent":17420,"posable":17421,"-fluid":17422,"Ġthorough":17423,"Ġpublicly":17424,"apters":17425,"ĠWilson":17426,"_PRE":17427,"yard":17428,"ä¼":17429,"ĉin":17430,"Ġrevers":17431,"Ġbullet":17432,"cribed":17433,"nesota":17434,"Ġ($_":17435,"annon":17436,"cursor":17437,"Ġclothing":17438,"ĠMulti":17439,":',":17440,"Ġvess":17441,"ordinator":17442,"Ġeinem":17443,"Cannot":17444,"Ġarmed":17445,"ĉV":17446,"ä¸Ĭ":17447,".Flat":17448,"ĠSep":17449,"ĠSubject":17450,"_font":17451,"Ġcharacteristics":17452,"Done":17453,"eln":17454,"############":17455,"POS":17456,"Ġdensity":17457,"ĠPlatform":17458,"-items":17459,"Ġovers":17460,"Ġpushing":17461,"ç¤":17462,".Connection":17463,"_term":17464,"Ġinitialization":17465,"________________________________":17466,"ç¬":17467,".document":17468,"lesh":17469,"ĉdocument":17470,"ĠPin":17471,"ça":17472,"Ġdefinitions":17473,".Path":17474,"_WRITE":17475,"ĠĉĊ":17476,"?>ĊĊ":17477,"Ġterrible":17478,"bean":17479,"ickets":17480,"ĠSV":17481,"Buy":17482,"(task":17483,"Ġregime":17484,"google":17485,"Ġcrack":17486,".visit":17487,"NUM":17488,"energy":17489,"Ġstruck":17490,"_sample":17491,".payload":17492,"Ġrevis":17493,"ĠScene":17494,"Ġpg":17495,"Ġbreakfast":17496,"URRENT":17497,".charAt":17498,"_exception":17499,"ĠAnton":17500,"Ġguidelines":17501,"Ġexhaust":17502,"ĠFinancial":17503,"Ġindent":17504,"Ġdesktop":17505,"Hidden":17506,"Failure":17507,"Ġprinciple":17508,"Ġiv":17509,"Ġseks":17510,"network":17511,"ĠnumberOf":17512,"ĠAlbert":17513,"ĉlong":17514,",.":17515,"Ġzeros":17516,"fade":17517,"ĠTyp":17518,"ĠTerm":17519,"ĠArts":17520,".Application":17521,"Ġbehalf":17522,"æĪ·":17523,"Ġmere":17524,"(`${":17525,"Ġawareness":17526,"elpers":17527,"flix":17528,"Ġweigh":17529,"Ġestimates":17530,".child":17531,"/O":17532,"ĠBitmap":17533,".bottom":17534,"Ġ**************************************************************************":17535,"Expect":17536,"ento":17537,"ĠForum":17538,"veral":17539,"Ġjail":17540,"Ġabilities":17541,"ĠHOLD":17542,"ĠCit":17543,"Ġdynam":17544,"Ġgray":17545,"ĉĉĉĉĉĉĉĉĉĉĉĉĉ":17546,".nextInt":17547,"antly":17548,"ĠARISING":17549,"(private":17550,"Ġrejected":17551,"ĠNic":17552,"Ġleather":17553,"={Ċ":17554,"alytics":17555,"thetic":17556,".Top":17557,".Page":17558,"={`":17559,"Ġ;čĊ":17560,"depth":17561,"mann":17562,"WD":17563,"ĠSom":17564,".Right":17565,"Ġ)}Ċ":17566,"Ġtrait":17567,"ÃĹ":17568,"iac":17569,"Ġrv":17570,"Sample":17571,".Xml":17572,"opped":17573,"ĠÑĦ":17574,"lists":17575,"Ġtear":17576,"iversary":17577,".collection":17578,"ĠConstitution":17579,"ĠHttpResponse":17580,"Ġbrill":17581,"ĠProm":17582,"hover":17583,"ĠMiami":17584,"Ġargue":17585,"_float":17586,"ĠãĤ":17587,"Ġnat":17588,"ĠTal":17589,"Ġintegration":17590,"(cur":17591,"Ġremoving":17592,"Ġcoeff":17593,"ĠThough":17594,"Ġforecast":17595,"ĠVegas":17596,"Site":17597,"Ġtrab":17598,"ĠHenry":17599,"-i":17600,"Ġinvolves":17601,"BT":17602,"Ġslo":17603,"Invoke":17604,"Ġlucky":17605,"rat":17606,"Ġ?Ċ":17607,"Ġhandled":17608,"(fd":17609,"contents":17610,"ĠOFF":17611,"RF":17612,"Ġsty":17613,"ĠMotor":17614,"tery":17615,"tax":17616,"MAP":17617,"ĠMrs":17618,"Ġphones":17619,"ĠUIView":17620,"\")));Ċ":17621,"(dev":17622,"ĠIrish":17623,"Ġws":17624,"DI":17625,"_OFFSET":17626,"ĠEvents":17627,"Ġstages":17628,"Ġ}//":17629,"Ġhaben":17630,"STANCE":17631,"ĠSin":17632,"ĠMoney":17633,"(top":17634,"Ġappointment":17635,"VERSION":17636,"metadata":17637,"_comment":17638,"Ġcolleagues":17639,"maps":17640,"âĺ":17641,"ĊĉĊ":17642,"(al":17643,"_req":17644,"Ġfut":17645,"Ġarchitecture":17646,"ĠWHETHER":17647,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":17648,"_screen":17649,"ĠstyleUrls":17650,"Ġmonster":17651,".up":17652,"phia":17653,"Ġprocessor":17654,"ĠTerr":17655,"=',":17656,"ĠManufact":17657,"ĠNT":17658,"kel":17659,"ibern":17660,"ĉfile":17661,"Ali":17662,"rientation":17663,"Ġ//!":17664,"apore":17665,"aneous":17666,"ĠCreat":17667,"folder":17668,"Ġhay":17669,"Suppress":17670,"(left":17671,"Ġeuro":17672,"Ġdisclaimer":17673,"ustry":17674,"ships":17675,"_fd":17676,"ĠFa":17677,"_insert":17678,"Ġrol":17679,"ifting":17680,"ĠComments":17681,"_br":17682,"Ġlosses":17683,"ĠAdded":17684,"charg":17685,"Ġпо":17686,"_system":17687,"ĠSometimes":17688,"ĠSpain":17689,"(group":17690,"ialis":17691,"Ġdollar":17692,"ĠArgs":17693,"quires":17694,"ĠTen":17695,".scss":17696,"Ġsurvive":17697,"usage":17698,"Ġjun":17699,"imiter":17700,"ï¼ģĊĊ":17701,"Ġfifth":17702,"toggle":17703,"Ġdecline":17704,"($\"":17705,"(Long":17706,"inge":17707,"Ġpilot":17708,"-light":17709,"-radius":17710,"Ġpodcast":17711,"Ġnaturally":17712,"Pages":17713,"为":17714,"ĠDespite":17715,"Ġlighting":17716,"Ġcrate":17717,"ĠBinary":17718,"Ġreducing":17719,"Ġeleg":17720,"ĠMouse":17721,"ĠTestBed":17722,"ĠbeforeEach":17723,"_ARRAY":17724,"Redirect":17725,"Ġflood":17726,"Ġships":17727,"Ġelectricity":17728,")*(":17729,"ê¸":17730,"ĠViet":17731,"hero":17732,"Ġdia":17733,"ĠKent":17734,"heart":17735,"Ġthreats":17736,"_acc":17737,"Ġsymbols":17738,"ischen":17739,"_inst":17740,"Criterion":17741,"ĠTIM":17742,".Height":17743,"ĠâĢĻ":17744,"();ĊĊĊ":17745,"Products":17746,"_SP":17747,"ĠCy":17748,"Ġdependent":17749,"este":17750,"Ġdatos":17751,"dit":17752,"ав":17753,"IGNAL":17754,"Ġlesson":17755,"\">'":17756,"ĠCover":17757,"ĠHope":17758,"ĠTimer":17759,"Ġdad":17760,"viders":17761,"ĠPhot":17762,"/?":17763,"ropy":17764,"oming":17765,"asion":17766,"Ġ\\(":17767,"ĠET":17768,"ĠReading":17769,"Ġepisodes":17770,"lm":17771,"echa":17772,"Ġneuro":17773,"Ġharmon":17774,"Ġliberal":17775,"-ind":17776,"DATA":17777,"Ġeveryday":17778,"Ġdivided":17779,"ĠActiveRecord":17780,"figure":17781,"UA":17782,"ä¹":17783,"riendly":17784,"tech":17785,".gameObject":17786,"иÑĤÑĮ":17787,"Ġmoon":17788,"ftime":17789,"Ġnoch":17790,"ĠTORT":17791,"ĠVM":17792,".initial":17793,"(child":17794,"Ġmusical":17795,"Ġoc":17796,"bas":17797,"ĠHay":17798,"_long":17799,"Ġmemset":17800,"iley":17801,"adelphia":17802,"SV":17803,"roat":17804,"_tx":17805,"Ġlon":17806,"ĠngOnInit":17807,"bp":17808,"ĠGolden":17809,"ACHE":17810,"Ġworried":17811,"azi":17812,"Ear":17813,"Take":17814,"(fp":17815,"burgh":17816,"_Data":17817,"gres":17818,"ĠOnt":17819,"pus":17820,"Ġtransparent":17821,"Ġpocket":17822,"Ġram":17823,"igrations":17824,".čĊčĊ":17825,"Ġ[(":17826,"Ġadopted":17827,"Ġreportedly":17828,"ĠDream":17829,"Ġ}));Ċ":17830,"losing":17831,"Ġteeth":17832,"ĠBooks":17833,"\",&":17834,"enny":17835,"LEMENT":17836,"Ġgel":17837,"ĠPlant":17838,"!âĢĿ":17839,".host":17840,"ĠReply":17841,"rength":17842,"Ġrecognition":17843,"Ġ}}>Ċ":17844,"LA":17845,"Ġmirror":17846,"Ġassistant":17847,"(device":17848,"Ġspiritual":17849,"builder":17850,"§":17851,"Ġoutr":17852,"Ġtt":17853,"ĠPER":17854,"Ġradical":17855,"Methods":17856,"Ġpace":17857,"udy":17858,"Ġgut":17859,"ĠGreek":17860,"Ġnonatomic":17861,"ĠPaper":17862,"_GPIO":17863,"Ġobst":17864,".Ad":17865,"vironments":17866,"ĠSov":17867,"(con":17868,"ĠTransaction":17869,".assign":17870,"ĉcatch":17871,"elter":17872,"Ġbitcoin":17873,"_GR":17874,"ĠčĊ":17989,"metic":17990,"Ġtransformation":17991,"åı·":17992,"Ġrgb":17993,"istributions":17994,"Ġimplicit":17995,"/in":17996,"destination":17997,"аÑĤÑĮ":17998,"Zero":17999,"Ġunset":18000,".where":18001,".go":18002,"Ġformation":18003,"Ġdeclaration":18004,"()čĊčĊ":18005,"ĠExpl":18006,"ĉĉĉĠĠ":18007,"/pro":18008,".JSON":18009,"Ġdesk":18010,".substr":18011,"//----------------------------------------------------------------------------":18012,"lyn":18013,"pson":18014,"disable":18015,"ĠFunc":18016,"ĉAssert":18017,"ĠMARK":18018,"Ġdefeat":18019,"Ġblind":18020,"Ġconstants":18021,".headers":18022,"UILD":18023,"Ġexpenses":18024,"Pixel":18025,"Ġhr":18026,"Ġfel":18027,"ĠEastern":18028,"_del":18029,"ĠCub":18030,"Ġsq":18031,"ĉcount":18032,"ĠDirectory":18033,"Ġexclus":18034,"Ġhistoric":18035,"Ġ------------------------------------------------":18036,"Ġcomposition":18037,"ĠdataGridView":18038,"ĠBurn":18039,"ĠBC":18040,"Master":18041,"Ġspawn":18042,"Ġbearing":18043,".SetActive":18044,"ilo":18045,"Ġgallery":18046,"Ġfounded":18047,"Ġavailability":18048,".sqrt":18049,"Ġpes":18050,"ĠDOM":18051,"mate":18052,"Oct":18053,"Ġmatched":18054,"itivity":18055,"Ġanxiety":18056,".price":18057,"ĠInstant":18058,"ìĬ":18059,"Ġtut":18060,"ICollection":18061,".shared":18062,"_sql":18063,"tbl":18064,"library":18065,"_destroy":18066,"ermal":18067,"ĠNotes":18068,"ĠEin":18069,"Ġsouthern":18070,"ĠOTHERWISE":18071,"Ġmacro":18072,".lower":18073,"cls":18074,"ContentView":18075,".link":18076,"constant":18077,"ĠBes":18078,"Ġsomebody":18079,"nb":18080,"\">{":18081,"(local":18082,".....":18083,"ĠNull":18084,"mx":18085,"Ġç":18086,"Ġpause":18087,"-----------":18088,"_MO":18089,"ĠCM":18090,"ĠforKey":18091,"ĠDVD":18092,"Ġclosest":18093,"_DEVICE":18094,"ĠStephen":18095,"ĠBBC":18096,"ĠTravel":18097,"Paint":18098,"ĠResults":18099,"ĠRule":18100,"Ġtp":18101,"Ġratings":18102,"cin":18103,"csv":18104,">/":18105,"ĠGOP":18106,"lad":18107,"ĠÑĢ":18108,"ĠindexPath":18109,"matrix":18110,"=f":18111,"arsed":18112,"Ġ});":18113,"ĠCos":18114,"ĠScore":18115,"Ġtak":18116,"ĠESP":18117,"ĠINC":18118,"_NULL":18119,"-flex":18120,"\"][":18121,"into":18122,"eland":18123,"Authorization":18124,"_FALSE":18125,"Ġgate":18126,"Ġvid":18127,"istent":18128,"TIME":18129,"Ġrewrite":18130,"Ġtie":18131,"Ġarchive":18132,".events":18133,".getParameter":18134,"ĠPermission":18135,"Ġprogramme":18136,"Ġé":18137,"jud":18138,"Ġcameras":18139,"(sys":18140,"ĠSyrian":18141,"Ġimprovements":18142,"Ġhip":18143,"Ġsuicide":18144,"Ġscholar":18145,"Ġcompatible":18146,"remote":18147,".down":18148,"FUNCTION":18149,"Ġmanaging":18150,"ĠUIKit":18151,".raw":18152,">>>>":18153,"Ġdemands":18154,"ellite":18155,"Ġdent":18156,"ĠMicro":18157,"åıĸ":18158,"'][$":18159,"ĠIE":18160,"imension":18161,"Ġtrem":18162,"Ġgained":18163,".with":18164,".ok":18165,"hou":18166,"Ġbom":18167,"ampaign":18168,"Ġjoining":18169,"fish":18170,"ĠaddSubview":18171,"Ġnorthern":18172,".cor":18173,"oret":18174,"Die":18175,"inish":18176,"_comp":18177,"Ġattended":18178,"Ġcollapse":18179,"ĠSS":18180,"acent":18181,"_EQUAL":18182,"ĠDeep":18183,"RGB":18184,"ĉtest":18185,"olves":18186,"uset":18187,"UnityEngine":18188,"writer":18189,"Resolver":18190,",%":18191,"ifference":18192,"_remove":18193,"onda":18194,"Ġfemme":18195,"decode":18196,"Branch":18197,"Ġflush":18198,"Ġinnovative":18199,"Tests":18200,"Ġ['./":18201,"Ġcovering":18202,".admin":18203,"ultipart":18204,"(lambda":18205,"namespace":18206,"ĠSport":18207,"Ġ!(":18208,"acles":18209,"Ġdepression":18210,"ĠKong":18211,"Ġpert":18212,"ĠConn":18213,"ĠOtherwise":18214,"/home":18215,"supported":18216,"Ġpink":18217,"Ġinvited":18218,"ños":18219,"_enabled":18220,"Ġ-Ċ":18221,"FW":18222,"eners":18223,"ĠMY":18224,"Ġsuggestions":18225,"Canvas":18226,"Ġfer":18227,"ĠMarketing":18228,"@Test":18229,"untu":18230,"ĠVen":18231,"ĠCou":18232,"ivals":18233,"Donald":18234,"limited":18235,"ĉĉĉĉĉĉĊ":18236,"Ġanalyst":18237,"(entry":18238,"Ġrepresentative":18239,"_attributes":18240,"Ġfur":18241,".hide":18242,"resp":18243,"adores":18244,"rides":18245,"ĠJosh":18246,"robot":18247,"ĠNAT":18248,"Ġsesso":18249,"Ġintegrated":18250,":true":18251,"parts":18252,"Ġstupid":18253,":event":18254,"@endsection":18255,"Ġpu":18256,".Table":18257,"ĠYii":18258,"`;ĊĊ":18259,"Ġclang":18260,"=\"\">":18261,"engan":18262,"_parameters":18263,".internal":18264,"ĠModern":18265,"Ġmetric":18266,"Ġsemi":18267,"={{Ċ":18268,".amazon":18269,"ĠBB":18270,"ainty":18271,"viewport":18272,"ĠstartActivity":18273,"dispatch":18274,"*****":18275,"Ġflav":18276,"ifferent":18277,"[this":18278,"Ġstake":18279,"Ġargued":18280,"viously":18281,".work":18282,"ĠOak":18283,"Old":18284,"(async":18285,"notes":18286,"Ġflip":18287,"Ġdisag":18288,"ĠTE":18289,"ĉerror":18290,"<'":18291,"Ġ»ĊĊ":18292,"Ġfiltered":18293,"ĠMach":18294,"Ġhung":18295,"_dump":18296,"_samples":18297,"-dismiss":18298,"Ġray":18299,"Implemented":18300,"DK":18301,"Ġjed":18302,"Ġbreaks":18303,"Ġfits":18304,".gr":18305,"ĠZero":18306,"oro":18307,"Ġequally":18308,"Ġ'[":18309,"Ġconcerning":18310,"<":18407,"Ġpromot":18408,"Ġincl":18409,"_only":18410,"를":18411,"ĠAttorney":18412,"-date":18413,"Ġlandscape":18414,"Ġfu":18415,"SY":18416,".prop":18417,"ĠArr":18418,"pag":18419,"ParallelGroup":18420,"':čĊ":18421,"Ġlogs":18422,"aunch":18423,"unci":18424,"nama":18425,"TableCell":18426,"issues":18427,".{":18428,"ecurity":18429,"_exec":18430,"olds":18431,"Ġhosts":18432,"Ġproto":18433,"_import":18434,"_sort":18435,"ĠBow":18436,"ĠNormal":18437,"ĠFarm":18438,".createParallelGroup":18439,"Rotation":18440,".err":18441,"Ġpleased":18442,"itage":18443,".Wh":18444,"ĉĉĠĠĠĠ":18445,"MR":18446,"ĠMORE":18447,"ĠNatural":18448,"_transform":18449,"BASE":18450,"eneral":18451,"utdown":18452,".commons":18453,"WT":18454,"Ġaan":18455,".Result":18456,"dog":18457,"Ġclicking":18458,"),ĊĊ":18459,"#line":18460,"Operator":18461,"Ġciv":18462,"Ġmerg":18463,"obuf":18464,"ngthen":18465,"Ġ[{":18466,"Ġcancell":18467,"trigger":18468,".:":18469,"WORK":18470,"declare":18471,"Ġdecrease":18472,"ÅĽci":18473,"loom":18474,".None":18475,"ĠMI":18476,"ĠJason":18477,"Ġhealthcare":18478,"iamond":18479,"sylvania":18480,"*x":18481,"ĠRa":18482,"[b":18483,"Ġprinting":18484,"phabet":18485,"ĠLabour":18486,"opper":18487,"Ġzijn":18488,"-target":18489,"_FUNCTION":18490,"Ġoct":18491,"ениÑı":18492,"åľ¨":18493,"Ġwestern":18494,"Ġcomputers":18495,"ĠRET":18496,"HashMap":18497,"[String":18498,"getValue":18499,"_DATE":18500,".Next":18501,"ĠFif":18502,"él":18503,"icked":18504,"æİ":18505,"-MM":18506,"Ġ{ĊĊĊ":18507,"Ġcontacts":18508,"Ġdigits":18509,"Produ":18510,"Ġunusual":18511,"Ġrapidly":18512,"tures":18513,"Ġangry":18514,"cancel":18515,"xxxx":18516,"_parser":18517,"idity":18518,"_PREFIX":18519,"Ġmehr":18520,"Ġrarely":18521,"ethe":18522,"opes":18523,"Ġ%.":18524,"works":18525,"Ġtheta":18526,"Ġcontribution":18527,"ĠTony":18528,"Ġsquad":18529,"ай":18530,"Ġîn":18531,"there":18532,"outed":18533,"ĉq":18534,"ĻĤ":18535,"good":18536,"LI":18537,"页":18538,"ĠLiving":18539,"izabeth":18540,"Ġkt":18541,"ĠDallas":18542,"]],Ċ":18543,"Ġ/>ĊĊ":18544,"Ġraising":18545,"/router":18546,"_game":18547,"ĠCUR":18548,"zens":18549,".es":18550,"ĠfontWeight":18551,"(func":18552,"notification":18553,"Ġ'../../../":18554,"Ġblame":18555,"ãĢĤĊĊĊĊ":18556,"anco":18557,"Identity":18558,"follow":18559,"Ġarts":18560,"xs":18561,"Ġofficially":18562,"ĠStudio":18563,"Ġrecommendations":18564,"Ġlocale":18565,"Ġamateur":18566,"ĠEnable":18567,"Ġcaps":18568,".End":18569,"-add":18570,"_gshared":18571,"ĠCT":18572,"Force":18573,"ĊĠĠĠĠĠĠĠĠĠĠĠĠĊ":18574,"Ġorange":18575,"Ġlp":18576,"Ġanswered":18577,".Grid":18578,"Ġdual":18579,"Ġstrategic":18580,"Ġnobody":18581,"Ġfatal":18582,"_est":18583,"(el":18584,"Ġìł":18585,"ĠBudd":18586,"AIT":18587,"_factor":18588,"-one":18589,"ĠHAVE":18590,"\"čĊčĊ":18591,"Prof":18592,"Ġär":18593,"strings":18594,"Ġdirty":18595,"ĠFace":18596,"ĠBegin":18597,"ĠBus":18598,"Ġwis":18599,"åŃĹ":18600,"Ġspeaker":18601,"Ġcarrier":18602,"ĠOm":18603,"Ġhadn":18604,"Allow":18605,"::__":18606,"Ġverb":18607,"ĠComplete":18608,"ĠEasy":18609,"Ġbills":18610,"ĠĠĊĊ":18611,"Vertical":18612,"Ġpron":18613,"ĠDefine":18614,"Ġlookup":18615,"variables":18616,"Ġpandas":18617,"umes":18618,"Ġinnoc":18619,"ĠsetUp":18620,"ĠChampionship":18621,"artist":18622,"ĠCType":18623,"Foundation":18624,"à¹Ī":18625,"ĠSetup":18626,"Ġrecipes":18627,"ĠUIColor":18628,"ĠFight":18629,"Ġauthorized":18630,"_click":18631,"_success":18632,"angan":18633,"ĠMountain":18634,"ĠDoctor":18635,"Ġegg":18636,"ĠMedicine":18637,"cles":18638,"`.Ċ":18639,"[int":18640,"dashboard":18641,"ĠAppro":18642,"-dr":18643,"Ġproduces":18644,"Ġrental":18645,"Ġreload":18646,"Ġarrival":18647,"spot":18648,"Ġundert":18649,"Ġequipped":18650,"Ġproved":18651,"Ġcenters":18652,"Ġdefines":18653,"also":18654,"Ġopacity":18655,"ĠUnfortunately":18656,"ĠIllinois":18657,"Ġне":18658,"ĠTemple":18659,"ĠTrail":18660,"ĠKelly":18661,"Ġmeasurement":18662,"Ġseparated":18663,"-circle":18664,"Hey":18665,"ĠREAD":18666,"igits":18667,"Ġib":18668,"ĠMOD":18669,"attery":18670,"аз":18671,"Ġvend":18672,"енÑĤ":18673,"ĠHttpClient":18674,"safe":18675,"_ASS":18676,"icit":18677,"ĠConstruct":18678,"ĠClo":18679,"ĠSix":18680,"_TOKEN":18681,"(block":18682,"Ġwarned":18683,"/*!":18684,"!Ċ":18769,"Ġinnovation":18770,"_\"":18771,"Ġ);čĊčĊ":18772,"Ġspots":18773,"Ġchoosing":18774,".cs":18775,"Ġflexible":18776,"UInt":18777,"Ġscratch":18778,"-al":18779,"Ġfestival":18780,"Ġoutstanding":18781,"================================================":18782,"Mean":18783,"ĠOregon":18784,"symbol":18785,".account":18786,"dney":18787,"'''":18788,"!\",":18789,"Ġparticle":18790,"Ãĥ":18791,"[MAX":18792,"IVER":18793,"ERENCE":18794,"NSMutable":18795,"ĠColumbia":18796,"_ĊĊ":18797,".fr":18798,"Ġcogn":18799,"VR":18800,"ĠMethods":18801,"ĠMade":18802,"ĠBR":18803,"ĠElse":18804,"Ġeggs":18805,"Ġswing":18806,"ĠInv":18807,"Ġdiseases":18808,"Ġfirms":18809,"Ġlemma":18810,"}`);Ċ":18811,"lings":18812,"Ġgym":18813,"uminum":18814,".Trim":18815,"Mem":18816,"Ġcriticism":18817,"ibernate":18818,"_TX":18819,"ioni":18820,"Ġguidance":18821,"Ġrepeatedly":18822,"Ġsupplier":18823,"Ġpainting":18824,".Fragment":18825,"edException":18826,"Ġwiring":18827,"Ġcourts":18828,"WEB":18829,"æľī":18830,"\\.":18831,"illance":18832,"Ġbrows":18833,"ĠPattern":18834,"PLICATION":18835,"ĠSummer":18836,"Chain":18837,"Ġcute":18838,"mercial":18839,"Ġdil":18840,"ĠFranklin":18841,"ĉglobal":18842,"INCLUDING":18843,"history":18844,"Ġlst":18845,"Qt":18846,"SDL":18847,"alia":18848,"iere":18849,"(...":18850,"ĉcin":18851,"iffs":18852,"velope":18853,"ĠRoot":18854,"cluster":18855,"UserName":18856,"igne":18857,"()Ċ":18949,"Ġapplying":18950,"Ġpromised":18951,"Ġox":18952,"ncia":18953,"ĠValidation":18954,"orts":18955,"_cur":18956,"elect":18957,"eye":18958,"(Data":18959,"Ġreporter":18960,"ĠBuff":18961,"Ġsr":18962,"Ġ\";":18963,"icky":18964,"Ġtempor":18965,"SN":18966,"Ġresident":18967,"pires":18968,"ysical":18969,"Ġendorse":18970,"ĠSong":18971,"isEmpty":18972,"leet":18973,"_util":18974,"Ġdistingu":18975,"ĠTalk":18976,"ĠMot":18977,"(default":18978,".Arg":18979,"gorithms":18980,"_words":18981,"immer":18982,"_reset":18983,"family":18984,"WW":18985,"Ġsavings":18986,"ĠâĢĿ":18987,"_enable":18988,"sidebar":18989,"Running":18990,"Ġali":18991,"Ġtestim":18992,"Ġwarnings":18993,"ĠChem":18994,"ĠExit":18995,"Ġfounder":18996,"pector":18997,"Ġrm":18998,"_dataset":18999,"ĠDas":19000,"Ġhan":19001,"Getty":19002,"ál":19003,"Ġny":19004,"Ġpoverty":19005,"Ġresulted":19006,".by":19007,"ĠVisit":19008,"Ġobtaining":19009,"/'.$":19010,"ĠĠĠĠĠĠĠĠĠĠĠĊ":19011,"shall":19012,"_LEFT":19013,"UIImage":19014,"_Name":19015,"have":19016,"ĠNob":19017,"lr":19018,"-footer":19019,"Ġnaked":19020,"ĠGarden":19021,"\\Facades":19022,"Ġgraduate":19023,"Ġfranchise":19024,"plane":19025,"Ġcontributions":19026,"ĠstringWith":19027,"Ġcrypto":19028,"Ġmovements":19029,"athers":19030,"Ġlifetime":19031,"Ġcommunicate":19032,"jar":19033,"ĠFragment":19034,"_IF":19035,"ĠNavy":19036,"ĠFigure":19037,"Ġsimulation":19038,"_stop":19039,"Ġreporters":19040,"Ġversus":19041,"aja":19042,"Ġα":19043,"Ġgovernor":19044,"ListItem":19045,"Ġsealed":19046,".Background":19047,"edi":19048,"ashing":19049,"Ġlip":19050,"ĠIh":19051,"merge":19052,"Ġnec":19053,"elocity":19054,"ATEG":19055,"Ġseeds":19056,"Ġfloating":19057,"_FA":19058,"walk":19059,"ĉuser":19060,"_depth":19061,"Ġwage":19062,"@app":19063,"Nil":19064,"([\"":19065,"(vector":19066,"Ġsecretary":19067,"ĠjPanel":19068,"vez":19069,"³³³³":19070,"direction":19071,"ĠEP":19072,"Ġhunt":19073,"JsonProperty":19074,"ĠPORT":19075,"]\",":19076,"ап":19077,"ĠForeign":19078,"panic":19079,"Ġtrials":19080,"ĠAle":19081,"Ġrural":19082,"-value":19083,"authorized":19084,"ĠScotland":19085,".drop":19086,"ĠMT":19087,"ç±":19088,"rowth":19089,"FilePath":19090,"Ġrecall":19091,"ifle":19092,"Ġcel":19093,"ĠSELECT":19094,"kn":19095,"_case":19096,"Ġcrop":19097,"sure":19098,"pot":19099,"ICS":19100,"Ġstem":19101,"Ġindustries":19102,"Put":19103,"Ġaber":19104,"roadcast":19105,"Icons":19106,")\")Ċ":19107,"æĪIJåĬŁ":19108,"gui":19109,"Ġassumed":19110,"Ġrx":19111,"EA":19112,"è§":19113,"ELL":19114,"Ġdose":19115,"Ġine":19116,"Ġdeeper":19117,"lider":19118,"Ġordinary":19119,"Ġgolf":19120,"_IMAGE":19121,"ĠNAME":19122,"(module":19123,"Ġatom":19124,"Ġbelt":19125,"Ġoffices":19126,"beta":19127,"Ġphilosophy":19128,"(JSON":19129,"-field":19130,"Ġintroduce":19131,"Ġconvenience":19132,"optim":19133,">\"Ċ":19134,"athy":19135,"Ġemployer":19136,"quate":19137,"Ġedited":19138,"Arguments":19139,"ĠNations":19140,"__)":19141,"Ġnose":19142,"ĠSample":19143,"')ĊĊĊ":19144,"Ġcake":19145,".getAttribute":19146,"HD":19147,"Modified":19148,"Ġpredicted":19149,"ÅĦ":19150,"anie":19151,"Sorry":19152,"(doc":19153,"wind":19154,"ieve":19155,"Ġprovisions":19156,"ATER":19157,"OTE":19158,"MY":19159,".Autowired":19160,"ĠBath":19161,".Boolean":19162,"Ġbackend":19163,".Mouse":19164,"ateral":19165,"paper":19166,"Const":19167,"ĠVR":19168,"_entity":19169,"_CTRL":19170,"ĠProtection":19171,"ĠGM":19172,"ĠStudy":19173,"Ġsoup":19174,"otime":19175,"'use":19176,"]\"":19177,"/users":19178,"aug":19179,"ĠHong":19180,"_norm":19181,"ãģ¨":19182,"Ġsecre":19183,"(Build":19184,"ĠContract":19185,"olas":19186,"Ġsauce":19187,"Ġaggressive":19188,"Ġracial":19189,"character":19190,"@@":19191,"Ġcompile":19192,"ĠVoid":19193,"_rem":19194,"_memory":19195,"kk":19196,"Ġmic":19197,"Same":19198,"Utility":19199,"ĠHtml":19200,"ĠXml":19201,"Ready":19202,"Ġgall":19203,"Ġallegedly":19204,"ĉĉĉĉĠĠĠ":19205,"ĠMetal":19206,"ĠPersonal":19207,"ĠborderRadius":19208,"rxjs":19209,"objects":19210,"Ġwanting":19211,"Ġbowl":19212,"vendor":19213,"offsetof":19214,"ĠRs":19215,"ĠRating":19216,"Ġrally":19217,"_NODE":19218,"ĠMix":19219,"Ġadvertis":19220,"Ġnarrative":19221,"sal":19222,"Ġmc":19223,"SError":19224,"Ġfingers":19225,"Ġaccompany":19226,"Ġtired":19227,"Ġstride":19228,"Ġgui":19229,"elist":19230,"Locale":19231,"Ġreleases":19232,"iking":19233,"Ġanger":19234,")))ĊĊ":19235,"allest":19236,"Summary":19237,"(O":19238,"(for":19239,"Ġbasketball":19240,"Ġroads":19241,"ĠInstall":19242,"ĠFab":19243,"itmap":19244,"Ġ))Ċ":19245,"Ġintersection":19246,"ighbor":19247,"ĠBry":19248,"ĠHERE":19249,"Software":19250,"elfare":19251,"acs":19252,"Ġtrailer":19253,".getClass":19254,"chars":19255,"Ġregulation":19256,"Ġrefers":19257,"Ġdestruction":19258,"Ġcontinuous":19259,"ĠAustin":19260,"é¢":19261,"akan":19262,".window":19263,"ĠTemplates":19264,"Ġabsence":19265,":n":19266,"Ġdisorder":19267,"flash":19268,"Ġdelet":19269,"boards":19270,"ĠĠĉ":19271,"ROP":19272,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":19273,"Ġacqu":19274,"Ġlawsuit":19275,"ĠReviews":19276,"Ġgarage":19277,"timer":19278,"Ġej":19279,"ĠRectangle":19280,"Ġflowers":19281,"ilst":19282,"ĠInstance":19283,"Super":19284,"det":19285,"disposing":19286,"ĠES":19287,"ĠIC":19288,"vere":19289,"Sk":19290,"_channels":19291,"puted":19292,"/null":19293,"nnen":19294,"ĠGallery":19295,"_global":19296,"Authentication":19297,"ĠRank":19298,"Ġblocked":19299,"Ġcalm":19300,"market":19301,"ĉval":19302,"Ġaug":19303,"period":19304,"ĠConstant":19305,"Ġ?>\">Ċ":19306,"Ġlobby":19307,"pal":19308,"Ġsink":19309,"iah":19310,"С":19311,"urname":19312,"Ġconver":19313,"Ġinvestigate":19314,"Christ":19315,"Hub":19316,"ĠIND":19317,"ĠPed":19318,"uras":19319,"ĉurl":19320,"ĠTro":19321,"Ġpreferences":19322,"Ġguaranteed":19323,"`ĊĊ":19324,"Ġportions":19325,"Ġevalu":19326,"'>;ĊĊ":19421,".AutoScaleMode":19422,"Ġcats":19423,"Ġregistry":19424,"ulus":19425,"FI":19426,"payload":19427,"-search":19428,"Ġstaying":19429,"acious":19430,"Decoration":19431,"Review":19432,"Inf":19433,"Keep":19434,"itis":19435,",String":19436,"Coord":19437,"Ġpero":19438,"Sex":19439,"ĠAtlanta":19440,"uesta":19441,"Argb":19442,">*":19443,"}_":19444,"Footer":19445,"Ġemployed":19446,"_bound":19447,"vide":19448,".func":19449,"$scope":19450,"Ġspo":19451,"ĠAnal":19452,"ounced":19453,"around":19454,"Ġrestriction":19455,"Ġshops":19456,"åĢ":19457,"ĠLatin":19458,"-col":19459,"Ġbarely":19460,"ĠEuro":19461,"Er":19462,"Ġfaire":19463,"_distance":19464,"_unlock":19465,"Quote":19466,"IVATE":19467,"ĠåĪ":19468,"Ġaimed":19469,"ĠRetrie":19470,".iter":19471,"Ġwrapped":19472,"Ġagreements":19473,"strument":19474,"(product":19475,"Ġstudied":19476,".setValue":19477,"Ġye":19478,"ĠCache":19479,"MBOL":19480,"Ġquarterback":19481,"Ġsyntax":19482,".getElementsBy":19483,".version":19484,"website":19485,"Runner":19486,"_single":19487,"ativ":19488,"ĠAltern":19489,"ĠBeautiful":19490,"rightarrow":19491,"Ġdiversity":19492,"plash":19493,"(co":19494,".Fill":19495,"Ġtyping":19496,"Ġclar":19497,"Hit":19498,"OO":19499,"acco":19500,"worth":19501,"Ġscripts":19502,"ĠMuslims":19503,"ĠLL":19504,"erving":19505,"(boolean":19506,"Ġbaseball":19507,"ĠCAN":19508,"MAIL":19509,"depend":19510,"Ġrespective":19511,"Ġconstexpr":19512,".*;ĊĊ":19513,"']))Ċ":19514,"Ġyard":19515,"Ġidentical":19516,"ifecycle":19517,"USH":19518,"upiter":19519,".validate":19520,"cli":19521,"ISTER":19522,"Indicator":19523,"Fail":19524,"Ġdemocracy":19525,".var":19526,"Ġsatisfied":19527,"-------------":19528,"encer":19529,"hor":19530,"Ġrounds":19531,"DAO":19532,"oa":19533,"Ġflask":19534,"=c":19535,"[]Ċ":19536,"/dist":19537,"Ġparte":19538,"Ġconfirmation":19539,"eron":19540,"aware":19541,"":19542,"Ġdependencies":19543,"ĠVideos":19544,"-row":19545,"Ġ**/Ċ":19546,"Ġnou":19547,"Ġhover":19548,"æŀ":19549,"Ġnin":19550,"ĠUSD":19551,"Mac":19552,"_Load":19553,"Ġoutcomes":19554,"_socket":19555,"Ġqueries":19556,"wm":19557,"Ġhitting":19558,"inux":19559,"Mich":19560,"udge":19561,"ATAB":19562,"Ġvulnerable":19563,"ä¾":19564,"Ġportfolio":19565,":YES":19566,"ĉmap":19567,"Bound":19568,"Ġiteration":19569,"incess":19570,"Ġactors":19571,"ĠQual":19572,"_clean":19573,"ãĢijãĢIJ":19574,"MSG":19575,"Green":19576,"ĠOfficer":19577,"Ġsmoking":19578,">',":19579,"ĠFlo":19580,"++;":19581,"olygon":19582,"Ġbulk":19583,"Ġdrama":19584,"Ġexceptions":19585,"osed":19586,"Ġ+čĊ":19587,"Ġlegacy":19588,"CV":19589,"Ġcontributed":19590,"ĠTerms":19591,"Ġbt":19592,"Ġuntuk":19593,"Ġalien":19594,"===Ċ":19595,"ĉVector":19596,"Ġls":19597,"Online":19598,".facebook":19599,"numeric":19600,"ockets":19601,"Aut":19602,"bury":19603,"-redux":19604,"ĠRedistributions":19605,"GLOBALS":19606,"urrencies":19607,"Ġtons":19608,"âĢĻ,":19609,"Ġê":19610,"(col":19611,"ĠSymbol":19612,"Ġstayed":19613,"ĠML":19614,"Ġmunicip":19615,"Ġsexo":19616,"Sen":19617,"nr":19618,"Ġgains":19619,"Ġshortly":19620,".Menu":19621,"ý":19622,"KNOWN":19623,"Ġoperators":19624,"-V":19625,"ĠPatrick":19626,"/add":19627,"_CO":19628,"iration":19629,"(post":19630,"Posts":19631,"/_":19632,"Ġplug":19633,"Ġintellectual":19634,"Ġmetab":19635,"Ġpregnancy":19636,"ĠPremier":19637,"nm":19638,"Ġprediction":19639,"ĠMinistry":19640,"Three":19641,"valuate":19642,"ĠMini":19643,"bu":19644,"оз":19645,"\";čĊ":20078,"ĠSav":20079,".Bold":20080,"Ġenables":20081,"ĉtmp":20082,"Ġmanually":20083,"ĠSqu":20084,"userid":20085,".function":20086,".cache":20087,"LOPT":20088,".Services":20089,"ddit":20090,"tim":20091,">>":20154,"station":20155,"lore":20156,"atype":20157,"ishop":20158,"/****************************************************************":20159,"ComboBox":20160,"Ġvacation":20161,"Ġinitiative":20162,"ĠdefaultValue":20163,"concat":20164,"ĠKh":20165,"ĠWelcome":20166,"izedName":20167,"Migration":20168,"Ġgradient":20169,"Hot":20170,"Ġhardly":20171,"elo":20172,"ĠStudents":20173,"Ġloose":20174,"atz":20175,".Send":20176,"'/":20177,"Ġuniversal":20178,"Ġenterprise":20179,"Ġregex":20180,"Ġvisitor":20181,"ĠFly":20182,"Seq":20183,"à¸Ļ":20184,"ĠVisual":20185,"Ġlibraries":20186,"atoes":20187,"Payment":20188,"Ġpent":20189,"Ġgathered":20190,"VRTX":20191,"ĠDM":20192,"Split":20193,"Ġletting":20194,"ÐĿ":20195,"_errors":20196,"epoch":20197,"PARAM":20198,"cu":20199,"ÑģÑĤв":20200,"olutions":20201,"Editing":20202,"fonts":20203,"Ġallocated":20204,"ĠBased":20205,"(Y":20206,"ĠJudge":20207,"Ġbrothers":20208,"FILES":20209,"ço":20210,"wb":20211,"_PI":20212,"'^":20213,"Ġsword":20214,".services":20215,"Ġnl":20216,"Tim":20217,"igg":20218,"ĠMoore":20219,"Ġcryptoc":20220,"åĩº":20221,"_posts":20222,"otate":20223,"?'":20224,"....ĊĊ":20225,"Ġkl":20226,"=\"$":20227,"Ġdecoration":20228,"ạ":20229,"ĠDIRECT":20230,"GUI":20231,")=>{Ċ":20232,"Ġnewsletter":20233,"Ġprecis":20234,"(point":20235,"ĠEquipment":20236,"uty":20237,"ĠDave":20238,"Ġparticipation":20239,"uarios":20240,"xit":20241,".As":20242,"ETER":20243,"orous":20244,"Ġshield":20245,"[]>":20246,"ilitary":20247,".origin":20248,"Ġpromotion":20249,"Unt":20250,"Ġct":20251,"TRA":20252,"ViewHolder":20253,"Ġsigma":20254,"delta":20255,"arehouse":20256,"contract":20257,"(Vector":20258,"Ġcompete":20259,"/form":20260,"/components":20261,"Ġnr":20262,"ĠIndones":20263,"ĠоÑĤ":20264,"ĠVolume":20265,".files":20266,"(resp":20267,"/models":20268,"Ġsurf":20269,"standard":20270,"/o":20271,"ĠXCTAssert":20272,"VICES":20273,".Code":20274,"SED":20275,"Ġactivate":20276,"Delta":20277,"Ġlimitation":20278,"rij":20279,"Ġpregnant":20280,":^(":20281,"Ġsour":20282,"pie":20283,"Ġexpense":20284,"ication":20285,"ĠLarge":20286,"Ġ±":20287,"ĠBowl":20288,"(models":20289,"/N":20290,"Pa":20291,".reload":20292,"Ġwondering":20293,"Execution":20294,"ĉĠĠĠĠĠĠ":20295,"ĠGraphics":20296,"ĠContin":20297,"_job":20298,"ĠgetName":20299,"ĠMagn":20300,"ĠDWORD":20301,"mad":20302,"Ġnh":20303,"features":20304,"}\");Ċ":20305,"heets":20306,"(train":20307,"zn":20308,"Ġrecruit":20309,".connection":20310,"Ġbarrel":20311,"Ġsteam":20312,"_setting":20313,"Ġangular":20314,"aneously":20315,"Ġbil":20316,"ĠNorm":20317,"(!$":20318,"ibt":20319,"%(":20320,"Ġposit":20321,"ĠFather":20322,"intendo":20323,"Live":20324,"Ġports":20325,"Ġmej":20326,"Ġlanding":20327,"ponder":20328,"Ġcod":20329,"_HEADER":20330,".Margin":20331,"Ġballs":20332,"Ġdiscussions":20333,"Ġblend":20334,"Hex":20335,"Ġfarmers":20336,"Ġmaintaining":20337,"ĠĠĠčĊ":20338,"syn":20339,"[T":20340,"rus":20341,"uffers":20342,"Ġcontributors":20343,"_sys":20344,".Debug":20345,"Ġconstructed":20346,"omes":20347,"?id":20348,"slider":20349,"Ġsuppliers":20350,"scriber":20351,"pes":20352,"Ðŀ":20353,"\":čĊ":20354,"\\Controller":20355,"))ĊĊĊ":20356,"Ġlua":20357,"Multi":20358,"ENS":20359,"Src":20360,"Ġpetition":20361,"Ġslave":20362,"looking":20363,"VERT":20364,"ĉvector":20365,"Special":20366,"hh":20367,"anne":20368,"ĠNiger":20369,"/views":20370,"zing":20371,"endant":20372,"(":20591,".Product":20592,"Forms":20593,"NEW":20594,"Pay":20595,"ĉboolean":20596,"_contact":20597,"ĠElectric":20598,"skip":20599,"Ġwur":20600,"Ġchronic":20601,"_driver":20602,"ĠSab":20603,"ĠUlt":20604,"ĠRad":20605,"STATUS":20606,"ĠLewis":20607,"OB":20608,"Ġgifts":20609,".Rec":20610,"TRUE":20611,"Ġintensity":20612,"Marker":20613,".compare":20614,"ffic":20615,"Cookie":20616,"ĠBaby":20617,"ĠBigDecimal":20618,"ilet":20619,"ĠHOLDERS":20620,"ĠLady":20621,"Ġlung":20622,"ĠAlabama":20623,"Ġdess":20624,"`);Ċ":20625,"ĠBuilder":20626,"_region":20627,"Ġneutral":20628,"Both":20629,"Ġhp":20630,"Ġhorn":20631,"Ġsegments":20632,"ĠEC":20633,"\"=>\"":20634,"(rec":20635,"ĠPi":20636,"GM":20637,"Ġlaptop":20638,"Scalar":20639,"isd":20640,"-dialog":20641,"ĠAnderson":20642,"Ġmistakes":20643,"ĠHan":20644,"jes":20645,"estination":20646,"Ġpromises":20647,"bid":20648,"ĠScient":20649,"GIN":20650,"ĠPerformance":20651,"bage":20652,".users":20653,"leading":20654,"Ġoral":20655,"Graphics":20656,"_PTR":20657,"hang":20658,"Ġinev":20659,"processing":20660,"Factor":20661,"ĠNA":20662,"$string":20663,"Ġgrounds":20664,".SaveChanges":20665,"clock":20666,"cripcion":20667,"ĠNewton":20668,"gc":20669,".includes":20670,"Ġblast":20671,"Ġ'-'":20672,"Ġpuede":20673,".Session":20674,"Ġgrep":20675,"_final":20676,"ĠGay":20677,"ĠGive":20678,"iri":20679,"-star":20680,"ĠUIImage":20681,"_epoch":20682,"ubb":20683,"enth":20684,"Ġelite":20685,"Ġcampaigns":20686,"ĠPorno":20687,"_assign":20688,"Protocol":20689,"ĠBeing":20690,"ĠAirport":20691,"Ġconventional":20692,"ĠWat":20693,"ĠCI":20694,"ETA":20695,"ĠAnthony":20696,"Ġtablet":20697,"(format":20698,"Ġconsistently":20699,"ĠIowa":20700,"Ġavatar":20701,".cursor":20702,"![":20703,"Ġhanging":20704,"Her":20705,"Such":20706,"';ĊĊĊ":20707,"orgeous":20708,"()==":20709,"ĠviewModel":20710,"Ġãĥ":20711,"Ġels":20712,"ĠAgent":20713,"Fetch":20714,"apor":20715,"Ġcx":20716,"pread":20717,"ĠPier":20718,"oeff":20719,"Sn":20720,"ĠVirtual":20721,"Apr":20722,".White":20723,"_MOD":20724,"ĠPoints":20725,"失":20726,"Ġgenes":20727,"Ġvendor":20728,"Ġmainstream":20729,"Ċ":20758,"Filename":20759,"Ġsne":20760,"ĠFootball":20761,"Ġrival":20762,"Ġdisaster":20763,"ionic":20764,"ĠDamage":20765,".Resource":20766,"-en":20767,"ĠTypes":20768,"getString":20769,"(board":20770,"Ġbol":20771,"plain":20772,"zym":20773,"า":20774,"Ġscanner":20775,"ilder":20776,"_msgs":20777,"æı":20778,"(intent":20779,"Ġdestruct":20780,"Ġbust":20781,"ĠEmploy":20782,"oni":20783,"ĠUIViewController":20784,"Ġodds":20785,"earer":20786,"Geometry":20787,"Ġyii":20788,"_EXPORT":20789,"ĠAttack":20790,"Ġniet":20791,"Ġimpression":20792,"ĠGil":20793,"_prob":20794,"ĠCF":20795,"ĠExperience":20796,"/plugins":20797,".Method":20798,"Ġbeliefs":20799,"Native":20800,"_build":20801,"Ġvig":20802,"Ġranks":20803,"covered":20804,"such":20805,"Guard":20806,".pack":20807,"adder":20808,"ivia":20809,"lng":20810,"ĠвÑĭ":20811,"Timestamp":20812,"_now":20813,"Ġpoker":20814,"Ġunc":20815,"Ġshapes":20816,"-types":20817,"_period":20818,"pk":20819,"Ġveteran":20820,"Ġsono":20821,"Ġappointed":20822,"overflow":20823,".driver":20824,"_cat":20825,"utt":20826,"plant":20827,"imb":20828,"ĠAccept":20829,"Ġconcert":20830,"ĉnode":20831,"ĉz":20832,"?>čĊ":20833,"Ġbanned":20834,"ĉĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":20835,"Ġtoxic":20836,"Ġdisappe":20837,"ÈĽ":20838,"Ġgrace":20839,"ateful":20840,"Reply":20841,"ĠCruz":20842,"Ġscrap":20843,"Ġkeywords":20844,"simp":20845,"Ġmortgage":20846,"Ġcyber":20847,"ĠExecute":20848,"Ġlatitude":20849,"ifu":20850,".COM":20851,"dbo":20852,"Ġsorts":20853,"ĠGas":20854,"omial":20855,".Local":20856,"Cells":20857,".Replace":20858,"Strings":20859,".fit":20860,"ĠThird":20861,"%\",Ċ":20862,"Ġ{}\".":20863,"ĠSony":20864,"Ġ[:":20865,"Ġfallen":20866,".')Ċ":20867,"inh":20868,"ĠMC":20869,"Ġredis":20870,"Codes":20871,"Ġprofiles":20872,"hook":20873,"Reducer":20874,"_FUNC":20875,"Ġnavigate":20876,"strlen":20877,"Ġhorm":20878,"áŀ":20879,"ĠSR":20880,".boot":20881,"Ġdigest":20882,"ĉheader":20883,".findOne":20884,"æģ":20885,"DbType":20886,"nia":20887,"_merge":20888,"Ġdonne":20889,"/Getty":20890,"_CHAR":20891,"Ġbands":20892,".URL":20893,"artial":20894,"Ġfreq":20895,"Ġsist":20896,"Ng":20897,"Ġrendering":20898,"\\Core":20899,"Widgets":20900,"ĠVA":20901,"Ġactivists":20902,"Ste":20903,"=_":20904,"alla":20905,"Stamp":20906,"Ġloads":20907,"Ġxx":20908,"ĠLearning":20909,".Mvc":20910,"uir":20911,"(\"$":20912,"Ġconnecting":20913,"ReadOnly":20914,"uru":20915,"ĠEag":20916,"BIT":20917,"_DEL":20918,"å§":20919,"arrass":20920,"external":20921,"ĠYOUR":20922,"ĠBrew":20923,"ĠFive":20924,"Ġresize":20925,"igid":20926,"eration":20927,"ĠÑį":20928,"åĬł":20929,"ĠCatch":20930,"Ùģ":20931,"ĠLeon":20932,"amil":20933,".Body":20934,"Clip":20935,"/list":20936,".br":20937,"EditText":20938,"ĉdb":20939,".Game":20940,"(BuildContext":20941,"backend":20942,".Red":20943,"facebook":20944,".urls":20945,"mr":20946,"rolled":20947,"-------":20948,"Ġintervention":20949,"Ġretirement":20950,"ĠKit":20951,"ĠPRE":20952,"UpperCase":20953,"ĠSocket":20954,"Ġ:-":20955,"Ġstudying":20956,"ĠMetro":20957,"arded":20958,"Ġconversations":20959,"Called":20960,"Ġexamine":20961,"ertificate":20962,".gz":20963,"-responsive":20964,"Ġrefund":20965,"_network":20966,"allowed":20967,"empt":20968,"Ġmeals":20969,"Categories":20970,"Ġtraveling":20971,"Ġkg":20972,"Ġshame":20973,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":20974,"Ġexplicitly":20975,"Ġmathematic":20976,"ĠSuite":20977,"ĠRGB":20978,"******/":20979,"Ġmixture":20980,"learning":20981,".template":20982,"atts":20983,"wx":20984,"ĉctx":20985,".properties":20986,"Ġdrinks":20987,"ĠEither":20988,"setText":20989,".getData":20990,".zip":20991,"Ġreveals":20992,".Ċ":21005,"Ġranked":21006,"_impl":21007,"ĠHandles":21008,"Ġhosted":21009,"Ġupdating":21010,"album":21011,"éĿ":21012,"Ġshader":21013,"Editors":21014,"-round":21015,"[]{":21016,"Ġsep":21017,"ĠHi":21018,"TEM":21019,"lookup":21020,".man":21021,"_INPUT":21022,"Ġthreatened":21023,"_IMPORT":21024,"Ġdrops":21025,"ruit":21026,"sid":21027,"both":21028,"ĠExcel":21029,"Ġjer":21030,"ordinary":21031,"ей":21032,"VIEW":21033,"reply":21034,"Ġ):Ċ":21035,"colors":21036,"verified":21037,"_Tr":21038,"_parse":21039,"Ġcongress":21040,"Promise":21041,"ints":21042,"ĠMother":21043,".Api":21044,"ĠDuration":21045,"ĠfirstName":21046,"inheritdoc":21047,"ĠMars":21048,"Ġapr":21049,"ODY":21050,"Ġvisits":21051,"Ġhealing":21052,"letters":21053,")));čĊ":21054,"future":21055,".Framework":21056,"Ġkiss":21057,"Ġinvolve":21058,"Ġsilent":21059,"adows":21060,"Ġanybody":21061,"sch":21062,"Ġsolely":21063,"-img":21064,"Ġpropri":21065,"Ġinstruct":21066,"Ġlicenses":21067,"Ġmeth":21068,"Ġcondem":21069,"ĠDomain":21070,"ĠHarris":21071,"ĠsÃ¥":21072,"CEPT":21073,"Batch":21074,"@extends":21075,"ĠCONTRIBUT":21076,".DataFrame":21077,"_packet":21078,"recision":21079,"Ġfocusing":21080,".ht":21081,"__\":Ċ":21082,":Get":21083,"ĠKC":21084,"Ġpassage":21085,"Segment":21086,"_center":21087,"-zA":21088,"_BL":21089,"Ġconvin":21090,"Ġclassified":21091,"ĠNSMutable":21092,"_ap":21093,"tile":21094,"Rectangle":21095,"(nums":21096,"vens":21097,"ĠUIButton":21098,"ĠFeder":21099,"amo":21100,"Ġoutline":21101,"ĠParser":21102,"Ġâī":21103,"ĠWorks":21104,".Schema":21105,"Ġengines":21106,"_common":21107,"_old":21108,"ĠsetContentView":21109,"Ġ///<":21110,"ĠBT":21111,"fm":21112,"Ġdivers":21113,"_weights":21114,"emark":21115,"ĠACT":21116,"Ġproportion":21117,"overlay":21118,".dirname":21119,"ĠGit":21120,"_REFERENCE":21121,"<>":21122,"lb":21123,"_rule":21124,"è´¥":21125,"ĠPutin":21126,"Ġsleeping":21127,"():čĊ":21128,"Ġpreserve":21129,"Ġparliament":21130,"ĠLooking":21131,"Ġpicking":21132,"ĠDispatch":21133,"Ġslip":21134,"ëĵ":21135,"ĠLyn":21136,"_signal":21137,"configuration":21138,"ĠPitt":21139,"aden":21140,"procedure":21141,"Ġenthusi":21142,"fight":21143,"ĠConsider":21144,"Ġtorn":21145,"Connected":21146,".cos":21147,"_groups":21148,"ĠThink":21149,"Ġdeliber":21150,"Ġresid":21151,"working":21152,".columns":21153,"ĠCalled":21154,"Ġeslint":21155,">\",":21156,"_DOWN":21157,"hist":21158,"ĠAdvanced":21159,"Ġrewards":21160,"actors":21161,"Ġsilence":21162,"Ġmyth":21163,"Ġneur":21164,"Ġauction":21165,".GetString":21166,"eks":21167,"(project":21168,"ĉmsg":21169,"ĉoutput":21170,"Ġcomplaints":21171,",S":21172,"Ġtbl":21173,"Ġ,ĊĊ":21174,"riors":21175,"ahren":21176,"Ġlawyers":21177,"redux":21178,"_symbol":21179,"offee":21180,"_RESULT":21181,"(Name":21182,"UTC":21183,".currentTime":21184,"Ġorganis":21185,".arg":21186,"Ġminim":21187,"wick":21188,"Ġreceives":21189,"Balance":21190,"Ġspeaks":21191,"ĠDays":21192,"ĠBelow":21193,"tipo":21194,"Present":21195,"Ġreserv":21196,"hp":21197,"Ġrit":21198,"_RIGHT":21199,"--)":21200,"Ġchairman":21201,"DIS":21202,"ĠBOOST":21203,"Ġexperiments":21204,"__);Ċ":21205,"Ġstamp":21206,"Ġfert":21207,"Ġfond":21208,"Ter":21209,"elve":21210,"uren":21211,"+i":21212,"endency":21213,"Ġvirtually":21214,"...\"":21215,"ï½ŀ":21216,"-cent":21217,"_unique":21218,"Ġpricing":21219,"mic":21220,"RESH":21221,"Ġ:::":21222,"Ġannotation":21223,"ĠCircle":21224,"ongodb":21225,"itas":21226,"Ġ%(":21227,"(component":21228,"Ġоб":21229,"(port":21230,"-hour":21231,".obj":21232,"LBL":21233,"Ġjury":21234,"GBT":21235,"Ġspy":21236,"ĠProfessional":21237,"Ġ\"\";ĊĊ":21238,"Ġstriking":21239,"Ġdiscrimination":21240,"Ġpays":21241,"lict":21242,"entes":21243,"Ġthrowing":21244,"ĠPlugin":21245,"(def":21246,"ĠRuntimeException":21247,"ĠMigration":21248,"Ġdic":21249,"bag":21250,"onia":21251,"Ġcorruption":21252,"(Map":21253,"Ġprz":21254,".dto":21255,"Ġacquire":21256,"StateToProps":21257,"Ġloving":21258,"ож":21259,"_pattern":21260,"Ġemotions":21261,"Ġpublisher":21262,"_be":21263,"Ġcouples":21264,"oj":21265,"ĠChart":21266,"Ġtrop":21267,".tool":21268,"Ġestablishment":21269,"Ġdol":21270,"Ġtower":21271,"Ġlane":21272,"ĠSydney":21273,"Ġfilling":21274,"claimed":21275,"Ġdialogue":21276,"Ġconvention":21277,"booking":21278,"parency":21279,"æ±":21280,"ĠGeneric":21281,"\\Schema":21282,"Ġranges":21283,"/ch":21284,"Ġpanels":21285,"Ġruled":21286,"çĶŁ":21287,".ts":21288,"_sets":21289,"Ġcleanup":21290,"Previous":21291,"ĠAnimal":21292,"($(":21293,"ĠAve":21294,"ollar":21295,"_eval":21296,"ĉName":21297,"(tree":21298,"Ġ\"]":21299,"Ġduties":21300,"='/":21301,"Clicked":21302,"Ġdifferently":21303,"ĠClark":21304,"Ġdit":21305,"ologists":21306,"Ġsynd":21307,"Ġsends":21308,"-known":21309,"kb":21310,"ĠModal":21311,"itative":21312,"Ġracing":21313,"Ġhighlights":21314,"ĠSimon":21315,"ĠCaptain":21316,"ä¿¡":21317,"ĠCB":21318,"contin":21319,"aran":21320,"Ġphysics":21321,"retty":21322,"etal":21323,".md":21324,"axios":21325,"Ġspeakers":21326,"Ġprep":21327,"Ġawarded":21328,"ì§Ģ":21329,"ĠCorn":21330,"ĠNature":21331,"UDIO":21332,"Ġproj":21333,"-pre":21334,"[u":21335,"Features":21336,"ĠisEqual":21337,"Binary":21338,"sig":21339,"Ġconfusion":21340,"ĠHat":21341,"Ġktó":21342,".configure":21343,"MON":21344,"/edit":21345,"_Add":21346,",true":21347,"Ġcli":21348,"ErrorMessage":21349,"-loader":21350,"Dimensions":21351,"ultiply":21352,"Ġ{!!":21353,"ĠSqlCommand":21354,"Ġspoken":21355,"Ġpics":21356,"Ġtoy":21357,"(Key":21358,"ĠLoop":21359,"ب":21360,"EATURE":21361,"inction":21362,"_setup":21363,"wrapper":21364,"Ġtong":21365,"cular":21366,"Opt":21367,".Pl":21368,"=\",":21369,"(length":21370,"umn":21371,"Ġchrom":21372,"Ġsevent":21373,"ĠIllegalArgumentException":21374,"ĉstart":21375,"Ġbegun":21376,"CEPTION":21377,"dataset":21378,"ĠFailed":21379,"cols":21380,"Ġknee":21381,"imore":21382,".splice":21383,"shell":21384,"iggers":21385,"Ġthemes":21386,"ĠDJ":21387,"ĠAssistant":21388,"-$":21389,"Maybe":21390,"Ġordering":21391,"ĠIntelligence":21392,"ĠMassachusetts":21393,"Ġfailing":21394,"elson":21395,"Great":21396,"=i":21397,".rest":21398,"Ġinvite":21399,"-disable":21400,".GroupBox":21401,"âĢĻest":21402,"Ġtackle":21403,"gv":21404,"etter":21405,"Ġ),čĊ":21406,"_rules":21407,".warn":21408,"functions":21409,"ĠChristians":21410,"Ġbacked":21411,"Ġslider":21412,"Ġenjoying":21413,"nest":21414,"Ġhij":21415,"_ms":21416,"//*":21417,"Annotations":21418,"ĠVariables":21419,"":21620,"cycle":21621,"ĠBull":21622,"paths":21623,"Ġunp":21624,"ĠviewDidLoad":21625,"_Model":21626,"ĠassertTrue":21627,"Ġrated":21628,"Decl":21629,"verted":21630,"ĠDat":21631,"brew":21632,"Ġpointing":21633,"Ms":21634,"ĠPointer":21635,")'":21636,"_non":21637,"ĠSEC":21638,"Ġyeah":21639,"gency":21640,"initialize":21641,"fly":21642,"[pos":21643,",g":21644,"Tele":21645,"Ġjoke":21646,"Ġclause":21647,".findById":21648,"enes":21649,"(instance":21650,"£":21651,"Ġslic":21652,"_home":21653,"Ġ*/}Ċ":21654,"_pages":21655,"(service":21656,"RP":21657,"ĠAmong":21658,".getCurrent":21659,"ãĤ¹":21660,"Ġslee":21661,"=[Ċ":22071,"oler":22072,"Ġlibert":22073,"Ġ`Ċ":22074,"Ġwenn":22075,"lated":22076,"Ġimmune":22077,"(Node":22078,"ĠProblem":22079,"ĠAbs":22080,"logs":22081,"Ġ../":22082,"ĠADC":22083,"Ġ}}\">Ċ":22084,">');Ċ":22085,"=b":22086,"ĠWind":22087,"lahoma":22088,"Ġallocate":22089,"orian":22090,"Ġprescription":22091,"-quality":22092,"ĠMayor":22093,"inely":22094,"endforeach":22095,"ĠComplex":22096,"kom":22097,"TY":22098,"]].":22099,".Style":22100,"_many":22101,"','$":22102,"Ġbarrier":22103,"ĠFetch":22104,"ĠMarvel":22105,"Ġresist":22106,"ого":22107,"bidden":22108,"ĠRunnable":22109,":false":22110,"Ġbuilds":22111,"ĠStage":22112,"Ġdub":22113,"empo":22114,".site":22115,";ĊĊĊĊ":22116,"ĠDenver":22117,"Ġrevel":22118,"Ġtriggered":22119,"Ġdice":22120,"_fail":22121,"Ġgc":22122,"ĉX":22123,"ĠThrowable":22124,".router":22125,"ĠRevolution":22126,"ÑĢа":22127,"_NON":22128,"Ł¥":22129,"Ġelder":22130,"Ġabroad":22131,"Ġе":22132,"ĠAdult":22133,"blr":22134,"glyphicon":22135,"Ġpromoting":22136,"Ġiz":22137,"ĠSolid":22138,"_loader":22139,"early":22140,".enabled":22141,"-edit":22142,"ĠUL":22143,"_play":22144,"ĠInterrupt":22145,"Ġadvantages":22146,"ucle":22147,"Ġmechanical":22148,".tableLayoutPanel":22149,"ĠWorking":22150,"Ġanonymous":22151,"Rating":22152,"igious":22153,"_phone":22154,".addActionListener":22155,"Ġfran":22156,"unden":22157,"Ġ*)&":22158,"_bool":22159,"ulative":22160,"Ġcone":22161,"ĠMult":22162,"Ġmö":22163,"ĠForward":22164,"]):Ċ":22165,"Ġconvinced":22166,"acted":22167,"ãģĵ":22168,"ĠConfigure":22169,"Ġceiling":22170,"Der":22171,"Ġpassengers":22172,"Groups":22173,"Ġsoccer":22174,"/W":22175,"aviors":22176,"swith":22177,"ĠZone":22178,".Options":22179,"ĠMom":22180,"ieder":22181,"Arrays":22182,"Ġtreatments":22183,"Ġprotecting":22184,"fac":22185,"Ġpickle":22186,"ButtonItem":22187,"Ġblocking":22188,"strar":22189,"ò":22190,"ĠExport":22191,"Ġthrew":22192,"otta":22193,"ĠBASE":22194,".ws":22195,".LEADING":22196,"orderBy":22197,"_delay":22198,"ĠPu":22199,".dll":22200,"ĠChoose":22201,"Police":22202,"ĠBEGIN":22203,"boxes":22204,"Ġdiamond":22205,",l":22206,"Ġĉĉĉ":22207,"Ġcurious":22208,"tv":22209,"Ġerotische":22210,"ackages":22211,"ĉSet":22212,"Tick":22213,".border":22214,"staticmethod":22215,"Ġcher":22216,"invoice":22217,"Ġcru":22218,"Ġdefect":22219,"_metadata":22220,"relation":22221,"ikan":22222,"[N":22223,"(Qt":22224,"(Base":22225,"æģ¯":22226,"beat":22227,"ĠEmpty":22228,"ĉo":22229,"_shift":22230,"Ġregret":22231,"Those":22232,"Cent":22233,"ĠPortug":22234,"ĠIslands":22235,"ĠTIME":22236,"Management":22237,"-sp":22238,"ême":22239,"Ġnotion":22240,"unifu":22241,"PK":22242,"è¡Į":22243,"ĠCURLOPT":22244,"\\\"\\":22245,"UV":22246,"çº":22247,"dra":22248,"cou":22249,"=`":22250,"ĠDestroy":22251,"rp":22252,".cancel":22253,"GG":22254,"runtime":22255,"ĠVue":22256,"Ġprogressive":22257,"/services":22258,"Ġrunner":22259,"_FRAME":22260,".ToolStripMenuItem":22261,"Ġ','":22262,"delay":22263,"=utf":22264,"Ġscreening":22265,"Ġpulling":22266,"omas":22267,"Ġanth":22268,"-new":22269,"/local":22270,"ĠiPad":22271,"Ġtwitter":22272,"Ġdying":22273,"Ġheaven":22274,"ĠUInt":22275,"ĠSenator":22276,"Ġpresum":22277,"ĠWalker":22278,"Ġovercome":22279,"etection":22280,"Ġembarrass":22281,"China":22282,"Include":22283,"ROLL":22284,"ĠdataType":22285,"David":22286,"ร":22287,"lop":22288,"-month":22289,"Ġscar":22290,"ĠSafe":22291,"Ġ****************************************************************":22292,"Ġaccessories":22293,"Ġramp":22294,"_USE":22295,"Ġcontrad":22296,"))]Ċ":22297,"Ġprest":22298,"ĠHR":22299,"ĠRap":22300,"Ġusize":22301,"Ġcapability":22302,"Ġcort":22303,"-next":22304,"Ġburden":22305,"_reader":22306,"Ġ@@":22307,"regular":22308,"ĠKa":22309,"MAN":22310,"Ġastr":22311,"Ġ'')Ċ":22312,"Ġfed":22313,"Ġparsing":22314,"ĠYears":22315,"Ġbroker":22316,"\":{\"":22317,"Ġakt":22318,"Inventory":22319,"abeled":22320,"Ġargparse":22321,"*******Ċ":22322,"versation":22323,"Ġcord":22324,"ĠTi":22325,"Ġhopefully":22326,"Ġah":22327,"verb":22328,"Ġstolen":22329,".Entry":22330,"Ġexpecting":22331,"Orientation":22332,"Ġpowered":22333,"Ġpersist":22334,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":22335,"']);":22336,"')),Ċ":22337,"ĠCash":22338,"ĉitem":22339,"grades":22340,"ropol":22341,"basic":22342,"Ġ\");čĊ":22343,"Ġawards":22344,"(range":22345,"-all":22346,"ĠIBOutlet":22347,"ĠIndeed":22348,"----------------------------------------------------------------------------":22349,"Ġstomach":22350,"Ġflower":22351,"Ġsew":22352,"_times":22353,"avis":22354,"QString":22355,"ĠRoutes":22356,"_prot":22357,"Ġcomedy":22358,"Ġlogout":22359,"Ġwooden":22360,"Ġposter":22361,"piece":22362,".Join":22363,"ĠPok":22364,"celona":22365,"mutex":22366,";čĊčĊčĊ":22367,"Ġstrikes":22368,"Loaded":22369,")arg":22370,"esa":22371,"United":22372,"Ep":22373,"PELL":22374,"ĠAtlantic":22375,"ullet":22376,"apple":22377,"Ġsettled":22378,"acon":22379,"Ġprinter":22380,"ĠGC":22381,"å®ļ":22382,"Ġrendered":22383,",âĢĻ":22384,"heit":22385,"social":22386,".ge":22387,"ĠRick":22388,"ĠUtah":22389,"got":22390,"onical":22391,"ĠScroll":22392,"ĠSciences":22393,"Ġjug":22394,"Ġampl":22395,"enti":22396,"LEFT":22397,"Ġtabs":22398,"Ġenormous":22399,".getKey":22400,"locate":22401,".EX":22402,".storage":22403,".We":22404,"Ġtoast":22405,"ĠAdditionally":22406,"ĠNOW":22407,"_UPDATE":22408,"Ġtransferred":22409,"tha":22410,".Display":22411,"_ui":22412,"IDEO":22413,"Ġmeaningful":22414,"ĠMoscow":22415,",this":22416,"ĠVictoria":22417,"æĶ¹":22418,"ĠÐŁ":22419,".stack":22420,"ĠBarn":22421,"paredStatement":22422,":string":22423,"Ġbij":22424,"ĠSTATE":22425,"Ġemployers":22426,"ĉinput":22427,"(|":22428,"Ġlex":22429,"invoke":22430,"ĉnum":22431,"++,":22432,"atial":22433,"orses":22434,"Ġfork":22435,"_txt":22436,"ĠAntonio":22437,"Ġ(<":22438,"averse":22439,"Ġdevast":22440,"ãĢĢ":22441,".Dec":22442,"ĠGard":22443,"/ui":22444,".%":22445,"tri":22446,"Ġrolled":22447,"ValuePair":22448,"itten":22449,"ĠTher":22450,"Ġvrou":22451,"ĠFlow":22452,"ĠFinance":22453,"ĠComb":22454,"HC":22455,".setVisible":22456,"isl":22457,"Ġpk":22458,"Ġupset":22459,"(raw":22460,"ĠVice":22461,"eatures":22462,"ĠLang":22463,"Looking":22464,"ĠAST":22465,"Ġtrips":22466,"ĠJustin":22467,"browser":22468,"=\"'.$":22469,".vertices":22470,"-co":22471,"}/{":22472,"Ġ?,":22473,"ĠDomin":22474,"ĠBelg":22475,"\"<":22476,"Ġsuppose":22477,"addy":22478,"Ġwalks":22479,"ERRU":22480,"_filters":22481,"Preferred":22482,"scene":22483,"еÑģ":22484,"ĠAffairs":22485,"Ġ\"#{":22486,"ĠonSubmit":22487,"Ġstocks":22488,"/view":22489,"gree":22490,"-get":22491,"hit":22492,"Jo":22493,".getC":22494,"Initialized":22495,"ÑĤи":22496,"cuts":22497,"(Type":22498,"ĠAgreement":22499,"ĠVietnam":22500,"Ġ/*!":22501,"Ġpizza":22502,"-view":22503,"_em":22504,"Ġlhs":22505,"Ġmuy":22506,"ĠIdent":22507,"ĠFriends":22508,"Ġabund":22509,"_AD":22510,".timestamp":22511,"-'":22512,"Ġduplicate":22513,"Ġhunting":22514,"Ġregulatory":22515,"iao":22516,"amous":22517,"ĠEntertainment":22518,"[A":22519,"iatric":22520,"_CLIENT":22521,"ĠKids":22522,"/pkg":22523,"Break":22524,")));ĊĊ":22525,"ĠShape":22526,"Ġrelating":22527,"Interrupt":22528,"ableOpacity":22529,"embre":22530,"Ġmystery":22531,"Ġjournalists":22532,"ritable":22533,".Link":22534,"Ġstopping":22535,"CRET":22536,".DB":22537,"Ġpopularity":22538,"Ġgew":22539,"Ġimpr":22540,"setValue":22541,"FLAG":22542,"ĉmax":22543,"Ġbake":22544,"wy":22545,"ĠEconomic":22546,"Ġencontr":22547,"Ġfname":22548,"/de":22549,"Rank":22550,"Ġbugs":22551,".sm":22552,"Ġmedian":22553,"DOWN":22554,"ĠSure":22555,"AtIndex":22556,"ĠDick":22557,"Ġ(__":22558,".delta":22559,"Fr":22560,"Ġsuggesting":22561,"ĠRecyclerView":22562,",e":22563,"START":22564,"/****************************************************************************":22565,"xford":22566,"Ġreceipt":22567,"CLAIM":22568,"readonly":22569,"Ġengaging":22570,"Ca":22571,"asma":22572,"Ġensuring":22573,"English":22574,"ĠVancouver":22575,"hyth":22576,"Ġpurchasing":22577,"ĠPI":22578,".word":22579,"(sp":22580,".home":22581,":def":22582,"Ġgig":22583,"ĠVe":22584,"forum":22585,"ĠMitch":22586,"Bay":22587,"_FL":22588,"Ġsoll":22589,"_columns":22590,"Ġminority":22591,"bird":22592,"Ġhanded":22593,"SSL":22594,"STAT":22595,"Ġnervous":22596,"ĥ½":22597,"ĠfilePath":22598,"CREATE":22599,"Aw":22600,"Ġpens":22601,"seed":22602,"ĠCompute":22603,"olk":22604,"ĠAsset":22605,"reach":22606,"'),čĊ":22607,"navigation":22608,"LF":22609,"/util":22610,"ĠPub":22611,"ĠâĶ":22612,"cion":22613,"##Ċ":22614,"III":22615,"TagName":22616,"Ġamid":22617,"permission":22618,"ifiable":22619,"xFFFFFFFF":22620,"ни":22621,".Buffer":22622,"_irq":22623,"dark":22624,"Ġretval":22625,".fire":22626,"production":22627,".listen":22628,"ĠWeather":22629,"Ġbuyers":22630,".ne":22631,"erp":22632,"ĠPent":22633,"Ġwelfare":22634,"ĠpageSize":22635,"ĠStadium":22636,"erta":22637,"Ġlev":22638,"ampa":22639,"Pager":22640,"Ġcharging":22641,"ĠNetflix":22642,"|null":22643,"_random":22644,".xpath":22645,"Ġstere":22646,"ĠISIS":22647,"ponses":22648,"(loc":22649,"eyond":22650,"ĠOfficial":22651,"ĠMaryland":22652,"DataType":22653,"_par":22654,"{},":22655,"ĠEnjoy":22656,"_SHIFT":22657,"ĠAwards":22658,"_ENTRY":22659,"Ġseemingly":22660,"enticate":22661,"Ġhearts":22662,"_;ĊĊ":22663,"ĠHIV":22664,"Ġindivid":22665,"ĠFlag":22666,"_ctrl":22667,"ĠCallback":22668,",z":22669,"ĠGPU":22670,"ĉobj":22671,"ĠPhoenix":22672,"ĠBUS":22673,"Ġrubber":22674,"_AUTH":22675,"ĠSolutions":22676,"(location":22677,"Variables":22678,".setEnabled":22679,"_high":22680,"WO":22681,"Gesture":22682,"Ġretry":22683,"ĠobjectForKey":22684,"alloween":22685,"Ġmos":22686,"ĠCele":22687,"Ġikke":22688,"(cell":22689,"ĠMODE":22690,"rena":22691,"Ġdescribing":22692,"Ġphi":22693,"Ġrd":22694,"Ġdeserve":22695,"Ġwheels":22696,"å¸Ĥ":22697,"Ġcritics":22698,"Namespace":22699,"ĠFra":22700,"ĠĊĊĊĊ":22701,"Ġalla":22702,"Ġrequiring":22703,"æľŁ":22704,"utation":22705,"Ġdelayed":22706,"Ġadministrative":22707,"Ġbay":22708,".hidden":22709,"Tex":22710,"Ġboundaries":22711,"Ġ]);ĊĊ":22712,"ĠFollowing":22713,"~/":22714,"Fi":22715,"_conv":22716,"_TITLE":22717,"Ġdesde":22718,"ICollectionView":22719,"Alias":22720,"Ġbite":22721,"patient":22722,"_COMMAND":22723,"Completed":22724,"ĉelif":22725,"(<":22726,"Business":22727,"ĠPool":22728,"Ġpursue":22729,"ĠBan":22730,"_steps":22731,"_DECL":22732,"umble":22733,"Ġcombo":22734,"ĠLayer":22735,".xr":22736,"Ġdup":22737,"---------":22738,"Ġmodifier":22739,"rob":22740,"rez":22741,"Ġathletes":22742,"Used":22743,"wear":22744,"Ġlegitimate":22745,"Ġ\"ĊĊ":22746,"Ġhv":22747,"Std":22748,"ĠHold":22749,"Ġsurviv":22750,"ĠAlliance":22751,"ĠEarly":22752,"Behavior":22753,"(font":22754,"/libs":22755,"Ġrectangle":22756,"Ġsinger":22757,"Ġamp":22758,"EqualTo":22759,"Ġ\".\"":22760,"Ġgirlfriend":22761,"å±":22762,"linear":22763,"observ":22764,"Ġpiù":22765,"Ġcomplement":22766,"WithValue":22767,"(password":22768,"take":22769,"Blank":22770,"ĠCompar":22771,"'\",":22772,"_policy":22773,"mongoose":22774,"_FAILED":22775,".report":22776,"Ratio":22777,".PerformLayout":22778,"usable":22779,"mers":22780,"_render":22781,"PEED":22782,"Ġlesb":22783,"ĉE":22784,"_tool":22785,"Ġladies":22786,"оÑģ":22787,"))))Ċ":22788,";;;;":22789,".dot":22790,"Ġnest":22791,"peak":22792,"ukkit":22793,"eca":22794,"_SW":22795,"Ġ&(":22796,"ĠOklahoma":22797,"Ġbanking":22798,"ĠNintendo":22799,"Ġreproduce":22800,"_elements":22801,"_mac":22802,"proxy":22803,"Ġremarkable":22804,"}/${":22805,"Ġouts":22806,".hasNext":22807,"MODE":22808,"Ġanime":22809,".conn":22810,"Unique":22811,"Dom":22812,"Ġimportantly":22813,"itty":22814,"Ġjuice":22815,"Tw":22816,"ĠPartners":22817,"Ġattacking":22818,"Ġportable":22819,"amiento":22820,".PictureBox":22821,".gen":22822,"Ġoptimal":22823,"Ġrecre":22824,"Ġjournalist":22825,"ĠExtract":22826,"ĠMoreover":22827,"ĠmarginTop":22828,".Ap":22829,"Ġfiring":22830,"NaN":22831,"ĉtemplate":22832,"ад":22833,".En":22834,"Ġdefence":22835,"ĠTel":22836,"ilen":22837,"jan":22838,"=data":22839,"ĠUrl":22840,"ĠReuters":22841,"(total":22842,"ĠFifth":22843,"Ġessays":22844,"Ġinterpretation":22845,"Ġcharity":22846,"ĠRules":22847,"Ġsubsection":22848,"styled":22849,"azer":22850,"lags":22851,"LIST":22852,"Ġuploaded":22853,"Ġtrash":22854,"Ġregistr":22855,"Ġseller":22856,">';čĊ":22857,"ĠstartTime":22858,"çĻ":22859,"sy":22860,"(HttpServletRequest":22861,"Ġtrap":22862,"GC":22863,"Ġembedded":22864,"Ġsurrounded":22865,"imits":22866,"TX":22867,"ylinder":22868,"ĠFal":22869,"Ġsentences":22870,"ĠJa":22871,"IFICATION":22872,"weapon":22873,"ovation":22874,"Ġcoat":22875,"Ġinterpol":22876,"Ġlips":22877,"ĠKy":22878,"Ġvectors":22879,"_am":22880,"Ġintake":22881,".world":22882,"Ġinbox":22883,"ĠMAC":22884,"_ab":22885,"(nameof":22886,"Ġentert":22887,"Ġgathering":22888,"ĠSIM":22889,"++.":22890,"nya":22891,"'}}":22892,"ĠUPDATE":22893,"Ġpac":22894,"(html":22895,"ĠSant":22896,"iating":22897,"ĠIdeas":22898,"Ġspray":22899,"ĠHart":22900,"Ġverification":22901,"adesh":22902,"/modules":22903,"ĠMind":22904,"ĠSizedBox":22905,"Ġshelter":22906,"Ġheroes":22907,"atty":22908,"Ġcertified":22909,"sj":22910,"Ġêtre":22911,"ÅĤo":22912,"Ġpublishing":22913,"ĠMalays":22914,".getUser":22915,"ĠProvider":22916,"ĠLinkedList":22917,"ĠBor":22918,"ROUND":22919,"did":22920,"tain":22921,"pire":22922,"ĠJenn":22923,"tel":22924,"ande":22925,"_front":22926,"ĠMcG":22927,"TestMethod":22928,"à¸Ń":22929,"Ġoccasionally":22930,"ĠWales":22931,"Ġexercises":22932,"ĠÐĴ":22933,"-plus":22934,"Ġvalidator":22935,"Ġprayer":22936,"LATED":22937,"_author":22938,"Ġlabour":22939,"++Ċ":22940,"-equiv":22941,"ĠGPL":22942,"Ġfacebook":22943,"simple":22944,"gly":22945,"Processor":22946,"ipy":22947,"Ġ*>":22948,"Ġcleared":22949,"ĠPush":22950,"Ġpenis":22951,"Structure":22952,"lij":22953,"ĠMorgan":22954,"Ġhandful":22955,"\".Ċ":22956,"|\\":22957,"Ġ********************************":22958,"ĠAqu":22959,"_IC":22960,".loads":22961,"Ġmeter":22962,"ĠMarine":22963,"::{":22964,"ĠTS":22965,"ĠArrays":22966,".Title":22967,"GRAM":22968,"termin":22969,"Ġcoinc":22970,"Else":22971,"_states":22972,"-run":22973,"members":22974,"astro":22975,"ĠonPress":22976,"Ġbeings":22977,"Ġabandoned":22978,"Ġtaxp":22979,"owners":22980,".mode":22981,"Ġdiagnosis":22982,"Ġ_Ċ":22983,"ĠKnight":22984,"ĉA":22985,"Ġobserve":22986,"),'":22987,"!\")Ċ":22988,"ĠPara":22989,"Ġvariation":22990,"(False":22991,"ĠAnti":22992,"Ġgri":22993,"Ġhomeless":22994,"?v":22995,"Ġbez":22996,".Server":22997,"release":22998,"ĠPatri":22999,"Ġchars":23000,"Ġranking":23001,"activation":23002,"Ġwides":23003,"qr":23004,".Sql":23005,"acular":23006,"ĠBot":23007,"_sync":23008,"Ġhappiness":23009,"Ġvolunteers":23010,"Ġsits":23011,"/<":23012,"[e":23013,"(fileName":23014,"Ġcapac":23015,"ĠMaria":23016,"father":23017,"Ġgram":23018,"*i":23019,"Ġcaso":23020,"_draw":23021,"ĠRaw":23022,"ĠIterator":23023,"ĠPadding":23024,"PD":23025,"BOX":23026,"ĠSPECIAL":23027,"Ġfecha":23028,"Ġvide":23029,"ĠLeader":23030,"以":23031,"$(\".":23032,"Ġdiameter":23033,"Ġmild":23034,"Ġrocks":23035,"appings":23036,"directory":23037,".flush":23038,"ĠJess":23039,"UNIT":23040,"ĠPear":23041,"Ġmandatory":23042,"Sur":23043,"qt":23044,"Ġstreams":23045,"Ġcooperation":23046,"ĠSac":23047,"Ġcheaper":23048,"ĉch":23049,"animation":23050,"fare":23051,"(height":23052,"(True":23053,"NY":23054,"Ġwrest":23055,"Ġpolls":23056,"Ġencountered":23057,"ĠMarketable":23058,"_PASSWORD":23059,"_SELECT":23060,"ĠArabia":23061,"_clock":23062,"Ġvoy":23063,"Ġиз":23064,"Ġstir":23065,"isible":23066,"-effect":23067,".created":23068,"Ġtoys":23069,"ĠTradable":23070,"Ġrust":23071,"Ġstrcpy":23072,"_timestamp":23073,"Ġtalented":23074,",null":23075,"ĠJobs":23076,"ĠPortland":23077,"Ġweakness":23078,"Throw":23079,"ĠAngel":23080,"ä¿®":23081,"Ġuncert":23082,"ï¼īĊ":23083,"ĠìĿ´":23084,"Which":23085,"Ġ[-]:":23086,"Something":23087,"Ġconvicted":23088,"kle":23089,"edium":23090,"Ġbranches":23091,"Ġbases":23092,"ç®":23093,"Ġcomplexity":23094,"ĠFig":23095,".reshape":23096,"$db":23097,"_CONST":23098,"ĠTes":23099,".runtime":23100,"Ġdeny":23101,"ĠBSD":23102,"Ġkr":23103,"hatt":23104,"ĠStatic":23105,"Ġuniversities":23106,"Replace":23107,"Ġdrove":23108,"Ġadoles":23109,"_plugin":23110,"ĠLGBT":23111,"Ġtex":23112,"duction":23113,"EDI":23114,"ĠTed":23115,"_URI":23116,"Ġreception":23117,"arten":23118,".Single":23119,"rice":23120,"scious":23121,"_bg":23122,"Ġwages":23123,"ĠServlet":23124,"UILayout":23125,"Ġformatted":23126,".Mod":23127,"',Ċ":23174,"Ġexpanding":23175,"ĠHamilton":23176,"ĠContrib":23177,".Tables":23178,"Activ":23179,"HH":23180,"ocommerce":23181,"_;":23182,"Ġamongst":23183,"owing":23184,"ĠCold":23185,"APH":23186,"Ġpsychological":23187,"_tensor":23188,"Ġpackaging":23189,"ĠSweden":23190,"Ġpare":23191,"Ġaggregate":23192,"Ġmoderate":23193,"_hand":23194,"Ġdesignated":23195,"Ġdrum":23196,"ĠgetUser":23197,"ĠCreek":23198,"_scope":23199,"ĠTransfer":23200,"ĠMarg":23201,"Ġfighters":23202,"Wnd":23203,"ĠSel":23204,"ĠLaunch":23205,"Ġemerging":23206,"iframe":23207,"ĠAdditional":23208,"Ġfears":23209,"Ġsatellite":23210,"_:":23211,"Ġdisposing":23212,"GetValue":23213,"HttpPost":23214,"ATIVE":23215,"ulary":23216,"Views":23217,"Ġattending":23218,"ĠTennessee":23219,"ĠMission":23220,"Ġmedication":23221,"ĠWy":23222,"ĠAnna":23223,"ع":23224,"ĠVertex":23225,".types":23226,"Organ":23227,".DataGridViewTextBoxColumn":23228,"ĠRS":23229,"Ġtempo":23230,"(App":23231,"VersionUID":23232,".point":23233,"ĠDutch":23234,"Hours":23235,"LU":23236,"Ġquoted":23237,".builder":23238,"ĠPerfect":23239,"ĠAlways":23240,"_two":23241,"Ġexclusively":23242,"ĠCra":23243,"ificar":23244,"ĠAWS":23245,"ingham":23246,"complex":23247,"kernel":23248,"Ġgravity":23249,"Ġwi":23250,"Ġoverview":23251,"ĠWant":23252,"ĠWP":23253,"(sh":23254,".rotation":23255,"States":23256,"ĠTeen":23257,"_components":23258,"ìĪĺ":23259,"Received":23260,"Ġlyrics":23261,"rites":23262,"ĉĉĉĉĉĠ":23263,"-American":23264,"[num":23265,"/python":23266,"ĠUART":23267,"Ġapple":23268,"ĠJonathan":23269,"Ġmomentum":23270,"ั":23271,"Ĥ¹":23272,"Ġmich":23273,"andra":23274,"Ġbiological":23275,"ĠMens":23276,"Ġ%%":23277,"elsea":23278,"ĠMexican":23279,".randint":23280,"Ġtale":23281,"ĠValidate":23282,"Ġdefeated":23283,".htm":23284,"Ġcopper":23285,"=/":23286,"cosystem":23287,"Ġrip":23288,"decimal":23289,".VISIBLE":23290,"ĠTa":23291,"ĉĉĉĉĉĉĉĉĉĉĉĉĉĉ":23292,"Ġdownloaded":23293,"environment":23294,"Ġnomine":23295,"building":23296,"ĠSpot":23297,"ipheral":23298,"Ġalto":23299,"quet":23300,"ĠFT":23301,"/get":23302,"/master":23303,"WIN":23304,"åħĥ":23305,"West":23306,"argc":23307,"Ġproducers":23308,"ĠMuch":23309,"_storage":23310,"credit":23311,"CONT":23312,"Ġvet":23313,"Ġvoices":23314,"('',":23315,"Ġinstruments":23316,"ĠMSG":23317,"esse":23318,"repository":23319,"omics":23320,"Ġdealer":23321,"Still":23322,"Ġbanner":23323,"ascii":23324,"Ġremarks":23325,"[js":23326,"Ġshorter":23327,"gulp":23328,"Ġmyster":23329,"Ġkun":23330,"ĠBird":23331,"Ġtiene":23332,"nut":23333,"ĠUm":23334,"Ġwise":23335,"Yeah":23336,"INESS":23337,"_begin":23338,"-heading":23339,"Course":23340,"ĠčĊčĊ":23341,"ombie":23342,"graded":23343,"ĠGPS":23344,"Ġże":23345,"Fit":23346,"caption":23347,"ön":23348,"/image":23349,"lia":23350,"(mod":23351,"Ġleak":23352,"enza":23353,"/H":23354,"ĠHappy":23355,"Dist":23356,"nx":23357,"ĠGovernor":23358,"(last":23359,"teacher":23360,"ĠSent":23361,"support":23362,"jectory":23363,"ĠÙħ":23364,"Registration":23365,"ĠGray":23366,",false":23367,"Ġadjusted":23368,"(settings":23369,"'Ċ":23431,"-fold":23432,"æĬ":23433,"ĠBetter":23434,"Ġ\"\\<":23435,"spacing":23436,"Ġfurnished":23437,"oser":23438,"]}Ċ":23439,"Ġ$\"":23440,"pull":23441,".Post":23442,"(ip":23443,"Ĺı":23444,".front":23445,"nte":23446,"ĠFM":23447,"guid":23448,"Ġnegotiations":23449,"agonal":23450,"Ġtremend":23451,"ungeon":23452,"Adv":23453,"carousel":23454,"ÃŁe":23455,"_DESC":23456,"Ġhammer":23457,"áºŃ":23458,"ĠĠĠĠĠĠĠĠĊĊ":23459,"-core":23460,"-service":23461,"Ġcorners":23462,"ĠSF":23463,"pred":23464,">A":23465,"ĠJLabel":23466,"Ġromantic":23467,"Ġtestimony":23468,"osc":23469,"ĠGeneration":23470,"asures":23471,"_internal":23472,"Ġprints":23473,"Ġ])Ċ":23474,"ĠCleveland":23475,"repo":23476,"Disc":23477,"Ġ\">Ċ":23478,"����":23479,"Ġnearest":23480,"_tb":23481,"(require":23482,"EOF":23483,"-child":23484,"Ġbudd":23485,".XtraEditors":23486,"alties":23487,"\\\":\\\"":23488,"Words":23489,"Ġlocally":23490,"Ġpurchases":23491,"Drawer":23492,"extract":23493,"Ġexecut":23494,"}'.":23495,"userdata":23496,"Ġfocuses":23497,"-minute":23498,"ĠPublish":23499,"ogo":23500,"Ġmountains":23501,"Bot":23502,"}>{":23503,"Ġtension":23504,"rod":23505,"mesh":23506,"Ġtransformed":23507,",R":23508,"()}Ċ":23509,".long":23510,"Ġgorgeous":23511,"ĠSchedule":23512,"Ġoldest":23513,"Ġsubprocess":23514,"(IN":23515,"yect":23516,"ĠCooper":23517,"arness":23518,"ĠMonitor":23519,".part":23520,"ĠNBC":23521,"Ġcotton":23522,"Ġhol":23523,"Ġrgba":23524,"ĠBio":23525,"Continue":23526,"Pod":23527,"Ġparticipating":23528,"clusions":23529,"(ByVal":23530,"ì":23531,"ĠHOW":23532,"_setopt":23533,"Ġaccompanying":23534,"aton":23535,"Ġ/\\":23536,"ĠAuthentication":23537,"ién":23538,"ĠBarack":23539,"/*.":23540,"Ġeager":23541,"ĠCancel":23542,"$":23586,"OLEAN":23587,"OKIE":23588,"IBILITY":23589,"UAGE":23590,"ĠSurvey":23591,"Ġresign":23592,"wing":23593,"Ġsecrets":23594,"Ġchips":23595,"JSONObject":23596,"Desktop":23597,"_SYMBOL":23598,"(resource":23599,"ĠĊ":23600,"Ġnewest":23601,"uli":23602,"Ġdesert":23603,"Ġdip":23604,"ĠPow":23605,"Ġequation":23606,"Ġpossibilities":23607,"ĠFed":23608,"osph":23609,"Ġ[%":23610,"Ġbubble":23611,"etherlands":23612,"Ġcement":23613,".auto":23614,"_AN":23615,"âĢĻ.":23616,"selection":23617,"ĠBond":23618,"Den":23619,"-O":23620,".getType":23621,".Window":23622,"pres":23623,"Ġswinger":23624,"\"})Ċ":23625,"Ġpip":23626,"Ġmice":23627,"Ġcompound":23628,"-plugin":23629,"iko":23630,"Ġcenturies":23631,"icular":23632,"-inline":23633,"ĉkey":23634,">\\<":23635,"ENSION":23636,"Ġ[čĊ":23637,"Ġprecisely":23638,"Ġété":23639,"ĠPast":23640,"ĠCambridge":23641,"-full":23642,"Ġanalyze":23643,"ĠSteven":23644,"Ġnem":23645,"due":23646,"oren":23647,"Ġmuscles":23648,"ijing":23649,"/-":23650,"ĠKennedy":23651,"RM":23652,"ossible":23653,"Ġactress":23654,"Ġdolor":23655,"å½ķ":23656,"Need":23657,".toggle":23658,"ĠRace":23659,"wers":23660,".material":23661,"ĠDue":23662,"ĠPel":23663,"#print":23664,"Ġindependence":23665,"exus":23666,"Shadow":23667,"Ġencoder":23668,"(level":23669,"ĠSwift":23670,".doc":23671,"_selection":23672,"ĠserialVersionUID":23673,"Labels":23674,"Ġperformances":23675,".Tag":23676,"ĠNHL":23677,"izen":23678,"/UIKit":23679,"_CONTROL":23680,"Ġearnings":23681,"ĠAlt":23682,"_HANDLE":23683,"Ctx":23684,"Ġpersu":23685,"Ġtran":23686,"ç¨":23687,"_CHANNEL":23688,"Ġsatisfaction":23689,"ĠGP":23690,"iox":23691,"mitt":23692,"lando":23693,"Ġpig":23694,"inals":23695,"ência":23696,"Surface":23697,"ĠUUID":23698,"Ġbeneficial":23699,"Ġsequences":23700,"ĉmemset":23701,"Ġmagical":23702,"«":23703,"Ġworn":23704,"ASC":23705,"popup":23706,"COMP":23707,"_before":23708,"eness":23709,"Ui":23710,"Les":23711,".require":23712,".Serializable":23713,"addGap":23714,"Ġauthorization":23715,".pyplot":23716,"urray":23717,"latitude":23718,"frames":23719,"ajs":23720,"Ġcompass":23721,"Ġobservations":23722,"_sup":23723,".environ":23724,"Ġtriple":23725,"ĠRuby":23726,"Ġdrain":23727,"_FILTER":23728,"San":23729,"UMP":23730,"NullException":23731,"ĠGab":23732,"owe":23733,"ĠTurkish":23734,"_sequence":23735,"ĠGrant":23736,"uela":23737,"Ġwo":23738,"Ġcube":23739,"iq":23740,"Ġdisorders":23741,"Ġextraordinary":23742,"Ġctrl":23743,"ĠSeq":23744,"entr":23745,"Ġsanctions":23746,"utsch":23747,"Reports":23748,"Ġinherit":23749,"Period":23750,"Ġphotography":23751,"ĠFramework":23752,"Ġspecialist":23753,"Ġ?ĊĊ":23754,"_selected":23755,".Player":23756,"Ġallocation":23757,"(account":23758,"Ġstructural":23759,"vable":23760,"-offset":23761,".AppCompatActivity":23762,"ам":23763,".AddWithValue":23764,"Ġicons":23765,"Ġshutdown":23766,"_low":23767,"ĠCompare":23768,"ĠCe":23769,"=head":23770,"lam":23771,".predict":23772,"_DEC":23773,"ĠSleep":23774,"ĠGratis":23775,"Ġsuggestion":23776,"ĠDEL":23777,"caff":23778,"avirus":23779,"Nothing":23780,"ŀĭ":23781,"Ġwidespread":23782,"Ġmechanisms":23783,"ĠtextAlign":23784,"occup":23785,"ĠRail":23786,":NS":23787,"Ġfiber":23788,"Ġmk":23789,"Ġvintage":23790,"-long":23791,".reduce":23792,".Entities":23793,"(record":23794,"Ġpleasant":23795,"FRING":23796,".Cells":23797,"OTT":23798,"ĉelseif":23799,"_confirm":23800,"ĠViewGroup":23801,"sym":23802,"Ġpray":23803,"Ġsuspected":23804,"Contains":23805,"Ġborders":23806,"ĠcomponentDid":23807,"ASSERT":23808,"Ġinfinite":23809,"-order":23810,"Ġhello":23811,"ĠGrade":23812,".currentTimeMillis":23813,"apolis":23814,"zh":23815,"ĉObject":23816,":\\\\":23817,"HO":23818,"valuation":23819,"Ġvocab":23820,"Ġcoupon":23821,"atabases":23822,".GetType":23823,"Learn":23824,"]=\"":23825,"ĠGary":23826,"otive":23827,"Ġash":23828,"Ġbib":23829,"XXXX":23830,"Ġbalanced":23831,"VALUE":23832,"ĠNat":23833,"_Ad":23834,"<":23976,"Ġfool":23977,"Ġesk":23978,".Null":23979,"ĠDies":23980,"_OUTPUT":23981,"_TYPED":23982,"Ġpainted":23983,"Ġsophistic":23984,"ĠBear":23985,"*n":23986,"_PACK":23987,"Ġdelivering":23988,"ĠCOUNT":23989,"åįķ":23990,"Ġjeg":23991,"-car":23992,"fname":23993,"Ġranging":23994,"ĠNeg":23995,"/******/":23996,"ĠCHAR":23997,"Ġultra":23998,"Grad":23999,"=t":24000,"Ġjudges":24001,"ĠDise":24002,"anners":24003,"Ġscal":24004,"_cal":24005,"ĠCONNECTION":24006,"_embed":24007,"(fn":24008,"ĠCraft":24009,"ĠPas":24010,"\")->":24011,".convert":24012,".resource":24013,"ĠSTATUS":24014,"ông":24015,"ĠTit":24016,"Ġclassroom":24017,"ĠArchitect":24018,"ĠKings":24019,"Ġsteady":24020,"/*!Ċ":24021,"ĠGene":24022,")\";Ċ":24023,"icia":24024,"stan":24025,"ĠConstruction":24026,"umper":24027,"wc":24028,"ĠCBS":24029,"inging":24030,"-party":24031,"(driver":24032,"MARK":24033,"Ġnested":24034,"eward":24035,"Ġdependency":24036,"Ġmales":24037,"ĠONE":24038,"ĠProduction":24039,"][$":24040,"ãĥ¼ãĥ":24041,"_LOAD":24042,"ĠBol":24043,"elry":24044,"łéĻ¤":24045,"ĠRequire":24046,"Ġplacing":24047,"xxx":24048,"CALE":24049,"Ġthumb":24050,"Choose":24051,"Ġprototype":24052,"VOID":24053,"Ġlesbian":24054,"Ġtraits":24055,"Sharp":24056,"Ġconsume":24057,"Truth":24058,"ĠactionPerformed":24059,"ĠEnvironmental":24060,"ĠDean":24061,"Ġestado":24062,"same":24063,"Ġnumeric":24064,"Ġtransit":24065,".Email":24066,"-side":24067,"_RUN":24068,"ĠVillage":24069,"_OPEN":24070,"è¦":24071,".rem":24072,"-warning":24073,"anya":24074,"PropertyChanged":24075,"Ġ(!_":24076,"(check":24077,"ilia":24078,"ĠSoft":24079,"steps":24080,"ĠMadrid":24081,"MemoryWarning":24082,"Ġhandlers":24083,"Ġexperiencing":24084,"Ġinspect":24085,"buttons":24086,"ReceiveMemoryWarning":24087,"chemy":24088,"Links":24089,"Ġurllib":24090,".SystemColors":24091,"ĠEigen":24092,"Ġpunishment":24093,":UIControl":24094,"bara":24095,"-set":24096,"Ġ}čĊčĊčĊ":24097,"Ġtolerance":24098,"Ġinterfaces":24099,".redirect":24100,"ighbors":24101,"csrf":24102,"_background":24103,".Utils":24104,"_HT":24105,"ĠInterest":24106,"imos":24107,"Ġgrants":24108,"Ġexamined":24109,"ÐĶ":24110,"Ġcf":24111,"forge":24112,"backs":24113,"ĠObjects":24114,"_sent":24115,".entry":24116,"ĠTHEN":24117,"ellido":24118,"cia":24119,",res":24120,"/stdc":24121,".nd":24122,"(Int":24123,"ĠAuthors":24124,"ĠAppCompatActivity":24125,"'{":24126,"Ġmedi":24127,"Music":24128,"igm":24129,"ceipt":24130,"Ġauss":24131,"Ġtargeting":24132,"ĠKeys":24133,"hn":24134,":]Ċ":24135,"Ġmineral":24136,"î":24137,".ca":24138,"omed":24139,"Ġsheets":24140,"Ġcamb":24141,"Ġdeadly":24142,".inject":24143,"(unit":24144,"ĠSelection":24145,".gms":24146,"(connection":24147,"Ġ$(\"":24148,"émon":24149,"ĠCurrently":24150,"pte":24151,"_paths":24152,"leaf":24153,"Ġimplications":24154,"posal":24155,"ä½į":24156,"[/":24157,"ancia":24158,"éĽ":24159,"mul":24160,"cie":24161,"Ġgeile":24162,"imals":24163,"UIView":24164,"Ġsurre":24165,"serialize":24166,"ISO":24167,"Ġarbitrary":24168,"Ġsockaddr":24169,".fn":24170,"ĠMerc":24171,"Ġcasting":24172,"KeyDown":24173,"ĠnewValue":24174,"opens":24175,"Todo":24176,"Ġflexibility":24177,"ĉĉĉĉĠĠ":24178,"Velocity":24179,"ún":24180,"rowing":24181,"Ġcomputed":24182,"`)Ċ":24183,"statement":24184,"Ġri":24185,"_cart":24186,"Low":24187,"transfer":24188,".nav":24189,"Ġgrave":24190,"ĠDoor":24191,"ĉalert":24192,".subscribe":24193,"-profile":24194,"ĉbase":24195,"ĠâĪĴ":24196,"__ĊĊ":24197,"Ġengineers":24198,"Ġexplosion":24199,"Ġdari":24200,"ĉLog":24201,"onal":24202,"Ġisolated":24203,"{i":24204,"ĠMsg":24205,"Future":24206,"Ġracist":24207,"-wrap":24208,"ĠVers":24209,"borg":24210,"ISION":24211,"ĠÑĢаÐ":24212,"ĠYan":24213,"initWith":24214,"Ġnomin":24215,"(empty":24216,"ÃŃn":24217,"ãĤ¤":24218,"ĉwidth":24219,"Ġchamber":24220,"/ajax":24221,"EMP":24222,"Ġneces":24223,"ivos":24224,"logic":24225,"*)&":24226,"cripts":24227,"RowAt":24228,"iblings":24229,"Ġears":24230,"Ġcomputing":24231,"Ġmaker":24232,"ĠNeither":24233,"breadcrumb":24234,"Ġserialize":24235,"ĠWithin":24236,"Ġdell":24237,"_TRACE":24238,"=a":24239,"Ġwishes":24240,"-inch":24241,"ĠDor":24242,"Ġinnocent":24243,"ĠDol":24244,"Ġintens":24245,"forced":24246,"ĠBIT":24247,"Ġphotographs":24248,"Ġcasa":24249,"ĠLen":24250,"\\Framework":24251,".Simple":24252,"Ġdear":24253,")/(":24254,"ippi":24255,"Ġowns":24256,"Players":24257,"Ġproposals":24258,".pi":24259,"usalem":24260,"Damage":24261,"Ġcalories":24262,"ĠCreative":24263,"Ġ[$":24264,"Ġ//čĊ":24265,"AndView":24266,"ème":24267,".custom":24268,"_factory":24269,"commands":24270,"_look":24271,"Ġstrcmp":24272,"YN":24273,"aired":24274,"Ġaudit":24275,"оÑģÑĤ":24276,"ĠReverse":24277,"ropriate":24278,"etics":24279,"';Ċ":24352,"Ġpepper":24353,"Ġshed":24354,"ĠMedium":24355,"ĠCookie":24356,"Ġoverseas":24357,"edor":24358,"asurement":24359,"åŃĺ":24360,"Ġ'.'":24361,"Ġphp":24362,"ĠPROC":24363,"Ġexceptional":24364,"(th":24365,"ĠJet":24366,"Ġoccupied":24367,".setImage":24368,"ĠRelated":24369,"ucker":24370,"Members":24371,"PRINT":24372,"ĠGlo":24373,"_VIEW":24374,"}\",Ċ":24375,"Ġadoption":24376,"[])Ċ":24377,"ĠMissouri":24378,"ĠLincoln":24379,"erald":24380,"Popup":24381,"Ġfate":24382,"-bootstrap":24383,"fections":24384,"ĠPoll":24385,"_ARGS":24386,"inance":24387,"-home":24388,".),":24389,"_done":24390,":ĊĊĊ":24391,"Ġdiscussing":24392,"ĠSQLException":24393,"Ġelectro":24394,"ĉreq":24395,"Ġzw":24396,"Ġlui":24397,"Ġovernight":24398,"$user":24399,"ĠWAY":24400,"Ġallerg":24401,"Ġdisappointed":24402,"Ġradiation":24403,"Ġimpressed":24404,"ificates":24405,"Ġtob":24406,"CLASS":24407,"Ġcuda":24408,"_det":24409,"-post":24410,"ulu":24411,"Translation":24412,"-hand":24413,".year":24414,"ĠMongo":24415,"Ġunclear":24416,".engine":24417,"WEBPACK":24418,"rices":24419,"_ACCESS":24420,"Ġholidays":24421,"percent":24422,".Identity":24423,"ĠGov":24424,"Ġpassionate":24425,"!!.":24426,"ĠGreece":24427,"plusplus":24428,"'));":24429,"GP":24430,"Ġexcit":24431,".tabPage":24432,"_cond":24433,"Ġsponsor":24434,"MODULE":24435,"_proc":24436,"Ġ$Ċ":24437,"Ġrational":24438,".Tool":24439,"Ġihr":24440,"cca":24441,"åĵģ":24442,"ĠEstate":24443,"IBUTE":24444,"ActionPerformed":24445,"ĠSolar":24446,"¦Ĥ":24447,"Ġequity":24448,"tid":24449,"Ġrecip":24450,".simple":24451,"mk":24452,"ĠLuke":24453,"ĠGuardian":24454,"Ġencrypted":24455,"Ġdominant":24456,".place":24457,"ĠNV":24458,"Ġtongue":24459,"(Get":24460,"Ġstainless":24461,".Play":24462,"Ġeb":24463,"aci":24464,".buffer":24465,"readcrumbs":24466,"Ġvaccine":24467,"prom":24468,"ĠuserInfo":24469,"Ġslug":24470,"SerializedName":24471,"-wide":24472,"Ġreactions":24473,"ĠYang":24474,"ĠAdds":24475,"(userId":24476,"Ġplates":24477,"ĠMEM":24478,"Ġbail":24479,"Inside":24480,"eted":24481,"Ġelsif":24482,"Ġsake":24483,"Ġcycles":24484,"ĠìĹ":24485,"ĉI":24486,"-collapse":24487,"ĠGMT":24488,"Declaration":24489,"Ġgros":24490,"Ġreaches":24491,"Ġcustody":24492,"Until":24493,"tu":24494,"ĠChen":24495,"Ġnx":24496,"(addr":24497,"ĠOffer":24498,"Ġcolleg":24499,"assador":24500,"Ġmapper":24501,"ĠSIGNAL":24502,"ĠBloom":24503,"ĠHoll":24504,"ĠImper":24505,"-des":24506,"_site":24507,"Proc":24508,"Equ":24509,"Ġatomic":24510,"ĠWoman":24511,"sent":24512,"scar":24513,"Ġintelligent":24514,"ĠGetting":24515,"ĠRegistration":24516,"ĠPhill":24517,"Ġkiller":24518,"unicode":24519,"ĊĉĉĊ":24520,"ĠJacob":24521,"ĠConst":24522,"Ġlocate":24523,"Ġcaus":24524,"ĠScholar":24525,"Ġconstitutional":24526,"Ġinflation":24527,"ĠGot":24528,"=array":24529,"endum":24530,"Ġtranslated":24531,"Ġdivorce":24532,"Entries":24533,"Ġsor":24534,"ĠQuote":24535,"irlines":24536,"UK":24537,"Ġexcel":24538,"(opt":24539,"ĠADV":24540,",:,":24541,"Ġcontacted":24542,"ĠDA":24543,"Ġrings":24544,"ĠIndustrial":24545,".getContext":24546,"Ġforgotten":24547,"ĠTan":24548,"Ġpants":24549,"Ġov":24550,"Ġdecoder":24551,"ĠPartial":24552,"Ġvc":24553,"Ġbattles":24554,"Arial":24555,"FRINGEMENT":24556,"irates":24557,",w":24558,"aintenance":24559,"ĠOd":24560,"ĠTechnologies":24561,"åīį":24562,"ĠCarter":24563,".findAll":24564,"Nome":24565,"Ben":24566,"ĠUsage":24567,"ĠPicture":24568,"Ġbadly":24569,"_panel":24570,"Ġpatent":24571,"ĠProtocol":24572,"lotte":24573,"ĉplayer":24574,"jections":24575,"Ġdou":24576,"_release":24577,"urniture":24578,"_tax":24579,"ĠFields":24580,".dataset":24581,"_master":24582,"CLUDE":24583,"ĠPharm":24584,"bst":24585,"Ġoperational":24586,".cell":24587,"Ġidentifying":24588,"Ġjwt":24589,"tuple":24590,"ĠTC":24591,"ĠCro":24592,"ixmap":24593,"-components":24594,"general":24595,"Ġoz":24596,"_De":24597,"_double":24598,"ĠToo":24599,".ViewGroup":24600,"gate":24601,"dings":24602,"photos":24603,"Ġgrande":24604,"ollect":24605,"_lin":24606,"Ġawful":24607,"filters":24608,"Ġalternate":24609,"esp":24610,"Ġcompress":24611,"eo":24612,"ĠScale":24613,"Ġindirect":24614,"Ġinvoice":24615,"ĊĊĊĊĊĊĊĊĊĊĊĊĊĊĊĊ":24616,"Starting":24617,"ĠPlayers":24618,"iele":24619,".then":24620,"Ord":24621,"ĠTuple":24622,"Ġbout":24623,"ĠStatistics":24624,"Preview":24625,"Ġpuzzle":24626,"ĠWidth":24627,"STATE":24628,"Ġoverlay":24629,"ĉon":24630,"Ġinfr":24631,"Ġsmallest":24632,"locked":24633,"ÑĤо":24634,"ssl":24635,"Ġdeemed":24636,"Ġsco":24637,"reck":24638,"ĠjButton":24639,"Ġmissions":24640,"称":24641,".SelectedIndex":24642,"TABLE":24643,"Sept":24644,"Ġacknowledge":24645,"Ġstrtotime":24646,"ĠTell":24647,"ĠDak":24648,"Ġaluminum":24649,"Ġfence":24650,"ĠStars":24651,"CONFIG":24652,"Ġretrofit":24653,"Ġemphasis":24654,"/header":24655,"ĠSomething":24656,"inished":24657,"='\".$":24658,"ĠValidators":24659,"Ġpolar":24660,"sections":24661,".aspx":24662,"Ġaspir":24663,".Mock":24664,"CodeGen":24665,"Ġpeut":24666,"Ġaccepting":24667,"Ġbacking":24668,"Picture":24669,"/ap":24670,"ег":24671,"_SEC":24672,"-use":24673,"annotation":24674,"Ġcognitive":24675,"Ġgrip":24676,"hour":24677,"ĠLegal":24678,"Ġepic":24679,".toolStrip":24680,".notify":24681,".Last":24682,"ORIZ":24683,"Middleware":24684,"criptions":24685,"lash":24686,"_FOUND":24687,"ĠLiverpool":24688,"Ġ{}\",":24689,"Install":24690,"Ġnit":24691,"Ġfigured":24692,"[len":24693,".Win":24694,".platform":24695,"Ġgambling":24696,"(dt":24697,"avery":24698,"ĉinclude":24699,"Whether":24700,"Routing":24701,"Ġtherap":24702,"Remote":24703,"ĠLoss":24704,"yll":24705,"Ġapproached":24706,"ĠVehicle":24707,"ĠAlpha":24708,"Ġvocê":24709,"answers":24710,"NSDictionary":24711,"consider":24712,"unused":24713,"ĠFan":24714,"orable":24715,"fre":24716,"ĠDISCLAIM":24717,"ĠActor":24718,".]":24719,"toHave":24720,".userId":24721,"Ġspeeds":24722,"eway":24723,"Ġrecurs":24724,"Ġг":24725,"_priv":24726,"!âĢĿĊĊ":24727,"Choice":24728,"Ġsettle":24729,"Ġplanes":24730,"'},":24731,"Tom":24732,"ITER":24733,"!\"Ċ":24734,"å»":24735,"achelor":24736,"Ġseparation":24737,"Ġdal":24738,"adj":24739,"Ġregisters":24740,"riz":24741,"ĠNotice":24742,"Ġlu":24743,"Ġcourage":24744,"Ġaxes":24745,"cellent":24746,".async":24747,"Ġcompatibility":24748,"ç«":24749,"Ġ!ĊĊ":24750,"ĉtitle":24751,"YLE":24752,"ĉmessage":24753,"UUID":24754,"OLDER":24755,"ĠHH":24756,"ĠStyleSheet":24757,"Ġaccessed":24758,".validation":24759,"tasks":24760,"Ġpollution":24761,".canvas":24762,"Ġingredient":24763,"ĠCabin":24764,"Ah":24765,"oldown":24766,"ĠNOI":24767,"ĠÃĹ":24768,"[f":24769,"educ":24770,"yalty":24771,"(not":24772,"_State":24773,"amen":24774,"Ġdao":24775,"udad":24776,"ellers":24777,"}&":24778,"licity":24779,"_WINDOW":24780,"Ġtatto":24781,"valor":24782,".Range":24783,"Ġreferenced":24784,"ĠReserve":24785,"Money":24786,"SCRIPT":24787,"/product":24788,"choices":24789,"Ġtin":24790,"ãĤĵ":24791,"Ġseparator":24792,"Ġpkg":24793,"ammed":24794,"ĠMAT":24795,"!!ĊĊ":24796,"Ġraid":24797,"Ġmotivation":24798,"ĠXP":24799,"ĠBackground":24800,"ĠQuaternion":24801,".defineProperty":24802,"iker":24803,"ĉparent":24804,"ĠOriginally":24805,"antage":24806,"ĠHans":24807,"Ġtimeline":24808,".cur":24809,"opic":24810,"ĠSequ":24811,"must":24812,"ĠCoal":24813,"Ġformatter":24814,"_RGB":24815,"Ġ_(\"":24816,"'}),Ċ":24817,"Ġ=================":24818,"ĠFUNCTION":24819,"Ġlng":24820,"icates":24821,"live":24822,"_engine":24823,"Ġtowns":24824,"'))ĊĊ":24825,"ĠPK":24826,"(api":24827,"ĉscanf":24828,"packet":24829,".phone":24830,"áĢ":24831,"ĠAndy":24832,"_NAMES":24833,"PLY":24834,"Ġmins":24835,"imi":24836,"Ġbrick":24837,"Ġblade":24838,".stdout":24839,"}`;Ċ":24840,"Shift":24841,"ĉsb":24842,"ĠChecks":24843,"Ġphenomenon":24844,"Avatar":24845,"Ġministry":24846,"rose":24847,"ĉFile":24848,"Ġtitled":24849,"(LOG":24850,"Ġgan":24851,"design":24852,"(),čĊ":24853,"Ġbones":24854,"stm":24855,"ÅĽÄĩ":24856,"ĠInputStream":24857,"Ġvolunt":24858,"ĠSerializable":24859,"Ġfighter":24860,"ĠDrag":24861,"Twitter":24862,"Ġsubsid":24863,"ç¼":24864,"Ġforums":24865,".loading":24866,"logged":24867,"_this":24868,"Ġterrain":24869,"Ġirre":24870,"ĠIng":24871,"ĠCN":24872,"_objects":24873,".uid":24874,"Ġconsciousness":24875,"TINGS":24876,"ĠGall":24877,"Ġportray":24878,"ĠDeveloper":24879,"Ġparticipant":24880,"Ġ\";čĊ":24881,"/model":24882,"ĠOperations":24883,"^\\":24884,"ĠLater":24885,"Ġraises":24886,"-none":24887,".meta":24888,"='.$":24889,"Finished":24890,"Ġreplacing":24891,"Ġsampling":24892,"ĠJen":24893,"\"There":24894,"REAL":24895,"ALE":24896,"ìĬ¤":24897,"Orders":24898,"_parameter":24899,"ĠOlympic":24900,"Ġtrès":24901,"Ġarena":24902,"iol":24903,";?>":24904,"Ġimpacts":24905,"ĠWS":24906,":get":24907,"Ġflights":24908,"ĠRussell":24909,"camera":24910,"Fn":24911,"sigma":24912,"Ġforcing":24913,"Ġlocals":24914,"Ġdeparture":24915,"Ġcelebration":24916,"ĠSay":24917,"ï¼Ĵ":24918,"ĠHills":24919,".hasOwnProperty":24920,"Ġtypings":24921,".API":24922,"Ġdonation":24923,"OperationException":24924,".Activity":24925,"cplusplus":24926,"ĠCharlie":24927,"Ġimported":24928,"Ġdann":24929,"Ġoccasions":24930,"Ġimplementing":24931,"Ġpurple":24932,".dialog":24933,"SQLException":24934,"erno":24935,"Ġwars":24936,"Ġpaste":24937,"Ġdecreased":24938,"Ġharsh":24939,"Ġelabor":24940,"inputs":24941,"ĠViews":24942,"ĠerrorMessage":24943,"_mul":24944,"ĉwrite":24945,"ĠCop":24946,"ĠAnnual":24947,"(button":24948,"Ġvida":24949,"bars":24950,"ĠHarvard":24951,"ĉexpect":24952,"Ġindexes":24953,"Ġdocumentary":24954,"Ġflesh":24955,"ORLD":24956,"ĠDelta":24957,"MAND":24958,"Brush":24959,"-column":24960,"Ġdevelopments":24961,"methodVisitor":24962,"slice":24963,"ĠPDO":24964,"Ġinvesting":24965,"irable":24966,"Ġxmlns":24967,"ï¼Ľ":24968,"arta":24969,"Ġtheories":24970,"_city":24971,"Ġ$__":24972,"Creating":24973,"(pr":24974,"Dropdown":24975,"ismatch":24976,"ĠNET":24977,"'])){Ċ":24978,"ĠValues":24979,"ĠSEO":24980,"ĠSTAT":24981,"Ġecosystem":24982,"Ġtempt":24983,"Ġ\\\\":24984,"Ġ//{Ċ":24985,"ĠChristopher":24986,"ĠKentucky":24987,"ĠHttpServletResponse":24988,"Ġhybrid":24989,"yon":24990,"Ġfeeding":24991,"ĠExtra":24992,"Norm":24993,"ITCH":24994,"ĠSean":24995,"ĠUpload":24996,"mun":24997,"pur":24998,"Ġpersistent":24999,"ĠIDC":25000,"ĠPerform":25001,".merge":25002,"_room":25003,"Meanwhile":25004,"!='":25005,"ĠWel":25006,"ArgsConstructor":25007,".Database":25008,"Ġcounting":25009,"()*":25010,"ĶåĽŀ":25011,"ĠTOP":25012,"mill":25013,"ĠDT":25014,"IGNED":25015,"ĠKB":25016,"Ġcomply":25017,"South":25018,"_collection":25019,"Chapter":25020,"Ġexplaining":25021,"_AM":25022,"_ts":25023,"cards":25024,"Ġquel":25025,"Ġpole":25026,"Ġtouchdown":25027,"ĠOthers":25028,"Ġpeers":25029,"ĠTypeError":25030,"Ġsixth":25031,"Ġcheer":25032,"Ġdispute":25033,"usc":25034,")],":25035,"thumb":25036,"Ġhiding":25037,"ĠSIG":25038,"likes":25039,"ĠPAGE":25040,".Reflection":25041,"Ġheadquarters":25042,"TING":25043,"ĠGhost":25044,"MLE":25045,"$Ċ":25046,"Ġcontrary":25047,"extend":25048,"']).":25049,"FFECT":25050,"ĠPinterest":25051,"úmero":25052,"ricane":25053,"ĉsession":25054,"Ġcrystal":25055,"-Control":25056,"overnment":25057,"ograf":25058,"-action":25059,"volume":25060,"ften":25061,"Ġuncon":25062,"Ġanimate":25063,"Ġlease":25064,"scr":25065,"Ġrefuse":25066,"ãĢĭ":25067,"ftp":25068,"information":25069,"Ġevaluated":25070,"Ġinjection":25071,"Ġjack":25072,"Ġworkshop":25073,"注":25074,"PTH":25075,"ĠTs":25076,"offer":25077,"ĉos":25078,"Ġkingdom":25079,"Missing":25080,"Ġlawmakers":25081,"extField":25082,"Ġsinging":25083,"abi":25084,"/client":25085,".media":25086,"ATEGORY":25087,"Signature":25088,"%',Ċ":25089,"ĠFuck":25090,"][:":25091,"Ġsensors":25092,"/com":25093,"ĠPrimary":25094,".SQL":25095,"_program":25096,"Ġpills":25097,"Ġintegral":25098,"Ġfleet":25099,"Ġdropping":25100,".sl":25101,"Been":25102,"Ġpets":25103,"Ġadvised":25104,"Ġdragon":25105,"_EDIT":25106,"(im":25107,"FER":25108,"ĠDrug":25109,"(random":25110,"Ġcompression":25111,"oust":25112,"[%":25113,"Ġbuyer":25114,"hop":25115,"Roles":25116,"manage":25117,"Ġpainful":25118,"ĠBranch":25119,"-modal":25120,"enant":25121,"ĠMesh":25122,"/font":25123,"ĠGraham":25124,"Ġâĺ":25125,"Ġnc":25126,"ĠFrancis":25127,"Ġspecification":25128,"Ġdamages":25129,"-config":25130,"Ġtheoret":25131,"secure":25132,"_multi":25133,"aceutical":25134,"Ġdemanding":25135,"enne":25136,"ISTS":25137,"()));ĊĊ":25138,"Reason":25139,"Recent":25140,"phase":25141,"Ġpsy":25142,"_MAN":25143,"Ġvolunteer":25144,"å¿":25145,"istributed":25146,"lio":25147,"Ġproductivity":25148,"_comm":25149,"Spring":25150,"nis":25151,".weight":25152,"ĠCancer":25153,"Alloc":25154,"ĠTweet":25155,"Ġseparately":25156,"ĉcheck":25157,"_properties":25158,".Unit":25159,"_CLK":25160,"Ġgt":25161,"Ġ();ĊĊ":25162,"Ġhandy":25163,"ĠThompson":25164,"Ġunnecessary":25165,"ĠReader":25166,"GN":25167,"=request":25168,"ĠUtility":25169,".Repository":25170,"ĠAx":25171,"hydr":25172,"ieu":25173,"Ġthy":25174,"Ġlt":25175,"_mail":25176,"ä¿®æĶ¹":25177,"ailand":25178,"ĠPhilip":25179,"Ġbitter":25180,"Ġbetting":25181,"Ġtimed":25182,"ocks":25183,"'a":25184,"Ġalgorithms":25185,"Ġreinterpret":25186,"Ġtoss":25187,"rogen":25188,"Ġhoped":25189,"(selected":25190,"Ġventure":25191,"TEX":25192,"ĠLeave":25193,".Substring":25194,"Ġgrateful":25195,"uka":25196,"ĠConsumer":25197,"Ġaggreg":25198,"Circle":25199,"à¸ģ":25200,"_blocks":25201,"Ġlegally":25202,"Ġ\"|":25203,"ãĥĥ":25204,".board":25205,".Ab":25206,"Functions":25207,"recipe":25208,"èĩ":25209,"ĠOxford":25210,"Ġwholes":25211,".Build":25212,"_changed":25213,"hai":25214,"Ġdepartments":25215,"Imp":25216,"Ġcoalition":25217,"INFRINGEMENT":25218,"Ġempower":25219,"itches":25220,"North":25221,"Ġinflamm":25222,"ONSE":25223,"Ġmissile":25224,"ĠRaj":25225,"ĠIssue":25226,"Ġatoi":25227,"caled":25228,".Controllers":25229,"ĠWolf":25230,"Ġcrushers":25231,"á»ĩ":25232,".Auth":25233,".addAttribute":25234,"his":25235,"Ġboots":25236,".clean":25237,"camp":25238,"Ġtenant":25239,"Ġtune":25240,"Ġ{}'.":25241,"Ġworkout":25242,"Repo":25243,"Ġpartially":25244,"MISSION":25245,"jamin":25246,"ĠSB":25247,"Ġdetermination":25248,"Ġ'');Ċ":25249,"ĠBeng":25250,"Ġvos":25251,"Ġinhab":25252,"/lang":25253,"sburgh":25254,"Executor":25255,"hone":25256,"ĠChallenge":25257,"_links":25258,".Level":25259,"Ġunderground":25260,"-code":25261,"Ġoptimization":25262,"logging":25263,"_dest":25264,"Ġsnake":25265,"Ġchemicals":25266,"_IMPORTED":25267,"adoop":25268,"ĠTHAT":25269,"managed":25270,"Ġreduces":25271,"ĠREAL":25272,"ĠGuy":25273,"_GENERIC":25274,"/********************************":25275,".amount":25276,"Ġdere":25277,"getTime":25278,"Ġpant":25279,"anonymous":25280,"Ġharmony":25281,"ĠAlan":25282,"Ġscenarios":25283,"Ġdirt":25284,"htags":25285,"Mc":25286,"Shell":25287,"rin":25288,"{čĊčĊ":25289,".pow":25290,"ĉclient":25291,"Ġconspiracy":25292,"Ġadmission":25293,"ĠRegional":25294,"ĠViewController":25295,"ĠPhilippines":25296,"Ġdepos":25297,"Ġpap":25298,"ĠPad":25299,"Paul":25300,".ComboBox":25301,"Ġtutor":25302,"ĠRecipe":25303,"writing":25304,"Ġcontributor":25305,"OTH":25306,"Small":25307,"VI":25308,"Ġhacer":25309,"equ":25310,"ĠExamples":25311,"human":25312,".messages":25313,"ĉtyp":25314,"Ġ(čĊ":25315,"ĠSSL":25316,"LEN":25317,"ĠRomney":25318,"(grid":25319,"ĉmin":25320,"Ġ>ĊĊ":25321,"Ġfruits":25322,"Ġvoter":25323,"Inline":25324,"pane":25325,"ĠCollections":25326,"charset":25327,"Ġspam":25328,"zb":25329,"itemap":25330,"Ġsucceeded":25331,"_COL":25332,"Ġelapsed":25333,"imeter":25334,"Ġrecovered":25335,"Tensor":25336,"hattan":25337,".setup":25338,"isto":25339,"(head":25340,"ĠSIZE":25341,"Ġtactics":25342,"Ġdistur":25343,"Ġpreval":25344,"icios":25345,"(Value":25346,"_cols":25347,"ĠFat":25348,"Ġseal":25349,"Ġsons":25350,"Ġensures":25351,"Ġpressing":25352,"=&":25353,"igenous":25354,"Ġharassment":25355,"_JSON":25356,"Ġignor":25357,"ynomial":25358,"omer":25359,"_static":25360,"Ġsignificance":25361,"Ġcircles":25362,"_System":25363,"Ġdiscipline":25364,"Ġdressed":25365,"Ġsphere":25366,"Ġclimb":25367,"_actions":25368,"ĠBab":25369,"Ġ'=',":25370,"_schema":25371,"\"use":25372,"Ġunders":25373,"Ġcups":25374,".screen":25375,"/new":25376,"Ġappearing":25377,"TOP":25378,"vised":25379,"clang":25380,"Ġinvestigators":25381,"Ġmysterious":25382,"Ġpromising":25383,"Ġqualify":25384,"Ġcave":25385,"Ġequip":25386,"=x":25387,"GT":25388,"(link":25389,".velocity":25390,".erase":25391,"oter":25392,"++++++++":25393,"profit":25394,"Ġzones":25395,"_uid":25396,"-ser":25397,"Ġobjectives":25398,"Ġmilf":25399,"webkit":25400,"(match":25401,"neh":25402,"ĠAssociated":25403,"ĠTodo":25404,"=d":25405,"Cam":25406,"Ġvocal":25407,"Ġsudo":25408,"(EX":25409,"Ġtrou":25410,"ABC":25411,".bean":25412,"ĠGround":25413,"ĠREST":25414,"weets":25415,"Ing":25416,"imon":25417,"_bus":25418,"ĠCOLOR":25419,"unto":25420,"Ġfoss":25421,"ĠLinks":25422,"äng":25423,"/forms":25424,"prises":25425,"Ġachievement":25426,"CALL":25427,"елÑĮ":25428,"ĠVerify":25429,"_SOURCE":25430,"aptcha":25431,"IDD":25432,"_reference":25433,"Gold":25434,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":25435,"Receiver":25436,"Ġaj":25437,"_direction":25438,"}]":25439,"ĠCompet":25440,"Ġbang":25441,"ĠCass":25442,"-url":25443,"techn":25444,"ĠJerusalem":25445,"longitude":25446,"');čĊčĊ":25447,"Ġwinners":25448,"Tasks":25449,"ĠDMA":25450,"Ġtooltip":25451,"İ·":25452,"ĠBra":25453,"_duration":25454,"cury":25455,"parents":25456,"---->(":25526,"ĠKir":25527,"Ġintros":25528,"Ġsketch":25529,"Ġskilled":25530,"Ġimmer":25531,"Ġadequate":25532,"_rep":25533,"(header":25534,"_like":25535,"Ġperceived":25536,"ssh":25537,"Ġassuming":25538,"Ġff":25539,"_uuid":25540,"ulas":25541,"Ġdemocratic":25542,".entities":25543,"Series":25544,"aphore":25545,"Ġnewer":25546,"}(":25547,"SEC":25548,"airo":25549,"Ġcommod":25550,"Ġprivilege":25551,"Ġdeux":25552,"ĠHop":25553,".'/":25554,"ctic":25555,".';Ċ":25556,"C":25630,"ĠWarren":25631,"Ġoptimizer":25632,"ĠSERVICES":25633,"_oper":25634,"getAttribute":25635,"ĠMcK":25636,"_self":25637,".rs":25638,"\")ĊĊĊ":25639,"GetComponent":25640,"erce":25641,"Ġtous":25642,"units":25643,"']);čĊ":25644,"Zoom":25645,"/E":25646,"Ġobsc":25647,"Ġfastest":25648,"online":25649,"Ġpeaceful":25650,"ffen":25651,"Ġcargo":25652,"ĉpr":25653,"Ġseeks":25654,"zu":25655,"Trim":25656,"Ġward":25657,"Ġverd":25658,"Ġblogs":25659,".exceptions":25660,"ĠPremium":25661,"ĠNetherlands":25662,"Safe":25663,"Finish":25664,"ĠAlbum":25665,"_ACC":25666,"=this":25667,"virtual":25668,"]>":25669,"_LABEL":25670,"ĠNich":25671,"_win":25672,"ĠAaron":25673,"WP":25674,";$":25675,"aims":25676,"ĠImageView":25677,"Ġendless":25678,"ERA":25679,"_DISABLE":25680,"Ġcancelled":25681,"-us":25682,"Ġinspection":25683,"emin":25684,"ĠGrey":25685,"-open":25686,"Ġiterations":25687,".owner":25688,"Ġkeras":25689,".Password":25690,"ĠRy":25691,"ĠINS":25692,"Air":25693,"ĠSeveral":25694,".TabStop":25695,"INGLE":25696,"ĠHair":25697,"ĠCanvas":25698,"AAAA":25699,"Ġflaw":25700,"cedes":25701,".Report":25702,"íĬ":25703,"ĠTips":25704,"criptors":25705,".transaction":25706,".Spring":25707,"Ġviewer":25708,"Ġinsights":25709,"è¾ĵ":25710,"ordion":25711,"UINT":25712,"seek":25713,"ĠAuf":25714,"ìŀIJ":25715,"Ġstrain":25716,"Tooltip":25717,"Ġdz":25718,"ignal":25719,"adt":25720,"Ġuc":25721,"finite":25722,"Ġnm":25723,".cmd":25724,"ĠMySql":25725,"[data":25726,".jackson":25727,".tree":25728,"RequestParam":25729,"_agent":25730,"\")]čĊ":25731,"Ġassass":25732,"(Constants":25733,":ss":25734,"ĠMAN":25735,"+-+-":25736,"ĠBottom":25737,"prints":25738,"ĠSame":25739,"@Autowired":25740,"swap":25741,"ición":25742,"Ġprotesters":25743,"Ġhoney":25744,"ĠVeter":25745,"(Calendar":25746,"-ad":25747,"ĠBrooklyn":25748,"Life":25749,"_VAR":25750,"zech":25751,"ĠCALL":25752,"_CAST":25753,"ĠElection":25754,"Ġthickness":25755,"Very":25756,"_INTEGER":25757,"-dev":25758,"))))":25759,"apat":25760,"oooo":25761,"demo":25762,"ĠparseFloat":25763,"ĠRather":25764,"STIT":25765,"maker":25766,"[current":25767,"chrono":25768,"Ġchrist":25769,"ãģª":25770,"ĠDetail":25771,"Æ°á»":25772,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":25773,"Ġsul":25774,"idency":25775,"Que":25776,"Ġelegant":25777,"apons":25778,"Ġdishes":25779,"Ġintegers":25780,"(read":25781,"findViewById":25782,"ĠAmount":25783,"ĠSkip":25784,"Ġhabits":25785,"*)(":25786,"Ġmonsters":25787,"MAC":25788,":end":25789,"Ġfrank":25790,"Assembly":25791,"Ġdfs":25792,"Ġneut":25793,"_TYPES":25794,"equal":25795,"loyd":25796,"(uri":25797,"Ġchi":25798,"Ġdefendant":25799,"Ġconflicts":25800,"Ġvil":25801,"-js":25802,"ĠPeace":25803,"Ġmutable":25804,")sender":25805,"ĠFocus":25806,"建":25807,"Ġappreciated":25808,"sleep":25809,"ĠRED":25810,"Culture":25811,"Ġdesigners":25812,"_generator":25813,"codes":25814,"/ex":25815,".GetValue":25816,"umbled":25817,".scalajs":25818,"peror":25819,"Ġveterans":25820,"Ġ})čĊ":25821,"Ġunfortunately":25822,"_CREATE":25823,"Mass":25824,"ĠCLAIM":25825,"ĠMeet":25826,"_support":25827,"Bank":25828,"().Ċ":25829,"Dark":25830,"_LOW":25831,"ĠMining":25832,"ĠOwner":25833,"iera":25834,"Cliente":25835,"Ġencouraging":25836,">S":25837,"Ġboyfriend":25838,"ĠHalf":25839,"ĠACC":25840,"Aff":25841,"_ar":25842,"-life":25843,"cx":25844,".JButton":25845,"izado":25846,".zero":25847,".openqa":25848,"oton":25849,".textContent":25850,"Ġtoll":25851,"atie":25852,"Ġballot":25853,"-number":25854,".Exception":25855,"ĉparams":25856,"circle":25857,"-map":25858,"Ġnap":25859,"ĠRobot":25860,"ĠIch":25861,"registration":25862,"Amazon":25863,"rollment":25864,"(exp":25865,"Ġtanks":25866,"ĠGordon":25867,"Ġmachinery":25868,"Ġbaseline":25869,"æĭ":25870,"Ø©":25871,"ĠConvention":25872,"ĉconfig":25873,"ookies":25874,"mult":25875,"Records":25876,"ĠEST":25877,"Ġgarbage":25878,"Ġconform":25879,"idal":25880,"Ġbarg":25881,"Ġsurvived":25882,"Ġinvestigations":25883,".containsKey":25884,"--------------------------------------------------------------------------Ċ":25885,"ortion":25886,"Ġhorr":25887,"_http":25888,"Ġmant":25889,"];čĊčĊ":25890,"binary":25891,"empl":25892,"Ġinquiry":25893,"ĠMeanwhile":25894,"Ġcollecting":25895,".EntityFramework":25896,"\",ĊĊ":25897,"ĠPic":25898,"@Inject":25899,"ickness":25900,"ĠBinding":25901,"Ġcontrolling":25902,"reverse":25903,"Ġchairs":25904,"sembled":25905,"(add":25906,"Disabled":25907,"anas":25908,".translate":25909,"-----------Ċ":25910,"Ġreflected":25911,"\"]ĊĊ":25912,"External":25913,"Arrow":25914,"Singleton":25915,"%x":25916,"ĠÅ":25917,"Ġancest":25918,"ĠOrleans":25919,"ĉcmd":25920,"Ġprohibited":25921,"ithmetic":25922,"(channel":25923,"_css":25924,"Forward":25925,".socket":25926,"Ġluc":25927,"âĨ":25928,"ĠFirefox":25929,"ĠMovies":25930,")_":25931,".ends":25932,"(shape":25933,"Ġdealt":25934,"Ġsaves":25935,"Ġglory":25936,"Ġmejor":25937,"Ġbreathing":25938,"Ġeller":25939,"getData":25940,"Ġangles":25941,"Ġtoolbar":25942,"Ġspacing":25943,"IPS":25944,"Ġfloors":25945,"_ACTIVE":25946,"Ġshuffle":25947,"/shared":25948,"ĠEle":25949,"edish":25950,"Ġwebcam":25951,".expect":25952,"iloc":25953,"ĠIncludes":25954,"Ġtweeted":25955,"Ġ:)":25956,"ĠEssay":25957,"Fix":25958,"-between":25959,"_web":25960,".conv":25961,"Ġracism":25962,"Ġreflects":25963,"umm":25964,"иÑĤе":25965,"_footer":25966,"/docs":25967,"ĠPour":25968,"NgModule":25969,".initialize":25970,"patterns":25971,"_In":25972,"ĠAbb":25973,"*čĊ":25974,"Ġsentiment":25975,"buff":25976,"_counts":25977,"Ġreuse":25978,"chunk":25979,"Ġimposed":25980,"PrimaryKey":25981,"Foreground":25982,"Ġconsumed":25983,"?!":25984,"Ġdick":25985,"Ġchron":25986,"ĠFern":25987,"Ġresponsive":25988,"Ġinsect":25989,"iculty":25990,"Ġrw":25991,"Ġalike":25992,"Ġsubset":25993,"ĠCookies":25994,"ĠPair":25995,"Ġtier":25996,"IFO":25997,"avour":25998,"ĠQU":25999,",sizeof":26000,"Ġmerged":26001,"mv":26002,"itol":26003,"ylon":26004,"Ġjumped":26005,".role":26006,"ensaje":26007,"Rules":26008,"Ġbrowse":26009,"Animator":26010,"Ġyoga":26011,"Ġvariants":26012,"Ġcourtesy":26013,"uran":26014,"pbs":26015,"elseif":26016,"Alt":26017,"ĠLane":26018,"CLK":26019,"IMARY":26020,"_PROPERTY":26021,"ï¼IJ":26022,"Ġchan":26023,"Ġgradually":26024,"Ġshake":26025,"Ġblonde":26026,"...\");Ċ":26027,"-sex":26028,"Ġgameplay":26029,"acies":26030,".refresh":26031,"USB":26032,"ĠPlot":26033,"Was":26034,"issippi":26035,"ĠTensor":26036,"Ġcryptocurrency":26037,"Ġdifficulties":26038,"Deleted":26039,"Without":26040,"_append":26041,"_ver":26042,"\"))čĊ":26043,"Ġhonestly":26044,"Ġpivot":26045,"Ġtemps":26046,"_ps":26047,"ĠUnlike":26048,"[:-":26049,"VS":26050,"_inf":26051,"Ġjunior":26052,"Ġanimations":26053,"Ġfilepath":26054,"?{{$":26076,"Ġunicode":26077,"places":26078,"ĠCoffee":26079,".SE":26080,"ĠPAR":26081,"(txt":26082,"gebra":26083,"Ġfires":26084,"MainWindow":26085,"medium":26086,"Ġ(âĢľ":26087,"Ġlg":26088,"Ġcmp":26089,"/base":26090,"_layers":26091,"_entries":26092,"Ġadminister":26093,"ĠSUCH":26094,"BP":26095,"ĠScottish":26096,"ĉčĊĉčĊ":26097,"guard":26098,"ĠStrong":26099,"Insn":26100,"ĠCAP":26101,"asury":26102,"ĠSEE":26103,"Clock":26104,"erie":26105,"\\models":26106,"Ġ$$":26107,"ĠCab":26108,"Ġwurde":26109,"Ġsoldier":26110,"Ġclips":26111,"Ġarrangement":26112,"ĠWonder":26113,"ĠHorn":26114,"Ġscared":26115,"Ġcure":26116,"mkdir":26117,"Ġaligned":26118,"ĠPink":26119,"Ġlanded":26120,"Dimension":26121,"ScrollPane":26122,".chat":26123,".With":26124,"ĠTrain":26125,"].Ċ":26126,"Ġthirty":26127,"Ġdurable":26128,"Ġld":26129,"Ġlateinit":26130,"Ġcharts":26131,"Ġinsult":26132,".Fatal":26133,"_ct":26134,"Ġmasks":26135,"CLUDED":26136,"President":26137,"Ġcolours":26138,"gments":26139,".attributes":26140,"ĠFlex":26141,"ĠClock":26142,"ÃŃcul":26143,"imen":26144,"JO":26145,"ĠRegex":26146,"_LINK":26147,"Ġcouch":26148,"ĠINPUT":26149,"Ġbeating":26150,"business":26151,"preced":26152,".unit":26153,"ĠFel":26154,"Never":26155,"ospel":26156,".startswith":26157,"ĠEPA":26158,".only":26159,"Ġpreventing":26160,"yer":26161,"ColumnName":26162,"Ġelevation":26163,"flu":26164,"icycle":26165,"Ġoffline":26166,"Toolbar":26167,"Ġcompeting":26168,")].":26169,"Ġmog":26170,"ĠisValid":26171,"Ask":26172,"_av":26173,"_lat":26174,"ANC":26175,"ĠJoh":26176,"kers":26177,"Ġguards":26178,"Ġchains":26179,"ĠSimpleDateFormat":26180,".static":26181,"Ġvessel":26182,"Ġmud":26183,"Ġstabil":26184,"Ġstret":26185,"gm":26186,"amation":26187,"çľ":26188,"-with":26189,"Ġros":26190,"_PA":26191,"Ġresultado":26192,"Ġconfidential":26193,"ĠTokyo":26194,"ĉusing":26195,"ĠMathf":26196,"ombine":26197,"ĠESPN":26198,"Ġdealers":26199,"Ġdismissed":26200,"TRY":26201,"Ġteens":26202,"records":26203,"Ġwings":26204,"gallery":26205,"accounts":26206,"_LIB":26207,"Ġjacket":26208,"ĠNSObject":26209,"Ġstones":26210,"ĠDelivery":26211,"ĠDiet":26212,"/watch":26213,"Ġtoilet":26214,"ĠGuest":26215,".day":26216,"Ġintval":26217,"Visit":26218,"Ġinvestigated":26219,"Ġpentru":26220,"ĠTheatre":26221,"andidates":26222,"Lang":26223,"ĠServ":26224,"Ġcontrollers":26225,"ĠsetTitle":26226,"NP":26227,"amy":26228,"flat":26229,"(ui":26230,"_document":26231,"èĥ½":26232,"ĠCoin":26233,"ĠAdams":26234,"ptic":26235,"Ġproductive":26236,"Ġaccomplished":26237,"čĊčĊčĊčĊ":26238,"Ġdeferred":26239,"ientes":26240,"Ġsinc":26241,"olars":26242,"Rightarrow":26243,"Ġvariations":26244,"(offset":26245,".LayoutInflater":26246,"Ġsuspend":26247,"Ġprevention":26248,"_private":26249,"_js":26250,"âĺħ":26251,"Ġwieder":26252,"atum":26253,"ĴĮ":26254,"Ġappearances":26255,".Document":26256,"Ġvalidates":26257,"calendar":26258,"}\";Ċ":26259,".demo":26260,"conut":26261,"Ġcorrection":26262,"ĠDeal":26263,"Ġbatteries":26264,".duration":26265,",\\":26266,"_marker":26267,"multi":26268,"Ġhalt":26269,"Ġcms":26270,"Ġshaped":26271,"Bro":26272,"reduce":26273,"Ġ####":26274,"CTOR":26275,"ĠBenef":26276,"Ġiconic":26277,"Ġpiano":26278,"Ġeffectiveness":26279,"|.Ċ":26280,"Ġajax":26281,"Ġvolumes":26282,"ม":26283,"Ġcljs":26284,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":26285,"aths":26286,"raits":26287,"大":26288,"Ñĸ":26289,"_mult":26290,"Ġfascinating":26291,"Average":26292,"Ġpré":26293,"ĠChairman":26294,".findElement":26295,"_pin":26296,"Ġcomparing":26297,"Ġdarkness":26298,"-Fi":26299,"-server":26300,"Ġselecting":26301,"sterdam":26302,"ĠParts":26303,"FORMATION":26304,"Ġnoting":26305,"Ġpile":26306,"ogs":26307,"Ġpalette":26308,"_do":26309,"itize":26310,"()(":26311,"Ġdefining":26312,"Ġremainder":26313,"Units":26314,"_TASK":26315,"HttpClient":26316,"Social":26317,"Ġfundra":26318,"NR":26319,"chest":26320,"Currency":26321,".adapter":26322,"Ġdop":26323,"unting":26324,"ANGUAGE":26325,"\"He":26326,"ĉindex":26327,"_package":26328,".Icon":26329,"Ġrepet":26330,"mass":26331,"=\".$":26332,"ĠSud":26333,"Ġlid":26334,"province":26335,"ìľ":26336,"GPIO":26337,"Ðļ":26338,"ĠMySQL":26339,"Ġdocs":26340,"ĠGA":26341,"Ġipsum":26342,"Kernel":26343,"Ġaccepts":26344,"Ġfitting":26345,"Ġcuando":26346,"Ġduplic":26347,"ĠBrother":26348,"ĠKle":26349,"nums":26350,"Ġmorph":26351,"Ġ########":26352,"ĠCGPoint":26353,"manual":26667,"ĠTechnical":26668,"Ġcorporation":26669,"ĠHW":26670,"anka":26671,"TAIL":26672,"istas":26673,"Ġperforms":26674,"ĠBehavior":26675,".For":26676,"_ORDER":26677,"ĠKick":26678,"Ġcallbacks":26679,"_dr":26680,"uego":26681,"hub":26682,"ufficient":26683,"sky":26684,"Ġbp":26685,"htable":26686,"ĠONLY":26687,"ĠAUTHORS":26688,".Argument":26689,"\"};Ċ":26690,"ĠThunder":26691,"ĠKom":26692,".Should":26693,"AUTH":26694,"ahu":26695,"_payment":26696,"Ġstarter":26697,"ìĦľ":26698,"ìļ©":26699,"Blog":26700,".patch":26701,"Ġgoverned":26702,"assy":26703,"-found":26704,"Ġtheater":26705,"ĠFontWeight":26706,"ĠBatman":26707,"\"If":26708,".Random":26709,"_delta":26710,"ĠCE":26711,"Authenticated":26712,"Ġdrone":26713,"Ġcous":26714,"radius":26715,"Mer":26716,"(None":26717,"ĠNJ":26718,"_headers":26719,"Ġamer":26720,"pytest":26721,"ĠActions":26722,"ĉĉĉĠĠĠĠ":26723,"Ġett":26724,"Ġholy":26725,"Ġuncomfort":26726,"ĠNin":26727,"ĠDecimal":26728,"ĠMessages":26729,".sender":26730,"]])Ċ":26731,"Ġembrace":26732,"Though":26733,"/sp":26734,"Ġcultures":26735,"Ġhighway":26736,"tar":26737,".fail":26738,"_hidden":26739,"ĠcomponentDidMount":26740,"ĠWright":26741,"Ġjag":26742,"_il":26743,"../../../":26744,"igu":26745,"Food":26746,"Ġace":26747,"Ġaños":26748,"USD":26749,"Ġmutual":26750,"Logic":26751,"Ġtemple":26752,"Ġbriefly":26753,"ĠTrip":26754,"classmethod":26755,"defaults":26756,"Ġchunks":26757,",,,,":26758,"ĠReason":26759,"$id":26760,"-ups":26761,"Ġdamn":26762,"Ġtrucks":26763,"Ġunlimited":26764,"Ġsculpt":26765,"ĠCards":26766,"Ġautor":26767,"ĠTesting":26768,"Ġdiese":26769,"shops":26770,"ç´":26771,"(payload":26772,"ĠPATH":26773,"ĠMemorial":26774,"Ġridiculous":26775,"egree":26776,"-winning":26777,"Ġrehab":26778,"Ġsophisticated":26779,"wpdb":26780,"ĉpath":26781,"!\";Ċ":26782,"_SYS":26783,".speed":26784,"Ġsoap":26785,"suffix":26786,"Wrap":26787,"Ġenhancement":26788,"Ãī":26789,"úb":26790,"Ġplaylist":26791,"Ġmixing":26792,"antidad":26793,"=\"\";Ċ":26794,"ĠRevision":26795,"ĠBeat":26796,".inc":26797,"-way":26798,"encias":26799,"ulers":26800,"Cat":26801,"idel":26802,"ĠShip":26803,".setColor":26804,"Ġthreatening":26805,".modules":26806,"Ġafterwards":26807,"ĠDashboard":26808,"ĊĠĊ":26809,"Signal":26810,"Ġprimer":26811,"orneys":26812,"iciary":26813,"Ġligne":26814,"_predict":26815,"Ġaest":26816,"_https":26817,">:":26818,"ĠLex":26819,"Ġrencontres":26820,"egral":26821,"scala":26822,"_family":26823,"ÃŁen":26824,"_sym":26825,"Ġuncertainty":26826,"ĠVALUE":26827,"Ġ};čĊčĊ":26828,"Ġbroader":26829,"Ġhorses":26830,"ãģĿ":26831,"ĠKal":26832,"oba":26833,"_INET":26834,"ĠKill":26835,"jquery":26836,"amination":26837,"[@\"":26838,"Ġmuj":26839,"###Ċ":26840,"FirstOrDefault":26841,"thenReturn":26842,"Che":26843,"/footer":26844,"Ġparks":26845,"asje":26846,"ĠGulf":26847,"Ġmodest":26848,".Init":26849,"ï¼ŁĊĊ":26850,"Ġprospects":26851,"Ġsvg":26852,"Ġåı":26853,".Dialog":26854,"_NET":26855,"Ġ(($":26856,"Ġek":26857,"ĠWarning":26858,"ĠMK":26859,"":27166,"ĠRepair":27167,"_BE":27168,"Brand":27169,"uart":27170,"preview":27171,"Ġinitiatives":27172,"running":27173,"bang":27174,"ĉupdate":27175,"ĠCoach":27176,"Rich":27177,"Ġyoutube":27178,"Ġritual":27179,"appa":27180,"ĠRobinson":27181,"precision":27182,"////////////////////////////////////////////////////////////////////////////":27183,"=[]Ċ":27184,"Ġcelebrated":27185,"OTO":27186,"Ġinclusion":27187,"JP":27188,"';čĊčĊ":27189,"Ġnotable":27190,"(_.":27191,"Managed":27192,"Ġguides":27193," ":27194,"atedRoute":27195,"ĠAdjust":27196,"Ġcolored":27197,"_scores":27198,"ĠTesla":27199,"_progress":27200,".inst":27201,"['_":27202,".flags":27203,"Ġfclose":27204,"_OPER":27205,"ży":27206,"_note":27207,"Ġtransgender":27208,"åķ":27209,"RIPT":27210,"Ġabsent":27211,"Ġamet":27212,"Ġoperand":27213,"ë©":27214,"Ġhood":27215,"toLowerCase":27216,"avo":27217,"ĠCircuit":27218,"ĠLind":27219,"--}}Ċ":27220,"=m":27221,"Ġsuppress":27222,"ĠMAP":27223,"iang":27224,"-admin":27225,"Ġsidebar":27226,"ĠBu":27227,"ĠHex":27228,",F":27229,"ĠSignal":27230,"Ġtransparency":27231,"ĠFederation":27232,"/V":27233,"Req":27234,"Ġpulse":27235,"Ġtends":27236,"Numbers":27237,"%'":27238,"Ġdeport":27239,"datas":27240,"_UINT":27241,"_tra":27242,"oko":27243,"Ġ\"?":27244,"compet":27245,"solete":27246,"undry":27247,"Ġoverlap":27248,"}`,Ċ":27249,".ly":27250,"_summary":27251,"ĠLost":27252,".Center":27253,"Ġdisability":27254,".Serialization":27255,"Ġgeom":27256,"Ġ?:":27257,"ĠWo":27258,"Ġshipped":27259,"Ĥæķ°":27260,"Ġugly":27261,"Ġexcitement":27262,"Ġexterior":27263,"Ġcheckout":27264,"Ġkur":27265,",D":27266,"ĠAlaska":27267,"Ġsynthetic":27268,"ĠBudget":27269,"ĠSubscribe":27270,"Ġ&Ċ":27271,"ÈĻi":27272,"ĠYu":27273,"ĉquery":27274,"}.Ċ":27275,"Ġtraged":27276,"assen":27277,"Ġaccommodation":27278,"Ġphysician":27279,"Ġrenamed":27280,"Ġtidak":27281,"zÄħ":27282,"Ġminus":27283,"nych":27284,"_EXCEPTION":27285,"threads":27286,"Ġtire":27287,"_created":27288,"ensure":27289,"Ġworthy":27290,"Ġexcuse":27291,"Ġcloth":27292,".parentNode":27293,"/platform":27294,"ĠUFC":27295,"ĠGtk":27296,"unny":27297,"Ġgibt":27298,"keley":27299,"hum":27300,"(tx":27301,"ĉdev":27302,"Ġoutfit":27303,"doors":27304,"Ġfon":27305,"icut":27306,"volatile":27307,"Ġhomosex":27308,"Maximum":27309,"Ġexpend":27310,"Ġ});ĊĊĊ":27311,"Eq":27312,"onders":27313,"department":27314,"ĠPhysics":27315,"\"});Ċ":27316,"Ġparad":27317,".Str":27318,"Ġsele":27319,"IFIED":27320,"Ġdelivers":27321,"ivan":27322,"Ġresponsibilities":27323,"Ġadvocates":27324,"èµ":27325,"ĠRID":27326,".parameters":27327,"Metrics":27328,"ronics":27329,"ĠUITableViewCell":27330,"Absolute":27331,"ipse":27332,"ylum":27333,"MLElement":27334,"_VALID":27335,"\\<^":27530,"Ġios":27531,"sound":27532,"\"];":27533,"Ġfreed":27534,"rottle":27535,"ĠLower":27536,"[count":27537,"åĿ":27538,"Ġpale":27539,"ĠWayne":27540,"earth":27541,"_categories":27542,"UCK":27543,".metadata":27544,"Ġsummon":27545,"HOME":27546,"олÑĮз":27547,"Ġmanufactured":27548,"Ġdock":27549,"Ġcompetitors":27550,"_MODEL":27551,"okia":27552,"ĠHey":27553,"ο":27554,"Ġbackward":27555,"ĠPOSS":27556,"ropa":27557,"Ġcri":27558,"_OBJ":27559,"Transport":27560,"-high":27561,"Ġerotik":27562,"_slot":27563,"Ġartic":27564,"_framework":27565,"-serif":27566,"ĠSqlDbType":27567,"')(":27568,"+\"/":27569,"Ġwore":27570,"Sil":27571,"Ġstoring":27572,"ĠPhase":27573,"uant":27574,"Ġbump":27575,"inho":27576,"Ġdign":27577,"Ġbacks":27578,"qq":27579,"(hash":27580,"Ġgeo":27581,"Ġtender":27582,"Logo":27583,"!)Ċ":27584,"ĠMX":27585,"ĠArthur":27586,"essoa":27587,"_Ch":27588,"Ġbedrooms":27589,"=\"#\"><":27590,"Ġthroat":27591,"insic":27592,".integer":27593,"Ġprimitive":27594,"Truthy":27595,"Ġfacilitate":27596,"Ġcreativity":27597,"ĠDNS":27598,"Ġgra":27599,"uez":27600,"Ġcountless":27601,"ĠPoland":27602,"'M":27603,"ĠDist":27604,"Ġvest":27605,"Ġcertification":27606,"á»ij":27607,"held":27608,"extensions":27609,"(static":27610,"Ġgrades":27611,"ĠUber":27612,"ãģŁ":27613,"Ġ[])Ċ":27614,"datos":27615,"ĠgetData":27616,"ĠCharg":27617,"ĠBS":27618,".microsoft":27619,".video":27620,".direction":27621,"->{'":27622,"lua":27623,"apest":27624,"Ġboiler":27625,"erek":27626,"Ġdecides":27627,".jar":27628,"ISC":27629,"ĠWords":27630,"(CON":27631,"EMPLATE":27632,"reeze":27633,"shots":27634,"apps":27635,"unted":27636,".setName":27637,"::<":27638,"-bold":27639,"ê²":27640,"å¯Ĩ":27641,"Longrightarrow":27642,"Ġunfair":27643,"Ġearning":27644,"Ġshelf":27645,"UREMENT":27646,"Ġidle":27647,"_MENU":27648,".Custom":27649,"AGER":27650,"-\"":27651,"_switch":27652,"because":27653,")view":27654,"mare":27655,"_condition":27656,"ĠStarting":27657,"Mvc":27658,"(pre":27659,"dump":27660,"_LOCK":27661,"atetime":27662,".callback":27663,"ĠCer":27664,"opol":27665,"ibrary":27666,"Ġreservation":27667,"ĉĉĉĉĉĉĉĊ":27668,"lector":27669,"graduate":27670,"Ġgenerous":27671,"Ġion":27672,"ricao":27673,"mq":27674,"_complete":27675,"(cursor":27676,"ĠFormControl":27677,":center":27678,"Ġsubstitute":27679,"ĠPlanning":27680,"Ġpension":27681,"Ġrecommendation":27682,"ĠTags":27683,"Ġgef":27684,"Ġalbums":27685,"Ġwashing":27686,"roc":27687,"Ġtrains":27688,"atings":27689,"Ġexponent":27690,"ackbar":27691,"-ln":27692,"ág":27693,".DataAnnotations":27694,"ĠEIF":27695,"ĠMalaysia":27696,"ĉPORT":27697,"onus":27698,"Ġclever":27699,"Ġpeu":27700,">ĊĊĊĊ":27701,"ĠArguments":27702,"Ġdebugging":27703,"(right":27704,"'D":27705,"compute":27706,"Ġfinest":27707,"ORAGE":27708,"Ġspectacular":27709,"phrase":27710,"Ġindia":27711,"Ġlegendary":27712,"birth":27713,"Ġcomposite":27714,"Ġgrows":27715,"ĠTD":27716,"Ġepid":27717,"Ġlaunching":27718,"]][":27719,"Minutes":27720,"ĠCha":27721,"Ġcleaned":27722,"Ġwitnesses":27723,"ukan":27724,"ĉType":27725,"Ġhabe":27726,"paragraph":27727,"ĠJPanel":27728,"ĠHann":27729,"Ġvaried":27730,"ĠPokemon":27731,"ĠMUST":27732,"åĬ¨":27733,".visibility":27734,"opup":27735,"^[":27736,".expand":27737,"Ġ\"',":27738,".fasterxml":27739,"_auto":27740,"ĠSheet":27741,"marker":27742,"Parcel":27743,"ews":27744,"ĠStrategy":27745,"-making":27746,"Ġunve":27747,"Ġtrailing":27748,"Ġclicks":27749,"ĠGetComponent":27750,"ĉcontent":27751,"IGENCE":27752,"ERNEL":27753,"NSMutableArray":27754,"Ġbreat":27755,"Ġharmful":27756,"¶Ī":27757,"Ġbesides":27758,"Ġboring":27759,"Ġbrutal":27760,"vang":27761,"(parse":27762,"quick":27763,"Ġpytest":27764,"Ġswitching":27765,"()]Ċ":27766,"ĠìĦ":27767,"LER":27768,"ĉfont":27769,"Ġnett":27770,")]ĊĊ":27771,"(/\\":27772,"æŀľ":27773,"toArray":27774,"Ġbreed":27775,"ĠCAR":27776,"ĠWeapon":27777,"Abs":27778,"tot":27779,"ĠsetName":27780,"aptive":27781,"Ġ:,":27782,"Ġescaped":27783,"orden":27784,"ĠPri":27785,"thumbnail":27786,"Ġdescriptions":27787,"/styles":27788,"ĠPCI":27789,"Ġalphabet":27790,"asticsearch":27791,"NOTE":27792,"Ġcialis":27793,"ĠGriff":27794,"Ġporque":27795,"Ġproteins":27796,"plays":27797,"Ġstating":27798,"Ġimagination":27799,"Ġfacial":27800,"ĠMechan":27801,"Ġarranged":27802,"_used":27803,"Ġarrangements":27804,"ĠPipe":27805,"hostname":27806,"Ġprovinc":27807,"Tit":27808,".FlatStyle":27809,"ĠSplit":27810,"ĠLoader":27811,".cc":27812,"Ġclinic":27813,"----------------------------":27814,"Ġbaking":27815,"ĠENT":27816,"neath":27817,"ãĢģĊĊ":27818,"ANE":27819,".EntityFrameworkCore":27820,"appers":27821,".ic":27822,"ĠNgModule":27823,"ĠFORM":27824,"Ġ';":27825,"-profit":27826,"hw":27827,"enemy":27828,"ĠEye":27829,"Ġcaution":27830,"town":27831,"Ġurged":27832,"ĠJimmy":27833,"ynchronous":27834,"-sized":27835,"making":27836,",{":27837,"]',":27838,"_Object":27839,"ahoma":27840,"Ġactivist":27841,"INVAL":27842,"ĠCommercial":27843,"ĠOrlando":27844,"(tab":27845,"Ġب":27846,"Algorithm":27847,"Ġheritage":27848,"GetMapping":27849,"Ġfailures":27850,"rios":27851,"ativa":27852,"Ġtet":27853,"Ġcarpet":27854,"(Z":27855,"three":27856,"Ġdisclosure":27857,".ERROR":27858,"_called":27859,"Ġdial":27860,"Ġoccasional":27861,".Err":27862,"Ġfuncion":27863,"caffold":27864,"Ġreleasing":27865,"ï¼īĊĊ":27866,"_Value":27867,"ĠVari":27868,"yellow":27869,"Ġstruggles":27870,".cal":27871,"ĠDakota":27872,"ĉclose":27873,"Ġsandwich":27874,"Ġanalytics":27875,"Ġ**)":27876,"&#":27877,"ĠJos":27878,"Ġpassive":27879,"ATTR":27880,"Throwable":27881,"ĠMun":27882,"ĠUint":27883,"(disposing":27884,"arak":27885,"ĠLeaders":27886,"Ġaffecting":27887,"ĠitemView":27888,"Ġeconomics":27889,"fv":27890,"à¹Ģ":27891,".rb":27892,"ĠOverall":27893,"Ġwealthy":27894,"Ġevolved":27895,"nda":27896,"ĠHus":27897,"restrict":27898,"umen":27899,"ĠAgricult":27900,"!ĊĊĊ":27901,"Ġexpires":27902,"Ġspokesperson":27903,"interval":27904,"Ġâ":27905,"Ġqueen":27906,"(nil":27907,"ingo":27908,"Heap":27909,"Ùİ":27910,"Ġcomplain":27911,"Sym":27912,"ĠClone":27913,"ĠRu":27914,"ĠWILL":27915,"ĠCrystal":27916,"/content":27917,"ingen":27918,"ointment":27919,"LastName":27920,"avicon":27921,"ĠIBM":27922,"ĠDimension":27923,"anh":27924,"icipants":27925,"ĠAnne":27926,".progress":27927,"Ġalgo":27928,"obil":27929,"ĠVoice":27930,"ĠFE":27931,"Ġgli":27932,"Ġved":27933,"Ġprevents":27934,"\\Column":27935,"Ġfolk":27936,"etti":27937,"Ġmn":27938,"ĠCLASS":27939,"Ġdisplaying":27940,"ĠKl":27941,"ĠFerr":27942,"duto":27943,".ib":27944,"Ġdados":27945,"'name":27946,"-space":27947,"Ġitalian":27948,"Ġinverse":27949,"Ġdense":27950,"uter":27951,"ĠIEnumerator":27952,"-sign":27953,"Ġnationwide":27954,"Ġpersona":27955,"Ġsolved":27956,"Ġdramatically":27957,"Logout":27958,"Ġgrav":27959,"Ġanalyses":27960,"ollo":27961,"Ġlamp":27962,".team":27963,"ĠErot":27964,"=[\"":27965,"Ġdancing":27966,"Ġ?>/":27967,"Ġcater":27968,"ffe":27969,"ĠSha":27970,"ĠBos":27971,"ĠREQUIRE":27972,"ĠMonster":27973,"ĠRB":27974,"ĠIDE":27975,"Ġsuits":27976,"ĠformData":27977,"(theta":27978,"Ġspatial":27979,"=NULL":27980,"ĠSqlConnection":27981,"Ġà":27982,"ĠVenez":27983,"ĠMorning":27984,"Ġpublications":27985,"ĠNONINFRINGEMENT":27986,"firstName":27987,"uds":27988,"Would":27989,"_HEAD":27990,"Ġinvested":27991,"stable":27992,"fred":27993,"Ġcommander":27994,"SES":27995,"âĢĶa":27996,"anche":27997,"ĠMovement":27998,"ë³":27999,"Suite":28000,"Ġjurisdiction":28001,"리":28002,"ĠBeth":28003,"jQuery":28004,"ĠIsa":28005,"Ġdental":28006,",*":28007,"ĠLimit":28008,"iliation":28009,"=\"{":28010,"bast":28011,"Ġturb":28012,"isy":28013,"OOK":28014,"Ġadvocate":28015,"imag":28016,"LECTION":28017,"лÑĮ":28018,"(category":28019,".dec":28020,"Ġuniqu":28021,"_sn":28022,"Ġattracted":28023,"ĠÃī":28024,"ĠRunning":28025,"_edges":28026,"ĠDisable":28027,"_AS":28028,"åĽ¾":28029,"Ġnetworking":28030,"_branch":28031,"Having":28032,"toBeTruthy":28033,"GI":28034,"Ġcamps":28035,"sep":28036,"-part":28037,"Ġ)ĊĊĊĊĊĊĊĊ":28038,"ustralia":28039,"ĠReports":28040,"rito":28041,"Ġwaist":28042,"_plus":28043,"ĠWW":28044,"-person":28045,"April":28046,"Ġsar":28047,".tar":28048,"Ġagricultural":28049,"tic":28050,"Ġtcp":28051,"ĠsetValue":28052,"agento":28053,"ĠAppe":28054,"piler":28055,"CADE":28056,"Ġanche":28057,"atcher":28058,"Ġcomics":28059,"Ġlbs":28060,"_segment":28061,"']=$":28062,"itters":28063,"icher":28064,"GINE":28065,"Ġutilize":28066,"ĠCursor":28067,"_expression":28068,"Ġdag":28069,"x":28257,".Task":28258,"money":28259,"ibaba":28260,"'});Ċ":28261,"ĠSpecific":28262,"ĠLinear":28263,"_OPT":28264,"HashCode":28265,"(Player":28266,".ContainsKey":28267,"Ġcollapsed":28268,"transparent":28269,"_RANGE":28270,"Viewer":28271,"(cfg":28272,"Ġsorting":28273,"Ġinfected":28274,"ĠNach":28275,"Ġaccommodate":28276,".elements":28277,"_PART":28278,"ĠSexy":28279,"=get":28280,"(year":28281,"Ġxhr":28282,":]":28283,"owski":28284,"Ġsummar":28285,"Ġ¿":28286,"Ġinte":28287,"Ġworkflow":28288,"ĠTaiwan":28289,"versions":28290,"åıij":28291,"Ġsurprisingly":28292,"Ġoptical":28293,"Ġproces":28294,"Ġdisagree":28295,"Ġnuevo":28296,"ĠCAM":28297,"sorted":28298,"leases":28299,"istle":28300,"Ident":28301,"ĉevent":28302,"jected":28303,"Chunk":28304,"Vars":28305,".provider":28306,"Ġproceedings":28307,"Ġinclusive":28308,"Ġartwork":28309,"endants":28310,"ï¼ļĊ":28311,"seen":28312,"Ġlig":28313,"Ġmakers":28314,"_fun":28315,"Ġlengths":28316,"PathVariable":28317,"[item":28318,"ี":28319,"Dead":28320,"FFFFFF":28321,"ĠUrban":28322,"uples":28323,"ichen":28324,"(nullptr":28325,".spec":28326,",System":28327,"URATION":28328,"(job":28329,"å¼ı":28330,"Ġtracker":28331,"ÅĻ":28332,"ĠMR":28333,"ĠSQLite":28334,"Ġdto":28335,"Ġ;;Ċ":28336,"Ġmint":28337,"ĠIntroduction":28338,"cao":28339,"Ġquestioned":28340,"Ġfitted":28341,"revision":28342,"sq":28343,"Ġmig":28344,"_units":28345,"_async":28346,"Ġflick":28347,"});ĊĊĊ":28348,"Ġnotre":28349,"}`,":28350,"Filters":28351,"Ġmundo":28352,"_days":28353,"Ġfrm":28354,"utc":28355,"Ġvals":28356,"ewidth":28357,"ĠGenerator":28358,"ĠArtist":28359,"ĠIDs":28360,"ĠArticles":28361,"reater":28362,"ĠComponentFixture":28363,".=":28364,"Ġrou":28365,"-no":28366,".bukkit":28367,"egg":28368,"ĠDiff":28369,"atics":28370,"ÑĥÑĩ":28371,"âĢĶĊĊ":28372,"ĠCharlotte":28373,"bye":28374,"Ġ});čĊčĊ":28375,"ĠVik":28376,"ĠBrow":28377,"Ġlv":28378,"ĠGib":28379,"-wing":28380,"GLIGENCE":28381,"(Il":28382,"ĠEngineer":28383,".Wait":28384,"ĠPictures":28385,"Ġrhet":28386,"Ġthermal":28387,"Ġpraise":28388,"<>();ĊĊ":28389,"ĠSpider":28390,"Pause":28391,"ĠBaker":28392,"Ġslower":28393,"Ġ}]Ċ":28394,"_enqueue":28395,"Ġdisappeared":28396,"ĠTicket":28397,"INUX":28398,"_LOCAL":28399,"аÑģÑģ":28400,"@Injectable":28401,"community":28402,"GestureRecognizer":28403,"åĽ½":28404,"Ġscales":28405,"Ġ-(":28406,"/'+":28407,"ĠSit":28408,"Ġexecutives":28409,"arding":28410,"Ġadvers":28411,"Ġbackwards":28412,"ĉcontext":28413,"ĠHamp":28414,"ĠPF":28415,"ĠDeck":28416,"ĠCraig":28417,"American":28418,"Ġbell":28419,"Ġprol":28420,"ufen":28421,"Ġrng":28422,"arshal":28423,"ĠSimply":28424,"firstname":28425,"shore":28426,"July":28427,"Ġmortality":28428,"ĠâĨĴĊĊ":28429,"Helpers":28430,"Ġbenchmark":28431,"emade":28432,"Ġorganisations":28433,".gson":28434,"ĠTextField":28435,"Ġcivilians":28436,".Arrays":28437,"ĠMississippi":28438,"Ġintermediate":28439,"getUser":28440,"_cluster":28441,"Relative":28442,"foreign":28443,".querySelectorAll":28444,"ForeignKey":28445,"Ġreasonably":28446,"---------Ċ":28447,"Cards":28448,"ĠKam":28449,"ĠThor":28450,"Ġroller":28451,"-element":28452,"ĠCurrency":28453,"ddie":28454,"ALLY":28455,"ĠRA":28456,"Ġpermet":28457,"aaaa":28458,"Ġhomework":28459,"ĠVit":28460,"Ġmold":28461,"ĠFer":28462,"[start":28463,"Ġstatistical":28464,"Ġscary":28465,"_HOME":28466,".Begin":28467,"Construct":28468,"ogenic":28469,"ĠDEALINGS":28470,"Ġtambién":28471,"ixon":28472,".ind":28473,"acre":28474,"Ġtransforms":28475,"ĠNap":28476,".Block":28477,"ussia":28478,"piration":28479,"ulent":28480,"Ġceil":28481,"Clause":28482,"naire":28483,"TES":28484,"Ġneat":28485,"STD":28486,"ĠRegExp":28487,"perform":28488,":)":28489,"Ġunions":28490,"Ġsublic":28491,"Ġwinds":28492,"loating":28493,"glich":28494,"Ġpagination":28495,"Skill":28496,"Apply":28497,"ĠOperator":28498,"istogram":28499,"Ġqualities":28500,"Cross":28501,"Ġdecom":28502,"],\"":28503,"ĠJuan":28504,".modal":28505,".Child":28506,"ĠRoger":28507,"STITUTE":28508,":CGRectMake":28509,"alette":28510,"Ġsta":28511,"aside":28512,"Ġblur":28513,"ĠWa":28514,"ifetime":28515,"reed":28516,"controls":28517,"Ġbins":28518,"Ġпол":28519,"*/,Ċ":28520,"UIS":28521,"ĠRou":28522,"ĠDemo":28523,"-awesome":28524,"ĠChain":28525,"Ġhasta":28526,"ĠBart":28527,".KEY":28528,"Ġvendors":28529,"nofollow":28530,"ĠDest":28531,"_builder":28532,"Ġargues":28533,"_answer":28534,"goto":28535,"ĠRESULT":28536,"ĠMON":28537,"Ġpoder":28538,"oons":28539,"_CASE":28540,"Ġreplic":28541,"Ġfinancing":28542,"ĠDATE":28543,"cern":28544,"_track":28545,"ties":28546,"/logo":28547,"ĠNEGLIGENCE":28548,"getType":28549,">T":28550,"bet":28551,"girl":28552,"ĠINCIDENTAL":28553,"-site":28554,".trigger":28555,"ĠLisa":28556,"_inputs":28557,"Ġrelatives":28558,"LoggedIn":28559,"Configure":28560,"IK":28561,".accept":28562,"Resume":28563,"ĠDraft":28564,"Ġ*>(":28565,"ĠWA":28566,"edian":28567,"erness":28568,"ĠLayoutInflater":28569,"*/čĊčĊ":28570,"othy":28571,"Ġobligation":28572,"Subscribe":28573,"Ġthumbnail":28574,"exist":28575,"Ġinsisted":28576,"ĠUICollectionView":28577,"ĠAngular":28578,"Ġtablets":28579,"ĠImpact":28580,"ãĢįĊĊ":28581,"aho":28582,"Ġcharacteristic":28583,"gd":28584,"Ġ=================================================":28585,"ourt":28586,"`.":28587,"Appro":28588,"Coordinate":28589,"Remember":28590,"Ġmarine":28591,"]=='":28592,"ĠAdministrator":28593,".getDefault":28594,"Ġforgot":28595,"ĠStructure":28596,"Vue":28597,"arsing":28598,"moment":28599,"kw":28600,"_cursor":28601,"Attack":28602,"Ġathletic":28603,"Ġdiagnosed":28604,"Ġende":28605,"åĪłéĻ¤":28606,"House":28607,"ĠPARAM":28608,"Ġwiki":28609,"ĠOpp":28610,"Ġconservation":28611,"Ġsnd":28612,"_tem":28613,"substr":28614,"ĠCape":28615,".sim":28616,"UTION":28617,"anan":28618,"âĢĻun":28619,"Ġgy":28620,"-work":28621,"Ġcompelling":28622,"='#":28623,"ĉsub":28624,"Ġdirectories":28625,"íĬ¸":28626,"Ġtouches":28627,"outines":28628,".Collection":28629,"schedule":28630,".lat":28631,"ĠDoctrine":28632,"CAA":28633,"ĠRefer":28634,"Ġshifts":28635,"Ġlikelihood":28636,"preter":28637,"ĠFemale":28638,"Ġintercept":28639,"Ġlou":28640,"çĻ»":28641,"Ġrug":28642,"ĠCrown":28643,"Ġ****************************************************************************":28644,"-product":28645,"Ġprompted":28646,"ungle":28647,"docker":28648,"ĠTu":28649,"ĠUnique":28650,"_Error":28651,"ulos":28652,"ĠâĦ":28653,"Ġ(`":28654,"Getting":28655,"_scal":28656,"ĠEnh":28657,"üt":28658,"Ġsustained":28659,"Ġpatches":28660,"Ġprosper":28661,"ĠGaza":28662,"_light":28663,"Ġincons":28664,"--------Ċ":28665,"ĉĉĠĠĠĠĠĠ":28666,"SF":28667,"CN":28668,":\";Ċ":28669,"ĠCollins":28670,"(*)":28671,"Ġcompilation":28672,"']čĊ":28673,"Ġconsequence":28674,",...":28675,"Ġdm":28676,"ĠBLOCK":28677,"Cluster":28678,"Ġski":28679,"(argc":28680,"Tuple":28681,"Ġjoins":28682,"ĠSheriff":28683,"War":28684,"indi":28685,"Ġcommented":28686,"HOST":28687,"Ġinvitation":28688,"apanese":28689,"Ġpermits":28690,"precedented":28691,"_zone":28692,"ĠAmy":28693,"_RD":28694,"Minimum":28695,"Ġinvocation":28696,".enable":28697,"ichten":28698,"-owned":28699,"\"id":28700,"_POINTER":28701,"Fac":28702,"Ġspecifications":28703,"Ġnomination":28704,"Ġgp":28705,"<(":28706,"Ġrobots":28707,"ĠJerry":28708,"Ġholders":28709,"Ġwand":28710,"cms":28711,"Ġ}))Ċ":28712,".Toast":28713,"ĠIList":28714,"Based":28715,"zoom":28716,"/style":28717,"ĠBeck":28718,"Men":28719,"Ġcontributing":28720,"Ġundo":28721,"ĠOH":28722,"ĠaddObject":28723,"Ġeigen":28724,"signup":28725,"éĶĻ":28726,"Ġdistant":28727,"PARATOR":28728,"ĠMari":28729,"Ġmá":28730,"Emp":28731,"ós":28732,"ĠìĪĺ":28733,"evt":28734,"+j":28735,"park":28736,"ĠStay":28737,"ĠDun":28738,"Ġsoy":28739,">%":28740,"azines":28741,"Ġtiempo":28742,"(me":28743,"present":28744,".This":28745,"Ġeditors":28746,"FIELD":28747,".Work":28748,"ĠUniverse":28749,"Ġdrunk":28750,".timer":28751,"Ġaltered":28752,"ĠNar":28753,"ëł¥":28754,".Active":28755,"idor":28756,"çŃ":28757,".deltaTime":28758,"Ġawkward":28759,""":28760,"ĠSafari":28761,"Ġtricks":28762,"MENTS":28763,"division":28764,"Ġvarying":28765,"ĠHighway":28766,"Ġphotographer":28767,"ĠStewart":28768,"Ġlasting":28769,".Pre":28770,".amazonaws":28771,"ĠLuck":28772,".Description":28773,"ĠNaz":28774,"neg":28775,"Ġcó":28776,"<<\"\\":28777,"ĠSurv":28778,"ĠUnc":28779,"Recipe":28780,".BorderStyle":28781,"Ġmodifications":28782,"-at":28783,"ATFORM":28784,"hdr":28785,"ako":28786,"Ġsublicense":28787,"ĠJump":28788,"Ġbeim":28789,"ĠManhattan":28790,".bool":28791,"_hw":28792,"ÑĤÑĮ":28793,"Bin":28794,"Ġgateway":28795,"\"\":":28796,"ĠUIS":28797,":\"+":28798,"-def":28799,"ĠRegular":28800,"/testing":28801,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":28802,"stringstream":28803,"Ġdispar":28804,"Ġmobil":28805,"-read":28806,"ĠAdapter":28807,"ĠChampions":28808,"Ġscheduler":28809,"Ġkills":28810,"ĠMultiple":28811,"irror":28812,"Ġgods":28813,"ADO":28814,"akte":28815,"ĠUsuario":28816,".circular":28817,"Ġrecept":28818,"ĠExpr":28819,"Ġelderly":28820,"Ġnicely":28821,"Ġbeste":28822,"Want":28823,"Ġclassical":28824,".sprite":28825,"objc":28826,"ĠMason":28827,"Ġsistema":28828,".Black":28829,"eso":28830,"ĠZeit":28831,"Ġdivid":28832,"Ġenters":28833,"_subject":28834,"ĠPlanet":28835,".warning":28836,"ĠGram":28837,"_tokens":28838,"Ġhouseholds":28839,"_customer":28840,"userName":28841,"cross":28842,"Ġpione":28843,"Ġassists":28844,"_SM":28845,"ibo":28846,"Ġloyal":28847,"Ġuseless":28848,"#elif":28849,"ĠUltimate":28850,"Come":28851,"gel":28852,"Ġdich":28853,"xyz":28854,"ikel":28855,"obra":28856,"_scan":28857,"ĠInterior":28858,"ĠNice":28859,"Ġplac":28860,"ĉtarget":28861,"Ġviral":28862,"asso":28863,"()/":28864,"unde":28865,"ĠAdobe":28866,"Os":28867,"visited":28868,"ĠOW":28869,"ĠFeed":28870,"ĠSequence":28871,"Ġmanages":28872,"inson":28873,"ĠLouisiana":28874,"{})":28875,"ĠHab":28876,"ĠLD":28877,"Ġbip":28878,"prites":28879,"(elem":28880,".hibernate":28881,"élé":28882,"Ġohne":28883,"_transaction":28884,"Ġannunci":28885,"Published":28886,"ĠHonda":28887,"ĠTam":28888,"ĠPacket":28889,"_selector":28890,"Ġchallenged":28891,"Processing":28892,"-hover":28893,"Ġtrainer":28894,"_cancel":28895,"ĠNSDictionary":28896,"abric":28897,"ĠMLS":28898,"_sensor":28899,"Ġshrink":28900,"ĠFX":28901,"threshold":28902,"ĉHX":28903,"-mark":28904,"`.`":28905,"Scheme":28906,"(full":28907,"_writer":28908,"ĠSys":28909,"Ġfled":28910,"ĠCin":28911,"-widget":28912,"ĠPrevious":28913,"Gender":28914,"_question":28915,"Feed":28916,"Ġscrut":28917,"(prefix":28918,"ãĢĤãĢĤ":28919,"Ġinfections":28920,"Parts":28921,"Ġhierarchy":28922,"_DELETE":28923,"ĠPatient":28924,"_pay":28925,"Ġpromoted":28926,"Ġìĭ":28927,"Ġcivilian":28928,"Ġagriculture":28929,"ĠPiece":28930,"Ġstance":28931,"utsche":28932,"Assign":28933,".ACTION":28934,"Fig":28935,"_radius":28936,"ĠSync":28937,"ducer":28938,"failure":28939,"ensed":28940,"ptime":28941,"BM":28942,"_datetime":28943,"quivo":28944,"QUEUE":28945,"èĢħ":28946,"Appear":28947,"Ġsummit":28948,":void":28949,"Ġvine":28950,"认":28951,"onne":28952,"_TRANS":28953,".green":28954,"_cc":28955,"Ġhungry":28956,"Ġ\">":28957,"());čĊčĊ":28958,"Extract":28959,"izens":28960,"Ġsolver":28961,"Notify":28962,"Ġenglish":28963,"ĠShopping":28964,"interfaces":28965,"REQ":28966,"Ġilleg":28967,"ĠUIImageView":28968,"Ġdisconnect":28969,"ĠUntil":28970,"ĠConservative":28971,"@Column":28972,"Ġshifted":28973,"Ġ:čĊ":28974,"Ġfich":28975,"Ġdla":28976,"Ġshoe":28977,"\"),čĊ":28978,"ularity":28979,"_RESP":28980,"Weather":28981,"UIApplication":28982,".iterator":28983,"Ġaging":28984,".Parent":28985,"owie":28986,"(equal":28987,"ĠConv":28988,"/default":28989,"Ġmeasuring":28990,".prev":28991,".IsValid":28992,".Fat":28993,"ĠsÄĥ":28994,"keywords":28995,"without":28996,"Ġsovere":28997,"Ġexchanges":28998,"Ġmelt":28999,"Ġislands":29000,"ĠIntegr":29001,"Ġjumping":29002,"Ġgle":29003,"Ġjournalism":29004,"Ġdated":29005,"Localized":29006,"ĠRefresh":29007,"Particle":29008,"Ġaa":29009,"ĠSTRICT":29010,"Ġbod":29011,".Process":29012,"_AUTO":29013,"ĠPublished":29014,"every":29015,"Ġtechnological":29016,"lsx":29017,"Ġirrit":29018,"Additional":29019,"Ġdelimiter":29020,"_language":29021,"-area":29022,"boys":29023,"ĠTube":29024,"Ġwat":29025,"Ġmechanics":29026,"_owner":29027,"Spell":29028,"ĠStories":29029,".AppendLine":29030,"TableView":29031,"hem":29032,"stick":29033,"ollower":29034,"IFF":29035,"ĠUV":29036,"ollision":29037,"SUB":29038,"Ġcomparable":29039,"Ġdonde":29040,"sales":29041,"llvm":29042,"Ġ}],Ċ":29043,"OTTOM":29044,"ĠPurpose":29045,"Lab":29046,"Ġinterviewed":29047,"ois":29048,"asil":29049,".setId":29050,"ĠInstruction":29051,"-->":29052,"ĠModified":29053,"ationally":29054,"ĠMeeting":29055,"误":29056,"#region":29057,"Ġrouting":29058,".focus":29059,"ĠYouth":29060,"<":29348,"Ġunto":29349,"ologically":29350,"ĠMul":29351,"VIDIA":29352,"Ġslim":29353,"ĠCommissioner":29354,"(on":29355,"Ġunderneath":29356,"/db":29357,"vote":29358,"(Message":29359,"ĠPope":29360,"Defined":29361,"Ġswift":29362,"urf":29363,"Ġadapted":29364,"SEL":29365,"Ġrevenues":29366,"Ġdivine":29367,"=y":29368,"Gradient":29369,"_act":29370,"Ġ/*!<":29371,"Ġpolygon":29372,"ĠFDA":29373,"ĠCarr":29374,"atables":29375,"(stdout":29376,"Ġrefriger":29377,"Ġcoordin":29378,"avorites":29379,"ÑĪи":29380,"Ġcompassion":29381,"ĠPOSSIBILITY":29382,"-secondary":29383,"uracy":29384,"Ġcompromise":29385,"_AV":29386,"_os":29387,"Ġbeside":29388,"ĥĿ":29389,"Ġln":29390,".plugins":29391,"Capacity":29392,"alah":29393,".bin":29394,"ĠCRC":29395,"_balance":29396,"ĠflexDirection":29397,"Ġambit":29398,"Ġnickname":29399,"ĠForces":29400,"CLE":29401,"ĠShell":29402,"Ġsail":29403,"ĠWriter":29404,"ĠAlice":29405,"dw":29406,"ĠIndians":29407,"ĠMarshall":29408,"_SRC":29409,"Ġnormalized":29410,"ĠJag":29411,"ãĤĴ":29412,"zeit":29413,"rpc":29414,"ÃŃc":29415,".inline":29416,"Ġtravers":29417,"_numeric":29418,"Ġutilities":29419,"Ġevac":29420,"INPUT":29421,"ĉregister":29422,"MX":29423,"ĠCampbell":29424,"Ġdatasets":29425,"Ġdemanded":29426,"ĠinitialState":29427,"gan":29428,"Ġei":29429,"Unexpected":29430,"-web":29431,"trait":29432,",Y":29433,"ĠTodd":29434,"Ġskeleton":29435,"Ġoptimize":29436,"第":29437,"ĠUpon":29438,"ĠStObject":29439,"Ġaplic":29440,".'P":29478,"vron":29479,".UN":29480,"Ġpainter":29481,"izarre":29482,"Ġlav":29483,"Ġpom":29484,"preg":29485,"=function":29486,"(serial":29487,"ifica":29488,"uming":29489,"åľ°":29490,"ãģĤ":29491,"-op":29492,"UCH":29493,"ĠHend":29494,".propTypes":29495,"Ġyo":29496,"Ġroutines":29497,"Ġcaring":29498,"Sem":29499,"Ġreserves":29500,"Ġpriorities":29501,"redits":29502,"ISTR":29503,"ContentType":29504,"ĠSchw":29505,"/media":29506,"Ġestr":29507,"Ġclimbing":29508,"-week":29509,"cherche":29510,"sensor":29511,"ToArray":29512,"ĠMontreal":29513,"Ġclouds":29514,"ĠInjectable":29515,"ĠRice":29516,"Ġpropaganda":29517,"_provider":29518,"Ġindoor":29519,"Ġinaug":29520,"Ġdiplom":29521,"Ġmessaging":29522,"_mut":29523,"å¦Ĥ":29524,"Ġkw":29525,"ONS":29526,"arians":29527,"RPC":29528,")]čĊ":29529,"-ray":29530,"ĠSor":29531,"mall":29532,"Ġmarketplace":29533,"Ġvtk":29534,"Ma":29535,"ogan":29536,"igi":29537,"Ġsponsored":29538,"ĠDani":29539,".SEVER":29540,">'.$":29541,"multipart":29542,"ĠWol":29543,"ĠtableName":29544,"ĠUsername":29545,"BackgroundColor":29546,"Ġfright":29547,"_EMAIL":29548,"September":29549,"_vals":29550,"opia":29551,"Ġspotted":29552,"-Ch":29553,"ĠdataSource":29554,"/\"Ċ":29555,"екÑĤ":29556,"ĠRequestMethod":29557,"ĠReplace":29558,"-do":29559,"ahn":29560,"ĠPhD":29561,"].ĊĊ":29562,"NON":29563,"gement":29564,"ĠThr":29565,"Ġquietly":29566,"Ġtorture":29567,"Ġteas":29568,"ĠCY":29569,"Ġatr":29570,"development":29571,"-detail":29572,"Ġlighter":29573,"Ġarguing":29574,"Ġdeserves":29575,"Ġcurriculum":29576,"_CONTEXT":29577,"ÅĤy":29578,"HITE":29579,"ĉID":29580,"/uploads":29581,"Ġtits":29582,"reo":29583,"_drop":29584,".UTF":29585,"Ġpickup":29586,"Ġgrocery":29587,"ĠPure":29588,"Ġeasiest":29589,"Phil":29590,".feature":29591,"(\"*":29592,"Ġinvestor":29593,"tok":29594,"Ġjar":29595,"Los":29596,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":29597,".queue":29598,"-speed":29599,"Mal":29600,"umblr":29601,"ĠCONST":29602,"ĠHRESULT":29603,"ĠDance":29604,"(filePath":29605,"Ġattributed":29606,"à¥į":29607,"ĠBund":29608,"coins":29609,"Ġsão":29610,"Ġpir":29611,"personal":29612,"Ġprelim":29613,"Ġpropose":29614,"ĠTL":29615,"]])":29616,"ĠSubscription":29617,"ĠKre":29618,",len":29619,".FirstOrDefault":29620,")--":29621,"_products":29622,".GetBytes":29623,"Ship":29624,"Ġencrypt":29625,"ĠSG":29626,"ĠMyst":29627,"hir":29628,"Ġiterate":29629,"Ġintend":29630,".mockito":29631,"Ġchapters":29632,"(angle":29633,"ĠVlad":29634,"设":29635,"'.ĊĊ":29636,"ResponseBody":29637,"ĠAbd":29638,"deal":29639,"Ġbarriers":29640,"-outline":29641,"bill":29642,"ĠFalls":29643,"_second":29644,".include":29645,".ceil":29646,"Ġoccupation":29647,"phony":29648,".moveTo":29649,"ĠJennifer":29650,"ASTER":29651,";\"><":29652,"ĠEnabled":29653,"Ġterminate":29654,"ĠIo":29655,"lations":29656,"ĠTHEORY":29657,"Ġearliest":29658,"Ġrack":29659,"ĠScar":29660,"shake":29661,"chip":29662,"Ġuv":29663,"Ġalliance":29664,"пиÑģ":29665,"ĠGOODS":29666,"zione":29667,"ĠVI":29668,"Ġ{-":29669,"Ġfiltering":29670,"Ġmiscon":29671,".DockStyle":29672,"Ġbush":29673,"Ġjunk":29674,"æĮ":29675,"ĠQUE":29676,"Ġhooks":29677,"Ġfirmware":29678,"Ġmiddleware":29679,"dic":29680,"ĠOakland":29681,"Ġarrives":29682,"Payload":29683,"pixel":29684,"]|":29685,"ĠstartDate":29686,".PRO":29687,"_audio":29688,"Ġmidfield":29689,"igidbody":29690,"ĠSwiss":29691,"ĠClip":29692,"ĠDump":29693,"ĠTextBox":29694,"Ġgeh":29695,"yield":29696,"ods":29697,"Ġreferendum":29698,"Backend":29699,"ĠCream":29700,"Ġdominated":29701,"ĠArchive":29702,"Ġriders":29703,".prepareStatement":29704,"Ġquando":29705,"Ġchef":29706,"wiki":29707,"inel":29708,"ampling":29709,"(\"\\\\":29710,"Ġsag":29711,"_proxy":29712,"ãģķ":29713,"pdo":29714,".getElementsByTagName":29715,"Ġdemonstration":29716,"ĠNPC":29717,"Ġarchivo":29718,"endance":29719,"Ġefficiently":29720,"(actual":29721,".tableView":29722,"Ġmush":29723,"Ġbears":29724,"_threads":29725,"jas":29726,"ahun":29727,"Ġneural":29728,"Ġdesigning":29729,"ĠGDP":29730,"Ġlifted":29731,"缮":29732,"ĠJoint":29733,"ĠInclude":29734,"ĠGiants":29735,"Ġwithdrawal":29736,"ĠRent":29737,"native":29738,"ĠSeek":29739,"gression":29740,"_CPU":29741,"\\S":29742,"ĠShield":29743,"Ġsolic":29744,"Ġboom":29745,"yecto":29746,"Ġmanufacture":29747,"ĠâĢĭ":29748,"Ġbbox":29749,"Ġearthqu":29750,"ollectors":29751,":@\"%":29752,"Ġloops":29753,"Je":29754,"alking":29755,"ĠWhats":29756,"ĠBoys":29757,".book":29758,"ARGE":29759,"_pixel":29760,"Ġsuspects":29761,"ι":29762,"usp":29763,"ĠBMW":29764,"ieces":29765,"(person":29766,"å¼Ģ":29767,"é»":29768,"ĠPodcast":29769,"Ġbou":29770,"(Item":29771,"û":29772,"(Input":29773,"HttpGet":29774,"Ġburg":29775,")^":29776,"BOARD":29777,"*/,":29778,"Ġgulp":29779,"ĠBenn":29780,"Ġdecks":29781,".statusCode":29782,"Ġacute":29783,"Ġhug":29784,"ugu":29785,"Ġpled":29786,",\"%":29787,"hape":29788,"Ġзап":29789,"ĠMaine":29790,".real":29791,"Ġdalam":29792,"ĠMinor":29793,".Float":29794,"disp":29795,"Ġtl":29796,"Ġencount":29797,"=>$":29798,"Ġfg":29799,"tees":29800,"ĠRecomm":29801,"äl":29802,"Ġchemistry":29803,"Blocks":29804,"OID":29805,"Ġforex":29806,"ĠAppend":29807,"Ġ{*":29808,"ĠSupply":29809,"CGFloat":29810,"(bl":29811,"Ġate":29812,"adora":29813,"Ġgust":29814,"Associ":29815,">.Ċ":29816,"FETCH":29817,".serial":29818,"widgets":29819,"ardless":29820,"iefs":29821,"_FULL":29822,"ernetes":29823,"ĠPred":29824,"ØŃ":29825,"äºĭ":29826,"ubernetes":29827,"ĠLaura":29828,"Ġlabeled":29829,"Highlight":29830,"Ġannoying":29831,"/update":29832,"(description":29833,"Ġintimid":29834,"$c":29835,"\")))Ċ":29836,".AP":29837,"Ġ[]*":29838,"ĠEXIT":29839,".Host":29840,"ĠOPEN":29841,".sendMessage":29842,"_camera":29843,"_tile":29844,"Ġtherm":29845,"onomous":29846,"Ġdisadv":29847,"Ġnaar":29848,"indexOf":29849,"ĠPP":29850,".protocol":29851,"AFE":29852,"Ġtextures":29853,"################################################":29854,"umbai":29855,".stats":29856,"ĠGE":29857,"Ġie":29858,"ĠSTD":29859,"ĠMann":29860,".reflect":29861,"KB":29862,"Ġdive":29863,".wav":29864,"/*----------------------------------------------------------------":29865,"/settings":29866,".lifecycle":29867,"Ġdaughters":29868,"orus":29869,"uber":29870,"NING":29871,"stri":29872,"ĠTip":29873,"Ġzn":29874,"Ġswitched":29875,"inet":29876,"uffy":29877,"ĠTransportation":29878,"(conf":29879,"frica":29880,"ĠXL":29881,"ĠLead":29882,"_percent":29883,"__":29899,"permissions":29900,"ĠDetermine":29901,".Man":29902,"Ġadvances":29903,".InputStream":29904,"Ġstrongest":29905,"ĠeBay":29906,"Ġ#-":29907,"Ġdirname":29908,"ĠSMS":29909,"Ġmedications":29910,"Ġamended":29911,"Ġchurches":29912,"ĠImperial":29913,"$row":29914,"ĠMadison":29915,"ĠInsp":29916,"Ġaffair":29917,"Ġpsychology":29918,"vh":29919,"Ġseverity":29920,"âĢIJ":29921,"Ġstrips":29922,"AH":29923,"vertising":29924,"Ġconse":29925,"IMAGE":29926,"ĠStats":29927,"ĉsc":29928,".Cursor":29929,"Ġfreeze":29930,"sson":29931,"(xml":29932,"ĠSusan":29933,".tile":29934,"eded":29935,"ĠĠĠĠĉĉĉ":29936,"uelle":29937,"ĠMitchell":29938,"based":29939,"Operand":29940,"½æķ°":29941,"ĠFF":29942,"ĉstrcpy":29943,"ounces":29944,"ildo":29945,".executeQuery":29946,"Ġapproaching":29947,"ĠSeven":29948,"Ġnuts":29949,"Ġric":29950,"assignment":29951,"Ġcalculator":29952,"ĠMurphy":29953,"ĠBou":29954,"íĦ":29955,"Ġbutt":29956,"Ġticks":29957,"Projects":29958,"ilib":29959,".textColor":29960,"mov":29961,"_logo":29962,"(template":29963,"ĠINIT":29964,"ĠimageView":29965,"scriptions":29966,"ORITY":29967,"Consumer":29968,"Ġunprecedented":29969,"Ġtourist":29970,"Ġbron":29971,"Ġcontractor":29972,"Ġlicence":29973,"ĠNam":29974,"æ¯":29975,"(transform":29976,"_ATT":29977,"Pref":29978,"ĠGam":29979,"Ġvessels":29980,"Ġhav":29981,"Later":29982,".ToLower":29983,"Ġurls":29984,"Ġbreakdown":29985,"Ġpenalties":29986,"Ġfoster":29987,"ĠUE":29988,"Ġclue":29989,"comed":29990,"åIJį称":29991,"-main":29992,"Ġpts":29993,"Ġcounted":29994,"icts":29995,"/post":29996,"Ġgetattr":29997,"Ġping":29998,"ANCEL":29999,"Ġpec":30000,"Ñħод":30001,"antom":30002,"ĠBlueprint":30003,"ĠEventEmitter":30004,"Ġlä":30005,"æ²":30006,"Ġstraw":30007,"(comp":30008,"'une":30009,">N":30010,"-client":30011,"esModule":30012,"-base":30013,"Ġretreat":30014,"_simple":30015,"ĉĉĉĉĉĉĠ":30016,"fee":30017,"')čĊčĊ":30018,"ControlItem":30019,"Ġsubscribers":30020,"please":30021,"ĠEff":30022,"Ġpound":30023,"ĠBytes":30024,"ĠTea":30025,"_activity":30026,"Ġmaxim":30027,"Ġopcode":30028,"BSD":30029,".constant":30030,";}":30031,"ombres":30032,"Ġcareers":30033,").ĊĊĊĊ":30034,"Ġspreading":30035,"-expanded":30036,"ĠOrd":30037,"amarin":30038,"Ġmobility":30039,"Unfortunately":30040,"akk":30041,"NL":30042,"_redirect":30043,"ĠPG":30044,"ĠSensor":30045,"bol":30046,"tap":30047,"_MEMORY":30048,"ĠUIAlert":30049,"plitude":30050,"Website":30051,"ĠLogo":30052,"love":30053,"[ind":30054,"Ġaltogether":30055,"Ġwondered":30056,"Ġesper":30057,"ĠLiberal":30058,"Ġoss":30059,"Ġelit":30060,"Ġstiff":30061,"odox":30062,"_mentions":30063,"ĠDouglas":30064,"_pid":30065,"ĠCK":30066,"ĠinitWithFrame":30067,".blog":30068,"pkg":30069,"anghai":30070,"QUIRED":30071,"uu":30072,"Ġmkdir":30073,"ATAL":30074,"Ġunh":30075,"inces":30076,"sth":30077,"Ġhypothesis":30078,"Ġcata":30079,"ĠTB":30080,"ĠClar":30081,"Ġpredecess":30082,"Ġsituated":30083,"-world":30084,"))/":30085,"Ġheadlines":30086,".stat":30087,"Ġoutbreak":30088,"spath":30089,"_FLAGS":30090,"ĠServletException":30091,"Sun":30092,"FROM":30093,"ĠDir":30094,"ãĥ»ãĥ»ãĥ»":30095,"_coord":30096,"ĠOptim":30097,"Monitor":30098,".bit":30099,"XXX":30100,"Ġtodas":30101,"feld":30102,"ÑĢи":30103,"imir":30104,"Ġpolitically":30105,"Ġmolecular":30106,"Ġtraded":30107,"Ġ{{$":30108,"ĠSwedish":30109,"Ġ'@/":30110,"_REAL":30111,"Ġwarehouse":30112,"today":30113,",L":30114,"orp":30115,"false":30392,"Ġspa":30393,"ĠNear":30394,"ìķ":30395,"Ġintrig":30396,"_members":30397,"wave":30398,"Ġanalysts":30399,"_OS":30400,"edin":30401,"ĠFri":30402,"Ġretrieved":30403,"Regular":30404,"_obs":30405,"EXPORT":30406,"')}}\"":30407,"\"class":30408,"__((":30409,"bucket":30410,"Ġstro":30411,"ĠPatch":30412,"ystick":30413,"fulness":30414,"apos":30415,"Da":30416,"ĉĉĉĉĉĠĠĠ":30417,"Ġenrich":30418,"unordered":30419,"hole":30420,"Cong":30421,"';ĊĊ":30463,"STRUCT":30464,"QR":30465,"IDs":30466,"(arguments":30467,"_aux":30468,"(Event":30469,"_PRIVATE":30470,"ĠTrek":30471,"Ġdownloads":30472,"mutable":30473,"_STRUCT":30474,"(wx":30475,"Ġdomains":30476,"jspx":30477,"ĠViagra":30478,"Commands":30479,"Js":30480,".cfg":30481,"ContentPane":30482,"ĠEditText":30483,"à¥įà¤":30484,"Attach":30485,"ĠARM":30486,"positive":30487,"ĠGenerated":30488,"Ġseized":30489,"=:":30490,"Ġelectronics":30491,"ĠAppComponent":30492,"/',Ċ":30493,".equalsIgnoreCase":30494,"Doctrine":30495,"disk":30496,"ĠPolitical":30497,"CHO":30498,"":30584,"ĠBeauty":30585,"Ġ`<":30586,"Ġtouching":30587,"Ġ|--":30588,"ĉflag":30589,"normalize":30590,"Ġtrapped":30591,"Ġestablishing":30592,"/build":30593,"AJ":30594,"fy":30595,"-react":30596,"avn":30597,"RIPTION":30598,"Ġkut":30599,"ĠFashion":30600,"ĠInform":30601,"curities":30602,"{Ċ":30634,"Ġgarlic":30635,"Ġrepr":30636,"Ġreplies":30637,"(prop":30638,"Ġspirits":30639,"Ġinspire":30640,"Ġbasement":30641,".reject":30642,"Ġhints":30643,"Ġpolling":30644,"ĉĠĊ":30645,"_rating":30646,"Ġcath":30647,"avier":30648,"Ġcompressed":30649,"ĠVS":30650,"]'":30651,"Ġjudicial":30652,"ĠTrend":30653,"training":30654,"ESTAMP":30655,"ognition":30656,"Äģ":30657,"SENT":30658,"ventions":30659,"Ġconsultant":30660,"umph":30661,"ĠuserService":30662,",NULL":30663,"kh":30664,"Dear":30665,"_BAD":30666,"itations":30667,"Ġmetaph":30668,"'é":30669,"andise":30670,"-font":30671,".chart":30672,"Ġsg":30673,"_Controller":30674,".jpeg":30675,"ĠULONG":30676,"ĉgame":30677,"(ss":30678,"ĠMaj":30679,"ĉgo":30680,"ĠSad":30681,"ĠBerg":30682,"ĠMine":30683,"Pack":30684,"Ġresistant":30685,"ĠROM":30686,"Ġpeg":30687,"ĠStanford":30688,"ĠYahoo":30689,"Ġscaled":30690,"Ġlan":30691,"=[]":30692,"\"/>ččĊ":30736,"Ġsud":30737,"ĉbackground":30738,"Ġscholars":30739,"-muted":30740,"ará":30741,"Ġ=====":30742,"Ġ____":30743,"Creat":30744,"enever":30745,"/wp":30746,"ĠVPN":30747,"ErrorCode":30748,")],Ċ":30749,"(builder":30750,"ĠEnemy":30751,"Sensor":30752,"usa":30753,"Ġtriggers":30754,"Ġplayoffs":30755,"_REQ":30756,"Ġ(~":30757,"ĠBarry":30758,"Ġpermanently":30759,"ĠRUN":30760,"Ġbure":30761,".Fatalf":30762,"Ġchick":30763,"ĉpanic":30764,"psi":30765,"oka":30766,"éĢī":30767,">[":30768,"Ġunderstands":30769,"ĠJunior":30770,"ĠINFO":30771,"=mysqli":30772,"ustain":30773,"-source":30774,"serv":30775,"ĠCREATE":30776,".au":30777,"Ġsells":30778,"ĠĠĊĠĠĊ":30779,"Europe":30780,"zw":30781,"preh":30782,"ĠNSA":30783,"Ġxy":30784,"ิ":30785,"ĠBeyond":30786,"Instead":30787,"NonQuery":30788,"Ġarise":30789,"Ġavoided":30790,".emplace":30791,"_models":30792,"}),Ċ":30793,"Ġhid":30794,"Ġ&_":30795,".points":30796,".getWidth":30797,".Exec":30798,"Ġ////":30799,"ĠSessions":30800,"...\\":30801,"ĠColomb":30802,"Ġacceleration":30803,"restore":30804,"Ġile":30805,"obic":30806,"}Ċ":31296,"plaint":31297,"getText":31298,"Ġindividually":31299,"Ġcheckbox":31300,"UY":31301,"ĠLamb":31302,"Ġdysfunction":31303,"ĠLar":31304,"à°":31305,"ĠCreating":31306,"');ĊĊĊ":31307,"\"They":31308,"locations":31309,"_CORE":31310,"Interaction":31311,"umbnails":31312,"ĠPartner":31313,"brit":31314,"Ġlesser":31315,"ĠSlot":31316,"setAttribute":31317,"ĠWave":31318,".po":31319,"/store":31320,"Ġbrowsing":31321,"_pd":31322,"sume":31323,"sed":31324,"Curve":31325,"Ġplasma":31326,"Ġsuspicious":31327,"ìĿ¸":31328,"ĠBah":31329,"ĠExplicit":31330,"_CC":31331,".ClientSize":31332,"\\View":31333,"Ġsubstit":31334,"loon":31335,"ĠGAME":31336,"ĠBrid":31337,"Ľå»º":31338,"_User":31339,"Ġsquares":31340,"fone":31341,"Ġsacred":31342,"ughs":31343,"]interface":31344,"ĠThrow":31345,"ĠKirk":31346,"Ġempire":31347,"Ġassessed":31348,"Tax":31349,"ĠHeaven":31350,"-buffer":31351,"_STATIC":31352,"éné":31353,"-bordered":31354,"Ġpunct":31355,"(mode":31356,"Ġkeine":31357,"Sent":31358,"ĠCalcul":31359,"ĠEve":31360,"Ġstylish":31361,"Ġoils":31362,".TestCase":31363,"Ġtrademark":31364,"Ġliterary":31365,"Ġconcentrations":31366,"ĠRelations":31367,"(Class":31368,"Ġstdin":31369,"Ġvæ":31370,"backup":31371,".VERSION":31372,".AutoScaleDimensions":31373,"starter":31374,"Transactional":31375,"-panel":31376,"Studio":31377,"kc":31378,"ĠChamber":31379,"ĠSpiel":31380,"Ġrho":31381,"اÙĦ":31382,"!'":31383,".Attributes":31384,"Ġmurdered":31385,"apeutic":31386,"Ġintimate":31387,"ĠtextField":31388,"ĠBuffalo":31389,"dummy":31390,"\"%":31391,"ĠLiberty":31392,"obar":31393,"ĠTank":31394,"ĠPopular":31395,"ervisor":31396,"ĠIniti":31397,"ĠMall":31398,"ĠPrior":31399,"CAP":31400,"ĠClay":31401,"ĠCertificate":31402,".Lock":31403,"-strip":31404,"-driven":31405,"/all":31406,"ĠMessageBoxButtons":31407,"_SECRET":31408,"_pb":31409,"Ġrats":31410,"ाà¤":31411,"Ġnt":31412,".Router":31413,"_topic":31414,"Ġtennis":31415,"ĠPUBLIC":31416,"ĠActivatedRoute":31417,"Ġ',Ċ":31418,"Ġcostume":31419,"Ġjokes":31420,".Handle":31421,"ĉbyte":31422,"Ġflavors":31423,"(cc":31424,"Ġpersonas":31425,"ĉimage":31426,"ĠNazi":31427,"Ġgrammar":31428,"Ġúlt":31429,"Ġvalve":31430,"Ġvic":31431,"ĠRachel":31432,"_invalid":31433,"Prefs":31434,"stdint":31435,"(route":31436,"Ġhtmlspecialchars":31437,"Ġpeoples":31438,"pline":31439,"Ġnv":31440,"ĠQuant":31441,"oppers":31442,"ĠcurrentUser":31443,"ĠCatal":31444,"Ġreconc":31445,"Ġconjunction":31446,"lx":31447,"amburg":31448,"Ġinfluential":31449,"danger":31450,"inders":31451,"Ġ%@\",":31452,".configuration":31453,"osome":31454,".identity":31455,"Ġpicker":31456,"nost":31457,"ĠDIY":31458,"August":31459,"ablo":31460,"Leaf":31461,"ĠReco":31462,"cko":31463,"DOC":31464,"ĠHerm":31465,":any":31466,"ĠInterview":31467,"ĠTex":31468,"xfe":31469,"(work":31470,"Ġleap":31471,"Heading":31472,"Ġquarters":31473,"\\Bundle":31474,"reb":31475,"Perhaps":31476,"ĠGmbH":31477,"Birth":31478,"ĉsum":31479,"ĠWatson":31480,".nil":31481,"ç¡":31482,"{}ĊĊ":31483,"icaid":31484,"Getter":31485,"\"name":31486,"Ġ\"čĊ":31487,"_none":31488,"zm":31489,"acute":31490,"uesto":31491,"Ġsous":31492,"Ġrebuild":31493,"Ġnewspapers":31494,"ĠHaz":31495,"Ġkits":31496,"ifo":31497,"Blur":31498,"Ġsuited":31499,"-In":31500,"à¯":31501,"ĠKeith":31502,"ĠNorway":31503,"INIT":31504,"ireccion":31505,"ieties":31506,"_usage":31507,"ĠDoug":31508,"rise":31509,"Ġtrillion":31510,"imited":31511,"ĠREL":31512,"alic":31513,"Ġcriticized":31514,"theorem":31515,"Ġcease":31516,"Ġsidew":31517,"ĠTerry":31518,"Ġsubsidi":31519,"Ġfirmly":31520,"Ġaws":31521,"Ġhott":31522,"Ġdressing":31523,"badge":31524,"ĠApplications":31525,"è¿ĶåĽŀ":31526,"Ġlaughed":31527,"Ġhobby":31528,"Ġmusicians":31529,"Ġ*.":31530,".placeholder":31531,"Ġcounters":31532,"ĠCapitol":31533,"SDK":31534,"Ġhelmet":31535,"andbox":31536,"quit":31537,"Ġcriminals":31538,"Ġteenager":31539,"(update":31540,"Gl":31541,".selection":31542,"Ġdischarge":31543,"Ġpresenting":31544,"ufacturer":31545,"_UNKNOWN":31546,"Ġstressed":31547,"åĻ¨":31548,"Proto":31549,"_correct":31550,"haus":31551,"Ġrenov":31552,"Ġfirearms":31553,"Ġtechnically":31554,"-browser":31555,"Ġcandy":31556,"Stroke":31557,"Ġexecutor":31558,"Ġoccurrence":31559,"ĠIPv":31560,"_INTERFACE":31561,"ĠRetrieve":31562,".bad":31563,"Exchange":31564,"Navbar":31565,"ĠKid":31566,"(getApplicationContext":31567,"_STOP":31568,"ĠBoss":31569,"Listeners":31570,"Ġshooter":31571,"ĠAlb":31572,"äch":31573,"Ġpix":31574,".keyCode":31575,"alone":31576,"Ġabsurd":31577,"ĠCum":31578,"ĠNewtonsoft":31579,"ikt":31580,"Ġlaughing":31581,"Ġcapitalism":31582,"reeNode":31583,"Tx":31584,"_QUERY":31585,".Sleep":31586,"(login":31587,"WebElement":31588,"Ġcelebrating":31589,"Ġdeprecated":31590,"Ġmaar":31591,"Ġartistic":31592,"_ASSOC":31593,"ĠBorderRadius":31594,"ĉwp":31595,"Ġsurvivors":31596,"Inner":31597,"-red":31598,"Ġprosecution":31599,"_pp":31600,"(\"$":31682,"Ġcomma":31683,"unchecked":31684,"graphics":31685,"rors":31686,"GROUND":31687,"(public":31688,"Ġcustomized":31689,"ĠArkansas":31690,"ĠRew":31691,"Ġexpiration":31692,"×ķ":31693,"ĠCul":31694,"Ġnons":31695,".Filter":31696,"Ġsenator":31697,"_definition":31698,"ashington":31699,"ymph":31700,"/J":31701,"Ġfuse":31702,"ramid":31703,"ĠSupplier":31704,"Ġautocomplete":31705,"Ġ}),":31706,".\"ĊĊĊ":31707,"_functions":31708,"ĉto":31709,".eval":31710,"ĠTObject":31711,"References":31712,"Ġheated":31713,"HAL":31714,"Ġ))}Ċ":31715,"}$":31716,"ĠBarr":31717,"_UNIT":31718,"+$":31719,"ĠgetValue":31720,"iped":31721,"chied":31722,"(vm":31723,"cue":31724,"_integer":31725,"_course":31726,"third":31727,"Ġrevised":31728,"**/Ċ":31729,"_DIRECT":31730,"OutOf":31731,"(\"(":31732,"ĠFeel":31733,"Ġreass":31734,"Ġsubtitle":31735,"peri":31736,"nf":31737,"Ġenjoys":31738,"Ġtreats":31739,")this":31740,"-tabs":31741,"ancers":31742,"Ġcontinent":31743,"Ġcardio":31744,"Ser":31745,".question":31746,"Ġphrases":31747,"Validators":31748,"Ġpopul":31749,"ĠlÃŃ":31750,"song":31751,"_INTERNAL":31752,"Ġadviser":31753,"Ġpuzz":31754,"Ġambitious":31755,"ĠTob":31756,"ĠDP":31757,"Ġpresidency":31758,"Ġsurrender":31759,"Ġwatches":31760,"_binary":31761,"ĠSoon":31762,"Ġcanada":31763,"(\"\")Ċ":31764,"]='":31765,"ĠBrandon":31766,"epsilon":31767,"rw":31768,".addChild":31769,".Copy":31770,"Principal":31771,"Photos":31772,"Ġmarginal":31773,"Ġbasics":31774,"eing":31775,"Must":31776,"_String":31777,"Ġole":31778,"Magento":31779,".customer":31780,"(prev":31781,"ล":31782,"Ġloyalty":31783,"Cog":31784,"Ġprotocols":31785,"ĠCompanies":31786,"Ġtheoretical":31787,"Ġaccessing":31788,"ĠZen":31789,".ones":31790,"attice":31791,"_world":31792,"zes":31793,"Ġtattoo":31794,"Ġmenos":31795,"Ġintersect":31796,"\"];ĊĊ":31797,"belie":31798,"Ġinactive":31799,".readline":31800,"-labelled":31801,".done":31802,"lickr":31803,"ĠWORK":31804,"Ġderivative":31805,"Ġdatabases":31806,"âĤĤ":31807,"Ġsx":31808,".isArray":31809,"Ġys":31810,"Ġpada":31811,"ĠBullet":31812,"(`/":31813,"isActive":31814,"ĠCGSize":31815,"(equalTo":31816,"ĠColumbus":31817,"Ġmarry":31818,"DEV":31819,"_limits":31820,"rones":31821,"IAS":31822,"Ġtau":31823,"mino":31824,"_Write":31825,"ĠWine":31826,"Ġ[['":31827,"ĠPull":31828,"riters":31829,"rients":31830,"Ġshifting":31831,"upp":31832,"_TIMER":31833,"ĠConditions":31834,"ấ":31835,"ĠOrders":31836,"ĠStrength":31837,"æīĢ":31838,"Ġvalidity":31839,"Ġfot":31840,"etur":31841,"Ġbolt":31842,"åĨħ":31843,"ĠAlong":31844,"oshi":31845,"Ġassumptions":31846,"Ġmagazines":31847,"_SPI":31848,"Ġpunt":31849,"_PRODUCT":31850,"Ġrelay":31851,"ĠJavascript":31852,".te":31853,"-es":31854,"Ġwidgets":31855,"(fs":31856,"\";":31923,"atching":31924,"ĠKnowledge":31925,"ĉThe":31926,";margin":31927,"lessness":31928,"opard":31929,"umatic":31930,"()));čĊ":31931,"Ġfals":31932,"(cache":31933,"TypeId":31934,"éĢļ":31935,"_choice":31936,"ĠGoth":31937,"ĠSites":31938,"MG":31939,"_border":31940,"Indices":31941,"Comparer":31942,"ĠRedistribution":31943,"Ġcloset":31944,"Ġversatile":31945,"Inputs":31946,"********************":31947,"Ġobesity":31948,"quiz":31949,"gra":31950,"(global":31951,"åĬ¡":31952,"Ġcollector":31953,"Ġkor":31954,"ovable":31955,"ADC":31956,"ĠEventHandler":31957,".nc":31958,"Ġplayback":31959,"ientos":31960,"_perm":31961,"_WARNING":31962,"ĠOlympics":31963,".norm":31964,"ĠBroadcast":31965,"_small":31966,"drive":31967,".iloc":31968,"Ġtyped":31969,"MEM":31970,"_cons":31971,"DMETHOD":31972,"Ġlun":31973,".distance":31974,"(par":31975,"poon":31976,"Ġbast":31977,"activities":31978,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":31979,":čĊčĊ":31980,"SER":31981,")&&":31982,"_lst":31983,"ĠPolish":31984,"Ġknocked":31985,"Ġfrustration":31986,"aukee":31987,"Ġphosph":31988,"iquid":31989,"_coeff":31990,"æѤ":31991,"Latest":31992,"ĠDust":31993,"Tipo":31994,"Ġmaintains":31995,"Ġmarsh":31996,"incinn":31997,"lbl":31998,"Care":31999,"Ġneighborhoods":32000,"_gpio":32001,"ĠArsenal":32002,"Dem":32003,"ĠWhe":32004,"_hook":32005,"Ġldc":32006,"ĠHarper":32007,"ĠBerkeley":32008,"Ġgraduated":32009,"Percent":32010,"Ġarriving":32011,"ĠAdventure":32012,"(scope":32013,"('*":32014,"quarter":32015,"ĠMarie":32016,"Speaking":32017,"_codegen":32018,"Ġimmun":32019,"caster":32020,"ãĤĮ":32021,"åķĨ":32022,"ĠDimensions":32023,".record":32024,"Ġtexto":32025,"ĠMichelle":32026,"Pending":32027,"(by":32028,"_PAR":32029,"ucht":32030,"bee":32031,".Thread":32032,"ampire":32033,"know":32034,"ĠClinical":32035,"ĠmarginBottom":32036,"Ġdistinguish":32037,".Full":32038,".undefined":32039,"ĠSequelize":32040,"############################################################################":32041,"Ġeducated":32042,"_OVER":32043,"åºı":32044,"ĠÂłĠÂł":32045,"_each":32046,"Ġurge":32047,"depart":32048,"Ġdonors":32049,"ĠAu":32050,"Ġbillions":32051,"Ġbelonging":32052,"_age":32053,"_Int":32054,"Ġsubstances":32055,"machine":32056,"!!!ĊĊ":32057,"Ġjsonify":32058,"ibbean":32059,"ĠCad":32060,"ĠendTime":32061,"Ġcycling":32062,"ĠUITextField":32063,"Ġleverage":32064,"Ġvanilla":32065,"eat":32066,"Launch":32067,"(pt":32068,"states":32069,"ĠControls":32070,"ĠRespons":32071,"ĠJake":32072,"Ġasleep":32073,"fortunate":32074,".nextLine":32075,"SizeMode":32076,"ìĿ¼":32077,"TestingModule":32078,"German":32079,"ĠInvestig":32080,".reverse":32081,"ĠBACK":32082,"(DateTime":32083,"Ġnonprofit":32084,"ĠExpect":32085,"Ġtanto":32086,"']),":32087,"ĉthe":32088,"Multiple":32089,"(getActivity":32090,"_WAIT":32091,"Ġjá":32092,"decor":32093,"levance":32094,"ĠGitHub":32095,"mination":32096,"_quantity":32097,".Scanner":32098,"ĠLion":32099,"éĶĻ误":32100,"Ġdre":32101,"Ġtantra":32102,"ĠcontentType":32103,"Ġfid":32104,"_alt":32105,"NSIndexPath":32106,"-pl":32107,"åĮĸ":32108,"Ġantibiot":32109,"tables":32110,"acial":32111,"ĠRegistry":32112,"Ġolive":32113,"igers":32114,"Ġsubscriber":32115,"_pres":32116,"ĠSyntax":32117,"Ġlovers":32118,".Byte":32119,"olders":32120,"_forward":32121,"always":32122,"Caption":32123,"Priv":32124,"ĠTampa":32125,"isateur":32126,"-labelledby":32127,"ĠToString":32128,"ĠìĤ¬":32129,"Ġinitiated":32130,"WF":32131,"Ġinstitutional":32132,"inject":32133,"ĠScr":32134,"Ġdoctrine":32135,"Ġspacious":32136,"isure":32137,"ĠAna":32138,"\"time":32139,"essaging":32140,"Ġcid":32141,"ĠNan":32142,"Ġincomplete":32143,"TAG":32144,"-build":32145,"December":32146,"Ġresidual":32147,"(PDO":32148,"ĠListen":32149,"Ġglyph":32150,"Ġgaps":32151,"nea":32152,".Rect":32153,"Ġsau":32154,"ĠPhotograph":32155,"Ġexecutable":32156,"ĠExpert":32157,"Coroutine":32158,"_sizes":32159,"ĠNL":32160,".isValid":32161,");}Ċ":32162,"-reg":32163,"Ġciting":32164,"cwd":32165,"ĠOttawa":32166,"ĠBatt":32167,"Ġrenewable":32168,"Ġpreliminary":32169,"Ġasylum":32170,"Ġwrist":32171,"Ġutiliz":32172,"Ġdetention":32173,"Fast":32174,"Ġange":32175,"incinnati":32176,"Ġsteering":32177,"ĠNaN":32178,"iosity":32179,"/page":32180,"Ġè¿":32181,"sterol":32182,"Ġdisg":32183,"(DB":32184,"ĠDESCRIPTION":32185,"Ġ_$":32186,"Ġobstacle":32187,"Ġbizarre":32188,"Ġextraction":32189,"_expected":32190,"Ġloses":32191,"ĠCelebr":32192,"ĠhtmlFor":32193,"Ġexploit":32194,"олÑĮзов":32195,"XYZ":32196,"Ġmagnet":32197,"amped":32198,"Ġatoms":32199,"Sources":32200,"pectives":32201,"Ñģли":32202,"Ġ=čĊ":32203,"Ġdare":32204,"ĠWalter":32205,"Ġbrightness":32206,"Ġannotations":32207,"ëı":32208,"iske":32209,"Schedule":32210,".images":32211,"rosso":32212,"Ġ\"..":32213,"gamma":32214,"Ġinstructor":32215,"Ġoverwrite":32216,"-am":32217,"Ġdevastating":32218,"ĠSaints":32219,"Ġhs":32220,"Ġbonuses":32221,"$output":32222,"ijd":32223,"(ActionEvent":32224,"monitor":32225,"Ġmattress":32226,"January":32227,".jp":32228,"Ġcaracter":32229,"Ġimpose":32230,"_rest":32231,"ĠSignature":32232,"Ġcoronavirus":32233,"ãģĬ":32234,"_compare":32235,"Measure":32236,"itated":32237,"elijk":32238,"igos":32239,"esar":32240,"Ġrushed":32241,"metry":32242,"_SEPARATOR":32243,"_WE":32244,"_ATTRIBUTE":32245,"Ġyaml":32246,"Ġspecs":32247,"ĠRah":32248,"pheric":32249,"ĠInvestment":32250,"äll":32251,"Ġappealing":32252,"Ġviewport":32253,"ç©":32254,"ĠmarginLeft":32255,"Ġsubtract":32256,"ĠEDIT":32257,"ĉArrayList":32258,"grading":32259,"ĠFailure":32260,"asper":32261,"EEK":32262,"(now":32263,")Ċ":32279,"Collision":32280,"ĠGreater":32281,"ĠRacing":32282,"alan":32283,"Ġmonetary":32284,",new":32285,"ĠSorry":32286,".Enable":32287,"ĠInstantiate":32288,"ollen":32289,"ë©´":32290,"ĠCalling":32291,"_hour":32292,"ADA":32293,"Ġshy":32294,")**":32295,"Ġ==>":32296,"Ġespecial":32297,"Ġinterpreted":32298,"!=\"":32299,"Ġpharmacy":32300,".single":32301,"ĠCialis":32302,"Ġparas":32303,".toUpperCase":32304,"ĠDemon":32305,"Prime":32306,"Ġrankings":32307,"Adding":32308,"_HASH":32309,"ĠExam":32310,"Ú©":32311,"ĠVictor":32312,"Okay":32313,"\"];čĊ":32314,"Ġfortune":32315,"ĠFETCH":32316,"expand":32317,".Interop":32318,"Ġbarn":32319,"æ¶Ī":32320,"uevo":32321,"Ġspeculation":32322,"âĶĢâĶĢâĶĢâĶĢ":32323,"ĠNu":32324,"ĠBlues":32325,"(fname":32326,"Ġinhabit":32327,"Ġ\\\"%":32328,"CES":32329,"ulario":32330,"_cr":32331,"Ġvalidated":32332,"Ġmidnight":32333,"anking":32334,"Ġincorporate":32335,"Ġpursuit":32336,"EXP":32337,"prime":32338,"Pid":32339,"-US":32340,"ĠNurs":32341,"ĠWheel":32342,"éĺ":32343,"Ġinp":32344,"Ġsupportive":32345,".member":32346,"ĠShot":32347,".CheckBox":32348,"Ġaffirm":32349,"Tor":32350,"FullYear":32351,"Ġconsiderably":32352,"credentials":32353,"_opts":32354,"Roll":32355,"(round":32356,"Ġcoment":32357,"_UART":32358,"Ġextending":32359,"RG":32360,"resultado":32361,"itu":32362,".getSession":32363,"Ġattraction":32364,"&D":32365,"$html":32366,"ĠJessica":32367,"ĠAssociate":32368,"añ":32369,"_ed":32370,"ĠLag":32371,"Ġorigins":32372,"())->":32373,"addEventListener":32374,"IALOG":32375,"åIJ¦":32376,".Compare":32377,"Album":32378,"ĠKu":32379,"\";ĊĊ":32423,"quisite":32424,"channels":32425,"/res":32426,"ĠAnalytics":32427,".appcompat":32428,"/to":32429,"ĠonError":32430,"(attr":32431,"IRM":32432,"Ġragaz":32433,"-as":32434,".Second":32435,"oriented":32436,"Ġdonn":32437,"Ġlightning":32438,"fid":32439,"ĠPle":32440,"ãģ¾ãģĻ":32441,"tro":32442,".True":32443,"Observable":32444,"×Ļ":32445,"umbing":32446,"Ġprospective":32447,"-filter":32448,"Ġpursuant":32449,"(points":32450,".Bind":32451,"Ġpalm":32452,"clearfix":32453,"ös":32454,"ĠGonz":32455,"Ġweaken":32456,"Drive":32457,"enido":32458,"lld":32459,"obox":32460,"anean":32461,"Got":32462,"ä¿Ŀ":32463,"Regex":32464,"æĥ":32465,"Ġsalad":32466,"assis":32467,"\"net":32468,"inheritDoc":32469,"ĠRV":32470,"quier":32471,"Ġclazz":32472,"Ä±ÅŁ":32473,"osterone":32474,"Ġairline":32475,".listdir":32476,"Ġdownloading":32477,"ĠPalm":32478,"waukee":32479,"<":32480,".BL":32481,"_INLINE":32482,"offs":32483,"<<(":32484,"_news":32485,"Ġchase":32486,"/><":32487,"Ġeuros":32488,"ĠEgyptian":32489,"ĠStainless":32490,"_BOOL":32491,"ĠGuild":32492,"ĠDynam":32493,"[indexPath":32494,"Ġï":32495,"Ġmemorable":32496,"ĠChampion":32497,"ResourceManager":32498,".Login":32499,"ĠFormer":32500,"yped":32501,"Ġlleg":32502,";\",":32503,"DWORD":32504,"Ġtaxi":32505,"Ġbombs":32506,"rah":32507,".tags":32508,"_tests":32509,"stones":32510,"âĢĿ)":32511,"[g":32512,"rtype":32513,"Ġvu":32514,"Ġhostile":32515,"Chars":32516,"ĠPatriots":32517,"/status":32518,"());Ċ":32872,"ajÄħ":32873,"_OCC":32874,"Ġplanets":32875,"æŁ¥":32876,"ĠDublin":32877,"Ġserie":32878,".printf":32879,"deep":32880,"`)":32881,"Ġ\\$":32882,"Ġμ":32883,"_VIDEO":32884,"endors":32885,"ĠCrypto":32886,"Far":32887,".Transparent":32888,".TR":32889,"iasm":32890,"_training":32891,"Ġteaches":32892,"ĠBelt":32893,"Ġlimiting":32894,"ĠKath":32895,"ĠIndexPath":32896,"Ġachievements":32897,"Ġserá":32898,"interopRequire":32899,"Ġdisse":32900,".If":32901,"arming":32902,"ulsion":32903,"Po":32904,"_DETAIL":32905,"Prototype":32906,"ĠCAL":32907,"Ġagrees":32908,".vo":32909,".ExecuteNonQuery":32910,"ĠTopic":32911,"Ġ'{}":32912,"Arm":32913,"Ġecc":32914,"Mag":32915,"Ġserialized":32916,"ĉconn":32917,"cached":32918,"=tf":32919,"ĠByteArray":32920,"protobuf":32921,"varchar":32922,"ĉASSERT":32923,"Ġliste":32924,"_trigger":32925,"·¸":32926,"Feel":32927,"Tahoma":32928,"ĠLik":32929,"Ġstructured":32930,"ergus":32931,".Initial":32932,"_ge":32933,"cljs":32934,".contact":32935,"Ġandere":32936,"$stmt":32937,"_CURRENT":32938,"ĠDiscover":32939,"$res":32940,"formatter":32941,"Ha":32942,"vangst":32943,"Ġemerge":32944,"ãĢĤâĢĿ":32945,"ĠCabinet":32946,"-square":32947,"éĥ¨":32948,"Ġrage":32949,"ĠAJ":32950,"ĠVT":32951,"shadow":32952,"ĠFaith":32953,"enames":32954,"pretty":32955,"hasil":32956,"party":32957,"Ġvarchar":32958,"Ġfotos":32959,"Ġalum":32960,"ĠBelgium":32961,".ylabel":32962,"Ġdej":32963,"_numbers":32964,"Ġhu":32965,".setAdapter":32966,"ĠUsually":32967,"(sample":32968,".Shared":32969,"Ġbooked":32970,"Ġ>>=":32971,"Ġminerals":32972,"\">":32991,"prog":32992,"boo":32993,"_md":32994,"_pack":32995,"(express":32996,"utz":32997,"\\Auth":32998,",id":32999,"ĠChile":33000,"actice":33001,"Ġrecruitment":33002,"Ġposes":33003,"Ġvulnerability":33004,"instanc":33005,"orum":33006,"dess":33007,"Ġxl":33008,"%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%":33009,"(fig":33010,"Ġdeleting":33011,".del":33012,")')Ċ":33013,"ĠWeekly":33014,"???":33015,"(strcmp":33016,"smith":33017,"Ġpursuing":33018,"-so":33019,"ĠApps":33020,"/'Ċ":33021,"Ġdecis":33022,"FORE":33023,"Everyone":33024,"Ġlanes":33025,"Virtual":33026,".attach":33027,"(Log":33028,"ĠMedicaid":33029,"(Path":33030,"ĠTurner":33031,"/application":33032,"Ġportrait":33033,"Ġoppose":33034,"checkout":33035,"Ġfinishes":33036,"_ME":33037,"Barrier":33038,"Song":33039,"VAR":33040,"Earlier":33041,"rella":33042,"Ġhast":33043,"azar":33044,"Ġpulls":33045,"ngx":33046,"Ġinspiring":33047,"ÑĥÑİ":33048,"-direction":33049,"Ġexplosive":33050,"ĠcreatedAt":33051,"sto":33052,"Ġwheat":33053,"ĠBuilt":33054,"'ai":33055,"Ġtracked":33056,"hammad":33057,"RowAtIndexPath":33058,"_heap":33059,"Due":33060,"Ġconnects":33061,".publish":33062,"emu":33063,"Ġbullets":33064,"BAR":33065,"olate":33066,"Ġinternally":33067,"Ġcatching":33068,"-password":33069,"ouched":33070,"æĢ§":33071,"eous":33072,"Ġxrange":33073,"Quality":33074,"vv":33075,"Manage":33076,"(($":33077,"acements":33078,"ĠBrothers":33079,"ĠHEAD":33080,"ĠUnsupported":33081,"san":33082,"esi":33083,"***Ċ":33084,"Ġadaptation":33085,"ĠWorker":33086,"']/":33087,".savefig":33088,"(trans":33089,"ج":33090,"nee":33091,"Correct":33092,"...\")Ċ":33093,"Ġsubmitting":33094,"-path":33095,"ĉlast":33096,"issan":33097,".xlabel":33098,"ĠSepar":33099,"/no":33100,"_best":33101,"ĠMills":33102,"_sock":33103,"(flag":33104,"Ġdestinations":33105,"emption":33106,"ĠFAIL":33107,"åĴĮ":33108,"Ġrp":33109,"fact":33110,"ĉlen":33111,"DAY":33112,"Ġseiz":33113,"_dst":33114,"lip":33115,".Linear":33116,"ĠBasket":33117,"$t":33118,"$i":33119,"-brand":33120,"ĠNeil":33121,"ĠEq":33122,"Ġthou":33123,"ogene":33124,"Ġscholarship":33125,"æĽ´":33126,"Ġswo":33127,"aginator":33128,"eni":33129,"(book":33130,"Ġblink":33131,"thus":33132,"ĠcancellationToken":33133,"ĠPalestinians":33134,"Ġprofitable":33135,"Ġbackpack":33136,"enson":33137,"true":33284,"ĠNYC":33285,"Ġbored":33286,"ĠDetect":33287,"Ġappar":33288,"Ġjeans":33289,"ĠTak":33290,"IOD":33291,"ĠHorse":33292,"(FILE":33293,"(?":33294,"rique":33295,"optimizer":33296,"nat":33297,"loys":33298,"ĉToken":33299,"oubted":33300,"uess":33301,"ocoa":33302,"DataMember":33303,"_POWER":33304,"classList":33305,"PushButton":33306,"ĠWiFi":33307,".Stream":33308,".guild":33309,"Ġnog":33310,"ĠPortugal":33311,"ĠUnter":33312,"Primitive":33313,"boss":33314,"ĠDeutsch":33315,"Ġerotic":33316,"Ġstrconv":33317,".TryParse":33318,"Ġgrams":33319,".Success":33320,"_pk":33321,"ĠHarvey":33322,"-minded":33323,".country":33324,"[]\"":33325,"Ġangel":33326,"Ġbeats":33327,"ĠVor":33328,"ilio":33329,".master":33330,"something":33331,"ĠPACK":33332,"(if":33333,"RequestBody":33334,"Ġantes":33335,"/widget":33336,"Ġmodo":33337,"ĠAW":33338,"finder":33339,"Ġoptimized":33340,"Ġmissiles":33341,"NB":33342,"ĉinternal":33343,"tex":33344,"ĠSri":33345,"Ġdamaging":33346,"ĠMais":33347,"-Allow":33348,"ĠZh":33349,"-alt":33350,"Ġ));ĊĊ":33351,"èī":33352,"Ġinfluences":33353,"Ġcatal":33354,"_REGISTER":33355,"ĠAPIs":33356,"-century":33357,"Ġbiology":33358,"ĠActual":33359,"Ġheels":33360,"TRACE":33361,"_DIG":33362,"Dataset":33363,"ĠMatter":33364,"Ġclassifier":33365,".wikipedia":33366,"ĠRogers":33367,"Ġdonated":33368,"rawler":33369,"enen":33370,"Ġcasinos":33371,"ortal":33372,"Ġprive":33373,"spe":33374,"ducers":33375,".ep":33376,"Ġgrasp":33377,"acji":33378,"Ġdairy":33379,"Ġbuses":33380,".comm":33381,".ins":33382,"ĠIRS":33383,"ĠBeer":33384,"adc":33385,"oard":33386,"_MET":33387,"Ġ'+'":33388,"rans":33389,"Ġkinda":33390,"ĠâĶĤ":33391,"ĠMaur":33392,"аг":33393,"Ġbandwidth":33394,"ibus":33395,"ĠDifferent":33396,"(mat":33397,"ĠResume":33398,"_UNS":33399,"establish":33400,"Ġfonction":33401,"Subscription":33402,"_company":33403,"Ġlightly":33404,".confirm":33405,".yaml":33406,"ĠBoost":33407,"Commerce":33408,"-template":33409,"_DELAY":33410,"ĠHI":33411,"Ġnavig":33412,"(Sender":33413,"ĠHS":33414,"_\"+":33415,"ĠREQUEST":33416,"Ġwifi":33417,"=\"\"Ċ":33418,"])->":33419,"Ġrope":33420,"Ġviolated":33421,"Ġglance":33422,"ĠKurd":33423,"Ġè®":33424,"deck":33425,"ĠISBN":33426,"Ġinfect":33427,"ĠFoo":33428,"Ġgetter":33429,"Ġtener":33430,"appe":33431,".hh":33432,"_hot":33433,"\".$":33643,"Ġrelies":33644,"(Console":33645,"International":33646,"->{$":33647,"Mid":33648,"Ġdissert":33649,"dds":33650,"Ġdeposits":33651,"ĉdriver":33652,"#ga":33653,"prising":33654,"println":33655,"Ġpresenter":33656,"Ġmines":33657,"CSS":33658,"ĠDual":33659,"(!(":33660,"Ġkam":33661,"ĠisLoading":33662,"ĠProtect":33663,".upper":33664,"arium":33665,"]:ĊĊĊ":33666,"Yii":33667,"-shirt":33668,"ĠIMAGE":33669,"_colors":33670,"Ġurgent":33671,".Container":33672,"!(Ċ":33673,"Saturday":33674,"Ġsocieties":33675,"ĠThan":33676,"ĠCod":33677,"=@":33678,"Ġattachments":33679,".mobile":33680,"Ġspite":33681,"Ġbounce":33682,"rawl":33683,"instancetype":33684,"ĠTruck":33685,"Ġmanipulation":33686,"(Config":33687,"-inst":33688,"Ġstor":33689,"itution":33690,"PreferredGap":33691,"ĠmainAxisAlignment":33692,"Ġlistened":33693,"'''ĊĊ":33694,"ottage":33695,"-project":33696,".APPLICATION":33697,"ĉroot":33698,"Ġwhit":33699,"Ġbilder":33700,"Ġker":33701,"Ġappliances":33702,"rowave":33703,"ìĿĢ":33704,"ematics":33705,"ĠOrg":33706,"oping":33707,"_SEARCH":33708,"Ġcham":33709,"addContainerGap":33710,"Ġ().":33711,"ĠArrow":33712,"Illegal":33713,"Currently":33714,"Ġusa":33715,"Ġpasswords":33716,"Ġrenown":33717,"avern":33718,"ĠEvil":33719,"Ġconcat":33720,"Ġduo":33721,"Ġvale":33722,"ĠBean":33723,"Ġindicators":33724,"cmath":33725,"ĠPump":33726,"November":33727,"ificant":33728,"_DOMAIN":33729,"regar":33730,"ĠPortal":33731,"\"$":33732,"Ġformerly":33733,"\"]:Ċ":33734,"ĠVisibility":33735,".getElementsByClassName":33736,"_RED":33737,"Ġchampions":33738,"à´":33739,"Valor":33740,"_es":33741,"*a":33742,"-repeat":33743,"Band":33744,".stage":33745,"Ġbureauc":33746,"Cnt":33747,"eten":33748,"-function":33749,"Ġmuito":33750,"PID":33751,"_editor":33752,"Ġcrashed":33753,"dead":33754,"kat":33755,"agh":33756,"ĠEXT":33757,"asser":33758,"-small":33759,"Ġrealiz":33760,"(Entity":33761,"ús":33762,"ĠActually":33763,"ĠElite":33764,"Ġhelm":33765,"(nonatomic":33766,"asher":33767,"Community":33768,"alleng":33769,"iry":33770,"ĠGrowth":33771,"Ġsue":33772,"Ġfrequencies":33773,"_descriptor":33774,".Attribute":33775,"Ġrecipients":33776,"_NS":33777,"/\"+":33778,"iban":33779,"Ġathlete":33780,"ĠIgn":33781,"_DMA":33782,"(ds":33783,"ĠRequirements":33784,"ADI":33785,"erez":33786,"\\Admin":33787,"braska":33788,"ĠRust":33789,"Relation":33790,"COD":33791,"ĠVERSION":33792,"emma":33793,")){":33794,".Duration":33795,"ĠCamb":33796,"-logo":33797,"Ġreadable":33798,"Ġcreators":33799,"()];Ċ":33800,"UpDown":33801,"-half":33802,".getMonth":33803,"(sf":33804,"Pic":33805,"Ġhunger":33806,".tx":33807,"Ġexceeded":33808,"_seed":33809,"(^":33810,"_sk":33811,".perform":33812,"Ġ>::":33813,"Ġmongo":33814,"=float":33815,"bindParam":33816,"Smart":33817,"ifa":33818,"Ġsecurities":33819,"Ġprejud":33820,"Ġ,\"":33821,"Ġcorps":33822,"Ġvra":33823,"amacare":33824,"iterr":33825,"(Media":33826,"uche":33827,"Ġcob":33828,"Ġliber":33829,".geometry":33830,"Locator":33831,"Ġsliding":33832,"Ġsurgical":33833,"_CUR":33834,"Ġconsect":33835,"[*":33836,"ĠResort":33837,"Stub":33838,"_DOUBLE":33839,"ĠSoph":33840,"Ġelectoral":33841,"_disable":33842,"ĠÑģо":33843,"ĠLightning":33844,"Ġmentions":33845,"ocy":33846,"Ġleaked":33847,"Ġrelaxing":33848,"Presenter":33849,"vsp":33850,"Ġguilt":33851,"=-=-":33852,".reply":33853,"ĠMirror":33854,"Camp":33855,"Ġ+#+#+#+":33856,"Ġ+#+#+#+#+#+":33857,".Author":33858,"Ġdirective":33859,"-hook":33860,"íĦ°":33861,"}ĊĊĊĊĊ":33862,"@pytest":33863,"_rand":33864,"mis":33865,"Ġcolorful":33866,"uje":33867,"lasses":33868,"ĠClasses":33869,".have":33870,"%),":33871,"é¢ĺ":33872,"Ġdisturbing":33873,"substring":33874,"ĠKoh":33875,"Invest":33876,"purchase":33877,"Ġrecycling":33878,"ĠART":33879,"ierarchy":33880,"Ġfps":33881,".checkBox":33882,"íķ´":33883,"_material":33884,"ducation":33885,"Ġfw":33886,"udit":33887,"Ġreviewing":33888,"ĠSid":33889,"Syntax":33890,"ĠWritten":33891,"argar":33892,"UME":33893,"/q":33894,"Classifier":33895,"Official":33896,"Ġjazz":33897,"Ġomega":33898,"Physics":33899,"Ġlugar":33900,"_accessor":33901,".commands":33902,"Ability":33903,"ĠBatch":33904,"RAM":33905,"Ġencounters":33906,".Qu":33907,"BYTE":33908,"ĠDistribution":33909,"Ġuso":33910,"ĠRecovery":33911,"approved":33912,"Ġdenial":33913,"/share":33914,"LinkedList":33915,")čĊčĊčĊ":33916,"uddy":33917,"Ġfines":33918,"Ġry":33919,"Unicode":33920,"ĉrender":33921,"Ġpremises":33922,"Ġpon":33923,"aliases":33924,"/Foundation":33925,"cuda":33926,"ĠCock":33927,",:)":33928,"(folder":33929,"Ġméd":33930,"drag":33931,"Ġtalents":33932,"ĠĠĠĊĊ":33933,"еÑģÑĤв":33934,"mob":33935,".yml":33936,"Ġaster":33937,"Ġdiscre":33938,"goal":33939,"ĠGTX":33940,"ĠSUCCESS":33941,"ĠLONG":33942,"(find":33943,"Ġsingular":33944,"_sz":33945,"ĠEthereum":33946,"..Ċ":33947,"Ġirres":33948,"')){Ċ":33949,"Ġministers":33950,"Steps":33951,"iversal":33952,"ĠNevertheless":33953,"-led":33954,"Ġ(%)":33955,"ç¡®":33956,"Ġtimezone":33957,"Ġstranger":33958,"(render":33959,"Ġshutil":33960,"Ġmph":33961,"Ġtrio":33962,"ppy":33963,"Ġpredomin":33964,"Ġendors":33965,"ĠRussians":33966,"ĉrow":33967,"Ġwizard":33968,".serialize":33969,"Ġcomplained":33970,"Ġsido":33971,"Ġdelighted":33972,"-me":33973,"ĠRav":33974,"Human":33975,"adays":33976,"recv":33977,"Working":33978,"Jump":33979,"ĠÃ¥r":33980,"ĠAutomatic":33981,"_Base":33982,"æł¼":33983,"aurants":33984,"¯":33985,"æ¸":33986,"(CType":33987,"IFI":33988,"(amount":33989,"Ġbelieving":33990,"=mysql":33991,"Ġfir":33992,"Ġrestoration":33993,"ereco":33994,"Т":33995,"_'+":33996,"Ġebook":33997,"Ġdebris":33998,"(inputs":33999,"AYOUT":34000,"Ġscreaming":34001,"avia":34002,"lander":34003,"Ġdistress":34004,"Ġassembled":34005,"ĠAvoid":34006,"(thread":34007,"ĠRPC":34008,"_EXIT":34009,"(queue":34010,"иÑģÑĤ":34011,"Dll":34012,"Ġskull":34013,"_pub":34014,"chez":34015,"minate":34016,"ensen":34017,"Ġinsane":34018,"bounds":34019,"ĠRosen":34020,"Ġconditioning":34021,"processed":34022,"videos":34023,"four":34024,".Conv":34025,"|;Ċ":34026,"Personal":34027,"cerpt":34028,":UIControlStateNormal":34029,"Ġdoses":34030,"ĠKarl":34031,"ĠFrequ":34032,".BASE":34033,"ĠVote":34034,"Ġconcurrent":34035,"ĠMessageBoxIcon":34036,"ĠÃĸ":34037,"ĠDubai":34038,"ĠRetail":34039,":number":34040,"ĠObserver":34041,"ĠBigInteger":34042,"_origin":34043,"_WORK":34044,"Frames":34045,"Ġnotably":34046,".âĢľ":34047,"Ġtropical":34048,"Ġniche":34049,"amina":34050,".sys":34051,"(tokens":34052,"modify":34053,"osit":34054,"strom":34055,"ĠComics":34056,"OPTION":34057,"Ticket":34058,"Ġfactories":34059,"Ġdisput":34060,"_File":34061,"ĠFinn":34062,"eee":34063,"ĠDiscord":34064,"_money":34065,".tpl":34066,"_safe":34067,"LB":34068,"Ġglut":34069,"JK":34070,".flow":34071,"-cont":34072,"gos":34073,"Ġhorizon":34074,"ĠRush":34075,"::*":34076,"Pipe":34077,"ulla":34078,"borough":34079,"heimer":34080,"(move":34081,"(Text":34082,"});čĊčĊ":34083,"welcome":34084,"ĠComponents":34085,"Ġgovernance":34086,"closed":34087,"ĉmargin":34088,"Ġlaundry":34089,"ĠTerminal":34090,"izards":34091,".âĢĶ":34092,".remote":34093,".radius":34094,"ĠQuebec":34095,"Ġdh":34096,"Tech":34097,"ĠMist":34098,"seller":34099,"_literal":34100,"Ġgenius":34101,"Ġbrains":34102,"gem":34103,"ĠMeasure":34104,"Ġcatast":34105,"rance":34106,".TextField":34107,"Ġconsuming":34108,"Ġ'\\''":34109,"oubtedly":34110,"ĠCertain":34111,"Ev":34112,"erti":34113,"being":34114,"Experience":34115,"Ġ//[":34116,"ĠArabic":34117,"ĠCrist":34118,"ĠAzure":34119,"Ġhora":34120,"ladesh":34121,"\\Blueprint":34122,"dar":34123,".rel":34124,"Ġsuprem":34125,"ĠReagan":34126,"ĠAttributes":34127,"-sidebar":34128,"ĠuseStyles":34129,"ĠAirlines":34130,"Ġhills":34131,"/xhtml":34132,"vinc":34133,"_mock":34134,"ĊĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":34135,"ĠPill":34136,".LayoutStyle":34137,"ĠCommander":34138,"]<":34139,"signature":34140,"Ġ{}čĊ":34141,"Ġhatred":34142,"Ġëĭ":34143,"olesterol":34144,"Ġ********":34145,"ancellor":34146,"crop":34147,"TIM":34148,"ĉĉĊĊ":34149,"ysqli":34150,"uitive":34151,"ĉunset":34152,"_sel":34153,"Ġmenus":34154,"tick":34155,"Ġconstitute":34156,"ĠElements":34157,"ĠRedis":34158,"aggio":34159,"_fp":34160,"_depend":34161,"emas":34162,"CAST":34163,"orange":34164,"jon":34165,"ĠEmily":34166,"Ġpotatoes":34167,"Ġreceptor":34168,"ĠElectronic":34169,"ĠLights":34170,"Ġcombining":34171,"ĠSomeone":34172,"Ġ########.":34173,"ĠTOD":34174,"/show":34175,"Xd":34176,".\"'":34177,"afx":34178,"Ġtragic":34179,"Styled":34180,"ĠMarco":34181,"Gallery":34182,"dale":34183,".âĢĿĊĊĊĊ":34184,"érie":34185,"/service":34186,"äºĨ":34187,"Ġambient":34188,"_SETTINGS":34189,".Adapter":34190,"lene":34191,"Ġtravels":34192,"Notice":34193,"Ġcleans":34194,"ĠFem":34195,"chair":34196,"Ñĥн":34197,"/my":34198,"_bad":34199,"ĠEconomics":34200,"ISA":34201,"_CNT":34202,"(Menu":34203,"äºİ":34204,"ĠRidge":34205,"Ġlengthy":34206,"Dot":34207,"Ġjumps":34208,"Ġhey":34209,"$pdf":34210,"Ġworm":34211,"Ġsut":34212,"Ġsher":34213,"iamo":34214,"ĠCalc":34215,"trieve":34216,"Ġcops":34217,"ĠChrom":34218,"Ġregulated":34219,"reatment":34220,"ĠHigher":34221,"oks":34222,"Ġdeze":34223,"LOCATION":34224,"ongsTo":34225,"Ġfinite":34226,"Ġvaries":34227,"Ġpositioned":34228,"'il":34229,"éĩij":34230,"Ġhike":34231,"(done":34232,"playlist":34233,"Ġada":34234,"Ġcoastal":34235,"ĠNancy":34236,".DateTimeField":34237,"CppCodeGen":34238,"ĠSimilarly":34239,"reur":34240,"ĠContr":34241,"ĠHidden":34242,"ĠBeta":34243,"atched":34244,"_install":34245,".Output":34246,"Lookup":34247,"ĠRichmond":34248,"quared":34249,"Ġmanga":34250,"-controls":34251,"ĠBernard":34252,"Large":34253,"Ġslices":34254,"Ġoffence":34255,"ĠMega":34256,"Ġestar":34257,"Ġjoints":34258,"Ġsumm":34259,"_platform":34260,"Buff":34261,".addSubview":34262,"Ġretained":34263,"Letter":34264,".dim":34265,"Ġessere":34266,"ĠScaffold":34267,"EXPECT":34268,"ĉRE":34269,".longitude":34270,"ünd":34271,"Ġstatue":34272,".addWidget":34273,"ĠCaribbean":34274,"addPreferredGap":34275,"ilde":34276,"UILabel":34277,"ĠOpport":34278,"Ġimperial":34279,"ursion":34280,"Ġmandate":34281,"Ġpromotional":34282,"Ġvk":34283,"iaÅĤ":34284,"Ġpyl":34285,"ĠCreation":34286,"озд":34287,"Ġsimpler":34288,".what":34289,"ĠRecent":34290,"Storm":34291,".quantity":34292,"ĠLov":34293,"\"-":34294,"ubbles":34295,"_notification":34296,"(world":34297,"urger":34298,"*(-":34299,":\"Ċ":34300,"hm":34301,"anship":34302,"ĠAlmost":34303,"Ġmotorcycle":34304,"_fee":34305,"Ġabsorb":34306,"ĠVincent":34307,"Ġsounded":34308,"ÃŃst":34309,"Ġpharmaceutical":34310,"htag":34311,"ĠKindle":34312,"italize":34313,"ĠEmperor":34314,"oustic":34315,"Ġspecialists":34316,"åħ¬":34317,"BorderStyle":34318,"/\\":34319,"RELATED":34320,"(',',":34321,"(expr":34322,"Ġht":34323,"åįĪ":34324,"_Create":34325,"Ġspecially":34326,"Ġ[];čĊ":34327,"Ġheel":34328,"Ġsept":34329,"_arch":34330,"(initial":34331,"%.ĊĊ":34332,"\\\",\\\"":34333,"Ġdiscusses":34334,"Ġupt":34335,"Ġ[&":34336,"Ġmanus":34337,".hand":34338,"ĠMAIN":34339,"ĠDenmark":34340,"Ġ],čĊ":34341,"Ġcryst":34342,"Ġnack":34343,"Coords":34344,"_inner":34345,"Ġmidst":34346,"Ġawake":34347,"ĠÐŀ":34348,"-break":34349,"ÃŃvel":34350,"_PASS":34351,"ĠParams":34352,"Ġdetr":34353,"Ġspider":34354,"ĠConcept":34355,"Ġprend":34356,"CHED":34357,".Exit":34358,"Ġpopulated":34359,"Ġvirtue":34360,"_SESSION":34361,"Ġnouvel":34362,"oauth":34363,"ĠданнÑĭ":34364,"rink":34365,".HeaderText":34366,"aturated":34367,"Ġerst":34368,"Ġåħ":34369,"à¥ĩ":34370,"_visible":34371,"eyer":34372,"Ġliable":34373,"Ġdebe":34374,"Ġbw":34375,"{-#":34376,"_WIN":34377,"dfs":34378,"Hover":34379,"ĠPUT":34380,"-angle":34381,"Ġnoble":34382,"Ġtraces":34383,"encv":34384,"ĠuserData":34385,"_ins":34386,"ĠSuz":34387,"Ġnewsletters":34388,"ĠModi":34389,"Ġentrepreneurs":34390,"Ġtribute":34391,"Ġrumors":34392,"Ġrr":34393,"ĠQuarter":34394,"ê³ł":34395,"Ġfeeds":34396,"óg":34397,"Ġenvelope":34398,"Ġlear":34399,"Ġkø":34400,"developer":34401,"Similar":34402,":\")Ċ":34403,"subscription":34404,"Modifier":34405,"italic":34406,"Ġnasty":34407,"Ġtermination":34408,"Ġcharming":34409,"ĠâŁ":34410,"tons":34411,".trace":34412,"hots":34413,"ĠUR":34414,"Mont":34415,"Ġjustified":34416,"ĠGang":34417,"inea":34418,"Ġbog":34419,"(ap":34420,"_$":34421,"Ġcontamin":34422,".Dot":34423,"ĉDebug":34424,"(exports":34425,"Ġpaired":34426,"ĠAssignment":34427,"Ġautomobile":34428,"ĵį":34429,"Ġphases":34430,"vw":34431,"@SuppressWarnings":34432,"=\\":34433,"rant":34434,"-ed":34435,"ĉawait":34436,"Ġcertificates":34437,"'>\"":34438,"Ġintact":34439,"CTRL":34440,"Mike":34441,"gregation":34442,"ATTERN":34443,"Ġrepublic":34444,"_upper":34445,"iliary":34446,"Ġcomputation":34447,"hire":34448,"ĠShin":34449,"_ANY":34450,"ĠManufacturer":34451,"ĠCarm":34452,"Ġbearings":34453,"_comb":34454,"cad":34455,"uristic":34456,"Ġwholesale":34457,"Ġdonor":34458,".interfaces":34459,"presso":34460,"ĠBrun":34461,"-close":34462,"prove":34463,"_SK":34464,"ĉframe":34465,"etros":34466,"ĠPain":34467,"_EXP":34468,"ĠLT":34469,"_fs":34470,".datas":34471,"ĉss":34472,"voir":34473,"ĠAxis":34474,"Major":34475,"=\"<":34476,"[h":34477,"Ġprofess":34478,"igrate":34479,"(score":34480,"Keyword":34481,"\"os":34482,"ĠĠĠĠĉĊ":34483,"analysis":34484,"Ġreplay":34485,".pass":34486,"\\d":34487,"tls":34488,"Ġsanct":34489,".light":34490,"_mobile":34491,"ÑģÑĤÑĮ":34492,"ĉtotal":34493,"uity":34494,"Ġpaused":34495,"NAS":34496,"Ġencore":34497,"loe":34498,"Ġ-*-ĊĊ":34499,".high":34500,"ampler":34501,"ĠSecure":34502,"Ġfragments":34503,"_vel":34504,"illary":34505,"ĠStein":34506,"ĠDawn":34507,"Ġmaximize":34508,"ย":34509,"Ġ/^":34510,"Ġcontinually":34511,"Ġshadows":34512,"ĉĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":34513,"ĠIActionResult":34514,"Ġinformación":34515,"CHECK":34516,".SelectedItem":34517,"bundle":34518,"olley":34519,"<":34681,"Ġtrajectory":34682,"_ring":34683,"Ġhydrogen":34684,"tron":34685,"Ġstatute":34686,"Ġconditional":34687,"Ġtray":34688,"-school":34689,"(widget":34690,"$config":34691,"Ġrequesting":34692,".uint":34693,"eton":34694,"brities":34695,"OfType":34696,"ADMIN":34697,"predict":34698,"Ġgegen":34699,"ĠHapp":34700,"OCUMENT":34701,"ĠApart":34702,"Ġ-----":34703,"roe":34704,"uide":34705,"justify":34706,"ĠSquad":34707,"Ġprofes":34708,".bot":34709,"_currency":34710,"innen":34711,"ĠMumbai":34712,"ĠNumbers":34713,"avanaugh":34714,"agnitude":34715,"âĢľThere":34716,"=http":34717,"çīĩ":34718,"Ġvb":34719,"+'{{$":34802,"Ġinode":34803,"sil":34804,"Ġhace":34805,"Ġseverely":34806,"ĠOverview":34807,"Ġspraw":34808,"Ġbeaches":34809,":left":34810,"·»":34811,"(${":34812,"ĠFIRST":34813,"ĠSpa":34814,"-ass":34815,"Ġbaise":34816,"ĠNODE":34817,"ĠPizza":34818,"Pet":34819,"(seq":34820,"\\\">Ċ":34821,"CppMethodPointer":34822,"Ġvp":34823,"Ġia":34824,"_seconds":34825,"emet":34826,"/blob":34827,"_THRESH":34828,"...čĊ":34829,"Dest":34830,"ĠNH":34831,".dataSource":34832,"ités":34833,"ĠJak":34834,"sell":34835,"Ġworkshops":34836,"\",Ċ":35452,"_Pin":35453,"uese":35454,"Ġoverrides":35455,"_ready":35456,"Advanced":35457,"Ġopi":35458,"-cart":35459,"(\"/\",":35460,"ĠDeb":35461,"CRY":35462,"ĠVertical":35463,"ĠOVER":35464,"ĠCorporate":35465,"Ġ\"\";":35466,"Ġstepping":35467,"ej":35468,"Ġaccusations":35469,"Ġoraz":35470,"_tail":35471,"Ġinduced":35472,"Ġelastic":35473,"Ġblown":35474,",//":35475,"Ġbackgrounds":35476,"âĢĻune":35477,"-sdk":35478,"ĠsetInterval":35479,"Ġincentives":35480,"Ġvegetable":35481,"_On":35482,"expanded":35483,"pix":35484,"_shader":35485,"ĠSPDX":35486,"@example":35487,"ĠWrapper":35488,".Zero":35489,"Positive":35490,"Ġspinner":35491,"Ġinvented":35492,"ĠGates":35493,"оÑĤоÑĢ":35494,"Ġcomparisons":35495,"è·":35496,".primary":35497,"dataProvider":35498,"additional":35499,"ĉoptions":35500,"snapshot":35501,".setHorizontal":35502,"Ġ\"{}":35503,"ĠFisher":35504,"halten":35505,"":35538,"ĠRegistered":35539,"INED":35540,"kal":35541,"parison":35542,"Ġobjeto":35543,"Vi":35544,"manda":35545,"Ġrenewed":35546,"ĠSof":35547,"essel":35548,".ndarray":35549,"Ġcrap":35550,"管":35551,".abspath":35552,"(up":35553,"Ġclearance":35554,"ĠTW":35555,"_COPY":35556,"ĠĠĠĠĠĠĠĠĠĠĠĠĉ":35557,"Ġforests":35558,"Ġarguably":35559,"ĠASS":35560,"hey":35561,"amel":35562,"_fore":35563,"ĠSoutheast":35564,"Ġabused":35565,"Ġpracticing":35566,"akedirs":35567,"主":35568,"_resources":35569,"Ġpond":35570,".Fixed":35571,"LastError":35572,"ĠPsychology":35573,"Ġ\"//":35574,"!:":35575,"Reusable":35576,"Ġmensaje":35577,"Ġrospy":35578,"Ġbour":35579,"Ġvarieties":35580,"Ġempath":35581,"(({":35582,"_org":35583,"ĠMes":35584,"ĠMagento":35585,"ISTORY":35586,"Unless":35587,"Ġhj":35588,"ĠDuty":35589,"Jun":35590,",size":35591,"Ġpaintings":35592,"Ġdispens":35593,"dart":35594,"Ġbehavioral":35595,"Ġrpc":35596,"calculate":35597,"fruit":35598,"_mm":35599,"ĉpthread":35600,"MaxLength":35601,"Ġcurrencies":35602,"_capacity":35603,"ĠOz":35604,"Ġfirearm":35605,"Ġcoefficient":35606,"Ġbankruptcy":35607,"wart":35608,"Ġfatigue":35609,"AVA":35610,"Ġespa":35611,"_pc":35612,"ĠQuotes":35613,"_LIGHT":35614,"ĠTickets":35615,"Ġrelates":35616,"Ġpublishers":35617,"Ġunlocked":35618,"Ġ//----------------------------------------------------------------":35619,"ĠInterruptedException":35620,"Ġoutlook":35621,"rn":35622,"Ġrebels":35623,"Written":35624,"Ġasian":35625,"otto":35626,"Ġĉĉĉĉ":35627,"_gpu":35628,"Txt":35629,".ImageView":35630,"Ġsuis":35631,"_tables":35632,".RecyclerView":35633,"Ġwhatsoever":35634,"èģ":35635,"]++;Ċ":35636,"assertTrue":35637,"_verify":35638,"ĠRivers":35639,"Ġ][":35640,"Jet":35641,"idian":35642,"Sibling":35643,"Ġgenres":35644,".Access":35645,"OPS":35646,"Ġtrivial":35647,"ส":35648,"alen":35649,"вед":35650,"ĠSword":35651,"Ġscrutiny":35652,"(cb":35653,"Ġcommerce":35654,"Ġguarantees":35655,"_adv":35656,"ĠLET":35657,"recio":35658,"Ġhilar":35659,"Ġbackyard":35660,"ãĢı":35661,"Ġillustrated":35662,"/vendor":35663,".Util":35664,"Ġwow":35665,"LOY":35666,"ĠMarshal":35667,"\">'.$":35668,"ĠBak":35669,"Ġmodifiers":35670,"dictionary":35671,"ĠStre":35672,"multiple":35673,"\")),":35674,"ĠCort":35675,"']\").":35676,"(admin":35677,"ĠCreator":35678,"Internet":35679,"(ms":35680,"logy":35681,"DECLARE":35682,"ĠMarcus":35683,"<<<<":35684,"ãģł":35685,"_my":35686,"(inst":35687,"Ġsciences":35688,"NDER":35689,".enter":35690,"Ġitu":35691,"Ġbehave":35692,"Pan":35693,"ombies":35694,"='<":35695,"'));čĊ":35696,"ĠMENU":35697,"ĠWorkers":35698,".NoError":35699,"Ġbindings":35700,"Ġdisabilities":35701,"{\\":35702,"ĠMunicip":35703,"Ġcores":35704,"urple":35705,"ĠNokia":35706,"usions":35707,"ĠFitness":35708,".handleChange":35709,"Ġjavascript":35710,"ìļĶ":35711,"(dec":35712,"Ġpacking":35713,"-depend":35714,"Ġtranscript":35715,"zeros":35716,"_alert":35717,"?\",Ċ":35718,"libs":35719,"±Ð¾ÑĤ":35720,"Ġ|ĊĊ":35721,"trained":35722,"ĠGent":35723,"ĠRab":35724,"xp":35725,"_configuration":35726,"天":35727,"_accept":35728,".recyclerview":35729,":url":35730,"ĠMuhammad":35731,"Ġprivileges":35732,"_bank":35733,"uku":35734,"wallet":35735,"ĠROOT":35736,"Ġencuent":35737,"?family":35738,"ĉposition":35739,"Ġcg":35740,"Ġprecip":35741,"methods":35742,"_fast":35743,"increment":35744,"ĠTiger":35745,"_OCCURRED":35746,"quip":35747,"ĠHAS":35748,"_dom":35749,"Ġwreck":35750,"bj":35751,"Ġdern":35752,"Ġorgans":35753,".entries":35754,"Ġ_('":35755,"ramento":35756,"ĠJamie":35757,"Ġpunk":35758,"IPP":35759,"Ġprograma":35760,"Ġattain":35761,"Ġproves":35762,"/sign":35763,"Ġanswering":35764,"Ġladder":35765,"****************************":35766,"ĠWalmart":35767,"ĠCONTENT":35768,"ductor":35769,"Ġverbal":35770,"ĠPID":35771,"crypto":35772,"_CALLBACK":35773,"Ġ=================================":35774,"Ġpotent":35775,"Ġshorts":35776,".Uri":35777,".uniform":35778,";border":35779,"ĠWer":35780,"Ġherein":35781,"lla":35782,"ĠIhr":35783,"Pixmap":35784,"literal":35785,"!)ĊĊ":35786,"generic":35787,"rust":35788,"_scripts":35789,"osto":35790,"itus":35791,"ĠCoalition":35792,"Ġremot":35793,"deploy":35794,"ĠEagle":35795,"ãĢģãĢĮ":35796,"Ġimportante":35797,"ĉobject":35798,"Ġseasonal":35799,"nej":35800,"aidu":35801,"BindView":35802,"ĠSierra":35803,"-bg":35804,"ĠmakeStyles":35805,"[offset":35806,"Games":35807,"Ġhormone":35808,"ARIO":35809,"heads":35810,"(select":35811,"ĠStarted":35812,"@param":35813,"_decl":35814,"_blog":35815,"Ġaño":35816,"\\Api":35817,"ĠMilwaukee":35818,"Provid":35819,"Animated":35820,"Ġcooler":35821,"ĠSeed":35822,".Edit":35823,"ÏĦ":35824,"ĠTaking":35825,"ĠborderColor":35826,"-founder":35827,".LoggerFactory":35828,"Ġ\"\"ĊĊ":35829,"ALT":35830,"ĠLate":35831,"EDIATE":35832,"Ġ);ĊĊĊ":35833,"afa":35834,"Ġcancellation":35835,"Atom":35836,"ĠBirmingham":35837,"empresa":35838,"HEMA":35839,"ascal":35840,"Ġupside":35841,".Version":35842,"ĠFolder":35843,"ĠEight":35844,"ĠVintage":35845,"ĠAppDelegate":35846,"ĠPrevention":35847,".separator":35848,"STM":35849,"(room":35850,"generator":35851,"Ġcattle":35852,"ĉZ":35853,"ĠParticle":35854,"'};Ċ":35855,"Ġneighbours":35856,"ĠStateless":35857,"Ġaltitude":35858,"Ġsaint":35859,"обав":35860,"Ġconvinc":35861,"ĠContents":35862,"Ġjeune":35863,"(ts":35864,"Serialization":35865,"(collection":35866,"ĠJazz":35867,"ĠDod":35868,"ĠRoch":35869,"acio":35870,"commended":35871,"DEFINE":35872,".onload":35873,"Ġspecialty":35874,"PLACE":35875,"_MOVE":35876,"Ġaccountable":35877,"Reuters":35878,"Ġficken":35879,"Ġdepr":35880,"Wow":35881,"Void":35882,".space":35883,"à¸Ĺ":35884,"Ġtq":35885,"ĠPets":35886,"<$":35887,"(Current":35888,"berries":35889,"planation":35890,"ĠlistOf":35891,"ĠThu":35892,"ĠPRINT":35893,"Ġmismo":35894,"Ġdoi":35895,"chk":35896,"ĠUnicode":35897,"(role":35898,"Ġvirgin":35899,"-->Ċ":36360,"Vol":36361,"ĠSSD":36362,"))),":36363,".Optional":36364,"Ġnurses":36365,"Ġorb":36366,"_pe":36367,");čĊčĊčĊ":36368,"placed":36369,"esser":36370,"Ġtherapeutic":36371,"Ġwhitespace":36372,"Ġaston":36373,"Successful":36374,"Ġpraised":36375,"ĠWes":36376,"Ġeighth":36377,"iral":36378,"Ġvrouw":36379,"Ġfaction":36380,"_bias":36381,"Ġwitch":36382,"Ġnpc":36383,"(sb":36384,"ĠRodrig":36385,"_big":36386,"Dependency":36387,"ĠAbraham":36388,"ardi":36389,"CAR":36390,"nos":36391,"Ġabundance":36392,"Ġnutrients":36393,"instein":36394,".Vert":36395,"ĠISS":36396,"D":36495,"Ġservlet":36496,"bastian":36497,"Ġ>&":36498,"SID":36499,"_clk":36500,"Ġdivisions":36501,"}',Ċ":36502,"Ġdildo":36503,"Ġparade":36504,"major":36505,"Ġaboard":36506,";++":36507,"Ġfusion":36508,"\"},{\"":36509,"ĠDialogResult":36510,"ĉarr":36511,"-em":36512,"_nr":36513,"(handler":36514,".NET":36515,".XtraReports":36516,"ĠShah":36517,"ĠBrief":36518,"-,":36519,"Ġprecio":36520,"ĉĉĉĠĠĠĠĠĠ":36521,"Ġtant":36522,"ĠGrande":36523,"/xml":36524,"_ICON":36525,"ĠRetro":36526,"unque":36527,"Ġnag":36528,"toFixed":36529,"XL":36530,"Ġdeclaring":36531,"ĠConcrete":36532,"ĠAmazing":36533,"ĉprintk":36534,"Ġdebates":36535,"DATED":36536,"Ġaesthetic":36537,"emetery":36538,"RoutingModule":36539,"ĠNashville":36540,"WAYS":36541,"Ġwolf":36542,"Ġobservers":36543,"OTA":36544,"anson":36545,"Ġea":36546,"Ġgreenhouse":36547,"ĵįä½ľ":36548,"Ġstair":36549,"Ġimmigrant":36550,"_apply":36551,"peare":36552,"ĠBloomberg":36553,"_PLAYER":36554,"Resp":36555,"æŃ£":36556,"Chooser":36557,"ĠICollection":36558,"Peter":36559,"Erro":36560,".detectChanges":36561,"Maps":36562,"Ġsqueeze":36563,"ĠHomes":36564,"wegian":36565,"Ġformatting":36566,"Ġnegotiate":36567,"uld":36568,"ĠNep":36569,"ĠQB":36570,"Ġeconomies":36571,"Ġ*/,":36572,"Ġredund":36573,"ĠAber":36574,".IsNullOrWhiteSpace":36575,"ycled":36576,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":36577,"_Sh":36578,"Ġskept":36579,"Ġrecreated":36580,"ĠgetType":36581,"Ġmargins":36582,"Ġcolonial":36583,"charts":36584,"//@":36585,"Ġprocessors":36586,"说":36587,"batis":36588,"æĦı":36589,"atorio":36590,"mentioned":36591,"Patient":36592,"Ġprey":36593,"Checkbox":36594,"_xpath":36595,".skip":36596,"ĠMormon":36597,"ĠMemoryStream":36598,"CREMENT":36599,"Ġku":36600,"meld":36601,"\\Data":36602,"ĠKernel":36603,"iltr":36604,"éĢģ":36605,"(profile":36606,"Carbon":36607,"ROLE":36608,"(pl":36609,"]*(":36610,".memory":36611,"Ġmedal":36612,"Ġadvisor":36613,"ität":36614,"Ġhdr":36615,"ierung":36616,"ĠProvides":36617,"(alpha":36618,"Ġteenagers":36619,"-parser":36620,".LatLng":36621,"]()Ċ":36622,"Ġfelony":36623,"ĉĉĉĊĉĉĉĊ":36624,"BOOK":36625,"Ġslash":36626,"Ġclearfix":36627,"ĠProphet":36628,"容":36629,"rightness":36630,"-fi":36631,".kind":36632,"erton":36633,"Jim":36634,"Ġmanipulate":36635,"Ġworksheet":36636,"olin":36637,"stars":36638,"Ġartifact":36639,"_EMPTY":36640,"ĉmain":36641,"-------------';":36709,"Ġexpressing":36710,"ĠIQ":36711,"ĠFact":36712,"/*******************************************************************************Ċ":36713,"_mass":36714,")):":36715,"Ġcondom":36716,"ĠcreateState":36717,"ometown":36718,"Ġirr":36719,"Ġ>(":36720,">B":36721,"iteration":36722,"ãĥª":36723,"Ġshirts":36724,"ounty":36725,"->$":36726,"_SIGN":36727,"ĠDale":36728,"Ġjj":36729,"Easy":36730,"Fre":36731,"ĠNy":36732,"Ġchlor":36733,"matched":36734,"ĠGerm":36735,"-UA":36736,"ĠNathan":36737,"education":36738,"-yard":36739,"-che":36740,"houses":36741,"ritional":36742,"Ġproximity":36743,"Ġdiesem":36744,"áºŃp":36745,"Ġdrought":36746,".audio":36747,"ĠLeo":36748,"Ġfavorable":36749,"inch":36750,"ĠDaw":36751,"ribly":36752,"_student":36753,"idable":36754,"OVE":36755,"Ġlacks":36756,"ouncing":36757,".business":36758,"Ġreopen":36759,"maybe":36760,"_GLOBAL":36761,"Ġdresses":36762,"ĠEdwards":36763,"ensible":36764,"ĠHardware":36765,"ĠExcellent":36766,"ĠTimeUnit":36767,"CTIONS":36768,"Ġschedules":36769,"Ġsegue":36770,"Opens":36771,"ammen":36772,"-Identifier":36773,"Ġstaring":36774,"Ġhappily":36775,"ĠHob":36776,"'_":36777,"Ġ\");":36778,"amentos":36779,"etched":36780,"Ġ/>}Ċ":36781,".Users":36782,"Ġinterrupted":36783,"Contacts":36784,"Ġregistro":36785,"inburgh":36786,"CHA":36787,"_imp":36788,"phis":36789,"say":36790,"Ġretailer":36791,".NODE":36792,"/maps":36793,"_LAST":36794,"ĠCharge":36795,"_guard":36796,"Collider":36797,"ĠStatelessWidget":36798,"\":[\"":36799,"(\"../../":36800,"ioxide":36801,"ĠSund":36802,"Ġ'';":36803,"unset":36804,"addWidget":36805,"лÑİ":36806,"elles":36807,"alker":36808,"Arc":36809,"Ġdeduct":36810,"GUILayout":36811,"ĠVilla":36812,"Ġforbidden":36813,"_where":36814,"Ġ\\/":36815,"ĠTib":36816,"_AX":36817,"]čĊčĊ":36818,"ĠBir":36819,"Ġbend":36820,"ĠMAKE":36821,"ĠMET":36822,"Ġfutures":36823,"Ġweighted":36824,"\"\"\"čĊ":36825,"Ġauthorize":36826,"(program":36827,"},{\"":36828,"Ġcoefficients":36829,"ês":36830,"PerPage":36831,"ĠBathroom":36832,"ĠPublishing":36833,"GPL":36834,"Ġsubmissions":36835,"ĠNUMBER":36836,"jÄħ":36837,"Ġadditionally":36838,"empre":36839,"ĠShel":36840,"otyp":36841,"Solution":36842,"Ġthunder":36843,"_ec":36844,"ĠĊĠĠĠĠĊ":36845,"ĠFellow":36846,"Ġkay":36847,"ĠnewState":36848,"ONTAL":36849,"Implementation":36850,".Look":36851,"Ġents":36852,"Ġlors":36853,"ĠBIG":36854,"fab":36855,"Ġaveraged":36856,"ĠFeedback":36857,"ĠWells":36858,"Ġmartial":36859,"Ġindul":36860,"ĠCommunist":36861,"ĠForex":36862,"ĠAgriculture":36863,"\"[":36864,"Ġquar":36865,"ĠKont":36866,"ĉview":36867,".Bytes":36868,"desktop":36869,"ĠMakes":36870,"akespeare":36871,".Nullable":36872,"Ġspotlight":36873,"VB":36874,"owy":36875,"(torch":36876,"tridge":36877,"_bounds":36878,"Ġapologize":36879,".addItem":36880,"antd":36881,"*);Ċ":36882,",u":36883,"(gen":36884,"ç»ĵ":36885,"reator":36886,"ĠCord":36887,"oupper":36888,".metro":36889,"Ġew":36890,"ĠWORD":36891,".After":36892,"Ġdetained":36893,"ĠHammer":36894,"existing":36895,"Ġost":36896,"Ġmonument":36897,"-custom":36898,"UserID":36899,"ĠNom":36900,"Ġrejection":36901,"(dim":36902,"Ġsingleton":36903,"ĉdie":36904,"ariance":36905,"reports":36906,"]!=":36907,"elda":36908,"Ġprevalence":36909,"_regs":36910,".\".":36911,"Ġfeminist":36912,"Codec":36913,"Ġ**Ċ":36914,"(labels":36915,"_MARK":36916,"FAILED":36917,"Ġadministered":36918,"WN":36919,"ĠĠĠĠĠĠĠĠĉĉ":36920,"Ġnoun":36921,"wig":36922,"Ġgotta":36923,"Ġrif":36924,"-im":36925,"ĠPaulo":36926,"ĠCommandType":36927,"]))ĊĊ":36928,"-zero":36929,"Training":36930,"Ġlord":36931,"_art":36932,"reddit":36933,"Cert":36934,"Ġpeso":36935,"Rot":36936,"Ġendanger":36937,".dr":36938,"userInfo":36939,"unts":36940,"nv":36941,"ĠTrailer":36942,"-first":36943,"(make":36944,"Ġbenefici":36945,"-black":36946,"iÃŁ":36947,"Ġundoubtedly":36948,"Ġmex":36949,"ĠAncient":36950,"(as":36951,"Ġdescent":36952,"Pick":36953,"Ġreplica":36954,"$obj":36955,"ähr":36956,"Ġarrows":36957,"fty":36958,"ĠLibya":36959,"uga":36960,"charged":36961,"Tur":36962,"Ġhomic":36963,"issen":36964,"ĠFake":36965,"Ġbeers":36966,"Ġscattered":36967,"(Time":36968,"UTIL":36969,"Ġbureaucr":36970,"/plain":36971,"Ġsticking":36972,"FAIL":36973,"ĠCovid":36974,"Third":36975,"_present":36976,"ĠPierre":36977,"Ġëª":36978,"Ġ[...]ĊĊ":36979,"Prob":36980,"ĠTraffic":36981,"icao":36982,"doctor":36983,"Ġ),ĊĊ":36984,"Tabs":36985,"alu":36986,"ï¼ļâĢľ":36987,"Ġinherent":36988,"_No":36989,"ritis":36990,"ĠProof":36991,".basename":36992,"ä¼ļ":36993,"Ġchim":36994,"ĠProtected":36995,"crit":36996,"Ġprone":36997,"Ġкон":36998,"ĠHeroes":36999,"Ġanxious":37000,"Ġanos":37001,"Ġweekends":37002,"Ġsext":37003,"Ġreducer":37004,"=UTF":37005,"half":37006,"ĠSaw":37007,".mm":37008,"Ġnueva":37009,".currentTarget":37010,".lua":37011,"_EXTENSION":37012,"ĉreg":37013,"ĠCtrl":37014,"_align":37015,"acceptable":37016,"Ġrushing":37017,"frac":37018,"Ġboasts":37019,"Five":37020,"±":37021,"ĠTemperature":37022,">):":37023,"Ġcharter":37024,"REATED":37025,"Ġsubjected":37026,"Ġopc":37027,"healthy":37028,"使çĶ¨":37029,"ĠScientific":37030,"Ġfrau":37031,"riages":37032,"à¸Ķ":37033,".inventory":37034,"ationale":37035,"Mad":37036,"minutes":37037,">>();Ċ":37038,"ĠEnv":37039,"Ġrecordings":37040,"Ġsuspicion":37041,"sqlite":37042,"ĉread":37043,"ãģ¦":37044,"Ġworries":37045,".putString":37046,"ĠShanghai":37047,"(uid":37048,"rer":37049,"ĠvÃŃde":37050,"\"):":37051,"Ġmethodology":37052,"ĠкоÑĤоÑĢ":37053,"ccc":37054,"avad":37055,"Ġinduction":37056,"ĉThread":37057,",string":37058,"ại":37059,"nehmen":37060,"uition":37061,"Ġ*__":37062,".emf":37063,"Ġìľ":37064,"/themes":37065,"ĠNine":37066,".One":37067,"ĠEmbed":37068,"Ġfaz":37069,"uations":37070,"Ġprivately":37071,"Ġling":37072,"[F":37073,"ushi":37074,"Ġlaunches":37075,"(KEY":37076,"GMT":37077,"Ġaiming":37078,"patible":37079,"ĠBiden":37080,"iw":37081,"ĠDegree":37082,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":37083,"Ġ$('<":37084,"ários":37085,"toUpperCase":37086,"ìłľ":37087,"ĠEUR":37088,"Ġoversight":37089,"Ġtablesp":37090,"Updates":37091,".makedirs":37092,"Ġhumidity":37093,"/template":37094,"Always":37095,"(IS":37096,"_cert":37097,"Dig":37098,"Ġunderway":37099,"orton":37100,"ĠHurricane":37101,"Ġspends":37102,"ĠSegment":37103,"Ġflies":37104,"ĠToggle":37105,"ĠLynch":37106,"Ġsenses":37107,"ĠKos":37108,"setEnabled":37109,"istically":37110,"Ġtester":37111,"Ġadministrators":37112,"Ġtagged":37113,"Ðĵ":37114,"Ġshortcut":37115,"ĠResolution":37116,"Ġsupervision":37117,"ĠAshley":37118,"Tracking":37119,"ulatory":37120,"andel":37121,"isten":37122,"Ġunre":37123,"(diff":37124,"ANTS":37125,"Ġrider":37126,"ĠsÄħ":37127,".Series":37128,"_orders":37129,"ORIZONTAL":37130,"Ġretention":37131,"ãĢĤčĊčĊ":37235,"Ġdiagonal":37236,"ĠCancellationToken":37237,"_Internal":37238,"Ġruin":37239,".Qt":37240,"ocratic":37241,"Tel":37242,"ĠAnswers":37243,"matic":37244,"Ġxp":37245,"atem":37246,"_jobs":37247,"_any":37248,"Ġseniors":37249,"Ġlandmark":37250,"ĠQList":37251,"Ġmaneu":37252,"otify":37253,"/\";Ċ":37254,"/server":37255,"ĠPhilosoph":37256,"utenant":37257,"(io":37258,"hz":37259,"Ġauthenticated":37260,"dv":37261,"-Compatible":37262,"Originally":37263,",function":37264,"ãĢĤčĊ":37265,"ĠRepresentative":37266,"asily":37267,"ircuit":37268,".dt":37269,"(math":37270,".Marshal":37271,"[,":37272,"ĠCities":37273,"_turn":37274,"|)Ċ":37275,"Ġcantidad":37276,"alter":37277,"ĉui":37278,"ĠNebraska":37279,"Ġskirt":37280,".bg":37281,"SharedPreferences":37282,"(style":37283,"Ġgrief":37284,"gew":37285,"Ġsafeg":37286,"olang":37287,"_lists":37288,"ìĽ":37289,"Ġgranite":37290,"Ġhottest":37291,".jdbc":37292,".Customer":37293,"Ġâī¤":37294,"Ġwaar":37295,"_scene":37296,"+'/":37297,"ĠJTextField":37298,"Ġseating":37299,"Ġwears":37300,"Ġ`/":37301,"Cases":37302,"ĠYoutube":37303,"ım":37304,"Ġbalcon":37305,",G":37306,"MetaData":37307,"-price":37308,"SCR":37309,"Unity":37310,"Ġtrunk":37311,"={`${":37312,"Ġearthquake":37313,"Partial":37314,"Ġsubst":37315,"Ġelimin":37316,"=\"'.":37317,"//*[@":37318,"Ġsupervisor":37319,"vrolet":37320,"_article":37321,"Ġpane":37322,"bio":37323,"Ġmotors":37324,"NM":37325,"Frank":37326,"Ġonion":37327,"-word":37328,"ItemClickListener":37329,"Ġbrit":37330,"endencies":37331,"Computer":37332,"_running":37333,"(day":37334,"-he":37335,"(named":37336,"ĠSach":37337,"оÑĩ":37338,"campaign":37339,".Abstract":37340,"(wrapper":37341,".pay":37342,"Ġuw":37343,"Geo":37344,"rails":37345,"/select":37346,"ichte":37347,"sons":37348,"EVENT":37349,"Ġaliment":37350,"Providers":37351,"Await":37352,"_INTERVAL":37353,".off":37354,"Ġgluten":37355,"_cloud":37356,"Ġwen":37357,".extract":37358,"ĉbutton":37359,"/MM":37360,"Party":37361,"Ġdemographic":37362,"_errno":37363,"Ġhiking":37364,"('')Ċ":37365,"\",@\"":37366,"Ġwit":37367,"rá":37368,"ologie":37369,"ĠStyles":37370,"ĠBrowserModule":37371,".RequestMapping":37372,"icans":37373,"PAGE":37374,"creation":37375,"ĠFerguson":37376,"uded":37377,"numbers":37378,"ĠGTK":37379,"Ġpresentations":37380,"ĠBobby":37381,"_span":37382,"estyle":37383,"Ġillegally":37384,"abela":37385,"Ġbattlefield":37386,"capacity":37387,"terror":37388,"]\");Ċ":37389,"Ġwarrior":37390,"leader":37391,"ĠDBG":37392,"ĠRevenue":37393,"Ġvigil":37394,"Ġcounterparts":37395,"(Error":37396,"ACTER":37397,"Ġheeft":37398,"Ġselections":37399,"zeug":37400,"tom":37401,"-two":37402,".;Ċ":37403,"_statement":37404,"ĠAid":37405,"ĠVul":37406,"_rgb":37407,"Ġprizes":37408,"Ġeditable":37409,"ĉform":37410,"ını":37411,".decor":37412,"Demo":37413,"lices":37414,"Ġenctype":37415,"ratulations":37416,"ĠROS":37417,"_chars":37418,"ĠJahr":37419,"partial":37420,"ÑĥÑĤ":37421,"ĠReceive":37422,"ĠLands":37423,"APTER":37424,"Ġchopped":37425,"..\"":37426,"ĠAnaly":37427,"ĠUID":37428,"ĠRadeon":37429,"ĠBee":37430,"Ġunm":37431,">M":37432,".findall":37433,"Tokenizer":37434,"ĠWHAT":37435,"Ġsj":37436,"Drawing":37437,"Ess":37438,"OND":37439,"Ĭ¶":37440,"(packet":37441,"âĢĶbut":37442,"Invocation":37443,"ĠNuclear":37444,"?;Ċ":37445,"Ġgrandes":37446,"ĠCrypt":37447,"remark":37448,"Ġ'../../../../":37449,"Ġinability":37450,"magic":37451,"cats":37452,"Ġsimulate":37453,":${":37454,"inflate":37455,"Ġener":37456,":NO":37457,"iples":37458,"Ġmerit":37459,"ĠRated":37460,"Ġglue":37461,"/blog":37462,"Ġgren":37463,"Ġthrilled":37464,".CH":37465,"uncan":37466,"ĠPRIMARY":37467,"Ġpersec":37468,"Ġfeared":37469,".MIN":37470,"ĠTheater":37471,"éĴ":37472,"ategorie":37473,"段":37474,"Ġappetite":37475,"square":37476,"ĠAlexand":37477,".UserId":37478,"_gt":37479,"_enter":37480,"Ġgraduates":37481,"FragmentManager":37482,"Authorize":37483,"-NLS":37484,"(My":37485,"Ġtriumph":37486,"usting":37487,"_PARAMS":37488,"Characters":37489,"(:,:,":37490,"_BUILD":37491,"MHz":37492,"Ġwashed":37493,"Ġuncle":37494,"Steve":37495,"ardown":37496,"${":37680,"_confirmation":37681,"Ġtrophy":37682,"Works":37683,"ĠElectronics":37684,"ĠMediterranean":37685,"_metrics":37686,"Ġannouncing":37687,"ĠDAY":37688,"_proto":37689,"Ġpear":37690,"baseUrl":37691,"ĉĉĉĉĉĉĉĉĊ":37692,"Ġcoordination":37693,":N":37694,".animate":37695,"ĠCotton":37696,"_hit":37697,"âľ":37698,"Ġjetzt":37699,"ifter":37700,"(fields":37701,"ownload":37702,"ificacion":37703,".cuda":37704,"ĠLiu":37705,">equals":37706,"ĠAce":37707,"ÑĢам":37708,"ĠSuperman":37709,"ĠGarcia":37710,"Ġarrests":37711,"agar":37712,"Ġ{})":37713,"Ġmacros":37714,"roupe":37715,"être":37716,"Ġtwisted":37717,"struments":37718,"_(\"":37719,"_vertices":37720,"ĠTransition":37721,"ик":37722,"[max":37723,"mind":37724,"ĠaccessToken":37725,"Ġunle":37726,"mus":37727,"cop":37728,"ĠFactor":37729,"Ġconced":37730,"Ġretr":37731,".linalg":37732,"-slider":37733,"obl":37734,"_StaticFields":37735,"Ġzombie":37736,"selling":37737,"Ġchap":37738,"Ġshaking":37739,"ĠTranslate":37740,"ĠAmsterdam":37741,"ĠETH":37742,"_EXTERN":37743,"kd":37744,"_disc":37745,"Ġpreceding":37746,"Ġprix":37747,"ObjectName":37748,"_modified":37749,"ardware":37750,"Ġ?>\">":37751,"ĠDW":37752,"`${":37753,"Ġ?>\">ĊĊ":37859,"Ġspinning":37860,"_pending":37861,"Matchers":37862,".Keys":37863,"ĠPV":37864,"enus":37865,"antis":37866,"Ġdiscard":37867,"Ġhaul":37868,"Ġempir":37869,"Ġpathway":37870,"Ġoak":37871,"мен":37872,"-induced":37873,"Ġimpair":37874,"ĠCalgary":37875,".isHidden":37876,"dz":37877,"_include":37878,"Ġgm":37879,"Ġ'('":37880,"PY":37881,"uggestions":37882,"Ġcommodity":37883,"cro":37884,"/sub":37885,"ĠgetInstance":37886,"ĠLegacy":37887,"ĠKil":37888,"Bal":37889,"(short":37890,"Inform":37891,"+x":37892,"*r":37893,"ĠHopefully":37894,"orate":37895,"Ġmachen":37896,"Ġtreaty":37897,"ĠOri":37898,".public":37899,"-horizontal":37900,"Ġtactic":37901,"Ġbord":37902,"wares":37903,"Ġammo":37904,"ĠLists":37905,"Ġequations":37906,"/her":37907,"ĠNSW":37908,"Bounding":37909,"_Collections":37910,"Ġavail":37911,".DropDown":37912,"è°":37913,"Ġhh":37914,"ĠlÃł":37915,".pb":37916,"Ġmemorial":37917,"ĠATTR":37918,"Ġexhausted":37919,"Ġtsp":37920,"ĉredirect":37921,"Ġlikewise":37922,"STER":37923,"Ljava":37924,"Ġcondemned":37925,"ocaust":37926,"(strict":37927,"Ġexempt":37928,"Ġsms":37929,"Ġexagger":37930,"SYS":37931,"Ġlounge":37932,":^":37933,"Ġtodd":37934,"deb":37935,"atorial":37936,"ĠPorter":37937,"Ġtuition":37938,"Ġexempl":37939,"Ġparen":37940,".lineTo":37941,"Ġkidney":37942,"Ġça":37943,"Ġcui":37944,"ï¼Į请":37945,"XC":37946,"Ġmoż":37947,"Ġnominated":37948,"lung":37949,"ImGui":37950,"ĠBuzz":37951,"Ġstereo":37952,"portal":37953,"resas":37954,"Ġklass":37955,"Ġdrafted":37956,"Ġprojectile":37957,"/gpl":37958,"(parameters":37959,"*)Ċ":37960,"Ġassisted":37961,"ĠNSInteger":37962,"sitemap":37963,":nth":37964,".Views":37965,".ArgumentParser":37966,"Ġmeer":37967,"zier":37968,"ĠDig":37969,"Ċ":38036,"Ġplag":38037,"pine":38038,"Ġblanket":38039,"Ġ:-":38643,"Ġlcd":38644,"---------------":38645,"(\"\"":38646,"Ġtactical":38647,"ĠRonald":38648,"extr":38649,"ĠFest":38650,"Ġfuer":38651,"-navigation":38652,"Ġkb":38653,"ghost":38654,"ĠhandleChange":38655,"_cls":38656,"()!=":38657,"Comparator":38658,".vm":38659,"ĠCox":38660,"_review":38661,"/@":38662,"_cookie":38663,"Ġrecognised":38664,"ldap":38665,"Threads":38666,"ĠSexual":38667,"ĠBearing":38668,"(SQL":38669,"Ġxr":38670,"Ġthigh":38671,"URLConnection":38672,"ĠSUV":38673,"ĠmContext":38674,"Ġincidence":38675,"ĠEste":38676,".sup":38677,"_te":38678,"(EXIT":38679,"CMD":38680,"/\">":38681,"Almost":38682,"ĠUne":38683,"Ġanderen":38684,"ĠSingleton":38685,"Ġbore":38686,"Think":38687,"Ġnarc":38688,"]initWith":38689,"_shop":38690,"(strategy":38691,"!',":38692,"herits":38693,"ĠDesk":38694,"_machine":38695,".netty":38696,"ında":38697,"=<":38698,"ĠQR":38699,"ĠSidebar":38700,".splitContainer":38701,"ĠonSuccess":38702,"Ġmonkey":38703,"Enjoy":38704,"(nodes":38705,"pectrum":38706,"Ġ(*(":38707,"ĉUINT":38708,",height":38709,"ĠNetworks":38710,".tail":38711,".linspace":38712,"Ġ\"...":38713,"Listen":38714,"Æ¡":38715,".Channel":38716,"-defined":38717,"Repeat":38718,"adjust":38719,"ERM":38720,"_application":38721,".assertNotNull":38722,"-stream":38723,"Ġrabbit":38724,"Ġpositioning":38725,"Ġwoke":38726,"Ġfing":38727,"Ġmultiplayer":38728,"Ġregistering":38729,"until":38730,"Ã¥n":38731,"(::":38732,"ussions":38733,"Ġpotato":38734,"ĠEquals":38735,".Sup":38736,"/apache":38737,"Ġ(=":38738,".\")":38739,".ptr":38740,"ĠSpeech":38741,".clip":38742,"ĠGabriel":38743,"Ġmusician":38744,"/issues":38745,".shop":38746,"ĠHier":38747,"_RET":38748,"_bucket":38749,"ãĥ¡":38750,"avs":38751,"Ġroz":38752,"flower":38753,"WriteBarrier":38754,"ĠMilan":38755,"Ġlegislature":38756,"ĠDoll":38757,"Ġproving":38758,".concatenate":38759,"âķIJ":38760,"Ġgchar":38761,"cdnjs":38762,"bles":38763,"ĠListing":38764,"ло":38765,".xrLabel":38766,"ĠSak":38767,"justice":38768,"ĠValentine":38769,"unless":38770,"Ġpiger":38771,"(run":38772,"Ġtestified":38773,"ANA":38774,"ĠRemoves":38775,"))));Ċ":38776,"recated":38777,"ĠRuntimeMethod":38778,"Ġconqu":38779,"ãĤ¢":38780,"Ġtissues":38781,"ailer":38782,"été":38783,"-Star":38784,"Ġflames":38785,".setIcon":38786,"Ġsupern":38787,"Ġvagina":38788,"-variable":38789,"Ġwellness":38790,"CUR":38791,"Ġbelle":38792,".getRequest":38793,"Ġpoco":38794,"benh":38795,"agens":38796,"Ġspill":38797,"ĠJur":38798,"Ġdispatcher":38799,"ного":38800,"emonic":38801,"(dirname":38802,"ĠÐĶ":38803,"Ġpasse":38804,"Ġganz":38805,"ricing":38806,"EU":38807,"Ġmujeres":38808,"essen":38809,".attribute":38810,"jj":38811,"ĉĉĠĊ":38812,"[^":38813,"Ġstrtolower":38814,"lexer":38815,"ectar":38816,"hotel":38817,".square":38818,"Ġrall":38819,"Ġlowered":38820,"handled":38821,"Market":38822,"ĠUses":38823,"ivas":38824,".Business":38825,"ãģĹãģ¦":38826,"DIV":38827,"Ġwasted":38828,"Ġavoir":38829,"êm":38830,"_ACCOUNT":38831,".et":38832,"ĉSDL":38833,"kap":38834,"Ġfox":38835,"uppet":38836,"{},Ċ":38837,"\",'":38838,"Favorite":38839,"PEND":38840,"ĠAES":38841,"}),":38842,"Ġdeduction":38843,"ĠpolÃŃt":38844,"ĠcomponentWill":38845,"ĠTelerik":38846,"_SELF":38847,"Ġmuse":38848,"Craft":38849,"Ġdens":38850,"ि":38851,"(tp":38852,"Ġtasty":38853,"Ġbalances":38854,"Ġdedication":38855,"ĠWallace":38856,"Ġunlaw":38857,"\\\">\\":38858,"Ġmum":38859,"-update":38860,"emente":38861,"Ġsoda":38862,"Republic":38863,"asmine":38864,"éric":38865,"(Status":38866,"ĠJsonConvert":38867,"ĠDisk":38868,".Redirect":38869,"Ġfilming":38870,"/mol":38871,"Ro":38872,"Ġville":38873,"Ġtrabaj":38874,"Ġsynthesis":38875,"rega":38876,"Ġrl":38877,"Scheduler":38878,"ISHED":38879,"currentUser":38880,"(errors":38881,"'h":38882,"_bot":38883,"ximo":38884,"ĠUSART":38885,"_super":38886,"_DECREF":38887,"ной":38888,"_ROW":38889,"Ġpromotes":38890,"ĠTA":38891,"Ġhoras":38892,"ĠRepresents":38893,"Ġnameof":38894,"ĠExc":38895,"ĠGarage":38896,"Ġseine":38897,",#":38898,"Ġherb":38899,"/resources":38900,"Ġpleaded":38901,".radioButton":38902,"Ġæĺ":38903,"Ops":38904,"ĠNest":38905,"cstring":38906,"ĠDefence":38907,"Ġrefere":38908,"_leaf":38909,"Ġrevelation":38910,"ë§":38911,".executeUpdate":38912,"_WORLD":38913,"Ġexpans":38914,"(\"\\\"":38915,"jab":38916,"Ġdoubts":38917,"ĠGeometry":38918,"Ġintroduces":38919,"Ġsenators":38920,"Ġcanal":38921,".helper":38922,"ĠBiology":38923,"_SENS":38924,".previous":38925,"-touch":38926,"abit":38927,"Ġimpacted":38928,"Ġbrackets":38929,".direct":38930,"accum":38931,"Ġtestosterone":38932,"ĉaction":38933,"ĠChance":38934,"Ġpeaks":38935,"CppCodeGenWriteBarrier":38936,"Ġunbelie":38937,"_press":38938,".Rel":38939,"angled":38940,"/templates":38941,"-->čĊ":38942,"lime":38943,"Ġsufficiently":38944,"_nt":38945,"Expand":38946,".isfile":38947,"ĠisEmpty":38948,"Ġqt":38949,"Ġmulher":38950,"acob":38951,"George":38952,"常":38953,"Ġassim":38954,"aso":38955,"Ġcomprised":38956,"OV":38957,"(CONFIG":38958,"ĉwriter":38959,"Ġdesp":38960,"Ġtenure":38961,"(cr":38962,".pool":38963,"ĠBrend":38964,"Ġcensor":38965,"(timeout":38966,"Ġplea":38967,".Wrap":38968,"Ġtightly":38969,"ĠWere":38970,"ĠIgnore":38971,"abei":38972,"Ġbridges":38973,"Ġcondemn":38974,"Ġsimplicity":38975,"Ġroutinely":38976,"Ġblacks":38977,"jb":38978,"ĠPit":38979,"Utf":38980,"Ġ/Ċ":38981,"reload":38982,"ĠsetObject":38983,"/global":38984,"Ġfatty":38985,"Ġsocks":38986,"Couldn":38987,"Ġerotisk":38988,"æĿ¡":38989,"ĠPressure":38990,"ĠMaz":38991,"npos":38992,"tolower":38993,"ĠEQ":38994,"uteur":38995,"ĠMoment":38996,"Ġeta":38997,"{{--":38998,"Ġgraphs":38999,"ĠGuar":39000,"rine":39001,"(--":39002,"ĠHttpStatus":39003,"(student":39004,"*np":39005,"Ġrailway":39006,"Ġasynchronous":39007,"_vm":39008,"'],'":39009,",text":39010,"merchant":39011,"(Guid":39012,"ĠGra":39013,"ixer":39014,"fetchAll":39015,".addListener":39016,"flip":39017,"*$":39018,">(),":39019,"Ġsunlight":39020,"assigned":39021,"Ġabc":39022,"ĠCOLUMN":39023,"ĠðŁĻĤĊĊ":39024,")...":39025,"Ġensemble":39026,"Ġnewline":39027,"_SINGLE":39028,"iedad":39029,"Ġdarker":39030,"ormap":39031,"Ġlion":39032,"plits":39033,"Ġillustration":39034,"ĠIEEE":39035,"Ġvista":39036,"ousands":39037,"*******":39038,"ĠTommy":39039,"Ġhue":39040,"Sel":39041,"Ġaura":39042,"ĠTherapy":39043,"Ġanimator":39044,".constraints":39045,"Ġvague":39046,"(\"\")":39047,"Ġvillain":39048,"Ġblessing":39049,"ĠstringBuilder":39050,"ĠMisc":39051,"ĠDIR":39052,"fax":39053,"-node":39054,"ĠWalking":39055,"ĠAU":39056,"sess":39057,"Ġgrill":39058,"VERTISE":39059,"ĠFoods":39060,"Ġtournaments":39061,"Ãĵ":39062,"ĠMarsh":39063,"Ġwonders":39064,"Longitude":39065,".CommandText":39066,"=input":39067,"_encoder":39068,"pageSize":39069,"ĠgetState":39070,">>Ċ":39071,".grey":39072,"pod":39073,"Ġreadings":39074,"Ġreconsider":39075,"Startup":39076,"Ġexcer":39077,".balance":39078,"_cycle":39079,"_Time":39080,"LOCAL":39081,"ĠEFI":39082,"ĠReyn":39083,".setForeground":39084,"byn":39085,"Ġdisconnected":39086,"ACTIVE":39087,"Ġembedding":39088,"ickers":39089,"Ġsurroundings":39090,"*c":39091,"Ġgarant":39092,"Ġbf":39093,"Ġwipe":39094,"Ġä¸ĭ":39095,"_TRA":39096,"adox":39097,"çķ":39098,"Ġsucks":39099,"ĠSongs":39100,"ĠAssociates":39101,"ĠBald":39102,"ĠBrett":39103,"venile":39104,"Ġvt":39105,"Ġinade":39106,"Ġresigned":39107,"ĠGlenn":39108,".pattern":39109,".DataBind":39110,"Ñĥм":39111,"LayoutInflater":39112,"chet":39113,"ĠTestament":39114,".ms":39115,"Ġpav":39116,"ĠReactDOM":39117,"urdy":39118,"ADATA":39119,"Mu":39120,"/actions":39121,"ĠJs":39122,"_extract":39123,"ĠBring":39124,":id":39125,"strt":39126,"ivation":39127,"Ġoutright":39128,"azu":39129,"loyment":39130,"иÑı":39131,"aldo":39132,"ĠPublisher":39133,"Education":39134,"Palette":39135,"_drv":39136,"Ġ($(":39137,"ĠAnda":39138,"Ġremedy":39139,"Ġinconsistent":39140,"tection":39141,"Ġregulators":39142,"Ġshortest":39143,"(pair":39144,"ĠInstallation":39145,"Ġdefendants":39146,"Ġ();":39147,"-large":39148,"Mel":39149,"Ġthreaten":39150,"нÑı":39151,"Ġfetish":39152,"otine":39153,"_dic":39154,"Ġ<$":39155,"Ġstagger":39156,"spi":39157,"$response":39158,"Serv":39159,"-born":39160,"jos":39161,"ĉimg":39162,"ĉWHERE":39163,"_lt":39164,"å½ĵ":39165,".cost":39166,"ĠTue":39167,".labels":39168,"ĠLV":39169,"wcsstore":39170,"ĠJesse":39171,"ห":39172,"Trade":39173,"Ġpredecessor":39174,"ëĤ":39175,"finally":39176,"_general":39177,"oggler":39178,"_REGION":39179,"nement":39180,"Ġblogger":39181,"ĠHarbor":39182,"ĠDataset":39183,"[w":39184,"Ġattendees":39185,".ico":39186,"maximum":39187,".Unlock":39188,"_SYNC":39189,"ágina":39190,"Ġdowns":39191,"ĠWii":39192,"])/":39193,"Ġkicking":39194,"unication":39195,"ĠDAC":39196,"ĠIDS":39197,"ĠRental":39198,"ĠcurrentTime":39199,"Ġvaccines":39200,"ĠDevil":39201,"Ġnors":39202,"_mouse":39203,"urrection":39204,"(no":39205,"Ġ>čĊ":39206,"Ġaggression":39207,"Ġbreeding":39208,".symbol":39209,"iman":39210,"AbsolutePath":39211,"ĠWHO":39212,"_flush":39213,"-root":39214,"arna":39215,"&M":39216,"Ġfathers":39217,"ĠRocket":39218,"iveau":39219,"Ġwander":39220,"Ġcompos":39221,"ĠWarrior":39222,"ĠSeat":39223,"ĠClinic":39224,"_invoice":39225,"(dispatch":39226,"Producto":39227,"aturing":39228,"ossier":39229,"ĠMAY":39230,"Ġdagger":39231,"Ġsanitized":39232,"ĠRFC":39233,"Ġproph":39234,"Ġurine":39235,"Ġgrind":39236,"ĠExpanded":39237,"descripcion":39238,"-fw":39239,"ĠKerry":39240,"=name":39241,"Ġchk":39242,"Ġnationally":39243,"Ġthee":39244,"Inc":39245,"Ġ?>>":39246,".RadioButton":39247,".HttpServletResponse":39248,"/Y":39249,"ĉfield":39250,"Ġhomme":39251,"yper":39252,"Physical":39253,"=v":39254,"Ġdriv":39255,"ĠErrors":39256,"ĠcÄĥ":39257,"Death":39258,"ĠWINDOW":39259,"Ġpoet":39260,"ĠSharp":39261,"ĠImmutable":39262,"ĉcreate":39263,"Ġgeht":39264,"ĠReform":39265,"aiser":39266,"ĠInitialization":39267,"Ġimmunity":39268,".compose":39269,"Ġlatency":39270,"ĠLebanon":39271,"ĠParad":39272,"Ġfuels":39273,"ĠExhib":39274,"coh":39275,"%\">Ċ":39276,"ĠCLI":39277,")initWith":39278,"-Za":39279,"_CLEAR":39280,"regn":39281,"Ġfinances":39282,".standard":39283,"_CATEGORY":39284,".library":39285,"Ġtravelers":39286,"_wp":39287,"ĠEvaluation":39288,"starting":39289,"Ġ)),Ċ":39290,"episode":39291,"ĠVariant":39292,"Ġdaemon":39293,"ĠJulia":39294,"ĠNR":39295,"Ġdoubles":39296,"'":39526,"Ġqueryset":39527,";}čĊ":39528,"ĠPopulation":39529,"utedString":39530,"resident":39531,"_FONT":39532,"ĠRespond":39533,"Ġobscure":39534,"Ġobservable":39535,"ĠContributors":39536,"kon":39537,"ĠMusk":39538,"exao":39539,"ĠTub":39540,"BootApplication":39541,"SOR":39542,".Horizontal":39543,".findBy":39544,".power":39545,"Ġpositively":39546,"venience":39547,"ĠJong":39548,"Ġwhistle":39549,"ĠзнаÑĩ":39550,"Ġlending":39551,"Ġdestructive":39552,"ĠonDelete":39553,"authorization":39554,"();?>":39555,"_original":39556,"science":39557,"atra":39558,"?,?,":39559,"ĠAsc":39560,"Ġconvincing":39561,"$a":39562,"orgen":39563,"_Date":39564,"ĠProvide":39565,"Ġlonely":39566,")'Ċ":39567,"exchange":39568,";?>Ċ":39569,".fast":39570,"Samples":39571,"London":39572,"'])čĊ":39573,"ĠIonic":39574,"Ġpesso":39575,"ĠKnights":39576,"ĠRaf":39577,"_attrs":39578,"Ġrepeal":39579,">Main":39580,"ĠOrdered":39581,"_New":39582,"=\"\">\";Ċ":39663,"ĠSERVER":39664,"ĠHEADER":39665,"_velocity":39666,"ĠInvoke":39667,".timestamps":39668,"Ġsulf":39669,"IQUE":39670,"Ġinhabitants":39671,"phins":39672,"azzo":39673,"Ġmono":39674,"Legend":39675,"Ġnonce":39676,"IFE":39677,";\";Ċ":39678,"-create":39679,"\"\",Ċ":39680,"permit":39681,"ĠImmigration":39682,"Ġpathname":39683,"ffective":39684,"âĻĢâĻĢ":39685,"Ġexams":39686,"-event":39687,"ĠTill":39688,"[mid":39689,"FIX":39690,";color":39691,"(Order":39692,"_traits":39693,"ĠorderBy":39694,"Ġsunt":39695,"ĠNicholas":39696,"ز":39697,"Ġsunny":39698,"iners":39699,"Ġaccessibility":39700,"ĠHB":39701,".comp":39702,"ĉop":39703,"Ġminorities":39704,"etheus":39705,"Ġcollaborative":39706,"prit":39707,"HIR":39708,"Ġwraps":39709,"ĉdraw":39710,"god":39711,"ĠIX":39712,".apps":39713,"ĠNM":39714,"Ġirrelevant":39715,"ĠTigers":39716,"Ġdiag":39717,"GV":39718,"ĠAccessories":39719,"kont":39720,"Ġsimplify":39721,"ĠFavorite":39722,"_tools":39723,"([]);Ċ":39724,"Ġtowers":39725,"Bes":39726,"Ġhunter":39727,"Ġsalon":39728,"(buff":39729,"ĉdebug":39730,"Ġmalware":39731,"Moving":39732,"-options":39733,")+'":39734,"ĠLOVE":39735,"_SOCKET":39736,"_fin":39737,"ĠDelaware":39738,"Ġsheriff":39739,"-invalid":39740,"ĠFULL":39741,"Ġпод":39742,"elas":39743,"\"strings":39744,"ĠRepresentatives":39745,"surface":39746,"resolved":39747,"htdocs":39748,")):čĊ":39749,"Ġpressures":39750,"Ġnorms":39751,"Ġpla":39752,"Ġsurname":39753,"Ġpostal":39754,"ĠDepart":39755,"Ġslaughter":39756,"orida":39757,"Ġhebben":39758,"Ġdesar":39759,"compact":39760,"_LANG":39761,"åIJĪ":39762,"opoly":39763,"_rad":39764,"ĠSTDMETHOD":39765,"Lazy":39766,"ĠĠĠĉ":39767,"...,":39768,"(web":39769,"ĠPont":39770,"Ġetwas":39771,"Ġupward":39772,"_hat":39773,"Ġ],ĊĊ":39774,"ĠbaseUrl":39775,"Ġworrying":39776,"-addon":39777,"(getClass":39778,"SPI":39779,"Ġcapturing":39780,")},Ċ":39781,"Effects":39782,"Ġcompetent":39783,"Ġfoul":39784,"Ġsubscribing":39785,"ĠOBJECT":39786,"IXEL":39787,"bucks":39788,"(edge":39789,"(pass":39790,"ĠPeterson":39791,"Ġboobs":39792,"ĠDelay":39793,"_square":39794,"elim":39795,"oters":39796,"_PC":39797,"%E":39798,"onclick":39799,"ĠSVG":39800,"Ġtopped":39801,"Ġfist":39802,"smart":39803,"ĠRalph":39804,"(owner":39805,"jours":39806,"Ġbronze":39807,"ĠArgumentException":39808,"(original":39809,"_SCALE":39810,"_cp":39811,"Ġrecommends":39812,".setStyle":39813,"Sure":39814,"LAND":39815,"Ġrepeating":39816,"Matt":39817,".Visibility":39818,"Ġenterprises":39819,".Setup":39820,"(scene":39821,"ĠReactive":39822,"urge":39823,"bw":39824,".Put":39825,"persist":39826,".cookie":39827,"ĠAudi":39828,"`s":39829,"supplier":39830,"(Form":39831,"¡":39832,"_so":39833,"ĮĢ":39834,"ĠLegion":39835,"tte":39836,"Nd":39837,"Loss":39838,"(attrs":39839,".scatter":39840,"Ġgroom":39841,"Ġglimpse":39842,"Ġnails":39843,"Ġcumulative":39844,"Ġfazer":39845,"_services":39846,".Num":39847,"ibilit":39848,"_resolution":39849,"ĠTx":39850,"uminium":39851,"opa":39852,".schedule":39853,"smtp":39854,"à¸ķ":39855,"urry":39856,"ük":39857,"goog":39858,"_signature":39859,".into":39860,"ĠSteps":39861,"Ġhomeowners":39862,"ĠNSURL":39863,"ĠPAC":39864,"ĠĠĠĠĠĠĠĠĠĠĠĠĊĊ":39865,">')Ċ":39866,"enh":39867,"Ġincap":39868,"$MESS":39869,"Ġmoins":39870,"ĠFi":39871,"Ġoffseason":39872,"pressions":39873,">.Ċ":39945,"ĠGrass":39946,"ĠGoal":39947,"_pdf":39948,"Handlers":39949,"Ġstacks":39950,".getFullYear":39951,"=[];Ċ":39952,"车":39953,",V":39954,"(split":39955,"Ñĥнк":39956,"Ġbakeca":39957,"Ġ~/.":39958,"pez":39959,"tails":39960,"ĠGlen":39961,"ĠsetImage":39962,"ĠComic":39963,"BLOCK":39964,"ĉThis":39965,"oader":39966,"Ġcapitalist":39967,"_STEP":39968,"(Boolean":39969,"ĠCorrect":39970,"rina":39971,"Ġconcaten":39972,"å®ŀ":39973,"():ĊĊ":39974,"Ġunanim":39975,"lli":39976,"alars":39977,"-ne":39978,"Ġdivor":39979,"ĠKickstarter":39980,"]._":39981,"*'+":40622,"åĿĢ":40623,"acency":40624,"(URL":40625,"_half":40626,"=l":40627,"ĠlistView":40628,"(section":40629,".toArray":40630,"+/":40631,"ĠRodriguez":40632,"istream":40633,"Ġeligibility":40634,"::-":40635,".newInstance":40636,"PB":40637,"ĠAssets":40638,"ĠComposite":40639,"ĠLabs":40640,"ĠHamas":40641,"++);Ċ":40642,"Ġblk":40643,"ĠNeo":40644,"Luc":40645,"@login":40646,"Ġunaware":40647,".met":40648,"_RELEASE":40649,"(ST":40650,"AMIL":40651,"rike":40652,"Ġ(){Ċ":40653,"(sprintf":40654,"ĠAccounts":40655,"ĠVIEW":40656,"ĠAj":40657,"ãĤ°":40658,"Ġwhisk":40659,"Ġidi":40660,"Ġrode":40661,"Ġihn":40662,"ĠElementary":40663,"Qty":40664,"Ġintriguing":40665,"Ġå¤":40666,"Jobs":40667,"ĉoffset":40668,"ĠAhmed":40669,"ĠTaliban":40670,"Ġèİ·åıĸ":40671,"Ġinjected":40672,".Authentication":40673,"_linear":40674,".Decimal":40675,"Ġapples":40676,"Ġshareholders":40677,"Ġbaked":40678,".diff":40679,"ĠEddie":40680,"okers":40681,"Ġconfronted":40682,"voices":40683,"Ġtus":40684,"ĠSpin":40685,"NODE":40686,"_Un":40687,"CTX":40688,"/google":40689,"Temperature":40690,"Ġ'').":40691,"Ġmagnificent":40692,"ĠstartIndex":40693,"sembles":40694,"Anyone":40695,"zk":40696,"ehen":40697,"ĠDame":40698,".strict":40699,"Ġreplaces":40700,"Ġlineback":40701,"Ġpushes":40702,"Ġcheek":40703,"ĠShi":40704,"_BYTES":40705,"REA":40706,"ản":40707,"_CONNECTION":40708,"Gateway":40709,"ĠTravis":40710,"ĠAX":40711,"ĠBasically":40712,"ĠUpgrade":40713,"àª":40714,"themes":40715,"ermo":40716,"kor":40717,"Female":40718,"_attach":40719,"ĠìĤ¬ìļ©":40720,"Ġpoz":40721,"==============Ċ":40722,"(symbol":40723,"ĠSector":40724,"__)ĊĊ":40725,"_padding":40726,"ï¼ļ\"":40727,"Ġfabs":40728,"Ġranged":40729,"setName":40730,"Ġperror":40731,"âĹ":40732,"ĠFileReader":40733,"Ġfulfilled":40734,"_Current":40735,"Ġdominate":40736,"Ġsmugg":40737,"PostMapping":40738,"_force":40739,"Ġbloc":40740,"ĠGiant":40741,"(video":40742,"ĠCU":40743,"SystemService":40744,"Ġelf":40745,"Ġkontakt":40746,"ëª":40747,"kees":40748,"gtk":40749,"ĠparamInt":40750,"Ġmarkup":40751,"uales":40752,"Ġaccounted":40753,"Ġgangbang":40754,"RYPT":40755,"ĠWrong":40756,"Ġcredited":40757,"ĠMESSAGE":40758,"Ġflaws":40759,"Ġbbw":40760,"Ġmetabolic":40761,"ĠOEM":40762,"/event":40763,"(Collectors":40764,"monton":40765,"appear":40766,"Ġopted":40767,"Ġcheat":40768,"Ġdav":40769,"ĠProceed":40770,"Ġê¸":40771,"anked":40772,"из":40773,"ansk":40774,"ĠHang":40775,"ĠCler":40776,"Ġdisgu":40777,"Ġcmap":40778,".cljs":40779,"Ġaument":40780,"lez":40781,"ĠJoined":40782,"_received":40783,"Ġaerial":40784,"otel":40785,"Ġgreet":40786,"\"s":40787,"ĠGenesis":40788,"ĠCalif":40789,"panion":40790,"Ġtailored":40791,"mapping":40792,"andExpect":40793,".track":40794,"atomy":40795,"ĠOw":40796,"ullah":40797,".Yes":40798,"ĠSimpleName":40799,"dbh":40800,"'en":40801,"Ġnonsense":40802,"Ġphilosophical":40803,"(getContext":40804,"Ġisso":40805,"ĠACE":40806,"startDate":40807,"ĠbÄĻd":40808,"ĠAUTHOR":40809,"ĠGlobe":40810,"Ġinsects":40811,"_Al":40812,"ushing":40813,"è®°":40814,"/Home":40815,"ĠLocalDate":40816,"needed":40817,"hesive":40818,"Ġillusion":40819,"äºĮ":40820,"Ġtrat":40821,"xo":40822,"/detail":40823,"_MATCH":40824,"Ġbroadband":40825,"Ġwal":40826,"ĠIllegalStateException":40827,"IRECTION":40828,"Ġnortheast":40829,"esium":40830,"ĠCliente":40831,"ulance":40832,"nty":40833,"Ġtecn":40834,"Devices":40835,"Ġgrains":40836,"ĠOg":40837,"ĠSEL":40838,"udiant":40839,"Ġ++;Ċ":40840,"Ġexplanations":40841,"occo":40842,"Ġdiets":40843,"Ġcohort":40844,"(controller":40845,".Iterator":40846,"-rich":40847,"rocess":40848,"GD":40849,"Ġcarbohydr":40850,"Ġfried":40851,"ĠEmployment":40852,"ìŀ¥":40853,"ĠLeonard":40854,"_${":40855,"quares":40856,"Ġcompanions":40857,"Ġparis":40858,"Ġstimulation":40859,"ĠZoo":40860,"Ġrelevance":40861,"ĠColour":40862,"Ġspear":40863,"otional":40864,"ĠLite":40865,"ĠKosten":40866,"Ġó":40867,"_attachment":40868,"orphic":40869,"Ġdamit":40870,"Ġdlg":40871,"Ġthrive":40872,"CHANGE":40873,"ĠApparently":40874,"Ġatual":40875,"Ġrooted":40876,"(images":40877,"awi":40878,"ariat":40879,"Ġcherry":40880,"STATIC":40881,"mnt":40882,"ĠUserId":40883,"illet":40884,"ĠHispanic":40885,"Ġnak":40886,"Ġcentro":40887,"Ġdims":40888,"_initialize":40889,"ık":40890,"ĠCenters":40891,"REN":40892,"Ġevolutionary":40893,"ĠTopics":40894,"_damage":40895,"emer":40896,"Ġrund":40897,"Ġpunished":40898,"Ġcubic":40899,"fair":40900,"[];ĊĊ":40901,"Ġinstantiate":40902,"Ġoversee":40903,"-delete":40904,"unteer":40905,"startTime":40906,"ĠPipeline":40907,"_GAME":40908,"ĠCir":40909,"ĉNull":40910,".Formatting":40911,"ucumber":40912,"ĠRide":40913,"Ġzoo":40914,"Ġchecker":40915,"åIJĮ":40916,"=C":40917,"Ġgrit":40918,"\");//":40919,"_xy":40920,"ĠDeclaration":40921,"Ġcallable":40922,"Foo":40923,"ĠListItem":40924,"Ġinaccur":40925,"mlin":40926,"ĉData":40927,"Ġevolving":40928,"awan":40929,"Ġcafe":40930,"folk":40931,"_IDX":40932,"ĠAnything":40933,"ĠPalestine":40934,"ĠGridView":40935,"Ġcolony":40936,"ĠGermans":40937,"(+":40938,".pid":40939,".jsx":40940,"ĠSuperior":40941,"Christian":40942,"ĠLect":40943,"ĉGame":40944,"Ġinstrumental":40945,"Animations":40946,"дал":40947,"ĠMoses":40948,"ĉĉčĊĉĉčĊ":40949,"zs":40950,"kte":40951,"ä¸ļ":40952,"_DIST":40953,"bitmap":40954,"dB":40955,"Ġpersistence":40956,"ÑĢоÑģ":40957,"$l":40958,"Bron":40959,"Ġ{|":40960,"_chart":40961,"ĠConsum":40962,"Ġhemp":40963,"Ġ\"))Ċ":40964,"Ġattackers":40965,"Ġknowledgeable":40966,"Ġcet":40967,"Ġviruses":40968,"'I":40969,"Ġpitcher":40970,"Ġsweeping":40971,"=list":40972,"aptops":40973,".depth":40974,"Ġinstructed":40975,"ĠRus":40976,"benhavn":40977,"Ġин":40978,"Sports":40979,"Ġonset":40980,"æĿĥ":40981,".RED":40982,"_si":40983,"ĠPST":40984,".onChange":40985,">tag":40986,"ĠRoh":40987,"_character":40988,"ĠLaws":40989,"ĠBachelor":40990,"_swap":40991,".reactivex":40992,"Ġrewarding":40993,"Medium":40994,"-[":40995,"ĠRecently":40996,"Joint":40997,"partition":40998,"ĠMinutes":40999,"Ġindo":41000,"Ġabsorbed":41001,"ĠGN":41002,"_IND":41003,"Ġsaber":41004,"Spawn":41005,"outputs":41006,"ĠJeffrey":41007,"Ġmedieval":41008,"hed":41009,"Guide":41010,"Ġpsycho":41011,"Ġglam":41012,"Elim":41013,"ädchen":41014,"_plain":41015,"ĠSau":41016,"-four":41017,"Ġanalyzing":41018,"QUERY":41019,"Ġtomato":41020,"_buttons":41021,"VEN":41022,".setStatus":41023,".Url":41024,"+ĊĊ":41025,"Ġcomplaining":41026,"degree":41027,"confirmed":41028,"Ġsubt":41029,"parsed":41030,"Ġtorque":41031,"Ġtroubled":41032,"ĠTARGET":41033,"Ġtrademarks":41034,"ĠCoordinate":41035,"ĠViv":41036,"Ġ//}ĊĊ":41037,"Ġaprès":41038,".getPosition":41039,"(KeyCode":41040,"ĠSilva":41041,"Ġmeteor":41042,"Ġendorsement":41043,"Overview":41044,"ĠPoss":41045,".Inject":41046,"Ġevenly":41047,"Ġvisualization":41048,"Ġwchar":41049,"ĠHDMI":41050,"Ġfunct":41051,"ickname":41052,"','','":41053,"Ġforwards":41054,"ManagedObject":41055,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":41056,"ĉserver":41057,"ĠOutlook":41058,"ĠChronicle":41059,"Ġdubbed":41060,"Ġdok":41061,"ĠWear":41062,".AL":41063,"paren":41064,".Interface":41065,"Interfaces":41066,".cod":41067,"Ġdib":41068,".Globalization":41069,"ĠAcademic":41070,"Ġassms":41071,"Autom":41072,"Ġlw":41073,"ĠNW":41074,"Ġ&&čĊ":41075,"Ġproblema":41076,"ĠManufacturing":41077,"limits":41078,"-mobile":41079,"Ġfilme":41080,"/map":41081,"Ġdoit":41082,"ĠInk":41083,"Ġsued":41084,".arr":41085,"Ġundermin":41086,"ĠProc":41087,"crollView":41088,"__$":41089,"Ġsidewalk":41090,"(that":41091,"ื":41092,"[q":41093,"grammar":41094,"Ġtë":41095,"quito":41096,"Ġspiral":41097,"extended":41098,"Ġfocal":41099,"Ġdigging":41100,"pas":41101,"ĠTall":41102,".proxy":41103,"itures":41104,"TRACT":41105,"ĠRealm":41106,"Ġfeder":41107,"Ġoriented":41108,"ĠAlternative":41109,"Ġowe":41110,"Ġsourced":41111,"inker":41112,".det":41113,"Sep":41114,"ĠQui":41115,"ĠPalmer":41116,"(_,":41117,"samples":41118,"oyer":41119,"ullan":41120,"quez":41121,"Edges":41122,"Ġshout":41123,"ĠAchie":41124,"Ġhaar":41125,"_Construct":41126,"Ġpremature":41127,"Ġrevert":41128,"').Ċ":41129,"Ġschn":41130,"filtered":41131,"nullptr":41132,"Saved":41133,"itecture":41134,"CLA":41135,"Ġvl":41136,"stell":41137,"ĉMe":41138,"ĠLip":41139,"national":41140,"Ġwholly":41141,"Ġsprings":41142,".Timer":41143,"ĉsrc":41144,"elsen":41145,"åħ¶":41146,"Ġcommunicating":41147,"ĠQuiz":41148,"Ġteng":41149,"Ġgez":41150,"ĠOutside":41151,".Sign":41152,"(cs":41153,"Ġdisputes":41154,"ĠWeiss":41155,"annes":41156,">No":41157,"ĠBach":41158,".removeAll":41159,"refer":41160,"/dashboard":41161,"ĠAjax":41162,"IndexChanged":41163,"ĠWeak":41164,"'\"Ċ":41165,"Ġsights":41166,"accessToken":41167,"ĠJoi":41168,"(domain":41169,"ĉcv":41170,"Ġcontinuation":41171,"Ġplum":41172,"adir":41173,".setMessage":41174,"Ġï¼Į":41175,"Ġswallow":41176,"ĠLamp":41177,"Ġqw":41178,"Ġuu":41179,"Coin":41180,"ubic":41181,"ĠDeals":41182,"race":41183,"Ġdictator":41184,"Ġmeme":41185,"turned":41186,"ĠJulie":41187,".gridColumn":41188,"Ġpuppy":41189,"Ġpam":41190,"Ġ){čĊ":41191,"Ġinviting":41192,"Ġfrench":41193,"vim":41194,"Ġwrapping":41195,"Ġ#-}Ċ":41196,"([-":41197,"Early":41198,"Ġshiny":41199,".faces":41200,"Ġrebell":41201,"abcdef":41202,"ält":41203,"Ġestimation":41204,"phys":41205,"losures":41206,"_REL":41207,"Ġexclusion":41208,"ĠSkype":41209,"weise":41210,"-stop":41211,"nothing":41212,"ĠEgg":41213,"isors":41214,"Richard":41215,"Ġcounseling":41216,"Ġcommem":41217,"ĠQMessageBox":41218,"ĠSynd":41219,"ĠFrost":41220,"ĠCompetition":41221,"ĠAwake":41222,"Ġted":41223,"iciones":41224,"ĠDevComponents":41225,"VERTISEMENT":41226,"otti":41227,".runner":41228,"Ġuniquely":41229,".flag":41230,"ĉrs":41231,"_generic":41232,"Ġ```Ċ":41233,"ACHINE":41234,"Ġmein":41235,"(Application":41236,"(br":41237,"Ġratios":41238,":,":41239,"ĠXCTest":41240,"ustainable":41241,"-www":41242,"itles":41243,"_TEMP":41244,"Ġsyst":41245,"umericUpDown":41246,"ĉassertTrue":41247,"Ġwf":41248,".peek":41249,"ĠBulg":41250,"Ġterrifying":41251,".MODE":41252,"ĠGW":41253,"ár":41254,"Ġfic":41255,"Ġcommitments":41256,"-tech":41257,"ĠLiquid":41258,"opez":41259,"zheimer":41260,"aña":41261,"-media":41262,"(animated":41263,"_goal":41264,"Ġgum":41265,"ystone":41266,".SET":41267,"ĠWend":41268,"setCellValue":41269,"Ġmsgs":41270,"cash":41271,"ALLOC":41272,"/aws":41273,"Ġmicrowave":41274,".Pointer":41275,"ĉConsole":41276,"_sorted":41277,"ĠFilip":41278,"Prod":41279,"Ġ//!<":41280,"ingroup":41281,"Ġks":41282,"_TRI":41283,"Ġteaspoon":41284,"ĠATT":41285,"Ġrecovering":41286,"ĠGLOBAL":41287,".Par":41288,"Ġ/>;Ċ":41289,"Ġmarble":41290,"ulators":41291,"ĠCycle":41292,"Ġherbs":41293,"_metric":41294,")!":41295,"_CLOCK":41296,"_Button":41297,"Harry":41298,"è¿Ľ":41299,"Ġstrains":41300,"ĠAppBar":41301,"ĠChan":41302,"/video":41303,"Ġbam":41304,".Progress":41305,"$f":41306,"lemen":41307,"Ġirregular":41308,"ĠDuncan":41309,"ĠMint":41310,"-video":41311,"া":41312,"ówn":41313,"ĠEMPTY":41314,"Ġstacked":41315,"ĠHA":41316,"_cut":41317,"Ġwherein":41318,"ĠWays":41319,"(counter":41320,"è¯ķ":41321,"FormGroup":41322,"Ġblew":41323,"courses":41324,"Ġproductos":41325,"rys":41326,"ĠRestr":41327,"Ġstyling":41328,">s":41329,"Ġpiv":41330,"Ġitertools":41331,"getRepository":41332,"ĠIk":41333,"_devices":41334,"layui":41335,"Ġhalfway":41336,"Ġfranç":41337,"Ġtuning":41338,"OA":41339,"_Node":41340,"arde":41341,"Ġfierce":41342,"licted":41343,"#čĊ":41344,"Ġbreakthrough":41345,"ĠErik":41346,"Ġbride":41347,"Ġ.\"":41348,"culus":41349,"inside":41350,"ĠIndianapolis":41351,"ĠEE":41352,"Ġyog":41353,"urret":41354,".fs":41355,".grad":41356,"_cards":41357,"_accuracy":41358,"_epi":41359,"queda":41360,"/org":41361,"éªĮ":41362,"Ġcompte":41363,"))[":41364,"Outside":41365,"Greater":41366,"ĠRenderer":41367,".actor":41368,"Accounts":41369,"Idle":41370,"_hours":41371,"erner":41372,"Joined":41373,"Ġmenj":41374,"requires":41375,"ĠOPER":41376,".removeChild":41377,"ĉsp":41378,"Ġesse":41379,"rift":41380,"xFE":41381,"ĠShakespeare":41382,"____________":41383,"Ġbudgets":41384,"ModelState":41385,"fillable":41386,"-component":41387,"ocos":41388,"ĠBUTTON":41389,"/io":41390,",out":41391,"sms":41392,"Thomas":41393,"ĠArmed":41394,"resume":41395,"Ġrotating":41396,"ĠVault":41397,"Ġseus":41398,".(*":41399,"Ġamino":41400,"Ġ[]);ĊĊ":41401,"Ġprovoc":41402,"nox":41403,".GetEnumerator":41404,"=======Ċ":41405,"æĸĻ":41406,"_scroll":41407,"Ġfilmed":41408,"ĠSoci":41409,"gap":41410,"gro":41411,"Vote":41412,"\"But":41413,"_RC":41414,"Animal":41415,"ÂĢ":41416,"ibile":41417,"Ġawaken":41418,"orest":41419,"inja":41420,"ĠIvan":41421,"(Command":41422,"Ġ*****":41423,"η":41424,"Ġkvinder":41425,"/helpers":41426,"_cases":41427,"tg":41428,"ìĦ¸":41429,"Registered":41430,"ĉpass":41431,"_digits":41432,"Ġcontour":41433,"Ġinfants":41434,"Ġjustification":41435,"ĠFortunately":41436,"Contr":41437,"ĠonCreateView":41438,"_SAMPLE":41439,"ĠallowNull":41440,"Ġnud":41441,"Ġfetched":41442,"_equ":41443,"ĠUnable":41444,"=\\\"\"":41445,">{Ċ":41446,"Ġcommittees":41447,"istema":41448,"+\".":41449,"ÃŃan":41450,"mant":41451,"Ġsoutheast":41452,"ï¼ĮĊ":41453,"dialogs":41454,"PROJECT":41455,"charger":41456,"-port":41457,"(uuid":41458,".export":41459,"Six":41460,"ĠRP":41461,"Prem":41462,"Ġconscience":41463,"ĠmarginRight":41464,"_distribution":41465,"yaml":41466,"resizing":41467,"Dock":41468,"ĠLocations":41469,"GY":41470,"Seed":41471,"BUFFER":41472,"ossip":41473,"ullen":41474,"Things":41475,"-self":41476,".poll":41477,"PLAYER":41478,"Ġå®":41479,"GROUP":41480,"ĠAway":41481,"Ġgospel":41482,"xfd":41483,"Mary":41484,"ĠPortable":41485,"TURE":41486,"Ġutilis":41487,"Ġseit":41488,"Ġstrand":41489,"Ġtransc":41490,"Ġ(^":41491,"ĠAlfred":41492,".mem":41493,".circle":41494,"Ġ~/":41495,"forcing":41496,"Ġriot":41497,"prox":41498,"THON":41499,"ización":41500,"ĠNI":41501,"rost":41502,"Ġdispro":41503,"_instances":41504,"ï¼ĮâĢľ":41505,"ographer":41506,"endas":41507,"ĠIsaac":41508,"ĠPine":41509,"/dis":41510,"ĠcolorWith":41511,"iterate":41512,"_stride":41513,"Ġpunto":41514,".EventArgs":41515,"(center":41516,"Ġneighboring":41517,"ĠPrison":41518,"ĠMessenger":41519,"Ġepidemic":41520,"dao":41521,"_complex":41522,"Ġgravel":41523,"_DIP":41524,"ément":41525,"ĠAri":41526,"_bitmap":41527,".quit":41528,"(valid":41529,"Ġpend":41530,"Ġrespiratory":41531,"Ġrebound":41532,"DefaultValue":41533,"ãĥŃ":41534,"Ġcommits":41535,".tests":41536,"_fr":41537,"itet":41538,".sf":41539,"Ġspacecraft":41540,"critical":41541,"Ġdepressed":41542,"ĠAnyObject":41543,"Ġunb":41544,"Ġdiscern":41545,"(mysql":41546,"Latin":41547,"ĠBog":41548,"ĠWildlife":41549,"ToFile":41550,"ioxid":41551,"@RestController":41552,"Ġ\"$(":41553,"Ġ<<\"":41554,"Ġdefects":41555,"Ġdatum":41556,"hin":41557,"Ġrealizar":41558,"anyahu":41559,"ĠSig":41560,"@Data":41561,"adaptive":41562,"ĠCatherine":41563,".cr":41564,"ĠCOOKIE":41565,"Ġpictured":41566,"ĠFighter":41567,"Queryable":41568,"ĠAnyway":41569,"ĠGLFW":41570,"_namespace":41571,"_ft":41572,"Ġ])":41573,"Organization":41574,"Ġconstitutes":41575,"Ġquand":41576,"(chunk":41577,"\"/>čĊ":41578,"ĠLakes":41579,"mainwindow":41580,"Carthy":41581,"spin":41582,"(csv":41583,":red":41584,"-commerce":41585,"ู":41586,"Ġdiscovering":41587,"Ġeco":41588,"_fac":41589,"inceton":41590,"ĠGreens":41591,"jwt":41592,"ص":41593,"ĠBroncos":41594,"ĠGoods":41595,"(GTK":41596,"ĠreturnValue":41597,"Ġsiempre":41598,"Ġneutr":41599,"went":41600,"ĠNatal":41601,"Ġenthusiastic":41602,"á»į":41603,"FN":41604,"/database":41605,"Catalog":41606,"Ġbrun":41607,"ĠKash":41608,"_Pl":41609,"iscrim":41610,",width":41611,"Ġinmates":41612,"Assignment":41613,"ĠHaven":41614,"Ġplayground":41615,"exam":41616,"@Controller":41617,"uliar":41618,".getParent":41619,"Ġ\";ĊĊ":41620,":size":41621,"issors":41622,"Ġfis":41623,"Ġalc":41624,"ensation":41625,"ĠNixon":41626,"Ġmighty":41627,"-str":41628,"_special":41629,"_ADC":41630,"ĠTwig":41631,"umbling":41632,"-address":41633,"Ġheroin":41634,"YTE":41635,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":41636,"Friend":41637,"Ġave":41638,"ĠPNG":41639,"ĠKurdish":41640,"DataSetChanged":41641,"Ġblades":41642,"bral":41643,"Steam":41644,"Ġsigu":41645,"IRTUAL":41646,"acos":41647,"UDP":41648,"(database":41649,"hec":41650,"ĠStrings":41651,"_scalar":41652,"ĉdesc":41653,"ĠTLS":41654,";\"Ċ":41655,"ĠCorbyn":41656,"SimpleName":41657,"uell":41658,"ĠEntre":41659,"ellites":41660,"-place":41661,"Ġfrankly":41662,"ĠErf":41663,"CEL":41664,"ĠpaÃŃs":41665,"Ġhedge":41666,"Ġlatent":41667,"ĠIRQ":41668,"ĠHerald":41669,"ĠPrec":41670,"ë³´":41671,".TEXT":41672,"Salary":41673,"Ġautumn":41674,"Ġtravail":41675,".Sum":41676,"Ġcared":41677,"Mor":41678,"Ġintuitive":41679,"Ġjournals":41680,"_IT":41681,"ĠTrou":41682,"ä¼ł":41683,"HasColumnName":41684,"Composite":41685,"Ġspice":41686,"_disk":41687,"_CODES":41688,"ĠIntroduced":41689,"iona":41690,"Ġnuestra":41691,"oct":41692,"ĠĠĠĠĊĠĠĠĠĊĠĠĠĠĊ":41693,"(parameter":41694,"Ġstudios":41695,"ĠprojectId":41696,"Ġbdsm":41697,".SqlClient":41698,"imizer":41699,"ĠCARD":41700,"+t":41701,"aan":41702,".sol":41703,"_Adjust":41704,"Ġrighteous":41705,"ĠLogging":41706,".filters":41707,"_TAB":41708,"ĉsys":41709,"rophic":41710,"otherapy":41711,"ĠBrowse":41712,"keyboard":41713,"RON":41714,"+\\":41715,"ropped":41716,"Ġextensively":41717,"fk":41718,"Ġlime":41719,"years":41720,"Exc":41721,"Ġsph":41722,"Ġcheating":41723,"andro":41724,"ÃŃo":41725,"Ġprince":41726,"oire":41727,"ĠDestination":41728,"ĠConverts":41729,"Ġupstream":41730,"oled":41731,"Ġservants":41732,"Ġsemantic":41733,"Ġcrunch":41734,"Ġeventual":41735,"runner":41736,"/error":41737,"Spin":41738,"Ġsecretly":41739,"Ġassemble":41740,".Person":41741,"enderror":41742,"_<":41743,"Ġpendant":41744,"Sleep":41745,"ĠChemistry":41746,"Ġbosses":41747,"lk":41748,"))),Ċ":41749,"Blockly":41750,"DEVICE":41751,"Ġreflecting":41752,"Ġample":41753,"Milliseconds":41754,"ĠPresidential":41755,"Ġusuarios":41756,"ĠNZ":41757,"ĠSalary":41758,"ĠAmanda":41759,"_np":41760,"jury":41761,"Ġkön":41762,"Ġtherapist":41763,"Ġhomosexual":41764,"ĠDrake":41765,"-window":41766,"ĠLocated":41767,".Driver":41768,"ĠVIDEO":41769,"Ġmerchants":41770,"ĠChest":41771,"-lock":41772,"/php":41773,"Ġmilano":41774,"_STYLE":41775,"arger":41776,"idea":41777,"GUID":41778,"advanced":41779,"meal":41780,"OptionsItemSelected":41781,"='%":41782,"ĠCham":41783,":data":41784,"(stat":41785,"WillAppear":41786,"Ġinformal":41787,"aji":41788,"Ġreproductive":41789,"ĠCAS":41790,"ãģ£":41791,"FUNC":41792,"ĠRuth":41793,")+(":41794,"CONST":41795,"ĠFans":41796,"ĠgroupId":41797,"xffffffff":41798,"Ġsampler":41799,"Ġ}}\">":41800,".the":41801,"Ġhollow":41802,"WAY":41803,"ĠFaculty":41804,"AttributedString":41805,"ĠLooks":41806,"ĠRex":41807,"jk":41808,"ĠMIL":41809,"Ġbard":41810,".Long":41811,"Ġlivest":41812,"Ġskal":41813,"icism":41814,"MAIN":41815,"Ġmucho":41816,"BODY":41817,"Ġese":41818,"ĉuse":41819,"Foot":41820,".SQLException":41821,"Ġinheritance":41822,"received":41823,"Ġputas":41824,"edis":41825,"alsa":41826,"ĠErrorMessage":41827,"Booking":41828,"Ġtract":41829,"acz":41830,"ĠCant":41831,"_regex":41832,"Ġideological":41833,"Ġjihad":41834,"hos":41835,"/sys":41836,"colm":41837,"(pool":41838,"Ġestán":41839,"ĠPending":41840,"emás":41841,"Ġktóry":41842,"));ĊĊĊ":41843,"transactions":41844,"Ġwield":41845,"itere":41846,"erture":41847,"_ss":41848,"Ġstretching":41849,"Ġprisoner":41850,".ReadAll":41851,"Ġbesch":41852,"--;čĊ":41853,"Ġcrisp":41854,"_SCAN":41855,"Ġae":41856,"Strict":41857,"ĠMinneapolis":41858,"ĠBoeing":41859,"aris":41860,"rek":41861,"_pipe":41862,"Ġpriests":41863,"(EIF":41864,"ehicles":41865,"ĠInteractive":41866,"between":41867,"ĉNullCheck":41868,"ĠBlair":41869,"ĠLt":41870,"_inline":41871,"ethyl":41872,"¼":41873,"_packages":41874,"Ġbarrels":41875,"_he":41876,"Ġregexp":41877,"_pts":41878,"_Handler":41879,"ingular":41880,"ĠNissan":41881,"ĠRanch":41882,"Ġperch":41883,"Unsupported":41884,"Smith":41885,"ĠLegends":41886,"Mi":41887,"Ġgf":41888,"steder":41889,"Ġacquiring":41890,"Ġsimulator":41891,"(),\"":41892,"receive":41893,"Ġinplace":41894,"ACTION":41895,"ĠWebDriver":41896,"filesystem":41897,"'+Ċ":41909,"Ġcredible":41910,"amat":41911,"playing":41912,".setImageResource":41913,"quel":41914,"Ġpodr":41915,"geom":41916,"Ek":41917,"ĠQatar":41918,"Ġgeld":41919,"?',Ċ":41920,"Ġcyl":41921,"(ax":41922,"ĠWI":41923,"urally":41924,"ĠBrasil":41925,"Ġsenza":41926,"aley":41927,"onen":41928,"Ġbah":41929,"Ġmolecule":41930,"Rad":41931,"è¿°":41932,"ANCH":41933,"-background":41934,"-agent":41935,"Ġprolifer":41936,":boolean":41937,"Ġtide":41938,"erializer":41939,"_;čĊ":41940,"Fee":41941,"**)":41942,"ergy":41943,"ĠHonor":41944,".Logging":41945,"iris":41946,"Ġundermine":41947,"ĠDy":41948,"Ġtyr":41949,"Ġdeque":41950,"Ġdamer":41951,"([])Ċ":41952,".layoutControlItem":41953,"peated":41954,"CAN":41955,"ragments":41956,"Land":41957,")]);Ċ":41958,"ĠSah":41959,"ĠDECL":41960,"Within":41961,"ĠNamespace":41962,"another":41963,"sembling":41964,".describe":41965,"Consum":41966,"ĠFear":41967,"given":41968,"Orange":41969,"This":41993,"ĠdataIndex":41994,"Ġprintable":41995,"ĠEyes":41996,"_targets":41997,"(Py":41998,".over":41999,"Ġbru":42000,"ampton":42001,"Ġplaintiff":42002,");Ċ":42013,"invest":42014,".*ĊĊ":42015,"Ġtélé":42016,"Ġsuperf":42017,"Ġcascade":42018,"DTD":42019,"Ġvivid":42020,"Ġsubsidies":42021,"ĠHass":42022,"Ġcollaps":42023,"Ġceramic":42024,"{}\".":42025,"ĠLeakage":42026,"-trash":42027,"collapsed":42028,"-social":42029,"ĠChad":42030,"Ġinclined":42031,"Ġsto":42032,"Ġstoryboard":42033,".payment":42034,"stackoverflow":42035,"ĠRaiders":42036,"Ġ#'":42037,"olicies":42038,"ìľ¼ë¡ľ":42039,"emap":42040,"Ġkj":42041,"Ġquota":42042,"ĠGardens":42043,"ë²Ī":42044,"ĠAngels":42045,"Ġoft":42046,"Ġlowercase":42047,"ĠiParam":42048,"Ġcheapest":42049,"unta":42050,"_pkt":42051,"icators":42052,"Ġleurs":42053,"Ġdecreases":42054,"ĉdefine":42055,"PREC":42056,"ammers":42057,"ĠPreparedStatement":42058,"(direction":42059,"Ġcrews":42060,"arked":42061,"ĠMemphis":42062,"ĠSell":42063,"GTK":42064,"Ġmaid":42065,":disable":42066,"éĽĨ":42067,"ĠPf":42068,"Ġalbeit":42069,"openh":42070,"?>\">Ċ":42071,".getSource":42072,"(scale":42073,"Du":42074,"ĠPIL":42075,"_refresh":42076,"Ġbets":42077,"(car":42078,"ĠVon":42079,"|--------------------------------------------------------------------------Ċ":42080,"ĠGrat":42081,"Much":42082,"(Dialog":42083,".stopPropagation":42084,"Ġtek":42085,"Ġexits":42086,"'],$":42087,"ĠphoneNumber":42088,"ucs":42089,"ecimal":42090,"--------------":42091,"inp":42092,".pojo":42093,"Ġcorpus":42094,"Ġpractitioners":42095,".pic":42096,"\"testing":42097,"ĠstringBy":42098,".NotNull":42099,"Ġrang":42100,".Dynamic":42101,"_Render":42102,"аÑĤа":42103,"Waiting":42104,"ĠWik":42105,"Ġoverwhelmed":42106,"%\">":42107,"ĠAE":42108,"}}>Ċ":42109,"uw":42110,"_typ":42111,"Ġbuckets":42112,"Ġgreeting":42113,"Ġlaughter":42114,"Ġantagon":42115,"uggestion":42116,"-email":42117,"ĉtop":42118,"Ġeros":42119,"_tri":42120,"Ġissuing":42121,"Ġhá":42122,"Ġisolate":42123,"Overflow":42124,",E":42125,"Ġnutritional":42126,"ĠAbbott":42127,"Ġnf":42128,".touch":42129,".fetchall":42130,"_zip":42131,"\")}Ċ":42132,"Ġamat":42133,"ĠCisco":42134,"ĠnÃ¥":42135,"PLEX":42136,"Ġsei":42137,"foto":42138,".toJson":42139,"å¤ļ":42140,"ĠKlein":42141,"Ġlibc":42142,"Ġminers":42143,"å¢":42144,"-print":42145,"ĠPride":42146,"Todos":42147,"Ġmasked":42148,"ĠsetData":42149,"Ġtelefon":42150,"Ġunhappy":42151,"ĠTables":42152,"geb":42153,"(debug":42154,"_allowed":42155,"-access":42156,"Ġlogistics":42157,"Ġgems":42158,"ĠMature":42159,"Ġrsp":42160,"ĠAlle":42161,".getBytes":42162,"\\web":42163,"ynchronized":42164,"Paragraph":42165,"Ġthrottle":42166,".sqlite":42167,"consulta":42168,"ĠSeah":42169,"Ce":42170,"Ġsubmar":42171,"ERE":42172,"Vous":42173,"Ġreddit":42174,"Ġsqlalchemy":42175,"-mile":42176,"ocide":42177,"Pour":42178,"}}\">Ċ":42179,"stead":42180,"Ġ@(":42181,"Ġ[])":42182,"ĠAds":42183,"Ġoverload":42184,"ridden":42185,"ĠDesert":42186,"ĠWrap":42187,"ĠPortuguese":42188,"etz":42189,"ĉfirst":42190,"Ġmilestone":42191,"æĹł":42192,"ÑĥÑī":42193,"(success":42194,"\")Ċ":42363,"ĠDollar":42364,"Ġemoji":42365,"Carousel":42366,"-player":42367,"Ġadjusting":42368,"Ġjuga":42369,"allenges":42370,"gene":42371,"(bodyParser":42372,"lopedia":42373,"ĠBehind":42374,"Ġsleeves":42375,"Ġdragging":42376,"ĠChevrolet":42377,"Ġbiz":42378,"ivities":42379,"ĠFrequency":42380,",char":42381,".WHITE":42382,"_preview":42383,")';Ċ":42384,"_ax":42385,"IONS":42386,".cpu":42387,".inputs":42388,"UBE":42389,"_feed":42390,"ĠSupplement":42391,"!).":42392,"esus":42393,"ĠUDP":42394,"Ġmicrophone":42395,"Ġconfirms":42396,".isNotEmpty":42397,"\":\"\",Ċ":42398,"_SCREEN":42399,"ĉexpected":42400,"+-+-+-+-":42401,"ĠHait":42402,"fastcall":42403,"Ġdepict":42404,"vb":42405,"_picture":42406,"ĉdescription":42407,"ĠWife":42408,"uci":42409,"Ġvicious":42410,"ä»ĸ":42411,"ueba":42412,"ĠsetUser":42413,"ãģ¡":42414,"Ġdiving":42415,"Ġopera":42416,"usercontent":42417,"arah":42418,")},":42419,"yun":42420,"velt":42421,"Ġuncovered":42422,"Ġhips":42423,"Ġoscill":42424,"Ġasserting":42425,"ĠXi":42426,".restore":42427,"kea":42428,"Ġspelling":42429,"Ġderive":42430,"abwe":42431,"ĠDow":42432,".setType":42433,"_vs":42434,"Ġcozy":42435,".categories":42436,"Org":42437,"_mgr":42438,"Ġdungeon":42439,"collectionView":42440,"ĠBlank":42441,"acias":42442,"ää":42443,"_cleanup":42444,"_ACTIVITY":42445,"Ġtriangles":42446,".MenuItem":42447,"Ġiphone":42448,"ĠWon":42449,"]]ĊĊ":42450,"ĠComparison":42451,".Doc":42452,"Ġcanonical":42453,"ĠSudan":42454,"'){":42455,"UpInside":42456,"builtin":42457,"ENCY":42458,"xbe":42459,"Ġchuck":42460,"Ġcontradict":42461,"Ġnuestro":42462,"Ġarchitectural":42463,"ĠFib":42464,"Ġcompares":42465,"*k":42466,"Cfg":42467,"çĦ¡":42468,"nten":42469,"Matches":42470,"ĠDOWNLOAD":42471,"_HANDLER":42472,"management":42473,"[S":42474,"ENG":42475,"ÂĢÂ":42476,"fang":42477,"Ġslipped":42478,"ĠLanka":42479,"escaping":42480,"Ġtackles":42481,"ĠPedro":42482,".Prop":42483,".''":42484,".Generated":42485,".NewGuid":42486,"atrigesimal":42487,"illon":42488,"Ġstatistic":42489,"species":42490,"holding":42491,"Drupal":42492,"Ġfundamentally":42493,"Ġbondage":42494,"Ġresolutions":42495,"InlineData":42496,"\\Type":42497,"estion":42498,".wrap":42499,"Ġwarriors":42500,"ĠLOCAL":42501,"Archive":42502,"Ġembraced":42503,"ủ":42504,".Ver":42505,"ĠAffordable":42506,"olesale":42507,"ĠApplied":42508,"ĠConversion":42509,"mega":42510,"_cam":42511,"Ġceremon":42512,"aurus":42513,"ĠVolk":42514,".opens":42515,"/about":42516,"ĠStd":42517,"journal":42518,"()){čĊ":42519,",\"\\":42520,"(Arrays":42521,"ĠDense":42522,"aseña":42523,"änner":42524,"/stat":42525,"userData":42526,"Ġgerman":42527,"Ġtz":42528,"worthy":42529,"FormatException":42530,"pherd":42531,"Ġsmiles":42532,"ĠWhenever":42533,"(adapter":42534,".badlogic":42535,"Ġbriefing":42536,".GridColumn":42537,"-char":42538,"dimension":42539,"ĠCopper":42540,"Ġninth":42541,"Ġ'{{":42542,"Ġrav":42543,"_Table":42544,"Ġderivatives":42545,"ĠRaise":42546,"ĠFut":42547,"armor":42548,"-padding":42549,"Ġremin":42550,"ĉstyle":42551,"ĠMembership":42552,"Ġspreads":42553,"Ġgalleries":42554,"ĠClarke":42555,"Ġconception":42556,"minute":42557,"Ġabusive":42558,"_adj":42559,"Ġterrific":42560,"Ġovert":42561,"ourcing":42562,"Ġentrada":42563,"levels":42564,"Ġcritique":42565,"Ġrespects":42566,"ĠMMA":42567,"iene":42568,"Ġencaps":42569,"ĠRaymond":42570,"Divider":42571,"ivable":42572,"baz":42573,"Ġ@_;Ċ":42574,"ĠClaire":42575,"Ġurging":42576,"CEE":42577,"Ġtransformer":42578,"discord":42579,"ĠJourney":42580,"tos":42581,"Ġcompetitions":42582,"ĠOBJ":42583,"ĠBis":42584,"Ġrelaxation":42585,"idy":42586,"_INSTANCE":42587,"ĠPref":42588,"dados":42589,"iciencies":42590,"ĠMediaQuery":42591,"ĠCube":42592,"ĠStrange":42593,"gpu":42594,"(days":42595,"_InitStruct":42596,"Ġfingerprint":42597,"emat":42598,"ĠGecko":42599,"Ġrails":42600,"ĠLum":42601,"straction":42602,"igung":42603,"(movie":42604,"_dictionary":42605,"_interrupt":42606,"ĠQC":42607,"iked":42608,"appendChild":42609,"recipient":42610,"ré":42611,"Ve":42612,"Ġtowel":42613,".lastIndexOf":42614,"Ġplacebo":42615,"ĠWie":42616,".esp":42617,"(Debug":42618,"operative":42619,"Ġdeceased":42620,"&id":42621,"ĉmutex":42622,"elic":42623,"Ġbapt":42624,"ĉčĊčĊ":42625,"Ġfarther":42626,"Half":42627,".disable":42628,".menuStrip":42629,"leccion":42630,"ĠresultCode":42631,"Ġcans":42632,"-election":42633,"female":42634,"_FIX":42635,"ausible":42636,"ĠPOWER":42637,"Ġreconstruction":42638,"Ġscans":42639,".XtraBars":42640,"âĢĺs":42641,"Removed":42642,"Ġparagraphs":42643,"_margin":42644,"Ġlymph":42645,"Ġbos":42646,"lington":42647,"ĠBaptist":42648,"Ġadvertisements":42649,"ĠManage":42650,"/yyyy":42651,"IOUS":42652,"ENCES":42653,"ĠFiction":42654,"ĉmenu":42655,"ĠFileOutputStream":42656,"ovan":42657,"ĠFeng":42658,"Ġskipping":42659,"getClass":42660,"anni":42661,"Ġrebounds":42662,"Ġpublicity":42663,"Ġingres":42664,"usement":42665,"Ġthoughtful":42666,".Chart":42667,"Ġhatte":42668,"passport":42669,"Ġhooked":42670,"ĠLens":42671,"Ġflagship":42672,"Ġstip":42673,"ĠGEN":42674,"Ġclues":42675,"ipv":42676,"ĠRise":42677,"ĠGew":42678,"tablename":42679,"Ġforemost":42680,"_validate":42681,"_analysis":42682,"olla":42683,"Ġqualifications":42684,"Ġdistributions":42685,"ĠFlower":42686,"Ġtense":42687,"Ġthankful":42688,"Ġclutch":42689,"Ġunified":42690,"roads":42691,"Ġsiti":42692,"Ġstall":42693,"_PRIORITY":42694,"cstdlib":42695,"_USERNAME":42696,".bytes":42697,"?page":42698,"ermalink":42699,"ĠVeget":42700,"/vnd":42701,"-author":42702,".NONE":42703,"ĠConcurrent":42704,"ĠCry":42705,"Ġstarters":42706,"ĠInteraction":42707,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":42708,"ĠLEVEL":42709,"Ell":42710,"ĠcomboBox":42711,"ĠTheresa":42712,"tek":42713,"_Handle":42714,"Ġaby":42715,".gdx":42716,",end":42717,"(Local":42718,"Ol":42719,"knife":42720,"arial":42721,"ĠHoff":42722,"Ġprostituerade":42723,"Doctor":42724,"Instances":42725,".SetValue":42726,"ĉfrom":42727,"Ġluxurious":42728,"Indent":42729,"Allocator":42730,"_DRAW":42731,"(\",\",":42732,"ĠFrances":42733,"ĠgroupBox":42734,"(schema":42735,"Printf":42736,"ORIES":42737,"-gradient":42738,"Ġreput":42739,"arin":42740,"_DONE":42741,"incre":42742,"ignty":42743,"Ġexert":42744,"Ġ-.":42745,"/App":42746,"-through":42747,"Ġdeclining":42748,"Ġdessert":42749,"Ġincumb":42750,"Ġdesignation":42751,".PORT":42752,",strong":42753,"Ġsandbox":42754,"Ġwines":42755,"ĠPav":42756,"$str":42757,"askell":42758,"Ġhö":42759,"ĠPY":42760,"GetInstance":42761,"TextInput":42762,"gameObject":42763,"/events":42764,"createdAt":42765,"ĠlocalVar":42766,"ĠWHITE":42767,"pered":42768,"ilege":42769,"efficient":42770,",color":42771,"cate":42772,"ĠCafe":42773,"Ġsimilarities":42774,"Ġpumps":42775,"ĠHungary":42776,".Username":42777,"Ġskate":42778,"Ġtouchdowns":42779,"Ġaccelerate":42780,"ĠHelen":42781,"OMEM":42782,"ĠKun":42783,"_vol":42784,"ĠfindAll":42785,"ĠMenschen":42786,"ahead":42787,");\"":42788,"kommen":42789,"Ġpossessed":42790,".argmax":42791,".transition":42792,"ARP":42793,"OLUME":42794,"(script":42795,"ĠÐĺ":42796,"ĠFinding":42797,"onces":42798,"Io":42799,"Bold":42800,"Ġrenewal":42801,"_DIALOG":42802,"Ġdisreg":42803,"INTERN":42804,"Ġtoute":42805,"Ġelectr":42806,"ĠGross":42807,"ĉtrue":42808,".Fields":42809,"ĠWIDTH":42810,"ĠDent":42811,"ĠÃģ":42812,"NSNotification":42813,"Ġaos":42814,"Ġmelee":42815,".Validation":42816,"ĠDEC":42817,"-dependent":42818,"Ġsuic":42819,"Traits":42820,"$message":42821,"ĠDear":42822,"ĉFILE":42823,"languages":42824,".Prot":42825,".addr":42826,"-generation":42827,"ICON":42828,"Ġtransplant":42829,"-description":42830,"Ġchasing":42831,"Ġchees":42832,"Ġ}*/Ċ":42833,"Trad":42834,"queries":42835,"/widgets":42836,"subpackage":42837,"Ġespec":42838,"Ġcracked":42839,"Ġcompetitor":42840,"Purchase":42841,"-team":42842,"olecular":42843,"orThunk":42844,"&P":42845,"Ġrelent":42846,"/#{":42847,"ĠproductId":42848,"Ġè¾":42849,"ĠLav":42850,"ĠAlter":42851,".Mode":42852,"ADIO":42853,"grp":42854,"æ·»åĬł":42855,"Quit":42856,"Ġdepths":42857,"-category":42858,"ĠDATABASE":42859,"SPELL":42860,"ĠFalcon":42861,"ĠQStringList":42862,"Ġ''.":42863,"ĠInstitution":42864,"damage":42865,"azor":42866,"belongsTo":42867,"verages":42868,"ĠNONE":42869,"ippets":42870,",\\Ċ":42871,"Ġfootprint":42872,"_archive":42873,"nak":42874,".getField":42875,"ĠReflection":42876,"Ġ']":42877,"ĠHBO":42878,"_discount":42879,"Ġincest":42880,"ĠDodge":42881,"ĠWade":42882,".NO":42883,"\"encoding":42884,"ĠBlockchain":42885,"Ġlawsuits":42886,"ĠMaint":42887,"chten":42888,"Ġétait":42889,"Ġktóre":42890,"_ctl":42891,"(timer":42892,"Battle":42893,"izo":42894,"ayed":42895,"IOR":42896,"ĠGlasgow":42897,"Ġsynth":42898,"_logs":42899,".pose":42900,"_AdjustorThunk":42901,"((&":42902,"Ġunsure":42903,"ystate":42904,"íķĺëĬĶ":42905,"OULD":42906,".ng":42907,"Ġdefaultdict":42908,"workspace":42909,"Ġselective":42910,"PickerController":42911,"YNAMIC":42912,".methods":42913,"Ġpathways":42914,"ĠFew":42915,"KG":42916,"CRYPT":42917,"following":42918,"ĠDLC":42919,"ĠSara":42920,"Ġpreset":42921,"estructor":42922,"ĠKurt":42923,"Ġairplane":42924,"Ġomp":42925,"ĠParents":42926,"ĠMartinez":42927,".complete":42928,"Ġbroadly":42929,"Ġscare":42930,"ĠMé":42931,"Ġelimination":42932,"Ġpoured":42933,"/sw":42934,"Ġcomun":42935,"Ġmasc":42936,"ĠOrganic":42937,"ĠStringUtils":42938,"ilateral":42939,"Ġreluctant":42940,"-age":42941,"Ġnz":42942,".\"\\":42943,"Ġpastor":42944,"alez":42945,"Ġefect":42946,"prov":42947,"/init":42948,"Ġpenn":42949,"unds":42950,"Ġssize":42951,"ĠProj":42952,"basename":42953,"Ġshells":42954,"ĠNeck":42955,"ĠEnforcement":42956,"vided":42957,"stown":42958,"Sphere":42959,"$r":42960,"ussen":42961,"afil":42962,"ĠTelegram":42963,"Ġanalytical":42964,"нÑĭе":42965,"usually":42966,"xn":42967,"Ġhistorian":42968,"ĠGregory":42969,"olph":42970,"ĠUna":42971,"Ġcontributes":42972,"%-":42973,"antiago":42974,"ÑĢед":42975,".region":42976,"Ġabrupt":42977,"ĠUnsupportedOperationException":42978,"ĠTASK":42979,"_finish":42980,"Ġnotorious":42981,"ĠVs":42982,"ĠMQ":42983,"Ġsunset":42984,"Ġunacceptable":42985,"arcer":42986,"Ġillumin":42987,"ĠOrb":42988,"Ġbh":42989,"Este":42990,"_dispatch":42991,"Ġripped":42992,"Ġtoujours":42993,"ĠParcel":42994,"_ll":42995,".userName":42996,".classes":42997,"SOURCE":42998,"(Number":42999,"елÑı":43000,"Ġheadphones":43001,"(side":43002,"constitution":43003,"annah":43004,"čĊĠĠĠĠĠĠĠĠčĊ":43005,"Ġcliff":43006,"-ref":43007,"Ġmostrar":43008,"ĠPowell":43009,"+y":43010,"ĠBG":43011,"_fragment":43012,".Port":43013,"Ġrealizing":43014,"paramref":43015,"Ġhometown":43016,"@Table":43017,"+\"--}}Ċ":43196,"French":43197,"EntityManager":43198,"ĠPlain":43199,"////////////////////////////////////////////////////////////////////":43200,"³":43201,"(RE":43202,"capt":43203,"Ġorganisms":43204,"Ġjets":43205,"olocation":43206,"ĠAppRoutingModule":43207,"Ġglorious":43208,"æľį":43209,"Ġdiscarded":43210,"ĉĉĉĉĠĠĠĠĠ":43211,"ĠArnold":43212,"lug":43213,"Ġparl":43214,"Ġhormones":43215,"Ġmah":43216,"ĠSonic":43217,"Ġorganizers":43218,"_PLATFORM":43219,".inv":43220,"Ġchord":43221,"ventional":43222,"ĉof":43223,"Episode":43224,".Enum":43225,"unkt":43226,"ĠDh":43227,"ĠJared":43228,"ĠNak":43229,"Ġintends":43230,"Endian":43231,"Ġaustralia":43232,"_cv":43233,"(resolve":43234,"Ġclinics":43235,"liked":43236,"ASHINGTON":43237,"inha":43238,"'*":43239,"ĠNP":43240,"_beh":43241,"Ġhf":43242,"Ġwür":43243,"categoria":43244,"$form":43245,"Ġsubway":43246,"ĠisActive":43247,"popular":43248,"Cour":43249,"Ġcooldown":43250,"Ġainsi":43251,"ĠGLuint":43252,"ereal":43253,"ĠarrayOf":43254,"Ġhatch":43255,"==========":43256,"resses":43257,"_PP":43258,".^":43259,"_decay":43260,"ĠBless":43261,"metrics":43262,"ĠCOPYING":43263,"ĠDumpster":43264,"ĠJosé":43265,"ĠDesigns":43266,"<":43269,"Ġ\"}Ċ":43270,"timezone":43271,"Ġeer":43272,"maxcdn":43273,"ĠESC":43274,"igaret":43275,"_connected":43276,"_reverse":43277,"Ġquestionable":43278,"ĠUSC":43279,"Ġtutti":43280,"Ġdropout":43281,"ĠActivities":43282,"ĠWinds":43283,"')));Ċ":43284,"Ġcongest":43285,"ģı":43286,"Ġprolonged":43287,"è¿Ļ":43288,"ĠCrossAxisAlignment":43289,"LEEP":43290,"ĠVALID":43291,"ĠGaz":43292,"Ġdependence":43293,"ĠPrix":43294,".CompilerServices":43295,"jump":43296,"Ġstrat":43297,"circ":43298,"ĠCUSTOM":43299,"xaa":43300,"Ġbmp":43301,"Ġbureau":43302,"Ġwaren":43303,"NX":43304,"(Window":43305,"ĠChristie":43306,"_FE":43307,"Ġtn":43308,"ĠOmega":43309,"communications":43310,"HomePage":43311,"completion":43312,"Ġsupplying":43313,"YPES":43314,"ável":43315,"åĪ¶":43316,"(click":43317,"\\Contracts":43318,"/questions":43319,"Ġez":43320,"AMS":43321,".mesh":43322,"Ġ'\\Ċ":43373,"Robot":43374,"JsonObject":43375,"ĠDF":43376,"ĠProcessor":43377,"_should":43378,".protobuf":43379,"-users":43380,"Ġembry":43381,"FONT":43382,"Ġstartups":43383,"ĠDataSource":43384,")#":43385,"uros":43386,"_Color":43387,"Ġstandalone":43388,"}[":43389,"jd":43390,"Ġforgive":43391,"Ġngx":43392,"ĠGenerally":43393,"Ġconfigurable":43394,"/order":43395,"Ġvas":43396,"')\";Ċ":43397,"ĠRR":43398,"ĠTroy":43399,"Ġcompromised":43400,"ĠSwan":43401,"intendent":43402,"Central":43403,"_keeper":43404,"Ġarquivo":43405,"ĠReadOnly":43406,"_curve":43407,"kv":43408,"entin":43409,"è±":43410,"ĠEy":43411,".imread":43412,"ĠPam":43413,"iffe":43414,"ativity":43415,"xbc":43416,"Ġgrim":43417,"-filled":43418,"namese":43419,"']:":43420,"Ġaur":43421,"ĠGibson":43422,".MouseEvent":43423,"Ġlado":43424,"avadoc":43425,"Ġfamil":43426,"ĠModer":43427,"fps":43428,"ãĢĢãĢĢ":43429,"-example":43430,"ĠAlzheimer":43431,"ĠUtf":43432,"_arguments":43433,"Conclusion":43434,"textContent":43435,"remaining":43436,"Ġinterrupts":43437,"ĠBackup":43438,"ĠMong":43439,"Ġreceptors":43440,"histor":43441,".coroutines":43442,"Ġshouted":43443,"Alarm":43444,"Ġcombust":43445,"Ġgrote":43446,"ultural":43447,"(ids":43448,"--------------------------------------------------------------------------------":43449,"iplinary":43450,"Opts":43451,"ĠYale":43452,"localStorage":43453,"Ġequival":43454,"ĠFleet":43455,"\\b":43456,"*pi":43457,"ĠQLabel":43458,"æ¡":43459,"Ġvx":43460,"ĠACL":43461,"Ġsucesso":43462,"Ġperc":43463,"ĠNotre":43464,"Ġanarch":43465,"Ring":43466,"spb":43467,"Ġstrpos":43468,"stores":43469,"ĠMaple":43470,"(MainActivity":43471,"(\"\"))":43472,"ĠviewHolder":43473,"Quad":43474,"Ġigual":43475,"orsche":43476,".margin":43477,"Ġindie":43478,"Ġfranc":43479,"ĠFormBuilder":43480,"ĠParticip":43481,".flash":43482,"Ġstorms":43483,"Ult":43484,"Ġfen":43485,"[new":43486,"Ever":43487,"=\"Ċ":43488,"Ġlocalized":43489,"_follow":43490,"Ġnave":43491,"Ġdominance":43492,"(tile":43493,"Journal":43494,"ĠVC":43495,"Ġpenetration":43496,"ï¼ķ":43497,"Ġcompartment":43498,"Ġbids":43499,"Formatted":43500,"******/ĊĊ":43501,"(city":43502,"âĢĶit":43503,"[C":43504,"ĠuseCallback":43505,"aub":43506,")?.":43507,"ĠVAR":43508,"ĠSebastian":43509,"ĠMoss":43510,"Ġabundant":43511,"Greg":43512,"ÑĤа":43513,"_ci":43514,"Ġbibli":43515,"CRM":43516,"ĠAttempt":43517,"isme":43518,"dash":43519,"ãĢİ":43520,"_mu":43521,".FormattingEnabled":43522,"Indeed":43523,"-direct":43524,"Ġsucking":43525,"Ġpne":43526,"ocabulary":43527,"ĠPackers":43528,".Navigation":43529,"Ġpied":43530,"cribing":43531,"ĠStuart":43532,".ToDouble":43533,"ĠSecondary":43534,"Saving":43535,"ĠDut":43536,"ĠMadd":43537,"Magic":43538,",H":43539,".documentElement":43540,"ĠBST":43541,"Ġdiffers":43542,"Ġmoreover":43543,"_nd":43544,"SEARCH":43545,"пÑĢав":43546,"æ´":43547,"toMatch":43548,"Ġdecreasing":43549,"-member":43550,"ampus":43551,"(boost":43552,"Daily":43553,"DataGridView":43554,"ĠHttpContext":43555,"Ġhipp":43556,"_workers":43557,"-language":43558,"éĵ":43559,"Ġconsisted":43560,"athing":43561,"ĠMercury":43562,"$content":43563,"Ġpracticed":43564,"ĠModules":43565,"_DAY":43566,"Ġweaknesses":43567,"ĠLodge":43568,"Ġnar":43569,"ĠMate":43570,"Ġjp":43571,"ĠHttpHeaders":43572,"Ġsmo":43573,"ĠTOKEN":43574,"])(":43575,"Ġaqui":43576,"swagen":43577,"Ġsrv":43578,"ĉans":43579,"Around":43580,"ĠManuel":43581,"Ġfictional":43582,"ĠIMG":43583,"Ġ.'":43584,"ĠBerry":43585,"Ġwallpaper":43586,"sexual":43587,"iero":43588,"ĠçļĦ":43589,"ìĨĮ":43590,"BackingField":43591,"ĠAdrian":43592,"BASEPATH":43593,"Ġrepeats":43594,"Ġblues":43595,"Ġunpredict":43596,"_coll":43597,"stacle":43598,"ĠTumblr":43599,"ĠElf":43600,"Ġassurance":43601,"Ġcensus":43602,"ĠIMPORT":43603,"ENDER":43604,"anos":43605,"Ġ=(":43606,"ĠEllis":43607,"\"ĊĊĊĊ":43608,".win":43609,"ĠAbove":43610,"alon":43611,"_tick":43612,"Ġrepresentations":43613,"Ġæķ":43614,"wid":43615,"ĠArms":43616,"Lista":43617,"_failure":43618,"_cm":43619,".FlatAppearance":43620,"Ġthrone":43621,"Patch":43622,"ĠVoy":43623,"engl":43624,"Ġnegotiating":43625,">`":43626,"Ġshoots":43627,"ĠFPS":43628,".Year":43629,"ĠKiss":43630,"ención":43631,"reeting":43632,"FromFile":43633,"Ġresignation":43634,"Ø·":43635,"Ġtwins":43636,"ượ":43637,"Ġgebru":43638,".getContent":43639,".Tree":43640,"ĠEmployees":43641,"ĠFIFA":43642,"Ġcertainty":43643,"(Cl":43644,"Ġtotals":43645,"editable":43646,"à¥Ģ":43647,".Reporting":43648,"Mas":43649,"quiet":43650,".rules":43651,"ĠVO":43652,"conexion":43653,",K":43654,"Ġallocator":43655,"ĠPowder":43656,"\\Repository":43657,"Beat":43658,"_tipo":43659,"Ġ['',":43660,"_INTR":43661,"Ġ<<<":43662,"\");čĊ":43691,"dropIfExists":43692,"ĠBeg":43693,"_HAL":43694,"ĠcrossAxisAlignment":43695,"ĠEvidence":43696,"Ġpeculiar":43697,"Ġinstitute":43698,"veis":43699,"Ġfft":43700,"Ãģ":43701,"Ġzoekt":43702,"analy":43703,"ĠHomeland":43704,"Ġpenetr":43705,"uddenly":43706,"ĉelement":43707,"ĠBren":43708,"ĠTrudeau":43709,"ĠCuban":43710,"jam":43711,"uslim":43712,"_ev":43713,"Ġstems":43714,"}%":43715,"Ŀå§ĭ":43716,"Ġbranding":43717,"Ġcorrespondence":43718,".jquery":43719,"¢åįķ":43720,"ĠReads":43721,"(HttpStatusCode":43722,"assin":43723,"(slot":43724,"ĠGraduate":43725,"///<":43726,"Ġinformations":43727,"ENABLE":43728,"Ġpuis":43729,"Ġfinder":43730,"ĠBris":43731,"Ġnettsteder":43732,"_mid":43733,"Ġogs":43734,"ĠSterling":43735,"Ġarrog":43736,"strftime":43737,"|ĊĊ":43738,"Ġvox":43739,"ĠRegardless":43740,"Ġeso":43741,"ĠComfort":43742,".BooleanField":43743,"Ġuh":43744,"ACY":43745,"Ġsqueez":43746,"ĠVic":43747,"contro":43748,".lo":43749,"Ġire":43750,"ĠComedy":43751,"ë¶":43752,"Ġoriginated":43753,"Ġshipment":43754,"|max":43755,"_guid":43756,"levation":43757,"наÑı":43758,"(undefined":43759,"ĠDDR":43760,"Ġshootings":43761,"ĠLatino":43762,"ENDOR":43763,"Ġaveraging":43764,"Ġgreeted":43765,"Ġtheaters":43766,"ое":43767,"ĠdB":43768,"Ġgst":43769,"Ġdefinite":43770,".Storage":43771,".her":43772,"Ġafore":43773,"ĠReality":43774,"ĠGods":43775,"versed":43776,"Ġhandsome":43777,"Ġexcluding":43778,"(ad":43779,"Quotes":43780,"ĠScheme":43781,"?q":43782,"ĠTamil":43783,"Ticks":43784,"Ġpest":43785,"'n":43786,"Ġpornography":43787,"_modal":43788,"Ġ----------":43789,"Ġdisposable":43790,"FREE":43791,"Ġshark":43792,"CHE":43793,"Ġdepicted":43794,"Ġdemonstrations":43795,"ĠKilled":43796,"ĠRULE":43797,"Ġobsessed":43798,"Ġsimplified":43799,"Postal":43800,"Ġconceptual":43801,"Ġpst":43802,"Las":43803,"_PROJECT":43804,"ucceeded":43805,"olu":43806,"ÄŁi":43807,"Ġpersonalities":43808,"Ġreshape":43809,"Ġenclosed":43810,"ĉptr":43811,"Ġtutorials":43812,"Ġexploded":43813,"_DIRECTORY":43814,"åĨħ容":43815,"Ġcanon":43816,"Ġrecognise":43817,"PAD":43818,"ĠApprox":43819,"ĠRestore":43820,"ĠImportant":43821,"Ġheavier":43822,".Sequential":43823,"Earth":43824,"ĠMilk":43825,".setRequest":43826,".tem":43827,"Ġreconstruct":43828,"Ġskeptical":43829,"_Private":43830,"BUF":43831,"qua":43832,":a":43833,"Ġsek":43834,"Ġdwell":43835,"ossa":43836,"Ġrewarded":43837,"ий":43838,"(topic":43839,"_partition":43840,"Ġ__________________":43841,"Keywords":43842,"ĠFranco":43843,"Lite":43844,"Ġnaken":43845,"Ġза":43846,"OBJECT":43847,"Ġcrafts":43848,"ĠSwap":43849,".Xna":43850,".Connect":43851,"Ġbalcony":43852,"(real":43853,"ĠBarnes":43854,"bir":43855,"ĠTwenty":43856,"ayan":43857,"atars":43858,"ĠPropel":43859,"ĠIhnen":43860,"Upgrade":43861,"Ġcurb":43862,"-second":43863,"Ġneph":43864,".pres":43865,"ìŀħ":43866,".seq":43867,"Ġpadded":43868,"\"?":43869,"jl":43870,"ãĥ¬":43871,"')a":43875,"Coordinates":43876,"Ġenacted":43877,"ENTS":43878,"Ġlac":43879,".final":43880,"ĠPhpStorm":43881,"called":43882,"Ġinquiries":43883,".middleware":43884,"ĠDowntown":43885,"/';Ċ":43886,"Ġkilomet":43887,"accel":43888,"Ġquien":43889,"wstring":43890,"setData":43891,"Ġmanera":43892,"Ġmodular":43893,"rimp":43894,"Ġtariffs":43895,"âĢĻil":43896,"_THROW":43897,"/color":43898,"ĠHTMLElement":43899,"Ġcarro":43900,"Ġprere":43901,"Ġplotting":43902,"ĠPositive":43903,"ĠMachines":43904,"OTES":43905,"Ỽ":43906,"pleasant":43907,"Ġalte":43908,"Ġainda":43909,"these":43910,"Ġcors":43911,"ipay":43912,"ĠAdvisory":43913,"ĠRubio":43914,"jq":43915,"Ġlimestone":43916,"Ġdetached":43917,"设置":43918,"tenant":43919,"ĠDepth":43920,"alore":43921,"ĠÑģÑĤÑĢок":43922,"ĠFORE":43923,"ĠLay":43924,"presentation":43925,")');Ċ":43926,".subplots":43927,"Ïĥ":43928,"NOW":43929,"Gar":43930,"handles":43931,"abra":43932,"puties":43933,"ĠElectrical":43934,"Middle":43935,"ropic":43936,"ĠJD":43937,"ĠDyn":43938,"ĠBristol":43939,"ĠMcCarthy":43940,"Ġstriker":43941,"Ġenumerable":43942,"ĠEvan":43943,".defaults":43944,"quences":43945,")||":43946,"ĉtoken":43947,"âĹı":43948,"-dropdown":43949,"STORE":43950,"ĠGraphic":43951,"(pp":43952,"Expl":43953,"Ġupwards":43954,"ĠDistributed":43955,"ĠWEB":43956,"Jer":43957,"isNaN":43958,"çĶŁæĪIJ":43959,">R":43960,"üssen":43961,"efs":43962,"Ġuncover":43963,"Ġlud":43964,".calculate":43965,"Ġintptr":43966,"Ġmidfielder":43967,".Headers":43968,"Ġmf":43969,"eref":43970,".Metro":43971,"ĠSpeaking":43972,":b":43973,"Ġcryptocurrencies":43974,"Ġdemons":43975,"ĉEXPECT":43976,"Ġwicked":43977,"youtube":43978,":Int":43979,"ĠHindi":43980,"ĠCAT":43981,"Ġع":43982,"rar":43983,"omore":43984,"/per":43985,"/license":43986,"Ġreim":43987,"Ġawaiting":43988,"Ġlethal":43989,"ĠEF":43990,"rounded":43991,"ĠPlatinum":43992,"ĠвÑģе":43993,".coords":43994,".Device":43995,"/item":43996,"ĠWenn":43997,"compileComponents":43998,"ĠKinder":43999,".removeItem":44000,"Ġanda":44001,"bnb":44002,"Ġpra":44003,"(transaction":44004,"Ġembarrassing":44005,"ĉBOOL":44006,".contentView":44007,"Ġeventdata":44008,"atore":44009,"ĠprovidedIn":44010,"irma":44011,"Ġzona":44012,"_HW":44013,"æĻ":44014,"Ġstove":44015,"Ġcounterpart":44016,"_Product":44017,"_MANAGER":44018,"Ġinfring":44019,"ĠERA":44020,"_party":44021,"Ñij":44022,"Ġinici":44023,"_Request":44024,"Ġmiracle":44025,"ĠcancelButton":44026,"Spy":44027,"ató":44028,"Ġpolish":44029,"ĠNicole":44030,".displayName":44031,"\\Requests":44032,"ĠuseHistory":44033,"RouterModule":44034,"Ġstared":44035,"IDER":44036,"ÑĥнкÑĨи":44037,"Ġnota":44038,"$arr":44039,"pecified":44040,"Ġtopp":44041,"_DRIVER":44042,"/ng":44043,"åł":44044,"_tm":44045,"%timeout":44046,"\"":44488,"tlement":44489,"$(\"":44490,"FromString":44491,"ĠBild":44492,"Ġconventions":44493,"_native":44494,"ĠInspector":44495,"ĠPist":44496,"ubar":44497,"Ġregs":44498,"ĠPilot":44499,"Thus":44500,">'+":44501,"Ġcela":44502,".news":44503,"(Product":44504,"Living":44505,"Russia":44506,"Ġfacet":44507,"etical":44508,"Ġ['$":44509,"/[":44510,"ĠDire":44511,"Ġgases":44512,"ĠINFORMATION":44513,"ĠEat":44514,"ĠForums":44515,"ĠCharacters":44516,"_met":44517,"Ġìĭľ":44518,"Ġkings":44519,"achie":44520,"ĠLambda":44521,"Ġtimers":44522,"ĠLighting":44523,"ĠCasey":44524,"addir":44525,"andex":44526,".answer":44527,"ĠHip":44528,"ĠPrincip":44529,"StartDate":44530,"ĠãĢĮ":44531,"tres":44532,"Ġ&#":44533,".MaxValue":44534,"ĠProblems":44535,"Ġlatex":44536,"OfClass":44537,"ĠLynn":44538,"//'":44539,"Ġvoyage":44540,"Ġshuttle":44541,"ĠRoller":44542,"ĠRuntimeError":44543,"uya":44544,"Dic":44545,"ĉbuilder":44546,"Ġbullying":44547,"Ġsimplest":44548,".called":44549,"ĠLR":44550,"Ġmorality":44551,"Ġsturdy":44552,"tracking":44553,".swagger":44554,"_BIND":44555,"ITOR":44556,"-urlencoded":44557,"ĠÑħ":44558,"ĠTrinity":44559,"Ġtraps":44560,"Ġ|-":44561,"ĠsetText":44562,"Ġbargain":44563,"Ġbrakes":44564,".getCode":44565,"Ġmigrate":44566,"Ġribbon":44567,")return":44568,"Ġcharger":44569,"acom":44570,"ADIUS":44571,"ĠAmbassador":44572,"-after":44573,"Ġanni":44574,"ĉspin":44575,"Concept":44576,"ĠHenderson":44577,"ĠHOST":44578,".rank":44579,"ĠNortheast":44580,"Ġberlin":44581,"Ġrequis":44582,".feed":44583,"ĠsourceMapping":44584,"ĠRencontre":44585,".ajax":44586,"nestjs":44587,"Ġtrek":44588,"ĠNacional":44589,"Ġ&[":44590,"Ġpayable":44591,"ortex":44592,"Ġdept":44593,"fieldName":44594,"Ġcompletes":44595,"ĠRVA":44596,"Ġonions":44597,"alignment":44598,"Formats":44599,"Ġ'{$":44600,"HashSet":44601,"ĠBod":44602,".InvariantCulture":44603,"Ġsettlements":44604,"Ġhydr":44605,".updated":44606,"venth":44607,"(seconds":44608,"=\"/\"":44609,"Ġwebpage":44610,"(ĊĊ":44611,"Ġtir":44612,"Ġtoes":44613,"ĠBrick":44614,"Ġambition":44615,"Pot":44616,"=max":44617,"ETIME":44618,"Ġdepot":44619,"calls":44620,"ĠNorwegian":44621,"`:":44622,"Ġburger":44623,"Ġprofessors":44624,"ĠAllocate":44625,"-thirds":44626,"-chart":44627,"Ġford":44628,"*N":44629,".kotlin":44630,"Ġpaperwork":44631,"ĠDEVICE":44632,"%@\",":44633,"respect":44634,"(mp":44635,"é«ĺ":44636,"-if":44637,"Ġcushion":44638,"obot":44639,"Ġparc":44640,"SPACE":44641,"ĠNetanyahu":44642,"Ġselfish":44643,"feat":44644,"Ġclientes":44645,"-tools":44646,"Ġporch":44647,"Ġjq":44648,".verbose":44649,"Ġliberals":44650,"])ĊĊĊ":44651,"pies":44652,"NotBlank":44653,"(term":44654,"ÈĽi":44655,"_Params":44656,".normalize":44657,"Bullet":44658,"ASIC":44659,"(hex":44660,"_cliente":44661,"+,":44662,"_DI":44663,"Ġforthcoming":44664,"}\")]Ċ":44665,"seo":44666,"Um":44667,">Name":44668,"Ġcomfortably":44669,"irectional":44670,"WITH":44671,"/pr":44672,"ĠPoor":44673,"ĠVitamin":44674,"vic":44675,"GH":44676,"Ġpriorit":44677,"ĠNN":44678,"ĠClosed":44679,"¤í":44680,"ĠisOpen":44681,"\\Console":44682,"AndFeel":44683,".SUCCESS":44684,"_OPERATION":44685,"polation":44686,"ĠTas":44687,"psz":44688,">'.":44689,"CURRENT":44690,"Vendor":44691,"hosts":44692,"ĠErd":44693,">tagger":44694,"ĠsourceMappingURL":44695,"Ġmarathon":44696,"_closed":44697,"Ġexemption":44698,"Ġrecognizes":44699,"ideshow":44700,"'$":44701,"('/');Ċ":44702,"mits":44703,"warz":44704,"ĠCherry":44705,"µ¬":44706,"nor":44707,"porte":44708,"Ġwl":44709,"_backup":44710,".getBoolean":44711,".getResource":44712,"Ġdefinitive":44713,".EditText":44714,"ĠsÃŃ":44715,".CONT":44716,"ĠPLAYER":44717,".cards":44718,"ĠShore":44719,"('/')Ċ":44720,"cluir":44721,"WebDriver":44722,"(month":44723,"-release":44724,"Ġinspector":44725,"å£":44726,"ĠNF":44727,"_clip":44728,"åŃIJ":44729,"Ġinteracting":44730,".tmp":44731,"Ġ'''ĊĊ":44732,"Ġdee":44733,"Ġfrost":44734,"\"]))Ċ":44735,"ĠPlaces":44736,"Throws":44737,"fork":44738,"/day":44739,"iPhone":44740,"ĠMIC":44741,"Ġfolding":44742,"Ġcrore":44743,"ĠChiefs":44744,"pherical":44745,"(price":44746,".WriteString":44747,"Ġexiting":44748,"]',Ċ":44749,"ighting":44750,"Ingredient":44751,"(vertex":44752,"ĠscrollView":44753,"hf":44754,":new":44755,"SEN":44756,"sector":44757,"Ġspins":44758,"ĠScheduler":44759,"otechn":44760,"semicolon":44761,"FontOfSize":44762,"ĠSpecifically":44763,"flamm":44764,".ObjectId":44765,"Ġconta":44766,"_permissions":44767,"ĉFROM":44768,"ICODE":44769,"/kg":44770,"ĠHotels":44771,"-med":44772,"ĠDin":44773,"Ġnavy":44774,"getParam":44775,"Ġmend":44776,"Ġportrayed":44777,"ĠMetropolitan":44778,"Painter":44779,"Ġreferral":44780,"_good":44781,"Ġmarvel":44782,"osaic":44783,">(&":44784,".ur":44785,"Ġestos":44786,"William":44787,"Ġtimber":44788,"Ġquelques":44789,"ĠDocuments":44790,".Xaml":44791,"Ġbatches":44792,"éģĵ":44793,"ĠReleased":44794,"Tail":44795,"COOKIE":44796,"heid":44797,"_station":44798,"ĠVia":44799,"Sale":44800,"ĠRepeat":44801,"Ġpromin":44802,"ĠZo":44803,"-forward":44804,"ĠIon":44805,"itary":44806,"Ġjus":44807,"-request":44808,"Ġproudly":44809,"ĠStreaming":44810,"(MouseEvent":44811,"ĠSprint":44812,"_rotation":44813,"Repositories":44814,"Ġtart":44815,"ĠÑģв":44816,"Ġmappings":44817,"èª":44818,"Cu":44819,"Cycle":44820,"Ġbun":44821,"ĉlua":44822,"ãĥī":44823,"Ġ((!":44824,"Ġcollectively":44825,"ĠCond":44826,"Ġwszyst":44827,"(lib":44828,"openhagen":44829,"_skip":44830,".ColumnHeader":44831,"éĤ":44832,"perienced":44833,"ıè¿°":44834,"_props":44835,"Ġcontrace":44836,"Ġmatchup":44837,"abetic":44838,".members":44839,"RECT":44840,"(dat":44841,"Ġsog":44842,"renom":44843,"_Method":44844,"Customers":44845,"fullname":44846,"ZN":44847,"retry":44848,"Ġkap":44849,"ĠNeu":44850,"èĬ":44851,"addChild":44852,"willReturn":44853,"_permalink":44854,"Ġenergetic":44855,"ĠWet":44856,"ĠMorr":44857,"Ġgcd":44858,"counts":44859,",type":44860,"dig":44861,"(Login":44862,"Ġcracks":44863,"Ġbacterial":44864,"ĠMeat":44865,"ĠArmstrong":44866,"ĠBronze":44867,"Ġapproximate":44868,"_dirs":44869,"liga":44870,"ÅĤad":44871,"Ġkindness":44872,"Ġcontre":44873,"ĠEVERY":44874,"MET":44875,"Ġannouncements":44876,"gpio":44877,"ĠWaitForSeconds":44878,"ĠPhotoshop":44879,"Ġdiscontin":44880,"/dd":44881,"Ġtopology":44882,"anical":44883,".interface":44884,"aucoup":44885,".HashSet":44886,"ARIANT":44887,"(routes":44888,"ĠTeh":44889,"Ġhype":44890,"]\").":44891,"Ġslam":44892,"Ġbroth":44893,"-inter":44894,"ĠRid":44895,"-manager":44896,"Cancelar":44897,"ĠPagination":44898,"Ġsoundtrack":44899,"Ġposterior":44900,"Ġscrub":44901,"creating":44902,"-*":44903,"irteen":44904,".dy":44905,".symmetric":44906,"Ġ\"\".":44907,"===============":44908,"Ġchassis":44909,"ĠnumberOfRows":44910,"Developer":44911,"_bins":44912,"ĠOUR":44913,"rieb":44914,"Pros":44915,"ĠwiÄĻ":44916,"\"d":44917,"Ġasyncio":44918,"zeigen":44919,"_spi":44920,".ALL":44921,"Ġscrews":44922,"Chinese":44923,"ĠapiKey":44924,"Ġunsuccessful":44925,"ĠSeahawks":44926,"ORG":44927,"ç«ł":44928,"Ġprofessionally":44929,"ĠCoupon":44930,"åŃĹ段":44931,"Convention":44932,"Ġpolym":44933,"æīĭ":44934,"Ġsalvation":44935,"Ġengineered":44936,"ĠWrest":44937,"ĠGCC":44938,"Ġwarmer":44939,"LayoutConstraint":44940,"Ġaggrav":44941,"Scripts":44942,"venture":44943,"Ġrefrigerator":44944,"Ġinnovations":44945,"ĠRunner":44946,"NIC":44947,"ĠRolling":44948,"ControlEvents":44949,"Ġloos":44950,"pac":44951,"ĉpanel":44952,"efe":44953,"ĠBuddha":44954,"--------------Ċ":44955,"åºĵ":44956,"(forKey":44957,"Ġlumin":44958,"Ġ(?":44959,"ĠAIDS":44960,",user":44961,"imientos":44962,"contentType":44963,"antlr":44964,"é¦":44965,"ĠWelt":44966,"Production":44967,"might":44968,"ĠVII":44969,"\",(":44970,"Ġobserving":44971,"Ġdeliberate":44972,"(control":44973,"Ġwithd":44974,"Ġsemana":44975,"STACK":44976,"uchen":44977,"Nice":44978,"ĠDeutschland":44979,"ĠSpecifies":44980,"dma":44981,"izio":44982,"ĠFacts":44983,"_popup":44984,"ĠDirectors":44985,"{:":44986,"[R":44987,"ĠÑįлеменÑĤ":44988,"Ġplat":44989,"Ġdirecting":44990,"ä¸ī":44991,"ĠGilbert":44992,"âĢ¦.ĊĊ":44993,".qml":44994,"Ġthereafter":44995,"Ġdisposition":44996,"draft":44997,"Ġsurgeon":44998,"ĠInsider":44999,"Blend":45000,"ĠTrev":45001,"trinsic":45002,"Topics":45003,"rieve":45004,"_FILENAME":45005,"Ġautres":45006,"Jose":45007,"Producer":45008,"erus":45009,"Ġpetit":45010,"ĠNEXT":45011,"ĠFilters":45012,"Ġreplicate":45013,"\"]).":45014,"Ġlenders":45015,"]\",Ċ":45016,";charset":45017,"CppObject":45018,"Ġfloral":45019,"ĠTipo":45020,"Ġcircuits":45021,"easy":45022,"(&$":45023,"itta":45024,"eryl":45025,"_COMMON":45026,"'}}>Ċ":45027,"-backed":45028,"(variable":45029,"(Index":45030,"Ġvoir":45031,"_locations":45032,"++){":45033,"ĠLouisville":45034,"Ġgratitude":45035,".Mockito":45036,"ĠPowers":45037,"ieurs":45038,"Ġgeographic":45039,"rale":45040,"Ġcra":45041,"ĠSpurs":45042,"iphertext":45043,"ACION":45044,"-common":45045,"Ġvictories":45046,"ĠFinals":45047,".shuffle":45048,"-million":45049,"_PROC":45050,"assume":45051,"Ġils":45052,"DBC":45053,"BootTest":45054,"Ġlavor":45055,".testing":45056,".ast":45057,"\"]/":45058,"moid":45059,"Ġqualification":45060,"gesch":45061,"ĉput":45062,"Ġairports":45063,"JI":45064,"Teacher":45065,"_uniform":45066,"Ġnama":45067,"ĠBast":45068,"ertype":45069,"capture":45070,"getAll":45071,"ĠReynolds":45072,"ooled":45073,".comments":45074,"Ġchin":45075,").*":45076,"Ġили":45077,"tgl":45078,"udos":45079,"ĠdÃŃas":45080,"chai":45081,".program":45082,"Ġpsz":45083,"ĉicon":45084,"phil":45085,"entral":45086,"_WRAP":45087,"ovi":45088,"Ġnostalg":45089,"Infinity":45090,"ĉyield":45091,"Ġvitamins":45092,"Quaternion":45093,"Sink":45094,"_goods":45095,"Ġ........":45096,"ĠWings":45097,"uridad":45098,"-story":45099,"\"])ĊĊ":45100,"idelity":45101,"TypeDef":45102,"Gtk":45103,"ĠíĮ":45104,"_Main":45105,"Ġchez":45106,"ĠRaven":45107,"Ġpayroll":45108,"Ġfreelance":45109,"LLU":45110,"ĠMend":45111,"eday":45112,"ApiModelProperty":45113,".FormBorderStyle":45114,"Ġeconomist":45115,"stanbul":45116,"Ġfreight":45117,"-Agent":45118,"(meta":45119,"Ġsymmetry":45120,"Ġ'..":45121,".Calendar":45122,"-aut":45123,"gf":45124,"pent":45125,"yclopedia":45126,"Ġwishing":45127,"ĊĊĊĊĊĊĊĊĊĊĊĊ":45128,"Ġgentleman":45129,"Ġê³":45130,"=#":45131,"Ġlectures":45132,"âĢľIn":45133,"Ġ!_":45134,"Ġhb":45135,"ĠVendor":45136,"Recently":45137,"_notes":45138,"æıIJ示":45139,"\"My":45140,"HeadersHeight":45141,"_SO":45142,"Ġunwilling":45143,"Ġsuperhero":45144,"gio":45145,"psy":45146,"ĠPeer":45147,"javax":45148,"&apos":45149,"ĠCrisis":45150,"ordinal":45151,"Memcpy":45152,"++++++++++++++++":45153,"-val":45154,"Ġworkbook":45155,"-ap":45156,"=k":45157,"Ġmetallic":45158,"_peer":45159,"ByPrimaryKey":45160,"_SD":45161,"uator":45162,"_SHADER":45163,")Math":45164,".Transform":45165,"Ġcows":45166,"Phi":45167,"ĠClem":45168,"(_(\"":45169,"ĠLud":45170,"-delay":45171,"ĠSecurities":45172,"ĠOrthodox":45173,"Symfony":45174,"(report":45175,"Ġentertain":45176,"EPS":45177,"izoph":45178,"exual":45179,"IRD":45180,"ä»İ":45181,"Ġlith":45182,"Ġsanitize":45183,"Ġfeminine":45184,"ISBN":45185,".authentication":45186,"_pipeline":45187,"/constants":45188,"ĠCONF":45189,"Ġlucr":45190,"ricia":45191,".ttf":45192,".setContent":45193,"Ġstan":45194,"orean":45195,"ĠLloyd":45196,".rawValue":45197,"Ġgor":45198,"ĠBrowns":45199,"Regression":45200,"Ġlowering":45201,"naissance":45202,"Ġblows":45203,"Ġamazed":45204,"Ġunrelated":45205,"Reviews":45206,"Ġruby":45207,"ĠModifier":45208,"Ġgiants":45209,".thread":45210,"Ġcontainment":45211,"ĠStartCoroutine":45212,"umat":45213,"orelease":45214,"ĠRandy":45215,"@endif":45216,"Digest":45217,"Ġsuburban":45218,"=\");Ċ":45219,"Ġannonce":45220,".variable":45221,"\\Foundation":45222,"Ġacre":45223,"Van":45224,"Ġtuples":45225,"dns":45226,"ĠStanding":45227,"_large":45228,"Ġboxing":45229,"SupportActionBar":45230,"ĠFortune":45231,"ĠRum":45232,"_multiple":45233,"archical":45234,"Ġfwrite":45235,"_quote":45236,"Ġfoolish":45237,"Ġcomprising":45238,"Ġоп":45239,"-selected":45240,"vf":45241,"maid":45242,"Nama":45243,"(datetime":45244,"Ġindirectly":45245,"gart":45246,"fixtures":45247,"chos":45248,"ĠHalo":45249,"Ġrecurring":45250,"-news":45251,"vil":45252,"ĠNursing":45253,"-produ":45254,"ĠHQ":45255,"\\HttpFoundation":45256,"enci":45257,"auen":45258,"Ġvy":45259,"ocracy":45260,"Ġdelegation":45261,"Ġasphalt":45262,"ĠsetSelected":45263,"kok":45264,"/rest":45265,"metics":45266,"ĠNSDate":45267,"Ġtravelled":45268,"Ġrecib":45269,"Ġmime":45270,"CLIENT":45271,"ĠGU":45272,"ĠHANDLE":45273,"/Q":45274,"[z":45275,"Ġbothered":45276,"ĠBBQ":45277,"ças":45278,"_examples":45279,"_FIN":45280,"ĠwhiteColor":45281,"Ġastronom":45282,"-dir":45283,"Ġsovereign":45284,"Ġbreeze":45285,"Ġinning":45286,"ĠEdmonton":45287,"gli":45288,".blogspot":45289,"jsx":45290,"Ġversa":45291,"ĠMohammed":45292,".Job":45293,"-toggler":45294,"ĠполÑĮзоваÑĤ":45295,"ardon":45296,"Ġnewborn":45297,"Ġnaval":45298,"noteq":45299,"Ġtumblr":45300,"Ġhentai":45301,"ĠTypically":45302,"Ġloot":45303,".Sprite":45304,"Flight":45305,"Ġwavelength":45306,"-sk":45307,"ĠElle":45308,"_exports":45309,"ĠÑı":45310,"ĠIH":45311,"izophren":45312,"Ġíģ":45313,"_primary":45314,"Ġmois":45315,"ĠBN":45316,"Ġsystemic":45317,"Ġdiferentes":45318,"INCT":45319,"Ġ''ĊĊ":45320,"$q":45321,"WidgetItem":45322,"clide":45323,"$file":45324,"Lemma":45325,"/table":45326,"agrid":45327,"ĠMongoDB":45328,"inte":45329,"Ġapprent":45330,"ÂŃing":45331,".Db":45332,"ĠÃĤ":45333,"hammer":45334,"='';Ċ":45335,"Ġbrokers":45336,"itlement":45337,"semblies":45338,"Ele":45339,"{x":45340,"Ġlastname":45341,"<-":45342,"Ġflatten":45343,"_band":45344,".Root":45345,".readFileSync":45346,"======":45347,".rx":45348,"?čĊ":45349,"Ġmetaphor":45350,"Ti":45351,"conte":45352,"Ġdebit":45353,"Ġcontempt":45354,"CppType":45355,"æĶ¯":45356,"FormField":45357,"ratio":45358,"osopher":45359,"Ġimplant":45360,"PURE":45361,"Ġalta":45362,"_management":45363,"Ġrefine":45364,"ĠCheckBox":45365,"ĠCharl":45366,"-version":45367,"conditional":45368,"venues":45369,"Ġrifles":45370,"Ġoffspring":45371,"Ġmilling":45372,"Ġsharply":45373,"Ġunderwater":45374,"(origin":45375,"_Control":45376,"Ġ.$":45377,"Plugins":45378,"Ġdrying":45379,"Ġillustrates":45380,"-u":45381,"Ġvegetarian":45382,"npc":45383,"Heart":45384,";',Ċ":45385,"comma":45386,"teenth":45387,"asan":45388,"/spec":45389,"_moves":45390,"-margin":45391,"Ġingen":45392,"³³³":45393,"Ġprojet":45394,"Ġotra":45395,"Ġbras":45396,".utc":45397,"Ġslept":45398,"=sub":45399,"abilit":45400,"poster":45401,"Ġsdk":45402,"ouncill":45403,"Ġwd":45404,"PreparedStatement":45405,"ĠDrum":45406,"(attribute":45407,"ĠEthernet":45408,"ĉDB":45409,"California":45410,"cube":45411,"[I":45412,".Created":45413,"ĠHM":45414,"Ġtracing":45415,"FormsModule":45416,"-you":45417,".currency":45418,"feeding":45419,"Ġtbody":45420,"Li":45421,"accion":45422,"nas":45423,"Ġtrouver":45424,"NONE":45425,"\"},čĊ":45426,"Ġftp":45427,"WithIdentifier":45428,"polate":45429,"FileInfo":45430,"Ġpursued":45431,"ĠĠĠĠčĊĠĠĠĠčĊ":45432,"DESCRIPTION":45433,"}*/Ċ":45434,"FromNib":45435,"Ġdecorative":45436,"_SSL":45437,"(chat":45438,"TLS":45439,"Ġsurprises":45440,"alculate":45441,"ĠSplash":45442,"(Configuration":45443,"ĠSEM":45444,"imson":45445,"/library":45446,"":45521,"GED":45522,"faq":45523,"Ġoptionally":45524,"_Dis":45525,"ĠSuccessful":45526,"ĠCensus":45527,"Ġincarcer":45528,"_CARD":45529,"Ġaviation":45530,"ĠGym":45531,"Authority":45532,".Bean":45533,"shader":45534,"NotExist":45535,"_TextChanged":45536,"ĠSTOP":45537,"(team":45538,"\"H":45539,"wg":45540,"Ġgrinder":45541,"Ġstripe":45542,"Ġpreservation":45543,"Claim":45544,"aversal":45545,"warehouse":45546,"targets":45547,"Trust":45548,"Ġallev":45549,",www":45550,"ousse":45551,"_chan":45552,"_Size":45553,"systems":45554,"Ġobjection":45555,"ĠKane":45556,"Ġcorros":45557,"ĠDSL":45558,"Ġua":45559,"ĠMH":45560,"ĠStrategic":45561,"_tcp":45562,"Ġê°Ĵ":45563,"Ġborrowed":45564,"ĠAch":45565,"ĉcommand":45566,"Ġgps":45567,"leston":45568,"ichever":45569,"ĠUA":45570,"Ġassaulted":45571,"Ġspecializes":45572,"ĉsearch":45573,"Hotel":45574,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠčĊ":45575,"ĠPitch":45576,"ĠÙģ":45577,"READY":45578,"Ġparental":45579,"Ġgéné":45580,"Ġdonnées":45581,"Ġdetain":45582,"TARGET":45583,"Ġprotagonist":45584,"ĠclearInterval":45585,"ĠIconButton":45586,"ĠGetAll":45587,"TypeInfo":45588,"EH":45589,"âĢľThey":45590,"Ġ{[":45591,"Ġgag":45592,"ĠÚ©":45593,"ĠDropdown":45594,".free":45595,"gone":45596,"imens":45597,"Ġinstal":45598,"ĉcurl":45599,"_CAN":45600,"ĠBone":45601,"ï¼Ķ":45602,"onyms":45603,"-government":45604,".bindingNavigator":45605,"ĠDans":45606,"ĠMcL":45607,"(en":45608,">(_":45609,"ÐĴÑĭ":45610,".*;čĊ":45611,"=j":45612,"-cor":45613,"Son":45614,".ToolStripItem":45615,"-around":45616,"_XML":45617,"endDate":45618,"Ġslack":45619,"Ġrotated":45620,"Ġnoqa":45621,"Ġcottage":45622,"Ġencontrar":45623,"_skill":45624,"houette":45625,"!čĊ":45626,".weather":45627,"Ġemphasized":45628,"家":45629,"ĠÑģпиÑģ":45630,"ĠCompiler":45631,"(android":45632,"ĠâĢº":45633,".turn":45634,"Ġsuppression":45635,"_calls":45636,"Ġ*@":45637,"(strlen":45638,".hex":45639,"ĠBills":45640,"ĠRSA":45641,"ÏĤ":45642,"ĠEscape":45643,"ementia":45644,"Ġfrontend":45645,"Ġpint":45646,"_exc":45647,"zzo":45648,"[],Ċ":45649,"Ġ\"','\"":45650,".Environment":45651,"Ġaforementioned":45652,"Ġendure":45653,"prototype":45654,"therapy":45655,"ssi":45656,"Deg":45657,"_plugins":45658,".userInfo":45659,"Printer":45660,"ĠPROGRAM":45661,"Ġruins":45662,"Ġempirical":45663,"Ġcrawl":45664,"ĠBoiler":45665,"-comment":45666,".subplot":45667,"_et":45668,"Ġ'.',":45669,"minor":45670,"ĠCustoms":45671,"Ġyaw":45672,"underline":45673,"ĠComo":45674,"(('":45675,"(mean":45676,"Ġchaque":45677,"ĠBlocks":45678,".rad":45679,"ilibrium":45680,"Ġwebdriver":45681,"Ġmelhor":45682,"dana":45683,"ĠAbuse":45684,"ĠSouthwest":45685,"ĠParen":45686,"PERTIES":45687,"ĉIL":45688,"Ġscream":45689,"vu":45690,"Ġincomes":45691,"Ġnim":45692,"Ġlace":45693,"Ġcompensate":45694,"Reverse":45695,"Dat":45696,"_attack":45697,"Ġnour":45698,"achen":45699,"cek":45700,"\"+":45957,"Ġtokenizer":45958,"Ġsovereignty":45959,"ĠPence":45960,"()\");Ċ":45961,"Ġpessoas":45962,".Ge":45963,"ĠIncluded":45964,"Ġpagina":45965,"Ġexposing":45966,"еÑĪ":45967,"_SCRIPT":45968,"/$',":45969,"Thumbnail":45970,"×Ķ":45971,"webElementX":45972,"webElementXpaths":45973,"pressure":45974,"ĠCurry":45975,"_CP":45976,"OLUTION":45977,"ILES":45978,"protect":45979,"oola":45980,"Workspace":45981,"{};Ċ":45982,"ĠUNS":45983,"Ġsympathy":45984,"roker":45985,"Ġremodel":45986,"ĉcell":45987,"Ġatop":45988,".FullName":45989,"Ġfaut":45990,"ĠEasily":45991,"_dynamic":45992,"Ġframed":45993,"Ġmotive":45994,"è·¯":45995,"sam":45996,"Ġmarca":45997,"ĠTextEditingController":45998,"Ġdestructor":45999,"cream":46000,"Ġrude":46001,"ĠBold":46002,"ĠIndigenous":46003,"Ġgens":46004,"Ġrelacion":46005,"(system":46006,"ĠUIFont":46007,"_charge":46008,"USTER":46009,"EV":46010,".Namespace":46011,"Ġmerger":46012,"Ġcalloc":46013,"gang":46014,"BadRequest":46015,"Ġsper":46016,"-design":46017,"Ġâĩ":46018,"Chan":46019,"Ġorganism":46020,",)":46021,"=id":46022,"_plane":46023,"ĠCases":46024,"elfast":46025,"ĠLegislature":46026,"ĠFaker":46027,"Ġinvoking":46028,"-utils":46029,"().'":46030,".face":46031,"Ġguardian":46032,"myModal":46033,"Ġclipboard":46034,"ĠATM":46035,"Ġpeas":46036,"ĠSylv":46037,".calc":46038,"ĠContacts":46039,"intValue":46040,"Ġmodifying":46041,"ĠBarb":46042,".loss":46043,"_percentage":46044,"Asked":46045,"(lst":46046,"ategorical":46047,"-files":46048,"ĠRomania":46049,".Ac":46050,"Ġhai":46051,"ĠFlying":46052,"Ġż":46053,"jp":46054,"ĠTrainer":46055,".arc":46056,"_deg":46057,"Ġtraceback":46058,"OrFail":46059,"FLOW":46060,".old":46061,"oya":46062,"gmt":46063,"isempty":46064,"Ġvaccination":46065,"Ġobsolete":46066,"recognized":46067,"Ġruined":46068,"ĠRein":46069,"ĠTracking":46070,"xfb":46071,"اÛĮ":46072,"Ġvære":46073,"Ġbryster":46074,"ĠITS":46075,"Ġdestiny":46076,"Ġswear":46077,"Ġredes":46078,"Ġclf":46079,"Ġflipped":46080,"ĉhead":46081,"Bluetooth":46082,"ĠOverrides":46083,":Boolean":46084,"_=":46085,"_lr":46086,"spawn":46087,":index":46088,"VALUES":46089,"iskey":46090,"?\");Ċ":46091,".synthetic":46092,"ĠChecking":46093,"structures":46094,"iping":46095,"Ġvocals":46096,"-Up":46097,"ĠManufacturers":46098,"ĠMarriage":46099,"代çłģ":46100,"Ġgarner":46101,"_Client":46102,"parallel":46103,"RIEND":46104,"Ġvinegar":46105,"segue":46106,"JB":46107,"Ġcontacting":46108,"ĠCarroll":46109,"Ġoutreach":46110,"tensor":46111,"_variant":46112,"Ġtheat":46113,"licable":46114,"{|":46115,"tiny":46116,"_letter":46117,"Ġpencil":46118,"HeadersHeightSizeMode":46119,"iltro":46120,".autoconfigure":46121,".drag":46122,".useState":46123,"ĠBMI":46124,"hint":46125,"Compile":46126,"*\\":46127,"enary":46128,"Ġlvl":46129,".Cache":46130,"+=\"":46131,"_tv":46132,"ruitment":46133,"Ġfread":46134,"Articles":46135,"fila":46136,"Ġpackaged":46137,"âĺĨ":46138,"ATHER":46139,"ĠPlanned":46140,"scheme":46141,"Ġdiary":46142,"Ġoffenses":46143,"/F":46460,"ĠStick":46461,"Ġcerc":46462,"ĠSlee":46463,"ĉĉĠĠĠĠĠĠĠĠ":46464,"":46639,"ĉcol":46640,"VG":46641,"_boolean":46642,"recent":46643,"Ġ*)ĊĊ":46644,"ĠRainbow":46645,"ommen":46646,"Ġlur":46647,"Ġoppression":46648,"(\",\");Ċ":46649,"ĠFacility":46650,"DEFINED":46651,"Ġneon":46652,"Ġoffender":46653,"AFP":46654,"ĠCleaning":46655,"[]):":46656,"Ġundocumented":46657,".Repositories":46658,"ĠGuitar":46659,"аÑģÑģив":46660,"Skills":46661,"Ġtestimon":46662,"ryptography":46663,"ĠAmber":46664,"ĠStalin":46665,"Ġlone":46666,"Ġapenas":46667,"Ġdieses":46668,"ĠArduino":46669,"转":46670,"==-":46671,"_Act":46672,"Ġcoded":46673,"âĸł":46674,"amburger":46675,"-links":46676,"Ġarmour":46677,".High":46678,"getContent":46679,"stag":46680,"Ġheck":46681,"ĠìĹĨ":46682,"ĠMcConnell":46683,"ĠConcert":46684,"ĠAlloc":46685,"äre":46686,".replaceAll":46687,"Ġpartitions":46688,"rott":46689,"ĠFle":46690,"_TREE":46691,"reasonable":46692,"ĠReporting":46693,"Ġbillionaire":46694,"scores":46695,"mins":46696,"-eye":46697,"MORE":46698,"abort":46699,"ĠSWT":46700,"Ġinverted":46701,"ĠTeachers":46702,";n":46703,"Ġastro":46704,"нов":46705,"аниÑĨ":46706,"producto":46707,"countries":46708,"ĠOwen":46709,"Ġcontamination":46710,"Ġvibe":46711,"ĠElli":46712,".script":46713,"ĠOlive":46714,"DMA":46715,"vier":46716,":semicolon":46717,"-module":46718,"gressive":46719,"agu":46720,"_players":46721,"Ġresultados":46722,"started":46723,"scrollTop":46724,"=====":46725,"Ġweighing":46726,"Ġ[[[":46727,"zahl":46728,"(NS":46729,"ĠAssertion":46730,"league":46731,".setTextColor":46732,"ĉMessage":46733,"Ġmoms":46734,"_AF":46735,".wh":46736,"ALS":46737,"Ġautre":46738,"]ĊĊĊĊ":46739,".opacity":46740,"ĠBuddhist":46741,"Ġdeaf":46742,"ĠOrganisation":46743,"(Global":46744,"ensch":46745,"Ġheadache":46746,"ĠAlien":46747,"_inode":46748,"ĠStark":46749,"Ġæī":46750,"-lnd":46751,"oref":46752,"_feat":46753,"Ġpedestrian":46754,"Ġnominal":46755,"Ġballoon":46756,"Ġsprites":46757,"PrototypeOf":46758,"ĠApost":46759,"ĠFEATURE":46760,"OH":46761,"Ġrecess":46762,"ĠDonna":46763,"consumer":46764,"$GLOBALS":46765,"ĠGIF":46766,"-frame":46767,"Inicio":46768,"Ġpassages":46769,"DateString":46770,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":46771,".byte":46772,"Bug":46773,"initializer":46774,"pkt":46775,"odium":46776,"ĠDER":46777,".ops":46778,"leri":46779,"Ġgifted":46780,"Ġdetach":46781,"terrain":46782,"elters":46783,"ãģı":46784,".loader":46785,"ĠNGO":46786,"strncmp":46787,"Kh":46788,"(fontSize":46789,"rocket":46790,"Ġprecedent":46791,"ĠAurora":46792,"ĠExperiment":46793,"isphere":46794,"Encoded":46795,"ĠâĢĵĊĊ":46796,"Ġpyramid":46797,"ĠAnniversary":46798,"ofil":46799,"ëŁ":46800,"(plugin":46801,"Coeff":46802,"Ġcooperate":46803,"Ġpredominantly":46804,"ISM":46805,"Phrase":46806,"_DEFINE":46807,"Flip":46808,"AMILY":46809,"ĠMarkets":46810,"ĠStreamReader":46811,"ĠCombine":46812,"Ġmanuscript":46813,"zza":46814,",tp":46815,"Whatever":46816,"ITICAL":46817,"ighbour":46818,"DataProvider":46819,".Texture":46820,"privacy":46821,".SDK":46822,"Ġrecharge":46823,"Ġcpp":46824,"ĠCFG":46825,"(holder":46826,"(py":46827,"mot":46828,"Ġsavoir":46829,"ĠRosa":46830,"ĠPCs":46831,"ĠíĻ":46832,".heroku":46833,"Ġfren":46834,"ĠRiley":46835,"agate":46836,"Ġsond":46837,".xlsx":46838,"Ġhacked":46839,"stad":46840,"Gi":46841,"Ġsanity":46842,"ĠSqlDataAdapter":46843,"...\",":46844,"ĠPussy":46845,"Ġ****************":46846,"Ġhassle":46847,"_PARENT":46848,"ĠUAE":46849,"Ġbeginners":46850,"(Client":46851,"Ġstatistically":46852,".hour":46853,"edelta":46854,"Ġtraction":46855,"uelve":46856,"arat":46857,"Ġsauna":46858,"INVALID":46859,"Ġindictment":46860,"ALLE":46861,"Ġdissent":46862,"ĠTypography":46863,"Ġintentional":46864,"sit":46865,"ĠAnimals":46866,"Ġcountryside":46867,"Ġuart":46868,"}\\\"":46869,"Ġseamless":46870,"¾ç¤º":46871,"Ġautos":46872,"Ġ\"'\";Ċ":46873,"Flush":46874,"ANNOT":46875,"Ġalgebra":46876,"assoc":46877,"ĠWaters":46878,"Ġpreparations":46879,"ronym":46880,"[,]":46881,"Sans":46882,"Ġarmies":46883,"ipeg":46884,"Ġcreamy":46885,".art":46886,"etre":46887,"ĠAnimated":46888,"Ġunpleasant":46889,"emean":46890,"great":46891,"iÄħ":46892,"ĠEarlier":46893,"Ġchic":46894,"Ġpreserving":46895,"(exec":46896,"ĠInvestigation":46897,"ĉGPIO":46898,"Ġrigorous":46899,"ijo":46900,"=num":46901,"ĠtoolStrip":46902,")set":46903,"+\"&":46904,"ĠAcceler":46905,"Ġdevelopmental":46906,"isposable":46907,"Ġflawed":46908,"rene":46909,"Updating":46910,"Ġwatchdog":46911,"Ġdenominator":46912,"Ġsuburbs":46913,"Ġ...)":46914,"Ġconvictions":46915,"closure":46916,".IP":46917,"Ġtranslates":46918,".swt":46919,".Trace":46920,"Ġmettre":46921,".isEnabled":46922,"ĠEffective":46923,".toInt":46924,"Ġenchant":46925,"Ġstunned":46926,"Ġpoi":46927,"/code":46928,"adm":46929,".databinding":46930,"ĠLorem":46931,"________________________________________________________________":46932,"Ġledger":46933,"Ġcara":46934,"ĠGir":46935,"Ġwaits":46936,"Uno":46937,"Ġcwd":46938,"è¾ij":46939,"ĠTResult":46940,"Ġrejo":46941,"Ġemitted":46942,"ĠWestminster":46943,"ä¸Ģ个":46944,"nek":46945,"_Tis":46946,"Ġenact":46947,"ĉwith":46948,"orgia":46949,"Ġjue":46950,"Perform":46951,"SPATH":46952,".topic":46953,"ĠDaten":46954,"ầ":46955,"Ġsitio":46956,"_MM":46957,"\"So":46958,"bial":46959,"Ġscoped":46960,"Requires":46961,"ĠTOTAL":46962,"ĠChancellor":46963,"(contents":46964,"Ġstealth":46965,"devices":46966,"-pass":46967,"ilih":46968,"ĠMalcolm":46969,"ĠDepot":46970,"Ġconfigur":46971,"aussian":46972,"_constraint":46973,"веÑĤ":46974,"GRA":46975,"ĠRates":46976,".dataGridViewTextBoxColumn":46977,"ĠNobel":46978,"itics":46979,"Ġignorant":46980,"ĠReporter":46981,"ĠEbola":46982,"ĠShock":46983,"_relation":46984,"ĠNinja":46985,")c":46986,"Ġticker":46987,".isChecked":46988,"ĠSuppliers":46989,"ĠRapid":46990,"Levels":46991,"âĤ¬âĦ¢":46992,"ĉqueue":46993,"Ġchop":46994,"ĠUnix":46995,"reject":46996,"-calendar":46997,"(sort":46998,"ène":46999,"ercicio":47000,"Ġhect":47001,"CALLTYPE":47002,"roupon":47003,"Ġrentals":47004,"authors":47005,"{name":47006,"ĠFIFO":47007,"Ġlassen":47008,"ĠNous":47009,"Ġsnapped":47010,"Ġfertility":47011,"\"log":47012,"clicked":47013,"Ġplanting":47014,"Ġgb":47015,"/output":47016,"PEAT":47017,"Ġcategoria":47018,"Ġbach":47019,"Professor":47020,"inth":47021,"\"]čĊ":47022,"Recorder":47023,"serde":47024,"ĠTransmission":47025,"trad":47026,"Ġturbo":47027,"_VERTEX":47028,"\\Event":47029,"ilver":47030,"Ġbodily":47031,"ĠSources":47032,"Ġkillings":47033,".xrTableCell":47034,"Ġfolded":47035,"/legal":47036,"uner":47037,"ĠRifle":47038,"ĠMIDI":47039,"_SelectedIndexChanged":47040,".SizeType":47041,"ĠWebSocket":47042,"Ġseleccion":47043,"Sand":47044,"otros":47045,"Ġenvision":47046,"/etc":47047,"ĠMelissa":47048,"Spot":47049,"ное":47050,"_ARM":47051,"Attempt":47052,"ĠBI":47053,"ãģĶ":47054,"ĠDU":47055,"Ġbacklash":47056,"stride":47057,"/classes":47058,"ĠtextColor":47059,"_staff":47060,"oblin":47061,"agenta":47062,".collections":47063,"illage":47064,"'čĊčĊ":47065,"flatten":47066,"_sales":47067,"_MASTER":47068,"TW":47069,"_da":47070,"Pitch":47071,"phies":47072,"Ġzombies":47073,"ĠVERY":47074,"ĠPharmacy":47075,"ĠprogressBar":47076,"Ġhashtag":47077,"Sidebar":47078,"@stop":47079,"(pc":47080,"олж":47081,"MAKE":47082,"ĠCoron":47083,"Ġkvinner":47084,"ĠMaid":47085,"bob":47086,".titleLabel":47087,"Ġsuccesses":47088,"ĠDemocracy":47089,"ĠSurgery":47090,"Ġcougar":47091,"Ġcurso":47092,"Ġloro":47093,"istency":47094,"Senior":47095,"æk":47096,"ĠAAA":47097,"ĠBOOK":47098,"ко":47099,"WSTR":47100,"Ġ*/,Ċ":47101,"oyal":47102,".vector":47103,"ĠSPEC":47104,"SSF":47105,"Ġcompuls":47106,"ĠAppeals":47107,"ĠWinston":47108,"ĠMockito":47109,"contrib":47110,".available":47111,"entityManager":47112,"arias":47113,"_sale":47114,"_rs":47115,"Ġdecoding":47116,"Ġlocator":47117,"olith":47118,"Ġkol":47119,"Ġascii":47120,"ĠRut":47121,"/interface":47122,"ĉĉĉĉĉĉĠĠĠ":47123,"ĠNumer":47124,".flip":47125,"-del":47126,"Ġbolster":47127,"onomic":47128,"Ġzm":47129,"LG":47130,"FindBy":47131,"Ġadaptive":47132,"loo":47133,"Ġvue":47134,"(reverse":47135,"_canvas":47136,".roles":47137,"ificado":47138,"venient":47139,"\"As":47140,"ĠEntr":47141,"aligned":47142,"Ġbereits":47143,"///ĊĊ":47144,".gwt":47145,".employee":47146,"_cli":47147,"Ġanticipate":47148,"éĻIJ":47149,"Ġpik":47150,"Ġmushrooms":47151,"(tt":47152,"Ġoma":47153,"ĠSanchez":47154,"_google":47155,".Valid":47156,"ĠFileName":47157,"ivative":47158,"ked":47159,"-war":47160,"Ġmaturity":47161,"ид":47162,"Ġminer":47163,"Reducers":47164,"ĠLatLng":47165,"_STD":47166,"Digits":47167,"Calc":47168,"-upload":47169,"Ġhandic":47170,"ีà¹Ī":47171,"egrated":47172,"ĠSTM":47173,"Clients":47174,"ĠTurbo":47175,"SYNC":47176,"Ġphotographers":47177,".Out":47178,".character":47179,"BUILD":47180,".unlock":47181,"Ġarises":47182,"ĠCommands":47183,"(\"\");čĊ":47184,"_FORE":47185,";',":47186,"+\"'":47187,".Images":47188,"\"){":47189,"ĠMeyer":47190,"Ġnegatively":47191,"ĠDLL":47192,"Ġexe":47193,"Ġdeficiency":47194,"Ġwildly":47195,"-switch":47196,"construction":47197,"Ġexceptionally":47198,"ĠLiz":47199,"/java":47200,"Ġtheirs":47201,"ĠContemporary":47202,"lis":47203,".fillRect":47204,"ĠNFC":47205,"Ġrehe":47206,"(numbers":47207,"Ġraster":47208,"Ġfiguring":47209,"Ġshowc":47210,"ĠJill":47211,"Ġarcade":47212,"ĠConstructs":47213,"mdl":47214,"('|":47215,"Ġidentifiers":47216,"Ġstellar":47217,"(Connection":47218,"Ġ\"{{":47219,"yor":47220,"(mysqli":47221,"Ġdove":47222,"OfBirth":47223,".disconnect":47224,"_hi":47225,"Ġzwischen":47226,"ĠGrund":47227,"iros":47228,"_Array":47229,".onclick":47230,"ansom":47231,"Answers":47232,"ĉremove":47233,"Fa":47234,"Ġhurry":47235,"-inf":47236,"ĠgetClass":47237,"ĠRegulation":47238,"ĠFLAGS":47239,"misc":47240,"Ken":47241,"_heading":47242,"GHz":47243,"-entry":47244,"Ġbiography":47245,"Sig":47246,"-mf":47247,"Watcher":47248,"âĢľA":47249,"}px":47250,"Ġspicy":47251,"_sq":47252,"Lost":47253,"(track":47254,"али":47255,"Descending":47256,"((":47453,"survey":47454,"Ġíĺ":47455,"...')Ċ":47456,"ĠDivider":47457,"osl":47458,"_CANCEL":47459,"_prepare":47460,"stin":47461,"ĠHeath":47462,".PrimaryKey":47463,"ĠâĨIJ":47464,"ĠLocalDateTime":47465,"Ġcooperative":47466,"Learning":47467,".enqueue":47468,"Ġgoog":47469,"ĠRegression":47470,"imates":47471,"Ġvoyeur":47472,"ĠDrink":47473,"plug":47474,"Ġlender":47475,"mana":47476,"Ġpersonnes":47477,"ypse":47478,"Ġunlink":47479,"ĠRavens":47480,"Ġhurd":47481,"Ġperiodically":47482,"ARGS":47483,"ĠGH":47484,"characters":47485,"...\"ĊĊ":47486,"-establish":47487,"Ġdn":47488,"(condition":47489,"ĠGravity":47490,"Ġestas":47491,"_focus":47492,"Creature":47493,"(site":47494,"Ġcarr":47495,"ĠRL":47496,"ĠRI":47497,"ĠMoto":47498,"ASF":47499,"ĠLuckily":47500,"ĉRoute":47501,"Ġentropy":47502,"(\",\"":47503,"Collect":47504,"(contact":47505,"ĠFlorence":47506,"Ġpremiums":47507,"Ġlifecycle":47508,"Ġbans":47509,"xef":47510,"WebKit":47511,"ĠFloating":47512,"Ġcosa":47513,"Specific":47514,"ĠLoans":47515,"bread":47516,"Ġdescriptors":47517,"Ġ{:.":47518,"THREAD":47519,"ĠTrent":47520,"Ġscop":47521,"QA":47522,"ĠAntar":47523,"pel":47524,"_difference":47525,"_changes":47526,"(...)":47527,"ĠRotation":47528,"ĠLGPL":47529,"ĠJUST":47530,"(Task":47531,"_subset":47532,"ĠTRANS":47533,"åĬĽ":47534,"ĠScout":47535,"-popup":47536,"Ġsmoked":47537,"_Class":47538,"Ġturnover":47539,"brakk":47540,"ĠRocky":47541,"tas":47542,".RegularExpressions":47543,"ĠElliott":47544,"ĠSpinner":47545,"DUCTION":47546,"Ġlibre":47547,"Ġmolto":47548,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠ":47549,"ĠFTP":47550,"mpeg":47551,"(features":47552,"Ġbald":47553,"ĠVid":47554,"Ġshouting":47555,"Lint":47556,"Ġsockets":47557,"Ġprow":47558,"Ġnouvelle":47559,"iscard":47560,"ĠSponsor":47561,"Ġconsulta":47562,")));":47563,"Indian":47564,"ĠRaspberry":47565,"Ġteammate":47566,"ĠJWT":47567,"ĠGhana":47568,"Ġcakes":47569,"primer":47570,"forma":47571,"ergarten":47572,"_Manager":47573,"Ġpreseason":47574,"GAME":47575,"|\"":47576,"ĠBrock":47577,"Ġoccupy":47578,"Ġdecorations":47579,"ánd":47580,"Ġcot":47581,"Ġparan":47582,"Disk":47583,"remain":47584,">?":47585,"Strong":47586,"Ġfrance":47587,"ĠEra":47588,"-cr":47589,".BufferedReader":47590,"ĠParadise":47591,"ĠVAT":47592,"ĠAnders":47593,"Ġlimb":47594,"ampoo":47595,"Ġimperative":47596,"UTILITY":47597,"ĠRecognition":47598,"Ġragazze":47599,"Ġpops":47600,"ypress":47601,"Ġembargo":47602,"//{Ċ":47603,"Ġsyll":47604,"PTR":47605,"åŃĺåľ¨":47606,"Ġdidnt":47607,"Mailer":47608,"Ġacademics":47609,"ĠFrauen":47610,"neider":47611,"-rel":47612,"Ġrainbow":47613,"(In":47614,"Ġsliced":47615,"=============Ċ":47616,"(send":47617,"NSMutableDictionary":47618,"vos":47619,"(package":47620,"Ġordinance":47621,"viewer":47622,"ĠSantos":47623,"-selling":47624,"Ġgov":47625,"ettle":47626,"Ġfounders":47627,"Ġwaking":47628,"slashes":47629,"-pound":47630,"recht":47631,"ات":47632,".onClick":47633,"Ġnord":47634,"ständ":47635,"_when":47636,"UTERS":47637,"icc":47638,"Ġcapsule":47639,"ĠWid":47640,"Marc":47641,"ุ":47642,"rored":47643,"UGE":47644,"LOUD":47645,"ĠAudit":47646,"ipients":47647,"opian":47648,"ĠSue":47649,"Ġwurden":47650,".Helpers":47651,"Ġfactions":47652,"[np":47653,"-than":47654,"Ġreco":47655,"Ġkas":47656,"Ġcmds":47657,"/network":47658,"xbf":47659,"getColor":47660,"Ġbiased":47661,"ĠLak":47662,"Datas":47663,"vents":47664,"Ġë²":47665,"_PS":47666,".Validate":47667,"Invoker":47668,"Ġneuen":47669,"Ġjuvenile":47670,"VISION":47671,"Ġdevote":47672,"Ġlinha":47673,"Ġdiscounted":47674,"\\Config":47675,"Ġworthwhile":47676,"Ġskinny":47677,"ĠCourses":47678,"leys":47679,"ĠMortgage":47680,"Kevin":47681,"Ġannounces":47682,"])*":47683,"reservation":47684,"Ġæķ°":47685,"Ġprejudice":47686,"ĠStringComparison":47687,"Ġbeard":47688,"-win":47689,"ĠSão":47690,"ĉms":47691,"jal":47692,"ĠEarn":47693,"_ports":47694,"ĠNombre":47695,"_COR":47696,"ĠBUILD":47697,".sound":47698,"Yellow":47699,"Ġlinebacker":47700,"Ġcharitable":47701,"jug":47702,"_NONNULL":47703,"ĠDental":47704,"\">${":47705,"ĉmatch":47706,"Russian":47707,"Ġversch":47708,"Ġpinned":47709,"Ġadopting":47710,"OptionsMenu":47711,"Pag":47712,"Ġpairing":47713,"Ġtread":47714,"ercises":47715,"ĠSpread":47716,")i":47717,"ĠBAD":47718,"_tf":47719,"UIImageView":47720,"populate":47721,"bab":47722,"ĠÏĥ":47723,"[++":47724,"Ġopioid":47725,"Ġ##Ċ":47726,"dtype":47727,"ĠStarts":47728,"('/')":47729,"Ġpersonals":47730,"-market":47731,"Ġredundant":47732,"ĠEssential":47733,"Ġscrapy":47734,"Ġим":47735,"acl":47736,"Ġcrear":47737,"ĠBend":47738,"Ġrelieve":47739,"-room":47740,"wife":47741,"ĠvÃł":47742,"ĠQPoint":47743,"Ġquasi":47744,"ĠmethodName":47745,"\\xc":47746,"ĠPeru":47747,"/The":47748,".orm":47749,"Ġviz":47750,"/pdf":47751,"Located":47752,"Ġconfrontation":47753,"ĠChampionships":47754,"Ġhypert":47755,"Ġdj":47756,"ĠUserInfo":47757,"ĠåĪĽå»º":47758,"\\xb":47759,"(sim":47760,"Ġ==Ċ":47761,"Ġstaging":47762,"Ġdrastically":47763,"åѦ":47764,"lords":47765,".less":47766,"ведиÑĤе":47767,"ĠBucket":47768,"ĠMam":47769,".term":47770,"_pi":47771,"czy":47772,".pub":47773,"precio":47774,"ĠVirt":47775,"Ġroman":47776,"itat":47777,"Lex":47778,"_infos":47779,"Ä°":47780,".other":47781,"VELO":47782,"Ġponder":47783,"Ġhanno":47784,"(Page":47785,"doi":47786,"Ġpolite":47787,"Ġprogrammer":47788,"Dies":47789,"$d":47790,"Ġreplication":47791,"addColumn":47792,"frican":47793,"Ġleng":47794,"beer":47795,"oit":47796,"Ġwasting":47797,"ylim":47798,"measure":47799,"Neg":47800,"Ġpartie":47801,".console":47802,"ĠGuinea":47803,"TEL":47804,"_fact":47805,".chunk":47806,"Ġlent":47807,"Ġaller":47808,"Ġà¤ķ":47809,"_idle":47810,"Ġadmissions":47811,"JSONArray":47812,"Ġvibration":47813,".helpers":47814,"å¤ĸ":47815,"Ġhen":47816,"john":47817,"ĠìĥĿ":47818,"Ġjudgement":47819,"Ġgeen":47820,"terra":47821,"^{":47822,"ĠIz":47823,"Ġcâ":47824,"instances":47825,"Ġthreatens":47826,"Ġmüssen":47827,"KindOfClass":47828,"Ġstorytelling":47829,"_demo":47830,"rias":47831,"Privacy":47832,"hift":47833,"ĠYi":47834,"esor":47835,"íķł":47836,"ensitivity":47837,".Writer":47838,"à¸Ĥ":47839,"District":47840,".getJSONObject":47841,"Impro":47842,"(getResources":47843,"ĠSPELL":47844,"roduce":47845,"Ġslowed":47846,"Ġlinewidth":47847,"Ġhonesty":47848,"ĠCoord":47849,"ĠFork":47850,"ĠDispatchQueue":47851,"ĠCliff":47852,"ĠWiring":47853,"_TIMESTAMP":47854,"ollah":47855,"avoid":47856,"++];Ċ":47857,"semantic":47858,"-css":47859,"Ġveto":47860,"ĠMerr":47861,"Ġlegislators":47862,"CEEDED":47863,"Ġquestionnaire":47864,"ĠPills":47865,"Calculate":47866,"(core":47867,"'e":47868,"Ġdislike":47869,"ĠPreferences":47870,"_EXTERNAL":47871,"è°ĥ":47872,"Ġdodge":47873,"æľįåĬ¡":47874,".names":47875,".drawImage":47876,"_prom":47877,"uckland":47878,"Ġ<$>":47879,"ız":47880,"/site":47881,"项":47882,"rophe":47883,"Ġcompelled":47884,"Ġlaptops":47885,"Ġuni":47886,"CLOSE":47887,"Ġcasualties":47888,"ĠUniform":47889,"Terminal":47890,".\",\"":47891,"DAT":47892,"(TreeNode":47893,"ĠGandhi":47894,"(stmt":47895,"AXB":47896,"*M":47897,"Ġumbrella":47898,"animal":47899,"Ġgrpc":47900,"Ġwhereby":47901,"Ġfloats":47902,"ĉarg":47903,"Ġdbg":47904,"Ġexceeding":47905,"EventType":47906,".SaveChangesAsync":47907,"Ġ{{{":47908,"Ġowed":47909,"ahrenheit":47910,"Ġì§":47911,"Ġequipo":47912,"urai":47913,"Ġidol":47914,"]\")Ċ":47915,"_major":47916,"Ġentirety":47917,"ingerprint":47918,"ços":47919,"/account":47920,"ĉright":47921,"ursos":47922,"ĠEDT":47923,"_INSERT":47924,"Ġshining":47925,"Ġ<:":47926,"EdgeInsets":47927,"Ġcolonies":47928,".IM":47929,"ĉĠĉ":47930,"ROAD":47931,"CCCC":47932,"placing":47933,"ĠgetActivity":47934,"emacs":47935,"'%(":47936,".clicked":47937,"ĠThem":47938,"isia":47939,"Buscar":47940,".rename":47941,"Ġoath":47942,"Ġafterward":47943,"ĠUFO":47944,"APS":47945,"ĠJacksonville":47946,".some":47947,"Confirmed":47948,".scan":47949,"igInteger":47950,"Decorator":47951,"shield":47952,"ressive":47953,".did":47954,"请è¾ĵåħ¥":47955,"Ġshutter":47956,"Dam":47957,"Ġparenting":47958,"eyed":47959,"$item":47960,"-develop":47961,"Ġextracts":47962,"Ġdecentralized":47963,"ĠElsa":47964,"_spin":47965,"])+":47966,"-initial":47967,"Ġmultitude":47968,"Ġsensory":47969,"ĠMODEL":47970,"Ġsafeguard":47971,"ì¹":47972,"Ġhunters":47973,"ĠTiny":47974,"INO":47975,"decorate":47976,"ĠNoSuch":47977,"Ho":47978,"(Response":47979,"Ġruler":47980,"ĉshort":47981,"Ġcaster":47982,"ĠclientId":47983,"Ġpdb":47984,"ëıĦ":47985,"itic":47986,"ĠGameState":47987,"ĠnewItem":47988,")ĊĊĊĊĊĊ":47989,"ouis":47990,"noc":47991,".BLACK":47992,"_VECTOR":47993,"----------();":48281,".getP":48282,"anye":48283,"Ġneuron":48284,"ifold":48285,"ĠKnown":48286,"Bitcoin":48287,"Anyway":48288,"ayette":48289,"Ġ'['":48290,"Ãłnh":48291,"mgr":48292,"Ġcorrelated":48293,"Ġnause":48294,"Ġmentality":48295,"hasMany":48296,"ĠFG":48297,"ampie":48298,"ITU":48299,"Fs":48300,".Sp":48301,"_between":48302,"Dependencies":48303,"oug":48304,"Placeholder":48305,"=text":48306,"ĠManaging":48307,"ocalypse":48308,"åĮĹ":48309,"_mag":48310,"fld":48311,"âij":48312,"CAM":48313,"ĠHelpers":48314,"Ġdost":48315,"/out":48316,"Ġassassination":48317,".getImage":48318,"ĠKenny":48319,".')ĊĊ":48320,"){//":48321,"ĠRanger":48322,"Ġgek":48323,"Ġsincere":48324,"čĊ":48527,".getResources":48528,"Ġlump":48529,"_consts":48530,"(ext":48531,"ĉdir":48532,"âĿ":48533,"ĠpaddingTop":48534,"Ġobsession":48535,"Ġbanning":48536,"ĠAppModule":48537,"Ġpartisan":48538,"Ġcatalogue":48539,"Ġminors":48540,"Ġpitches":48541,"weep":48542,"Ġundertake":48543,"Ġthemed":48544,"audit":48545,".scrollTop":48546,"Ġrer":48547,"Ġsymptom":48548,"Ġopenings":48549,".blocks":48550,"openid":48551,"Ġassh":48552,"-save":48553,"ĠPig":48554,"Ġregain":48555,"Ġinicial":48556,"/favicon":48557,"ĉexp":48558,"Ġspices":48559,"iska":48560,"claims":48561,"mak":48562,"definitions":48563,"Ġcorrespondent":48564,"ĠCannabis":48565,"__,Ċ":48566,"ĠLucky":48567,"ĠGaussian":48568,"ĠNearly":48569,"CAD":48570,"']]Ċ":48571,"Ġadequately":48572,"ĠTITLE":48573,"constitutional":48574,"-mm":48575,"_override":48576,"Ġblas":48577,".readyState":48578,"Ġreminis":48579,"Ġreinforced":48580,"ĠCollabor":48581,"Ġdecorating":48582,"Ġbachelor":48583,"ERRUPT":48584,"Ġupright":48585,"ipation":48586,"ĠNoble":48587,"ĠvalueForKey":48588,"ĠsetLoading":48589,".Ignore":48590,"åģ":48591,"Globals":48592,"ĠMent":48593,"ASSES":48594,"Ġlimbs":48595,"ĠHUD":48596,"inci":48597,".iv":48598,"ĠQModelIndex":48599,"Fuse":48600,"Ġpedal":48601,"_FREQ":48602,"(verbose":48603,"Ġlongitud":48604,"ĠCharter":48605,"ê·¸":48606,"Ġbundles":48607,".ignore":48608,"umbo":48609,"EMA":48610,".......":48611,"sx":48612,".Card":48613,"Ġheute":48614,"Ġsteer":48615,"jumlah":48616,"Ġ{_":48617,"_Checked":48618,"Ġfax":48619,"ĠGust":48620,"itchens":48621,"Ġ))ĊĊ":48622,"Ġremarkably":48623,"/XML":48624,"-remove":48625,"_bt":48626,"Ġincub":48627,".package":48628,".currentThread":48629,"ĠHighlander":48630,".side":48631,"splash":48632,"Ġici":48633,"=D":48634,"Ġpuck":48635,"Ġballots":48636,"Ġhugely":48637,"coeff":48638,"ĠpData":48639,".COLUMN":48640,"ĠHealing":48641,"Ġordin":48642,"!),":48643,"Ġ'',čĊ":48644,"(md":48645,"ĠSask":48646,"čĊ":48668,"Ġrá":48669,"Ġblunt":48670,"ĠImageIcon":48671,"ifik":48672,"RTC":48673,"Ġfibers":48674,"Ġtoile":48675,".sent":48676,"ĠPyQt":48677,"$app":48678,"Ġmedio":48679,"Ġgranting":48680,"Ġtslint":48681,"ĠMö":48682,"(figsize":48683,"Ġhurricane":48684,"Ġlifes":48685,"ĠÃĦ":48686,"rocessing":48687,"_standard":48688,"-option":48689,"')))":48690,"Ġvacant":48691,"å·¥":48692,"ĠHollow":48693,"handleChange":48694,"Ġdivider":48695,"ĠEngineers":48696,"Ġsvens":48697,"Ġcompliant":48698,"tanggal":48699,"ĠCredits":48700,"ĠEmirates":48701,"RuleContext":48702,"Ġrealization":48703,"Ġdistracted":48704,"]+=":48705,"Ġaugment":48706,"ĠDw":48707,"otp":48708,"orrent":48709,"Editar":48710,".stock":48711,"Study":48712,"pections":48713,"ĠGameManager":48714,"=cut":48715,"Ġflock":48716,"ĠRomans":48717,"them":48718,"-hop":48719,"Ġscreenshots":48720,"Ġ/*!Ċ":48721,"Ġconversions":48722,"Ġnormalization":48723,"(configuration":48724,"Ġaeros":48725,"_security":48726,"!'Ċ":48727,"Bonus":48728,"ĠDRIVER":48729,"ĉDate":48730,"tie":48731,"ĠWyoming":48732,"Stand":48733,"itre":48734,"Ġshoppers":48735,"Ġdisadvantage":48736,"Ġliking":48737,"ç¬ij":48738,"Ġunderstandable":48739,"SEE":48740,"Ġhoy":48741,"Ġninete":48742,"Ġconfer":48743,"Ġnowrap":48744,"ĠVern":48745,",čĊčĊ":48746,"imestep":48747,"LayoutManager":48748,"à·":48749,"ĉwait":48750,"PLETED":48751,"Japan":48752,"Ġinduce":48753,"Ġå¯":48754,"озв":48755,"_ENDPOINT":48756,".horizontal":48757,"Ġaccelerated":48758,"rimon":48759,"IVES":48760,"Transactions":48761,"Lean":48762,"ĠSOUR":48763,"whether":48764,"yg":48765,"Ġoid":48766,"ĠEntityManager":48767,"OUNTRY":48768,"Ġfila":48769,"OLUMNS":48770,"INUE":48771,"ĠAnchor":48772,"TRAN":48773,"woo":48774,"blockquote":48775,"ĠNurse":48776,"ĠCarp":48777,"Ġredeem":48778,".try":48779,"ĠJP":48780,"Ġtimestamps":48781,"Ġ?>\"><":48782,"ĠREMOVE":48783,"ĠStarbucks":48784,"Really":48785,"Ġflooded":48786,".Callback":48787,"DropDown":48788,"ipro":48789,"Ġtended":48790,"lte":48791,"Ġproportions":48792,"-te":48793,"ĠRena":48794,"licate":48795,"forces":48796,".extra":48797,".authenticate":48798,"вод":48799,"¡°":48800,"ĠforControlEvents":48801,"Ġsenha":48802,"Ġkein":48803,"Ġminist":48804,"ĠPreference":48805,"ĠTelegraph":48806,"Ñĥп":48807,"strpos":48808,"Ġillnesses":48809,"Ġpigs":48810,"ĠgetIntent":48811,"Sol":48812,"Ġ¡":48813,"(cpu":48814,"[prop":48815,"screens":48816,"');?>":48817,"ĠActs":48818,"Ġstrdup":48819,"Ġaverages":48820,"anal":48821,"ĠCasual":48822,"GroupBox":48823,"ĠHandbook":48824,"/comments":48825,"Ġnumbered":48826,"Ġbroadcasting":48827,"çĽij":48828,".nativeElement":48829,".mu":48830,"ĠupdatedAt":48831,"ĠDoesn":48832,".AC":48833,".coll":48834,"Ġrecorder":48835,"_sha":48836,"Bg":48837,"bil":48838,"Ġbolts":48839,"Ġç¬":48840,"Ġimposing":48841,"ĠInformationen":48842,"_flashdata":48843,"economic":48844,"Remark":48845,"ucas":48846,"ĠOfficers":48847,"ĠTER":48848,"Walk":48849,"Ġmercado":48850,"_generate":48851,"HY":48852,"Calling":48853,"snap":48854,"scriptId":48855,".operation":48856,"ĠFlame":48857,"liness":48858,"Ġrented":48859,"_toggle":48860,"-changing":48861,"ĠTY":48862,"'util":48863,"EEP":48864,"Ġgraphql":48865,"ĠUni":48866,"Ġimpulse":48867,".Basic":48868,"Ġenergies":48869,"MARY":48870,"ĠMarcel":48871,"Ġmortal":48872,"Ġfres":48873,"mens":48874,"motion":48875,"Ġsampled":48876,"âĢľThat":48877,"iday":48878,"quipment":48879,"getInt":48880,"ĠAbsolute":48881,",'\"":48882,"uned":48883,".share":48884,"Ġ})(":48885,"mmm":48886,"ĠRising":48887,"ä»»":48888,"Ġunemployed":48889,"xfa":48890,".follow":48891,"ĉĉĉĉĠĠĠĠĠĠ":48892,"slt":48893,".Phone":48894,"Ġknives":48895,"Ġeve":48896,"onClick":48897,"]))čĊ":48898,"ĠWitness":48899,"ĉNS":48900,"ĠEOS":48901,"ĠStefan":48902,"ĠPriest":48903,"âĢĶwhich":48904,"GetString":48905,".By":48906,"Ġupstairs":48907,"Ġdetriment":48908,"broken":48909,"embro":48910,"Ġnicotine":48911,"ilion":48912,"Ġastonishing":48913,"_aff":48914,"ĠLesson":48915,"Ġaccidental":48916,"odor":48917,"Ġdecir":48918,"ĠnewName":48919,"+.":48920,"缸":48921,"igslist":48922,"ĠGithub":48923,"Ġsuccessive":48924,"racial":48925,"Ġenviron":48926,"éªĮè¯ģ":48927,"Ġredirected":48928,"TOTAL":48929,"Ġgrabbing":48930,"ĠLance":48931,"Ġforfe":48932,"_CB":48933,"å¾®":48934,"Elapsed":48935,"_way":48936,"(DialogInterface":48937,"_measure":48938,"xbb":48939,"Dog":48940,"Depart":48941,"-src":48942,"resolver":48943,"withstanding":48944,"_shell":48945,"ĠLastName":48946,"ĠAviation":48947,"Ġbeginner":48948,"(\"%.":48949,"(tool":48950,"Ġнов":48951,":init":48952,"(API":48953,"ĠMorrison":48954,"vtColor":48955,"Ġstaple":48956,"/INFO":48957,"Ġsupernatural":48958,"Ġsteak":48959,"timeline":48960,"zzle":48961,"\"`ĊĊ":48962,"Secondary":48963,"ĠNepal":48964,".StringUtils":48965,"Ġadam":48966,"Ġ(...":48967,"Ġsubstitution":48968,"Ġboarding":48969,"ĠKeyword":48970,"ĠAssault":48971,"dbcTemplate":48972,"ĠorderId":48973,"(engine":48974,".assertThat":48975,"ĠVenus":48976,"Ġhomicide":48977,"ĠAval":48978,"Ġgutter":48979,"ĠSupported":48980,"/part":48981,"Ġacclaimed":48982,"Histor":48983,"Ġmeses":48984,"über":48985,"ĠRenew":48986,"Ġgras":48987,"ĠEk":48988,"Ġinfile":48989,"indy":48990,".music":48991,".Scroll":48992,"ĠAges":48993,"ĠNaruto":48994,"ĠGather":48995,"Ġconfirming":48996,"=(\"":48997,"Ġpitched":48998,"oley":48999,"France":49000,"+'\"":49001,"$total":49002,"Ġonde":49003,"Ġditch":49004,"_sigma":49005,"Ġcontinuity":49006,"reward":49007,"-load":49008,"Ġproceso":49009,"Locked":49010,"staw":49011,"Ġspinal":49012,"lazy":49013,"!==":49014,"jest":49015,"Ġdun":49016,"ĠRodgers":49017,"ĉgrid":49018,"Ġlogos":49019,"ĠBengal":49020,".super":49021,"Provides":49022,"Ġnutrient":49023,".Timestamp":49024,"IZATION":49025,"åĨĮ":49026,"Ġfats":49027,"ĠXxx":49028,"ctica":49029,"Targets":49030,"Ġcontours":49031,"Ġreordered":49032,":Array":49033,"Ġtolerate":49034,"Vir":49035,"Ġterribly":49036,"Ġbricks":49037,"(&_":49038,"hb":49039,"Portal":49040,"ĠBread":49041,".which":49042,"ÂŃt":49043,"asInstanceOf":49044,"Ġjobject":49045,"ĉlength":49046,"_MT":49047,";\">čĊ":49048,"_EXIST":49049,"Ġmaternal":49050,"REL":49051,"Ġê²½ìļ°":49052,"hee":49053,"Ġlayouts":49054,"ĠLap":49055,"aisy":49056,"Ġstumbled":49057,"ĠUIG":49058,"ĠSco":49059,"Ġimpaired":49060,"RESSED":49061,"Ġabuses":49062,"VF":49063,"ARB":49064,".NAME":49065,"rch":49066,"primir":49067,"_completed":49068,"Ġpenny":49069,"Chrome":49070,"(begin":49071,"ernen":49072,"-checkbox":49073,"PlainOldData":49074,"ĠLPC":49075,"rade":49076,"spir":49077,"Ġconceived":49078,"Tips":49079,"ĠIoT":49080,"ĠGan":49081,"èģĶ":49082,"Ġbiases":49083,"Ġconsultants":49084,"pled":49085,"_ht":49086,"associated":49087,"],ĊĊ":49088,"Ġdelightful":49089,"ĠÑĤек":49090,"Helvetica":49091,"(load":49092,"-expand":49093,"_WIDGET":49094,"toa":49095,"ĠAkt":49096,"Ġomn":49097,"Ġclauses":49098,"Intel":49099,"*/}Ċ":49100,"_registration":49101,"ĠoldValue":49102,"Ġrestoring":49103,"Ġunreal":49104,"OVER":49105,"ĉĊĉĊĉĊ":49106,"ATS":49107,"_probe":49108,"Ġdivisor":49109,".updateDynamic":49110,"å¹³":49111,"Produces":49112,"stamp":49113,".jboss":49114,"ĉtask":49115,"!(:":49116,"Ġpsychic":49117,"@class":49118,"Martin":49119,"ĠPassed":49120,"clarations":49121,"hel":49122,"аÑĩ":49123,"ĉcopy":49124,"-bin":49125,"zan":49126,"igram":49127,"াà¦":49128,"(sig":49129,"ĠCaval":49130,"_##":49131,"Ġ%=":49132,"outlined":49133,"ĠAcid":49134,"Ġunpredictable":49135,"-dashboard":49136,"HexString":49137,"+c":49138,".Public":49139,"ẩ":49140,"Ġconveyor":49141,"ĠEB":49142,"Ġselects":49143,"Ġknocking":49144,"ĠCec":49145,"IBUTES":49146,"owaÄĩ":49147,"gatsby":49148,"*v":49149,"entropy":49150,"Ġdispatched":49151,"Ġcamel":49152,"ĠSaturn":49153,"Ġoverweight":49154,"(phone":49155,"parable":49156,"%B":49157,"_vectors":49158,"Ġbrewing":49159,"ĠTk":49160,"ĠDownloads":49161,"ĠSaved":49162,".Price":49163,"Ġcurved":49164,"ĠParenthood":49165,"è¶":49166,".pnl":49167,"pletely":49168,".Day":49169,"Ġadvertisers":49170,"Ġejec":49171,"Ġprzed":49172,"ë¯":49173,"!';Ċ":49174,"ĠKush":49175,"ĠTAB":49176,"Ġquests":49177,"Ġcoincidence":49178,"ummies":49179,"ĠKashmir":49180,"ĠEthics":49181,"_growth":49182,"Ġaktiv":49183,"Ġgrouping":49184,"å¢ŀ":49185,"_truth":49186,"åIJ¬":49187,"todos":49188,"iset":49189,"TexCoord":49190,"ätt":49191,"ĠZur":49192,"roys":49193,"_MAGIC":49194,"Ġbrewery":49195,"(State":49196,"ĠSMALL":49197,"ĠPlants":49198,"itbart":49199,"eacher":49200,"ĠAdelaide":49201,"Lu":49202,"Ġfick":49203,"undles":49204,"_loaded":49205,"ие":49206,"Poll":49207,"ritic":49208,"ELY":49209,"Ġ+'":49210,"ĠProfession":49211,"Ġstamps":49212,"ĠSew":49213,"scrollView":49214,"Ġcommunist":49215,"/problems":49216,"}čĊčĊčĊčĊ":49217,",o":49218,"Ġudp":49219,"Ġobese":49220,"approve":49221,"ancellation":49222,"_Game":49223,"ĠHashtable":49224,"adaptiveStyles":49225,"Ġpossesses":49226,".matcher":49227,"functional":49228,"Mrs":49229,"ĉsave":49230,"ĠDbType":49231,"Ġken":49232,"getContext":49233,"Ġmans":49234,"(rel":49235,"ĠBrotherhood":49236,")`Ċ":49237,"解":49238,".Information":49239,"OutOfRangeException":49240,"ĠSek":49241,"Cas":49242,"Ġbloggers":49243,"Either":49244,"(\"\"\"":49245,"Ġpinch":49246,"Ġcoarse":49247,")p":49248,"ĠPulse":49249,"Ġlearnt":49250,"Ġdentist":49251,"Ġonchange":49252,"Ġdirectives":49253,"(actions":49254,"nyder":49255,"ĠShir":49256,"Trait":49257,"_dep":49258,"ĠPET":49259,"ĠREP":49260,".AppSettings":49261,"cuador":49262,"idenav":49263,"Ġenvi":49264,"Ġslammed":49265,"ĠShoot":49266,"ĠdateFormat":49267,".joda":49268,"veys":49269,"Ġ).ĊĊ":49270,"Ġcareg":49271,"ĠParallel":49272,"_translation":49273,".functions":49274,".obs":49275,"RuntimeException":49276,"[]=":49277,"overview":49278,"ĠSchl":49279,"Ġnoisy":49280,"ĠOnPropertyChanged":49281,"Sending":49282,"Ġunfamiliar":49283,"Upon":49284,"ĠPrints":49285,".typ":49286,"Ġfleeing":49287,"ĉmove":49288,"(Un":49289,"Ġqr":49290,"׾":49291,"_beta":49292,"Ġskies":49293,"ĉme":49294,"WND":49295,"Ġstickers":49296,"blas":49297,"Ġinserts":49298,"Ġverses":49299,"ĠDew":49300,"Ġtangible":49301,"Ġhecho":49302,"POL":49303,"Ġteardown":49304,"omnia":49305,"IBE":49306,".cover":49307,"_strategy":49308,"^-":49309,"setPosition":49310,"uale":49311,"Signed":49312,"Ġiface":49313,"aseline":49314,".setTime":49315,"ĠMineral":49316,"ĠFighting":49317,"skins":49318,"Ġdiscrimin":49319,"Ġdansk":49320,"ĠPrinceton":49321,"acist":49322,"Ġ());Ċ":49323,"tracks":49324,"imonial":49325,"adecimal":49326,"EPROM":49327,"uggle":49328,".Notification":49329,"$mail":49330,"cantidad":49331,"ĠJung":49332,"Ġseekers":49333,"Ġplausible":49334,"tier":49335,"еж":49336,"Ġrapper":49337,"ĠMana":49338,"ĠHttpStatusCode":49339,"Ġburnt":49340,"loses":49341,"ĠFoto":49342,"ĠJsonObject":49343,"Instagram":49344,"Ġsyscall":49345,"Ġrealities":49346,"ĠMATLAB":49347,":^{Ċ":49348,"TERM":49349,"ĠCbd":49350,"ĠParagraph":49351,"Ġtravés":49352,"Ġconstructing":49353,"Ġswal":49354,"Ġpige":49355,"LLLL":49356,"-existing":49357,"Gets":49358,"Ġmelted":49359,"Ġmitigate":49360,"Hen":49361,"Ġhm":49362,"imas":49363,"ĠAo":49364,"ĠPerez":49365,"ĠDAL":49366,"Ġëĭ¤":49367,"Ġdivis":49368,"StoryboardSegue":49369,"ĠModify":49370,"ĠÃľber":49371,"_OVERRIDE":49372,".pem":49373,"untos":49374,"Ġespañ":49375,"Ġ{?":49376,"ĠPAY":49377,"_ipv":49378,"ĠFury":49379,"__.__":49380,"elow":49381,"-centered":49382,"checks":49383,"_Reg":49384,"-Javadoc":49385,"ĉload":49386,"ĠLikewise":49387,"اÙħ":49388,"UNE":49389,".sem":49390,"xcb":49391,"ĠCave":49392,"_sleep":49393,"Ġsilently":49394,"ĠExtreme":49395,".ToUpper":49396,"ĉCHECK":49397,"Ġcue":49398,"ĠQByteArray":49399,"Ġcorrupted":49400,"ĠDé":49401,"Ġimped":49402,"GetName":49403,"Ġinaccurate":49404,"Ġsober":49405,"ее":49406,"Ġbarcode":49407,"--){Ċ":49408,"inki":49409,"Ġép":49410,"Ġdri":49411,"ĠALT":49412,">>>>>>>>":49413,"onta":49414,"[L":49415,"Ġinteres":49416,"verting":49417,"Ġdiagnostics":49418,"pdev":49419,"è©":49420,"ĠIntegrated":49421,").'":49422,"_gc":49423,"$text":49424,".games":49425,"ĠTerra":49426,"'Re":49427,".transfer":49428,"_FIFO":49429,"getModel":49430,"Ġbland":49431,"ĠColeman":49432,"Ġprimes":49433,"ĠæĪ":49434,"Ġcrosses":49435,"nk":49436,"GING":49437,"Ġ'^":49438,"ĠBlob":49439,"Ġintercourse":49440,"ĠBlvd":49441,"Ġweighs":49442,"_regular":49443,"ĠPerth":49444,"Ġseparating":49445,"Ġbilled":49446,".tabControl":49447,"Ġpuppet":49448,"Ġutilization":49449,"Ġâĸł":49450,"Ġsucces":49451,"Ġlamps":49452,"_proj":49453,"Eric":49454,"Ġrenovation":49455,"ĠFamilies":49456,"ĠBits":49457,"partials":49458,"-Men":49459,"solution":49460,"Ġdwarf":49461,".INTEGER":49462,"ĠLOCK":49463,".ct":49464,"Ġexcerpt":49465,"ĠPix":49466,"ĠFirstName":49467,"ANTED":49468,"ĠAdmir":49469,"-help":49470,"Prior":49471,"ĠAlign":49472,".INSTANCE":49473,"LineEdit":49474,"('/:":49475,"Ġinet":49476,"odus":49477,".pkl":49478,"ĠKY":49479,"upert":49480,"Ġnerves":49481,"_gradient":49482,"}','":49483,"_unref":49484,"Ġsaturated":49485,"ĠConnected":49486,"ĠFN":49487,"EXIT":49488,"Ġteleport":49489,"Ġavait":49490,"PageRoute":49491,"Ġdivorced":49492,"(lang":49493,"fst":49494,"ĠTyr":49495,"Ġmessenger":49496,"ifstream":49497,"XS":49498,"ĠBanking":49499,"Ġinfectious":49500,"ĠMons":49501,"_LOOP":49502,"Ġzurück":49503,"Ġobtener":49504,"/repos":49505,"Vel":49506,"acro":49507,"ĠuserRepository":49508,"styleType":49509,"ĠSRC":49510,"VMLINUX":49511,"recursive":49512,"/bar":49513,"_chip":49514,"ominated":49515,"ĠNit":49516,"âĢĶto":49517,"ĠBuddh":49518,"омеÑĢ":49519,"ĠMAG":49520,"ĠCHE":49521,"_den":49522,".raises":49523,"_degree":49524,"Ġpumpkin":49525,"_templates":49526,"_MEDIA":49527,"ĠTimeline":49528,"Ġbots":49529,"ObjectType":49530,"Ġbuys":49531,".posts":49532,"CAL":49533,"waiting":49534,"ĠDaniels":49535,"Ġdabei":49536,"ĠSigma":49537,"ilor":49538,"igel":49539,",W":49540,"ADS":49541,"(panel":49542,"ì²´":49543,"itating":49544,".palette":49545,"Ġmosquito":49546,"Ġtego":49547,"(parseInt":49548,"Ġdespués":49549,"promise":49550,"Ġwij":49551,"typescript":49552,"ĠTv":49553,"_IDENTIFIER":49554,").ĊĊĊ":49555,"_flat":49556,"itsu":49557,"USR":49558,"experience":49559,"-fit":49560,"phinx":49561,"_thresh":49562,"Ġideally":49563,"ĠFreeman":49564,",DB":49565,"_rw":49566,"çŃī":49567,"Ub":49568,"_statistics":49569,"=\"\"><":49570,"Ġchore":49571,"Ġyork":49572,"installed":49573,"Additionally":49574,"Ġpstmt":49575,"ylko":49576,"::Ċ":49577,"Forest":49578,"Ġheadset":49579,"Ġgallon":49580,"ÑĢем":49581,"Ġwithdrawn":49582,"ĠCandidate":49583,"Ġmelting":49584,"Ġfreezer":49585,"Ġhl":49586,"_HELP":49587,"mime":49588,"(/*":49589,"Ġthirst":49590,"$return":49591,"memberof":49592,"еб":49593,"ĠHttpServletRequest":49594,"(ob":49595,"_Result":49596,"Ġasserted":49597,"Ġfulfilling":49598,"Ġstretches":49599,"parated":49600,"-funded":49601,"ĠåĽ":49602,"ingles":49603,"_ca":49604,".condition":49605,"ĠDisplays":49606,"Ġorang":49607,"ĠCRE":49608,"ĠglBind":49609,"ĠSelector":49610,"/type":49611,"ĠAlexa":49612,"chedules":49613,"ĠPeninsula":49614,"Ġparity":49615,"ĉdest":49616,"ĠDoors":49617,"čĊĉčĊ":49618,"_dimension":49619,"Ġaload":49620,".StoredProcedure":49621,"(paren":49622,"ĠBurke":49623,"')]Ċ":49624,"-engine":49625,"Ġquir":49626,"ĠHybrid":49627,"ĠDoe":49628,"Ġoutlines":49629,"ĠTrends":49630,"_NV":49631,"periments":49632,"ĠHin":49633,"?',":49634,"ĉText":49635,"FUL":49636,"Ġsmells":49637,"Ġslick":49638,"Ġmiserable":49639,"ĠArrayAdapter":49640,"ĠparamString":49641,"Hom":49642,"_literals":49643,"usuarios":49644,"Ġprompting":49645,"_lazy":49646,"ĠActivation":49647,"_oc":49648,"Weak":49649,"Ġanecd":49650,"ĠUCLA":49651,"=re":49652,"issement":49653,"ĠEscorts":49654,"Excellent":49655,"ĠPause":49656,"Ġrepositories":49657,"TOR":49658,"ariate":49659,"_iso":49660,"updates":49661,"halb":49662,"udiante":49663,"ë¡Ŀ":49664,"Ġnaive":49665,"ĠPeg":49666,"ĠLounge":49667,"ARGIN":49668,"(bin":49669,"OnClickListener":49670,"ĠFAILED":49671,"Ġlite":49672,"Ġdzie":49673,"ĠLiteral":49674,"ivor":49675,"fcntl":49676,"Ġeats":49677,"Ġqed":49678,"Unlock":49679,"riding":49680,"undai":49681,"=M":49682,"ATTER":49683,"ConfigureAwait":49684,"icias":49685,"ustomed":49686,"Ġsuccession":49687,"endTime":49688,"ĠJupiter":49689,"Ġjudging":49690,"dration":49691,"_docs":49692,".mo":49693,"Ġeducators":49694,"ĠVine":49695,"Cond":49696,"[out":49697,"qb":49698,"\\Validator":49699,"Ġmeanings":49700,"Ġpresently":49701,"Ġdividing":49702,"ottenham":49703,"ascular":49704,"Ġtrailers":49705,"ĠCLOSE":49706,"ами":49707,"âĢĻai":49708,"ĠGain":49709,"wor":49710,"Ġplanner":49711,"Ġdistributing":49712,"vat":49713,"months":49714,"xlabel":49715,"HF":49716,"Viol":49717,".BASELINE":49718,"еÑĤÑģÑı":49719,"ĠRotate":49720,"Ġtxn":49721,":bold":49722,"Ġbloss":49723,"Forgery":49724,"(embed":49725,"Ġjako":49726,"sprintf":49727,"their":49728,"Ġexhibits":49729,"-static":49730,"hecy":49731,"getActiveSheet":49732,".clients":49733,"ãģį":49734,"_hide":49735,"[word":49736,"Cb":49737,"addItem":49738,"axe":49739,"_radio":49740,"alion":49741,"modifier":49742,"Ġsaturation":49743,"Ġdenom":49744,"_pixels":49745,"mess":49746,"(fl":49747,"atif":49748,"Ġsecs":49749,"Ġprostitution":49750,"Ġgrandchildren":49751,"Ġparadise":49752,"ĠFeld":49753,"_BINARY":49754,"itous":49755,"à¹Ħ":49756,"Ġflashing":49757,"-sided":49758,"Ġcontradiction":49759,"/*ĊĊ":49760,"ylabel":49761,"ĠTet":49762,"Ġadmire":49763,"reso":49764,"Ġletz":49765,"ĠSEARCH":49766,"slots":49767,"ĠRewards":49768,"ĠHog":49769,"ĠNSData":49770,"stash":49771,"Fall":49772,"ĠAmer":49773,"LinearLayout":49774,"/photos":49775,"Ġfeather":49776,"Ġ|čĊ":49777,"Downloads":49778,".StartsWith":49779,"Ġ//#":49780,"ineTransform":49781,"Ġaffid":49782,"Vtbl":49783,"ĠRogue":49784,"scribed":49785,"Ġfauc":49786,"ĠMonroe":49787,"Ġdeclares":49788,"modern":49789,"reon":49790,"aybe":49791,"PASS":49792,"fers":49793,"_MULTI":49794,"ĠMathematics":49795,"Ġsudah":49796,"_ATTACH":49797,"ĠnumberWith":49798,"ĠSolomon":49799,"jin":49800,"ografia":49801,"öl":49802,"_design":49803,"culated":49804,"ĠLuna":49805,"iesz":49806,"Ġ=>'":49807,"Ġrevelations":49808,"Along":49809,"(ed":49810,"ĠFilename":49811,"Ġylabel":49812,"Secure":49813,"Ġbusca":49814,"agnosis":49815,"_RECE":49816,"Ġoverlapping":49817,"Extent":49818,"Ġanticipation":49819,"Checks":49820,"ĠALSO":49821,"orc":49822,"ilingual":49823,"itational":49824,"Ġadvancement":49825,"ouro":49826,"ĠPredicate":49827,"å¾Ĺ":49828,"eria":49829,"ĠPierce":49830,"orio":49831,"Ġmerits":49832,"Ġpeanut":49833,".Package":49834,"ĠConduct":49835,"_SENSOR":49836,"Ġboiling":49837,"Ġintra":49838,"ĠIGN":49839,"ĠFur":49840,".Refresh":49841,"ĠReach":49842,"_decoder":49843,".Exp":49844,"ĠÑĤак":49845,"pill":49846,",Q":49847,"ĠGrill":49848,"Ġpopping":49849,".Ag":49850,"Ġproyecto":49851,"Ġmileage":49852,"Ġecological":49853,"]]);Ċ":49854,"ĠÂŃ":49855,"subplot":49856,"acad":49857,"ĠTrying":49858,"recipes":49859,"$criteria":49860,"ĠPersian":49861,"-bound":49862,"MASK":49863,"ĠGesture":49864,"Ġkk":49865,"ĠPVC":49866,"Ġprohibition":49867,"Ġcomando":49868,"ĠLOOK":49869,"Shopping":49870,"Ġdistortion":49871,"čĊ":49917,".Dependency":49918,".QueryString":49919,".Owner":49920,"Ġexpiry":49921,"Thu":49922,"(Vec":49923,"Ġhazardous":49924,"Ġrpm":49925,"APON":49926,"ĠaddTarget":49927,"sville":49928,"pNet":49929,"ĠImg":49930,"ĠTIMER":49931,".Animation":49932,"Ġbek":49933,"Ġassort":49934,"Ġlebih":49935,"ĠbodyParser":49936,"Ġvibrating":49937,"IDL":49938,"Ġbutterknife":49939,"inters":49940,"Ġpersuade":49941,"ĠLGBTQ":49942,"èĭ":49943,".soft":49944,"Ġbeams":49945,"_sur":49946,".Def":49947,"Ġlabs":49948,"ĉplt":49949,"Ġskins":49950,"Ġtransferring":49951,"Ġimaginary":49952,"_End":49953,";background":49954,"Ġlaps":49955,"_COMMENT":49956,"(SDL":49957,"onds":49958,".Record":49959,"ĠImplements":49960,"_ticks":49961,"()))ĊĊ":49962,"Ġarose":49963,"]?":49964,"ĠMp":49965,"ĠICommand":49966,"Ġsculpture":49967,"Ġcontracted":49968,"\">'":50446,"kinson":50447,"Ġкол":50448,"ognitive":50449,"_li":50450,"Ġimminent":50451,"Ġaffinity":50452,".signal":50453,"Ġnotch":50454,"ĠSteelers":50455,"maxlength":50456,"KK":50457,"ĠEugene":50458,"_PWM":50459,"roi":50460,"ĠâĹı":50461,"ĠHamburg":50462,".Must":50463,"Ġaxe":50464,"enef":50465,"Ġambitions":50466,"ĠSpecies":50467,"ĠStress":50468,"Ġawhile":50469,"ĠбÑĥд":50470,"Ġwithstand":50471,"ĠDecoder":50472,"_inventory":50473,"Ġ{ččĊ":50474,"Ġtgt":50475,"Ġrailroad":50476,"WASHINGTON":50477,"Ġnegotiated":50478,"NST":50479,"-phone":50480,",U":50481,"Ġexercising":50482,"ụ":50483,"_PIXEL":50484,"avors":50485,"iterated":50486,"Ġvampire":50487,"adal":50488,"Ingrese":50489,"Ġung":50490,"jective":50491,".cells":50492,"Ġnano":50493,"Ġmarkdown":50494,"_RULE":50495,"(events":50496,"Ġluggage":50497,"MESSAGE":50498,"igkeit":50499,"$count":50500,"AttributeName":50501,"IGINAL":50502,"_Ent":50503,"ĠBF":50504,"ĠCOMMENT":50505,"_ini":50506,"ĠEuropeans":50507,"ĠBelle":50508,"åij½":50509,")['":50510,"åºĶ":50511,"ĠUseful":50512,".reference":50513,"()\",":50514,"_grade":50515,"ĠKaw":50516,"Ġsentencing":50517,"Ġsocialism":50518,"monster":50519,"_LAYER":50520,"Ġdeepest":50521,"wk":50522,"ĠNoise":50523,"###ĊĊ":50524,"Ġpréc":50525,"otle":50526,"ÑĤе":50527,"auf":50528,"ibal":50529,"Ġconquer":50530,">Email":50531,"Ġambulance":50532,"OAD":50533,"Ġ(\"%":50534,"ĠFI":50535,".fixture":50536,"Ġterse":50537,"ĠĠĠĠĉĉĉĉ":50538,"Ġsanctuary":50539,"ugi":50540,"ĠComparator":50541,"Definitions":50542,"Ġasthma":50543,"Ġlact":50544,"Ġhardwood":50545,".clock":50546,"Ġattracting":50547,"ĠMour":50548,"(distance":50549,"icits":50550,"Ġbonne":50551,"ĠACCESS":50552,".DeserializeObject":50553,"ĠTyped":50554,"Ġjeu":50555,"ĠappId":50556,"ĠClara":50557,"ĠHF":50558,"ĠReich":50559,"ipples":50560,"//--------------------------------------------------------------------------------":50561,"_delivery":50562,"erialization":50563,"Ġplaintiffs":50564,"Scient":50565,"shopping":50566,"ĠDummy":50567,"ĠWald":50568,"GroupName":50569,"Ġinscription":50570,"elog":50571,"::::::::":50572,"_ld":50573,"BackPressed":50574,".Raw":50575,"ĠOnTrigger":50576,"Ġmuseums":50577,"ĠBeen":50578,"ĠAdventures":50579,"Ġslate":50580,"Ġlett":50581,"Ġsund":50582,"ĠGin":50583,"ĠMechanical":50584,".ship":50585,"AppComponent":50586,"Ġdestined":50587,"Ġdwelling":50588,"Profiler":50589,"Prepare":50590,"zeich":50591,"Ġsilicon":50592,"(has":50593,"Ġ#%":50594,"VIDEO":50595,"Ġcollaborate":50596,"Lin":50597,"Ġscopes":50598,"(className":50599,"(sd":50600,"andin":50601,".ham":50602,"ServiceImpl":50603,"-described":50604,"Ġirony":50605,"stial":50606,"ĠHuawei":50607,"(repo":50608,"Ġunexpectedly":50609,"ĠKai":50610,".install":50611,"\\xf":50612,"Ġexhibited":50613,"_TCP":50614,"ĠOx":50615,"_CHO":50616,"Ġprostituerte":50617,"Ġvä":50618,"Ġsito":50619,"Ġconstituents":50620,"ĠContinued":50621,"ĠSAVE":50622,"rss":50623,"/message":50624,"ubes":50625,"Ġmisdemean":50626,"Ġtaxation":50627,"Ġstoryline":50628,"hair":50629,"ĠFinds":50630,"SIG":50631,"verification":50632,"~=":50633,".hp":50634,"Iterable":50635,"Ñĭе":50636,"atori":50637,"Ġctr":50638,"Rx":50639,"_);ĊĊ":50640,"dag":50641,".pin":50642,"Ġpseud":50643,"Ġinvo":50644,"ÑģÑĤÑĢ":50645,"_pix":50646,"为空":50647,"Ġsworn":50648,"âĢĶor":50649,"_registry":50650,"Ġdisasters":50651,"ĠROI":50652,"ĠâĢķ":50653,"aktu":50654,"forest":50655,"beiten":50656,"âĢĶI":50657,"ueva":50658,"egt":50659,"Ġspikes":50660,"URES":50661,"ĠRecommended":50662,"Ġexploited":50663,"ĠFrederick":50664,"_COMPLETE":50665,"ĠDrugs":50666,"!!!!!!!!":50667,"ĠRiv":50668,"STOP":50669,"ROOM":50670,"ĠPASSWORD":50671,"Cookies":50672,".El":50673,"á»Ń":50674,"ĠBert":50675,"Ġhashed":50676,"icester":50677,"Ġdecorator":50678,"ĠqueryString":50679,":;Ċ":50680,"Ġ\"[\"":50681,"otope":50682,"-Americ":50683,"ĠMatthews":50684,"URAL":50685,"âĢľ,":50686,"Summer":50687,"fos":50688,"_CONTAINER":50689,"_ACK":50690,"Ġfiltr":50691,"_disp":50692,"_Re":50693,"Ġfacile":50694,"аÑĪ":50695,"ĠìķĬ":50696,"Ġeben":50697,"Ġsprink":50698,"ĠQuint":50699,">V":50700,"Ġhistorians":50701,"ourmet":50702,"ĠMonitoring":50703,"ledger":50704,"cott":50705,"Ġware":50706,"GGLE":50707,"cars":50708,"ĠMEDIATEK":50709,"Ġvolupt":50710,"_View":50711,"HEL":50712,"(copy":50713,"(stats":50714,"Ġchromosome":50715,"ĠCurtis":50716,"-conf":50717,"(asset":50718,"Ġhvor":50719,"FileSystem":50720,"<>();čĊ":50721,"ocoder":50722,"ĠCannon":50723,")x":50724,"ĠSmooth":50725,"ĠSAS":50726,"_ce":50727,"ĉprev":50728,"_movie":50729,"Ec":50730,"_wall":50731,".ĊĊ":51278,"ogenesis":51279,"ĠOPTIONS":51280,"uptools":51281,"Ġmilitant":51282,"Ġexited":51283,"igar":51284,"ĠCOMM":51285,"ĠDisposable":51286,"aycast":51287,"Ġrowspan":51288,"Ġsynthes":51289,"Ġsondern":51290,"ĠĊ":54769,"ĠJacket":54770,"RATION":54771,".getSelectedItem":54772,"-init":54773,"ĠRegisters":54774,"_sep":54775,"ĠToolkit":54776,".dict":54777,"Ġxlabel":54778,"\\Table":54779,"toc":54780,"_combo":54781,"ĠCompact":54782,"Ġrugged":54783,"à¥ĩà¤":54784,"-management":54785,"')}}\">Ċ":54786,"ĠStamp":54787,"ıl":54788,"rox":54789,"Ġlandscapes":54790,"_NOTE":54791,"monary":54792,"cab":54793,"Ġmoet":54794,"xaf":54795,"rcode":54796,"-cli":54797,"_gate":54798,"[event":54799,"SPORT":54800,"gia":54801,"ĠSUPER":54802,"/Login":54803,"_shutdown":54804,"interrupt":54805,"Ġpretending":54806,"Ġfringe":54807,"ĠReds":54808,"ĠCUDA":54809,"ĠUNIX":54810,"vit":54811,"Ġbrig":54812,"drv":54813,"ĠConnector":54814,"Therefore":54815,"Ġlia":54816,"Detection":54817,"_actor":54818,"Ġtempfile":54819,"Ġeccentric":54820,"-role":54821,"Ġpadx":54822,"dent":54823,"Western":54824,"Ġê·¸":54825,"ĠApplicationRecord":54826,"Ġcampaigning":54827,"_runner":54828,"ĠCivic":54829,"aleigh":54830,"Ġdirekt":54831,".sul":54832,"ĠĠĉĉĉ":54833,"anten":54834,"Ġissuer":54835,"Ġassertions":54836,"(orig":54837,"ATIO":54838,"Ġleaned":54839,"äs":54840,".DTO":54841,"explode":54842,".Observable":54843,"Ġstaggering":54844,"Ġkidnapped":54845,"Ġprogrammers":54846,"ĠInnov":54847,".parameter":54848,"Ġdomination":54849,"Ġskeptic":54850,"Ġæĺ¯":54851,"Ġavoids":54852,".Verify":54853,"ubby":54854,"ĠASN":54855,"Ġformato":54856,"ĠBeatles":54857,"_brand":54858,"Ġinset":54859,"youtu":54860,"Ġtoc":54861,"-final":54862,"Showing":54863,"ĠDoub":54864,"ĠMesa":54865,"Adj":54866,"_medium":54867,"Creates":54868,"(endpoint":54869,"ĉUP":54870,"bbie":54871,"Ġstalk":54872,".databind":54873,".Scan":54874,"agents":54875,"$,":54876,"individual":54877,"+)/":54878,"ĉvm":54879,"(notification":54880,"Ġinex":54881,"ĠClassification":54882,"reno":54883,"Ġolig":54884,"-rated":54885,"Ġformulation":54886,"',{":54887,"Ġacept":54888,"_unpack":54889,"_CA":54890,".Pow":54891,"ĉim":54892,"Ġaluminium":54893,"ANO":54894,"Ġxn":54895,"Ġcómo":54896,"ĠIngredient":54897,"Ġseizures":54898,"åħ±":54899,"ificador":54900,"Ġsiguiente":54901,"ĠInfragistics":54902,"Ġduplicated":54903,"ĠDee":54904,"Ġnø":54905,"ĠACCEPT":54906,"(crate":54907,"иÑĤелÑĮ":54908,"-less":54909,"Ġinfinity":54910,"Analyzer":54911,"-Day":54912,"ritt":54913,"(cin":54914,"ĠGy":54915,"Ġmultiplied":54916,"uchi":54917,"ĠBaldwin":54918,"/ip":54919,"Ġshortcuts":54920,".ADD":54921,"Ġvigor":54922,"_instruction":54923,"(;":54924,"_eta":54925,"è¿ŀ":54926,"utorials":54927,"Ġboosting":54928,"bv":54929,"Ġacknowledges":54930,"Listening":54931,"FAQ":54932,";b":54933,"((-":54934,"Ġarchitects":54935,"Ġzwe":54936,"Ġpuls":54937,"ĠgetCount":54938,"verbs":54939,"ãĢľ":54940,"(Collection":54941,"kre":54942,"Ġjurisdictions":54943,"_bridge":54944,"ĠCrack":54945,"ĠDifficulty":54946,"KO":54947,"Reservation":54948,"_requires":54949,"Tour":54950,"ãģĹãģŁ":54951,".setCurrent":54952,"Ġky":54953,"ĠAlbany":54954,"Ġè§":54955,"ller":54956,"agna":54957,"workers":54958,".blank":54959,"ĠPrayer":54960,"MIC":54961,"Ġresilience":54962,"TeX":54963,"ĠLanguages":54964,"study":54965,"ĉcurr":54966,"Ġenzymes":54967,"Slug":54968,"ĠíĮĮ":54969,"stral":54970,"Ġtumors":54971,"Ġsegunda":54972,"='{":54973,"instruction":54974,"ĠLisp":54975,"/info":54976,"Ġ\"{$":54977,",:),":54978,"Ġgv":54979,"(ErrorMessage":54980,"Ġ'=":54981,"}-${":54982,".Documents":54983,"\"Well":54984,"Ġreminiscent":54985,"Ġgaz":54986,"iropr":54987,"ehr":54988,"Ġsuppressed":54989,"ersh":54990,".scrollTo":54991,"Ġcadena":54992,"ĠgameState":54993,"ÃŃm":54994,"(conv":54995,"ĠTomorrow":54996,"ĠCCT":54997,"Mongo":54998,"ulg":54999,".Camera":55000,".handlers":55001,"mph":55002,"Ġstk":55003,"Ġgenetics":55004,"ACING":55005,"Trivia":55006,"ĠBam":55007,"(marker":55008,".Stretch":55009,"ĠSunni":55010,"ĠBetty":55011,".tolist":55012,"unlikely":55013,".Rectangle":55014,"obsolete":55015,"ILON":55016,"innerText":55017,"embourg":55018,"aN":55019,"ĠVehicles":55020,"unlock":55021,":utf":55022,"nob":55023,"ĠSeeing":55024,"ĠNEVER":55025,"Ġtls":55026,"Ġfilles":55027,"Ġbenefited":55028,"ĠClint":55029,"*/),":55030,".fold":55031,"Ġposible":55032,"ADED":55033,"thouse":55034,".DAL":55035,"ĠOdd":55036,"rokes":55037,"ĠSunny":55038,"ĠPartialEq":55039,"_Buffer":55040,"ĠLevi":55041,"longrightarrow":55042,"eldon":55043,"gages":55044,"_warn":55045,".CreateTable":55046,"ĠDip":55047,"_questions":55048,".logic":55049,"Ġ#\"":55050,"={()=>":55051,"Ġtep":55052,"Ġjuicy":55053,"ìĤ¬":55054,"enko":55055,"ialect":55056,"Ùī":55057,"Ġonboard":55058,"Ġæı":55059,"ĉrt":55060,"_UTF":55061,"ĠQAction":55062,"âĢŀ":55063,"(Component":55064,"(audio":55065,".hit":55066,"gte":55067,"Ġprogrammed":55068,"stateParams":55069,"Ġpolyester":55070,"fires":55071,"byss":55072,"]=(":55073,"_quality":55074,"OfDay":55075,"ĠFairy":55076,"Ġyelled":55077,"opl":55078,"(userName":55079,"ĠDifference":55080,"Ġevaluations":55081,"iffany":55082,"Ġcyclists":55083,"Ġcidade":55084,"Ġtextbook":55085,"Ġprofiling":55086,"__),":55087,"dea":55088,".activate":55089,"Ġindications":55090,"Ðķ":55091,"TouchUpInside":55092,"Ġinvaluable":55093,"ĠMASK":55094,"Ġcontend":55095,"Freq":55096,"Ġrecruits":55097,"(interval":55098,"ĠUserProfile":55099,"Ġ'./../":55100,"edu":55101,"_Callback":55102,"Ġanalogy":55103,"ĠTrophy":55104,"apphire":55105,"Videos":55106,"ĠCher":55107,"ĠHav":55108,"âĢ¦\"":55109,".validator":55110,"gfx":55111,"ĠUObject":55112,"classnames":55113,"triangle":55114,"ĠEncoder":55115,".spy":55116,"Ġpredators":55117,"=status":55118,"-safe":55119,":\",Ċ":55120,"ĠIncluding":55121,"Ġ{};čĊ":55122,"*cos":55123,"Ġendured":55124,".sulake":55125,"Ġnursery":55126,"Ġfragrance":55127,"Ġrebuilding":55128,"Ġnth":55129,"ĠFraser":55130,".setDate":55131,"ĠVince":55132,"_REST":55133,"Ġventilation":55134,"æµ·":55135,"cribes":55136,".asm":55137,"lpVtbl":55138,"ĠAbe":55139,"uisine":55140,",array":55141,"ĉclassName":55142,"errals":55143,"Ġ'ĊĊ":55144,"Checkout":55145,"Ġsolicit":55146,"Aux":55147,"_capture":55148,"Ġribs":55149,"ragon":55150,"viol":55151,"topics":55152,"FunctionFlags":55153,"ĠMarty":55154,"bike":55155,"ĠTucker":55156,"(kernel":55157,"ĠOps":55158,"CloseOperation":55159,"/demo":55160,"ilda":55161,"ĠlÃŃnea":55162,"APPING":55163,"Ġsuites":55164,".visitVarInsn":55165,"urus":55166,"ĠMinute":55167,"(manager":55168,"Ġbutterfly":55169,"Ġapare":55170,"Ġwolves":55171,"JWT":55172,"ĠSalon":55173,"ĉdelay":55174,"-eslint":55175,"isations":55176,".rpc":55177,")|(":55178,"ĠSnapchat":55179,"/mm":55180,"MN":55181,"ceries":55182,".textAlignment":55183,"ĠFrankfurt":55184,"Ġado":55185,"(newValue":55186,"(access":55187,"(Expression":55188,"ĠSignIn":55189,"ĠHaiti":55190,"_tp":55191,".setParameter":55192,"Minute":55193,"Ġmanuals":55194,"ricanes":55195,"ĠPTR":55196,"ĠOuter":55197,"Ġgetline":55198,"ocations":55199,"_CD":55200,"ĠLyon":55201,"/gui":55202,"_live":55203,"idan":55204,".geom":55205,"ĠborderBottom":55206,"imuth":55207,"_checkpoint":55208,"Ġmeu":55209,"ĠIrving":55210,"Ġpeuvent":55211,"(MAX":55212,"ĠARCH":55213,"Ġpov":55214,".sourceforge":55215,"Ġjamais":55216,"Ġark":55217,"ĠBaghdad":55218,"ĠCLEAR":55219,"MenuBar":55220,"Ġtrois":55221,"CHEDULE":55222,"Ġ#čĊ":55223,"(Call":55224,"$order":55225,"(Material":55226,"Ġencontrado":55227,"$list":55228,"ĠMETHODS":55229,".beginTransaction":55230,"_MAG":55231,"StyleSheet":55232,"Ġmajors":55233,"Ġindefinitely":55234,"cleanup":55235,"Ġhomeland":55236,"(dto":55237,"Dates":55238,"Presentation":55239,"ĠDK":55240,"={`/":55241,"ĉKey":55242,"(Block":55243,"_checkbox":55244,"needs":55245,"ĠonComplete":55246,"rico":55247,"Ġgleich":55248,"Ġxm":55249,"OOD":55250,"Better":55251,"ĠSQLITE":55252,".Book":55253,"xad":55254,"ĠGone":55255,"ĉdp":55256,"Ġdevotion":55257,"Ġstm":55258,"Ġobsess":55259,"ĠBackend":55260,"Queries":55261,"Ik":55262,"//****************************************************************":55263,"Ġdividends":55264,".parentElement":55265,"}\")ĊĊ":55266,"ĠMaterialPageRoute":55267,":num":55268,"Ġexplic":55269,"ĠOL":55270,"least":55271,"Oops":55272,"imentos":55273,"Ġinsurers":55274,"Ġheroic":55275,"ĉfields":55276,".imgur":55277,".btnCancel":55278,"ĠDetective":55279,"(sm":55280,"ĠMutableLiveData":55281,".lab":55282,"(([":55283,"Ġhairst":55284,"ĠTransactions":55285,"å¼Ģå§ĭ":55286,"ĠstdClass":55287,"uento":55288,"GIS":55289,"_cod":55290,"Instructions":55291,"Calls":55292,"PointerType":55293,"ĠRw":55294,"Ġassortment":55295,"ĠDIG":55296,"+r":55297,"_CERT":55298,"Ġinstability":55299,"Ġvib":55300,"onas":55301,"Ġroku":55302,"apellido":55303,"Ġangl":55304,"preneur":55305,"Ġfluids":55306,"isease":55307,"Ġdeed":55308,"quist":55309,"_CONSTANT":55310,"Ġequilibrium":55311,"_delegate":55312,"ĠQuantum":55313,"rei":55314,"Capabilities":55315,"rectangle":55316,"?><":55317,"alien":55318,"ĠJug":55319,"DNA":55320,"Tickets":55321,"Occurs":55322,"ĠHawk":55323,".setHorizontalGroup":55324,"\\Collection":55325,"ffiti":55326,"Ġrearr":55327,".setVerticalGroup":55328,"Ġcavity":55329,"Ġadulte":55330,"Facade":55331,"-wh":55332,"ĠLOL":55333,"Ø°":55334,"Ġgrandparents":55335,"Swift":55336,"ĉwx":55337,"æīĢæľī":55338,"ifen":55339,"ffset":55340,"Beyond":55341,"//}ĊĊ":55342,"Ġwager":55343,"Ġbury":55344,"Ġcommence":55345,"registro":55346,"scient":55347,"ĠPercent":55348,"Ġдолж":55349,"(identifier":55350,".setModel":55351,"Ġseldom":55352,"nton":55353,"Ġappliance":55354,"amus":55355,"rysler":55356,"Ġpanties":55357,"enguins":55358,"Ġmimic":55359,"ĠonChanged":55360,"Ġalcoholic":55361,".reloadData":55362,"Charge":55363,"ĠFax":55364,"ĠjScrollPane":55365,"Empresa":55366,"Ġshattered":55367,"xba":55368,"Fonts":55369,"?s":55370,"Ġpostseason":55371,"retain":55372,"_rates":55373,"ĠrequestCode":55374,".todo":55375,"´s":55376,"CHK":55377,"ĠKeeping":55378,"engeance":55379,"Ġvscode":55380,"IPPING":55381,"DefaultCloseOperation":55382,"_raise":55383,"ĠOculus":55384,"ograms":55385,"raj":55386,"pci":55387,"Ġcorrosion":55388,".handleSubmit":55389,"Accessible":55390,"ĠPiano":55391,"little":55392,"ACL":55393,"Äĩe":55394,".unwrap":55395,"ĠConvers":55396,"ĠLeben":55397,"ioneer":55398,"ĠMerchant":55399,"ĠJorge":55400,"Ġembracing":55401,"Ġventa":55402,"ást":55403,"Ġviene":55404,"Ċ":55556,"-growing":55557,"Ġdeepcopy":55558,"Ack":55559,"eggies":55560,"Ġ__(\"":55561,"Ġnoir":55562,"terrorism":55563,"Ġanthem":55564,"agency":55565,"_PACKAGE":55566,"ĠClosure":55567,".registry":55568,"Ġmammals":55569,"L":55600,"Ġbluetooth":55601,".Deep":55602,"-standing":55603,"ácil":55604,"Ġrooft":55605,"ĠPaths":55606,"_iterations":55607,"InvalidArgumentException":55608,".spi":55609,"ĠUIAlertAction":55610,"uye":55611,"signin":55612,".priority":55613,"ĠEssays":55614,"='{$":55615,"Ġè¿ĶåĽŀ":55616,"_signed":55617,".persist":55618,"Ġredesign":55619,"ToLower":55620,"ĠNewman":55621,"=start":55622,"ĠIsraelis":55623,"asiswa":55624,"Speech":55625,"Ġnumeros":55626,"handlers":55627,"ĠWong":55628,"ĠмеÑĤод":55629,"Weights":55630,"ĠGujar":55631,"teil":55632,"ĠNonetheless":55633,"_EFFECT":55634,"Ġvect":55635,"ĠOsc":55636,"Ġcoats":55637,"ĠWheat":55638,"Ġgeek":55639,"ĠPROPERTY":55640,"worm":55641,"_constants":55642,"ĠBoulder":55643,"ĠParm":55644,"cole":55645,"ĠdefaultCenter":55646,"ĠRouge":55647,":A":55648,"xcf":55649,"ĠVenice":55650,"median":55651,"Ġredemption":55652,"Fresh":55653,"Ġcosm":55654,"Ġfigur":55655,"Ġrefurb":55656,"COPE":55657,".cd":55658,"Ġchords":55659,"ĠSgt":55660,"Åį":55661,"VPN":55662,"ĠSEND":55663,"ainen":55664,"_accounts":55665,"Ġtenth":55666,"Ġdissolved":55667,"":55907,"Ġlegitimacy":55908,"Ġoo":55909,"Slinky":55910,"Ġnationals":55911,".words":55912,";p":55913,"trap":55914,"omanip":55915,"Ġcues":55916,"Ġgraduating":55917,"Ġsemaphore":55918,"\"]);ĊĊ":55919,"acey":55920,"REET":55921,"Grab":55922,"ĠFelix":55923,"(Id":55924,"_neighbors":55925,"Ġmeaningless":55926,"(del":55927,"Ġjeder":55928,"ĠContentValues":55929,".absolute":55930,"/cl":55931,"Ġxb":55932,"datum":55933,"Ġtortured":55934,"Ġrubbing":55935,"Scores":55936,"ĠðŁĺī":55937,"Ġavons":55938,"Ġamsterdam":55939,"EOS":55940,"Hal":55941,"Ġtrustworthy":55942,"#=":55943,".EXTRA":55944,"Ġmano":55945,"isicing":55946,"-support":55947,"ĉcursor":55948,"ĠSpo":55949,"aimassage":55950,"Mission":55951,"[]{\"":55952,"Ġprinters":55953,"GREEN":55954,"Ġteg":55955,"Ġabdominal":55956,"!ĊĊĊĊĊĊ":55957,".Short":55958,"азв":55959,"ĠGifts":55960,"}\")":55961,"(binding":55962,"xce":55963,"âĢij":55964,"infos":55965,"FormData":55966,"Ġdart":55967,"Ġelems":55968,"(inv":55969,"YL":55970,"tin":55971,"GENER":55972,"ữ":55973,"ĠTaken":55974,"uckle":55975,":e":55976,"Ġspectral":55977,".baidu":55978,"/');Ċ":55979,"Ġgreedy":55980,"esion":55981,",,,,,,,,":55982,"Ġ/>,Ċ":55983,"InternalServerError":55984,"NSNotificationCenter":55985,"ĠAi":55986,"Ġspit":55987,"Ġaugmented":55988,"ĠstandardUserDefaults":55989,"FINITY":55990,"Race":55991,":C":55992,"ĠRECORD":55993,"ĠHighlight":55994,"Ġ'`":55995,"Ġdeficits":55996,"Ġnei":55997,"Ġresearched":55998,"Ta":55999,"Ġcopp":56000,".GetHashCode":56001,"):čĊčĊ":56002,"OnClick":56003,"ĠWellington":56004,"Ġrevival":56005,"æ¯Ķ":56006,"éĹ®":56007,"ĠNSS":56008,"Ġforn":56009,"Ġinté":56010,"ĠKuwait":56011,"_flip":56012,"_bo":56013,"_\\":56014,"Ġoccurrences":56015,"ĠScientists":56016,"SRC":56017,"ogens":56018,"igrant":56019,"REMOTE":56020,"ĠSID":56021,".opts":56022,"uve":56023,"()])Ċ":56024,"Ġlibertarian":56025,"ĠGlide":56026,"lesen":56027,"Ġforme":56028,"owania":56029,"Ġannoyed":56030,"Defs":56031,"ĠExecutor":56032,"Ġcasts":56033,".setChecked":56034,"ĠSharing":56035,".SerializeObject":56036,"Ġselectors":56037,"_OTHER":56038,"미":56039,"(super":56040,"(OS":56041,"_VERIFY":56042,"idunt":56043,"';Ċ":56045,"Ġvidéo":56046,"ĠNegro":56047,"ĠLords":56048,"ĠTours":56049,"Ġsoftly":56050,".receive":56051,"ĠERC":56052,"ĠdataSet":56053,"Badge":56054,"ĉEvent":56055,"Ġperl":56056,"Ġ{}\\":56057,"(sentence":56058,"OrUpdate":56059,"Ġdiminish":56060,"PIN":56061,"(draw":56062,".ToDateTime":56063,".EqualTo":56064,"(pin":56065,"-pencil":56066,"luent":56067,"ĠCaller":56068,"Ġplayful":56069,"-'+":56070,"xca":56071,"swick":56072,"){}Ċ":56073,"}:${":56074,"ĠMeth":56075,".getCell":56076,".break":56077,"Ġymax":56078,"='Ċ":56291,"ĠHiro":56292,"(TRUE":56293,"asurer":56294,"Ġcuer":56295,"Uber":56296,".Operation":56297,"Ġolan":56298,"Ġthrilling":56299,"'.":56321,"ĉvalid":56322,"\"\",":56323,"Instrument":56324,">J":56325,"Ġnostr":56326,"ĠRift":56327,"_Port":56328,"Ġveces":56329,"[['":56330,"Ġrallies":56331,"-series":56332,"Ġvv":56333,".uc":56334,"Ġrtn":56335,"StateChanged":56336,"(ins":56337,"ĠCla":56338,"------------Ċ":56339,"cus":56340,"ĠReload":56341,"//------------------------------------------------------------------------------------------------":56342,".seconds":56343,"_destination":56344,"Ġscrewed":56345,">c":56346,"Thickness":56347,"Designer":56348,"Ġgrids":56349,"nÄħ":56350,"(cookie":56351,"Trip":56352,"-Mobile":56353,"Ġvoll":56354,"Ġgenital":56355,"Ġconfisc":56356,"ĠConfederate":56357,"ĠwebView":56358,"Ġmise":56359,"Ġcler":56360,"(selection":56361,"$date":56362,"Ġsharpen":56363,"ragen":56364,"AndUpdate":56365,"Ġremix":56366,"Ġhtons":56367,"RW":56368,"MPI":56369,"Ġretrieval":56370,"Ġrichest":56371,".Decode":56372,":initComponents":56373,"ĠTValue":56374,"Saint":56375,"@include":56376,"ĠPERSON":56377,".sep":56378,"ĠLDAP":56379,"gba":56380,"ĠgroÃŁe":56381,"Ġreliably":56382,"ĠDFS":56383,".getItemId":56384,"Ġprésent":56385,".getToken":56386,"Ġchinese":56387,"ĠMeal":56388,"YOU":56389,"\">>ĊĊ":56948,"bower":56949,"Ġswapped":56950,"/install":56951,"Ġsinks":56952,"etrize":56953,"Ġdeclines":56954,"ĉmysql":56955,"ĠCString":56956,"ĠMotionEvent":56957,".Language":56958,"Road":56959,"ÑĤеÑĢ":56960,"ascimento":56961,"'))->":56962,".about":56963,"(editor":56964,"ĠRatings":56965,"income":56966,"Å¡e":56967,".dequeueReusableCell":56968,"ĠAustrian":56969,"Ġsulla":56970,"ĠTribunal":56971,"ĠDidn":56972,"оваÑĢ":56973,"Ġinspections":56974,"Boss":56975,"Ġcocktails":56976,"Ġapologized":56977,"_subplot":56978,"opal":56979,"+=(":56980,"Ġresonance":56981,"ibu":56982,"Ġ리":56983,"roma":56984,"reserve":56985,"pls":56986,"ĠTah":56987,"axies":56988,"OPLE":56989,"ĠDarren":56990,"ĠZombie":56991,"_Map":56992,"Ġ])ĊĊ":56993,"ĠQi":56994,"ĠSail":56995,"Ġrestrictive":56996,"Ġerosion":56997,"-par":56998,"WHITE":56999,"Ġoldu":57000,"Ġaperture":57001,"Ġbitcoins":57002,"texto":57003,"ĠComcast":57004,"Ġtimeless":57005,"enkins":57006,"Ġfeeder":57007,"/tmp":57008,"resden":57009,"+'_":57010,".Destroy":57011,"Ġçok":57012,"ĠDOCUMENT":57013,".lng":57014,".tagName":57015,"Ġkullan":57016,"egrate":57017,"Ġ(*.":57018,"ç¼ĸè¾ij":57019,"Ġhandshake":57020,"soc":57021,"_geometry":57022,"ĠDamascus":57023,"Minor":57024,"ĠKafka":57025,"ìŬ":57026,"Florida":57027,"_compute":57028,".expr":57029,"Ġparalle":57030,"ĠDiaz":57031,"cir":57032,"[target":57033,"Ġjoking":57034,"Ġglor":57035,"(setq":57036,"_handlers":57037,"Hang":57038,"Ġferr":57039,"riminal":57040,"ĉĠĠĠĠĉĉ":57041,"enties":57042,"defines":57043,"-tax":57044,"jsonp":57045,"ĠUPS":57046,"metro":57047,"__;Ċ":57048,"ĠUganda":57049,"])):Ċ":57050,"_td":57051,"xae":57052,"lw":57053,".OS":57054,"ĠLogged":57055,"acid":57056,"ĠMayo":57057,"aspect":57058,"Ġvaginal":57059,"Ġinitializing":57060,"Ġsteroids":57061,"fiction":57062,"GRE":57063,"gend":57064,"Ġliabilities":57065,"ĠLets":57066,"Mech":57067,"(nc":57068,"(change":57069,"Ġconnectors":57070,":k":57071,"Ġtast":57072,"!\");ĊĊ":57073,"things":57074,"rophy":57075,"luetooth":57076,"ĠSignUp":57077,".ctrl":57078,"Ġtherein":57079,"orda":57080,".escape":57081,"igator":57082,"Ġpetrol":57083,"Ġspecimen":57084,"Ġdebuted":57085,"-Pro":57086,"Ġcrises":57087,".addView":57088,"ëıĻ":57089,"-door":57090,"Ġmonet":57091,"Ġmillis":57092,"Ġvier":57093,"InternalEnumerator":57094,"Ġadmins":57095,"ĠLair":57096,"zin":57097,"getQuery":57098,"umbles":57099,"LIMIT":57100,"ĠVig":57101,"_song":57102,"":57415,"Ġpasado":57416,"thank":57417,"_Delete":57418,"ĠBrighton":57419,",unsigned":57420,"ä½ľèĢħ":57421,"Ġaspirations":57422,"-how":57423,"Rose":57424,"=((":57425,"_needed":57426,"_plural":57427,">ĊĊ":57545,"Ġsurfaced":57546,"ĠìłĢìŀ¥":57547,"platz":57548,"ĉemail":57549,"ceptors":57550,"\">(":57551,"Ġepile":57552,"读":57553,"ĠDebt":57554,"åijĬ":57555,"NOP":57556,"\"https":57557,":j":57558,"FormItem":57559,"_LICENSE":57560,".getDouble":57561,"ĠAgenda":57562,"ĉfinally":57563,"(filters":57564,"(av":57565,"ç¾İ":57566,"APER":57567,"Ġlava":57568,"еÑĢж":57569,"))))ĊĊ":57570,"Ġfaulty":57571,"_nm":57572,"Ġtrava":57573,"(Bitmap":57574,"Ġspeeding":57575,">').":57576,"Ġscreened":57577,"_roll":57578,"ĠMacBook":57579,"ĠAUD":57580,"Ġdiagnose":57581,".Generate":57582,"Ġ^^":57583,"Ġstrs":57584,"[Test":57585,"Ġransom":57586,"ĠDHCP":57587,"elden":57588,"Ġinterpretations":57589,"()].":57590,"flatMap":57591,"ĠlineHeight":57592,"_mount":57593,"ĠWizards":57594,"Ġsluts":57595,"ehler":57596,"odal":57597,"Ġmilitia":57598,"å²":57599,"earned":57600,"Ġmisery":57601,"intval":57602,"fund":57603,"Ġhides":57604,"Ġdiarr":57605,"ĠWesley":57606,"Ġxmm":57607,"Ġquem":57608,"ĠArabs":57609,"ifth":57610,"ategorized":57611,"Disposable":57612,"Pure":57613,"_NOTIFY":57614,"snippet":57615,"ĠGarrett":57616,".running":57617,".weights":57618,"Ġ(--":57619,"Ġinvariant":57620,"äºĭ件":57621,"ĠAllowed":57622,"dirs":57623,"Ġpassions":57624,"Ġlad":57625,"ĠFlush":57626,"menus":57627,":block":57628,"Ġcompra":57629,".chomp":57630,"allocator":57631,"Ġcurated":57632,"ĠKnowing":57633,"ĠPatterson":57634,"Ġtelah":57635,"'ex":57636,"Ġdoomed":57637,"Ġphilanth":57638,"otty":57639,".styles":57640,"Owned":57641,"Ġallergies":57642,"=params":57643,"ocese":57644,"itelist":57645,"ĠSending":57646,"bef":57647,"orrar":57648,"ĠNão":57649,"ĠFargo":57650,"ĠLub":57651,"ĠCombined":57652,"_given":57653,"ĉĉĉĉĉĠĠĠĠ":57654,"Ġreconciliation":57655,"Patterns":57656,"azard":57657,"Ġbiomass":57658,"ĠHouses":57659,"respuesta":57660,"cco":57661,"/topics":57662,"ĠYuk":57663,"Ġweakened":57664,"_calendar":57665,"Ġmulheres":57666,"ĠMarl":57667,"Ġsine":57668,"ĠTil":57669,"ĠSouls":57670,"ĠDeutsche":57671,"ĠFOLLOW":57672,"Ġpipelines":57673,"ĠBeverly":57674,"_DIPSETTING":57675,"\"#":57676,"ĠProto":57677,".big":57678,"ĠSavings":57679,"ĠTanz":57680,"jun":57681,"ĠGamma":57682,"ĠSadd":57683,"Ġadvisors":57684,"Ġroast":57685,"Ġunters":57686,"udies":57687,"_lon":57688,"-pointer":57689,"ĠElementRef":57690,"\\Builder":57691,"exampleInput":57692,".webdriver":57693,"dataType":57694,"ĠQuite":57695,"ĠCeltics":57696,"uil":57697,"-defense":57698,"bish":57699,"ĠUIWindow":57700,"ĠSuddenly":57701,".hot":57702,".reason":57703,"Ġgör":57704,"AMD":57705,".Multi":57706,"authenticated":57707,"regions":57708,";(":57709,"аÑĢам":57710,"ĠKirby":57711,"$route":57712,"PRECATED":57713,"ĠDurham":57714,"owo":57715,"ĠPerforms":57716,"Ġdisregard":57717,"nst":57718,"ĠPols":57719,"ĠgetP":57720,"\"]:":57721,"-colored":57722,"(Keys":57723,"ĠAlleg":57724,"_modify":57725,"_loading":57726,"strained":57727,"Ġatroc":57728,"_phr":57729,"":58721,"ceph":58722,".DateTimePicker":58723,".\";ĊĊ":58724,"ĠTie":58725,",item":58726,"Ġmenn":58727,"Gas":58728,"ocha":58729,"_virtual":58730,"Ġmasterpiece":58731,"_sequences":58732,"LTE":58733,"ĠSubmission":58734,"Caller":58735,"$\\":58736,"Sport":58737,"agus":58738,"ConstraintMaker":58739,"Ġcoloc":58740,"Ġwig":58741,"ĠУ":58742,"ĉArray":58743,"Looks":58744,"ĠGTA":58745,".steps":58746,"atchewan":58747,"_ranges":58748,"extAlignment":58749,"ĠBrennan":58750,"Ġabstraction":58751,"ulerAngles":58752,".misc":58753,"Ġantibodies":58754,"Ġexponential":58755,"ĠCHANNEL":58756,"expense":58757,"'y":58758,"Ġdetectives":58759,"Ġpurported":58760,"YSTEM":58761,"Ġradioactive":58762,"ĠLatina":58763,".Encoding":58764,".TAG":58765,"xin":58766,"Degree":58767,"uracion":58768,"prices":58769,"ĠReferentialAction":58770,"Ġrarity":58771,"Ġpiles":58772,"gende":58773,"_projects":58774,"_globals":58775,".startTime":58776,"Ġ구":58777,"SECTION":58778,"_publish":58779,"Fault":58780,"DDL":58781,"_prior":58782,"Mom":58783,"Ġthicker":58784,"Ġsequelize":58785,"Ġessentials":58786,"stras":58787,"intr":58788,">(()":58789,".management":58790,"eil":58791,"éĹŃ":58792,"Aware":58793,".City":58794,"ĠArbit":58795,"_DM":58796,"_keyboard":58797,"LObject":58798,"-webpack":58799,"ĠNewport":58800,"ĠprincipalColumn":58801,"legant":58802,"Ġpallet":58803,"Ġfracture":58804,"Ġgmail":58805,".Meta":58806,"Above":58807,".KeyEvent":58808,"jit":58809,"_macro":58810,"_PUSH":58811,"ứ":58812,"/controller":58813,"åĬłè½½":58814,"Ġsuperficial":58815,"exterity":58816,"Ġmensagem":58817,"Wind":58818,"iston":58819,".openapi":58820,"иÑĢов":58821,"ĠSerializer":58822,"uctive":58823,"Ġzar":58824,"Places":58825,".Static":58826,"Ba":58827,"Ġinadvert":58828,"ĠIndonesian":58829,"_IPV":58830,"(horizontal":58831,"ĠgetTitle":58832,"idepress":58833,"ĠConsoleColor":58834,"ipers":58835,"$out":58836,"Ġfestive":58837,"Ġevenings":58838,".GetData":58839,"uitka":58840,"ĠManuals":58841,"ussed":58842,"_Max":58843,".Chat":58844,"ĠAircraft":58845,"=com":58846,"FOUND":58847,"apro":58848,"Ġtreasures":58849,"_alive":58850,"Ġgadget":58851,"eking":58852,"ButtonDown":58853,"Browsable":58854,".PERMISSION":58855,"PASSWORD":58856,"ĠHASH":58857,"fé":58858,"\\TestCase":58859,"LOSS":58860,"others":58861,",J":58862,"Ġasshole":58863,"werk":58864,"Ġmã":58865,".ie":58866,"evil":58867,"kontakte":58868,"////////////////////////////////////////////////////////////////////////////////Ċ":58869,"=sys":58870,"ĉlock":58871,"--;ĊĊ":58872,"_FUN":58873,"FillColor":58874,"óa":58875,"prend":58876,"Ġcompressor":58877,"Mother":58878,"ĠArcher":58879,".goto":58880,"Ġwürde":58881,"Ġbamboo":58882,"ï¼İ":58883,"ĠTrees":58884,"Ġbumper":58885,"Ġsausage":58886,"ĠElasticsearch":58887,"Ġhorizontally":58888,"ĠGul":58889,"Immutable":58890,"Ġloser":58891,"Ġaborted":58892,"-demo":58893,"ĠHatch":58894,"Ġunde":58895,"Ġprocesso":58896,"-call":58897,"Income":58898,"åĥ":58899,"_returns":58900,"'].\"'":58901,"(sw":58902,"CBS":58903,"amilies":58904,"ĠYourself":58905,"ĠHolt":58906,".MON":58907,"à§ĩ":58908,"ÑĪе":58909,"anon":58910,"ĠFontAwesome":58911,"producer":58912,"jr":58913,"Ġmau":58914,"ĉinter":58915,"Ġdishonest":58916,"Ġmagna":58917,"ĠCollective":58918,"Ġvraiment":58919,"Ġchoix":58920,"stay":58921,"Ġwelding":58922,"rising":58923,",min":58924,"ĠFate":58925,"glob":58926,"RGBA":58927,"Ġdette":58928,"Ven":58929,"Ġembarrassment":58930,".DELETE":58931,"gregar":58932,"-render":58933,"(bucket":58934,"\">ĊĊĊ":58935,".waitKey":58936,"Busy":58937,"Ġdifferentiation":58938,"ĠCST":58939,".Constant":58940,"ĠlineNumber":58941,"(matches":58942,"Ġwebsocket":58943,"Ġbarred":58944,"Ġpuedes":58945,"Mono":58946,"CORE":58947,"IID":58948,"ĠĠĠĠčĊčĊ":58949,"Ġpúblico":58950,"leaning":58951,"Ġcleansing":58952,"Ġcris":58953,"ĠDevils":58954,"_SETTING":58955,"untary":58956,".);Ċ":58957,"ĊĠĠĠĊ":58958,"[curr":58959,"tsy":58960,"ĠAlexis":58961,"ritel":58962,"Ġpetroleum":58963,".preprocessing":58964,"matter":58965,"ForResult":58966,"-license":58967,"Ġtravellers":58968,"ĠDispatcher":58969,"ennifer":58970,"Ġdigestive":58971,"PED":58972,"hibition":58973,"MASConstraintMaker":58974,"ĠWatt":58975,"Benef":58976,".setView":58977,"dto":58978,"TEE":58979,"ĠPelosi":58980,"_EXTRA":58981,"Ġmedals":58982,"xhr":58983,"forecast":58984,"Ġnargin":58985,"ouns":58986,"-fill":58987,"_CURSOR":58988,"Ġsupervised":58989,"Ġturf":58990,"ĠEdgar":58991,"POSITION":58992,"ĠcategoryId":58993,"âī":58994,"_ER":58995,"ủa":58996,"Shown":58997,".ll":58998,"_POLICY":58999,"(),'":59000,"ĠPrev":59001,"ĠStringField":59002,"ĉGlobal":59003,"assed":59004,"Throughout":59005,"ostringstream":59006,".awtextra":59007,"Ġslopes":59008,"ĠSequential":59009,"Ġgiorn":59010,"Ġzelf":59011,"Ġversatility":59012,"leneck":59013,".cgi":59014,"Ġdoubling":59015,"ĠBangkok":59016,"Ġbuurt":59017,"Ġusuário":59018,"studio":59019,"Ġjeunes":59020,"Ġmuted":59021,"Ġips":59022,"_fraction":59023,"&&(":59024,"Ġstunt":59025,"');?>čĊ":59049,"Ġevapor":59050,"bable":59051,"ĠPRICE":59052,"Ġæ³":59053,"lucent":59054,"Ġvamp":59055,"ĠTechnician":59056,"Ġuniqueness":59057,"Mes":59058,"urban":59059,".parametrize":59060,"ĠReplay":59061,"Sessions":59062,"embr":59063,"-Americans":59064,"_PROXY":59065,"Ġpian":59066,"Ġtrie":59067,"ĠDestructor":59068,"GameState":59069,"ĠIMF":59070,"chin":59071,"Ġporte":59072,"ĠSwal":59073,"åŁİ":59074,"Substring":59075,"iming":59076,"/Library":59077,"Ġfrightened":59078,"writes":59079,"Ġrecursos":59080,"arResult":59081,"_INITIALIZ":59082,"ĠBadge":59083,"_crc":59084,"Eight":59085,"ĠDISTINCT":59086,"Ġthro":59087,"@Xml":59088,"ĠLegendary":59089,"-twitter":59090,"_easy":59091,"Ġ+++":59092,"(DATA":59093,".Locale":59094,"Ġkä":59095,"Ġnurt":59096,"Ġcruis":59097,"_ios":59098,"Ġsensing":59099,"_Line":59100,"ĊĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":59101,"pong":59102,"oleon":59103,"Ġwildcard":59104,"çĶ¨æĪ·åIJį":59105,"Ġbegging":59106,"Rod":59107,"ĠÃİ":59108,"_CELL":59109,"Researchers":59110,".selector":59111,"_ing":59112,"Ġaspiring":59113,"Ġimmortal":59114,"Ġymin":59115,"_robot":59116,"Ġplur":59117,"BTC":59118,"ĠDID":59119,"Ġpiercing":59120,"*u":59121,"_DEFINED":59122,"ĠThi":59123,"itaire":59124,"(media":59125,"-ons":59126,"Ġchefs":59127,"Ġ\"*.":59128,"/AP":59129,"Ġrazor":59130,"ĠsearchData":59131,"Ġ=&":59132,"ĠãĢĤ":59133,"Ġmourn":59134,"tingham":59135,"Ġoli":59136,"ĠVernon":59137,"_RS":59138,"ŀæĢ§":59139,"Ġfácil":59140,"angen":59141,"celain":59142,"Ġail":59143,"lest":59144,"ĠQCOMPARE":59145,"gain":59146,"Ġε":59147,"ĠKob":59148,"ĠFault":59149,"_configs":59150,"ç»ĵæŀľ":59151,".+":59152,"calar":59153,"(colors":59154,"Mul":59155,"_ART":59156,"Ġexperimenting":59157,"ermen":59158,"ĠAnglo":59159,".FixedSingle":59160,"Sea":59161,"Ġctxt":59162,".slider":59163,"Collapse":59164,"Grey":59165,"Ġfld":59166,"-proof":59167,".capacity":59168,"getParent":59169,"ĠCompliance":59170,"Ġburgl":59171,"-rec":59172,"Ġoverwritten":59173,"MU":59174,"Ġrouters":59175,"ĉModel":59176,"Ġfantasies":59177,"avian":59178,"_prec":59179,"ĠScandin":59180,"Ġ//<":59181,"/oct":59182,"Ġceremonies":59183,"Months":59184,"undy":59185,"Ġqued":59186,"ĠNou":59187,"ĠVibr":59188,".rgb":59189,"Ġcitrus":59190,"Ġbraces":59191,"-uppercase":59192,"getTable":59193,"Ġdopo":59194,"ĠKerr":59195,"_CHILD":59196,"-cloud":59197,"ĉMatrix":59198,"Ġgardening":59199,"Sing":59200,"almost":59201,"Requirements":59202,"uguay":59203,"(Property":59204,"subscriber":59205,"FAST":59206,"reaction":59207,"(lp":59208,")})Ċ":59209,"`).":59210,".wallet":59211,"_exchange":59212,".Maximum":59213,"ĠVerb":59214,"âĶģ":59215,"()<":59216,"ï¼ĽĊ":59217,"ROT":59218,"CARD":59219,"ubit":59220,"{@":59221,"_kel":59222,"ĠTooltip":59223,"MySQL":59224,"MainActivity":59225,"arf":59226,"Ġmalign":59227,"Ġseinen":59228,"apist":59229,"Ġ<%":59230,"MethodImpl":59231,"Mil":59232,"ĠMick":59233,".depend":59234,">&":59267,"ĉok":59268,"-low":59269,".usuario":59270,"nested":59271,"XB":59272,"OURS":59273,".BorderColor":59274,"Ġbrow":59275,"ĠÐķ":59276,"corr":59277,"ĠRedskins":59278,".getTag":59279,".getTransaction":59280,"Ġstigma":59281,"hardt":59282,"ĠPlayerPrefs":59283,"alsy":59284,"ucson":59285,"Languages":59286,"ĠOlivia":59287,"Ġtac":59288,"Ġbli":59289,"Ġcaval":59290,"Ġconsolidated":59291,"Ġperil":59292,"Ġdele":59293,"Ġformulated":59294,"Ġhighways":59295,".spawn":59296,"==$":59297,"ĠNiet":59298,"Ġveggies":59299,"ypo":59300,"-rule":59301,"ĠVie":59302,"/epl":59303,"Ġenfants":59304,"stringLiteral":59305,"Ġtoughest":59306,"buyer":59307,"Ġcovariance":59308,"Ġili":59309,"ĠSophie":59310,"ĠBAB":59311,"Ġ\"),":59312,"ĠUk":59313,"currentIndex":59314,"_userdata":59315,".codec":59316,"ĠPunjab":59317,"ĠSNP":59318,"lol":59319,"advance":59320,"Ġcomfy":59321,"JsonIgnore":59322,"Ġfashionable":59323,"ĠICON":59324,"Ġora":59325,"ĠPricing":59326,"E":59384,"tering":59385,"/screens":59386,"Ġheightened":59387,"аÑĢÑĤ":59388,"Authorities":59389,"_bbox":59390,"ünst":59391,".fontSize":59392,"ĠBOOLEAN":59393,"divide":59394,"ĠSloven":59395,"ucer":59396,"ÙĴ":59397,"stub":59398,"Ġnavigating":59399,":animated":59400,"_NOW":59401,"_vect":59402,"}{Ċ":59403,"@(":59404,"Ġtelecom":59405,"Ġcontracting":59406,"ĠAssange":59407,"Ġextracting":59408,"Ġgrö":59409,"cobra":59410,".DIS":59411,"Ġcrab":59412,"Ġtwitch":59413,"Ġverts":59414,"Ġrejects":59415,"ĉformat":59416,"Ġregeneration":59417,".Sys":59418,"solve":59419,"ĉdialog":59420,"shi":59421,"meter":59422,"(best":59423,"validators":59424,"Ġonwards":59425,"Ġguru":59426,"Ġmoderator":59427,"owied":59428,"experiment":59429,"rub":59430,"Ġmqtt":59431,"ĠCaucas":59432,"Ġnationalism":59433,"Ġmange":59434,"ĉImGui":59435,"/Edit":59436,"Ġinh":59437,"Ġintellig":59438,"erokee":59439,"ĉexport":59440,"Ġdiscriminate":59441,"subtract":59442,"ĠMoodle":59443,"enser":59444,"ĠGuides":59445,"RAP":59446,"-hot":59447,"_grp":59448,".picture":59449,"XA":59450,"ĠinitView":59451,"_Comm":59452,"Ġoverdose":59453,"Ġ+ĊĊ":59454,"ĠSilent":59455,"shows":59456,"Ġinterpolate":59457,"Formation":59458,"Ġbisc":59459,"markets":59460,"(SC":59461,"Ze":59462,"ĠNetworking":59463,"Ġadrenal":59464,"ĠGuns":59465,"eteor":59466,"Declared":59467,"orgetown":59468,"Ġkarena":59469,"/password":59470,"_addresses":59471,"ITERAL":59472,"Buzz":59473,"ĠConway":59474,"(case":59475,"PWD":59476,"heiro":59477,"(act":59478,"**čĊ":59479,"());ĊĊĊ":59480,"Ġanv":59481,"Ġ..ĊĊ":59482,"(MenuItem":59483,"(mail":59484,"_sections":59485,"ĉnet":59486,"Ġplut":59487,"Ġwrench":59488,"/object":59489,"ĠIst":59490,"ĠVIS":59491,"/pub":59492,"alten":59493,"Ġguitars":59494,"Ġantibiotic":59495,"ï¼ĸ":59496,"¹":59497,"Ġ\"+\"":59498,"formula":59499,"Ġbabes":59500,"ĠPrompt":59501,"Ġenim":59502,"/player":59503,"ĉref":59504,"ĠbyÄĩ":59505,"Ġconsumes":59506,"ĠHast":59507,"ĠTao":59508,"Ġ'))Ċ":59509,"Ġclam":59510,"Ġthighs":59511,"Ġmotif":59512,"ApiOperation":59513,"ĠWL":59514,"getC":59515,"ĉflags":59516,"ointments":59517,"Ġeconomical":59518,"needle":59519,"xls":59520,"practice":59521,"utzer":59522,"timeofday":59523,"-output":59524,"ĠfindById":59525,"ĠBuddy":59526,"ÐŀÑĤ":59527,"Seven":59528,"ĠBark":59529,"Ġenvoy":59530,"_algorithm":59531,"åĪ©":59532,"Ġballistic":59533,"移":59534,"rades":59535,"ĉdoc":59536,"roducing":59537,"ĠEating":59538,"Unmount":59539,"/dataTables":59540,"_bonus":59541,"Ġlitt":59542,"pps":59543,")localObject":59544,"perf":59545,"ĠHelvetica":59546,"shutdown":59547,"/ml":59548,".tokens":59549,"ĠHardcore":59550,",row":59551,"/bg":59552,"Scaler":59553,"âĢĶas":59554,"_logits":59555,"âĢĻint":59556,"ĉApp":59557,"Implicit":59558,".Fprintf":59559,"ETO":59560,"Ġterra":59561,"Ġpossessing":59562,".rstrip":59563,",),":59564,"=yes":59565,"ĠStripe":59566,"?=":59567,"neutral":59568,".good":59569,"Ġkennen":59570,"ĠSung":59571,"fault":59572,"ystatechange":59573,"Canadian":59574,"','\".$":59575,"ĠMits":59576,"ænd":59577,"ĠSTRUCT":59578,"ĠURLWithString":59579,"ĠCompass":59580,"Ġ--ĊĊ":59581,"ĠNSLayoutConstraint":59582,"|min":59583,"-adjust":59584,"Ġrebuilt":59585,"LIGHT":59586,"/se":59587,"-mount":59588,"vpn":59589,"validated":59590,"(QObject":59591,"Ġignition":59592,"ĠChargers":59593,"RYPTO":59594,"]initWithFrame":59595,"ĠFluid":59596,"Ġcadre":59597,"Ġnominations":59598,"Neill":59599,"ĠHou":59600,"Ġcurrents":59601,"_gene":59602,"(inp":59603,"Paris":59604,"zÄĻ":59605,"aggregate":59606,"Ġassoc":59607,"weeted":59608,"errat":59609,"âĢĵĊĊ":59610,"Ġ'/',Ċ":59611,"fixture":59612,"ĠHighest":59613,"ambient":59614,"Ġchmod":59615,"Ġconte":59616,"Ġsensual":59617,"Ġgarment":59618,"zers":59619,"ĠPowered":59620,"domains":59621,"Reward":59622,"iomanip":59623,"Ġcockpit":59624,"outfile":59625,"Ġbuiltin":59626,"Ġinsisting":59627,".vars":59628,"zipcode":59629,"Ġ����":59630,"fails":59631,"Ġconsolidation":59632,"_oid":59633,"Planet":59634,"Ġ=\",":59635,"ĉel":59636,"UILT":59637,"ätz":59638,"afari":59639,"ĠMcCl":59640,"Timeline":59641,"Esta":59642,"Ġfram":59643,"YE":59644,"Ġcerebral":59645,"OfMonth":59646,"ĠPregn":59647,"ĠклаÑģÑģ":59648,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":59649,"ĠFres":59650,"Approved":59651,".Special":59652,"ĠProtestant":59653,"Ġallergy":59654,"_pcm":59655,"ĉCopyright":59656,"ĠsuperClass":59657,"\"strconv":59658,"ĠMohamed":59659,"Ġ'//":59660,"ForeColor":59661,"Arthur":59662,"ĠJungle":59663,"Ġveins":59664,"Sad":59665,"Ġbackups":59666,"ĠOpinion":59667,"ût":59668,"Ġintermitt":59669,"odyn":59670,"ĠChristina":59671,"Ġandre":59672,"Ġevacuation":59673,"palette":59674,"horse":59675,"ĠResident":59676,"ĠHassan":59677,".Nil":59678,"Ġaisle":59679,"ĠGrowing":59680,"Ġbloginfo":59681,"/sql":59682,"_ioctl":59683,"Scaling":59684,"ĠMonad":59685,"_cpp":59686,"ĠHutch":59687,"ĠAppleWebKit":59688,"Expense":59689,"_JOB":59690,"Ġpointless":59691,"FromBody":59692,"antal":59693,"Ġdepicting":59694,"ĠCELL":59695,"Ġrefin":59696,"ĠCNC":59697,"ì¹ĺ":59698,"_dimensions":59699,"ĠSAN":59700,"Ġaft":59701,"Ġfootsteps":59702,"ccoli":59703,"_PHONE":59704,"/math":59705,"-kind":59706,"ĠMeans":59707,"ichael":59708,".guna":59709,"Ġinauguration":59710,"-driving":59711,"(delete":59712,"ĠtotalCount":59713,"_MC":59714,".Extension":59715,"Commercial":59716,"ĠzIndex":59717,"$":59849,"Ġebay":59850,"Ġcaptive":59851,"pliant":59852,"ĠCalculates":59853,"olta":59854,"esting":59855,"_revision":59856,"Ġmús":59857,"+m":59858,"\",\"\",\"":59859,"WHAT":59860,"Ġcompassionate":59861,"harga":59862,"[random":59863,"Ġmodulo":59864,"(sn":59865,"Ġoccupations":59866,"////Ċ":59867,"ĉboard":59868,"ĠBalk":59869,"wiÄħ":59870,"ĠWifi":59871,".Profile":59872,":maj":59873,"ĉmat":59874,"LOCKS":59875,"(jButton":59876,"Ġ('$":59877,"Mur":59878,"æĮī":59879,"bble":59880,"Ġfrog":59881,"-hide":59882,"Ġbroadcaster":59883,"à¸ŀ":59884,"haled":59885,"Ġamusing":59886,"_predictions":59887,"_intr":59888,"Ġeagle":59889,"аÑĤелÑĮ":59890,"ĠgetList":59891,"psilon":59892,"Ġcharacterization":59893,"ARDS":59894,"Ġrelocation":59895,"Ġrulers":59896,"PAY":59897,"ĠDefinitely":59898,"_Action":59899,"Ġclosures":59900,"Ġfactual":59901,"odynamic":59902,"Ġprecautions":59903,"niej":59904,"ĠParties":59905,"ĠSubaru":59906,"Ġcousins":59907,"arbeit":59908,".money":59909,"gunta":59910,"(and":59911,"getitem":59912,".StylePriority":59913,"Ġslid":59914,"singleton":59915,"Ġgarn":59916,"ĠPAS":59917,"Ġdazz":59918,"aż":59919,"Ġbogus":59920,"ĠMog":59921,"Ġrivalry":59922,"isol":59923,"Ġlandmarks":59924,"ñas":59925,"Bern":59926,"ĠSachs":59927,"Ġ\")ĊĊ":59928,"Ġhostility":59929,"_mex":59930,"mere":59931,"Mot":59932,"pictureBox":59933,"Defense":59934,"Ġaffidavit":59935,"otherwise":59936,".directory":59937,"_UnityEngine":59938,"-blog":59939,".skin":59940,"phem":59941,"Apellido":59942,"erchant":59943,"[class":59944,"Ġwart":59945,".\"[":59946,"aleur":59947,"/back":59948,"ĠĠĠĠĉĠĠĠ":59949,"Ġprecipitation":59950,"Ġobstruction":59951,"ĠpObj":59952,"Ġrupt":59953,"UCKET":59954,"aye":59955,"æİĴ":59956,"gx":59957,"Ġecl":59958,"Ġsecrecy":59959,"/Header":59960,"ĠLesb":59961,"Ġlei":59962,"ĠBulletin":59963,"Ġgiveaway":59964,".Home":59965,"_ROOM":59966,"\"W":59967,"Ġcowork":59968,"_ra":59969,"ĠCycling":59970,"ĠPaw":59971,"Ġpupil":59972,"/arch":59973,"ĠFileUtils":59974,"é¦ĸ":59975,"rsp":59976,"Ġfreedoms":59977,"ĠLear":59978,"}`).":59979,"Ġbowls":59980,"/block":59981,"_logging":59982,"Ġmethane":59983,"Ġhorns":59984,"Ġwonderfully":59985,"Ġalterations":59986,"Ġexile":59987,"lsen":59988,"_pause":59989,"_LANGUAGE":59990,"ĠUSDA":59991,"_mysql":59992,"_AMOUNT":59993,"ĠLIFE":59994,"Ġyoungsters":59995,"Ġriots":59996,"[E":59997,"Ġunforgettable":59998,",},Ċ":59999,"Disposed":60000,"ĠAssassin":60001,"UNG":60002,"ĠNewsp":60003,"UserService":60004,":aload":60005,"+',":60006,"Ġsettlers":60007,"Ġscreams":60008,"Ġinconvenience":60009,".Rotate":60010,"Ġjars":60011,"ĠPuzzle":60012,"Ġmest":60013,"arsi":60014,"ĠSharma":60015,"|(":60016,".ds":60017,"ĠSacred":60018,"_evt":60019,"Ġexpresses":60020,"Ġhoch":60021,"ĠDuch":60022,".calls":60023,"thr":60024,"ĠSheffield":60025,".AlertDialog":60026,"Ġradically":60027,"Ġtrous":60028,"Ġprevailing":60029,"ĠWWII":60030,"âĢĻn":60031,"ensely":60032,"ĠYesterday":60033,"ĠSirius":60034,"Ġkillers":60035,"ĠFFT":60036,"Ġoval":60037,"'):čĊ":60038,"Ġìłķë³´":60039,"ourage":60040,"ĠCheckbox":60041,"Workbook":60042,".defer":60043,"_floor":60044,"Ġcouncill":60045,"Ġnorske":60046,"moil":60047,"orea":60048,"Ġmarketed":60049,"_SUR":60050,"xAA":60051,"Ġstained":60052,"eut":60053,"ĠMeng":60054,"Ġieee":60055,".extern":60056,"egie":60057,"Ġrapp":60058,"ĠPyongyang":60059,"'class":60060,"Mob":60061,"ĠinitialValue":60062,"_wave":60063,"Ġjab":60064,"Ġmasculine":60065,"Ġamplifier":60066,"Ġtty":60067,"PathComponent":60068,"_xt":60069,"ĠGFP":60070,"/sec":60071,"ĉdispatch":60072,"markdown":60073,"ĠSchn":60074,"bole":60075,"··":60076,"mousemove":60077,"ĠerrMsg":60078,"Ġasign":60079,"_mono":60080,"ToSelector":60081,"ĠZu":60082,"(Rect":60083,"ĠErrorCode":60084,"latin":60085,"angible":60086,"vtk":60087,"CGSize":60088,"Pokemon":60089,"Ġclassmates":60090,"Ġattracts":60091,"ĠTatto":60092,"ultan":60093,"ológ":60094,"Ġhalted":60095,"न":60096,"ĠKart":60097,"Ġue":60098,"_InitStructure":60099,"TestClass":60100,"ĠAirbnb":60101,"_\",":60102,"Ġcharcoal":60103,"Ġipc":60104,"ĠStretch":60105,".glide":60106,"latesAutoresizingMaskIntoConstraints":60107,"Ġpotion":60108,"ITTLE":60109,"Ġcountert":60110,"_hd":60111,"prepared":60112,"Ads":60113,"ĠVampire":60114,"robots":60115,".CreateIndex":60116,"StatusLabel":60117,"Ġtucked":60118,"afür":60119,"Ut":60120,"Ġsweater":60121,"_FN":60122,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĉ":60123,"ataka":60124,"Ġeyebrows":60125,"acoes":60126,"uden":60127,".LinearLayoutManager":60128,"Ġsway":60129,"Ġmultin":60130,"())))Ċ":60131,"ĠNSUInteger":60132,"ĠMyBase":60133,"Partner":60134,"utschen":60135,"ĠCater":60136,".setBackgroundColor":60137,"Ġaccomplishment":60138,"_problem":60139,".dtd":60140,"ĠpageNumber":60141,"Ġjackets":60142,"Ġcropped":60143,"uels":60144,"ĠHep":60145,"Ġcapped":60146,"*Math":60147,"_callbacks":60148,"Ġpubb":60149,"ĠBrunswick":60150,".respond":60151,"[\"_":60152,"Ġbedding":60153,"hythm":60154,"OX":60155,"(speed":60156,"Ġpesticides":60157,"Ġ-------":60158,".Blue":60159,"Ġnoodles":60160,"ĠGoes":60161,"Ġsaver":60162,"oxy":60163,"_completion":60164,"ĠSwinger":60165,"ĠgetDate":60166,"Ġminded":60167,"integration":60168,"ĠLotus":60169,"(stop":60170,"(',');Ċ":60171,"Ġfloods":60172,"ĠWorkflow":60173,"Ġerupted":60174,"Macro":60175,"ĠSauce":60176,"ĠeventName":60177,"\\Input":60178,"Breaking":60179,"ĉwhen":60180,"_pw":60181,"INDER":60182,"ĠWellness":60183,"Ġvoxel":60184,"ĠMell":60185,"ĠMEDIA":60186,"SENS":60187,"ĠFunds":60188,"ĠMild":60189,"Ċ":60198,"Ġtempting":60199,"Ġtestament":60200,"Ġbible":60201,"Ġconsulted":60202,"ĠIndexError":60203,"è¨ĺ":60204,"Ġkeypad":60205,"izzo":60206,"(ok":60207,"Ġwhatsapp":60208,"ĠRemoteException":60209,"Ġteamed":60210,"âĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶâĢĶ":60211,"»,":60212,"ĠgetTime":60213,"diag":60214,"issy":60215,"Ġhed":60216,"Ġknots":60217,"jom":60218,"Ġfunnel":60219,"-mails":60220,"Ġexporting":60221,"ĠVL":60222,"ĠKarn":60223,"ĠBuddhism":60224,"ĠAllan":60225,"_RADIUS":60226,"Ġwording":60227,"ĠForget":60228,"ĠCorona":60229,"iphy":60230,"Ġlimburg":60231,"uggy":60232,"ĠUserRepository":60233,"imin":60234,"(ele":60235,"Ġlabelled":60236,"社":60237,"ĠHerman":60238,".qq":60239,"Ġ\"));Ċ":60240,"ieber":60241,".Translate":60242,"ryn":60243,"Ġdesenv":60244,"umd":60245,"Simply":60246,"ĉmode":60247,"Rpc":60248,"ĠValencia":60249,"Ġstaffers":60250,"Ġselv":60251,"ĠSpike":60252,"Ġdelic":60253,"Ġeru":60254,"_DT":60255,"Judge":60256,"á»ķ":60257,"ĠBasin":60258,".mutable":60259,"\"url":60260,"Ġtariff":60261,"ĠSleeve":60262,"Ġflare":60263,".dropout":60264,"Ġbrides":60265,")),čĊ":60266,"_constraints":60267,"destruct":60268,"Outline":60269,"Ġdisappears":60270,"_locked":60271,"ĠNSLocalizedString":60272,"cke":60273,"ĉnull":60274,"adresse":60275,"Ġtopping":60276,"ĠJoker":60277,"bishop":60278,"ноÑģÑĤÑĮ":60279,"andering":60280,"_amp":60281,"=time":60282,"_Space":60283,"_PULL":60284,"'=":60285,"Ġantiqu":60286,"Ġcach":60287,"___ĊĊ":60288,"ONES":60289,"оÑı":60290,"Ġunread":60291,".policy":60292,"oooooooo":60293,"룬":60294,"Ġusted":60295,"ĠRece":60296,"Ġallem":60297,"ãĥ¼ãĤ¹":60298,"ĠThoughts":60299,"veillance":60300,"istrate":60301,"_lane":60302,"Ġfamed":60303,".GetName":60304,"Ġsmoother":60305,"ĠQualified":60306,"azers":60307,"_geo":60308,"Fax":60309,"ĠMinds":60310,"ĠRaises":60311,"Ġtranscripts":60312,"Conversation":60313,"Ġremarked":60314,"ëĤĺ":60315,"dling":60316,"Ġdeploying":60317,"ĠsharedApplication":60318,"Ġkp":60319,"FontAwesomeIcon":60320,"_dummy":60321,"reiben":60322,"ĠJaneiro":60323,"Directions":60324,".getBean":60325,"sass":60326,"Ġcommanders":60327,"vation":60328,"errorCode":60329,"ĠAlloy":60330,".localized":60331,"Ðij":60332,"Ġdishwasher":60333,"ĠSoup":60334,"Nu":60335,"_Default":60336,"Ġuneven":60337,"Ġ/>\";Ċ":60338,"-Based":60339,"Ġseamlessly":60340,"-null":60341,"ĠXC":60342,"Ġstew":60343,"(delay":60344,"ATORS":60345,"ĠWheeler":60346,"\"H":60500,"east":60501,".air":60502,"âĢľBut":60503,"ObjectContext":60504,"successfully":60505,"_land":60506,"Ġfolds":60507,"_COORD":60508,"Ġsubpo":60509,".getAddress":60510,"instr":60511,"Materials":60512,"ÑĥÑģÑĤ":60513,"deposit":60514,"-last":60515,"_GRAY":60516,"=find":60517,"Ġmutant":60518,"Ġlesbienne":60519,"letcher":60520,"ROUGH":60521,"ureka":60522,".capture":60523,"Ġenn":60524,"Ġ([[":60525,"ĠFlu":60526,"ĠtaskId":60527,"ĠHussein":60528,".folder":60529,"Ġausterity":60530,"ISTRATION":60531,"_Impl":60532,"注æĦı":60533,"Ġdecree":60534,"-chat":60535,"Ġimplication":60536,"Ġguesses":60537,"ulkan":60538,"Analytics":60539,".plus":60540,"COMMAND":60541,"ели":60542,"»ĊĊ":60543,"_SITE":60544,"ĠequalTo":60545,"SupportFragmentManager":60546,"ĠRecording":60547,"å®ĮæĪIJ":60548,"Ġbaggage":60549,"Ġpitchers":60550,"ĠEh":60551,"oque":60552,"ĉcnt":60553,"Ġ=>$":60554,"/foo":60555,"IRA":60556,"ĠSatellite":60557,"borah":60558,"Ġ}}\"Ċ":60559,"ĠEnds":60560,"ĠSpray":60561,",param":60562,".Chrome":60563,"*q":60564,"thought":60565,"ibrated":60566,"Ġthieves":60567,"Ġbeneficiaries":60568,"Entered":60569,"ottesville":60570,"Ġveterin":60571,"ByID":60572,"quipe":60573,"umption":60574,"-unit":60575,"ExecutionContext":60576,"@s":60577,"ĠGiov":60578,".ToolTip":60579,"_friend":60580,"(attributes":60581,"Ġdumping":60582,"ĠJC":60583,"_DOCUMENT":60584,"ĠArmour":60585,"(insert":60586,".HorizontalAlignment":60587,"ĠQed":60588,"ãģĦãģ¾ãģĻ":60589,"/git":60590,"ĠYYYY":60591,"ĠCardiff":60592,"Ġapa":60593,"organic":60594,"ĠWhereas":60595,"ĠæĿ":60596,"ĠMia":60597,"Ġdemolition":60598,"Ġscars":60599,"Ġpai":60600,"Ġretries":60601,"Ġrq":60602,"ĠDenis":60603,"(Utils":60604,"Ġalleviate":60605,"ĠPIC":60606,"idue":60607,"Ġacknowledging":60608,"Ġ//////////////////////////////////":60609,"ç¡®å®ļ":60610,"Ä«":60611,"\\Json":60612,".binary":60613,"Ġxtype":60614,"signals":60615,"ĠAppearance":60616,"&r":60617,"}s":60618,"Ci":60619,"ĠIllum":60620,"porate":60621,"hog":60622,"ĠindexOf":60623,"\\Command":60624,"_parallel":60625,"ĠSherlock":60626,"íĥ":60627,"Ġ\"\")čĊ":60628,"////////////////////////////////////////////////////////////////////////////////////////////////":60629,"Ġcriticize":60630,"ĠSoap":60631,"ĠMatcher":60632,"Ġgrilled":60633,"*T":60634,"Ġadore":60635,"ulling":60636,"Ġjedoch":60637,"_refs":60638,"leanup":60639,"ĠJAXB":60640,"Ġroses":60641,"ĠLiam":60642,"sizei":60643,"Ġgetchar":60644,"Ġtarde":60645,"-tooltip":60646,"Ġqualifier":60647,"ĠIntermediate":60648,"_Window":60649,"ĠMalta":60650,"Disconnect":60651,"ewhere":60652,"Campo":60653,"Ġirrational":60654,"ledo":60655,"ĠDN":60656,"ARGV":60657,"Ġoutro":60658,"Ġthirteen":60659,"Joseph":60660,"MAR":60661,"/gl":60662,"Jess":60663,"ĠPsychiat":60664,"ĠpaddingBottom":60665,"-loop":60666,"/fonts":60667,"_seen":60668,"Teams":60669,"ReactDOM":60670,"(man":60671,"(xpath":60672,".getSimpleName":60673,">(*":60674,"ĠPvt":60675,"Ġelders":60676,"Ġpies":60677,".userAgent":60678,"-region":60679,"ĠGreeks":60680,"(fragment":60681,"stu":60682,"Ġcouncils":60683,"Ġstamina":60684,"ĠGoddess":60685,"西":60686,"Ġphilosophers":60687,"Ġpersone":60688,"ĠLose":60689,"ĠCLR":60690,"ĠDocs":60691,"Ġsoak":60692,"ĠHOLDER":60693,"Ġbells":60694,"hashCode":60695,"RATE":60696,"_WEIGHT":60697,"inous":60698,"endra":60699,"ophobic":60700,"Ġprose":60701,"Ġfinely":60702,"/oauth":60703,"(space":60704,"adge":60705,"ĠMama":60706,"ĠstringBuffer":60707,"Ġstint":60708,"Ġmisma":60709,"Ġvillains":60710,"ĠCrimea":60711,"Ġdiploma":60712,"ĠпоÑģл":60713,"ĠBea":60714,"(join":60715,"Ġíķ´":60716,"CHAT":60717,"pering":60718,"ĠCros":60719,"Ġmonkeys":60720,"Ġpreds":60721,"yla":60722,",,,":60723,"Ġvibrator":60724,"ĠNU":60725,"åħĪ":60726,"fant":60727,"zet":60728,"Ġbietet":60729,"unft":60730,"sworth":60731,".Flow":60732,"Ġpsyched":60733,"ĠContinental":60734,">t":60735,"Ġquilt":60736,".UP":60737,"Ġexpansive":60738,"Dispose":60739,"(language":60740,"Caps":60741,"_ZONE":60742,"Ġrecycle":60743,"ĠManaged":60744,"currentColor":60745,".broadcast":60746,"signIn":60747,".prom":60748,"llu":60749,"ueblo":60750,"Ġpunches":60751,"Ġautomat":60752,"Ġassigning":60753,"ĠcreateUser":60754,"ĠAllied":60755,"Ġconductor":60756,"Ĥ¨":60757,"Ġsaddle":60758,"Ġdni":60759,"omedical":60760,"-West":60761,"PositiveButton":60762,"Ġitalic":60763,"?[":60764,"(trigger":60765,"Ġelephants":60766,"\":\"\",\"":60767,"Ġcaliber":60768,"rafted":60769,"digits":60770,"Ġmarshal":60771,"milliseconds":60772,"markers":60773,"mom":60774,"/place":60775,"Ġholistic":60776,":t":60777,"#,":60778,"Ġboto":60779,"Ġnausea":60780,"ĠShooting":60781,"itech":60782,"ĠtextStatus":60783,"())Ċ":61004,"ADDRESS":61005,"BST":61006,"etzt":61007,"ĠQgs":61008,"Sense":61009,"ExceptionHandler":61010,"ĠChu":61011,".getOwnProperty":61012,"Ġexercised":61013,"iotic":61014,"ĠReleases":61015,"Ġpinterest":61016,"olie":61017,"isoft":61018,"Ġsequencing":61019,"Ġpadre":61020,"]));čĊ":61021,"(radius":61022,".med":61023,"ainties":61024,".ObjectModel":61025,"Ġemple":61026,"Ġseguro":61027,"Stars":61028,"Ġqualitative":61029,"lemn":61030,"á»±":61031,">\").":61032,"Ġgx":61033,"-cert":61034,"ĠASTM":61035,"Ġfullname":61036,"Ġtelemetry":61037,"ĠCambodia":61038,"_ul":61039,"ĠClare":61040,"CUSTOM":61041,"QC":61042,"ĠUns":61043,"ĠHTTPS":61044,"ĠParkinson":61045,"ancybox":61046,"','.":61047,"Tue":61048,".getLast":61049,"Ġabi":61050,"Äħd":61051,"Ast":61052,"ĠEditing":61053,".Unity":61054,"jmp":61055,"Ġmats":61056,"ĠsharedPreferences":61057,"Captain":61058,".pageSize":61059,"Ġrtl":61060,"Ġanmeld":61061,"RuntimeObject":61062,"Ġdemande":61063,"(\";":61064,"seite":61065,"-headed":61066,"ĠKra":61067,"ĠFONT":61068,"`\\":61069,"ClassNotFoundException":61070,".avg":61071,"atical":61072,"Aj":61073,"Ġpermitting":61074,"Proj":61075,"ERRQ":61076,"Ġcreampie":61077,"ĠBuyer":61078,"-modules":61079,"ĠSundays":61080,"|`Ċ":61081,"Ġdaytime":61082,"Ġ+(":61083,"Ġglitch":61084,"ĠOperand":61085,"Ġtoxins":61086,"inya":61087,"DNS":61088,"ĠSas":61089,"Cake":61090,"ĠNationals":61091,".addTo":61092,"Ġsinking":61093,"Ġcomprehension":61094,"Ġscor":61095,"agements":61096,"Ġtard":61097,"Ġmarching":61098,"ĠMTV":61099,"Ġsane":61100,"CreateInfo":61101,"ắ":61102,"ĠendIndex":61103,"ĉlayout":61104,"ĠåIJį":61105,"SITE":61106,"ĠTHERE":61107,"Ġ[{'":61108,"opathic":61109,"Ġtransmitter":61110,"/body":61111,"Ġpund":61112,"ĠClosing":61113,"Ġsetattr":61114,"Ġbounded":61115,"Atlas":61116,"suming":61117,"(times":61118,"parer":61119,"ynom":61120,"feit":61121,"Ġfrem":61122,"-leg":61123,"ĠBras":61124,">#":61125,"Ġì¶ľëł¥":61126,"ĠINSTANCE":61127,"ĠCouch":61128,"_hosts":61129,"likelihood":61130,".Marker":61131,"ĠMasks":61132,"Ġcereal":61133,"utilities":61134,"Ġelemental":61135,"Ġdistorted":61136,"inactive":61137,"cry":61138,"WL":61139,"UPPORTED":61140,".Throws":61141,"/schema":61142,"serie":61143,".\"',":61144,"ĠBenedict":61145,"-picker":61146,"iggs":61147,"ĠPirate":61148,"åij¨æľŁ":61149,"ĠThema":61150,"ĠSouthampton":61151,"ĠarrayWith":61152,"ĠPaula":61153,"Ġpredictor":61154,"-Ass":61155,".userid":61156,"Ġperi":61157,"Ġexaggerated":61158,"urate":61159,"arseille":61160,"ĠConcent":61161,"ĠPik":61162,"Ġ@_;ĊĊ":61163,"Ġformations":61164,"Ġdenomin":61165,"\"/>.Ċ":61166,"endedor":61167,"Ġpancre":61168,"Ġamt":61169,"ĠonResume":61170,"onDelete":61171,"ĠBCH":61172,")(\"":61173,"movement":61174,"Ġpotassium":61175,"":69726,"ĠPPC":69727,"isz":69728,"akeFromNib":69729,"ĠDisp":69730,"ĠAthletics":69731,"Ġnightclub":69732,"GOOD":69733,".setGeometry":69734,"+[":69735,"/send":69736,"Ġbinaries":69737,"Ġráp":69738,":req":69739,"-consuming":69740,"ertime":69741,"UPDATED":69742,"_nullable":69743,"VIN":69744,"ulia":69745,"cyan":69746,"Ġmisunderstanding":69747,"orical":69748,"degrees":69749,"Leading":69750,".AR":69751,"ickest":69752,"Nuevo":69753,"uforia":69754,"Ġgoodies":69755,"Ġfores":69756,"()<<\"":69757,"ademic":69758,"ActionCreators":69759,"servername":69760,"(nt":69761,"dbContext":69762,"Ġairborne":69763,"Ġexhibitions":69764,"cele":69765,"Ġtela":69766,"":69782,".setPreferredSize":69783,"ĠMID":69784,"ĠAless":69785,"Ġhorsepower":69786,"Ġatm":69787,"ĠPackaging":69788,"Ġciphertext":69789,"RequestMethod":69790,"Ġbeiden":69791,"è£":69792,"ĠPOW":69793,".WriteHeader":69794,"director":69795,"-but":69796,"ãģłãģķãģĦ":69797,"incer":69798,"_dn":69799,"!!!!!":69800,"Ġmanufactures":69801,".TextUtils":69802,"Ġconsciously":69803,"Ġbounced":69804,"culture":69805,"ĠSpar":69806,"ĠPiper":69807,".press":69808,"-owner":69809,"Ġevaluator":69810,"ĠSTREAM":69811,".PictureBoxSizeMode":69812,"Ġsugars":69813,"ScreenWidth":69814,"ĠnextState":69815,"Ġivory":69816,"Ġbrunch":69817,"density":69818,"_OW":69819,"ĠCoronavirus":69820,"ĠCFR":69821,"bak":69822,"\\Category":69823,"æķ°ç»Ħ":69824,"Ġinvokevirtual":69825,"}()Ċ":69826,"Ġsujet":69827,"-marker":69828,"isdigit":69829,"ĠMobil":69830,"ĠJsonRequestBehavior":69831,"_REMOTE":69832,".existsSync":69833,"Ġriches":69834,".presenter":69835,"ĠglColor":69836,"Ġhanya":69837,"Ġfortress":69838,"Ġflashed":69839,"viz":69840,"requently":69841,"buat":69842,"$con":69843,">|":69844,".Func":69845,"Ġhumorous":69846,"uem":69847,".ZERO":69848,"ĠSTL":69849,"ĠBuk":69850,"/sample":69851,"ĠGros":69852,"Recipes":69853,"Ġinflated":69854,"Ġswung":69855,":F":69856,"Facing":69857,".Theme":69858,"ник":69859,"Ġsplendid":69860,"ĠrequestId":69861,".CenterScreen":69862,"/autoload":69863,"embedded":69864,"_depart":69865,"ĠPorts":69866,"à¹ĥ":69867,"айд":69868,"discussion":69869,"_consum":69870,"Ġscouts":69871,"Ġcolabor":69872,".Stage":69873,".nano":69874,"eldorf":69875,"Ġgemacht":69876,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":69877,"Ġpolicymakers":69878,"_PKT":69879,",Th":69880,"oky":69881,"_UID":69882,"Ping":69883,"Ġorchest":69884,"Ġoptics":69885,"uhan":69886,"ĠXOR":69887,"Ġespañol":69888,"ĠAdidas":69889,"rng":69890,"mans":69891,".vstack":69892,"Ġgetaway":69893,"Ġhierarchical":69894,"anoia":69895,"ĠBitmapFactory":69896,"realm":69897,"ĉap":69898,"_apps":69899,"-divider":69900,".drawer":69901,"ĠHARD":69902,"'];?>Ċ":69903,"-packed":69904,"æ²»":69905,"_STRUCTURE":69906,"[Y":69907,"iParam":69908,"(eq":69909,"Ġencompasses":69910,"Ġ\\ĊĊ":69911,"->[":69912,"&utm":69913,"groupon":69914,"strate":69915,"DY":69916,"omorphic":69917,"':[":69918,"Ġgravitational":69919,"ĠMicha":69920,"ĠTencent":69921,"Ġcoached":69922,"ì¶ľ":69923,"ÑĥменÑĤ":69924,"/mobile":69925,"MouseDown":69926,"bud":69927,"ĠYas":69928,"ĠProviders":69929,"NZ":69930,"ĉreport":69931,"errmsg":69932,"ĠimagePath":69933,"acterial":69934,"ĠManga":69935,"wicklung":69936,"(usuario":69937,"\"));čĊčĊ":69938,"/***":69939,"Ġorganise":69940,"Indexed":69941,"_QUAL":69942,"(PyObject":69943,"Ġsurrendered":69944,"POCH":69945,"ĠNOTES":69946,"\\\\\"":69947,"-job":69948,"Ġseventy":69949,"####Ċ":69950,"ĠManor":69951,"Ġdownright":69952,"Ġtimeframe":69953,"insurance":69954,"checker":69955,"ĠSECRET":69956,"Ġechoes":69957,"ĠCarmen":69958,".setHorizontalAlignment":69959,"ĠisChecked":69960,"ĠTOR":69961,"_nn":69962,"('(":69963,"FetchRequest":69964,"ĠPrinted":69965,"Fluid":69966,"ĠSTACK":69967,"GES":69968,"aigned":69969,"igor":69970,".Unknown":69971,"CBC":69972,"ĠCarlson":69973,".URI":69974,"Ġplight":69975,"/start":69976,"ĠPersonnel":69977,"ĠPREFIX":69978,",**":69979,"Ġlimite":69980,"_heat":69981,"%ï¼Į":69982,"ĠDonne":69983,"getNode":69984,"ĠScientology":69985,"Ġcomet":69986,"Ġwenig":69987,"Aside":69988,"ĠMPEG":69989,"'?":69990,"variably":69991,".endDate":69992,"Ġuncont":69993,"ĠScores":69994,"ĠLoginForm":69995,".generated":69996,",ch":69997,"-mar":69998,"ĠNed":69999,"ĠeventId":70000,"+p":70001,"ĠSIN":70002,"/reset":70003,".REACT":70004,"ĠMessi":70005,"_RANK":70006,".writeFile":70007,"Ġcripp":70008,"esthetic":70009,"ERSIST":70010,"Ġreimbursement":70011,"CurrentValue":70012,"Ġunin":70013,"DownLatch":70014,"ĠpaddingRight":70015,"Ġstocked":70016,"/'.":70017,"Ġrepayment":70018,"trak":70019,"/backend":70020,"Ġизмен":70021,"CSR":70022,"Ġpreventive":70023,"Ġpantalla":70024,"_trim":70025,"Pedido":70026,"hospital":70027,"Ġmanageable":70028,"routeParams":70029,"textures":70030,"......ĊĊ":70031,"Ġsélection":70032,"NameValuePair":70033,"Ġpollut":70034,"Modes":70035,"ĠLaud":70036,"jay":70037,"ĠUrs":70038,"Ġsigner":70039,"ĠJJ":70040,"ĠCherokee":70041,"_EXISTS":70042,"Ġdwar":70043,"Ġ($('#":70044,"Ġreef":70045,">{$":70046,"ĠBaylor":70047,"ĠModelState":70048,"-_":70049,"ĠStructures":70050,"Ġsouvent":70051,"Specify":70052,"(pipe":70053,"Ġfracking":70054,"ĠGPA":70055,"Ġbele":70056,"ĉĉĉĉĉĉĉĠĠĠ":70057,"ĠMinority":70058,"Ġtud":70059,"Ġopenness":70060,"ĠIllustrated":70061,"Ġoxidation":70062,"ĠNK":70063,"ĉUpdate":70064,"ĠEMS":70065,"ĠTeddy":70066,"Ġgenerals":70067,"ĉMat":70068,"Ġradios":70069,"ĠAntique":70070,"conomy":70071,"ĠSquadron":70072,")','":70073,"声":70074,"Ġyoure":70075,"ĠMainPage":70076,"Ġbehaviours":70077,"enght":70078,"(@\"%@\",":70079,"Ġtestcase":70080,"ĠCompilation":70081,"Ġflavours":70082,"ĠExtend":70083,"illator":70084,"Ġcoh":70085,"Ġspline":70086,"ĠKG":70087,"-pay":70088,"Ġcommunism":70089,"ĠBusinesses":70090,"ocking":70091,".MaxLength":70092,"assandra":70093,"quiring":70094,"adden":70095,"ĠJeb":70096,"_fault":70097,"[file":70098,"Ġprominence":70099,"disciplinary":70100,"âĢĶthey":70101,"_extent":70102,"ĠVIC":70103,"Ġentails":70104,".partner":70105,"Ġhippoc":70106,"League":70107,"çĶ·":70108,"wipe":70109,"-spinner":70110,"Ġsalute":70111,"ĠSurgical":70112,"(outputs":70113,"worked":70114,"[strlen":70115,"appointed":70116,"ĠHeg":70117,"ĠACPI":70118,"([^":70119,"uala":70120,"_tol":70121,"ĠRit":70122,".Payment":70123,"kowski":70124,"Ġwalmart":70125,"requirements":70126,"ĠFINSEQ":70127,"_BACKGROUND":70128,"ĠOsborne":70129,"(errorMessage":70130,"Reporting":70131,"Ġauctions":70132,"Ġcombos":70133,"ĠNoticed":70134,"_oct":70135,"Ġprimero":70136,"taire":70137,"_hr":70138,"Ġмод":70139,"Ġcontradictory":70140,"=\"@":70141,"achines":70142,"(optarg":70143,"ĠPenguin":70144,"ĠAbbas":70145,"Ġsublime":70146,"Ġpageable":70147,"ĠDefensive":70148,"Ġdistinctly":70149,"ĠAutomatically":70150,"Understanding":70151,"EqualityComparer":70152,"gota":70153,"Ġ\"::":70154,"Ġpulver":70155,"ĠBattles":70156,"Ġunparalleled":70157,"TCHA":70158,"Ġconstrued":70159,"-aff":70160,"Ġprecursor":70161,"-lfs":70162,"Ġmaduras":70163,"ĠDaisy":70164,"ĠArbeits":70165,".Management":70166,"ĉIn":70167,"Ġrobes":70168,"Ġspéc":70169,"âĢľ(":70170,"Ġmaternity":70171,"extent":70172,"ĠSpacer":70173,"DidAppear":70174,"ĉus":70175,".getRequestDispatcher":70176,"(cols":70177,"Ġplummet":70178,"ìħ":70179,"Ġ{ĊĊĊĊ":70180,"érica":70181,"ĠSizes":70182,".enum":70183,".Highlight":70184,"Ġ!!}ĊĊĊ":70193,"Wenn":70194,"Ġclimax":70195,"Ġcrem":70196,"_that":70197,"[âĢ¦":70198,"_domains":70199,"_REPLY":70200,"Ġcompleta":70201,"VEST":70202,"_particle":70203,"Ġsop":70204,"Ġfatalities":70205,"implify":70206,"ĠSKF":70207,"Ġinfusion":70208,"ĠJavier":70209,"Ġballet":70210,"Ġamigo":70211,".want":70212,"Ġcollagen":70213,"ĠLawyer":70214,".Statement":70215,".rt":70216,"baar":70217,"EndPoint":70218,"ĠBek":70219,"SHIP":70220,"Ġpatriarch":70221,"ĠAunt":70222,"_TM":70223,"ĠmÃŃn":70224,"Ġmastered":70225,"WXYZ":70226,"Ġespos":70227,"=logging":70228,"Ġrighteousness":70229,"torrent":70230,"Ġbst":70231,"_CHAIN":70232,"Ġoutskirts":70233,"(rotation":70234,"Ġ'.')":70235,"igrants":70236,"+lsi":70237,"ĠCCTV":70238,"_PHASE":70239,".azure":70240,"_Process":70241,"vae":70242,"ĠTropical":70243,"ĠAnkara":70244,"imageView":70245,"_RUNNING":70246,"Ġ*)__":70247,"ến":70248,"(cli":70249,"scatter":70250,"Ġsche":70251,"Registrar":70252,"Ġairing":70253,"Ġpyplot":70254,"isión":70255,"/customer":70256,"Ġsimplement":70257,"Ġclassy":70258,"ĠDWC":70259,"ĠBashar":70260,"ĠDEVELO":70261,"ĠVick":70262,"avail":70263,"ĠHö":70264,"_extend":70265,"drFc":70266,".isNotBlank":70267,"Ġplais":70268,"|}Ċ":70269,"Ġpornofil":70270,"labs":70271,"Ġhaus":70272,"Ġoriginating":70273,"Ġsurrounds":70274,"ĠQUAL":70275,"meg":70276,"/logger":70277,"[obj":70278,"Ġirresponsible":70279,"ĠPublicKey":70280,"HONE":70281,":'/":70282,"ibox":70283,"ĠFVector":70284,"|{Ċ":70285,"ataloader":70286,"hawks":70287,"HDR":70288,"Ġescalation":70289,"ĠPodsDummy":70290,"elite":70291,"Ġpresup":70292,"Cached":70293,">G":70294,".optimizer":70295,"ĠVisible":70296,"´Ģ":70297,"Ġnen":70298,"Ġpcs":70299,"ĠIdle":70300,"[Any":70301,"Ġkeyboards":70302,"ĠCOMPONENT":70303,"Ġtitanium":70304,"(mut":70305,"ĠLedger":70306,"Ġprosperous":70307,"etrofit":70308,"_LL":70309,"_patient":70310,"Ġpdata":70311,"Ġkontakte":70312,"Swipe":70313,"Ġcheerful":70314,"ĠHonduras":70315,"\"][$":70316,"Ġhemorrh":70317,"\":\"+":70318,"Ġleasing":70319,"Ġinstalls":70320,"ĠPax":70321,"ĠLogistics":70322,"Ġkinetic":70323,"ĠPhon":70324,"_movement":70325,"ĉbytes":70326,"Ġcinco":70327,"ĠMadness":70328,"\")+":70329,"ĠJE":70330,"_ij":70331,"SceneManager":70332,"ĠBust":70333,"ptest":70334,"aea":70335,"Ġbesser":70336,"ÃŃg":70337,"дин":70338,"(tasks":70339,"(\"(\"":70340,"setType":70341,"(outfile":70342,"ĉreset":70343,"ĠARC":70344,"Ġmúsica":70345,"ĠShelf":70346,"ĠminY":70347,"pch":70348,"Ġweiber":70349,"issor":70350,"Ġtrouve":70351,"ĉButton":70352,"Ġregenerated":70353,"Å£i":70354,"imachinery":70355,"blocking":70356,".dataTables":70357,"_frac":70358,"ĠAdvantage":70359,".visitMethod":70360,"éĩįæĸ°":70361,"Ġextrapol":70362,"Ġteasing":70363,"ĠHitch":70364,"ĠGeek":70365,"ESCO":70366,"Ġwich":70367,"ĉax":70368,"_decor":70369,"ĠscreenWidth":70370,"ĠSophia":70371,"Forgot":70372,".uni":70373,"ĠVenture":70374,"_collision":70375,"Ġlawmaker":70376,"(Edit":70377,"blers":70378,"ĠgetNext":70379,"âĢĶyou":70380,"MediaPlayer":70381,"ĠHorde":70382,"ĠCongressman":70383,"observations":70384,"ĉproperty":70385,"Ġ<--":70386,"CreatedAt":70387,"ubyte":70388,"Ġquarantine":70389,"Ġdistressed":70390,"_APB":70391,"ĠGoodman":70392,"ãĤ«":70393,"Ġrecomend":70394,"_PRINTF":70395,"DONE":70396,"Bindable":70397,"rstrip":70398,"centaje":70399,"ĠUnexpected":70400,"ĠSCHOOL":70401,"ĠProfessionals":70402,"ĠGPUs":70403,"Lesson":70404,"Exclusive":70405,"Ġatrav":70406,"ĠDank":70407,"ĠLawyers":70408,"ĠWalton":70409,">[]":70410,"Ġaloud":70411,"=\"../../../":70412,"Ġdebating":70413,"ĠAVG":70414,"_VOL":70415,"/cgi":70416,".deg":70417,":g":70418,".Infof":70419,"MeasureSpec":70420,".song":70421,"mtree":70422,"ulls":70423,"Jordan":70424,"ĠCovers":70425,"Ġattributable":70426,"Ġjedis":70427,"iatrics":70428,"Ġrotterdam":70429,"Ġmeld":70430,"ĠContentType":70431,"Ġmantle":70432,"Ġalice":70433,"_duplicate":70434,"/Internal":70435,"Ġfilesize":70436,"ĉfire":70437,"rese":70438,"ondere":70439,"Ġfamiliarity":70440,"ĠCrest":70441,"Ġkarma":70442,"Ġtorino":70443,"Ġmesa":70444,"/temp":70445,"Ġchir":70446,"ĠOverflow":70447,"Ġtenemos":70448,"unik":70449,"NEXT":70450,"Alle":70451,"Ġnxt":70452,"Mart":70453,"Ġatl":70454,"Ġperiodo":70455,"_you":70456,"Ġ})).":70457,"intestinal":70458,".AdapterView":70459,"Ġhesitant":70460,"Ġcomparatively":70461,".UInt":70462,"(viewModel":70463,"Ġsangat":70464,"ĠResponsive":70465,"ĠZack":70466,"âħ":70467,"JAVA":70468,"ĠFuller":70469,"ĠâĿ¤":70470,".Consumer":70471,"Ġank":70472,"Ġreactors":70473,"fuck":70474,"_rat":70475,"ĠsessionFactory":70476,"_backward":70477,"Ġscrambled":70478,"ĉth":70479,"Ġinsensitive":70480,"Ġchamps":70481,"Ġnginx":70482,"Ġconhec":70483,"ĠJasper":70484,".fm":70485,"StrictEqual":70486,"achsen":70487,"-Nov":70488,"lassen":70489,".integration":70490,"(lbl":70491,"Compose":70492,"ĠFon":70493,"Ãļ":70494,"Gratis":70495,"ĠLime":70496,"ĠAdapterView":70497,"Ġpoisoned":70498,"anchors":70499,"设计":70500,"']?>\"":70501,"Ġprocur":70502,"Italy":70503,".MONTH":70504,"ĠLUA":70505,"ĠLithuania":70506,"ĠHeads":70507,"_CHUNK":70508,"ĠPUSH":70509,"AspectRatio":70510,"Ġweg":70511,"Ġvids":70512,"ĠWein":70513,"ĉINT":70514,"sessionId":70515,"Industry":70516,"Ġdenounced":70517,"JKLM":70518,"ĠVanessa":70519,".Identifier":70520,"propri":70521,"Ġиг":70522,"Ġtécn":70523,"Ġmosaic":70524,"StreamReader":70525,"-Th":70526,"forth":70527,"Ġadherence":70528,"bate":70529,"Ġknights":70530,"sounds":70531,"Ġsalle":70532,"OMET":70533,"ãĤ¹ãĥĪ":70534,"-tm":70535,"ĠRhe":70536,".FileOutputStream":70537,"åĪĨç±»":70538,"ĠENG":70539,"holiday":70540,"ĠCongratulations":70541,")(Ċ":70542,"Ġaggregates":70543,"HOOK":70544,"ewire":70545,"Senator":70546,"Ġembeddings":70547,"epy":70548,"(COM":70549,"Ġrobber":70550,"äter":70551,"wang":70552,"_teacher":70553,"Ġresentment":70554,"Ġlettuce":70555,"erreur":70556,"(ic":70557,"ĠTactical":70558,"ĠContracts":70559,"Ġmænd":70560,"Ġsitios":70561,"Ġbastante":70562,"Ġnuevos":70563,"ĉNdrFc":70564,"ĠprivateKey":70565,"ucch":70566,"MMdd":70567,"Ġè¾ĵåĩº":70568,"umba":70569,"@foreach":70570,":\");ĊĊ":70571,"Ġslippery":70572,"ĠKeystone":70573,"Ġpioneering":70574,"_triangle":70575,"(\"Ċ":70576,"ĉĉĉĉĉĉĉĉĠĠ":70577,"ĠIntervention":70578,"SCI":70579,"ĠcJSON":70580,"Ġterminating":70581,"ë¹Ħ":70582,"Ġbabys":70583,"Subset":70584,"Ġë¡":70585,"Ġseulement":70586,"Ġmuestra":70587,"Entre":70588,"以ä¸Ĭ":70589,"ngo":70590,"\"bytes":70591,"QRST":70592,"Ġypos":70593,"persona":70594,"ĠDeploy":70595,"cee":70596,"Ġà®":70597,".goal":70598,"Ġhabitats":70599,"ĠisAdmin":70600,"Ġexploiting":70601,"Ġventil":70602,"ĠBalls":70603,"اب":70604,"Ġmindfulness":70605,"(kwargs":70606,"Ġresembling":70607,"Ġchoir":70608,"ĠonBackPressed":70609,"ĠSECURITY":70610,"/gtest":70611,"Ġjustices":70612,"ĠintegerValue":70613,"blah":70614,"ĠAim":70615,"_finalize":70616,"keh":70617,"ĠComplexity":70618,"Ġaugust":70619,"getElementsByTagName":70620,"Ġpreach":70621,"Ġpronunciation":70622,"ĠTrash":70623,"-percent":70624,"_PRIV":70625,"ĠHunts":70626,"ĠCurse":70627,"uellen":70628,"Ġheavyweight":70629,"Xi":70630,"ĉselected":70631,"ĠMcCoy":70632,"å¼Ĥ常":70633,"|=Ċ":70634,"ĠBattlefield":70635,"ItemImage":70636,"Ġdeductions":70637,"ĠElemental":70638,"());//":70639,"ĠBurk":70640,"})čĊčĊ":70641,"swift":70642,"/function":70643,"Usually":70644,"_St":70645,"_feats":70646,"ĠIsValid":70647,"Ġzad":70648,"ImageContext":70649,"Ġclassname":70650,"Ġdonner":70651,"Ġ-->ĊĊĊ":70652,"Ġmotorcycles":70653,"+'/'+":70654,"ĠsetBackground":70655,"\\CMS":70656,".AllArgsConstructor":70657,"ĠLexington":70658,".examples":70659,"ĠPurs":70660,"PushMatrix":70661,"Ġ==============================================================":70662,".addTarget":70663,"pora":70664,"Fullscreen":70665,"Ġgoof":70666,"hlen":70667,"äge":70668,"ĠCURL":70669,"ĠInteresting":70670,"Ġretrieves":70671,"_Obj":70672,"inness":70673,"-----ĊĊ":70674,".tsv":70675,"(IM":70676,"ĠBraves":70677,"_ISR":70678,"osti":70679,"á»ĵ":70680,"ĠExterior":70681,"ĠCourtney":70682,"Ġresidues":70683,"Tier":70684,".*;čĊčĊ":70685,":black":70686,"webView":70687,"\"path":70688,"Ġmasa":70689,"]!='":70690,"ĠMatching":70691,"dur":70692,"Jvm":70693,"=context":70694,"_RING":70695,"Ġproponents":70696,"ĠQStringLiteral":70697,"Ġinflate":70698,"\">čĊ":70931,"_COST":70932,"ilinear":70933,"ĠWorkspace":70934,"Ġspel":70935,"agogue":70936,"ĠMillennium":70937,"ĠPopulate":70938,"Ġnid":70939,".parseColor":70940,"Solar":70941,"ĠGad":70942,"Ġì¤ij":70943,"ĠKamp":70944,"ĉrm":70945,"Ġbenz":70946,"ĠHonestly":70947,"Ġelectrode":70948,"ĠPrairie":70949,"ĠPROFILE":70950,"ĠOriental":70951,"ĠOLED":70952,"/copyleft":70953,"awaii":70954,"(products":70955,")\\<":70956,"-created":70957,".ManyToMany":70958,"\"How":70959,"ĠвÑĭп":70960,"Ġmitochondrial":70961,"_testing":70962,"(created":70963,"ĠgetField":70964,"_EVAL":70965,"].\"":70966,"ĠFSM":70967,"ĠRita":70968,"ĠåıĤæķ°":70969,"Ġcôt":70970,"ĠInsight":70971,"ĉmysqli":70972,"_timing":70973,"IDO":70974,")))))Ċ":70975,"COVERY":70976,".imag":70977,"CDF":70978,"lust":70979,"ickt":70980,"_FP":70981,".','":70982,"gcc":70983,"Ġkurz":70984,"_pwm":70985,"Ġodpowied":70986,"ĠBarrier":70987,"/***************************************************************************Ċ":70988,"pak":70989,"-Israel":70990,"ĠRutgers":70991,"ĠselectedItem":70992,"ĠRamirez":70993,"Farm":70994,"Ġcalendars":70995,"gzip":70996,"Ġblockbuster":70997,"ĠPlymouth":70998,"çľĮ":70999,"responses":71000,".DialogInterface":71001,"-grand":71002,"ĠgetSource":71003,"Ġdejtings":71004,"Ġtieten":71005,"Ġcondemnation":71006,"Ġcontinuar":71007,".MockMvc":71008,"/english":71009,"ĠMediaPlayer":71010,"computed":71011,"ĠClippers":71012,"(delegate":71013,".Slf":71014,"Ġë¡ľ":71015,"ĠTide":71016,"Ġihrem":71017,"ĠWan":71018,"ÑĥÑİÑī":71019,"}><":71020,"Discussion":71021,"Ġwatts":71022,"-minus":71023,"ĠJuliet":71024,"éĽħ":71025,"Ġconcluding":71026,"andscape":71027,"Ġúltima":71028,"ĠDERP":71029,"ĠsignUp":71030,"ĠSecondly":71031,"WAIT":71032,"lds":71033,".callbacks":71034,"(hour":71035,"imators":71036,"volent":71037,"AAF":71038,"edriver":71039,"ĠMathematic":71040,"'":71042,"{j":71043,"_ABORT":71044,"Ether":71045,"Ġeducator":71046,"Ġprecaution":71047,"Ġfingertips":71048,"getVar":71049,"camatan":71050,"-debug":71051,"ĠRAF":71052,"[arg":71053,"Ġraced":71054,"Ġtsunami":71055,".flink":71056,"Ġglyc":71057,"uko":71058,"ĠMultiply":71059,"Ġredistribution":71060,"AGO":71061,"ĠRoutine":71062,"Ġopr":71063,"(lower":71064,"ĠFunktion":71065,".dk":71066,"Ġegt":71067,"_BASIC":71068,"syscall":71069,"ĠLSD":71070,"ĠDuplicate":71071,"_sell":71072,"ĠerrorHandler":71073,"_ips":71074,"Ġerv":71075,"annie":71076,"(resourceName":71077,"Ġbottled":71078,"Ġcrawling":71079,"egment":71080,".setTag":71081,"Ġrss":71082,"ĠQuarry":71083,"_exact":71084,".jwt":71085,"ĠBoards":71086,"opi":71087,"Ġnasal":71088,"ĠXYZ":71089,".ud":71090,"Northern":71091,"Ġactivating":71092,"edx":71093,"ovah":71094,"Ġindx":71095,"AlertDialog":71096,"Ġtienes":71097,"annya":71098,"_pan":71099,"(decimal":71100,".Dict":71101,"Ġsubsidiaries":71102,"ProductName":71103,"Few":71104,"dato":71105,"odied":71106,"-under":71107,"Ġê²ĥ":71108,"çīĪæľ¬":71109,"atism":71110,"[Math":71111,".'<":71112,"(infile":71113,"Ġdenotes":71114,"$class":71115,"_SECURITY":71116,"Ġsewage":71117,"melon":71118,"(Character":71119,"/github":71120,"Ġglaring":71121,".Guid":71122,"_sparse":71123,"ĠMargin":71124,"_dns":71125,"Ġmeiner":71126,"Ġleftist":71127,"ĉloc":71128,"abytes":71129,"Ġequipments":71130,"expo":71131,"ĠSomerset":71132,"EK":71133,"æį¢":71134,"Ġlecturer":71135,"Ġmemiliki":71136,"æł¸":71137,"ç´ł":71138,"pron":71139,":pointer":71140,"borrow":71141,"ĠProtective":71142,"_cf":71143,"ĠÐķÑģли":71144,"bpp":71145,"';ĊĊĊĊ":71146,"aturally":71147,"_NAV":71148,"Ġpeptide":71149,">d":71150,"Ġifstream":71151,"_FACTORY":71152,"');//":71153,"joined":71154,"mong":71155,"Ġtimespec":71156,"Ġdestabil":71157,"Ġautop":71158,"-limit":71159,"publication":71160,"ĠDenn":71161,".Memory":71162,"(skb":71163,"ĠAnaheim":71164,"_RETURNTRANSFER":71165,"oueur":71166,"(_('":71167,"legt":71168,"istingu":71169,"ĉpriv":71170,"Ġredirects":71171,"Mt":71172,"Ġalleen":71173,"ĠPointF":71174,"Ġomin":71175,"Ġcitt":71176,"ĠTage":71177,"ĠWalls":71178,"á»ī":71179,"Ġoccupying":71180,"xBF":71181,"rangle":71182,"Ġrelational":71183,"-org":71184,"Ġjpg":71185,"-derived":71186,"Ġmalfunction":71187,"ĠBenson":71188,"(scroll":71189,"ĠXD":71190,"Holy":71191,"(commands":71192,"Ġtipping":71193,"Ġprimitives":71194,"Ġsexle":71195,"CallCheck":71196,"ĠMASTER":71197,"_TEAM":71198,".setRequestHeader":71199,"_specs":71200,"Ġserge":71201,".Master":71202,"Ġims":71203,".SpringBootTest":71204,"paypal":71205,"ĠWANT":71206,".Inst":71207,"ĠCarpet":71208,"Ġwrongly":71209,"($('.":71210,"Ġbild":71211,".Roll":71212,"ĠUrb":71213,"-can":71214,"ãģıãģłãģķãģĦ":71215,"oliberal":71216,"čĊčĊ":71610,"ĠMahm":71611,"}\";ĊĊ":71612,"Ġdq":71613,"ĠPublishers":71614,"ĠAmpl":71615,"ĠDanielle":71616,"Ġtern":71617,"èµ·":71618,"noÅĽÄĩ":71619,"ein":71620,"ĠAsyncStorage":71621,"unger":71622,"rouw":71623,"Ġscissors":71624,"/assert":71625,".bucket":71626,"/archive":71627,"_Man":71628,"Ġintoler":71629,"Ġ()=>":71630,"ĠÐĴÑĭ":71631,"Ġsai":71632,".xy":71633,".\"čĊ":71634,"Ġurinary":71635,"esub":71636,"ISTICS":71637,"Ġκ":71638,"Ġcompliments":71639,"ĠtypingsJapgolly":71640,"ihar":71641,"Expansion":71642,"ĠServing":71643,"_students":71644,"ĠXBOOLE":71645,"(il":71646,"Ġì²ĺ":71647,"Ġjó":71648,"(tol":71649,"(JS":71650,"ĉCG":71651,"ĠDRAW":71652,"twig":71653,"Ġoat":71654,"_smooth":71655,"ĠCSL":71656,"Ġosob":71657,"Ġensuing":71658,"Ġbanker":71659,"ĠBackpack":71660,"_ping":71661,"Ġwishlist":71662,"=ax":71663,"ĉĠĠĠĊ":71664,"Disney":71665,"steady":71666,"\">%":71667,"Ġprophets":71668,"ĠZX":71669,"Ġminimalist":71670,".PLAIN":71671,"Seattle":71672,".ordinal":71673,"ĠPIPE":71674,"Ġretorna":71675,"Ġjugador":71676,"ĠBret":71677,"ĠâĶľ":71678,"Ġplush":71679,"ULATOR":71680,"Sorting":71681,".gridy":71682,"ectomy":71683,"_activ":71684,"rack":71685,"Interactive":71686,"ĠAntarctica":71687,"Ġvengeance":71688,"enso":71689,"_known":71690,"upplier":71691,".Modules":71692,"ĠConnectionState":71693,"éļIJèĹı":71694,"@FindBy":71695,"Ġplacer":71696,"\\model":71697,"<()>":71698,".isSuccessful":71699,"-good":71700,"bz":71701,"ĠDraco":71702,"Assistant":71703,"-extra":71704,"аблиÑĨ":71705,"Ġhypocrisy":71706,"Ġtst":71707,"ĠAgr":71708,"$txt":71709,"Ġlogistic":71710,"licensed":71711,"ĠHof":71712,"Ġtat":71713,"(iv":71714,"Ġintoxic":71715,"postId":71716,"_strike":71717,"Ġhumiliation":71718,"pcodes":71719,"\"sync":71720,"(recipe":71721,"+N":71722,"rente":71723,"ĉClient":71724,"ycopg":71725,"ĠZurich":71726,"ĠProfiles":71727,"Countries":71728,"Ġpict":71729,"Ġrollout":71730,"requencies":71731,"Ġpatched":71732,"Ġcartridges":71733,"Ġshading":71734,"Jar":71735,"Ġsalvage":71736,"ĠTaxes":71737,"Ġstandby":71738,"aporan":71739,"Eigen":71740,".angular":71741,"ĠNested":71742,"享":71743,"ĠisVisible":71744,"ĠDwight":71745,"_BRANCH":71746,".Delay":71747,"Ġkend":71748,"Ġfacilitated":71749,".flatMap":71750,"Ġsanta":71751,"ĉSend":71752,"/messages":71753,"ĠofType":71754,"ĉswap":71755,"#plt":71756,"ĠTurks":71757,"NES":71758,"Ġprogressively":71759,"ĠResidence":71760,"ĠTREE":71761,"Ġnoen":71762,"dio":71763,"Ġnelle":71764,"Ġsogar":71765,"itti":71766,"weekly":71767,"Ġambiguity":71768,"_Settings":71769,"Ware":71770,".neo":71771,"_DST":71772,"Ġæĸ¹":71773,"prep":71774,"lobby":71775,"@email":71776,"/movie":71777,"Ġfunkc":71778,"ĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĠĊ":71779,"ÂŃs":71780,"Ġguardians":71781,"-pos":71782,"Ġconfiguring":71783,"ĠCPS":71784,"ĠDeus":71785,"Ġvidéos":71786,"_empresa":71787,"Ġslapped":71788,"',Ċ":71820,"_XDECREF":71821,"ĠBuzzFeed":71822,"_MARGIN":71823,"PLOY":71824,".small":71825,"ĠmimeType":71826,"Ġholog":71827,"ĉcamera":71828,"lias":71829,"Ġsuspense":71830,"odynam":71831,"bau":71832,"Ġgraveyard":71833,"_named":71834,"\":\"'":71835,"Ġ************************************************":71836,"ĠgameOver":71837,"ĠLENGTH":71838,"ĉscreen":71839,"ĠdoInBackground":71840,"_dependencies":71841,"Ġrtc":71842,"/up":71843,"_ROM":71844,"Hall":71845,"Ġdeficiencies":71846,"(te":71847,"'#":71848,"_equiv":71849,"Ġpreorder":71850,"ĠAxe":71851,"омÑĥ":71852,".sendFile":71853,"Ġfilt":71854,"ĠLimits":71855,"ĠCavaliers":71856,".discount":71857,"âĨIJ":71858,"ĠWit":71859,"QRSTUV":71860,"Ġij":71861,"Ġtegen":71862,"Ġ:\",":71863,"difficulty":71864,"punkt":71865,"ĠEmails":71866,"chlor":71867,"(fun":71868,".Uint":71869,"ĠStall":71870,"_verified":71871,"uD":71872,"FileType":71873,"Ġpleasures":71874,"Ġjudiciary":71875,"Ġsham":71876,"ipur":71877,"_PLUS":71878,"offers":71879,"(foo":71880,"_GT":71881,"ĉcore":71882,"ENTION":71883,"ĠLiberation":71884,"CommandLine":71885,"_department":71886,".Ar":71887,"_neighbor":71888,"ĠSubmitted":71889,"ĠĊ":96121,"Ġdroits":96122,"Ġhomosexuals":96123,"Ġabduction":96124,"ĉwidget":96125,"$headers":96126,"ĠDAR":96127,"Ġfla":96128,"threat":96129,"Ġlouis":96130,".GetProperty":96131,"\"Just":96132,"(frames":96133,"ryo":96134,"profession":96135,"|i":96136,"íķ´ìĦľ":96137,"(sv":96138,"Ġunrecognized":96139,"Ionic":96140,"Fashion":96141,"ScreenState":96142,"ĠIncoming":96143,"NotNil":96144,"Ġsyncing":96145,"emie":96146,"Ġthermo":96147,"_procs":96148,"Ġinconsistency":96149,"religious":96150,".mj":96151,"Ġpersonn":96152,"Ġmomentos":96153,"orarily":96154,"ĠæĬ":96155,"_neurons":96156,"Illustr":96157,"imoto":96158,"ilik":96159,"ĠWoj":96160,"Trading":96161,"Ġappare":96162,"Ġentreprises":96163,"achat":96164,"Ġ¬":96165,"Ġneigh":96166,"BUTTONDOWN":96167,"ĠMaher":96168,"aghan":96169,"-hash":96170,"\"f":96171,"Ġclientele":96172,".addButton":96173,"ĉSP":96174,"Qi":96175,"Ġgrated":96176,"POSITE":96177,":>":96178,"ĠHowell":96179,"ĠComparative":96180,"ĠISC":96181,"ÂŃi":96182,"Ocean":96183,"Davis":96184,"ĠFilme":96185,"Wins":96186,"ĠJIT":96187,"occer":96188,"ĠCorm":96189,"ENCHMARK":96190,"rchive":96191,"icação":96192,"Ġmata":96193,"Ġchildbirth":96194,"ĠOptionally":96195,"Ens":96196,"Ġxhttp":96197,"Ġelucid":96198,"_OscInitStruct":96199,"))):Ċ":96200,"Ġintuit":96201,"ĠDonate":96202,"Ġcorrelates":96203,">Delete":96204,"Ġequipe":96205,"Ġboca":96206,"Ġinflatable":96207,"erah":96208,"ĠDateTimeKind":96209,"Ġcalves":96210,"\\Lib":96211,"Ġemlrt":96212,"ĠTrilogy":96213,"ĠPanc":96214,"ĠDuis":96215,"ĠpelÃŃcula":96216,"WARDS":96217,"_DETECT":96218,"-sectional":96219,"dhcp":96220,"ForRow":96221,"-destruct":96222,"ĠPresenter":96223,"/slick":96224,",on":96225,"ĠCitadel":96226,"loggedin":96227,"_subtype":96228,"Ġsigue":96229,"Ġcuring":96230,"ĠFirewall":96231,"Ġfluorescence":96232,"ĠItalians":96233,"иÑĤÑģÑı":96234,".getStyle":96235,"InSeconds":96236,"jie":96237,"-Smith":96238,"Ġxlink":96239,"Ġsubmissive":96240,"онÑĤ":96241,"arbonate":96242,"ĠFaul":96243,"_goals":96244,"ĠCommissioners":96245,"chartInstance":96246,"_POSTFIELDS":96247,"Ġmedial":96248,"Ġmanos":96249,"Ġdelt":96250,"svm":96251,".Apis":96252,"ephy":96253,"Ġasympt":96254,"ĠappDelegate":96255,"Ġimprobable":96256,"cka":96257,"simd":96258,"/Error":96259,".âĢĵ":96260,"ĠPTS":96261,"deer":96262,"Ġsina":96263,"magnitude":96264,"IDADE":96265,"']}'":96266,"Ġmayores":96267,"ĉcomment":96268,"/console":96269,"\"@":96270,"volt":96271,".sell":96272,"ĠMacy":96273,"Ġmelod":96274,"Ġimágenes":96275,"_chg":96276,"Ġinout":96277,"idente":96278,")'),Ċ":96279,"dni":96280,".blob":96281,"Ġtypography":96282,"Ġeerie":96283,"_OID":96284,"pesan":96285,"ajan":96286,"Ġchopping":96287,"Ġbluff":96288,"adf":96289,"_bases":96290,".Formatter":96291,"Ġ\\%":96292,"ĠPageInfo":96293,"Carrier":96294,"ĠCalibration":96295,"como":96296,"-bodied":96297,"Ġfinancier":96298,"ĠINA":96299,".ERR":96300,"Ġhoodie":96301,"ĠSanity":96302,"guarded":96303,".opendaylight":96304,"ISMATCH":96305,"Highlights":96306,"ünk":96307,"aniem":96308,"angered":96309,"assignments":96310,"Ġregistrado":96311,"ĠUPPER":96312,"ampilkan":96313,"ashire":96314,"ĠNikola":96315,"ĠCFL":96316,"ĠHDC":96317,"Ġpoids":96318,"ĠIPs":96319,"Ġpreventative":96320,"ipsoid":96321,"ifix":96322,".camel":96323,".ga":96324,"Volumes":96325,"-ste":96326,"Yahoo":96327,"_sibling":96328,"Highest":96329,"optgroup":96330,"Ġkvinna":96331,"âĢĿãĢĤĊĊ":96332,"ĠAppliances":96333,"Ġ\"><":96334,"')\")Ċ":96335,"htt":96336,"ĠIdentified":96337,"Ġpencils":96338,"ĠmemberId":96339,"ĠappendString":96340,".loadData":96341,"ĠmockMvc":96342,"Ġjub":96343,"ĠSlut":96344,"ĠTaipei":96345,"statt":96346,"Polit":96347,"Ġpartager":96348,"DidChange":96349,"Increases":96350,")}.":96351,"ĠBaba":96352,"_CLIP":96353,"[unit":96354,"ĠклÑİÑĩ":96355,"Ġalcuni":96356,"ĠLola":96357,"Ġclinging":96358,"@PostMapping":96359,"(concat":96360,"Ġssid":96361,"ĠFauc":96362,"okit":96363,"ĠRecorded":96364,"ález":96365,"($('<":96366,".assertIsNot":96367,"Ġkali":96368,"Volt":96369,"Ġwarmly":96370,"Ġscares":96371,"getti":96372,"führt":96373,"_does":96374,".EMAIL":96375,"imations":96376,"Ġspringfox":96377,"ĠDecom":96378,"arcy":96379,"Ġglitches":96380,"ĠMoff":96381,"ĠVoll":96382,".between":96383,"Ġcoorden":96384,"ĠParticularly":96385,"GBP":96386,"Ġsemble":96387,"Eastern":96388,"_MSB":96389,"]){čĊ":96390,"morgan":96391,"ĠEVAL":96392,"dere":96393,"HOUSE":96394,"moire":96395,"istique":96396,"_lstm":96397,"-commit":96398,"ysterious":96399,"Ġtwink":96400,"-thumbnails":96401,"enÃŃ":96402,":'',":96403,"Ġblackout":96404,"ĠFloors":96405,"Ġsofas":96406,"Ġoui":96407,"leshoot":96408,"ĠRaq":96409,"-abs":96410,"Ġkra":96411,"Mining":96412,"shaft":96413,".setColumns":96414,"Clazz":96415,"PRETTY":96416,".playlist":96417,"éĸ¢":96418,"-Saharan":96419,"MING":96420,"ĉbl":96421,"è®®":96422,"jf":96423,"DOCKER":96424,"hopefully":96425,"(ignore":96426,"ĠUsersController":96427,"ĠMitarbeiter":96428,"ĠLES":96429,"Hamilton":96430,"-metadata":96431,"ĠKK":96432,"iktig":96433,"Ġwollte":96434,"egrator":96435,"]bool":96436,",current":96437,"ĠvalueType":96438,"Ġexcavation":96439,"oland":96440,"Ġverv":96441,"/filepath":96442,"AuthProvider":96443,"Ġprocrast":96444,"ĉULONG":96445,"_MEMBERS":96446,"Ġuplift":96447,"ĠAutonomous":96448,"Ġartworks":96449,"ĠOutreach":96450,"Ġpore":96451,"Homepage":96452,"DialogTitle":96453,"ĠGenerating":96454,"PARSE":96455,"Ġsemanas":96456,"Ġhumano":96457,"JSGlobalScope":96458,"Ġvolte":96459,"Ġbella":96460,"(isinstance":96461,"Ġplc":96462,"\\Catalog":96463,"Ġesteemed":96464,"鼷":96465,"(suffix":96466,"Ġsweeps":96467,"ĉORDER":96468,"Ġdoivent":96469,"ĠSwarm":96470,"ĠCompiled":96471,"getPage":96472,"ADR":96473,".RichTextBox":96474,"ĠNaming":96475,"agged":96476,"ĠGANG":96477,"rasing":96478,"odeled":96479,"Ġgala":96480,"ĠJSName":96481,"ddf":96482,"Ġillust":96483,"ĠLansing":96484,"[port":96485,"-death":96486,"Ġdinheiro":96487,"ĠEighth":96488,"Ġbian":96489,"stÃ¥":96490,"Ġversión":96491,"ĠLinearGradient":96492,"ĠHarding":96493,".*)":96494,"eczy":96495,"$header":96496,"ĠvÃ¥r":96497,"Unchecked":96498,"Ġkoje":96499,"ĠPaladin":96500,"())),":96501,"Giving":96502,"()})Ċ":96503,"Ġdips":96504,"Friendly":96505,"Ġportrays":96506,"Ġhelium":96507,"Ġinsurgency":96508,"_expiry":96509,"ĠstringByAppendingString":96510,"Ġaantal":96511,"slope":96512,"mast":96513,".getInteger":96514,"Ġ########################":96515,"_PIPELINE":96516,"Ġdensely":96517,"Ġmutating":96518,"midi":96519,"ĠSeit":96520,"ayne":96521,"NOWLED":96522,"ĠDesmond":96523,"ĠFName":96524,"ĠNairobi":96525,"\\Context":96526,"Ġcalcular":96527,"-den":96528,"Ġcott":96529,"]):čĊ":96530,"ĠRecommendation":96531,"ĠRolex":96532,"ĠvalidationResult":96533,".pat":96534,"ĠnÃły":96535,"ĠRestClient":96536,"ĠGPI":96537,"ĠAsheville":96538,"ĠOSP":96539,"ĠPERMISSION":96540,"ÐĶаÑĤа":96541,"/notification":96542,"Knight":96543,"_Word":96544,"ĠBender":96545,"ranking":96546,"Ġpartida":96547,"_reservation":96548,"ÌĢ":96549,"ĠmName":96550,"Ġgetch":96551,"Ġborr":96552,"Ġdiligent":96553,"Discuss":96554,"æŃ£åľ¨":96555,"apeake":96556,"ioned":96557,"-Nazi":96558,".cum":96559,"ĠKron":96560,"=$('#":96561,"/single":96562,"Ġerotisch":96563,"ĠVib":96564,"Ġratified":96565,"Ġconcerted":96566,"ĠREGARD":96567,"Ġdobr":96568,".DriverManager":96569,"'r":96570,"Portable":96571,"ĉsuite":96572,"Ġrelaciones":96573,"ĠDop":96574,"emploi":96575,"DOB":96576,"Ġcrumbs":96577,"Ġxls":96578,"_Application":96579,"(':',":96580,"Ġ------------------------------------------------------------------------Ċ":96581,"mse":96582,"Ġberk":96583,"ĠReturnValue":96584,"ĠBelly":96585,"Ġcamar":96586,"ĠPeek":96587,"elsing":96588,"Ġnotifies":96589,"ĠTristan":96590,"ĠGAR":96591,"emme":96592,"ĠElevated":96593,"_CSV":96594,"(chalk":96595,"Ġtwenties":96596,"ĠSearchResult":96597,"=search":96598,"ĠMixing":96599,"ýt":96600,"Ġrecruiter":96601,"ĠIDEOGRAPH":96602,"ĠAgo":96603,"(Operation":96604,"$values":96605,"Ġworldly":96606,"ĠRosenberg":96607,"ĠConfigureServices":96608,">*Ċ":96705,"Ġsnork":96706,"_opacity":96707,"ĠinitWithNibName":96708,"iado":96709,"AAC":96710,"Ġ]).":96711,";z":96712,"_paragraph":96713,"Ġnoses":96714,"stands":96715,"ifr":96716,"_mE":96717,"Iraq":96718,".Predicate":96719,"enaire":96720,"]]];Ċ":96721,"Ġunidad":96722,"Ġretirees":96723,"_hello":96724,"Ġmodele":96725,"ĠUITableViewController":96726,"fwrite":96727,"_numero":96728,"_visited":96729,"Ġrecebe":96730,"(Notification":96731,"Fantastic":96732,"_submenu":96733,"ĠPEM":96734,"ĠCupertino":96735,"approximately":96736,"classed":96737,".ReadString":96738,"Ġdomicile":96739,"_PW":96740,"Ġballpark":96741,"ĠKale":96742,"contra":96743,"_favorite":96744,"/of":96745,"Quite":96746,"ĠOTA":96747,"Ġaccelerometer":96748,"didn":96749,"|^":96750,"ĠRohingya":96751,"ivicrm":96752,"annabin":96753,"обÑĭÑĤи":96754,"orado":96755,"')+":96756,"Haunted":96757,",ID":96758,"(UIAlertAction":96759,"urv":96760,"_bel":96761,"ĠMexicans":96762,"/terms":96763,"ĠPainter":96764,"InputLabel":96765,"ĠVinci":96766,"ĠRosie":96767,"\\uc":96768,"":96929,"_gs":96930,"Ġcompil":96931,"nard":96932,"-exc":96933,"Ġrhyme":96934,"Ġbutto":96935,"says":96936,"antasy":96937,"ë¸":96938,"ĠcittÃł":96939,"Ġcheg":96940,"TimeString":96941,"Ġpositivity":96942,"ĠDabei":96943,"Ġwang":96944,"Ġescre":96945,"\"c":96946,"ĉvideo":96947,"ĠRanked":96948,".strings":96949,">>>(":96950,"ĠинÑĤеÑĢ":96951,"Ġresta":96952,"[:,:":96953,"Ġrendre":96954,"Ġdeser":96955,"Jos":96956,"Ġdisruptions":96957,"ĠопеÑĢ":96958,"sampling":96959,"suppress":96960,"ĠcontainerView":96961,"ĠSeamless":96962,"Ġairy":96963,"Ġonload":96964,".WindowManager":96965,"ĠPLA":96966,"braco":96967,".setPositiveButton":96968,"Ġpdu":96969,"Ġgsi":96970,"ĠCli":96971,"_gradients":96972,"Ñıд":96973,"ĠWhisper":96974,"cstdint":96975,"Ġläng":96976,"Ġformulations":96977,"énom":96978,"ournemouth":96979,"[$_":96980,"Ġordinarily":96981,".setUsername":96982,"Ġfaculties":96983,"MITTED":96984,"/values":96985,"Ġweir":96986,"ĠApt":96987,"MZ":96988,"ĉcf":96989,"ucken":96990,"ĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉĉ":96991,"defense":96992,"[iVar":96993,"ĠBusinessException":96994,"Selectors":96995,"(coordinates":96996,"ĠResets":96997,"ĠDrinks":96998,"oleans":96999,"(stypy":97000,"_IOC":97001,".xxx":97002,"ĠSlater":97003,"ĠBelize":97004,"Ġ/************************************************************************":97005,"addin":97006,"_episodes":97007,"Ġischem":97008,"legalArgumentException":97009,"Danny":97010,"Ġpared":97011,".codehaus":97012,"ĠAssy":97013,"ĉRect":97014,"âŀ":97015,".lista":97016,"ĠваÑĪ":97017,"Ġvets":97018,"HWND":97019,"isoner":97020,"Ġxo":97021,"Ġorally":97022,"ĠStmt":97023,".rnn":97024,"ĠDPI":97025,"ĠStrikes":97026,".setViewportView":97027,"ĠèĩªåĬ¨çĶŁæĪIJ":97028,"YELLOW":97029,"GLenum":97030,"partners":97031,"ĠImplicit":97032,"Ġtako":97033,"âĢĻelle":97034,"Ġermög":97035,"totalCount":97036,"Gil":97037,"ĉwork":97038,"Ġpratic":97039,"inati":97040,"abies":97041,"ĠSkinner":97042,"Ġspirited":97043,"Ġpancreatic":97044,"Ġhdf":97045,"'em":97046,"Ġpsychosis":97047,"olicit":97048,"Ġ\"{\"":97049,"_atual":97050,"Ġélect":97051,"TEAM":97052,"Ġdak":97053,"ĠSWAT":97054,".FragmentManager":97055,"Ġprovisioning":97056,"lifetime":97057,"_EXTENSIONS":97058,"ĠCASCADE":97059,"Ġ![":97060,"(KP":97061,"Ġvem":97062,"ĠInterracial":97063,"']},Ċ":97064,"spacer":97065,"_kv":97066,"Warehouse":97067,"RDD":97068,"_fsm":97069,".StretchImage":97070,",Yes":97071,"ĠRefugee":97072,"ĠBringing":97073,"Ġválido":97074,".intersection":97075,"Ġspooky":97076,"_portal":97077,"Ġmoth":97078,"ĠZodiac":97079,"ĠSOCIAL":97080,"MimeType":97081,"']}}":97200,"_Blue":97201,"Ġbotanical":97202,"Ġfrags":97203,"Ġfamilial":97204,"-du":97205,"Ġseizing":97206,"(blocks":97207,".rd":97208,".checkNotNull":97209,"Ġmiser":97210,"Ġmaxx":97211,"ĠKnee":97212,"ViewItem":97213,"InnerHTML":97214,"Danger":97215,"((__":97216,"Ġprzypad":97217,"createUrl":97218,"**,":97219,"ĠDecorating":97220,"ATEGY":97221,"?>/":97222,".Designer":97223,"hexdigest":97224,"ĠEverywhere":97225,"alleries":97226,".TEXTURE":97227,".Blocks":97228,"zell":97229,"Ġpreço":97230,"Suddenly":97231,"inputEmail":97232,"(sync":97233,".bd":97234,"golden":97235,">');":97236,"ĠDickinson":97237,">>(Ċ":97238,"ĠQUEUE":97239,"ĠgetColumn":97240,"ĠSAND":97241,".piece":97242,"licer":97243,"Flutter":97244,"ĠgetVersion":97245,"ĠresourceId":97246,"ogl":97247,"ÅĤaw":97248,".Branch":97249,"ĉweb":97250,"Ġframerate":97251,"PPP":97252,"Ġfray":97253,"CNT":97254,"Ġinformatie":97255,"']čĊčĊ":97256,"neas":97257,"HeaderCode":97258,"Ġæ¸":97259,"Ġtrg":97260,"rawtypes":97261,"Honda":97262,"Ġmarketer":97263,"ĠrequestData":97264,"ĠPg":97265,"ĉnot":97266,"ĠpageInfo":97267,"Ġaktuellen":97268,"ãģķãĤĵ":97269,"ĠAMS":97270,"pushViewController":97271,"ĉAL":97272,"Ġvests":97273,"produce":97274,"-même":97275,"ĠRahman":97276,"Funny":97277,"EZ":97278,"_Valid":97279,"Ġsquadron":97280,"Ġlash":97281,"Ġirm":97282,"iasco":97283,"ĠParan":97284,"Ġpetites":97285,"ĠDecay":97286,"Ġuninitialized":97287,"privileged":97288,"Ġmbedtls":97289,"å¤ĩ注":97290,"Ġ^.":97291,"Ġecstatic":97292,"Detroit":97293,"Ġparten":97294,"Ġsouvenir":97295,".getLogin":97296,"моÑĤÑĢ":97297,"enção":97298,"ĠmÃŃnimo":97299,"ĠAccessed":97300,"rió":97301,"Mic":97302,"ĠVocal":97303,".SetString":97304,"Ġmensajes":97305,"åĢį":97306,"Ġattravers":97307,"ĠAph":97308,"Ġ');čĊ":97309,"ünde":97310,"Ġenchanted":97311,"ĠRootState":97312,"ĠCLOSED":97313,"ĉĉĉĉĉĉĉĉčĊ":97314,"Ġcaliente":97315,"orris":97316,"Ġphysicists":97317,"hwnd":97318,"_vi":97319,"Ġrápido":97320,"Ġcapitalized":97321,"edBy":97322,"Ġmachining":97323,"Ġhubby":97324,"ĠStacy":97325,".Bus":97326,"drink":97327,"Hur":97328,"Ġpropia":97329,"UnitTest":97330,"Ġmisconception":97331,"__));Ċ":97332,"/dc":97333,"ĠMayweather":97334,"_mC":97335,".createFrom":97336,"ĠQPainter":97337,"ropsych":97338,"innitus":97339,"ayas":97340,"Ġgeg":97341,"(dw":97342,"Ġusado":97343,"Ġtrickle":97344,"Ġannihil":97345,"ĠPasta":97346,"Ġ++Ċ":97347,"(ExpectedConditions":97348,".postValue":97349,"icap":97350,"ĠDonetsk":97351,"_soup":97352,"-publish":97353,"ĠPb":97354,"mentions":97355,"ACCEPT":97356,".Pull":97357,",âĢĻâĢĻ":97358,"Ġretarded":97359,"_ATOM":97360,"ĠTerminator":97361,"-court":97362,"ĠCLLocationCoordinate":97363,"Ġreverence":97364,"ĠSSC":97365,"utely":97366,"ĠWON":97367,"ĠGSL":97368,"frei":97369,".getLongitude":97370,"ĠopenFileDialog":97371,".Butter":97372,"-important":97373,"_MANY":97374,"ĠGong":97375,"âĢľHow":97376,"Ġgorge":97377,"=msg":97378,"ĠEzek":97379,"createCommand":97380,":checked":97381,"Ġinfographic":97382,".WEST":97383,"Dirs":97384,"Ġguarda":97385,"Ġbeetle":97386,"Loading":97460,"_mA":97461,".getRandom":97462,"blings":97463,"Ġcheeses":97464,"tti":97465,".âĢ¢":97466,"ĠBurgess":97467,"enderit":97468,".',čĊ":97469,"(\"\"+":97470,"acb":97471,"%p":97472,"indexed":97473,"_predicate":97474,"nesia":97475,"Ġbied":97476,"ĠCIT":97477,"(Pos":97478,"_radi":97479,"ä»·æł¼":97480,"Biz":97481,"ĠAdolescent":97482,"Ġviên":97483,"cycl":97484,"_Cancel":97485,"Ġconclusive":97486,"Ġappellate":97487,"informatics":97488,"SJ":97489,"Ġelective":97490,"roleId":97491,"Fetcher":97492,"ĉCommand":97493,"(\"(%":97494,"Ġfart":97495,"ILA":97496,"getBlock":97497,"AUSE":97498,"Ġдан":97499,"ĠArte":97500,"Ġnotifying":97501,"Ġgele":97502,".same":97503,"ĠRegel":97504,"ĠBaÅŁ":97505,".creation":97506,"ĠVN":97507,"_community":97508,"Ġunsustainable":97509,"SEX":97510,"ĠgridSize":97511,"rescia":97512,"aversable":97513,"(',')[":97514,"ĠPhelps":97515,"á»ķi":97516,"ANCELED":97517,"-IS":97518,".runners":97519,"ĠStokes":97520,".Produ":97521,"Ġwhipping":97522,"_acquire":97523,"Ġinvestigación":97524,"fried":97525,".copyWith":97526,"ĠHardcover":97527,"-Se":97528,"áŀ¶áŀ":97529,"invitation":97530,"lesai":97531,"ĠDorm":97532,"ĠÑģпиÑģка":97533,"Ġconcatenated":97534,"ophil":97535,"Ġthinker":97536,"/fontawesome":97537,"ĠLeopard":97538,"Ġ\"/\");Ċ":97539,"Ġresiduals":97540,"ĠMicrowave":97541,"Ġconforme":97542,"throp":97543,"Ġdisemb":97544,"ĠOMG":97545,"ĠDiscipline":97546,"ĠAcrobat":97547,"/repository":97548,"dfa":97549,"_MED":97550,"bufio":97551,"Ġméthode":97552,"_HOLD":97553,"iasi":97554,"_legacy":97555,")ččĊ":97556,"æ£Ģ":97557,"GetProcAddress":97558,"Ġyay":97559,"otence":97560,"orderid":97561,"-tw":97562,"Ġdearly":97563,"Incoming":97564,"/il":97565,"Ġneurop":97566,"ucz":97567,");čččĊ":97568,"ĠInnovative":97569,"Ġprofund":97570,"igmat":97571,"SelectionMode":97572,"relevant":97573,".GO":97574,"Ġbruises":97575,"Ġsach":97576,"odef":97577,"Ġreimb":97578,"/desktop":97579,"-spot":97580,"undance":97581,"Entropy":97582,"\\core":97583,"Ġsuger":97584,"ĠMvc":97585,"ĠGNOME":97586,"_indx":97587,"ĠYYSTYPE":97588,"ĠMatlab":97589,"ĠCIF":97590,"Ġ*))":97591,"ĠproductList":97592,"ĠAlright":97593,"acemark":97594,"ÑĤив":97595,"modification":97596,"international":97597,"Ġhomers":97598,"Ġdicts":97599,"ĠQFont":97600,".SQLite":97601,"Ġtransplantation":97602,"ĠMessageBoxButton":97603,"ĠElves":97604,"']])Ċ":97605,"(QIcon":97606,"Ġcinemas":97607,"COORD":97608,"-China":97609,"Ġkhẩu":97610,"æĪijçļĦ":97611,"Ġskulls":97612,"Ġpainstaking":97613,"fce":97614,".XRLabel":97615,"Ġspecifier":97616,"Ġpreferring":97617,"/activity":97618,"(Photo":97619,"ált":97620,".lot":97621,"''.":97622,"annonce":97623,".googlecode":97624,"-pdf":97625,"ĠPoke":97626,"_ACL":97627,"Ġendowed":97628,"discover":97629,".omg":97630,"Ġwoodland":97631,".Magic":97632,"Ġvolont":97633,"NotAllowed":97634,"Ġchave":97635,"BMW":97636,"','=',":97637,"ĠSIX":97638,"æĪij们":97639,"Ġkosher":97640,"Ġaspiration":97641,"intl":97642,"_refptr":97643,"'+Ċ":97644,"mentor":97645,".club":97646,"WindowState":97647,".ARR":97648,"Ġzza":97649,"ĠmessageType":97650,".equ":97651,"Thor":97652,"Ġinjust":97653,"Ġgums":97654,"ĠborderSide":97655,"/////":97656,"ĠTransmit":97657,"Ġbufsize":97658,"Ġhak":97659,"Ġellas":97660,"RANDOM":97661,"ĉmc":97662,"Ġpea":97663,"eko":97664,"documento":97665,"Ġhysteria":97666,"Ġarenas":97667,"Ġgunmen":97668,"Ġmike":97669,"Ġimpunity":97670,"atisation":97671,"_Zero":97672,"_COMPANY":97673,"ĠGors":97674,"ĠuseClass":97675,"(redis":97676,"ĠRUNNING":97677,"ĠBair":97678,"velte":97679,"Ġ','.":97680,"аÑĤÑĮÑģÑı":97681,"öst":97682,"encodeURIComponent":97683,"_restrict":97684,"Ġdecals":97685,"ĠPedido":97686,"Ġaltercation":97687,"Displays":97688,"ĠApplicants":97689,"CUS":97690,"Textarea":97691,"ĠAngola":97692,".future":97693,"ĠUSHORT":97694,"Ġsuppressing":97695,"Ġsetzen":97696,"APolynomial":97697,"Ġtoch":97698,"Ġhallmark":97699,"Ġ$$$":97700,"ĠCHARSET":97701,".rpm":97702,"ĠDich":97703,"--------------------":97704,"_parm":97705,"è¿ĺ":97706,"acciones":97707,"hait":97708,"WARDED":97709,"_routing":97710,"ĠNOM":97711,"Ġenclave":97712,"ĠLotto":97713,"ĉfr":97714,"complexContent":97715,"ĠBallard":97716,"kube":97717,"/win":97718,".getColumnModel":97719,"_REPLACE":97720,"HeaderValue":97721,"Ġestudiantes":97722,"Ġapis":97723,"Ġbpm":97724,"ĠTypeName":97725,"AndGet":97726,"rita":97727,"Plans":97728,">Note":97729,"Ġfetisch":97730,"Ġtoned":97731,"_goto":97732,"onsense":97733,"Ġmolds":97734,"Ġinfiltration":97735,"ĠGuerrero":97736,"ubbo":97737,"cki":97738,"($(\".":97739,"_activities":97740,"(changes":97741,"ĠofApp":97742,"ĠKepler":97743,"ĠDemp":97744,"ĠContinent":97745,".Ticks":97746,"ĠUnsigned":97747,"ĠJahres":97748,"Ġfreshmen":97749,"ĠArchived":97750,"ĠкоÑĤоÑĢÑĭй":97751,"Ġ'::":97752,"Tutorial":97753,"Cc":97754,"ĠtableLayoutPanel":97755,"fromJson":97756,".levels":97757,"_transient":97758,"Ġendorsing":97759,"ĠDIC":97760,"lauf":97761,"Ġshred":97762,"_EMIT":97763,"ificantly":97764,"ALA":97765,"/proto":97766,"Ġnarrowing":97767,"Utc":97768,"Factors":97769,"Ġsentient":97770,"æŀIJ":97771,"lixir":97772,"ĠCROSS":97773,"meteor":97774,"Ġgroin":97775,"Ġmdb":97776,"ĠRotterdam":97777,"Ġcomida":97778,"ĠOpCode":97779,"ĠDefaultValue":97780,"PermissionsResult":97781,"Ġheterogeneous":97782,"Ġmoot":97783,"Ġdeceived":97784,"-independent":97785,"ĠObjectOutputStream":97786,"Ġoverpower":97787,".dup":97788,"Ġldb":97789,"Ġdomestically":97790,"Ġbestellen":97791,"Ġlov":97792,"ĠContractors":97793,"Triangles":97794,"Ġfodder":97795,"Ġfilmes":97796,"ä¼ģ":97797,"Ġrevolver":97798,"StartupScript":97799,"/validation":97800,"ĠResourceType":97801,"iÅŁ":97802,"ĠLaz":97803,"fef":97804,"Ġlstm":97805,"{*":97806,".attachment":97807,".hits":97808,"ewith":97809,"DOG":97810,"Alabama":97811,"Ġmediums":97812,".mContext":97813,"-cols":97814,"åıĭ":97815,".notice":97816,"Ġattn":97817,"ĠPacking":97818,"ĠLn":97819,"_COMPLEX":97820,"/Users":97821,".savetxt":97822,"ĠRounds":97823,"?,?,?,?,":97824,"Ġingl":97825,"ĠROC":97826,"_female":97827,"ĠStard":97828,"]];":97829,"Ġwrestlers":97830,"Ġtorrents":97831,"Ġsinh":97832,"ĊĊ":97833,"ë³µ":97834,"sense":97835,"however":97836,".Physics":97837,"Infrastructure":97838,"ĠSacr":97839,"Fel":97840,"ĠDISTRIBUT":97841,"éments":97842,"ĠValidates":97843,"############################################################":97844,"Ġ|/":97845,"Ġesl":97846,"Ġréseau":97847,"ĠBip":97848,"BYTES":97849,"_WATER":97850,"Turning":97851,"ELS":97852,"Ġjuxtap":97853,"Ġlesbische":97854,"ých":97855,"(Unknown":97856,"Neo":97857,"@JsonProperty":97858,"Ġalumnos":97859,"ĠRaqqa":97860,"imei":97861,".getBounds":97862,".MouseEventHandler":97863,"#######":97864,"GenericType":97865,"/cms":97866,"Ġturno":97867,"Ġмин":97868,"Ġfolklore":97869,"ĠEvo":97870,"Ġconductivity":97871,"Ġleben":97872,"Ġgearbox":97873,"-vs":97874,"ĠÏĨ":97875,"Ġdrinkers":97876,"Ġconexao":97877,"ĠTeeth":97878,"ĠgetArguments":97879,"ĠRAT":97880,"entious":97881,"Educ":97882,"+W":97883,"ĠInstitutional":97884,"ĠBord":97885,"isEqual":97886,"(pwd":97887,"Ġignited":97888,"ĠRousse":97889,"Ġimpactful":97890,"ĠMalk":97891,"Ġgeral":97892,"ĠPivot":97893,"Ġazt":97894,"Ġcsvfile":97895,"ĠRope":97896,"ĠSOLUTION":97897,"ĠArbitrary":97898,"Ġletto":97899,".MouseAdapter":97900,"Ġ}}}":97901,"ĠSailor":97902,"dera":97903,"Putting":97904,"Ġconcentrates":97905,"ĠauthDomain":97906,"âĢĿçļĦ":97907,"-finals":97908,",strlen":97909,"Muon":97910,"ĠOrdinary":97911,"firefox":97912,"ĠLaTeX":97913,"ĠHund":97914,"engineering":97915,"/blue":97916,"edTextBox":97917,"(\"\");":97918,"ĠCDDL":97919,"kept":97920,"ĠGetString":97921,"Kir":97922,"()='":97923,"ĠOCD":97924,"antium":97925,"$menu":97926,"ĠAppalachian":97927,"Secretary":97928,"ë¥ĺ":97929,"ีย":97930,"Semantic":97931,"Ġ*[":97932,"estone":97933,"ungkin":97934,"MaxY":97935,"-tone":97936,"\"};čĊ":97937,"_Part":97938,"ĊĊ":98140,"Lic":98141,"ĠMirage":98142,"ĠAssemblyFileVersion":98143,"TeV":98144,"ĠValueEventListener":98145,"-solving":98146,"Tho":98147,"roulette":98148,"_WP":98149,"Ġuninterrupted":98150,"ĠfieldType":98151,".Typed":98152,"Ġamour":98153,"Ġmockery":98154,"(vol":98155,"ĠSubcommittee":98156,"ĠRuf":98157,"erox":98158,":UIButtonTypeCustom":98159,"ĠBlur":98160,"Ġwykon":98161,"nces":98162,"ASHBOARD":98163,"!!\");Ċ":98164,"Ġmurderers":98165,".daily":98166,"ĠDIAG":98167,"jing":98168,"Ġdolphin":98169,"Ġlòng":98170,"Ġbö":98171,"ĠVocabulary":98172,".StObject":98173,"')\">":98174,"Ġzun":98175,"Ġscrimmage":98176,"tréal":98177,"ĠLig":98178,"[vi":98179,"Cole":98180,"Ġfrosting":98181,".Players":98182,"-translate":98183,"Feels":98184,"=\\\"/":98185,".ButterKnife":98186,"Ġ?>;Ċ":98187,"Ġavi":98188,"innie":98189,".Failure":98190,"Ġspindle":98191,"ConfigurationException":98192,"_hop":98193,"Ġposição":98194,"ĠAwait":98195,"UIImagePickerController":98196,"ĉday":98197,"Ġgenom":98198,"Cab":98199,"ĠÑĢезÑĥлÑĮÑĤаÑĤ":98200,"ORIGINAL":98201,"Ġejaculation":98202,"(tcp":98203,"SECOND":98204,"Ġtonic":98205,"ĠListBox":98206,"ĠĉĉĊ":98207,"()>Ċ":98208,"Ġquatre":98209,"ượng":98210,"withErrors":98211,".Maybe":98212,",âĢ¦":98213,"tokenId":98214,"_UNDEF":98215,"Ġfreshness":98216,"ĠAmendments":98217,".mapbox":98218,".CV":98219,"(blog":98220,"_gettime":98221,".quest":98222,"sparse":98223,"Ġresale":98224,"Ġenthusiastically":98225,"ĠProstitutas":98226,"Wa":98227,"Cargo":98228,".Parcelable":98229,"SENSOR":98230,"ĠRyu":98231,"Laughs":98232,"_Native":98233,"/pg":98234,"ysts":98235,"Ġphotoc":98236,"ç®Ģ":98237,"adopt":98238,".species":98239,"conciliation":98240,"Adjusted":98241,".FirebaseAuth":98242,"uttle":98243,"ordination":98244,"Ġmunch":98245,"ĠStake":98246,".ping":98247,"anker":98248,"(QStringLiteral":98249,"Ġsubscript":98250,"ĠĠĉĊ":98251,"ĠMCC":98252,"_Cmd":98253,"sexy":98254,"iou":98255,"ĠMANY":98256,"Ġnanny":98257,"TRAIN":98258,"Ġflourishing":98259,"ĠWatches":98260,"ĠQMap":98261,"ĠFerm":98262,"Ġwasm":98263,"ĠAbed":98264,"_UD":98265,"ĠGlasses":98266,"+v":98267,"Attend":98268,".Chain":98269,"Ġdecency":98270,"ĠSupplementary":98271,"hunter":98272,"-txt":98273,"Ġ\"}\";Ċ":98274,".setWindowTitle":98275,"(\"":98377,"Ġmascara":98378,"(Profile":98379,"åĬŁèĥ½":98380,"imité":98381,"Ġwildfires":98382,"-ROM":98383,".isOn":98384,"(groupId":98385,"Repair":98386,"accumulate":98387,"Ġ<\",":98388,"Ġhandwritten":98389,"Ġacheter":98390,"ĠMGM":98391,"ĠIrma":98392,"->{_":98393,"gee":98394,"criminal":98395,"Ġèĭ¥è¦ģ":98396,"Ġmomentarily":98397,"\")!=":98398,"_lit":98399,"ĠexpiresIn":98400,".\").":98401,"éķ¿åº¦":98402,"Ġfrække":98403,"vlc":98404,"Ġorbs":98405,"),$":98406,"Ġventured":98407,"/>\\":98408,"charm":98409,"Nuitka":98410,"eldig":98411,"atonin":98412,"Witness":98413,"-lat":98414,"ĠsetHidden":98415,"Ġrelics":98416,"Ġconsulate":98417,".IGNORE":98418,"\"After":98419,"ĠsetAddress":98420,"Ġbesteht":98421,"Ġ'')ĊĊ":98422,".xaxis":98423,"Ġserão":98424,"Ġmisled":98425,"_UNIFORM":98426,"ĠVIA":98427,"incr":98428,"Ġzenith":98429,"Ġviscosity":98430,"Ġthinly":98431,".getSharedPreferences":98432,".ErrorCode":98433,"\"),\"":98434,"ĠMillionen":98435,"Ġ/>)Ċ":98436,"ScrollIndicator":98437,"-seeking":98438,"ĠPOLITICO":98439,"asca":98440,"_rl":98441,"Navig":98442,"(fullfile":98443,"Ġsolitude":98444,"Ġjuven":98445,"Ġhauling":98446,"ĠMacros":98447,"ĠGry":98448,"Ġexercitation":98449,"ĠATTACK":98450,"TickCount":98451,"Ġrites":98452,"Ġdoe":98453,"ParticleSystem":98454,"Ġslu":98455,"WindowText":98456,"ĠClassName":98457,"Ġslander":98458,"ĉPort":98459,"jong":98460,"?a":98461,".Dial":98462,"âĢĶat":98463,"$objPHPExcel":98464,"Ġsoar":98465,"ENN":98466,"appeared":98467,"Ġquotid":98468,"emachine":98469,"Ġnip":98470,"Ġmicrotime":98471,"ĠAlma":98472,";!":98473,"------------------------------------------------------------------------------------------------":98474,"ĠPassage":98475,"Ġdumpsters":98476,"ĠExclude":98477,"Ġsuggestive":98478,"ĠCircularProgressIndicator":98479,"_clr":98480,"ArrayType":98481,"ILLA":98482,"ElapsedTime":98483,"Driven":98484,"ĠresourceName":98485,"ĠGarrison":98486,"serir":98487,"-ahead":98488,"Ġpinnacle":98489,"ĠEspresso":98490,"Sparse":98491,"Ġassays":98492,"ĠGirlfriend":98493,"imid":98494,"]='\\":98495,"ONGLONG":98496,"Ġportraying":98497,"Lane":98498,"Ġbúsqueda":98499,"Ġreinforcements":98500,"ĠSpreadsheet":98501,"ĠArrayCollection":98502,",arr":98503,"lightbox":98504,"icana":98505,"<\"":98506,"builders":98507,"Kid":98508,"ĠMatSnackBar":98509,"EXPR":98510,"odcast":98511,"ĠFoundations":98512,"Ġinds":98513,"='${":98514,"Fizz":98515,"-functional":98516,"(workspace":98517,"Ġstemmed":98518,"_patches":98519,"ĠJarvis":98520,"READING":98521,"Ġdisrespectful":98522,"ĠQDom":98523,"Ġ${Ċ":98524,"estatus":98525,"Reached":98526,"!.ĊĊ":98527,"ILT":98528,"ĠNDEBUG":98529,"ĠCourage":98530,"birthdate":98531,"ĠTing":98532,"Ġutilizado":98533,"ánchez":98534,"Outdoor":98535,"Ġhandguns":98536,"RefCount":98537,"ÉĻ":98538,"romo":98539,"Ġtts":98540,".She":98541,"ĠPane":98542,"ãĢij,ãĢIJ":98543,"ĠIOCTL":98544,"/black":98545,"inscription":98546,"Ġbiopsy":98547,"ĠTimeInterval":98548,".TestCheck":98549,"ĠGUIStyle":98550,"ĠCapability":98551,"ĠBeitrag":98552,"donnees":98553,"Treatment":98554,".backup":98555,"Ġsignings":98556,"ĠBoca":98557,"drm":98558,".MAIN":98559,"Ġgoede":98560,"ĠMarkup":98561,"GREE":98562,"ĠBaseService":98563,".Creator":98564,"Ġjails":98565,"ĠKahn":98566,"IpAddress":98567,"ACHI":98568,"Ġinhibited":98569,"Ġ@$_":98570,"ĠAssass":98571,"Ġenviado":98572,"Heroes":98573,"ÐŁÐµÑĢ":98574,"ĠMaven":98575,".ls":98576,"Ġive":98577,"|RF":98578,"ĠresizeMode":98579,"Ġrumpe":98580,"_attachments":98581,"TU":98582,"Ġtactile":98583,"Attempting":98584,"Ġrobin":98585,"yaw":98586,"Ġmercenaries":98587,"ĠHabitat":98588,"enddate":98589,"Ġoxy":98590,"ĉRandom":98591,"ohon":98592,"IsNull":98593,"ĠValidationResult":98594,"ãĥļ":98595,"umbed":98596,"ppv":98597,"Ġarp":98598,"ichick":98599,"_rnn":98600,"ĠTFT":98601,"TexImage":98602,"\"On":98603,"ĠSampler":98604,"topl":98605,"Ġjane":98606,"yling":98607,"ĠUNICODE":98608,"TabIndex":98609,"<{Ċ":98610,"suspend":98611,"uvian":98612,",application":98613,"олиÑĩеÑģÑĤво":98614,"yat":98615,"ezier":98616,"ĠCHUNK":98617,"ĠAdler":98618,"/Add":98619,"ĠKeyValue":98620,"Ġsposób":98621,"Sampling":98622,"chers":98623,"_AMD":98624,"Ru":98625,".MustCompile":98626,"Nation":98627,"Assoc":98628,"Managing":98629,"ĠEngl":98630,"_GB":98631,"Ġsuccinct":98632,"Ġdisliked":98633,"ĠIke":98634,"Bulletin":98635,"_ARCHIVE":98636,"Proposal":98637,"Ġjogging":98638,".CREATED":98639,"Ġchol":98640,"è£ħ":98641,"Į¨":98642,"-push":98643,"Ġreserva":98644,"corev":98645,"ètre":98646,"THR":98647,"Ġincompetence":98648,"Ġcharisma":98649,"æĦŁ":98650,"Ġ\"==":98651,"BTN":98652,"ĠLocator":98653,"ivet":98654,"('.')Ċ":98655,"ĠforIndexPath":98656,"ôme":98657,"Ġcapacit":98658,"waters":98659,"ĠWRONG":98660,"hoa":98661,"ĠMIPS":98662,"Ġemiss":98663,"ĠJacqueline":98664,"(cmp":98665,"Ġeens":98666,"Leo":98667,".timing":98668,"CLUSION":98669,"Ġ(\"-":98670,"åĵĪ":98671,".kode":98672,"ĠUndert":98673,"Ġbewild":98674,"ĠEssen":98675,".hd":98676,"Ġrenegot":98677,"Ġmower":98678,"Ġlsp":98679,"Ġpenchant":98680,"Ġmanoe":98681,"Ġagli":98682,"Ġrecal":98683,"ĠOPERATION":98684,"(^)(":98685,"Ġν":98686,"ĠScoped":98687,"Ġ@\"Ċ":98688,"=label":98689,"[loc":98690,"Intl":98691,"ĠNz":98692,"tablet":98693,".ColumnName":98694,"ĠscreenSize":98695,"DBus":98696,"cooked":98697,"-registration":98698,"âĢľOne":98699,"-non":98700,"ĠwiÄĻc":98701,"Ġcosta":98702,".addTab":98703,".conditions":98704,"ĠHess":98705,"MEMORY":98706,"ĠAvalanche":98707,"()}}Ċ":98708,"Ġtriplet":98709,"Ġlabyrinth":98710,"ĠNodeList":98711,"ĠNYT":98712,"Ġyeni":98713,"dff":98714,".HtmlControls":98715,"AVIS":98716,"/Math":98717,"Ġmemcmp":98718,"اء":98719,"оÑģÑĮ":98720,"crap":98721,"(pages":98722,"Ġlxml":98723,"ĠQDateTime":98724,"_tcb":98725,"Ġopenid":98726,"Ġsynaptic":98727,"ĠMDMA":98728,"(slug":98729,"igmatic":98730,"enor":98731,"Ġcramped":98732,"GOP":98733,"ŃIJ":98734,".isFile":98735,"ĠDifferential":98736,"Ġ=\"\";Ċ":98737,"ĉĉĉĠĠĠĠĉ":98738,"ĠCooke":98739,"ĉUFUNCTION":98740,"Ġperseverance":98741,"RelativeLayout":98742,"IMPORTANT":98743,"Ġexon":98744,"Ġон":98745,"ibase":98746,"(CONT":98747,"novation":98748,"ä½ķ":98749,"[sub":98750,"AdminController":98751,"HTTPHeader":98752,"crear":98753,"ĠNIR":98754,"ĠDropDownList":98755,"Ġvalide":98756,"Ġdehydration":98757,".']":98758,"(WIN":98759,"Ġ...\\":98760,"Ġphotoshop":98761,"ĉInit":98762,"_cou":98763,"ĠtimeZone":98764,"darwin":98765,"romatic":98766,"NavigationItemSelectedListener":98767,"brates":98768,"]--;Ċ":98769,"Ġtragedies":98770,"ĠPediatrics":98771,"SMART":98772,"-API":98773,"ĠMessageLookup":98774,"ĉvo":98775,"Ġprejudices":98776,"ĠmA":98777,"Ups":98778,"ĠMISSING":98779,"ĉad":98780,"Cream":98781,"ĠTb":98782,"ĠMona":98783,"_ghost":98784,"ĉtypes":98785,"Emb":98786,"ĠDocumentary":98787,"');ĊĊĊĊ":98788,"Ġlup":98789,"_Reference":98790,"ĠBATCH":98791,"Ġintertwined":98792,"":98915,"Ġfoyer":98916,"'utilisation":98917,"ĠMüller":98918,"ĠFetish":98919,"ĠdefaultManager":98920,"Ġbacktrack":98921,"Bah":98922,"Explicit":98923,"_ASCII":98924,"ĠmActivity":98925,"(Msg":98926,"Ġê²Į":98927,"ĠTERMS":98928,"ĠAngie":98929,"HSV":98930,"ĠMosque":98931,".Names":98932,"íĬ¼":98933,"reste":98934,"_parms":98935,"Ġgaping":98936,"Ġcropping":98937,"DataFrame":98938,"Ġresponsiveness":98939,"_undo":98940,"_tran":98941,".terminate":98942,"Ġitaliane":98943,"Ġwalkthrough":98944,"Ġattractiveness":98945,"де":98946,"_STS":98947,"_learn":98948,"Ġchocolates":98949,"ierarchical":98950,"-thinking":98951,"Ġ)))":98952,"ishments":98953,".Logf":98954,"ĠTMZ":98955,"ĠCanary":98956,"foil":98957,"ĠVaccine":98958,".vx":98959,"ĠSurround":98960,"Intermediate":98961,"Ġiov":98962,"vais":98963,"';\";Ċ":98964,"ï½ŀĊĊ":98965,"éĢģæĸĻ":98966,"âĢ¦it":98967,"Seats":98968,"Clar":98969,"Wars":98970,"ĠHutchinson":98971,"ĠHasan":98972,"!')ĊĊ":98973,"ĠRichie":98974,"cheiden":98975,"($('":98976,"York":98977,"Ġlids":98978,"Ġalphanumeric":98979,"ĠGlock":98980,".shapes":98981,"Ġsparking":98982,"_epsilon":98983,"uplicated":98984,".dirty":98985,"])==":98986,"ĠìľĦì¹ĺ":98987,"Ġscn":98988,"Ġ/****************************************************************":98989,"_PREVIEW":98990,"_HC":98991,"ielding":98992,"fgets":98993,"ĠAddison":98994,"ĠproductService":98995,"-figure":98996,"(retval":98997,"zano":98998,"Ġautob":98999,"ĉsd":99000,"_numer":99001,"ĠSetLastError":99002,"ĠFior":99003,"ificance":99004,"Untitled":99005,"Ġinfield":99006,"Ġ{}));Ċ":99007,"Ġspac":99008,"Ġrookies":99009,"(describing":99010,"ngen":99011,"ிà®":99012,".rdf":99013,".Mutex":99014,"Ġkneeling":99015,"ĠQE":99016,"setMax":99017,"ReadStream":99018,"Ġventas":99019,"sut":99020,"cmpeq":99021,".WriteAllText":99022,"ĠExperienced":99023,"$__":99024,"Ġkaum":99025,"ĠLIS":99026,"Ġdocumentos":99027,"_HEALTH":99028,"icontains":99029,"Ġartisans":99030,"OWNER":99031,"Ġblinked":99032,"getDisplay":99033,"Ġtoen":99034,"ĠrowNum":99035,"Ġavril":99036,"Ġinvis":99037,"ĠKear":99038,"toBeInTheDocument":99039,"apur":99040,"Ġracked":99041,"ĠMcMaster":99042,"_ATTRIB":99043,"Haz":99044,"Ġfactura":99045,"/ts":99046,"ĠÑĢазмеÑĢ":99047,"Ġzf":99048,"Ġshortfall":99049,".fasta":99050,"ĠCONSTANT":99051,".managed":99052,"gems":99053,"SharedPointer":99054,"Ġblurry":99055,"brightness":99056,"(components":99057,"Ġ...\"ĊĊ":99058,"SELL":99059,"ĠIllustrator":99060,".getChannel":99061,"Ġtrouvé":99062,"ysters":99063,"Ġvois":99064,"ĠLinden":99065,"Ġemojis":99066,"Ġbrawl":99067,"ĠMSR":99068,"ĠElo":99069,"ĠCroatian":99070,"PopupMenu":99071,"Lewis":99072,".JWT":99073,"Ġastonished":99074,"Bush":99075,"(itemId":99076,"Ġdetachment":99077,"ĠEncore":99078,"å°Ķ":99079,"Ġrekl":99080,"Ġcram":99081,")$/":99082,".getHost":99083,"_recommend":99084,"-HT":99085,"_calibration":99086,"Authenticate":99087,".firebaseapp":99088,"UNIX":99089,"ĉCamera":99090,"ĠHEAP":99091,"Ideal":99092,".office":99093,"Ġgoofy":99094,"(Symbol":99095,"Ġjouer":99096,"_partitions":99097,"Ġrapidement":99098,"ĠGNUNET":99099,"idUser":99100,"Ġsupervise":99101,"(Contact":99102,"AWN":99103,"ãģĺ":99104,"Ġnaam":99105,"Ġaust":99106,"åľ¨çº¿":99107,"_softmax":99108,"AllowAnonymous":99109,"ammable":99110,"ROUTE":99111,"*D":99112,"Ġaden":99113,"ĠCristina":99114,"ĠCristiano":99115,"Ġbloodstream":99116,"subclass":99117,"_persona":99118,"CHILD":99119,"-know":99120,"ĠnavigationOptions":99121,"ĠZukunft":99122,"ĠPixar":99123,"Tyler":99124,"Ġunderworld":99125,"Ġsincerity":99126,"Ġdispenser":99127,"Ġkter":99128,"idders":99129,".addNode":99130,"-checked":99131,"Ġkeyst":99132,"ĠWTO":99133,".signals":99134,"Ġadventurer":99135,"ĠPang":99136,"\\R":99137,"=pos":99138,"Ġdispensaries":99139,"ĠCloset":99140,"(\"{\\\"":99141,"ideon":99142,"Ġnécessaire":99143,"()\"Ċ":99144,"_RECEIVED":99145,"Ġrésultats":99146,"Ġmoden":99147,"ĠIcelandic":99148,";d":99149,".allowed":99150,"(newUser":99151,"Ġmerciless":99152,".WaitFor":99153,"Ġdaycare":99154,"ĠConveyor":99155,"çĸ":99156,"ð¬":99157,"çĥ":99158,"çĹ":99159,"çł":99160,"èĦ":99161,"é²":99162,"å¦":99163,"çĿĢ":99164,"å¾Ī":99165,"éħ":99166,"çĭ":99167,"éª":99168,"æĤ":99169,"é¥":99170,"èħ":99171,"æĥ³":99172,"å¨":99173,"é¹":99174,"çĤ":99175,"åĴ":99176,"çĮ":99177,"è´¨":99178,"æ¢":99179,"æ°Ķ":99180,"ð«":99181,"æķĻ":99182,"çŁ":99183,"åĦ":99184,"åıijå±ķ":99185,"åĪĽ":99186,"èij":99187,"æħ":99188,"åŀ":99189,"åģļ":99190,"æĪĺ":99191,"æIJ":99192,"强":99193,"æ·±":99194,"åĩł":99195,"ç¿":99196,"å©":99197,"èŀ":99198,"å§Ķ":99199,"åIJĦ":99200,"èİ":99201,"é¸":99202,"éº":99203,"åıĹ":99204,"èģĮ":99205,"åĺ":99206,"æ½":99207,"é£İ":99208,"èIJ¥":99209,"åħļ":99210,"èľ":99211,"éĤ£":99212,"é¢Ĩ":99213,"çij":99214,"é³":99215,"æľ¯":99216,"ä»Ģ":99217,"æĪ¿":99218,"ç²¾":99219,"åª":99220,"éĨ":99221,"太":99222,"èĤ¡":99223,"èĽ":99224,"åħī":99225,"æŀģ":99226,"åĬŀ":99227,"èĵ":99228,"çĺ":99229,"å´":99230,"åĹ":99231,"èĬ±":99232,"çłĶ":99233,"å¿«":99234,"å¸Ī":99235,"è¶Ĭ":99236,"è§Ĥ":99237,"æ¤":99238,"æ¦":99239,"çŀ":99240,"èĤ²":99241,"çĪ±":99242,"çĻ½":99243,"ä¸ĸ":99244,"ä»Ģä¹Ī":99245,"çľ¼":99246,"å³":99247,"èĴ":99248,"æĵ":99249,"被":99250,"å¹²":99251,"çĹħ":99252,"士":99253,"çĴ":99254,"è¸":99255,"æ¾":99256,"å·¥ä½ľ":99257,"让":99258,"çĥŃ":99259,"è¾ĥ":99260,"åĦ¿":99261,"åĬ©":99262,"积":99263,"ç³":99264,"çĵ":99265,"ç£":99266,"åĤ":99267,"è¹":99268,"èļ":99269,"å·±":99270,"çĻ¾":99271,"åĬ¿":99272,"èµĽ":99273,"æ¨":99274,"æ¿":99275,"èĸ":99276,"æĿij":99277,"带":99278,"å¢ĥ":99279,"æĬ¤":99280,"éŃ":99281,"å«":99282,"èĩªå·±":99283,"æµİ":99284,"ä½İ":99285,"åĮ»":99286,"éĺ²":99287,"åĨľ":99288,"èĨ":99289,"çĨ":99290,"é«":99291,"åĨĽ":99292,"æĪı":99293,"åįĩ":99294,"æĸ¯":99295,"ä½ı":99296,"èIJ½":99297,"åħ»":99298,"èĩ´":99299,"çĬ":99300,"çĩ":99301,"çħ":99302,"èĶ":99303,"ä¼ģä¸ļ":99304,"åĽ¢":99305,"æīį":99306,"æł¡":99307,"åĩĨ":99308,"å¥ĩ":99309,"åī¯":99310,"é¼":99311,"æ¼Ķ":99312,"马":99313,"èµ°":99314,"ç¥ŀ":99315,"åħĭ":99316,"æľĽ":99317,"æ²¹":99318,"è¾¹":99319,"åįĥ":99320,"å¾Ģ":99321,"åĪĩ":99322,"æ©":99323,"ç¶":99324,"åĻ":99325,"éĻħ":99326,"çīĮ":99327,"社ä¼ļ":99328,"游æĪı":99329,"æĸ½":99330,"çħ§":99331,"æݧ":99332,"满":99333,"è¯Ĩ":99334,"éĩįè¦ģ":99335,"足":99336,"çķĻ":99337,"ç»Ĩ":99338,"åįı":99339,"éĢĤ":99340,"æĩ":99341,"æ§":99342,"éĦ":99343,"èĿ":99344,"å¸Ĥåľº":99345,"ç»ıæµİ":99346,"ä¹ł":99347,"æĸĩåĮĸ":99348,"éļ¾":99349,"ä¹IJ":99350,"åĨ³":99351,"欢":99352,"è§ī":99353,"åĽŃ":99354,"åħ´":99355,"åħħ":99356,"举":99357,"æī¹":99358,"èķ":99359,"æĬĬ":99360,"æĬĢæľ¯":99361,"究":99362,"第ä¸Ģ":99363,"便":99364,"åĵį":99365,"çİ©":99366,"åĿļ":99367,"èŀį":99368,"åįĬ":99369,"åĸľ":99370,"å±Ĥ":99371,"离":99372,"ä»ħ":99373,"éŁ":99374,"åij³":99375,"念":99376,"åŃ£":99377,"ç´§":99378,"ä¹ħ":99379,"é¤":99380,"éŀ":99381,"è¤":99382,"åĢĻ":99383,"åĨµ":99384,"çŁ³":99385,"åģ¥":99386,"æĢİ":99387,"å®Ŀ":99388,"è¡Ģ":99389,"åŁŁ":99390,"æĹ©":99391,"çŁ¥éģĵ":99392,"è´Ł":99393,"åįļ":99394,"å·´":99395,"亲":99396,"å±ŀ":99397,"严":99398,"äºī":99399,"å¯Ł":99400,"èº":99401,"ç°":99402,"建设":99403,"产ä¸ļ":99404,"åIJĥ":99405,"åŃ©":99406,"æĹħ":99407,"æł¹":99408,"æĿIJ":99409,"ä¼Ĺ":99410,"éļı":99411,"å®ĺ":99412,"åºķ":99413,"彩":99414,"å¯Į":99415,"温":99416,"åį«":99417,"åī§":99418,"çĽĬ":99419,"æĬĹ":99420,"è´¢":99421,"纪":99422,"æĨ":99423,"çĶŁæ´»":99424,"红":99425,"çĶŁäº§":99426,"è¿ľ":99427,"éĴ±":99428,"åĶ®":99429,"群":99430,"çıŃ":99431,"楼":99432,"éĩĩ":99433,"èīº":99434,"å±ħ":99435,"åģĩ":99436,"è°Ī":99437,"æĻļ":99438,"é¬":99439,"èĪª":99440,"害":99441,"èĹ":99442,"çį":99443,"åµ":99444,"çİĭ":99445,"康":99446,"èİ·":99447,"ç»Ń":99448,"äºļ":99449,"é£Ł":99450,"åİĭ":99451,"æĭĽ":99452,"èĮĥ":99453,"许":99454,"åĽ´":99455,"é½":99456,"éĻį":99457,"纳":99458,"åĵª":99459,"æķĻèĤ²":99460,"å·²ç»ı":99461,"å¾·":99462,"æŀĹ":99463,"å®īåħ¨":99464,"é¾Ļ":99465,"大家":99466,"éĿĴ":99467,"åºľ":99468,"æ²³":99469,"åı¤":99470,"èį¯":99471,"åĿĩ":99472,"æĻº":99473,"乡":99474,"çķ¥":99475,"åĨ·":99476,"ç¦ı":99477,"室":99478,"ç»´":99479,"æī¿":99480,"å±Ĭ":99481,"è¯ī":99482,"åĪ»":99483,"èŁ":99484,"æª":99485,"å°±æĺ¯":99486,"è¿Ļ个":99487,"ä¸Ńå¿ĥ":99488,"ä¸ĸçķĮ":99489,"åŁİå¸Ĥ":99490,"éĿŀ常":99491,"åĪĴ":99492,"åıĮ":99493,"æĢİä¹Ī":99494,"åĪ°äºĨ":99495,"æľĥ":99496,"åı²":99497,"ä¾Ĩ":99498,"å¾ĭ":99499,"å¥ĸ":99500,"ç»Ī":99501,"åªĴ":99502,"å®ģ":99503,"课":99504,"èģĮä¸ļ":99505,"åħį":99506,"æµĭ":99507,"æĢ¥":99508,"æķij":99509,"çĭ¬":99510,"èѦ":99511,"é¤IJ":99512,"æĦ¿":99513,"è´«":99514,"çĸij":99515,"åļ":99516,"她":99517,"åıĪ":99518,"åĽłä¸º":99519,"ä¸įæĺ¯":99520,"å¤Ł":99521,"æĸ¹éĿ¢":99522,"éķĩ":99523,"äºĴ":99524,"éħĴ":99525,"讲":99526,"çĸĹ":99527,"æĺ¥":99528,"æ¹ĸ":99529,"å¤ľ":99530,"责任":99531,"人æ°ij":99532,"åħ°":99533,"çŁŃ":99534,"æķħ":99535,"åĩı":99536,"æĻ®":99537,"亮":99538,"ä¾Ŀ":99539,"åį°":99540,"éĿĻ":99541,"åĢĭ":99542,"å¾ģ":99543,"åIJ¸":99544,"缺":99545,"æĶ»":99546,"åĩĢ":99547,"åħ¸":99548,"åĽº":99549,"访":99550,"ç¹":99551,"çĢ":99552,"æıIJä¾Ľ":99553,"ç»ĩ":99554,"å¾Īå¤ļ":99555,"çłĶ究":99556,"è·Ł":99557,"主è¦ģ":99558,"æĥħåĨµ":99559,"çŃĸ":99560,"æŃ»":99561,"大åѦ":99562,"æĶ¿åºľ":99563,"å½±åĵį":99564,"ä¹°":99565,"åħŃ":99566,"éĻ©":99567,"åħ«":99568,"æŁIJ":99569,"è´¨éĩı":99570,"åįł":99571,"å·®":99572,"æĽ´å¤ļ":99573,"æľĭ":99574,"éĿ©":99575,"宣":99576,"çł´":99577,"è½»":99578,"座":99579,"æĺ¾":99580,"稳":99581,"è´µ":99582,"èĥĮ":99583,"èī¯":99584,"çĸ«":99585,"æ¯Ĵ":99586,"ä¹İ":99587,"åĢŁ":99588,"è¿·":99589,"çŃĶ":99590,"æ¿Ģ":99591,"åij¼":99592,"äºĨä¸Ģ":99593,"趣":99594,"ä¼´":99595,"ä¼Ļ":99596,"è¼":99597,"ð¬Ń":99598,"åĽ½å®¶":99599,"æ´»åĬ¨":99600,"çİ°åľ¨":99601,"ç§ijæĬĢ":99602,"åį¡":99603,"ä¸įåIJĮ":99604,"个人":99605,"è®°èĢħ":99606,"ä¸įæĸŃ":99607,"éĹ»":99608,"ä¹Ŀ":99609,"èijĹ":99610,"综":99611,"ä¸ĥ":99612,"æłij":99613,"æľĭåıĭ":99614,"åįĸ":99615,"伤":99616,"æ²Ļ":99617,"åĸĦ":99618,"å¥Ĺ":99619,"è½®":99620,"ç©¿":99621,"è¡¥":99622,"ä¸Ģå®ļ":99623,"çªģ":99624,"çĿ£":99625,"追":99626,"å¨ģ":99627,"åı¦":99628,"åĽ°":99629,"æŀ¶":99630,"ç»Ŀ":99631,"æķ£":99632,"æİ¢":99633,"æ´Ĺ":99634,"临":99635,"ä¼¼":99636,"è´¸":99637,"丰":99638,"æĺ¯ä¸Ģ":99639,"ç«ŀ":99640,"è¿İ":99641,"èģļ":99642,"è«":99643,"æįŁ":99644,"æī§":99645,"驾":99646,"è¿Ŀ":99647,"è¥":99648,"èł":99649,"ä»ĸ们":99650,"æŶåĢĻ":99651,"å®ĥ":99652,"人åijĺ":99653,"è¿Ļæł·":99654,"å·¥ç¨ĭ":99655,"åĪĽæĸ°":99656,"åŃ©åŃIJ":99657,"å¸Į":99658,"éĥ¨åĪĨ":99659,"éĵ¶":99660,"代表":99661,"é¦Ļ":99662,"帮":99663,"æİ¨è¿Ľ":99664,"çĽĺ":99665,"积æŀģ":99666,"éĥ¨éŨ":99667,"åŁ¹":99668,"æѦ":99669,"ä¸įä¼ļ":99670,"çŃij":99671,"éĢĻ":99672,"çݩ家":99673,"æĭ¿":99674,"åİĤ":99675,"æ¯Ľ":99676,"çģµ":99677,"æŃĮ":99678,"绿":99679,"å¦Ī":99680,"缼":99681,"é¦Ĩ":99682,"顺":99683,"èĦ¸":99684,"å°¼":99685,"丽":99686,"奥":99687,"éģĩ":99688,"è¯į":99689,"å°ģ":99690,"ä¸Ŀ":99691,"好çļĦ":99692,"æĭħ":99693,"èĦ±":99694,"æģ¶":99695,"åİļ":99696,"åĬ³":99697,"缣":99698,"æĬĺ":99699,"åı¥":99700,"æĢĢ":99701,"æŁĵ":99702,"书记":99703,"åĨł":99704,"é²ľ":99705,"æ¦Ĥ":99706,"éļIJ":99707,"å¹ħ":99708,"èµŀ":99709,"å¹ķ":99710,"æ¥Ń":99711,"éģĹ":99712,"åĪ¤":99713,"èĺ":99714,"å¶":99715,"æĬķèµĦ":99716,"è¡Įä¸ļ":99717,"äºij":99718,"çݯå¢ĥ":99719,"åѦçĶŁ":99720,"åIJĪä½ľ":99721,"åģ¥åº·":99722,"é£ŀ":99723,"ä¸ĢæŃ¥":99724,"ä¸Ģ缴":99725,"åıijçĶŁ":99726,"éĺ¿":99727,"é¢Ĩ导":99728,"åĸľæ¬¢":99729,"åºĶ该":99730,"çĤº":99731,"è®Ń":99732,"æĿĢ":99733,"港":99734,"交éĢļ":99735,"éĺ¶":99736,"éĴ¢":99737,"令":99738,"å°½":99739,"æ¯į":99740,"è¡£":99741,"ç²ī":99742,"顶":99743,"ä¹Łä¸į":99744,"æĬĵ":99745,"èĭ¦":99746,"幸":99747,"礼":99748,"第ä¸ī":99749,"大çļĦ":99750,"éģİ":99751,"çĥŁ":99752,"éģ¿":99753,"ä»į":99754,"åºĨ":99755,"æĢķ":99756,"è°¢":99757,"çĽĸ":99758,"å°Ħ":99759,"éľ²":99760,"æĸĹ":99761,"çĬ¶":99762,"åѸ":99763,"æ¯ķ":99764,"å·¨":99765,"çŁ¿":99766,"çļĩ":99767,"å¸Ń":99768,"çĹĩ":99769,"æī¬":99770,"延":99771,"侧":99772,"æ·¡":99773,"çļĦä¸Ģ":99774,"網":99775,"æ´ģ":99776,"ç¸":99777,"è§Ī":99778,"çŃ¹":99779,"ç§ĺ":99780,"è¯Ĭ":99781,"çı¾":99782,"èªī":99783,"毫":99784,"ð¨":99785,"åį´":99786,"æĪIJ为":99787,"èĥ½åĬĽ":99788,"é»Ħ":99789,"æĹħ游":99790,"èĪ¬":99791,"æ¯Ķè¾ĥ":99792,"èµ·æĿ¥":99793,"äºĨ解":99794,"èĩªçĦ¶":99795,"ä¸Ģ次":99796,"åŁºæľ¬":99797,"æĽ¾":99798,"综åIJĪ":99799,"èıľ":99800,"è§īå¾Ĺ":99801,"第äºĮ":99802,"è·ij":99803,"æ³¢":99804,"åĢĴ":99805,"ç¡Ģ":99806,"åħµ":99807,"èįī":99808,"çĶ³":99809,"çĶ°":99810,"æĤ£":99811,"è§Ħå®ļ":99812,"èĥľ":99813,"èµĦ产":99814,"梦":99815,"æľĿ":99816,"è¿ĻéĩĮ":99817,"夫":99818,"æĮ¥":99819,"ä½Ľ":99820,"å®Ī":99821,"鼶":99822,"æĸ¼":99823,"ç¯ĩ":99824,"å²Ľ":99825,"åĵ¥":99826,"éŃĶ":99827,"ä¸įåĪ°":99828,"æīĺ":99829,"åºĬ":99830,"欧":99831,"èį£":99832,"æ±ĩ":99833,"æī©":99834,"åģı":99835,"å¢Ļ":99836,"讯":99837,"å©ļ":99838,"æĥł":99839,"æ´ĭ":99840,"å®ľ":99841,"润":99842,"æħ¢":99843,"éĢı":99844,"宽":99845,"顾":99846,"ç´¯":99847,"污":99848,"çĪĨ":99849,"秣":99850,"æĥĬ":99851,"涨":99852,"饰":99853,"éĺµ":99854,"饮":99855,"æļĸ":99856,"åºŁ":99857,"æĹĹ":99858,"éļĶ":99859,"ç¶ĵ":99860,"åĭĻ":99861,"實":99862,"éĢĶ":99863,"æī«":99864,"çĥĪ":99865,"鼻":99866,"åĪij":99867,"éĹľ":99868,"éĹª":99869,"å¥ĭ":99870,"åĤ¨":99871,"缩":99872,"ä¾µ":99873,"å¬":99874,"ð¬¶":99875,"åĽ½éĻħ":99876,"ç»Ħç»ĩ":99877,"ä¸ĵä¸ļ":99878,"åıijçİ°":99879,"å¸ĮæľĽ":99880,"ç»ıèIJ¥":99881,"åı«":99882,"æĿ¥è¯´":99883,"éļľ":99884,"ä»»ä½ķ":99885,"交æĺĵ":99886,"éĩįçĤ¹":99887,"çļ®":99888,"ç»į":99889,"æ´¾":99890,"ç§ijåѦ":99891,"åºĶçĶ¨":99892,"建çŃij":99893,"èĤī":99894,"æĶ¹éĿ©":99895,"åŁºç¡Ģ":99896,"æ±ī":99897,"åĩºæĿ¥":99898,"è¿Ļä¹Ī":99899,"åĪļ":99900,"åĿIJ":99901,"ä¸įä»ħ":99902,"ä¼ļè®®":99903,"éĿł":99904,"åªĴä½ĵ":99905,"æ°¸":99906,"åĨ²":99907,"èĭı":99908,"央":99909,"çĪ¶":99910,"åłĤ":99911,"å®ŀéĻħ":99912,"è¡Ĺ":99913,"ç«¥":99914,"éĺħ":99915,"äºĭæĥħ":99916,"åİŁåĽł":99917,"éħ¸":99918,"以æĿ¥":99919,"娱":99920,"宫":99921,"åĿĹ":99922,"绩":99923,"éĩİ":99924,"ä¸įå¾Ĺ":99925,"ä¼łå¥ĩ":99926,"硬":99927,"åİħ":99928,"æĹ¢":99929,"ç»ĥ":99930,"èĦij":99931,"å¼±":99932,"æİĮ":99933,"è´´":99934,"æĮĤ":99935,"åħ³éĶ®":99936,"å°ļ":99937,"é¥Ń":99938,"åºĦ":99939,"çĻ¼":99940,"åľĭ":99941,"æİĪ":99942,"个æľĪ":99943,"äºĪ":99944,"å¸ģ":99945,"è·Ŀ":99946,"æ²ī":99947,"ç«Ł":99948,"åĨ¬":99949,"æĬ½":99950,"éĨĴ":99951,"å¼Ł":99952,"触":99953,"èģĺ":99954,"è±Ĩ":99955,"æļ´":99956,"åijĬè¯ī":99957,"豪":99958,"èµ¢":99959,"è·¨":99960,"è³ĩ":99961,"çĪ¸":99962,"æĬ±":99963,"浪":99964,"麻":99965,"仪":99966,"è¡¡":99967,"奶":99968,"çģ¾":99969,"赶":99970,"èĤ¥":99971,"å§IJ":99972,"åĢº":99973,"éľĩ":99974,"订":99975,"æ¬Ĭ":99976,"ç·":99977,"å»ī":99978,"ä¿Ĺ":99979,"å¿ĺ":99980,"å¦ĩ":99981,"ç¼ĵ":99982,"åŃķ":99983,"漫":99984,"è£ģ":99985,"çĩĥ":99986,"é»ĺ":99987,"çī¢":99988,"çĪ·":99989,"æĬµ":99990,"宾":99991,"æľīä¸Ģ":99992,"迹":99993,"è¿«":99994,"è²Į":99995,"æľīçļĦ":99996,"ð¬ĺ":99997,"è¿ĺæĺ¯":99998,"æīĢ以":99999,"ä¹Łæĺ¯":100000,"è¿ĻäºĽ":100001,"对äºİ":100002,"åIJ§":100003,"缮åīį":100004,"èĩªå·±çļĦ":100005,"èĥ½å¤Ł":100006,"å¦Ĥä½ķ":100007,"æľºæŀĦ":100008,"åıªæĺ¯":100009,"ç½ijç«Ļ":100010,"åħ¨éĿ¢":100011,"为äºĨ":100012,"å¼Ģåıij":100013,"æĸ°éĹ»":100014,"éĩijèŀį":100015,"继":100016,"客æĪ·":100017,"ä¸Ģèµ·":100018,"èĮ¶":100019,"åħ³æ³¨":100020,"æ°´å¹³":100021,"åİĨåı²":100022,"å¢ŀéķ¿":100023,"é±":100024,"åŁºéĩij":100025,"åºŃ":100026,"åı¶":100027,"ä¿ĥ":100028,"鼨":100029,"æ¶Īè´¹":100030,"èĪ¹":100031,"çŁ¥è¯Ĩ":100032,"æĪĺçķ¥":100033,"ç»ıéªĮ":100034,"å³°":100035,"æĽ²":100036,"èĦļ":100037,"åĨ°":100038,"å¤ı":100039,"å½Ĵ":100040,"ç¬Ķ":100041,"èĻij":100042,"çĶ²":100043,"åľĪ":100044,"è¯Ĺ":100045,"é½IJ":100046,"容æĺĵ":100047,"çłĶåıij":100048,"骨":100049,"纸":100050,"è·µ":100051,"æŧ":100052,"çķ¶":100053,"åĪ¸":100054,"è´·":100055,"åı¬":100056,"ç§ĭ":100057,"液":100058,"è¡ĮæĶ¿":100059,"çĮ®":100060,"èĤ¤":100061,"éĢIJ":100062,"è¶ĬæĿ¥":100063,"è¶ĬæĿ¥è¶Ĭ":100064,"æĦıè§ģ":100065,"èĪŀ":100066,"åīĤ":100067,"æ¶ī":100068,"ç¨ĭ度":100069,"åħ¬åħ±":100070,"械":100071,"æľ«":100072,"纯":100073,"åĶ±":100074,"æ´²":100075,"æĬ¢":100076,"æ¤į":100077,"å¿Ļ":100078,"ä¼°":100079,"å¼¹":100080,"æ³ī":100081,"æľĢ大":100082,"è¶ĭ":100083,"å·§":100084,"ç¦ģ":100085,"æī¶":100086,"åį±":100087,"çıł":100088,"çĨŁ":100089,"æĭľ":100090,"主ä¹ī":100091,"æĿĤ":100092,"éĻĦ":100093,"éģį":100094,"æIJŃ":100095,"æĮ¯":100096,"å¤ļå¹´":100097,"æķ¬":100098,"æijĦ":100099,"纷":100100,"å¼ĥ":100101,"湿":100102,"å¨ĺ":100103,"æ¡£":100104,"驶":100105,"æľĹ":100106,"æ®ĸ":100107,"æ¦ľ":100108,"åĵ¡":100109,"ä¸Ģä½ĵ":100110,"æŁ¥çľĭ":100111,"ç¹ģ":100112,"æµĵ":100113,"åħ¬å®ī":100114,"æ½ľ":100115,"è´¯":100116,"éªĹ":100117,"æIJľ":100118,"å·¡":100119,"è¬":100120,"éĬ":100121,"å§Ķä¼ļ":100122,"æĤł":100123,"åī©":100124,"æıŃ":100125,"åŃ£åº¦":100126,"ð«ĺ":100127,"ð¬¬":100128,"ä´":100129,"ðª":100130,"ä½Ĩæĺ¯":100131,"éĥ½æĺ¯":100132,"å¹³åı°":100133,"åŃ¦ä¹ł":100134,"åĵģçīĮ":100135,"ä¸Ķ":100136,"è¿Ļç§į":100137,"æĶ¿çŃĸ":100138,"æĭ¬":100139,"认为":100140,"ä¸ĢèĪ¬":100141,"æłĩåĩĨ":100142,"æĶ¯æĮģ":100143,"模å¼ı":100144,"åħ³ç³»":100145,"çļĦæĺ¯":100146,"è¿Ļä¸Ģ":100147,"ä¸įè¦ģ":100148,"çĶļ":100149,"ç²¾ç¥ŀ":100150,"æĭ¥":100151,"åĪ©çĶ¨":100152,"ä¿ĿæĬ¤":100153,"ä½ľçĶ¨":100154,"èĭ¥":100155,"åĽ½åĨħ":100156,"ä»ĭç»į":100157,"ä¸Ģä¸ĭ":100158,"å·¥ä¸ļ":100159,"缮æłĩ":100160,"æľĢåIJİ":100161,"ä»·åĢ¼":100162,"å°į":100163,"éĵģ":100164,"è°ģ":100165,"ç»ĵæŀĦ":100166,"éĽª":100167,"æĻºèĥ½":100168,"ä¼łç»Ł":100169,"ä½ĵèĤ²":100170,"çĶŁæĢģ":100171,"æĭį":100172,"æݪ":100173,"åĨľä¸ļ":100174,"çī¹èī²":100175,"è§Ħ模":100176,"æĹ¶ä»£":100177,"è¿ĩç¨ĭ":100178,"éĴĪ":100179,"æĿ¾":100180,"åĶIJ":100181,"åĮ»çĸĹ":100182,"çģ¯":100183,"åĪ¶éĢł":100184,"æł¸å¿ĥ":100185,"ä¸įåı¯":100186,"ç³»åĪĹ":100187,"åIJī":100188,"åľ£":100189,"åĢij":100190,"ä½³":100191,"æĿ¥çľĭ":100192,"æ¯ĶèµĽ":100193,"ä¸ĭæĿ¥":100194,"åĩºäºĨ":100195,"å¹²éĥ¨":100196,"微信":100197,"å½ĵåľ°":100198,"åį·":100199,"åį«çĶŁ":100200,"ä¼Ł":100201,"çĸ«æĥħ":100202,"è°·":100203,"åĩłä¸ª":100204,"éĺ´":100205,"çĶŁçī©":100206,"å°¤":100207,"ä¼Ĭ":100208,"èĤ¯":100209,"éĿ¢ç§¯":100210,"åĪĽéĢł":100211,"æı¡":100212,"åľĨ":100213,"æĻĵ":100214,"æĪIJäºĨ":100215,"åĩ¡":100216,"çĸ¾":100217,"ç«ŀäºī":100218,"讨":100219,"主é¢ĺ":100220,"é²ģ":100221,"迪":100222,"ä¿Ħ":100223,"æĢª":100224,"並":100225,"èĻļ":100226,"æ½®":100227,"çĥ§":100228,"èĢ³":100229,"æ±ł":100230,"éĢĤåIJĪ":100231,"æł¹æľ¬":100232,"åĬłçĽŁ":100233,"çĶµè§Ĩ":100234,"æ··":100235,"ç¼ĺ":100236,"çªĹ":100237,"çĬ¯":100238,"æĥ¯":100239,"æĦıä¹ī":100240,"åĬŀæ³ķ":100241,"ä¼ij":100242,"æ»ij":100243,"åĭĩ":100244,"æķ¢":100245,"寻":100246,"è¦Ĩ":100247,"éĢĥ":100248,"ç»ıçIJĨ":100249,"åĿı":100250,"æ³½":100251,"ä¹ĺ":100252,"åĪº":100253,"å±ı":100254,"é¡¿":100255,"亡":100256,"éĤĢ":100257,"åħ¼":100258,"åĭ¤":100259,"æ®ĭ":100260,"æĺł":100261,"æ¯ķä¸ļ":100262,"æĪª":100263,"è·Į":100264,"å£ģ":100265,"åı¦ä¸Ģ":100266,"羣å®ŀ":100267,"磨":100268,"è¯ļ":100269,"å¿ħè¦ģ":100270,"æģĭ":100271,"æĩĤ":100272,"å¾Ĵ":100273,"è°ĵ":100274,"æķı":100275,"æĻ¨":100276,"èĥ¸":100277,"æĭ¼":100278,"å¦Ļ":100279,"诸":100280,"èģĬ":100281,"æĤī":100282,"麼":100283,"åĩŃ":100284,"èĪĴ":100285,"æ¶Ĥ":100286,"è¿ģ":100287,"沿":100288,"å¡ij":100289,"æĽ¿":100290,"æ¾³":100291,"å¿į":100292,"èĢĹ":100293,"龸":100294,"åĩłå¹´":100295,"åĪĬ":100296,"èĦī":100297,"èħIJ":100298,"æ¡Į":100299,"çºł":100300,"æ»ļ":100301,"æĤ²":100302,"åĨĴ":100303,"妹":100304,"çķħ":100305,"纵":100306,"æijĩ":100307,"夺":100308,"è·¯ä¸Ĭ":100309,"忽":100310,"èĸª":100311,"æģIJ":100312,"æĦıæĢĿ":100313,"å«Į":100314,"æı´":100315,"æ°§":100316,"èĢĢ":100317,"éĺ»":100318,"轨":100319,"å¹»":100320,"æįķ":100321,"åĿ¦":100322,"åĵĪåĵĪ":100323,"çĭIJ":100324,"滨":100325,"è²»":100326,"è¿Ł":100327,"人éĥ½":100328,"ç»ĺ":100329,"åı¹":100330,"çµIJ":100331,"æī°":100332,"æ»ĭ":100333,"å¥ij":100334,"åĭŁ":100335,"確":100336,"ð¦":100337,"éĽĨåĽ¢":100338,"æĿİ":100339,"å¼Ģå±ķ":100340,"æıIJåįĩ":100341,"åħ¨åĽ½":100342,"汽车":100343,"åŃ¦æł¡":100344,"æł¹æį®":100345,"è¿Ļæĺ¯":100346,"åĩºçİ°":100347,"éĻĪ":100348,"ç½Ĺ":100349,"èİ·å¾Ĺ":100350,"åĪĺ":100351,"éĶĢåĶ®":100352,"æľªæĿ¥":100353,"éľĢæ±Ĥ":100354,"å®ŀæĸ½":100355,"åĿļæĮģ":100356,"åħ¨çIJĥ":100357,"éĵ¶è¡Į":100358,"æݧåĪ¶":100359,"é¡»":100360,"åľ°åĮº":100361,"æīĵéĢł":100362,"çļĦè¯Ŀ":100363,"帮åĬ©":100364,"ä½ĵç³»":100365,"è¾¾åĪ°":100366,"è§ĦåĪĴ":100367,"åŁ¹è®Ń":100368,"两个":100369,"æĬ¥åijĬ":100370,"åľ°æĸ¹":100371,"å®Įåħ¨":100372,"æİī":100373,"ç»ĵåIJĪ":100374,"å®£ä¼ł":100375,"æ³ķå¾ĭ":100376,"èīºæľ¯":100377,"çĶµå½±":100378,"說":100379,"ä¸ĢçĤ¹":100380,"è¶ħè¿ĩ":100381,"çĶµåŃIJ":100382,"æĢĿæĥ³":100383,"æķĻåѦ":100384,"éĺ¶æ®µ":100385,"åķĨä¸ļ":100386,"çī©æµģ":100387,"åĪĽä¸ļ":100388,"æĸ¹æ¡Ī":100389,"çݰ代":100390,"æ¡¥":100391,"èIJ½å®ŀ":100392,"带æĿ¥":100393,"产çĶŁ":100394,"ç§Ģ":100395,"æ³°":100396,"ä¹±":100397,"åħ·ä½ĵ":100398,"åĸĿ":100399,"èĵĿ":100400,"å®Ĺ":100401,"åįĩ级":100402,"æ·±åħ¥":100403,"ä¿ĿéĻ©":100404,"ç®Ģåįķ":100405,"çĹĽ":100406,"稳å®ļ":100407,"è¾Ĩ":100408,"å±ŀäºİ":100409,"å·Ŀ":100410,"ä¸įå°ij":100411,"åĴ¨":100412,"ä¸ľè¥¿":100413,"å½¢å¼ı":100414,"娱ä¹IJ":100415,"æŃ£å¸¸":100416,"鸡":100417,"åħħåĪĨ":100418,"å®ŀè·µ":100419,"éĩĮéĿ¢":100420,"è·³":100421,"èĻİ":100422,"æĪIJéķ¿":100423,"æļĹ":100424,"çĿ¡":100425,"罪":100426,"çIJĨ念":100427,"æĮij":100428,"èµĦæľ¬":100429,"å¤ļå°ij":100430,"ä¸ĭéĿ¢":100431,"å¸Ŀ":100432,"åħ¬å¼Ģ":100433,"æ¸IJ":100434,"éķ·":100435,"å±ĭ":100436,"欢è¿İ":100437,"å¿ĥçIJĨ":100438,"çĤİ":100439,"æ¹¾":100440,"è®ĵ":100441,"éĤĦ":100442,"ç³ĸ":100443,"ä¹Į":100444,"åĬ±":100445,"çīĻ":100446,"èħ¿":100447,"å²Ĺ":100448,"ä¼į":100449,"æĪIJåijĺ":100450,"åŃĶ":100451,"å°ıç¼ĸ":100452,"èij£":100453,"泡":100454,"åħĪè¿Ľ":100455,"åħ§":100456,"åĺ´":100457,"è´Ŀ":100458,"è»":100459,"æIJŀ":100460,"æ³Ľ":100461,"鸣":100462,"ç½²":100463,"èĽĭ":100464,"主任":100465,"缮çļĦ":100466,"ä¹ı":100467,"æ´¥":100468,"æĪ´":100469,"ä¸¥æł¼":100470,"çħ¤":100471,"çĮ«":100472,"åĶ¯":100473,"å°Ĭ":100474,"çĶľ":100475,"åŀĥ":100476,"åľ¾":100477,"æĭŁ":100478,"çĦ¦":100479,"é«Ķ":100480,"å®ı":100481,"æ©Ł":100482,"é©»":100483,"æĹģ":100484,"å½»":100485,"éĥ½ä¸į":100486,"æij©":100487,"ä»ĵ":100488,"ä¹³":100489,"岸":100490,"è°ĭ":100491,"大å¤ļ":100492,"çģŃ":100493,"èħ¾":100494,"æŁľ":100495,"èĪį":100496,"åħļçļĦ":100497,"å°ĺ":100498,"åįģå¹´":100499,"æĭĴ":100500,"裡":100501,"æŁĶ":100502,"å¹¼":100503,"éĶģ":100504,"ä¸ĵ项":100505,"æīİ":100506,"驾驶":100507,"ç¢İ":100508,"è¢ĭ":100509,"éĶĭ":100510,"壮":100511,"å°ĸ":100512,"çĶµæ±ł":100513,"è¿Ķ":100514,"æ¼ı":100515,"循":100516,"èıĮ":100517,"èĥĥ":100518,"è¾ħ":100519,"éĢĴ":100520,"èĥİ":100521,"éĻª":100522,"寿":100523,"å¥Ķ":100524,"çĮĽ":100525,"纹":100526,"çŁ¥åIJį":100527,"å¿Ĩ":100528,"æ¡ĥ":100529,"æ£ĭ":100530,"éĢĨ":100531,"çĤ¼":100532,"ç±į":100533,"çī§":100534,"æł·çļĦ":100535,"è¾Ľ":100536,"åłĨ":100537,"å®ŀåľ¨":100538,"ä¼ı":100539,"宿":100540,"èµı":100541,"è£Ĥ":100542,"åįĬå¹´":100543,"åĢ¾":100544,"满æĦı":100545,"梯":100546,"æĦıåij³":100547,"åѤ":100548,"ç¥Ŀ":100549,"æĻ¶":100550,"èµĶ":100551,"åģ¿":100552,"èĦĤ":100553,"ç½ļ":100554,"ç¢į":100555,"æ²ĥ":100556,"æĵį":100557,"å´ĩ":100558,"æļĤ":100559,"è·ĥ":100560,"æIJ¬":100561,"å©Ĩ":100562,"éī":100563,"éī´":100564,"åħ´è¶£":100565,"èIJ¥ä¸ļ":100566,"è®Ĭ":100567,"èĦı":100568,"è¾Ī":100569,"å·ŀå¸Ĥ":100570,"è´«åĽ°":100571,"ç©·":100572,"ä¸Ńå°ı":100573,"æ¼Ĥ":100574,"çĻĮ":100575,"èľľ":100576,"ä¼Ļä¼´":100577,"çīµ":100578,"æĤŁ":100579,"éĻ·":100580,"èµĽåŃ£":100581,"樣":100582,"åģ¶":100583,"æĺĨ":100584,"è¢Ń":100585,"æįIJ":100586,"èī°":100587,"æĤ¬":100588,"çĶ¢":100589,"èij¡":100590,"çĽĹ":100591,"å©´":100592,"å°İ":100593,"纽":100594,"åĢ¡":100595,"æī®":100596,"è¨Ń":100597,"æĬij":100598,"ç¡ķ":100599,"è¾ĸ":100600,"éĥģ":100601,"辩":100602,"éĤ»":100603,"çİ°åĩº":100604,"è¦ı":100605,"å½¹":100606,"éĺĶ":100607,"åīµ":100608,"诱":100609,"æĥij":100610,"æ·Ģ":100611,"é¢Ī":100612,"侦":100613,"æģ°":100614,"æ£Ģå¯Ł":100615,"éĨ«":100616,"çĦ¶æĺ¯":100617,"åĭĥ":100618,"èĮ«":100619,"äĵ":100620,"ð¬¸":100621,"ä½ľä¸º":100622,"çļĦ人":100623,"éĤ£ä¹Ī":100624,"ç¾İåĽ½":100625,"è¿ĺæľī":100626,"æıIJé«ĺ":100627,"èĻ½":100628,"åħ·æľī":100629,"åĮħæĭ¬":100630,"æĪĸèĢħ":100631,"ä¸įè¿ĩ":100632,"ä¸Ĭæµ·":100633,"åĮ»éĻ¢":100634,"èµĦéĩij":100635,"çĶļèĩ³":100636,"åĪ¶åº¦":100637,"解åĨ³":100638,"èģĶç½ij":100639,"继ç»Ń":100640,"建ç«ĭ":100641,"è¿Ľä¸ĢæŃ¥":100642,"æĿIJæĸĻ":100643,"ä»Ĭ天":100644,"å¿ħé¡»":100645,"åIJĦç§į":100646,"çİ°åľº":100647,"ä»ĸçļĦ":100648,"å¢ŀåĬł":100649,"é¢ĨåŁŁ":100650,"åıĤä¸İ":100651,"æĮģç»Ń":100652,"ä¹ĭä¸Ģ":100653,"çī¹åĪ«":100654,"é±¼":100655,"åħ±åIJĮ":100656,"åĬª":100657,"çİī":100658,"人们":100659,"åħĪçĶŁ":100660,"ä¼ĺåĬ¿":100661,"ä¿ĿæĮģ":100662,"ä½ľåĵģ":100663,"çīĽ":100664,"æĪIJæľ¬":100665,"æĶ¶åħ¥":100666,"åıĬæŶ":100667,"è´Łè´£":100668,"æİ¥åıĹ":100669,"èįIJ":100670,"åıªè¦ģ":100671,"羣çļĦ":100672,"导èĩ´":100673,"æľºåĪ¶":100674,"è¡ĮåĬ¨":100675,"æĸ°çļĦ":100676,"å®ĮåĸĦ":100677,"为ä»Ģä¹Ī":100678,"ä¸Ń央":100679,"æĪIJç«ĭ":100680,"æĦŁè§ī":100681,"åıĺåĮĸ":100682,"åıĹåĪ°":100683,"并ä¸į":100684,"åŃĻ":100685,"æĸ½å·¥":100686,"æĺİæĺ¾":100687,"è¿ĩåİ»":100688,"åıijæĮ¥":100689,"羣æŃ£":100690,"åŁºåľ°":100691,"æĺİç¡®":100692,"èĥ¡":100693,"许å¤ļ":100694,"ä¸Ģå¹´":100695,"æĸ¹åIJij":100696,"æģ©":100697,"çĽ¸ä¿¡":100698,"åľ³":100699,"详ç»Ĩ":100700,"äºĭä¸ļ":100701,"çĶŁåij½":100702,"åĴ¨è¯¢":100703,"æĸĩæĺİ":100704,"çijŀ":100705,"绿èī²":100706,"èİ«":100707,"æĦıè¯Ĩ":100708,"æĬķåħ¥":100709,"åĬłå¿«":100710,"æ¢ħ":100711,"ç¿»":100712,"å¼ĢæĶ¾":100713,"æĻ®éĢļ":100714,"åįıä¼ļ":100715,"æĪIJ绩":100716,"ä»Ļ":100717,"å¯Ĵ":100718,"è¯ģåĪ¸":100719,"认è¯Ĩ":100720,"丹":100721,"大éĩı":100722,"è¿ħ":100723,"åģļåĪ°":100724,"设æĸ½":100725,"è´¸æĺĵ":100726,"èĥ½æºIJ":100727,"æĹ¶æľŁ":100728,"ä¸Ģ天":100729,"æ²»çIJĨ":100730,"åĺī":100731,"å®ĩ":100732,"丰å¯Į":100733,"举è¡Į":100734,"æĪIJæŀľ":100735,"èĤ¯å®ļ":100736,"çĭĹ":100737,"åĬ¨åĬĽ":100738,"森":100739,"åĩłä¹İ":100740,"åĽłç´ł":100741,"æ°ijæĹı":100742,"æ´ŀ":100743,"ç½ijåıĭ":100744,"åIJĪçIJĨ":100745,"广大":100746,"æ®Ĭ":100747,"æ´Ľ":100748,"æĿ¯":100749,"èĴĻ":100750,"çĶ¨äºİ":100751,"èŀįèµĦ":100752,"ç¥ĸ":100753,"æľºæ¢°":100754,"举åĬŀ":100755,"èĩªåĬ¨":100756,"åĬŀåħ¬":100757,"é»ŀ":100758,"éĽĦ":100759,"åĢ¼å¾Ĺ":100760,"çĮª":100761,"以为":100762,"æĺĮ":100763,"è·Ŀ离":100764,"åIJ¸å¼ķ":100765,"ç»ķ":100766,"éļĨ":100767,"计ç®Ĺ":100768,"éĺŁä¼į":100769,"大ä¼ļ":100770,"å¼ķèµ·":100771,"çī¹çĤ¹":100772,"èĥ¶":100773,"å¹´è½»":100774,"æľ¬èº«":100775,"æľºåħ³":100776,"å®ĺæĸ¹":100777,"éĥij":100778,"æµĻ":100779,"è§Ĵèī²":100780,"èij£äºĭ":100781,"为主":100782,"æĹłè®º":100783,"ä¹łæĥ¯":100784,"æ¥ļ":100785,"æĭĵ":100786,"ç»Łè®¡":100787,"åħĦ":100788,"å¹¿æ³Ľ":100789,"åįĢ":100790,"污æŁĵ":100791,"è«ĭ":100792,"èĬĤ缮":100793,"伦":100794,"è¦ĨçĽĸ":100795,"èĢIJ":100796,"æī¶è´«":100797,"ç»ıåİĨ":100798,"éĩįè¦ģçļĦ":100799,"èĤ¡ä¸ľ":100800,"æĭĽèģĺ":100801,"åĽĽä¸ª":100802,"æĩī":100803,"èĥŀ":100804,"æijĨ":100805,"é«ĺéĢŁ":100806,"麦":100807,"åİŁåĪĻ":100808,"èݱ":100809,"æĽ´å¥½":100810,"éķľ":100811,"åĩĮ":100812,"åŀĥåľ¾":100813,"éĢ²":100814,"çģ°":100815,"éĵº":100816,"äºĭæķħ":100817,"çĶĺ":100818,"空æ°Ķ":100819,"é¾Ħ":100820,"èı²":100821,"çĵ¶":100822,"æĺ¨":100823,"æĹ¥æĬ¥":100824,"æµ®":100825,"åľ°åĽ¾":100826,"åijĪ":100827,"大åĬĽ":100828,"绪":100829,"å¸ħ":100830,"æľįåĭĻ":100831,"ä¸įéĶĻ":100832,"乡æĿij":100833,"å±¥":100834,"å¹³æĸ¹":100835,"éĹ²":100836,"æī£":100837,"ç´łè´¨":100838,"èµ´":100839,"éģŃ":100840,"èIJ¨":100841,"èĩªä¸»":100842,"éĩijå±ŀ":100843,"èī¯å¥½":100844,"两年":100845,"æ³¥":100846,"é¢ľ":100847,"精彩":100848,"ä¸Ńåįİ":100849,"æĻĭ":100850,"ä¹łè¿ij":100851,"ä¹łè¿ijå¹³":100852,"æĪĺ士":100853,"åģļçļĦ":100854,"éªij":100855,"æ»´":100856,"çĵľ":100857,"çīĪæĿĥ":100858,"èĤł":100859,"æľĥåĵ¡":100860,"çıį":100861,"種":100862,"仿":100863,"çī©ä¸ļ":100864,"åĢĭ人":100865,"妻":100866,"伸":100867,"æ±Ĺ":100868,"æĹº":100869,"çIJĨæĥ³":100870,"æij¸":100871,"è¿Ŀæ³ķ":100872,"å®Įæķ´":100873,"åݦ":100874,"è¸ı":100875,"æĸij":100876,"æ¡Ĥ":100877,"ä½ĵåĪ¶":100878,"師":100879,"æĿĨ":100880,"殿":100881,"æ¯ģ":100882,"é¦Ī":100883,"è§Ĵ度":100884,"欣":100885,"çĥ¦":100886,"èĤº":100887,"éĩĩ访":100888,"æijĺ":100889,"æĮ¡":100890,"æ·ĺ":100891,"åħ»èĢģ":100892,"çĤ¸":100893,"è¿Ī":100894,"åİī":100895,"åĿĬ":100896,"è¾£":100897,"åĩĿ":100898,"泪":100899,"çĸı":100900,"æİĺ":100901,"åĥıæĺ¯":100902,"éĽķ":100903,"ç¼Ŀ":100904,"èį·":100905,"æį·":100906,"åł¡":100907,"åı¥è¯Ŀ":100908,"çĸ¼":100909,"æłı":100910,"éģµ":100911,"碳":100912,"å·¥åķĨ":100913,"æIJº":100914,"åĪ¥":100915,"ä¹Ļ":100916,"æĹĭ":100917,"æĥľ":100918,"ä¸Ģ大":100919,"å±Ĥ次":100920,"èµĸ":100921,"æĬ¬":100922,"æ¨Ĥ":100923,"è¯ŀ":100924,"åħĴ":100925,"篮":100926,"èĤĥ":100927,"姿":100928,"æĬļ":100929,"çĵ·":100930,"çĶµåĬ¨":100931,"æĸ°åĨł":100932,"涵":100933,"ç¢ij":100934,"æ·®":100935,"æŨ":100936,"踪":100937,"æ¸Ķ":100938,"æĦĪ":100939,"åıĶ":100940,"åįĹçľģ":100941,"義":100942,"å§Ķ书记":100943,"貸":100944,"æ¶Į":100945,"è«ĸ":100946,"èIJĦ":100947,"æıı":100948,"忧":100949,"辦":100950,"å¦Ĩ":100951,"æīŃ":100952,"åijµ":100953,"éģ¥":100954,"許":100955,"ä»ĩ":100956,"åįģä¸ī":100957,"åī²":100958,"èªį":100959,"èĪ°":100960,"é¢ĩ":100961,"饱":100962,"çĭł":100963,"é«ĺçļĦ":100964,"çµ±":100965,"æħİ":100966,"é¢ģ":100967,"åIJĪéĢĤ":100968,"æµ´":100969,"èµĭ":100970,"æĬ¼":100971,"妥":100972,"éĻ¢éķ¿":100973,"èĢķ":100974,"辨":100975,"æħ°":100976,"åįģåĽĽ":100977,"æľµ":100978,"èĵĦ":100979,"æŀ¢":100980,"å»·":100981,"æĤĦ":100982,"涯":100983,"磩":100984,"åŃIJéĩĮ":100985,"çĬ¹":100986,"å±Ģéķ¿":100987,"éIJ":100988,"å¥ł":100989,"ä¼ļéķ¿":100990,"æĵļ":100991,"ä¸įåıĬ":100992,"åįģä¹Ŀ":100993,"欺":100994,"躺":100995,"éĺIJ":100996,"çºĮ":100997,"註":100998,"åĨĬ":100999,"èŃĺ":101000,"é«ĺçŃī":101001,"èħº":101002,"å¤ķ":101003,"ç»ij":101004,"åĶ¤":101005,"èķ´":101006,"çķľ":101007,"æħĭ":101008,"åıĻ":101009,"åıĥ":101010,"峡":101011,"人大":101012,"éħ¿":101013,"éģ©":101014,"奢":101015,"åı£æ°Ķ":101016,"éĮĦ":101017,"éı":101018,"åĭĺ":101019,"è´¿":101020,"éļª":101021,"éĭ":101022,"éļ¶":101023,"ð¥":101024,"ð¬£":101025,"ð£":101026,"ð«į":101027,"ð¬³":101028,"ð«ĵ":101029,"ð«Ħ":101030,"ð«Ł":101031,"ð¨±":101032,"äĹ":101033,"以åıĬ":101034,"æľīéĻIJ":101035,"åij¢":101036,"åIJĹ":101037,"çľĭåĪ°":101038,"计åĪĴ":101039,"è¿Ľåħ¥":101040,"缴æİ¥":101041,"åĪĨæŀIJ":101042,"åıªæľī":101043,"设å¤ĩ":101044,"åħ¶å®ŀ":101045,"åĬłå¼º":101046,"ä¸ŃçļĦ":101047,"ä¿Ŀéļľ":101048,"èĢģå¸Ī":101049,"人æīį":101050,"å¾ĹåĪ°":101051,"é£İéĻ©":101052,"ä¸Ģç§į":101053,"空éĹ´":101054,"æĪijåĽ½":101055,"ä¹ĭåīį":101056,"ä¸ĵ家":101057,"æĿ¨":101058,"æĹ¥æľ¬":101059,"群ä¼Ĺ":101060,"åıĤåĬł":101061,"æķĪæŀľ":101062,"æľīåħ³":101063,"家åºŃ":101064,"åĮºåŁŁ":101065,"åĬªåĬĽ":101066,"éļıçĿĢ":101067,"æĹłæ³ķ":101068,"交æµģ":101069,"è¡Į为":101070,"æ£ĢæŁ¥":101071,"æľŁéĹ´":101072,"å¦ĤæѤ":101073,"èĤ¡ä»½":101074,"å½ĵæŶ":101075,"è£ħå¤ĩ":101076,"åĩĨå¤ĩ":101077,"éħĴåºĹ":101078,"è¿IJåĬ¨":101079,"æıIJåĩº":101080,"å·¦åı³":101081,"æݪæĸ½":101082,"é£Łåĵģ":101083,"æ¶Īè´¹èĢħ":101084,"åѦéĻ¢":101085,"æĮĩ导":101086,"è¿IJèIJ¥":101087,"éĩį大":101088,"åĨľæĿij":101089,"éĢłæĪIJ":101090,"æĶ¿æ²»":101091,"éĴĪ对":101092,"æŃ£å¼ı":101093,"åıĸå¾Ĺ":101094,"éĤ£ä¸ª":101095,"éĽĨä¸Ń":101096,"åıªèĥ½":101097,"å¿«éĢŁ":101098,"身ä½ĵ":101099,"åħļåijĺ":101100,"èģĶåIJĪ":101101,"åĬĽéĩı":101102,"éĥ½æľī":101103,"æħ§":101104,"å¡Ķ":101105,"åĪ«äºº":101106,"表çİ°":101107,"æķħäºĭ":101108,"ä¸ĢåĪĩ":101109,"å°ĩ":101110,"èµĦæĸĻ":101111,"åŁ¹åħ»":101112,"éĺħ读":101113,"æľī人":101114,"èIJ¥éĶĢ":101115,"çĽijçĿ£":101116,"çݯä¿Ŀ":101117,"èĢĥèĻij":101118,"æ·±åľ³":101119,"严éĩį":101120,"èĮĥåĽ´":101121,"å§Ķåijĺ":101122,"çĽij管":101123,"ä¸ī个":101124,"è£ħä¿®":101125,"åħ¬éĩĮ":101126,"åĪĨåĪ«":101127,"çIJĨ解":101128,"飩":101129,"åĬłå·¥":101130,"è®¤çľŁ":101131,"ä¸į好":101132,"åݻ年":101133,"éĻįä½İ":101134,"æľºä¼ļ":101135,"åįıè®®":101136,"符åIJĪ":101137,"å¢ŀ强":101138,"æĬĢèĥ½":101139,"é¦ĸåħĪ":101140,"秦":101141,"ä¸ģ":101142,"å°¾":101143,"æľīäºĨ":101144,"åľ°äº§":101145,"æ¸ł":101146,"æĸ¹ä¾¿":101147,"移åĬ¨":101148,"éĢŁåº¦":101149,"å°¤åħ¶":101150,"éĢļçŁ¥":101151,"åĿĽ":101152,"éģ¿åħį":101153,"æģ¢":101154,"è´¡":101155,"èģĮå·¥":101156,"å®ŀåĬĽ":101157,"æĺ¯ä¸Ģç§į":101158,"åIJ¯åĬ¨":101159,"çĸ¾çĹħ":101160,"æĿ¥äºĨ":101161,"çĽ¸å¯¹":101162,"çİ°å®ŀ":101163,"èŀįåIJĪ":101164,"åIJĮæł·":101165,"åħ¬åijĬ":101166,"çī¹æ®Ĭ":101167,"ç´«":101168,"ä¸ĭåİ»":101169,"ä¼łæĴŃ":101170,"æľĢ好":101171,"ä¼ĺè´¨":101172,"æ²Ĵ":101173,"æĮº":101174,"æŦ":101175,"诺":101176,"ä¸ĢåIJį":101177,"éģĵè·¯":101178,"示èĮĥ":101179,"è¿ĩæĿ¥":101180,"åIJĮåѦ":101181,"é¼ĵ":101182,"æĿŃ":101183,"æľ¬æ¬¡":101184,"åIJĮæĦı":101185,"ä¸ĸ纪":101186,"ç¾Ĭ":101187,"欲":101188,"å·¥èīº":101189,"çĵ¦":101190,"人士":101191,"æľīæīĢ":101192,"ä»İäºĭ":101193,"æľīå¾Īå¤ļ":101194,"ä¸įäºĨ":101195,"å²Ĺä½į":101196,"åıĺå¾Ĺ":101197,"åĬ³åĬ¨":101198,"å¤Ħäºİ":101199,"å¹³åĿĩ":101200,"形象":101201,"å¡ŀ":101202,"åħ±äº«":101203,"çĿĽ":101204,"åĪ©æ¶¦":101205,"æŃ£æĺ¯":101206,"å¾Ģå¾Ģ":101207,"缸æ¯Ķ":101208,"横":101209,"åĪ·":101210,"æµĻæ±Ł":101211,"大éĥ¨åĪĨ":101212,"å¤ļ个":101213,"æĤ¨çļĦ":101214,"çĶµåķĨ":101215,"å¾®åįļ":101216,"å§ĭç»Ī":101217,"çĬ¯ç½ª":101218,"æĺ¯åľ¨":101219,"ç»ĦåIJĪ":101220,"åİŁæĿ¥":101221,"æ¸ħæ¥ļ":101222,"åIJĦåľ°":101223,"æĦŁåıĹ":101224,"å½ĵä¸Ń":101225,"è¶ĭåĬ¿":101226,"æĻ¯åĮº":101227,"羣æĺ¯":101228,"ä¾ĽåºĶ":101229,"转åŀĭ":101230,"çĭĤ":101231,"èĨľ":101232,"èĭĹ":101233,"å¿ł":101234,"å¾Ī大":101235,"èĤ¡æĿĥ":101236,"ç¾İåħĥ":101237,"æİĴåIJį":101238,"åĬ¨çī©":101239,"éĶħ":101240,"墨":101241,"主å¸Ń":101242,"å¾Ī好":101243,"ç»Ŀ对":101244,"æĿľ":101245,"转载":101246,"çĴĥ":101247,"æĿijæ°ij":101248,"åIJ¨":101249,"åĽŃåĮº":101250,"é«ĺ度":101251,"çī©è´¨":101252,"è¾ī":101253,"æĹ¥å¸¸":101254,"æıĴ":101255,"ä¸īå¹´":101256,"ä½ĵçİ°":101257,"æīįæĺ¯":101258,"代çIJĨ":101259,"ä¸į管":101260,"æģĴ":101261,"åľ°ä½į":101262,"ç²®":101263,"èĸĦ":101264,"æĺİçĻ½":101265,"ä¸Ģèĩ´":101266,"æĽ¼":101267,"åĵŃ":101268,"åĩ¤":101269,"åĬ²":101270,"æķĮ":101271,"æĪĺæĸĹ":101272,"主ä½ĵ":101273,"åħ¬å¸ĥ":101274,"åıĤèĢĥ":101275,"èĪªç©º":101276,"寺":101277,"åѦä¼ļ":101278,"åıįæĺł":101279,"ç¾İ丽":101280,"太éĺ³":101281,"建æĪIJ":101282,"æħ¢æħ¢":101283,"åIJĦ个":101284,"éĤ¦":101285,"ç»ĦæĪIJ":101286,"ä¸ī大":101287,"éĶ¦":101288,"大å¤ļæķ°":101289,"æ¦Ĥ念":101290,"éŃĤ":101291,"åħ¬çĽĬ":101292,"èįĴ":101293,"身份":101294,"æ·±åĪ»":101295,"åħ©":101296,"ç»ıåħ¸":101297,"åIJĦ项":101298,"èĻķ":101299,"è¿ĽæŃ¥":101300,"åįģäºĮ":101301,"æī§æ³ķ":101302,"æĥ³åĪ°":101303,"æĦŁæŁĵ":101304,"åķĨåĬ¡":101305,"å°ıç»Ħ":101306,"èĶ¬":101307,"çıŃåŃIJ":101308,"åIJĮå¿Ĺ":101309,"éĿ¢ä¸´":101310,"çĤĴ":101311,"å¤ļç§į":101312,"è§ĤçĤ¹":101313,"åĵªéĩĮ":101314,"å°Ŀ":101315,"å§Ĩ":101316,"èħ¹":101317,"åŁİåĮº":101318,"太å¤ļ":101319,"çĹħæ¯Ĵ":101320,"åľ¨äºİ":101321,"æīĢè°ĵ":101322,"æĻ°":101323,"æŀĿ":101324,"æĭĸ":101325,"å®ħ":101326,"æķ´æ²»":101327,"ä½ıæĪ¿":101328,"åģ·":101329,"çĨĬ":101330,"èµģ":101331,"æ°Ľ":101332,"æł¼å±Ģ":101333,"åŁºç¡Ģä¸Ĭ":101334,"èĥĨ":101335,"åħ½":101336,"鼶åĶ®":101337,"åĿ¡":101338,"女åŃ©":101339,"æĴŀ":101340,"åħ¨åĬĽ":101341,"åĴĸ":101342,"èĤ©":101343,"çľī":101344,"èĩ³äºİ":101345,"åħļç»Ħ":101346,"ä¸Ģ件":101347,"æĭĨ":101348,"äºĭå®ŀ":101349,"åĤ³":101350,"æ¹ĺ":101351,"網ç«Ļ":101352,"循çݯ":101353,"åIJĮæ¯Ķ":101354,"æĭĶ":101355,"åĮ»èį¯":101356,"åħ»æ®ĸ":101357,"åĽºå®ļ":101358,"å®ŀéĻħä¸Ĭ":101359,"è®°å¾Ĺ":101360,"åĪ©äºİ":101361,"æĤ¦":101362,"æĭ³":101363,"èĤĿ":101364,"æķĪçĽĬ":101365,"該":101366,"æ°ij主":101367,"çĹĩçĬ¶":101368,"風":101369,"å¹¼åĦ¿":101370,"å§ij":101371,"æĪĴ":101372,"ä¸ĭçļĦ":101373,"渡":101374,"å¹´åºķ":101375,"è®°å¿Ĩ":101376,"åIJIJ":101377,"大å¹ħ":101378,"å¾½":101379,"åħ¬ä¼Ĺ":101380,"ä¿¡å¿ĥ":101381,"çİĽ":101382,"ä¼ļä¸Ĭ":101383,"ä¹Ķ":101384,"æijĦå½±":101385,"æ£ĭçīĮ":101386,"éĻķ":101387,"åºĶæĢ¥":101388,"æĶ¶è´¹":101389,"æݧèĤ¡":101390,"仪å¼ı":101391,"çŀ¬":101392,"æīĢåľ¨":101393,"碰":101394,"å§ĵ":101395,"é¡Į":101396,"æĶ¯éĥ¨":101397,"使åij½":101398,"çĤī":101399,"å¯Ħ":101400,"翼":101401,"åľ°ä¸ĭ":101402,"è¾ŀ":101403,"俱":101404,"主æĮģ":101405,"è´§å¸ģ":101406,"æģ¨":101407,"èĤĮ":101408,"çĽĪ":101409,"éĶ»":101410,"å¿ĹæĦ¿":101411,"类似":101412,"æĮĸ":101413,"éĢ»":101414,"總":101415,"纪念":101416,"åķ¥":101417,"弯":101418,"åIJįåŃĹ":101419,"åģ¥èº«":101420,"çļĦå¿ĥ":101421,"驱":101422,"èĥĮåIJİ":101423,"æ³ķå¸Ī":101424,"ç²Ĵ":101425,"èĥ½éĩı":101426,"è¾°":101427,"èī³":101428,"å½¼":101429,"段æŶéĹ´":101430,"åIJĪæ³ķ":101431,"æĵ¦":101432,"ç¾½":101433,"åݨ":101434,"æĪij说":101435,"äºĭåĬ¡":101436,"åĩłå¤©":101437,"åħģ":101438,"ç¼´":101439,"åįĵ":101440,"两ç§į":101441,"çĭ¬çī¹":101442,"帶":101443,"éĴ»":101444,"æĥ©":101445,"é¢ĨåħĪ":101446,"è¶³å¤Ł":101447,"壳":101448,"æĦıåij³çĿĢ":101449,"åĪĨå¸ĥ":101450,"ä¹ĥ":101451,"éģĭ":101452,"佩":101453,"è°±":101454,"çģ£":101455,"èį¡":101456,"贯彻":101457,"å¹¾":101458,"ç£ģ":101459,"åħ¸åŀĭ":101460,"åīĩ":101461,"åĨ»":101462,"æ¬ł":101463,"ä¸įä¹ħ":101464,"浦":101465,"éŃħ":101466,"å¼ĢäºĨ":101467,"使çĶ¨èĢħ":101468,"è¿Ļ款":101469,"å°Ī":101470,"èĦ±è´«":101471,"æĶ»åĿļ":101472,"ç®Ĺæĺ¯":101473,"ç¨Ģ":101474,"æĹłäºº":101475,"åłµ":101476,"å¥ı":101477,"éĥ½å¸Ĥ":101478,"åı¯è§ģ":101479,"ä¸įåĩº":101480,"æ·»":101481,"äºı":101482,"ç¾İ好":101483,"èĥĸ":101484,"飵":101485,"æłĩå¿Ĺ":101486,"èĬĤèĥ½":101487,"æĬ«":101488,"å°º":101489,"寸":101490,"ä¸Ģ代":101491,"é¢Ĺ":101492,"èĢ¶":101493,"èĴ¸":101494,"åĸ®":101495,"滿":101496,"çĮľ":101497,"æµĨ":101498,"åŁĥ":101499,"åįĥä¸ĩ":101500,"èµĮ":101501,"èģ²":101502,"ä½ľé£İ":101503,"質":101504,"寨":101505,"年人":101506,"åį°è±¡":101507,"桶":101508,"æĴ¤":101509,"åįģäºĶ":101510,"æ¯ħ":101511,"沪":101512,"åĽ½æľī":101513,"大éĩıçļĦ":101514,"御":101515,"å¯ĵ":101516,"è¦ĸ":101517,"æ¼Ĥ亮":101518,"çľł":101519,"çĤŃ":101520,"é»İ":101521,"èĻ¹":101522,"åĪ©äºļ":101523,"èŃī":101524,"æµı":101525,"åįģåħ«":101526,"丢":101527,"è¾½":101528,"æľīä¸ĢäºĽ":101529,"æħĪ":101530,"åģľè½¦":101531,"å®ł":101532,"解æĶ¾":101533,"æľīå¤ļ":101534,"éĤĬ":101535,"常è§ģ":101536,"æĬ¹":101537,"纤":101538,"親":101539,"æ¡Ĩ":101540,"èİŀ":101541,"æ°§åĮĸ":101542,"è¿Ļ件":101543,"åĩ°":101544,"æŁ´":101545,"åıijçĶµ":101546,"é¼ł":101547,"转åĮĸ":101548,"å¨ĥ":101549,"æĮ¤":101550,"罩":101551,"å¯ĨåĪĩ":101552,"æĪijä¸į":101553,"é«ĺæĸ°":101554,"ä¸Ģç¯ĩ":101555,"è¿Ľç¨ĭ":101556,"è¡°":101557,"è¿ĺä¸į":101558,"çħĮ":101559,"æĸ°åįİ":101560,"èĤ¿":101561,"滩":101562,"ä¸Ģæµģ":101563,"è¯Ī":101564,"å®ŀä½ĵ":101565,"å¤ĸåĽ½":101566,"躲":101567,"èµł":101568,"覺":101569,"æ¢Ŀ":101570,"ä¸įè§ģ":101571,"è¨Ĭ":101572,"åĮ¹":101573,"åįµ":101574,"çĩ¥":101575,"æħķ":101576,"齿":101577,"å®´":101578,"饼":101579,"èij¡èIJĦ":101580,"å°ıå¿ĥ":101581,"æģ¼":101582,"éĻĮ":101583,"æĺĤ":101584,"åĥ¹":101585,"èĬĿ":101586,"æ¯ı个人":101587,"åīįæıIJ":101588,"ä½ĵä¼ļ":101589,"æ¨Ļ":101590,"æIJľçĭIJ":101591,"对åħ¶":101592,"丧":101593,"èľĤ":101594,"浸":101595,"調":101596,"åĿª":101597,"é¢ĸ":101598,"åIJį为":101599,"笼":101600,"èĪĮ":101601,"æľ¬ä¹¦":101602,"èģ¯":101603,"纺":101604,"ç®Ģ缴":101605,"éĽ¢":101606,"ç¾İçļĦ":101607,"éļ¨":101608,"é«ĺå³°":101609,"è¿Ļ家":101610,"åĤ¬":101611,"å°¸":101612,"ç¡ķ士":101613,"èŃ·":101614,"è°¨":101615,"æĺı":101616,"æĶ¿åįı":101617,"è¡Ķ":101618,"ç¿Ĵ":101619,"åľĴ":101620,"åĽ½æ°ij":101621,"主è§Ĵ":101622,"è£ķ":101623,"伪":101624,"åºŀ":101625,"æ°ijèIJ¥":101626,"æĥ§":101627,"ç§ĺ书":101628,"çĹķ":101629,"çĻ¾åĪĨ":101630,"溶":101631,"æĹłçĸij":101632,"çļĦçľ¼":101633,"æĵİ":101634,"ä¼Łå¤§":101635,"å½°":101636,"åħ¬å®īå±Ģ":101637,"ç³ķ":101638,"å¼¥":101639,"åĤĻ":101640,"ä¹¾":101641,"毫ä¸į":101642,"注æĺİ":101643,"åī¯æĢ»":101644,"æĦī":101645,"æķ¦":101646,"馨":101647,"æĶĢ":101648,"éĢĿ":101649,"åı¯éĿł":101650,"夸":101651,"åľĺ":101652,"éĿ¢ä¸Ĭ":101653,"æĬĸ":101654,"èĦĨ":101655,"é©°":101656,"ä¼IJ":101657,"妨":101658,"å®ļäºĨ":101659,"ç³Ĭ":101660,"æŃ¡":101661,"éĥ¨éķ¿":101662,"ç§ī":101663,"èĪĨ":101664,"åĪijäºĭ":101665,"åIJµ":101666,"æ¤Ĵ":101667,"è¡ĵ":101668,"豫":101669,"èı©":101670,"åѵ":101671,"饲":101672,"就好":101673,"åłª":101674,"ä¸īè§Ĵ":101675,"åľºæ¯ĶèµĽ":101676,"ä¸įåģľ":101677,"æĵħ":101678,"åħ¨æĸĩ":101679,"æ³ģ":101680,"åѦä½į":101681,"æ±°":101682,"éłĺ":101683,"åıł":101684,"éļĽ":101685,"å¸IJ":101686,"çľĭåĩº":101687,"åĮł":101688,"å±ĢéĿ¢":101689,"æ³Į":101690,"è°Ĭ":101691,"åIJĮæľŁ":101692,"æĬķæłĩ":101693,"奴":101694,"æĿ¥çľĭçľĭ":101695,"èĦ¾":101696,"èŀº":101697,"æŃī":101698,"çĽ¯":101699,"ç¨İåĬ¡":101700,"å»Ĭ":101701,"æİ©":101702,"æħ¨":101703,"çĽ¼":101704,"èĬĴ":101705,"è®Ģ":101706,"æĮ£":101707,"èĮħ":101708,"æĸ¥":101709,"æ¤ħ":101710,"åĪ°æĿ¥":101711,"èijĹä½ľ":101712,"çĭ±":101713,"äºĮæīĭ":101714,"ä»İæĿ¥":101715,"çĸ²":101716,"åºĬä¸Ĭ":101717,"æĸ°æµª":101718,"æ³Ħ":101719,"å¢ŀåĢ¼":101720,"丼":101721,"æļij":101722,"ä»İä¸ļ":101723,"æ·ĭ":101724,"å¤ļæł·":101725,"æľ´":101726,"份é¢Ŀ":101727,"æŀ£":101728,"西çľģ":101729,"æľ¬è´¨":101730,"深深":101731,"èīĩ":101732,"绵":101733,"产åĢ¼":101734,"æ¼ł":101735,"èħ»":101736,"çŃĽ":101737,"åİĮ":101738,"æģŃ":101739,"å«Įçĸij":101740,"æĪ¶":101741,"æ»ŀ":101742,"èĨĢ":101743,"åĬ£":101744,"座è°Ī":101745,"常æĢģ":101746,"çļĦæĥħ":101747,"覽":101748,"å¯Ĥ":101749,"åĮĨ":101750,"èĩº":101751,"顯":101752,"çķı":101753,"éģ£":101754,"åįľ":101755,"çŃīå¥ĸ":101756,"責":101757,"溯":101758,"éİ":101759,"çĤ¹å¤´":101760,"èĵ¬":101761,"決":101762,"éħ¬":101763,"éģĬ":101764,"è³¼":101765,"註åĨĬ":101766,"æľ¬æĬ¥":101767,"çµķ":101768,"æ´»æĢ§":101769,"åħij":101770,"éĮ¯":101771,"åĨ¶":101772,"åĸ»":101773,"æºĸ":101774,"èĤ¢":101775,"æºĥ":101776,"æŬ":101777,"åīĬ":101778,"çIJĨäºĭ":101779,"å±ł":101780,"沧":101781,"èļĢ":101782,"鼻åŃIJ":101783,"为æŃ¢":101784,"常å§Ķ":101785,"çµĤ":101786,"éĬ·":101787,"çĭĢ":101788,"ä¾£":101789,"èĥĢ":101790,"èŃ°":101791,"çĶ¨è½¦":101792,"åĻª":101793,"æŃ·":101794,"åįĶ":101795,"åĪ¹":101796,"ç«Łæĺ¯":101797,"é©Ĺ":101798,"èIJĿ":101799,"çĻ«":101800,"çĹ«":101801,"æѧ":101802,"å¼Ĭ":101803,"媽":101804,"çıĬ":101805,"è¡·":101806,"éľī":101807,"åŁºçĿ£":101808,"éļ±":101809,"æ°¨":101810,"绸":101811,"å°¼æĸ¯":101812,"çĥĺ":101813,"æľŁåĨħ":101814,"è°ħ":101815,"éĽĩ":101816,"éļĻ":101817,"åĸī":101818,"åī¥":101819,"çĹĺ":101820,"æĮ½":101821,"çĵ£":101822,"æ¹Ľ":101823,"樱":101824,"æ¾İ":101825,"æ¹ĥ":101826,"åĨ¬å¥¥":101827,"棵":101828,"å®°":101829,"åŀĴ":101830,"æ§ĭ":101831,"ä¾Ī":101832,"èĮĦ":101833,"åĺ¿":101834,"èıĩ":101835,"çĻĤ":101836,"åĬĥ":101837,"éį":101838,"èĶ½":101839,"çŀŃ":101840,"æķŀ":101841,"ä¹ĸ":101842,"飧":101843,"è¾ľ":101844,"æĩĪ":101845,"ä½£":101846,"çŀ»":101847,"åŁĶ":101848,"èĪħ":101849,"å®ŀäºĭ":101850,"é¨":101851,"姥":101852,"絡":101853,"åĺ»":101854,"çķ¢":101855,"æ²ĥå°Ķ":101856,"è¿Ħ":101857,"èĤĩ":101858,"æħij":101859,"ã§":101860,"äı":101861,"ðł":101862,"ð¬ĩ":101863,"ð«Ń":101864,"ð«IJ":101865,"ã³":101866,"©½":101867,"ð«ł":101868,"ãĽ":101869,"ð¬į":101870,"é¿":101871,"ð¬Ĵ":101872,"ãĻ":101873,"ð¬¤":101874,"ð¬´":101875,"ð«ĸ":101876,"ð¤":101877,"ã¬":101878,"ä²":101879,"ð«Ķ":101880,"ð«ļ":101881,"è¦ģæ±Ĥ":101882,"ä¸ĢäºĽ":101883,"å®ŀçİ°":101884,"èĢĮä¸Ķ":101885,"åĽłæѤ":101886,"çĶ±äºİ":101887,"åħ³äºİ":101888,"çĦ¶åIJİ":101889,"æݨåĬ¨":101890,"ä¸Ģæł·":101891,"æĮīçħ§":101892,"è¿Ļæł·çļĦ":101893,"å½¢æĪIJ":101894,"æľīäºĽ":101895,"æĽ´åĬł":101896,"ç»ıè¿ĩ":101897,"建议":101898,"æ²»çĸĹ":101899,"ä½łä»¬":101900,"æīįèĥ½":101901,"ä¿ĥè¿Ľ":101902,"åijĺå·¥":101903,"ä½ĵéªĮ":101904,"èĪĩ":101905,"åģļ好":101906,"ä¿Ŀè¯ģ":101907,"æķ´ä¸ª":101908,"æĺ¯ä¸Ģ个":101909,"éĩĩçĶ¨":101910,"çIJĨ论":101911,"æ¯Ķå¦Ĥ":101912,"ä¸ĬçļĦ":101913,"æݨèįIJ":101914,"çĶ³è¯·":101915,"天空":101916,"éĥ¨èIJ½":101917,"åįģåĪĨ":101918,"æĿ¥èĩª":101919,"ä¹ĭéĹ´":101920,"è°ĥæķ´":101921,"æ¯ı天":101922,"è°ĥæŁ¥":101923,"æĤ£èĢħ":101924,"è¿ĩç¨ĭä¸Ń":101925,"é¦Ļ港":101926,"广åijĬ":101927,"éĿ¢å¯¹":101928,"满足":101929,"éķ¿æľŁ":101930,"è§ĦèĮĥ":101931,"æķ´ä½ĵ":101932,"æĶ¹åıĺ":101933,"æĻºæħ§":101934,"å¦Īå¦Ī":101935,"å¦Ĥä»Ĭ":101936,"åIJĪåIJĮ":101937,"éĥ½ä¼ļ":101938,"åĦ¿ç«¥":101939,"åĩıå°ij":101940,"éŁ³ä¹IJ":101941,"ç»ı常":101942,"ä¸Ĭå¸Ĥ":101943,"ä¼ĺç§Ģ":101944,"çļĦéĩįè¦ģ":101945,"ä¸ĢæĿ¡":101946,"æµ·å¤ĸ":101947,"åı¦å¤ĸ":101948,"ä¸Ģ家":101949,"åİĭåĬĽ":101950,"大åŀĭ":101951,"çľĭçĿĢ":101952,"åĪĢ":101953,"幸ç¦ı":101954,"æݨ广":101955,"åIJĽ":101956,"å¾IJ":101957,"æī¾åĪ°":101958,"äºİæĺ¯":101959,"èĩªèº«":101960,"ä¸Ģä½į":101961,"åľŁåľ°":101962,"åĬłåħ¥":101963,"æݢ索":101964,"æ¢ģ":101965,"主åĬ¨":101966,"å°±ä¸ļ":101967,"女æĢ§":101968,"çªģçł´":101969,"ä¸įåIJĮçļĦ":101970,"è¿IJè¾ĵ":101971,"èĩªçĶ±":101972,"å±ħæ°ij":101973,"æŃ¤æ¬¡":101974,"çļĦæŶéĹ´":101975,"家éķ¿":101976,"ä¸Ģ个人":101977,"æ£Ģæµĭ":101978,"åĨħéĥ¨":101979,"广å·ŀ":101980,"缴æĴŃ":101981,"ä»İèĢĮ":101982,"贷款":101983,"åı¬å¼Ģ":101984,"æĶ¹éĢł":101985,"人çĶŁ":101986,"å±ķ示":101987,"æ¯ıå¹´":101988,"女人":101989,"çļĦæĸ¹å¼ı":101990,"æķĪçİĩ":101991,"å±±ä¸ľ":101992,"æ¸łéģĵ":101993,"ä¼¼ä¹İ":101994,"æ¡Ī件":101995,"åĪ©çĽĬ":101996,"çľĭçľĭ":101997,"å¿ĥéĩĮ":101998,"ç»´æĬ¤":101999,"å®Ŀå®Ŀ":102000,"ç½ijä¸Ĭ":102001,"论åĿĽ":102002,"å°±åı¯ä»¥":102003,"ä¸į足":102004,"æģ¢å¤į":102005,"å¸ĥå±Ģ":102006,"è´¡çĮ®":102007,"ä¸ĭéĻį":102008,"æİĮæı¡":102009,"çļ®èĤ¤":102010,"å·¥åħ·":102011,"éĩįåºĨ":102012,"åĵģè´¨":102013,"æݨåĩº":102014,"çĶ·äºº":102015,"æī¿æĭħ":102016,"çªģåĩº":102017,"èĢĮè¨Ģ":102018,"æ²Ł":102019,"åįıè°ĥ":102020,"æĺ¯ä»Ģä¹Ī":102021,"汤":102022,"æĴij":102023,"çĭ¬ç«ĭ":102024,"çݯèĬĤ":102025,"æī©å¤§":102026,"æ´ª":102027,"æĿ°":102028,"çĽIJ":102029,"ä»ģ":102030,"æ¶īåıĬ":102031,"èĢģ人":102032,"åį³ä½¿":102033,"åįĹ京":102034,"éħįåIJĪ":102035,"鬼":102036,"çĪ¶äº²":102037,"ç½Ĺæĸ¯":102038,"å°ıåĮº":102039,"æķĻæİĪ":102040,"åĨ³çŃĸ":102041,"é¢Ħ计":102042,"æľ¬äºº":102043,"伯":102044,"竹":102045,"åĪ°åºķ":102046,"å¸Ĥæ°ij":102047,"åĩºåı£":102048,"éĩĩè´Ń":102049,"æĢ»ç»ĵ":102050,"æѦæ±ī":102051,"åĬłå¤§":102052,"å¹¿ä¸ľ":102053,"æµģç¨ĭ":102054,"人åı£":102055,"å¦Ĥæŀľä½ł":102056,"åĩºåİ»":102057,"åĩī":102058,"åĨľæ°ij":102059,"çݰ象":102060,"åĬĽåº¦":102061,"ç»ĻäºĪ":102062,"åħļå§Ķ":102063,"è¯Ńè¨Ģ":102064,"线ä¸Ĭ":102065,"æĢİæł·":102066,"åĦ¿åŃIJ":102067,"ç¡®å®ŀ":102068,"ä¹ĭå¤ĸ":102069,"éĥ½åľ¨":102070,"èī¾":102071,"çļĦæĥħåĨµ":102072,"éĩĮçļĦ":102073,"åĽ´ç»ķ":102074,"æĽ´å¤ļçļĦ":102075,"ä¾Ŀæ³ķ":102076,"åħ¬åĽŃ":102077,"家éĩĮ":102078,"æ¯į亲":102079,"ä¸įåĨį":102080,"èĭ¹":102081,"æ³ķéĻ¢":102082,"éŁ©åĽ½":102083,"缸å½ĵ":102084,"ä¸įçŁ¥":102085,"è¯Ħä¼°":102086,"ä¸įçĶ¨":102087,"顺åĪ©":102088,"éĩįè§Ĩ":102089,"è´¢åĬ¡":102090,"ä»ĸåĢij":102091,"åıijè¡Į":102092,"ä¸ĵéŨ":102093,"åħ·å¤ĩ":102094,"并ä¸įæĺ¯":102095,"足çIJĥ":102096,"éŀĭ":102097,"åıij表":102098,"æ°¸è¿ľ":102099,"èIJ¥åħ»":102100,"éħįå¥Ĺ":102101,"æķ´åIJĪ":102102,"è´º":102103,"åĽŀçŃĶ":102104,"æĶ¶çĽĬ":102105,"ä¹Łè®¸":102106,"è»Ĭ":102107,"æݥ触":102108,"æĶ»åĩ»":102109,"åĽĽå·Ŀ":102110,"æĢ§èĥ½":102111,"åĽŀåĪ°":102112,"èħ°":102113,"ä¹Łæ²¡æľī":102114,"å¼Ħ":102115,"设ç«ĭ":102116,"éĺ²æݧ":102117,"æĬĢå·§":102118,"éĢļ常":102119,"è´¢æĶ¿":102120,"éĥ¨ç½²":102121,"åľºæĻ¯":102122,"æ±Łèĭı":102123,"表达":102124,"åĸ·":102125,"女åĦ¿":102126,"èĪ¶":102127,"給":102128,"ä¼ļåijĺ":102129,"æĪĸ许":102130,"亩":102131,"举æĸ¹":102132,"天津":102133,"è¿ijå¹´":102134,"çľĭæĿ¥":102135,"æ¯Ķä¾ĭ":102136,"岩":102137,"éĵľ":102138,"çİ»":102139,"å®ŀéªĮ":102140,"æĢĿç»´":102141,"æĭħå¿ĥ":102142,"æ²Ī":102143,"身边":102144,"æ·±åĮĸ":102145,"ç²¾åĩĨ":102146,"ç§ģæľį":102147,"æ¶Īéĺ²":102148,"åİ»äºĨ":102149,"ç»Ĩèĥŀ":102150,"çIJĥéĺŁ":102151,"æĺİæĺŁ":102152,"é£Łçī©":102153,"å¾Īå¿«":102154,"è®©ä½ł":102155,"ä¿¡çĶ¨":102156,"åĶ¯ä¸Ģ":102157,"åħ¶å®ĥ":102158,"çŃīæĸ¹éĿ¢":102159,"å¾ĭå¸Ī":102160,"æŃ»äº¡":102161,"æŁ³":102162,"ä¸Ģæī¹":102163,"ä¸Ĭ涨":102164,"æľºåľº":102165,"å½¢åĬ¿":102166,"æĦ¿æĦı":102167,"éĽĨä½ĵ":102168,"æĸ°åŀĭ":102169,"æįŁå¤±":102170,"æĽ¸":102171,"ä¸ĭåįĪ":102172,"æ¯ı次":102173,"æĪIJå°±":102174,"åħ¬è·¯":102175,"èĻ«":102176,"åĴ±":102177,"西å®ī":102178,"æľĢä½³":102179,"ç§ijçłĶ":102180,"å¤įæĿĤ":102181,"æľºåĻ¨":102182,"çĪ±æĥħ":102183,"çħ§çīĩ":102184,"å¹´é¾Ħ":102185,"è³ĩæĸĻ":102186,"ç²Ĺ":102187,"åĩĨç¡®":102188,"åĬłä¸Ĭ":102189,"åĩºçīĪ":102190,"è°IJ":102191,"家å±ħ":102192,"èĥĮæĻ¯":102193,"ä¸Ģ线":102194,"äºĭ项":102195,"åĬ¨ä½ľ":102196,"祥":102197,"æĢ»ä½ĵ":102198,"æĪ¿åŃIJ":102199,"ä¹Łå°±æĺ¯":102200,"大æ¦Ĥ":102201,"é«ĺæķĪ":102202,"åIJ¹":102203,"æİĪæĿĥ":102204,"éĻĦè¿ij":102205,"æ¡Īä¾ĭ":102206,"éĹ¹":102207,"çĪ¸çĪ¸":102208,"彩票":102209,"æĢĴ":102210,"举æĬ¥":102211,"æĻ®éģį":102212,"çķĻä¸ĭ":102213,"è¡£æľį":102214,"æĹłè®ºæĺ¯":102215,"åħħ满":102216,"深度":102217,"æ¡ij":102218,"æĪªèĩ³":102219,"带æĿ¥çļĦ":102220,"éĻµ":102221,"æĦŁæĥħ":102222,"èµļ":102223,"åĵªäºĽ":102224,"æķ´æĶ¹":102225,"æĪIJçĨŁ":102226,"å¨ľ":102227,"é¼»":102228,"磼":102229,"çĽ¾":102230,"好好":102231,"ç¬¬åĽĽ":102232,"åĨłåĨĽ":102233,"è´¢å¯Į":102234,"æľĢ好çļĦ":102235,"车åŀĭ":102236,"éĸĢ":102237,"åį³å°Ĩ":102238,"åĪĨ为":102239,"éĿĴå²Ľ":102240,"纷纷":102241,"ä»ĬæĹ¥":102242,"平衡":102243,"å¹³æĸ¹ç±³":102244,"éĤ£ç§į":102245,"åĩºçĶŁ":102246,"éĿĴæĺ¥":102247,"人群":102248,"人工":102249,"ä¹ĭä¸ĭ":102250,"æ¹ĸåĮĹ":102251,"åľ¨æѤ":102252,"åįļ士":102253,"æŶåĪ»":102254,"æ²³åĮĹ":102255,"æĶ¾å¼ĥ":102256,"éĢļéģĵ":102257,"森æŀĹ":102258,"çĸĨ":102259,"æķ¸":102260,"èĬ³":102261,"æīĵåĩ»":102262,"æĽ¹":102263,"åĮĸåѦ":102264,"æĥ³è±¡":102265,"ä¸ĩ人":102266,"è´¢ç»ı":102267,"åħĥç´ł":102268,"ä¼ļ计":102269,"åħ¨ä½ĵ":102270,"æĦĽ":102271,"é«ĺä¸Ń":102272,"æľºéģĩ":102273,"å£°éŁ³":102274,"æĹħè¡Į":102275,"浩":102276,"æŁ±":102277,"å°ijå¹´":102278,"åĽ½å¤ĸ":102279,"èijĹåIJį":102280,"çĶŁåŃĺ":102281,"å§ľ":102282,"带é¢Ĩ":102283,"é¢ľèī²":102284,"ä¸Ĭä¸ĭ":102285,"产ä¸ļéĵ¾":102286,"æĽ´å¥½çļĦ":102287,"å²Ń":102288,"ä¼ĺæĥł":102289,"便æĺ¯":102290,"åħ§å®¹":102291,"ä¸Ģåıª":102292,"çIJ´":102293,"梦æĥ³":102294,"秣èµģ":102295,"å¼ĢåIJ¯":102296,"è´Ńçī©":102297,"åĮħåIJ«":102298,"åĪ©çİĩ":102299,"èµ·äºĨ":102300,"æľīåĬĽ":102301,"éĤ£éĩĮ":102302,"审æī¹":102303,"对æīĭ":102304,"çİ°éĩij":102305,"天çĦ¶":102306,"çĽĴ":102307,"çĪ½":102308,"å¿ħçĦ¶":102309,"åĮĸå·¥":102310,"ä¸ĵåĪ©":102311,"åķ¡":102312,"å¼Ģå¿ĥ":102313,"人ä½ĵ":102314,"éģĵ士":102315,"æĢģ度":102316,"空è°ĥ":102317,"æĭĽåķĨ":102318,"姻":102319,"第äºĶ":102320,"æ£Ĵ":102321,"ä¸Ģç³»åĪĹ":102322,"åį±æľº":102323,"转åıĺ":102324,"åľºæīĢ":102325,"鸣":102326,"æĪ¿éĹ´":102327,"éĢ¼":102328,"è¯ķçĤ¹":102329,"对å¤ĸ":102330,"åĩºåı°":102331,"åľ¨è¿Ļ":102332,"åİĤ家":102333,"巨大":102334,"ç®Ģä»ĭ":102335,"çľĭäºĨ":102336,"åħļ建":102337,"æĮĩæĮ¥":102338,"çŁ³æ²¹":102339,"ä¸įåı¯èĥ½":102340,"èݲ":102341,"ä¸į太":102342,"åĪĽæĦı":102343,"第ä¸Ģ个":102344,"è´µå·ŀ":102345,"è¿ĩäºĨ":102346,"æľ¬æĿ¥":102347,"éģĵå¾·":102348,"çŃĶæ¡Ī":102349,"éĻ¶":102350,"ä¸Ģè·¯":102351,"èĤĸ":102352,"æ¸ħæ´ģ":102353,"æľīæľº":102354,"åIJįåįķ":102355,"æĿ±":102356,"åij¼åIJ¸":102357,"ä¸Ī":102358,"ç¦ı建":102359,"è¯ķéªĮ":102360,"å¼ķåıij":102361,"ä¹Łæ²¡":102362,"ä¸įä½ı":102363,"çĨŁæĤī":102364,"èIJ¬":102365,"ä¸įèī¯":102366,"çłĸ":102367,"èĩ´åĬĽ":102368,"çŃ¾è®¢":102369,"åIJĬ":102370,"侯":102371,"çĺ¦":102372,"å§ijå¨ĺ":102373,"æĸ¤":102374,"妻åŃIJ":102375,"æĺ¥èĬĤ":102376,"çĪ¬":102377,"æĽĿ":102378,"çĥŃæĥħ":102379,"éķ¿æ²Ļ":102380,"èIJ¥éĢł":102381,"éħ·":102382,"éĵĿ":102383,"åŁºæľ¬ä¸Ĭ":102384,"åij¨åĽ´":102385,"ä»Ģ麼":102386,"认åı¯":102387,"åĪĨåŃIJ":102388,"ä¸Ģæĸ¹éĿ¢":102389,"è½´":102390,"å¼·":102391,"马ä¸Ĭ":102392,"éĽ¾":102393,"èĩ£":102394,"å°¿":102395,"çĶŁæĦı":102396,"å®īå¾½":102397,"ç¥ŀç»ı":102398,"åĩºå¸Ń":102399,"èį¯åĵģ":102400,"çIJĨçĶ±":102401,"åįıåIJĮ":102402,"æµģåĬ¨":102403,"åıijåĬ¨":102404,"åĿļå®ļ":102405,"表æĺİ":102406,"åIJİéĿ¢":102407,"ä¹īåĬ¡":102408,"å¦ĸ":102409,"æľīåı¯èĥ½":102410,"年轻人":102411,"大éĻĨ":102412,"å²³":102413,"ä¸įèµ·":102414,"çŀ¬éĹ´":102415,"ä¸įå¾Ĺä¸į":102416,"çŃ¾çº¦":102417,"åIJĪæł¼":102418,"åħļæĶ¯éĥ¨":102419,"æµİåįĹ":102420,"便åĪ©":102421,"éļıæŶ":102422,"å¥ī":102423,"称为":102424,"产æĿĥ":102425,"åIJķ":102426,"çĽĨ":102427,"课åłĤ":102428,"ç·ļ":102429,"æ£ī":102430,"线ä¸ĭ":102431,"èĩªè¡Į":102432,"举æݪ":102433,"åݦéŨ":102434,"èĩªä¿¡":102435,"å½±è§Ĩ":102436,"ä»Ķ":102437,"çĶŁæ´»ä¸Ń":102438,"æĿĥçĽĬ":102439,"çĻ½èī²":102440,"å°±ä¸į":102441,"è¿Ľå±ķ":102442,"æ¯ıæĹ¥":102443,"ä¾Ľç»Ļ":102444,"æĿĥåĪ©":102445,"æĹłæķ°":102446,"çIJĨè´¢":102447,"ä¾Ŀæŧ":102448,"ä¸ĬåįĪ":102449,"è¯ĨåĪ«":102450,"çĽĪåĪ©":102451,"çłĤ":102452,"许åı¯":102453,"åIJĮäºĭ":102454,"åĺĽ":102455,"éģ¸":102456,"çĿĢåĬĽ":102457,"éŨåı£":102458,"ä¸įå¤ļ":102459,"åħ¶æ¬¡":102460,"碧":102461,"çī©çIJĨ":102462,"åĨħå¿ĥ":102463,"çĻ¾å§ĵ":102464,"æĢ»ç»Ł":102465,"å¹²åĩĢ":102466,"积累":102467,"åıįé¦Ī":102468,"æłijç«ĭ":102469,"社交":102470,"秩":102471,"åįģä¸Ģ":102472,"éĤĵ":102473,"驱åĬ¨":102474,"å±ķè§Ī":102475,"èĪĴéĢĤ":102476,"åŁºåĽł":102477,"å·®å¼Ĥ":102478,"转让":102479,"å°ıå§IJ":102480,"æł·åŃIJ":102481,"ç¿Ķ":102482,"é«ĺåħ´":102483,"å½±åĵįåĬĽ":102484,"æīĭç»Ń":102485,"缸åIJĮ":102486,"缸åºĶ":102487,"æĻĴ":102488,"è§Ģ":102489,"å¸Ĥå§Ķ":102490,"èĬ¯":102491,"å±ķçİ°":102492,"åľ°çIJĥ":102493,"éĤª":102494,"ä¸Ģå®ļçļĦ":102495,"åħģ许":102496,"ä¿¡ä»»":102497,"æīij":102498,"éĻ¢æł¡":102499,"ç®Ģ称":102500,"åģļæ³ķ":102501,"ä¹ĭè·¯":102502,"æĹĹä¸ĭ":102503,"èħĶ":102504,"æ¶Ī失":102505,"ä¸ĸçķĮä¸Ĭ":102506,"åŁİ乡":102507,"èĪŀåı°":102508,"å¾Ī大çļĦ":102509,"绣çŃ¹":102510,"åħ¬å¹³":102511,"èĤ¾":102512,"çļĦ好":102513,"æ±ģ":102514,"çľ¼åīį":102515,"éĽ£":102516,"å¹½":102517,"åħ±äº§":102518,"主åĬŀ":102519,"å¤Ħç½ļ":102520,"åºĻ":102521,"éģĵçIJĨ":102522,"å¼µ":102523,"æİ¥çĿĢ":102524,"çĮİ":102525,"çģĮ":102526,"çĶ±æѤ":102527,"人åĬĽ":102528,"æµģè¡Į":102529,"ä¾ł":102530,"åı¯ä»¥è¯´":102531,"èĴĭ":102532,"å½¢æĢģ":102533,"æĹ¥åŃIJ":102534,"æ¼Ĩ":102535,"çķĻåѦ":102536,"缸éĹľ":102537,"æľĢå¤ļ":102538,"åĩŃåĢŁ":102539,"åħ¬äº¤":102540,"æĮĸæİĺ":102541,"æĿĤå¿Ĺ":102542,"主人":102543,"éļľç¢į":102544,"æł¡éķ¿":102545,"æĸ¹ä½į":102546,"ä¸ĬçıŃ":102547,"å¤ļåħĥ":102548,"èĥģ":102549,"éŃħåĬĽ":102550,"èĮĤ":102551,"åħħçĶµ":102552,"强大":102553,"çĥ¤":102554,"å¥ĭæĸĹ":102555,"å®ŀçĶ¨":102556,"éĺģ":102557,"ç»ĻäºĨ":102558,"æľ¬ç§ij":102559,"æłĭ":102560,"æĭ¨":102561,"æķĻç»ĥ":102562,"éĥ½çŁ¥éģĵ":102563,"æ¯ķä¸ļçĶŁ":102564,"ç¢Ĺ":102565,"åŀĤ":102566,"讼":102567,"å®ģæ³¢":102568,"åѦèĢħ":102569,"谢谢":102570,"åŁİéķĩ":102571,"æĢİä¹ĪåĬŀ":102572,"éģĶ":102573,"æĪIJ交":102574,"æ½ľåĬĽ":102575,"åį§":102576,"æĸ°å¼Ģ":102577,"éħįå¤ĩ":102578,"主åĬĽ":102579,"åij³éģĵ":102580,"çĥĤ":102581,"é£ŀè¡Į":102582,"å«ģ":102583,"大大":102584,"ç»Ļ大家":102585,"å¤ĸéĿ¢":102586,"éĨī":102587,"åıijè¨Ģ":102588,"æĹ©é¤IJ":102589,"åIJĦèĩª":102590,"å®Ļ":102591,"èį£èªī":102592,"æĬ«éľ²":102593,"é¡ŀ":102594,"åĨħçļĦ":102595,"èĤª":102596,"è¾IJ":102597,"æ³µ":102598,"æĬĽ":102599,"æĺŁæľŁ":102600,"ä¸Ģ带":102601,"çĶŁç´ł":102602,"ç»ıéĶĢ":102603,"åĩ¶":102604,"åľ°ä¸Ĭ":102605,"åij½è¿IJ":102606,"åĵ²":102607,"ä¸Ĭåİ»":102608,"æĸĩçī©":102609,"è¯ij":102610,"æĮ¯åħ´":102611,"éķ¿æŶéĹ´":102612,"ç¥Ń":102613,"åIJĪèĤ¥":102614,"è¿Ŀè§Ħ":102615,"èģª":102616,"ä½İäºİ":102617,"éĢĤå½ĵ":102618,"æľīåºı":102619,"æľ¬ç½ij":102620,"çķĻè¨Ģ":102621,"æĥ³æ³ķ":102622,"çŃ¾ç½²":102623,"å§ļ":102624,"æĢ§æł¼":102625,"èĴĻåı¤":102626,"æŁı":102627,"åŀ«":102628,"åѦåİĨ":102629,"ä»ħä»ħ":102630,"讲è¯Ŀ":102631,"éĶIJ":102632,"æĢĸ":102633,"åīª":102634,"èĭį":102635,"åIJĵ":102636,"强çĥĪ":102637,"åģ¥åħ¨":102638,"çĸ¯":102639,"åı¤ä»£":102640,"å¥Ī":102641,"ä¸įçĦ¶":102642,"乡éķĩ":102643,"æľĭåıĭ们":102644,"åĤħ":102645,"èģ½":102646,"个æĢ§":102647,"æ³ķè§Ħ":102648,"å°ıéķĩ":102649,"çĶ»éĿ¢":102650,"第åħŃ":102651,"網路":102652,"åīįæĻ¯":102653,"åIJ¬è¯´":102654,"ä¼łåªĴ":102655,"æĿ¡ä¾ĭ":102656,"åĪ«çļĦ":102657,"ä¸įæĩĤ":102658,"顾éĹ®":102659,"强度":102660,"éĺ¿éĩĮ":102661,"èµ°åĬ¿":102662,"帽":102663,"çļĦç¡®":102664,"åĮºåĪ«":102665,"éĮ¢":102666,"主管":102667,"ä¸Ģçľĭ":102668,"æĸľ":102669,"åŃĺåľ¨çļĦ":102670,"仲":102671,"åį±å®³":102672,"éĵŃ":102673,"游æĪıä¸Ń":102674,"éħ±":102675,"é¾Ļ头":102676,"人å¿ĥ":102677,"éĢĢä¼ij":102678,"æµıè§Ī":102679,"åĬ«":102680,"éĺ²æ²»":102681,"ç®Ń":102682,"å±Ī":102683,"è¾½å®ģ":102684,"壤":102685,"è¿İæĿ¥":102686,"éŀį":102687,"çĶ¨æĿ¥":102688,"å¤§åľ°":102689,"ä»°":102690,"éĢļ讯":102691,"å¼Ģå·¥":102692,"裤":102693,"å¦ĤåIJĮ":102694,"骤":102695,"éĺŁåijĺ":102696,"轩":102697,"ç¾İæľ¯":102698,"èĻŁ":102699,"åIJĮä¸Ģ":102700,"åľĸ":102701,"书æ³ķ":102702,"æīĵåį°":102703,"åIJ«æľī":102704,"éĽĨæĪIJ":102705,"éĹ·":102706,"å¸Ĥåľºä¸Ĭ":102707,"æĹģè¾¹":102708,"åľ°æĿ¿":102709,"产çĶŁçļĦ":102710,"粤":102711,"éĩįç»Ħ":102712,"è¡Ģ液":102713,"çŃĭ":102714,"åĬŀäºĭ":102715,"常è§ģçļĦ":102716,"ä¸ĬåįĬå¹´":102717,"å±ıå¹ķ":102718,"åIJīæŀĹ":102719,"å·©":102720,"åĸľçĪ±":102721,"ç¿ł":102722,"ä¸īç§į":102723,"æ¡Ĩæŀ¶":102724,"举èİŀ":102725,"çĶĺèĤĥ":102726,"èĬ¬":102727,"åĽ¾ä¹¦":102728,"åĩ¤åĩ°":102729,"æ°ĶåĢĻ":102730,"å°´":102731,"å°¬":102732,"两天":102733,"è¾ħ导":102734,"åĢŁæ¬¾":102735,"æĹ¥èµ·":102736,"æ´Ĵ":102737,"ä¸Ģ度":102738,"è¹Ī":102739,"æ½Ń":102740,"æīĩ":102741,"çĻľ":102742,"æĸ°åħ´":102743,"åĤ²":102744,"诸å¤ļ":102745,"è´ª":102746,"éĻ·åħ¥":102747,"èĪŁ":102748,"èĤºçĤİ":102749,"ä¸Ģæł·çļĦ":102750,"åİĺ":102751,"åľ°çIJĨ":102752,"æĬķ注":102753,"éļĬ":102754,"åħīä¼ı":102755,"ä¿Ŀåģ¥":102756,"åħĶ":102757,"åħ¬åĬ¡":102758,"æīĵçł´":102759,"çĶ·åŃ©":102760,"åĬ³åĬ¡":102761,"ä½łä¼ļ":102762,"çĶ¨åľ°":102763,"溢":102764,"åıijè¾¾":102765,"èĤļ":102766,"è¿ĩäºİ":102767,"èĩĤ":102768,"éĢĻ樣":102769,"轻轻":102770,"ä¸Ńåħ±":102771,"åIJĦåĽ½":102772,"åĶĩ":102773,"å®ŀä¹ł":102774,"èĻ¾":102775,"槽":102776,"ä¸įä¸Ĭ":102777,"åħįçĸ«":102778,"åįłæį®":102779,"å·¥ä¼ļ":102780,"åĽĬ":102781,"èĪªå¤©":102782,"åı¯çĪ±":102783,"æĸĹäºī":102784,"çĺ¤":102785,"å¦Ĥæľī":102786,"éĽĸ":102787,"对æĪij":102788,"åĩºç§Ł":102789,"好çľĭ":102790,"太大":102791,"æ°´åĪ©":102792,"åĬ¿åĬĽ":102793,"åħ¨æ°ij":102794,"ç½¢":102795,"èµ¢å¾Ĺ":102796,"çĶµä¿¡":102797,"车éĹ´":102798,"æĻĤåĢĻ":102799,"å°ijæķ°":102800,"éĵ¸":102801,"åħ³èģĶ":102802,"ä¸įä»ħä»ħ":102803,"为æĤ¨":102804,"åĴ¸":102805,"æľºåĬ¨":102806,"è£Ļ":102807,"åĵįåºĶ":102808,"éģł":102809,"è²·":102810,"ç©´":102811,"å¢ħ":102812,"éĶ¡":102813,"çµĦ":102814,"çģ«è½¦":102815,"è³ĩè¨Ĭ":102816,"åĨ³èµĽ":102817,"污水":102818,"èªŀ":102819,"å´Ľ":102820,"ç´§å¯Ĩ":102821,"缺å°ij":102822,"å¤ļ人":102823,"æĢ»ä¹¦è®°":102824,"éĶĪ":102825,"èijĽ":102826,"å¿ĺè®°":102827,"éĻĮçĶŁ":102828,"éķ¿å¤§":102829,"åħĪè¿ĽçļĦ":102830,"ç¡ħ":102831,"åıijæĺİ":102832,"å©´åĦ¿":102833,"æīİå®ŀ":102834,"èĽĭçĻ½":102835,"ä¸ĢçĻ¾":102836,"缮åħī":102837,"æħĮ":102838,"åĬłæ²¹":102839,"åIJŀ":102840,"ä¸Ģ群":102841,"ä¸Ńä»ĭ":102842,"å¸ĸ":102843,"å¿Į":102844,"èģĮèĥ½":102845,"广æĴŃ":102846,"çĽijå¯Ł":102847,"ç§ĺå¯Ĩ":102848,"çĭ®":102849,"è¿ĻæĿ¡":102850,"éĢ¢":102851,"æĢ¨":102852,"åįģåħŃ":102853,"試":102854,"说åĪ°":102855,"åĩĿèģļ":102856,"æĮĩ示":102857,"æ°¢":102858,"å¼ĺ":102859,"éĺĢ":102860,"æĸ©":102861,"éłħ":102862,"ä¸Ģå¼Ģå§ĭ":102863,"æİĴè¡Į":102864,"åľ¨æĪij":102865,"纪å½ķ":102866,"æĬĦ":102867,"æłª":102868,"说æ³ķ":102869,"ä¸Ńèį¯":102870,"好å¤ļ":102871,"åıªä¸įè¿ĩ":102872,"çķĻåľ¨":102873,"个å°ıæŶ":102874,"è®¤çŁ¥":102875,"çķ«":102876,"è§ģè¿ĩ":102877,"å°ıå¾®":102878,"ä½Ľå±±":102879,"çľ¾":102880,"讲述":102881,"梳":102882,"称åı·":102883,"æĹ¥æĻļ":102884,"è¢ĸ":102885,"åķ¤":102886,"æľªç»ı":102887,"æľĢæĹ©":102888,"æī®æ¼Ķ":102889,"è¡Ģ管":102890,"纱":102891,"æĥħèĬĤ":102892,"第ä¸ĥ":102893,"æį§":102894,"ä»Ĺ":102895,"æ¿ĢçĥĪ":102896,"æĹłçº¿":102897,"ä¸į容æĺĵ":102898,"å¼Ģå¹ķ":102899,"æĸ°çĶŁ":102900,"ä¸ĵ注":102901,"èij±":102902,"åįĹæµ·":102903,"çĩŁ":102904,"èµ·ä¾Ĩ":102905,"æ´¾åĩº":102906,"åĦĴ":102907,"侨":102908,"è¼ĥ":102909,"åįļè§Ī":102910,"éĢ¾":102911,"åĮĢ":102912,"ç»ıæµİåѦ":102913,"æ¸Ĺ":102914,"ä¿ĿèŃ·":102915,"çīº":102916,"çī²":102917,"çİ«":102918,"çij°":102919,"æľĢåIJİä¸Ģ":102920,"æĶ¿åĬ¡":102921,"æ§Ľ":102922,"èĻķçIJĨ":102923,"éļIJæĤ£":102924,"æī¿åĮħ":102925,"極":102926,"æ¡©":102927,"çĽ²":102928,"导åIJij":102929,"èĩ´å¯Į":102930,"ç¼Ĩ":102931,"æģĭçĪ±":102932,"ä¸įåĬ¨":102933,"ç»Ļ人":102934,"å·¢":102935,"表æĥħ":102936,"举åįĹ":102937,"åĨħå¤ĸ":102938,"è¾ĪåŃIJ":102939,"åıī":102940,"åįļä¼ļ":102941,"åĬŁæķĪ":102942,"渴":102943,"屬":102944,"æİĴéĻ¤":102945,"éĢĽ":102946,"ä¸Ģä¼ļ":102947,"ä¸įå¼Ģ":102948,"å¼Ģå¥ĸ":102949,"é»ijé¾Ļ":102950,"é»ijé¾Ļæ±Ł":102951,"å¿«ä¸ī":102952,"度åģĩ":102953,"åĿ¤":102954,"éĤ®ä»¶":102955,"æĩĴ":102956,"ä¾ĽçĶµ":102957,"廣":102958,"好è¯Ħ":102959,"ç§ĺ书éķ¿":102960,"æĪĺåľº":102961,"好å¥ĩ":102962,"ä¾µæĿĥ":102963,"æĨ¾":102964,"æľĢåĪĿ":102965,"æī¹åıij":102966,"åİķ":102967,"è¼ķ":102968,"æŀ¯":102969,"ä¸ļåĨħ":102970,"è´ŃæĪ¿":102971,"ä¸įåľ¨":102972,"纪å§Ķ":102973,"æīĢéľĢ":102974,"å¸Ĥéķ¿":102975,"è³½":102976,"å¼ķæĵİ":102977,"çģµéŃĤ":102978,"éĬĢ":102979,"滤":102980,"çĿIJ":102981,"å¤ļ项":102982,"åĽŀ头":102983,"èīĺ":102984,"å¤įå·¥":102985,"éĥ¨ä»¶":102986,"紧紧":102987,"æŁIJç§į":102988,"使åħ¶":102989,"æĸ°äºº":102990,"æŀļ":102991,"æ³ķå®ļ":102992,"å·´å·´":102993,"涵çĽĸ":102994,"稻":102995,"æĭ¾":102996,"æĻķ":102997,"轿":102998,"éĢļè¡Į":102999,"åĵĢ":103000,"æ³Ĭ":103001,"温馨":103002,"éĽĨèģļ":103003,"çĨĻ":103004,"åĩij":103005,"åįģä¸ĥ":103006,"æ°Ķæģ¯":103007,"æıIJä¾ĽçļĦ":103008,"æ³³":103009,"奥è¿IJ":103010,"çģ¾å®³":103011,"åĩĢåĮĸ":103012,"è·¨è¶Ĭ":103013,"åĵªæĢķ":103014,"éŁ¿":103015,"å¢ŀæ·»":103016,"çĦĬ":103017,"æ®ĭçĸ¾":103018,"ç¢Į":103019,"æĤĶ":103020,"è§ģè¯ģ":103021,"è¾ĸåĮº":103022,"å¿ĥèĦı":103023,"éļ§":103024,"åį¸":103025,"åı¯èĥ½æĢ§":103026,"æľī趣":103027,"åī¯ä¹¦è®°":103028,"åĮĸå¦Ĩ":103029,"ä¿Ĥ":103030,"æ£ļ":103031,"éĨĩ":103032,"带头":103033,"éłĪ":103034,"追究":103035,"æijĶ":103036,"è¿Ļéĥ¨":103037,"ä¸į论":103038,"祸":103039,"å³»":103040,"éģķ":103041,"çĶŁèĤ²":103042,"å¤ł":103043,"å¤ĸ交":103044,"è¯Ħ为":103045,"ä»İå°ı":103046,"å°ıå°ı":103047,"饿":103048,"æĴ¼":103049,"è·¨å¢ĥ":103050,"被åijĬ":103051,"åįĹå®ģ":103052,"身å¿ĥ":103053,"åĨįçĶŁ":103054,"æīĢ说":103055,"æŶéĹ´åĨħ":103056,"åĪĹåħ¥":103057,"éĿĴæµ·":103058,"çĪ±å¥½":103059,"çªĦ":103060,"èĪĪ":103061,"è¿ĩ渡":103062,"æ¿Ł":103063,"éĽĢ":103064,"审议":103065,"åĽ½èµĦ":103066,"æŃ¥ä¼IJ":103067,"轨éģĵ":103068,"信念":103069,"ä¸īåĪĨ":103070,"çĨ¬":103071,"åѵåĮĸ":103072,"ç¼ł":103073,"éĥĬ":103074,"èĪĴæľį":103075,"纪æ£Ģ":103076,"ä¸Ģä¸ĭåŃIJ":103077,"éĽ»è©±":103078,"è²ł":103079,"éĴ¥":103080,"åĮĻ":103081,"çĹ´":103082,"è¶ģ":103083,"绣":103084,"çĪµ":103085,"è½°":103086,"éªĦ":103087,"姨":103088,"æĭĺ":103089,"çĮ´":103090,"讶":103091,"è¿Ļ座":103092,"çį¨":103093,"æ·ĺæ±°":103094,"çĹħä¾ĭ":103095,"æ²Ļåıij":103096,"è§Ĩ为":103097,"头æĿ¡":103098,"å¿ħè¦ģçļĦ":103099,"åı¯è°ĵ":103100,"è¯Ŀ说":103101,"ç¯Ħ":103102,"æĹ©çĤ¹":103103,"æŀ¢çº½":103104,"羡":103105,"çĪ±åĽ½":103106,"çªģåıij":103107,"éĢĬ":103108,"æ½į":103109,"èį£èĢĢ":103110,"èŁ¹":103111,"æ¦Ĥçİĩ":103112,"å¾Īä¹ħ":103113,"æĥķ":103114,"訴":103115,"åľĨ满":103116,"çļ±":103117,"åĪĨæ³Į":103118,"åħħ足":103119,"çľĭæ³ķ":103120,"è¾Ł":103121,"æĭ¦":103122,"æĭ©":103123,"对åºĶ":103124,"ä¸ºæł¸å¿ĥ":103125,"èħĬ":103126,"å¤ļä¹Ī":103127,"æµij":103128,"å®ıè§Ĥ":103129,"èĦĸ":103130,"åIJĪèµĦ":103131,"çĶŁæ¶¯":103132,"å®ŀè´¨":103133,"ä¼ĺçĤ¹":103134,"çĶ¨æ°´":103135,"寿åij½":103136,"沫":103137,"åIJģ":103138,"詹":103139,"åĽ½éĺ²":103140,"å´©":103141,"åĿİ":103142,"èĨı":103143,"ä¸Ģè½®":103144,"éģĹ产":103145,"æ¹¾åĮº":103146,"ç»İ":103147,"åįķ纯":103148,"æ¾Ħ":103149,"åīįåĪĹ":103150,"身影":103151,"é»ĺé»ĺ":103152,"æįī":103153,"çĴ°":103154,"èıĬ":103155,"æĢľ":103156,"åħĭæĢĿ":103157,"æĢ»å±Ģ":103158,"çĩĥæĸĻ":103159,"ä¸ļæĢģ":103160,"åIJĦæł·":103161,"åĴ½":103162,"åĩºèī²":103163,"åĪĿå¿ĥ":103164,"åıĽ":103165,"çłĶ讨":103166,"è¡«":103167,"åİĨç¨ĭ":103168,"禽":103169,"è¶³å¤ŁçļĦ":103170,"èįĨ":103171,"çľĭå¾ħ":103172,"è´©":103173,"åĨ³å¿ĥ":103174,"裹":103175,"å¸ĪèĮĥ":103176,"åŀĦ":103177,"æĿł":103178,"åĩ¸":103179,"çĬ¹è±«":103180,"çĥŃè¡Ģ":103181,"åIJĪä¼Ļ":103182,"éħµ":103183,"èIJ½åľ¨":103184,"åįłåľ°":103185,"衬":103186,"èĵī":103187,"æĦ¤":103188,"æ¸Ĭ":103189,"åĪĨæķ°":103190,"ç¬ijçĿĢ":103191,"太平":103192,"çĤ«":103193,"æݨä»ĭ":103194,"æĸ¯åĿ¦":103195,"形容":103196,"æĵĬ":103197,"æĦŁåħ´è¶£":103198,"åĨĽäºº":103199,"åĩĮæĻ¨":103200,"对çħ§":103201,"åıijçĹħ":103202,"å·¾":103203,"èĪī":103204,"檢":103205,"ç¬ijäºĨ":103206,"ç¡®è¯Ĭ":103207,"è´ŁåĢº":103208,"壮大":103209,"æĪļ":103210,"äºĴèģĶ":103211,"課":103212,"èħ¦":103213,"æű":103214,"åıĹ欢è¿İ":103215,"åįī":103216,"éĻ¢å£«":103217,"æ©¡":103218,"ä¸Ģ对":103219,"è¾±":103220,"æ²Ĥ":103221,"åı²ä¸Ĭ":103222,"æIJı":103223,"å´ĸ":103224,"代谢":103225,"磷":103226,"é¡ĺ":103227,"æµĩ":103228,"常çĶ¨":103229,"åįij":103230,"åĩºåĽ½":103231,"è¯ł":103232,"稳æŃ¥":103233,"ç»ı纪":103234,"å¤ļå¤ļ":103235,"æīĢå¾Ĺ":103236,"为主é¢ĺ":103237,"ä¸ĢåĪĨ":103238,"æł½":103239,"顧":103240,"纲":103241,"åĥħ":103242,"å£ĵ":103243,"åĦª":103244,"ç¿°":103245,"æİĢ":103246,"人为":103247,"媳":103248,"æ´½":103249,"èĿ¶":103250,"å¤įåħ´":103251,"ä¼ļå½±åĵį":103252,"åIJĦçķĮ":103253,"éĤ£ä¸Ģ":103254,"颤":103255,"çĢı":103256,"çĢı覽":103257,"å¯ŀ":103258,"åı¯æĢķ":103259,"åį³æŶ":103260,"çķ´":103261,"ä¸ĭåįĬå¹´":103262,"ç¬Ķè®°":103263,"éĻĦåĬł":103264,"çĥŃæ°´":103265,"奸":103266,"ç£ħ":103267,"æĿī":103268,"æ¸ħåįİ":103269,"éĸ±":103270,"ç°¡":103271,"å¤Ħå¤Ħ":103272,"åIJĪéĩij":103273,"æ²³æµģ":103274,"ç´°":103275,"è´ŁéĿ¢":103276,"çļĦ羣å®ŀ":103277,"åĻ¨æ¢°":103278,"èĴIJ":103279,"西äºļ":103280,"å·ħ":103281,"ç²¹":103282,"åİŁæĸĩ":103283,"æŀķ":103284,"è¡Ģåİĭ":103285,"åļ´":103286,"å¸ĺ":103287,"åĨĢ":103288,"æĮ«":103289,"çĶµè·¯":103290,"å°ıä¼Ļä¼´":103291,"èĿ´":103292,"æľĢå¿«":103293,"æĭĮ":103294,"宪":103295,"æĸ·":103296,"ç¿ħ":103297,"åĴ³":103298,"åĹ½":103299,"ç¾ŀ":103300,"èººåľ¨":103301,"èµĽè½¦":103302,"æ²IJ":103303,"éĻIJ度":103304,"为ä¸Ģä½ĵ":103305,"èĴľ":103306,"幫":103307,"æIJħ":103308,"åĭĭ":103309,"åīĸ":103310,"纳ç¨İ":103311,"éķ¿æķĪ":103312,"ç½ķ":103313,"åī¯æľ¬":103314,"ç©į":103315,"éĴ©":103316,"ç¹¼":103317,"åĽ½åľŁ":103318,"è¼ī":103319,"ä¸įå¿ĺ":103320,"èŃ¦ç¤º":103321,"çģ¿":103322,"å¿ĥå¾Ĺ":103323,"æĦļ":103324,"忽çķ¥":103325,"åĽŀäºĭ":103326,"åįłæľī":103327,"æ·Ħ":103328,"çī¡":103329,"çĽijäºĭ":103330,"ç¿¡":103331,"éĴĪ对æĢ§":103332,"çªĥ":103333,"製":103334,"èĨĿ":103335,"ç³Ł":103336,"港澳":103337,"太太":103338,"澡":103339,"ç»ĨåĮĸ":103340,"åĶ®åIJİ":103341,"å®ŀåľ¨æĺ¯":103342,"ç«£":103343,"çį²":103344,"åĢ¾åIJij":103345,"å¼ķçĶ¨":103346,"é¹ħ":103347,"ç¬ij容":103348,"ä¹IJ趣":103349,"æ°ijæĶ¿":103350,"éŨæĪ·":103351,"å±ģ":103352,"迷失":103353,"éĶĮ":103354,"å°ı康":103355,"åĭī":103356,"æ³¼":103357,"ä¾ĭåŃIJ":103358,"ä¸īä½į":103359,"å»ł":103360,"èĶĵ":103361,"广éĺĶ":103362,"èĢį":103363,"èĢģèĻİ":103364,"åĭŁéĽĨ":103365,"èĦļæŃ¥":103366,"æĭ¯":103367,"åŃĹåı·":103368,"çĦ°":103369,"é¢ł":103370,"èļĤ":103371,"èļģ":103372,"飯":103373,"人æĢ§":103374,"æĴ°":103375,"åİ¢":103376,"å±ĢéĻIJ":103377,"æľªæĪIJ":103378,"åĵªåĦ¿":103379,"大åıij":103380,"ä¸įå®ļ":103381,"å¾ģæ±Ĥ":103382,"éĥµ":103383,"åĢºæĿĥ":103384,"çĪ±ä½ł":103385,"èºģ":103386,"ä»ħä¾Ľ":103387,"è¿ľå¤Ħ":103388,"éĨĽ":103389,"åĥµ":103390,"积æŀģæĢ§":103391,"æİ¡":103392,"åīįä¸ī":103393,"äºİä¸Ģä½ĵ":103394,"çŀĦ":103395,"çĿģ":103396,"沸":103397,"åħ±èµ¢":103398,"éĢĢå½¹":103399,"è´Ŀå°Ķ":103400,"æİı":103401,"æĪ²":103402,"è¡į":103403,"éĶĤ":103404,"ä¸ĩä½Ļ":103405,"ç§ijåĪĽ":103406,"æ¼ĶåĶ±":103407,"欧åħĥ":103408,"æ·¡æ·¡":103409,"éĿĴå±±":103410,"èĹĿ":103411,"绽":103412,"令çīĮ":103413,"éĽĨ群":103414,"ä½ľçī©":103415,"çĢij":103416,"夯":103417,"ç½ij游":103418,"åħ«å¤§":103419,"éªļ":103420,"èªĵ":103421,"ä¼ļå±ķ":103422,"åħļåı²":103423,"æ£Ģå¯ŁéĻ¢":103424,"åĸĺ":103425,"éĺ±":103426,"èĢĮåĩº":103427,"éĢļ车":103428,"éĴĵ":103429,"æĥħ人":103430,"æ¸Ľ":103431,"ä¸Ńç§ĭ":103432,"çĪŃ":103433,"åıªåī©":103434,"æĺĶ":103435,"éĩİçĶŁ":103436,"ç¡«":103437,"èIJĿåįľ":103438,"æĬµæĬĹ":103439,"çĻ«çĹ«":103440,"éĻĢ":103441,"èĶļ":103442,"å¸ľ":103443,"满满":103444,"èı±":103445,"éļĨéĩį":103446,"æĺŁçº§":103447,"æ½ĩ":103448,"åħ¬åħĥ":103449,"è°£":103450,"æ¯Ķäºļ":103451,"æ¡ĮåŃIJ":103452,"èµ£":103453,"è²¼":103454,"æĦ¿æľĽ":103455,"顽":103456,"æ´¾éģ£":103457,"ç¥Ľ":103458,"åªļ":103459,"éĺľ":103460,"èij«":103461,"èĬ¦":103462,"æ³»":103463,"å¡Į":103464,"çĭŃ":103465,"å»īæĶ¿":103466,"å¥ijæľº":103467,"æĹĹèĪ°":103468,"æĥ«":103469,"严åİī":103470,"åıĭæĥħ":103471,"å¦Ĭ":103472,"å¨ł":103473,"åĵªå®¶":103474,"èĨ¨":103475,"è¶Ł":103476,"æĮª":103477,"èĻIJ":103478,"éłģ":103479,"çŀ©":103480,"éºŁ":103481,"稣":103482,"èģĶéĢļ":103483,"åı®":103484,"çİĭèĢħ":103485,"ä¸įç¡®å®ļ":103486,"çijľ":103487,"è°İ":103488,"çī¢è®°":103489,"碼":103490,"æĬ¤èĤ¤":103491,"é¡·":103492,"çĦķ":103493,"åģļ强":103494,"éļ±ç§ģ":103495,"éļ±ç§ģæ¬Ĭ":103496,"åıĹ害":103497,"ä¸įçĶ±":103498,"çĥ¹":103499,"饪":103500,"驳":103501,"ä¼½":103502,"ä¸Ŀ绸":103503,"è¥Ħ":103504,"åįģä½Ļ":103505,"éºĹ":103506,"æ¬ĬåĪ©":103507,"èģŀ":103508,"åı¤èĢģ":103509,"éģı":103510,"åIJĦå¼ı":103511,"å°±è¡Į":103512,"åħ¥å¢ĥ":103513,"çĥģ":103514,"èľĺ":103515,"èĽĽ":103516,"纬":103517,"磫":103518,"è»Ł":103519,"æ´Ĺè¡£":103520,"æĦ§":103521,"é¢Ħæ¡Ī":103522,"éľĨ":103523,"æ·±åİļ":103524,"éĺ¿æĭī":103525,"åĨĻåŃĹ":103526,"åį¦":103527,"éķĢ":103528,"æ¨¡æł·":103529,"åĤį":103530,"æIJį":103531,"èĸ¯":103532,"åłħ":103533,"åħ¬ç§¯":103534,"è¨İ":103535,"ä¼łæŁĵ":103536,"毯":103537,"çIJĨå·¥":103538,"åĨ·éĵ¾":103539,"ç«ĭæĸ¹":103540,"æ¢Ń":103541,"åľ£è¯ŀ":103542,"综èīº":103543,"çİ©ç¬ij":103544,"æĥ³ä¸įåĪ°":103545,"æijĩ头":103546,"æ·¹":103547,"åģĩæĹ¥":103548,"åĢĺ":103549,"èĢ½":103550,"èİĵ":103551,"åŁ·":103552,"èĩªè´¸":103553,"åįĬ天":103554,"æªĶ":103555,"æ¾İæ¹ĥ":103556,"éķij":103557,"丫":103558,"éĩĮç¨ĭ":103559,"å¼ĢèįĴ":103560,"èıı":103561,"å®Ŀè´µ":103562,"èѬ":103563,"åķŁ":103564,"æŁł":103565,"檬":103566,"é©Ń":103567,"æ±Ľ":103568,"çĨĬçĮ«":103569,"èķī":103570,"éļıä¹ĭ":103571,"å±ij":103572,"è¾ĥ强":103573,"èĥ³":103574,"èĨĬ":103575,"éĿĻéĿĻ":103576,"åĴª":103577,"æĭĽåij¼":103578,"代è¨Ģ":103579,"ä¿¡ç®±":103580,"è£ħéħį":103581,"æĤį":103582,"åįķ车":103583,"èIJİ":103584,"å¤ļ彩":103585,"éĻ¸":103586,"ä»İ严":103587,"æ©Ħ":103588,"æ¦Ħ":103589,"éĢ®":103590,"éĩĮæĸ¯":103591,"姿æĢģ":103592,"太æŀģ":103593,"éĩĿ":103594,"æºī":103595,"è¿Ń":103596,"秸":103597,"ç§Ĩ":103598,"å·¥å§Ķ":103599,"æ±ķ":103600,"èģĨ":103601,"佬":103602,"ç¼ħ":103603,"çĶ¸":103604,"åī¯å±Ģéķ¿":103605,"éĹº":103606,"誤":103607,"è¤IJ":103608,"ä¸įéĻIJ":103609,"èħķ":103610,"åijķ":103611,"磶":103612,"åĨľå®¶":103613,"管å§Ķä¼ļ":103614,"饺":103615,"èĬľ":103616,"æ¾Ī":103617,"è©¢":103618,"å¨ģå°¼æĸ¯":103619,"ä½ķåĨµ":103620,"å°ıä¼Ļ":103621,"奢ä¾Ī":103622,"è¿Ļç¯ĩ":103623,"诵":103624,"ç«łç¨ĭ":103625,"ç´Ģ":103626,"éIJĺ":103627,"éĤ¢":103628,"ç³Ļ":103629,"ç¼Ģ":103630,"ä¹Ĵ":103631,"ä¹ĵ":103632,"çī¢åĽº":103633,"åĿŀ":103634,"å¼Ī":103635,"ä¾ĭå¤ĸ":103636,"廳":103637,"è§Ħç«ł":103638,"èĬĻ":103639,"篷":103640,"躯":103641,"æłĪ":103642,"åĿļå®ŀ":103643,"åŁºå»º":103644,"çĿĢçľ¼":103645,"ç·´":103646,"èij©":103647,"ç¼ļ":103648,"æ¦Ĩ":103649,"主åĭķ":103650,"ç¥Ģ":103651,"äºĴéĢļ":103652,"尤为":103653,"å®Ľ":103654,"骼":103655,"æ±²":103656,"ä¾ĥ":103657,"æĤłä¹ħ":103658,"æij§":103659,"æĭĩ":103660,"é«ĵ":103661,"éºĴ":103662,"éĻĽ":103663,"æŀ¸":103664,"æĿŀ":103665,"è´¬":103666,"å°ıé¾Ļ":103667,"åĵ®":103668,"èĵ¬åĭĥ":103669,"åĮĪ":103670,"çķľçī§":103671,"娩":103672,"个å¤ļ":103673,"æ²¥":103674,"æĺ§":103675,"çĦļ":103676,"æĬijéĥģ":103677,"çĸ¡":103678,"èĺij":103679,"éģİç¨ĭ":103680,"橱":103681,"éĿĵ":103682,"大çIJĨ":103683,"髦":103684,"åĪĨ辨":103685,"渤":103686,"çĸ¤":103687,"åĬ¨èĥ½":103688,"å¼łå®¶":103689,"ä¸ĩåįĥ":103690,"滥":103691,"饥":103692,"åºŁå¼ĥ":103693,"帳":103694,"æ¼³":103695,"è±IJ":103696,"ä»ij":103697,"å«ī":103698,"å¦Ĵ":103699,"çŀĴ":103700,"è¡ħ":103701,"çĭ¸":103702,"å¾ģç¨ĭ":103703,"éĤ¯":103704,"éĥ¸":103705,"ç¥Ī":103706,"祷":103707,"趴":103708,"ç»ĵæŀĦæĢ§":103709,"è§ĨåIJ¬":103710,"è¬Ŀ":103711,"çĴĢ":103712,"çĴ¨":103713,"åĩºå¤Ħ":103714,"è¯Ģ":103715,"å¾ĺ":103716,"å¾Ĭ":103717,"羨":103718,"åĸĩ":103719,"åıŃ":103720,"åĺ²":103721,"çķ¸":103722,"å¹²äºĭ":103723,"æļ§":103724,"æ²Ľ":103725,"åĦĦ":103726,"å»ĵ":103727,"åİ¿éķ¿":103728,"èĥļ":103729,"çIJ¢":103730,"çŃ·":103731,"éĩĭ":103732,"ä¾®":103733,"åIJ©":103734,"åĴIJ":103735,"åĮ¿":103736,"æĬ¬èµ·":103737,"æ³£":103738,"涤":103739,"麽":103740,"æĽĻ":103741,"åī¯éĻ¢éķ¿":103742,"åħļåĴĮ":103743,"æķ£åıij":103744,"润æ»ij":103745,"åĵº":103746,"æĥ¬":103747,"漫éķ¿":103748,"ä¸įæĩĪ":103749,"åŁł":103750,"åĹĵ":103751,"èĢģçĪ·":103752,"讽":103753,"æĪĺç»ĦåIJĪ":103754,"æ£ł":103755,"åħ¨åŁŁ":103756,"èł¢":103757,"诡":103758,"åīįçŀ»":103759,"æķĽ":103760,"ä¸Ģå°ģ":103761,"å¹Ĥ":103762,"èİĨ":103763,"è¯Ŀè¯Ń":103764,"ç»ĨåĪĻ":103765,"屿":103766,"åµĮ":103767,"éĢį":103768,"åĺ±":103769,"渲":103770,"çĥ¯":103771,"çĿ¹":103772,"é¦Ĵ":103773,"èħ¥":103774,"æĬĹåĩ»":103775,"çĿ«":103776,"èįĶ":103777,"éļİ":103778,"æ³īæ°´":103779,"è¬Ĥ":103780,"çĤ¬":103781,"åĩıæİĴ":103782,"è¸Ĭ":103783,"è·»":103784,"æ·Į":103785,"éľ¾":103786,"å¥ĩ纳":103787,"å¯Ŀ":103788,"æ¤İ":103789,"æŁ¬":103790,"æĸ¯åŁº":103791,"åħ¬ç«ĭ":103792,"è¨ĵ":103793,"é£Ļ":103794,"é©¿":103795,"åĤµ":103796,"èĽĻ":103797,"ç¯ĩç«ł":103798,"åĪĨæĶ¯":103799,"ä¸Ĭå¹´":103800,"çŃĿ":103801,"缤":103802,"èĢģæŧ":103803,"åĻ¬":103804,"æľ¦":103805,"èĥ§":103806,"æ¶Īè²»":103807,"æĵĶ":103808,"榴":103809,"æ¿Ĵ":103810,"糯":103811,"泸":103812,"æįĨ":103813,"ç»ļ":103814,"èµİ":103815,"çIJIJ":103816,"èµĤ":103817,"æħ®":103818,"æ²Į":103819,"çĦĻ":103820,"æĴŃæĬ¥":103821,"æ·ĩ":103822,"åĪĩåħ¥":103823,"çijķ":103824,"çĸµ":103825,"éģ´":103826,"ç¨ļ":103827,"ç©©":103828,"èŀĥ":103829,"æ£ķ":103830,"æĨ§":103831,"æĨ¬":103832,"伺":103833,"æ¯Ĺ":103834,"æįį":103835,"æĬī":103836,"ç´Ĭ":103837,"å¼Ľ":103838,"æĭŃ":103839,"æĹıèĩªæ²»":103840,"åĿ·":103841,"競":103842,"詳":103843,"è¿Ħä»Ĭ":103844,"è°´":103845,"çŀŃ解":103846,"æŁ¿":103847,"é¢Ĭ":103848,"ç°§":103849,"çĥŁèĬ±":103850,"ä¾¥":103851,"çĿ¦":103852,"éħĿ":103853,"æ°ĵ":103854,"çIJī":103855,"å§Ĭ":103856,"æ²®":103857,"æħ·":103858,"èľķ":103859,"çijļ":103860,"éĩĩçŁ¿":103861,"åł°":103862,"åºķèķ´":103863,"èĨ³":103864,"è¾ķ":103865,"éŁŃ":103866,"åĴĻ":103867,"ç²½":103868,"åīĶ":103869,"沦":103870,"èĤ´":103871,"éķ¶":103872,"æĺ¼":103873,"è¾Ĺ":103874,"婪":103875,"åĮ®":103876,"æĸĵ":103877,"汶":103878,"éĥ´":103879,"éł»":103880,"çªĴ":103881,"袱":103882,"åĽ±":103883,"èĢĺ":103884,"èļĮ":103885,"çĭĻ":103886,"çĹ¹":103887,"ç¥ī":103888,"æı®":103889,"æ·Ĩ":103890,"ç£ĭ":103891,"éĺª":103892,"æ«":103893,"ã¸":103894,"Ļ¶":103895,"ãij":103896,"ð£²":103897,"ä¢":103898,"ãŃ":103899,"ð¬¨":103900,"ð¬Ģ":103901,"ð¬®":103902,"ð¬¯":103903,"ð¬ľ":103904,"ðª¨":103905,"ð«Ĺ":103906,"ð¬Ĭ":103907,"ð¬±":103908,"ð¬Ł":103909,"äİ":103910,"ð¡":103911,"äĥ":103912,"ãł":103913,"ð©":103914,"ð©¾":103915,"ð¬º":103916,"ð¬Ļ":103917,"ãĢĶ":103918,"ãĢķ":103919,"çļĦæŶåĢĻ":103920,"æľīéĻIJåħ¬åı¸":103921,"ä¹ĭåIJİ":103922,"ä¸ļåĬ¡":103923,"åķĬ":103924,"èĻ½çĦ¶":103925,"æĭ¥æľī":103926,"äºĴèģĶç½ij":103927,"éĤ£äºĽ":103928,"ä½łçļĦ":103929,"åĨ³å®ļ":103930,"éĻ¤äºĨ":103931,"åĽ¢éĺŁ":103932,"åı¯æĺ¯":103933,"以åIJİ":103934,"社åĮº":103935,"çļĦéĹ®é¢ĺ":103936,"并ä¸Ķ":103937,"æķĻå¸Ī":103938,"å°±ä¼ļ":103939,"天空éĥ¨èIJ½":103940,"æľĢç»Ī":103941,"å½ĵçĦ¶":103942,"ä¹Łæľī":103943,"ç¡®ä¿Ŀ":103944,"æĥ³è¦ģ":103945,"è´Ńä¹°":103946,"人çļĦ":103947,"åIJ´":103948,"çļĦåıijå±ķ":103949,"ä¸įçŁ¥éģĵ":103950,"软件":103951,"æĪij们çļĦ":103952,"çĪ¶æ¯į":103953,"åīij":103954,"èĢĮæĺ¯":103955,"å®īæİĴ":103956,"åIJİæĿ¥":103957,"çļĦåľ°æĸ¹":103958,"èµµ":103959,"èĢĥè¯ķ":103960,"çªģçĦ¶":103961,"ä¸Ģå®ļè¦ģ":103962,"åĪ¶ä½ľ":103963,"è¯Ħä»·":103964,"åħįè´¹":103965,"è´¹çĶ¨":103966,"绣ä¸Ģ":103967,"çĦ¶èĢĮ":103968,"è¿Ļ次":103969,"éĿĴå¹´":103970,"人类":103971,"亦":103972,"让人":103973,"è´Łè´£äºº":103974,"éĩĩåıĸ":103975,"çļĦäºĭæĥħ":103976,"ä¹Łä¼ļ":103977,"车è¾Ĩ":103978,"æĽ´æĺ¯":103979,"强åĮĸ":103980,"æĪijåĢij":103981,"以åīį":103982,"ä¼ĺåĮĸ":103983,"å§Ķåijĺä¼ļ":103984,"åĽ°éļ¾":103985,"年度":103986,"ä½įäºİ":103987,"æĮĩåĩº":103988,"åĨį次":103989,"åĬŀçIJĨ":103990,"æ¯ı个":103991,"对æĸ¹":103992,"è¿Ľè¡ĮäºĨ":103993,"æľĢé«ĺ":103994,"课ç¨ĭ":103995,"身ä¸Ĭ":103996,"æĽ¾ç»ı":103997,"åĮ»çĶŁ":103998,"å®īè£ħ":103999,"æľ±":104000,"è¿IJè¡Į":104001,"åıĮæĸ¹":104002,"æľĢ大çļĦ":104003,"æŀĦ建":104004,"è¿ŀç»Ń":104005,"çļĦå°ı":104006,"她çļĦ":104007,"çŃīçŃī":104008,"æĶ¹åĸĦ":104009,"åIJĦç±»":104010,"éģĩåĪ°":104011,"æľīçĿĢ":104012,"人çī©":104013,"æĢ»æĺ¯":104014,"è¿ħéĢŁ":104015,"åĪ¶å®ļ":104016,"å®ĥ们":104017,"å®ĺç½ij":104018,"è¿ĺè¦ģ":104019,"ç»Īäºİ":104020,"æĪ¿åľ°äº§":104021,"è¯ģæĺİ":104022,"èĤ¡ç¥¨":104023,"åºĶå½ĵ":104024,"èĭ±åĽ½":104025,"è¿IJçĶ¨":104026,"æľĢæĸ°":104027,"享åıĹ":104028,"让æĪij":104029,"æĻļä¸Ĭ":104030,"å¾ŀ":104031,"å°ı说":104032,"å°¤åħ¶æĺ¯":104033,"è®Ńç»ĥ":104034,"åħ¨å¸Ĥ":104035,"æĮijæĪĺ":104036,"æľīçĤ¹":104037,"带çĿĢ":104038,"çļĦä¸ľè¥¿":104039,"é£İæł¼":104040,"é»Ħéĩij":104041,"å¼ķ导":104042,"æѤå¤ĸ":104043,"æľĢè¿ij":104044,"追æ±Ĥ":104045,"强è°ĥ":104046,"ä¹Łåı¯ä»¥":104047,"æĦŁåĪ°":104048,"èĩªæĪij":104049,"çī¹åĪ«æĺ¯":104050,"æĪIJéĥ½":104051,"éĢIJæ¸IJ":104052,"å¿«ä¹IJ":104053,"ä¹ĭä¸Ń":104054,"æĬķèµĦèĢħ":104055,"ä»ĸ们çļĦ":104056,"æ°ı":104057,"å·¥ä½ľäººåijĺ":104058,"äºĨä¸Ģ个":104059,"åķ¦":104060,"ä¸ĢåĢĭ":104061,"åŁºå±Ĥ":104062,"æ²ŁéĢļ":104063,"第ä¸Ģ次":104064,"并没æľī":104065,"çļĦå·¥ä½ľ":104066,"åľ¨è¿ĻéĩĮ":104067,"æŀª":104068,"æĶ¯æĴij":104069,"æŶå°ļ":104070,"æĿ¥åĪ°":104071,"æĶ¶è´Ń":104072,"éĿ©åij½":104073,"æĺ¯ä¸įæĺ¯":104074,"讨论":104075,"ä¸ļ绩":104076,"å°±èĥ½":104077,"ç«ĭåį³":104078,"è¡Ĺéģĵ":104079,"åľ¨ä¸Ģèµ·":104080,"æľĪ份":104081,"é«ĺ端":104082,"å¾Īéļ¾":104083,"ä¿Ħç½Ĺæĸ¯":104084,"æīĭ段":104085,"åģļåĩº":104086,"ä¼Ĺå¤ļ":104087,"å®ŀè¡Į":104088,"æīĵå¼Ģ":104089,"游客":104090,"ä¾ĿçĦ¶":104091,"å°±åĥı":104092,"离å¼Ģ":104093,"说éģĵ":104094,"æĸ°èĥ½æºIJ":104095,"溪":104096,"äºķ":104097,"令人":104098,"ä¸Ģåľº":104099,"æĪijæĥ³":104100,"两人":104101,"èĩ³å°ij":104102,"çļĦçĶŁæ´»":104103,"æĺ¯ä¸ª":104104,"èĭ±è¯Ń":104105,"æ²Ĵæľī":104106,"æĢĿèĢĥ":104107,"éĻIJåĪ¶":104108,"åı°æ¹¾":104109,"ä¸ĢæŦ":104110,"çļĦä¸Ģ个":104111,"é«ĺ级":104112,"åĬŀåħ¬å®¤":104113,"å¾·åĽ½":104114,"æĪijå°±":104115,"å®ļä½į":104116,"éĢĤåºĶ":104117,"æĮĩæłĩ":104118,"åħ¨çľģ":104119,"ä¸Ĭè¿°":104120,"å®ĥçļĦ":104121,"åĽŀ家":104122,"欧洲":104123,"éĵģè·¯":104124,"é¼ĵåĬ±":104125,"çļĦå½±åĵį":104126,"é«ĺæł¡":104127,"天ä¸ĭ":104128,"é«ĺè´¨éĩı":104129,"æĿŃå·ŀ":104130,"èµĦ讯":104131,"æĶ¾åľ¨":104132,"æľīä¸Ģ个":104133,"å°±è¦ģ":104134,"ä¸ĬéĿ¢":104135,"解éĩĬ":104136,"éĢIJæŃ¥":104137,"尽管":104138,"æľīä»Ģä¹Ī":104139,"çļĦäºĭ":104140,"çĻ»è®°":104141,"人æ°ijå¸ģ":104142,"è§Ĥä¼Ĺ":104143,"è§Ĥå¯Ł":104144,"çĶµèĦij":104145,"çļĦåIJĮæŶ":104146,"ä½ľä¸ļ":104147,"宣å¸ĥ":104148,"çļĦä½ľçĶ¨":104149,"åĽŀæĿ¥":104150,"éļ¾ä»¥":104151,"æīĢæľīçļĦ":104152,"å°ıåѦ":104153,"æıIJåīį":104154,"æ¤įçī©":104155,"åĩ¯":104156,"ä¸ĬäºĨ":104157,"å°±åľ¨":104158,"åħĪåIJİ":104159,"æīĭæľ¯":104160,"éĥŃ":104161,"éĿ¢åīį":104162,"æ¯ķç«Ł":104163,"äºĮæĺ¯":104164,"红èī²":104165,"éĺ³åħī":104166,"èĭ¹æŀľ":104167,"å¾Īå¤ļ人":104168,"ç»ĻæĪij":104169,"åĵ¦":104170,"çľ¼çĿĽ":104171,"éłŃ":104172,"ä¸Ģæĺ¯":104173,"åıijå±ķçļĦ":104174,"åıįåºĶ":104175,"æĪ¿å±ĭ":104176,"æľŁå¾ħ":104177,"ç§įæ¤į":104178,"æĸĩåѦ":104179,"åį³åı¯":104180,"é¦ĸ次":104181,"èĭ±éĽĦ":104182,"å¤ļ次":104183,"åĮħè£ħ":104184,"æ²³åįĹ":104185,"ä¹ĭéĹ´çļĦ":104186,"ä»įçĦ¶":104187,"åIJ¬åĪ°":104188,"èij£äºĭéķ¿":104189,"è§ĦåĪĻ":104190,"ä¸Ģ份":104191,"大ä¼Ĺ":104192,"使å¾Ĺ":104193,"è¿Ľåı£":104194,"ä¸Ģçīĩ":104195,"æĢ§çļĦ":104196,"çļĦ大":104197,"æĪijæĺ¯":104198,"äºĴåĬ¨":104199,"æ°£":104200,"çļĨ":104201,"åħ¬åı¸çļĦ":104202,"ä¸Ģè¾¹":104203,"åıĬåħ¶":104204,"èī¯å¥½çļĦ":104205,"æĭĵå±ķ":104206,"å½ĵå¹´":104207,"å¹¿åľº":104208,"åģļäºĨ":104209,"åŁºäºİ":104210,"æıIJéĨĴ":104211,"åħĦå¼Ł":104212,"èĢģæĿ¿":104213,"è¿ijæĹ¥":104214,"çĬ¶åĨµ":104215,"注éĩį":104216,"åĪļåĪļ":104217,"è°ĥçłĶ":104218,"å¿ĥä¸Ń":104219,"æĬĬæı¡":104220,"éļıåIJİ":104221,"ä¸įå¤Ł":104222,"åĪĽä½ľ":104223,"ç«Ļåľ¨":104224,"缸äºĴ":104225,"çĸ«æĥħéĺ²æݧ":104226,"年代":104227,"带åĬ¨":104228,"伤害":104229,"ç«ŁçĦ¶":104230,"å¼ķè¿Ľ":104231,"累计":104232,"让æĪij们":104233,"åĽŀæĶ¶":104234,"æĬ¥åIJį":104235,"åĬ©åĬĽ":104236,"èģĶ缣":104237,"çŃĸçķ¥":104238,"åij¨è¾¹":104239,"åĭĴ":104240,"è¿ĺåľ¨":104241,"æµģéĩı":104242,"寻æī¾":104243,"çĶµåĬĽ":104244,"èĪ¹èĪ¶":104245,"è¿ĺèĥ½":104246,"æĭħä»»":104247,"çļĦæĥħåĨµä¸ĭ":104248,"çļĦåİŁåĽł":104249,"缺ä¹ı":104250,"çIJĥåijĺ":104251,"å²ģçļĦ":104252,"çĶ·åŃIJ":104253,"å·¥èµĦ":104254,"è¿ijå¹´æĿ¥":104255,"åijĢ":104256,"æıIJä¾ĽäºĨ":104257,"她们":104258,"家åħ·":104259,"çĩķ":104260,"è½»æĿ¾":104261,"æł¡åĽŃ":104262,"èĢĥæł¸":104263,"åį±éĻ©":104264,"åħļç»Ħç»ĩ":104265,"æĢ»ç»ıçIJĨ":104266,"çļĦæĸ°":104267,"çİ»çĴĥ":104268,"è¿Ļä½į":104269,"对æѤ":104270,"家人":104271,"çļĦè¦ģæ±Ĥ":104272,"温度":104273,"æĮĩæķ°":104274,"缴åĪ°":104275,"æѤæŶ":104276,"æ¹ĸåįĹ":104277,"éĥ½è¦ģ":104278,"ä½ľåĩº":104279,"åIJĦä½į":104280,"èĢĥçĶŁ":104281,"ä¾Ŀæį®":104282,"说è¯Ŀ":104283,"æĪijä¹Ł":104284,"å·¥åİĤ":104285,"åıĺæĪIJ":104286,"ä»ĸ人":104287,"æĪijè§īå¾Ĺ":104288,"åIJĦ级":104289,"ä¼łå¥ĩç§ģæľį":104290,"ä¸Ĭåįĩ":104291,"好åĥı":104292,"åĬłéĢŁ":104293,"äºĮåįģ":104294,"è¢ģ":104295,"è£ħ饰":104296,"éĥ½èĥ½":104297,"ä¸Ģå¼ł":104298,"åĬ¨æĢģ":104299,"å¹´çļĦ":104300,"è¿Ļå°±æĺ¯":104301,"ä¹Łè¦ģ":104302,"èµĦæł¼":104303,"æĪĺäºī":104304,"æĦŁè°¢":104305,"åŁ¹èĤ²":104306,"天æ°Ķ":104307,"女士":104308,"åı¯èĥ½ä¼ļ":104309,"çļĦ产åĵģ":104310,"ä¹Łå°±":104311,"主è¦ģæĺ¯":104312,"åĪºæ¿Ģ":104313,"ç»Ļä½ł":104314,"大æķ°æį®":104315,"åĮ»åѦ":104316,"åĪ¤æĸŃ":104317,"ä»ĸ说":104318,"表æ¼Ķ":104319,"äºļæ´²":104320,"ä¸ĵé¢ĺ":104321,"ç«ŀäºīåĬĽ":104322,"éĤ£æł·":104323,"å±ķå¼Ģ":104324,"å¹³æŶ":104325,"æİ¥ä¸ĭæĿ¥":104326,"æī¿è¯º":104327,"æ³ķåĽ½":104328,"åħ³å¿ĥ":104329,"ä¼ļæľī":104330,"éĤĢ请":104331,"é¢Ħéĺ²":104332,"对æİ¥":104333,"好äºĨ":104334,"åĴ±ä»¬":104335,"çļĦæĦŁè§ī":104336,"æĢĿè·¯":104337,"éĥ½æ²¡æľī":104338,"çļĦæĸ¹æ³ķ":104339,"女åŃIJ":104340,"åı¸æ³ķ":104341,"è¿ĺä¼ļ":104342,"è¶ĬæĿ¥è¶Ĭå¤ļ":104343,"åĽłçĤº":104344,"æµ·åįĹ":104345,"人æķ°":104346,"å°Ĩä¼ļ":104347,"ä¸ļ主":104348,"é¤IJ饮":104349,"å±ħä½ı":104350,"åıijåĩº":104351,"è¿ijæľŁ":104352,"å¼ķé¢Ĩ":104353,"æľºåĻ¨äºº":104354,"åĩºæĿ¥çļĦ":104355,"çľĭè§ģ":104356,"ä¿Ĭ":104357,"让ä»ĸ":104358,"ä¸įæĥ³":104359,"å·¥ä½ľçļĦ":104360,"è¡¥åħħ":104361,"æµħ":104362,"çī¹å¾ģ":104363,"ä¸Ĭå¸Ĥåħ¬åı¸":104364,"ç¾İé£Ł":104365,"广西":104366,"æ¯ıä¸Ģ个":104367,"èIJ½åľ°":104368,"åĵģç§į":104369,"åĴĮè°IJ":104370,"å½»åºķ":104371,"é«ĺèĢĥ":104372,"æĺ¨å¤©":104373,"åīįå¾Ģ":104374,"çĽijæµĭ":104375,"çĻ¾åº¦":104376,"åľ¨ä¸ŃåĽ½":104377,"çļĦéľĢæ±Ĥ":104378,"亿ç¾İåħĥ":104379,"åŃ¦æľ¯":104380,"æĶ¶åĪ°":104381,"æĿ¿åĿĹ":104382,"ä¸Ģ段":104383,"æŀĦæĪIJ":104384,"ä¼ģä¸ļçļĦ":104385,"表éĿ¢":104386,"æķ´çIJĨ":104387,"ç»ĵå©ļ":104388,"人家":104389,"åģľæŃ¢":104390,"åѦç§ij":104391,"æĺ¾å¾Ĺ":104392,"ä¼ijæģ¯":104393,"é¢ĦæľŁ":104394,"æĪĸæĺ¯":104395,"çļĦ主è¦ģ":104396,"åºĶ对":104397,"èµ°äºĨ":104398,"ä¸ŃéĹ´":104399,"èµ°è¿Ľ":104400,"åijĪçİ°":104401,"æIJŃéħį":104402,"é¹ı":104403,"æĺ¯åĽłä¸º":104404,"æĥħ绪":104405,"å®ļæľŁ":104406,"社ä¼ļ主ä¹ī":104407,"çŃī级":104408,"çŁĽçĽ¾":104409,"é£ŀæľº":104410,"èĩ³ä»Ĭ":104411,"æĶ¶éĽĨ":104412,"çļĦæķħäºĭ":104413,"åĪĩå®ŀ":104414,"å®ŀçİ°äºĨ":104415,"å½¢æĪIJäºĨ":104416,"åįĹæĸ¹":104417,"ä¸ŃåѦ":104418,"æµ·æ´ĭ":104419,"åIJ¦åĪĻ":104420,"æĭįæijĦ":104421,"大åѦçĶŁ":104422,"åĩºçİ°äºĨ":104423,"æĦıå¤ĸ":104424,"ä¹Łèĥ½":104425,"çļĦèĥ½åĬĽ":104426,"åĿIJåľ¨":104427,"åĪĻæĺ¯":104428,"èĢĥå¯Ł":104429,"å°Ĭéĩį":104430,"éĺ²æŃ¢":104431,"ç´§å¼ł":104432,"读书":104433,"åĩºè¡Į":104434,"å°±æľī":104435,"å±¥è¡Į":104436,"çݰ代åĮĸ":104437,"åĽ½åĬ¡":104438,"åĽ½åĬ¡éĻ¢":104439,"ç»´ä¿®":104440,"åİŁåĪĽ":104441,"æĺ¯æĮĩ":104442,"ä¼ijéĹ²":104443,"çĤ®":104444,"æĸ°æĹ¶ä»£":104445,"éĢĻåĢĭ":104446,"ä¸įæķ¢":104447,"å®Įç¾İ":104448,"ç»ĨèĬĤ":104449,"éŃı":104450,"èĶ¬èıľ":104451,"é¢Ĩ导çıŃåŃIJ":104452,"è¶ħ级":104453,"è¡Įæĥħ":104454,"人工æĻºèĥ½":104455,"åį°åº¦":104456,"åŁºç¡Ģ设æĸ½":104457,"åıĪæĺ¯":104458,"èį¯çī©":104459,"åIJ¸æĶ¶":104460,"åį´æĺ¯":104461,"éĥİ":104462,"å¥ĸåĬ±":104463,"çļĦæľĭåıĭ":104464,"ä¿ĿçķĻ":104465,"è§Ħå¾ĭ":104466,"æĸ°çĸĨ":104467,"è¿ĺåı¯ä»¥":104468,"æİ¥è¿ij":104469,"æѤåīį":104470,"æī¹åĩĨ":104471,"æĢİä¹Īæł·":104472,"çļĦä½įç½®":104473,"ä¸ĢåĿĹ":104474,"æĭĴç»Ŀ":104475,"顾客":104476,"ä¹Łåľ¨":104477,"ä¸ĢçĶŁ":104478,"éĥ¨éĺŁ":104479,"å¹´åīį":104480,"æĸ¹éĿ¢çļĦ":104481,"å°Ŀè¯ķ":104482,"羣æŃ£çļĦ":104483,"ç¦ģæŃ¢":104484,"è¿ĺ没æľī":104485,"æ°ijçĶŁ":104486,"èµ°åIJij":104487,"èĦ¸ä¸Ĭ":104488,"å½ĵ天":104489,"éĽĨåĽ¢åħ¬åı¸":104490,"çļĦä¸Ģç§į":104491,"西æĸ¹":104492,"åĽŀåºĶ":104493,"ä¸Ģ声":104494,"常常":104495,"æıIJåĪ°":104496,"èħ¾è®¯":104497,"æľįè£ħ":104498,"为ä½ķ":104499,"äºijåįĹ":104500,"å°±ç®Ĺ":104501,"ä¼łæī¿":104502,"åıįèĢĮ":104503,"ä¸ĩåIJ¨":104504,"财产":104505,"å¦Ĥä¸ĭ":104506,"æĹ¥åīį":104507,"åİŁæľ¬":104508,"æľĢéĩįè¦ģçļĦ":104509,"认è¯ģ":104510,"ä¸Ģéģĵ":104511,"ä¿¡æģ¯åĮĸ":104512,"å¾ĹåĪ°äºĨ":104513,"éĢ²è¡Į":104514,"æĪijè¦ģ":104515,"éĢļä¿¡":104516,"室åĨħ":104517,"èµļéĴ±":104518,"æĶ¶èĹı":104519,"解åĨ³æĸ¹æ¡Ī":104520,"æĪ¿äº§":104521,"çĭ¼":104522,"æ´»åĬĽ":104523,"ç»ıæµİåıijå±ķ":104524,"çŃīå¾ħ":104525,"ä¹Łå¾Ī":104526,"åĿij":104527,"å¾Ī好çļĦ":104528,"éļ¾åº¦":104529,"ä¸įå¦Ĥ":104530,"人æ°ijæĶ¿åºľ":104531,"åĩºåıij":104532,"åīįæľŁ":104533,"æ¼Ķåijĺ":104534,"女çĶŁ":104535,"èģļçĦ¦":104536,"审计":104537,"é¢Ħæµĭ":104538,"ä¾Ŀæīĺ":104539,"äºĶå¹´":104540,"补贴":104541,"æ¸ħæĻ°":104542,"éªĤ":104543,"çľĭèµ·æĿ¥":104544,"çļĦåŃ©åŃIJ":104545,"é¢ijéģĵ":104546,"ä½ıå®ħ":104547,"éĿ¢åIJij":104548,"æľĢä½İ":104549,"æĹ¢çĦ¶":104550,"ä¸Ģå¥Ĺ":104551,"æķ°åѦ":104552,"群ä½ĵ":104553,"åĮĹ京å¸Ĥ":104554,"å±ħçĦ¶":104555,"æ°ĽåĽ´":104556,"éĢĶå¾Ħ":104557,"çļĦåŁºç¡Ģä¸Ĭ":104558,"èģĮè´£":104559,"åı¯èĥ½æĺ¯":104560,"åĨĽäºĭ":104561,"æĪIJæķĪ":104562,"åŃ©åŃIJ们":104563,"计ç®Ĺæľº":104564,"赤":104565,"产ä¸ļåıijå±ķ":104566,"巨大çļĦ":104567,"工人":104568,"çĶŁéķ¿":104569,"éĥ½åı¯ä»¥":104570,"çļĦæľºä¼ļ":104571,"èµĦè´¨":104572,"çĹĽèĭ¦":104573,"ç²īä¸Ŀ":104574,"å¢ĵ":104575,"å¹³å®ī":104576,"管éģĵ":104577,"è·ŁçĿĢ":104578,"é¥®é£Ł":104579,"åķĨ家":104580,"å¤ļ家":104581,"åı¸æľº":104582,"åºĶ该æĺ¯":104583,"éĢıéľ²":104584,"认å®ļ":104585,"è¡Įä¸ļçļĦ":104586,"çļĦä¼ģä¸ļ":104587,"æ¯ıä¸Ģ":104588,"èĮĥåĽ´åĨħ":104589,"è¾ĥ大":104590,"è´¤":104591,"å¤§èµĽ":104592,"å¤ļäºĨ":104593,"鸿":104594,"临åºĬ":104595,"åľ¨è¿Ļ个":104596,"çļĦåĨħ容":104597,"éĶĢéĩı":104598,"å¾Īå°ij":104599,"åŃŁ":104600,"ç»´æĮģ":104601,"åĴĸåķ¡":104602,"æľ¬åľ°":104603,"èī²å½©":104604,"并éĿŀ":104605,"èĢĮå·²":104606,"温æļĸ":104607,"èIJ§":104608,"æĬĵä½ı":104609,"èĢĮä¸įæĺ¯":104610,"åĸĬ":104611,"çļĦåħ³ç³»":104612,"çī©åĵģ":104613,"éĤ£æĺ¯":104614,"åĨľäº§åĵģ":104615,"è¿ĻæŶ":104616,"å©ļ姻":104617,"æ°´æŀľ":104618,"æĶ¶èİ·":104619,"ä»ĺåĩº":104620,"客æĪ·ç«¯":104621,"æ¼Ķåĩº":104622,"åħ¨æĸ°":104623,"è¿Ļä¹Łæĺ¯":104624,"æĺ¯çĶ±":104625,"è§Ĥ念":104626,"æľī个":104627,"éĢłåŀĭ":104628,"èĥľåĪ©":104629,"ä¸īæĺ¯":104630,"è¶ħå¸Ĥ":104631,"åħļå»ºå·¥ä½ľ":104632,"æĶ¾å¿ĥ":104633,"线路":104634,"æĭĽçĶŁ":104635,"åIJĥé¥Ń":104636,"è½ī":104637,"å°½éĩı":104638,"è§ģåĪ°":104639,"åIJĮæ¯Ķå¢ŀéķ¿":104640,"åįİ为":104641,"æĪijå¸Ĥ":104642,"æıIJåĩºäºĨ":104643,"æ°ijèѦ":104644,"åįļçī©":104645,"åįļçī©é¦Ĩ":104646,"è¯ļä¿¡":104647,"åīįéĿ¢":104648,"山西":104649,"è¾ħåĬ©":104650,"转移":104651,"æĽ´ä¸º":104652,"丰å¯ĮçļĦ":104653,"åį¢":104654,"å¿«éĢĴ":104655,"æĺ¾èijĹ":104656,"çī©èµĦ":104657,"åĪ°è¾¾":104658,"æľīåĪ©äºİ":104659,"åijĨ":104660,"åŃ©åŃIJçļĦ":104661,"ä¸įä½Ĩ":104662,"çłĶ究éĻ¢":104663,"çĶ³æĬ¥":104664,"æļ¨":104665,"æ°ijéĹ´":104666,"åį»":104667,"çļĦå£°éŁ³":104668,"å¸ĤåľºçļĦ":104669,"ä¸Ģåı¥":104670,"çľģ级":104671,"æĿ¥çļĦ":104672,"åĵªä¸ª":104673,"æīįä¼ļ":104674,"åĪĨéħį":104675,"èĶ¡":104676,"ä»ĸåľ¨":104677,"åħ±æľī":104678,"å¡ĺ":104679,"èĴĤ":104680,"éľį":104681,"åıĤè§Ĥ":104682,"ä¸Ī夫":104683,"ä¾ĿéĿł":104684,"æľīæŶ":104685,"äºĨå¾Īå¤ļ":104686,"ä¸ĸçķĮæĿ¯":104687,"家æĹı":104688,"ä¸įéľĢè¦ģ":104689,"大å¸Ī":104690,"èŀįåħ¥":104691,"éĿŀæ³ķ":104692,"çĹħ人":104693,"åIJİæľŁ":104694,"大家éĥ½":104695,"ç½ijåĿĢ":104696,"åİŁæĸĻ":104697,"ä¾¿å®ľ":104698,"æ¶Ľ":104699,"ä»¿ä½Ľ":104700,"å·®è·Ŀ":104701,"åı¦ä¸Ģæĸ¹éĿ¢":104702,"产åĵģçļĦ":104703,"赫":104704,"æĥħåĨµä¸ĭ":104705,"éĴ¢éĵģ":104706,"æľ¬ç«Ļ":104707,"纳åħ¥":104708,"å·²æľī":104709,"æľī没æľī":104710,"估计":104711,"é£ĺ":104712,"æľŁè´§":104713,"åĢĭ人è³ĩæĸĻ":104714,"ä¸ĵä¸ļçļĦ":104715,"çĪĨåıij":104716,"èĩ´åĬĽäºİ":104717,"çİ°åľ¨çļĦ":104718,"æľīåĵªäºĽ":104719,"çł´åĿı":104720,"æķ°åŃĹåĮĸ":104721,"åľ°éĿ¢":104722,"é»ijèī²":104723,"å¹¼åĦ¿åĽŃ":104724,"çļĦç²¾ç¥ŀ":104725,"äºŃ":104726,"导æ¼Ķ":104727,"çİ°æľī":104728,"æѦåĻ¨":104729,"èĭıå·ŀ":104730,"çİĦ":104731,"æ±Łè¥¿":104732,"延伸":104733,"论æĸĩ":104734,"è¾ĥ为":104735,"çİ©æ³ķ":104736,"é¼İ":104737,"åIJĮæŃ¥":104738,"éĩĬæĶ¾":104739,"æĽĿåħī":104740,"åĿļåĨ³":104741,"å§Ķæīĺ":104742,"å°Ĩåľ¨":104743,"äºĪ以":104744,"ä½ľæĸĩ":104745,"èĢĮåľ¨":104746,"ä¼ĺåħĪ":104747,"åĽŀåİ»":104748,"ä¿®å¤į":104749,"åĽ½åĨħå¤ĸ":104750,"çŃĸåĪĴ":104751,"åıijæĶ¾":104752,"å¿ĥæĥħ":104753,"çļĦåİĨåı²":104754,"éĿ¢è¯ķ":104755,"举åĮĹ":104756,"ä¿¡åı·":104757,"ç²®é£Ł":104758,"è¯ģ书":104759,"æŁIJäºĽ":104760,"è¿IJä½ľ":104761,"åĨ²åĩ»":104762,"çĥŃçĤ¹":104763,"æŶæŶ":104764,"æŶæĹ¶å½©":104765,"åľ°çĤ¹":104766,"ä¸Ģä½ĵåĮĸ":104767,"éļ¾é¢ĺ":104768,"æĽ°":104769,"ç«ĭåĪ»":104770,"æĺ¯éĿŀ常":104771,"åħ±åĴĮ":104772,"åħ±åĴĮåĽ½":104773,"æ¿ĢåĬ±":104774,"æľīæķĪçļĦ":104775,"å¤Ħç½®":104776,"该åħ¬åı¸":104777,"æ£ĢéªĮ":104778,"èѦæĸ¹":104779,"è´¾":104780,"äºĨä¸Ģä¸ĭ":104781,"ä»ĬåIJİ":104782,"çħ®":104783,"çĶ¨åĵģ":104784,"读èĢħ":104785,"æĪijåľ¨":104786,"åĽŀå¤į":104787,"ä¸Ģ座":104788,"è¿ĺ没":104789,"å®ļåĪ¶":104790,"没æĥ³åĪ°":104791,"夹":104792,"ä¼łéĢĴ":104793,"ä¸Ģ款":104794,"强大çļĦ":104795,"çļĦè¡Į为":104796,"å¤ı天":104797,"åıijåĬ¨æľº":104798,"é¢ĨåŁŁçļĦ":104799,"å®ŀéªĮ室":104800,"ä¸ĢæĬĬ":104801,"æĺ¯ä¸ºäºĨ":104802,"éĻķ西":104803,"æĭħä¿Ŀ":104804,"è¾¾æĪIJ":104805,"è¦ģæĺ¯":104806,"æĺİ天":104807,"ç»Ļä»ĸ":104808,"建ç«ĭäºĨ":104809,"ä¸įè¡Į":104810,"ä¸Ńæĸĩ":104811,"åľ°è¯´":104812,"åIJİçļĦ":104813,"çĽijæݧ":104814,"éĢ¸":104815,"æĢ»éĥ¨":104816,"æľ¬æĸĩ":104817,"鹿":104818,"æĻ¯è§Ĥ":104819,"çļĦ缮æłĩ":104820,"èĽĩ":104821,"åĨ¯":104822,"ä¸ŃåĮ»":104823,"æķĪåºĶ":104824,"产éĩı":104825,"åŃĿ":104826,"è´¦æĪ·":104827,"è¿Ŀåıį":104828,"èij£äºĭä¼ļ":104829,"äº¬ä¸ľ":104830,"责任ç¼ĸè¾ij":104831,"åķıé¡Į":104832,"çĪ±å¿ĥ":104833,"èŃ¦å¯Ł":104834,"é¤IJåİħ":104835,"å¸ĤæĶ¿åºľ":104836,"天天":104837,"æĸ°é²ľ":104838,"éĥijå·ŀ":104839,"è¶ħè¶Ĭ":104840,"å½Ń":104841,"çŁ¥è¯Ĩ产æĿĥ":104842,"åĽŀå¿Ĩ":104843,"路线":104844,"å»īæ´ģ":104845,"éĿĴå°ijå¹´":104846,"åıĸå¾ĹäºĨ":104847,"çľĭåĪ°äºĨ":104848,"馬":104849,"ç²¾åĵģ":104850,"åľ°éĵģ":104851,"æĮģæľī":104852,"ä¸ĭäºĨ":104853,"æľīæŶåĢĻ":104854,"ä¸Ģ人":104855,"æĴĴ":104856,"ä»Ķç»Ĩ":104857,"èĢģåħ¬":104858,"äºĭå®ŀä¸Ĭ":104859,"èģĶèµĽ":104860,"ä¾ĽåºĶéĵ¾":104861,"é¢Ħç®Ĺ":104862,"åĪ¶éĢłä¸ļ":104863,"å®īåħ¨çĶŁäº§":104864,"俱ä¹IJ":104865,"俱ä¹IJéĥ¨":104866,"çļĦæł¸å¿ĥ":104867,"æīĵç®Ĺ":104868,"å½±çīĩ":104869,"æIJŃ建":104870,"ä¹Łä¸įä¼ļ":104871,"æĭħå½ĵ":104872,"å±ĤéĿ¢":104873,"åѦåijĺ":104874,"临æŶ":104875,"缸ç»ĵåIJĪ":104876,"对æ¯Ķ":104877,"ä»ĸæĺ¯":104878,"æĸ°åĮº":104879,"è¿Ľåİ»":104880,"çĻ¾å¹´":104881,"ä¿©":104882,"尽快":104883,"çĶµåŃIJåķĨåĬ¡":104884,"æĽ´æľī":104885,"æ¸ħçIJĨ":104886,"åı¦ä¸Ģ个":104887,"åĤ»":104888,"ä»Ģä¹Īæł·çļĦ":104889,"æĺ¯æľĢ":104890,"åij¨å¹´":104891,"å¾Ī容æĺĵ":104892,"åĽ¢ç»ĵ":104893,"ç´Ħ":104894,"æĹ©å·²":104895,"çļĦåıĺåĮĸ":104896,"éľŀ":104897,"æĹ¥ä¸ĬåįĪ":104898,"失åİ»":104899,"ä¸Ńåľĭ":104900,"çļĦä¸ĢäºĽ":104901,"å°ıåŃ©":104902,"ä¸ĭè·Į":104903,"éĶ»çĤ¼":104904,"éij":104905,"éij«":104906,"å¿ĹæĦ¿èĢħ":104907,"èĤ¡å¸Ĥ":104908,"èµĽäºĭ":104909,"许åı¯è¯ģ":104910,"åı¯æĮģç»Ń":104911,"åijĬè¯īè®°èĢħ":104912,"éĢ»è¾ij":104913,"å¼ķåħ¥":104914,"çļĦè¿ĩç¨ĭä¸Ń":104915,"è§Ĩè§ī":104916,"èĩªæ²»åĮº":104917,"è¯ģæį®":104918,"è£ħç½®":104919,"第ä¸īæĸ¹":104920,"å¹´æĿ¥":104921,"å¹¿ä¸ľçľģ":104922,"带æĿ¥äºĨ":104923,"éķ¿æ±Ł":104924,"访éĹ®":104925,"å·®ä¸įå¤ļ":104926,"æĺ¯æĪij":104927,"éģŃéģĩ":104928,"æĬĵ好":104929,"é«ĺè¾¾":104930,"å¹¶åľ¨":104931,"èĩªè§ī":104932,"ä¾ĽåºĶåķĨ":104933,"æĥħæĦŁ":104934,"ä½ıäºĨ":104935,"çļĦèģĮä¸ļ":104936,"çļĩå¸Ŀ":104937,"西éĥ¨":104938,"åĴĮå¹³":104939,"çļĦåĬĽéĩı":104940,"汪":104941,"åħħåĪĨåıijæĮ¥":104942,"æĬķè¯ī":104943,"èµ·åĪ°":104944,"äºĴ缸":104945,"æ¾³éŨ":104946,"æİ¥åĪ°":104947,"æ°´æ³¥":104948,"模åŀĭ":104949,"ä¸ĢåįĬ":104950,"秩åºı":104951,"æĪijä»¬åľ¨":104952,"æī¿è®¤":104953,"ä¸Ģéĥ¨åĪĨ":104954,"åįłæ¯Ķ":104955,"å¦ĩ女":104956,"ç²ĺ":104957,"äºĨ解åĪ°":104958,"ä¸Ģå®ļä¼ļ":104959,"åIJĦ大":104960,"èµ°åĩº":104961,"为大家":104962,"é«ĺéĵģ":104963,"åı¯ä»¥åľ¨":104964,"ä½Ĩåľ¨":104965,"çĶŁæĢģçݯå¢ĥ":104966,"èı¯":104967,"çļĦä»·æł¼":104968,"麻çĥ¦":104969,"æ¿Ģåıij":104970,"éĤ£å°±":104971,"çļĦæł·åŃIJ":104972,"为æѤ":104973,"å¤©åľ°":104974,"çļĦ缮çļĦ":104975,"åĢºåĪ¸":104976,"å·²ç¶ĵ":104977,"åĽĽå¤§":104978,"åIJĮæĹ¶ä¹Ł":104979,"å½¼æѤ":104980,"æĭ¿åĪ°":104981,"åIJ«éĩı":104982,"åįģ大":104983,"éļ¾éģĵ":104984,"å¼Ĺ":104985,"ä¸Ģ段æŶéĹ´":104986,"çħ§é¡¾":104987,"æķ°æį®æĺ¾ç¤º":104988,"æĪIJ为äºĨ":104989,"èµ°åĪ°":104990,"æľ¬åħ¬åı¸":104991,"ç»Ī端":104992,"ä¹Łä¸įæĺ¯":104993,"头åıij":104994,"大约":104995,"é£İæĻ¯":104996,"æ¶ĪèĢĹ":104997,"å®¡æŁ¥":104998,"äºīåıĸ":104999,"æ³ķæ²»":105000,"äºĭçī©":105001,"ç¼ĵ解":105002,"æĥ¨":105003,"缸åºĶçļĦ":105004,"çļĦæķĪæŀľ":105005,"åıįå¤į":105006,"åıijçĶŁäºĨ":105007,"éĢĻäºĽ":105008,"ç»ĥä¹ł":105009,"åݨæĪ¿":105010,"å¼Ģæĭĵ":105011,"欣èµı":105012,"夫妻":105013,"ä¸įä¸Ģæł·":105014,"产èĥ½":105015,"èĬ¯çīĩ":105016,"è¦ģç´ł":105017,"åıį对":105018,"çİĩåħĪ":105019,"è´§çī©":105020,"æĹ¥çĶµ":105021,"ä½ľå®¶":105022,"æĶ¹è¿Ľ":105023,"æĪIJåĪĨ":105024,"åĽłèĢĮ":105025,"åĩıèĤ¥":105026,"æ½ĺ":105027,"å±±ä¸ľçľģ":105028,"åĬĿ":105029,"åŁĭ":105030,"æѦè£ħ":105031,"æ±ĩæĬ¥":105032,"ä¸Ģ个æľĪ":105033,"çĥŃéŨ":105034,"大éģĵ":105035,"æ´»åĭķ":105036,"éĥ½å¾Ī":105037,"çĶµæ¢¯":105038,"ç´§æĢ¥":105039,"åĢºåĬ¡":105040,"客æľį":105041,"ä¸Ģéĥ¨":105042,"ä½łæĺ¯":105043,"çİ°çĬ¶":105044,"æŃ£ç¡®çļĦ":105045,"ä¹ĭå¤Ħ":105046,"ç¼ĸåĪ¶":105047,"ä½łåı¯ä»¥":105048,"çŃīåľ°":105049,"èİī":105050,"对è¯Ŀ":105051,"æ·ĺå®Ŀ":105052,"è°ĥèĬĤ":105053,"æİĴæĶ¾":105054,"åºĵåŃĺ":105055,"ç´ļ":105056,"çļĦä¼ĺåĬ¿":105057,"æĿĥå¨ģ":105058,"以ä¸ĭç®Ģ称":105059,"ä¸Ģ项":105060,"èģļéĽĨ":105061,"ä¼łç»ŁçļĦ":105062,"æ··åIJĪ":105063,"è¿Ļä¸ĢçĤ¹":105064,"ä¸Ģçľ¼":105065,"æĹłéĻIJ":105066,"èİ·å¾ĹäºĨ":105067,"éĢīæīĭ":105068,"åĪ¶åĵģ":105069,"åįıä½ľ":105070,"çĭ¬çī¹çļĦ":105071,"ä¸Ģ级":105072,"è¿Ļ个éĹ®é¢ĺ":105073,"æĸĮ":105074,"æĺ¯æĪij们":105075,"æķĮ人":105076,"æ¸ħæ´Ĺ":105077,"ä¸ĢçĽ´åľ¨":105078,"å°ıç±³":105079,"çļĦè¿ĩç¨ĭ":105080,"åľ¨åĮĹ京":105081,"ä¸ĢæĶ¯":105082,"æĹ©ä¸Ĭ":105083,"æĸĩèīº":105084,"ç¦ıåĪ©":105085,"é£ŁçĶ¨":105086,"æĦŁåĬ¨":105087,"åħ¨ç¨ĭ":105088,"æĶ¯åĩº":105089,"æĸ°å»º":105090,"å¸ķ":105091,"æĺ¾çĦ¶":105092,"羣çļĦæĺ¯":105093,"æĸ°éĹ»ç½ij":105094,"èĥ½åIJ¦":105095,"åįıåĬ©":105096,"亲èĩª":105097,"å¾Īæľī":105098,"çĻ¼å±ķ":105099,"æĦı大":105100,"æĦı大åĪ©":105101,"çĶµç½ij":105102,"æĹ¥çĽĬ":105103,"çĨ±":105104,"èĤĮèĤ¤":105105,"çĶ·æĢ§":105106,"ç»Ħ建":105107,"çŃīéĹ®é¢ĺ":105108,"æ¶ĪéĻ¤":105109,"æĬ¤çIJĨ":105110,"å¡ijæĸĻ":105111,"ä¹Įåħĭ":105112,"ä¹Įåħĭåħ°":105113,"åķĨæłĩ":105114,"çIJ³":105115,"æĸ°æīĭ":105116,"çļĦçī¹çĤ¹":105117,"åĴ¬":105118,"å½ĵä¸ĭ":105119,"设计å¸Ī":105120,"èµĶåģ¿":105121,"第åįģ":105122,"æĻºèĥ½åĮĸ":105123,"å¼ĢåıijåĮº":105124,"åı¯ä»¥éĢļè¿ĩ":105125,"åħ±äº§åħļ":105126,"åİī害":105127,"çģµæ´»":105128,"æŶåħī":105129,"éĥ¨ä½į":105130,"人æĸĩ":105131,"è¿ĽæĿ¥":105132,"ä¹ĭæīĢ以":105133,"ä¸īåįģ":105134,"çļĦåѦçĶŁ":105135,"éĺ²æĬ¤":105136,"åĽ½äº§":105137,"æ·±åľ³å¸Ĥ":105138,"éĤ£å°±æĺ¯":105139,"åĪ°ä½į":105140,"çī¹æľĹ":105141,"çī¹æľĹæĻ®":105142,"å®ŀæŶ":105143,"åı°çģ£":105144,"èĢĮä¸į":105145,"æĮĩå®ļ":105146,"åĿĿ":105147,"èħIJè´¥":105148,"çī¹å®ļ":105149,"å¢ŀéĢŁ":105150,"æłĩçŃ¾":105151,"æĪ¿ä»·":105152,"æĦģ":105153,"贯彻èIJ½å®ŀ":105154,"æĢ§è´¨":105155,"çłĶ究çĶŁ":105156,"ç¾İ容":105157,"æī¹è¯Ħ":105158,"究竣":105159,"人åĬĽèµĦæºIJ":105160,"éĸĭå§ĭ":105161,"åĽŀå½Ĵ":105162,"èIJ¥åķĨ":105163,"èIJ¥åķĨçݯå¢ĥ":105164,"ä¸ŃåĽ½äºº":105165,"çļĦåŁºæľ¬":105166,"è¯Ŀé¢ĺ":105167,"æłĩåĩĨåĮĸ":105168,"西èĹı":105169,"åĭ¾":105170,"çļĦ设计":105171,"ç®ĢåįķçļĦ":105172,"å¤įåĪ¶":105173,"æ¸IJæ¸IJ":105174,"以å¤ĸ":105175,"èģĶåĬ¨":105176,"两次":105177,"æĢ§åĴĮ":105178,"æĽ´å¤§":105179,"çļĦåIJįåŃĹ":105180,"飦":105181,"ä½łè¦ģ":105182,"å¢ĥå¤ĸ":105183,"æĹ©æľŁ":105184,"åĪĿæŃ¥":105185,"è´¦åı·":105186,"害æĢķ":105187,"æĺ¨æĹ¥":105188,"åĪļæīį":105189,"ç¥ŀç§ĺ":105190,"ç²¾å¿ĥ":105191,"æµģéĢļ":105192,"åħ¨æĸ¹ä½į":105193,"以å¾Ģ":105194,"ä¹Łå°Ĩ":105195,"æĺ¯ä¸ŃåĽ½":105196,"åĽ½å®¶çº§":105197,"å°ĨåĨĽ":105198,"æijĬ":105199,"æľĢ为":105200,"第ä¸ĢæŶéĹ´":105201,"æ¶Īæ¯Ĵ":105202,"å°Ĩäºİ":105203,"å¨ģèĥģ":105204,"èĭ±æĸĩ":105205,"æīĭä¸Ń":105206,"çIJĥè¿·":105207,"è§Ĥçľĭ":105208,"离å©ļ":105209,"æľ¬åľŁ":105210,"åĪĨæķ£":105211,"æĻ´":105212,"è¦ģ注æĦı":105213,"浪费":105214,"管æݧ":105215,"åĩºåĶ®":105216,"æĢ»è£ģ":105217,"ä¸Ģéĺµ":105218,"å¨ĩ":105219,"äºĶ个":105220,"å½ĵåĪĿ":105221,"çºłçº·":105222,"ä¸ĵçĶ¨":105223,"å¤ĩæ¡Ī":105224,"åĪĿæľŁ":105225,"å®ĥæĺ¯":105226,"åĮºåĿĹ":105227,"åĮºåĿĹéĵ¾":105228,"大è¿ŀ":105229,"è¿Ļç±»":105230,"åıĺæĪIJäºĨ":105231,"éĤĦæĺ¯":105232,"åįļ客":105233,"çı¾åľ¨":105234,"ä¸Ģæĸ¹":105235,"å®ĮæĪIJäºĨ":105236,"è¿Ļ个æŶåĢĻ":105237,"åħ¨å¹´":105238,"ä¸Ĭ线":105239,"ç½IJ":105240,"ç«ŀèµĽ":105241,"åĩºçīĪ社":105242,"åĵ¥åĵ¥":105243,"寫":105244,"å¾Ĺ以":105245,"èĬ±åĽŃ":105246,"äºĨèµ·æĿ¥":105247,"èĦ±è´«æĶ»åĿļ":105248,"çļĦåİŁåĪĻ":105249,"讲解":105250,"æ¶ĪåĮĸ":105251,"æįŁå®³":105252,"æļĤæŶ":105253,"å¾ĹçŁ¥":105254,"éĢĤçĶ¨":105255,"éŨåºĹ":105256,"解读":105257,"æĻ®åıĬ":105258,"人æ°ijæ³ķéĻ¢":105259,"åī¯ä¸»ä»»":105260,"å¿ĥçģµ":105261,"è¯ĬæĸŃ":105262,"ç¾İ女":105263,"æŁ¯":105264,"年以æĿ¥":105265,"æ´»è·ĥ":105266,"åĢŁåĬ©":105267,"åħ±å»º":105268,"è¯ī讼":105269,"æĶ¾æĿ¾":105270,"çªĹåı£":105271,"ä¼ģæ¥Ń":105272,"åĬłæĭ¿":105273,"åĬłæĭ¿å¤§":105274,"ä¹°äºĨ":105275,"主æµģ":105276,"æĩĤå¾Ĺ":105277,"å°Ĩåħ¶":105278,"éĢıæĺİ":105279,"å·¥ä½ľä¸Ń":105280,"èĤ¡ä»·":105281,"æ¡£æ¡Ī":105282,"没æľīä»»ä½ķ":105283,"åijĬçŁ¥":105284,"å¹´åĪĿ":105285,"æĹ¥ä¸ĭåįĪ":105286,"åİĤåķĨ":105287,"èĬĤå¥ı":105288,"主导":105289,"è£Ŀ":105290,"åħ³éĶ®è¯į":105291,"èģĬ天":105292,"åĨĻä½ľ":105293,"æĶ¹éĿ©å¼ĢæĶ¾":105294,"æľīæľĽ":105295,"éĢļæĬ¥":105296,"èIJĮ":105297,"æĢ»é¢Ŀ":105298,"çŁŃæľŁ":105299,"ä¸Ģçķª":105300,"çĶŁæ´»çļĦ":105301,"åĮĸçļĦ":105302,"æĺ¥å¤©":105303,"è¿Ļåľº":105304,"æĸ°å¼Ģä¼łå¥ĩ":105305,"æĺ¯è¦ģ":105306,"å°ļæľª":105307,"åıĺæĽ´":105308,"ä¸Ģåij¨":105309,"客è§Ĥ":105310,"æĹ¥èĩ³":105311,"é¹°":105312,"çݲ":105313,"å°ĨæĿ¥":105314,"客人":105315,"åıĺéĿ©":105316,"说äºĨ":105317,"åİŁçIJĨ":105318,"èģĮåĬ¡":105319,"åıĪæľī":105320,"ä¸Ģåı¥è¯Ŀ":105321,"æĦŁåıĹåĪ°":105322,"ç¬ĶèĢħ":105323,"移æ°ij":105324,"西åįĹ":105325,"ä¹ĥèĩ³":105326,"æŃ£è§Ħ":105327,"åĪĿä¸Ń":105328,"çĬ¬":105329,"å½ĵäºĭ":105330,"å½ĵäºĭ人":105331,"æĪij们è¦ģ":105332,"åħ¥åı£":105333,"éĤ£æŶ":105334,"æľīéĻIJ责任":105335,"å°ij女":105336,"è¿Ļä¹Īå¤ļ":105337,"åĪĨåħ¬åı¸":105338,"å®ĩå®Ļ":105339,"çļĦéĢīæĭ©":105340,"å§IJå§IJ":105341,"åıijèµ·":105342,"è»į":105343,"æĽ´å¥½åľ°":105344,"éĻĨç»Ń":105345,"æľ¬æľįåĭĻ":105346,"å«©":105347,"赶紧":105348,"èĦĤèĤª":105349,"第äºĮ天":105350,"æĪijä¼ļ":105351,"两ä½į":105352,"æķ²":105353,"åħ¬å®īæľºåħ³":105354,"ç§ijæĬĢåĪĽæĸ°":105355,"尺寸":105356,"è¾IJå°Ħ":105357,"å®ĹæķĻ":105358,"转æį¢":105359,"åĩºçİ°åľ¨":105360,"ä¸Ģé¢Ĺ":105361,"æľŁéĻIJ":105362,"åIJĮåѦ们":105363,"åĮĹæĸ¹":105364,"ä½łå°±":105365,"ä¸Ģ带ä¸Ģè·¯":105366,"èĢģå©Ĩ":105367,"游æĪıçݩ家":105368,"çļĦç»ĵæŀľ":105369,"è¡¥åģ¿":105370,"å¤ĸè´¸":105371,"对å¾ħ":105372,"ç»´çĶŁç´ł":105373,"ç»ıéĶĢåķĨ":105374,"è¿ĺå°Ĩ":105375,"åŃIJ女":105376,"æĽ´é«ĺ":105377,"ä¸į大":105378,"éī´å®ļ":105379,"让ä»ĸ们":105380,"æīĢè°ĵçļĦ":105381,"æŃ»äºĨ":105382,"帮æī¶":105383,"åĵ²åѦ":105384,"以ä¸ĬçļĦ":105385,"çļĦåħ³éĶ®":105386,"æĹ©å°±":105387,"æĬ¥ä»·":105388,"éģµå®Ī":105389,"æī©å¼ł":105390,"æĺ¯å¾Ī":105391,"å¼ĢéĢļ":105392,"æĸ°åĬł":105393,"æĸ°åĬłåĿ¡":105394,"ç¿»è¯ij":105395,"询éĹ®":105396,"é¸Ń":105397,"ä½ĵåĨħ":105398,"两个人":105399,"çĪ¹":105400,"éľľ":105401,"乡æĿijæĮ¯åħ´":105402,"çĿ¡è§ī":105403,"å®ĺåijĺ":105404,"åĪĽå§ĭ":105405,"åĪĽå§ĭ人":105406,"ä¼Ĺ人":105407,"åį³ä¾¿":105408,"çĸ«èĭĹ":105409,"ä¼ģä¸ļ家":105410,"渣":105411,"ç²¾åĬĽ":105412,"å¤ĸéĥ¨":105413,"èģªæĺİ":105414,"è¿Ļä¹Ł":105415,"å½ķåıĸ":105416,"åĨ²çªģ":105417,"åħ¨èº«":105418,"åŃ£èĬĤ":105419,"忽çĦ¶":105420,"çļĦæĢģ度":105421,"åĤ¨å¤ĩ":105422,"ä¿Ŀåħ»":105423,"çļĦæĥ³æ³ķ":105424,"ä¸Ĭæµ·å¸Ĥ":105425,"æIJºæīĭ":105426,"çļĦä¿¡æģ¯":105427,"åķĨåľº":105428,"çļĦæĢĿæĥ³":105429,"æĿĥåĬĽ":105430,"毫æĹł":105431,"æĢĢåŃķ":105432,"硬件":105433,"åĨħèĴĻåı¤":105434,"æݢ讨":105435,"åħ»çĶŁ":105436,"çļĦ表çİ°":105437,"空ä¸Ń":105438,"æģIJæĢĸ":105439,"å¾Īé«ĺ":105440,"ç»ıæµİ社ä¼ļ":105441,"ä¸ĬæĿ¥":105442,"延ç»Ń":105443,"éĩįå¤į":105444,"éĺ²èĮĥ":105445,"çļĦå½¢å¼ı":105446,"æľĪåºķ":105447,"èĢģ年人":105448,"绿åĮĸ":105449,"å±±åĮº":105450,"æĭ¿åĩº":105451,"æĹħ客":105452,"æĽ´æį¢":105453,"åħ¬ä¸»":105454,"èĬĤ约":105455,"åħ¨åİ¿":105456,"åĽŀæĬ¥":105457,"çIJĨæĢ§":105458,"çĸ¯çĭĤ":105459,"æ¶īå«Į":105460,"åī§æĥħ":105461,"åĨ¬åŃ£":105462,"åIJİç»Ń":105463,"è¿Ļæĺ¯ä¸Ģ个":105464,"æ¼Ķ讲":105465,"ä¸Ģå±Ĥ":105466,"æľīåħ³éĥ¨éŨ":105467,"æĹłå¥Ī":105468,"ç§įç±»":105469,"缸åħ³çļĦ":105470,"æĪĸèĢħæĺ¯":105471,"æī¶æĮģ":105472,"å¤ļæķ°":105473,"çļĦä½ľåĵģ":105474,"ä¸ĭä¸ĢæŃ¥":105475,"å¸ĪåĤħ":105476,"é«ĺéĢŁåħ¬è·¯":105477,"好åıĭ":105478,"ä¼ĺç§ĢçļĦ":105479,"è¿ĽäºĨ":105480,"æģIJæĢķ":105481,"äºĨåIJ§":105482,"大è§Ħ模":105483,"çļĦä¸ĸçķĮ":105484,"æĢĢçĸij":105485,"å··":105486,"åħ´å¥ĭ":105487,"æĪ°":105488,"æĿijéĩĮ":105489,"æľĭåıĭåľĪ":105490,"åĨ¬å¤©":105491,"ä¸Ńåįİ人æ°ij":105492,"åįıåķĨ":105493,"è¯ĦéĢī":105494,"æĹŃ":105495,"å¢ŀåĬłäºĨ":105496,"åıĹ伤":105497,"ä¸ĢèĤ¡":105498,"便æį·":105499,"ä¸ij":105500,"鹤":105501,"å¤ĸè§Ĥ":105502,"å·¥ç¨ĭå¸Ī":105503,"åĴĮåħ¶ä»ĸ":105504,"è¿Ļå°±":105505,"ä¸Ńå°ıä¼ģä¸ļ":105506,"西åĮĹ":105507,"åĽ½æľīä¼ģä¸ļ":105508,"èĭ¥æĺ¯":105509,"åı¯æĥľ":105510,"çĶŁæĹ¥":105511,"åĩ½":105512,"ä¹°åįĸ":105513,"ç¥Ŀç¦ı":105514,"人æ°ij群ä¼Ĺ":105515,"åħīæĺİ":105516,"åħ¬å¯ĵ":105517,"æĺ¯è°ģ":105518,"æĪijçŁ¥éģĵ":105519,"è¯Ńæĸĩ":105520,"æķıæĦŁ":105521,"ä¸įéĶĻçļĦ":105522,"æĿ¥è®²":105523,"æ³¢åĬ¨":105524,"çļĦ第ä¸Ģ":105525,"åľ°éľĩ":105526,"åľ¨åħ¨åĽ½":105527,"骨干":105528,"å®īç½®":105529,"家çĶµ":105530,"ä¸İæѤ":105531,"ä¸İæѤåIJĮæŶ":105532,"åıĹçģ¾":105533,"çĥŃ线":105534,"çļĦæĬĢæľ¯":105535,"æµĭéĩı":105536,"ä¾Ŀèµĸ":105537,"ä¸ŃåĽ½çļĦ":105538,"çī¹æĢ§":105539,"è¾ĥé«ĺ":105540,"踩":105541,"ä¼ļåľ¨":105542,"建éĢł":105543,"导èĪª":105544,"æĥ³èµ·":105545,"åħ¨ä¸ĸçķĮ":105546,"建æĿIJ":105547,"ç¯Ģ":105548,"çļĦåŁºç¡Ģ":105549,"èĩªåĬ¨åĮĸ":105550,"åīįåIJİ":105551,"çĿ¡çľł":105552,"æݨè¡Į":105553,"æį®äºĨ解":105554,"ä»Ģä¹ĪæŶåĢĻ":105555,"ä¸įåĸľæ¬¢":105556,"çħ¤çĤŃ":105557,"éĤ£ä¹Īå¤ļ":105558,"å¸ĤåľºåĮĸ":105559,"ä¸į管æĺ¯":105560,"ç«ĭåľº":105561,"éĥ½æ²¡":105562,"课é¢ĺ":105563,"æĪij们å°Ĩ":105564,"è¿ĩçļĦ":105565,"åĨįåĬłä¸Ĭ":105566,"çĪ¾":105567,"身æĿIJ":105568,"çĶ·å¥³":105569,"è¿ľè¿ľ":105570,"çĶ·çĶŁ":105571,"èĩªèº«çļĦ":105572,"è´Łæĭħ":105573,"çĻ¾ä¸ĩ":105574,"西çıŃ":105575,"西çıŃçīĻ":105576,"åĩĢåĪ©æ¶¦":105577,"澳大":105578,"澳大åĪ©äºļ":105579,"ä¸įåİ»":105580,"æī¿åıĹ":105581,"楼çĽĺ":105582,"å¢ĥåĨħ":105583,"æ··åĩĿ":105584,"æ··åĩĿåľŁ":105585,"æĢĿæĥ³æĶ¿æ²»":105586,"å¸ĤåĮº":105587,"æĭĽæłĩ":105588,"åĽ¢ä½ĵ":105589,"è¿Ľåº¦":105590,"åĨĽéĺŁ":105591,"åıįå¼¹":105592,"äºĨä¸ĢäºĽ":105593,"æİ¥å¾ħ":105594,"çļĦåŃ¦ä¹ł":105595,"éħįéĢģ":105596,"é£Łåĵģå®īåħ¨":105597,"æĽ¿ä»£":105598,"æĺ¯ä»¥":105599,"éĢļçĶ¨":105600,"çłĶ究æīĢ":105601,"ç¦ħ":105602,"æīĶ":105603,"éļĶ离":105604,"ä¸ĩå¹³æĸ¹ç±³":105605,"çļĦè§Ħå®ļ":105606,"ç»ĻæĪij们":105607,"æ¿Ģåħī":105608,"ä¼ļåĩºçİ°":105609,"çŁŃä¿¡":105610,"ç©¿çĿĢ":105611,"æ²Īéĺ³":105612,"æķĻæĿIJ":105613,"éĺ²çĸ«":105614,"ä¼ĺèī¯":105615,"约å®ļ":105616,"æĪijçľģ":105617,"åħ¬æ°ij":105618,"éģ¸æĵ":105619,"éģ¸æĵĩ":105620,"å·²æĪIJ为":105621,"ä¸įå¿ħ":105622,"ç¥ĸåĽ½":105623,"å¹¶æľª":105624,"åľŁå£¤":105625,"å¾®ç¬ij":105626,"äºĭä¸ļåįķä½į":105627,"çļĦ游æĪı":105628,"åħ¬ç¤º":105629,"åIJĪçIJĨçļĦ":105630,"çªĿ":105631,"æ°Ķ象":105632,"家ä¸Ń":105633,"äº®çĽ¸":105634,"åį«æĺŁ":105635,"è®°è½½":105636,"è§Ĩéĩİ":105637,"åľ°åĮºçļĦ":105638,"ä½Ĩä»ĸ":105639,"èĤĮèĤī":105640,"äºıæįŁ":105641,"åĬŀåѦ":105642,"ä¸Ģè¡Į":105643,"è¯ŀçĶŁ":105644,"åıijå¸ĥçļĦ":105645,"çļĦæľįåĬ¡":105646,"çļĦçłĶ究":105647,"åij¨æľ«":105648,"产ä¸ļåĽŃ":105649,"é«ĺ温":105650,"æĪIJåĬŁçļĦ":105651,"æŃ¥éª¤":105652,"åŃĺåĤ¨":105653,"åŃIJåħ¬åı¸":105654,"让她":105655,"ä¸Ńæľī":105656,"åĺī宾":105657,"妮":105658,"æĺİå¹´":105659,"äºĨåIJĹ":105660,"äºīè®®":105661,"æĪĪ":105662,"ä¸Ģæľ¬":105663,"ç¾İ丽çļĦ":105664,"ä½łè¯´":105665,"大人":105666,"æĶ»çķ¥":105667,"ä¸įæľĥ":105668,"å¾ħéģĩ":105669,"ä¸Ģè¾Ĩ":105670,"çīĪæĿĥæīĢæľī":105671,"æ°ijä¼Ĺ":105672,"åĬŁå¤«":105673,"å±ķä¼ļ":105674,"大èĦij":105675,"æ¯ıæľĪ":105676,"å°ı麦":105677,"æµĻæ±Łçľģ":105678,"çļĦæīĢæľī":105679,"ä¸ĭæ»ij":105680,"èĵĿèī²":105681,"è¦ģæĥ³":105682,"åѦçĶŁçļĦ":105683,"å½ĵä½ł":105684,"ä½ľæĪĺ":105685,"家乡":105686,"å¤ļåIJį":105687,"é«ĺäºİ":105688,"åĿļ强":105689,"è¿ŀéĶģ":105690,"åIJİæŀľ":105691,"人äºĭ":105692,"ç´ħ":105693,"æ¿ĢåĬ¨":105694,"è¿ĽæĶ»":105695,"ç©Ĩ":105696,"ä¸ĺ":105697,"让èĩªå·±":105698,"以æѤ":105699,"夫人":105700,"å¼Ģ设":105701,"æ°Ķè´¨":105702,"鸡èĽĭ":105703,"çĦ¡æ³ķ":105704,"åIJĥäºĨ":105705,"åĪĨåĪ«ä¸º":105706,"èģĶåIJĪåĽ½":105707,"å½ĵ代":105708,"å¦Ĥæŀľæĺ¯":105709,"è¿ľç¨ĭ":105710,"åĸĤ":105711,"è®°ä½ı":105712,"æ¸ħåįķ":105713,"åIJĪä½ľä¼Ļä¼´":105714,"åİ»åģļ":105715,"æķħéļľ":105716,"模æĭŁ":105717,"å¸ĪçĶŁ":105718,"åīįæĿ¥":105719,"çĶµè§Ĩåī§":105720,"çĥŃçĪ±":105721,"éľ²åĩº":105722,"é«ĺå±Ĥ":105723,"çĶµåĻ¨":105724,"纪å¾ĭ":105725,"å¼ĢåıijåķĨ":105726,"éķ¿å®ī":105727,"è½½ä½ĵ":105728,"çļĦå°±æĺ¯":105729,"被人":105730,"åıĹçIJĨ":105731,"篮çIJĥ":105732,"èİİ":105733,"交ç»Ļ":105734,"æľªæĿ¥çļĦ":105735,"两大":105736,"åIJķå¸ĥ":105737,"çŃī人":105738,"çļĦæĹ¥åŃIJ":105739,"åIJĪä½ľç¤¾":105740,"æĮijéĢī":105741,"åŃĺ款":105742,"ç³»ç»ŁçļĦ":105743,"æĬĬå®ĥ":105744,"没æľīä»Ģä¹Ī":105745,"ä»İæѤ":105746,"ä¸ŃåįĪ":105747,"çĸ¼çĹĽ":105748,"å·©åĽº":105749,"浪漫":105750,"缸åħ³éĥ¨éŨ":105751,"éķ¿åŁİ":105752,"纤维":105753,"ä¸ĬéŨ":105754,"çĪĨçĤ¸":105755,"èµ·çĤ¹":105756,"çļĦéĢļçŁ¥":105757,"èĢĮæĿ¥":105758,"çļĦèĢģ":105759,"æīĭéĩĮ":105760,"è¯ŃéŁ³":105761,"è¾Ľèĭ¦":105762,"æ±Łèĭıçľģ":105763,"çĶ¨äºĨ":105764,"身份è¯ģ":105765,"æľīåĬ©":105766,"æľīåĬ©äºİ":105767,"çī©èģĶç½ij":105768,"åĩºéŨ":105769,"å¼ŁåŃIJ":105770,"æĥ¹":105771,"è¿Ļ件äºĭ":105772,"æĪij们åı¯ä»¥":105773,"çļĦçĶŁåij½":105774,"æľīä¸Ģç§į":105775,"åºĹéĵº":105776,"åıĮæīĭ":105777,"çļĦæ¶Īæģ¯":105778,"èĢIJå¿ĥ":105779,"å°´å°¬":105780,"éĤ£å¤©":105781,"é¦ĸæī¹":105782,"æĺ¯ä¸Ģ家":105783,"人æ°Ķ":105784,"åıįæŃ£":105785,"æĪijåĴĮ":105786,"å®łçī©":105787,"ä¸į对":105788,"寻æ±Ĥ":105789,"çĽ¸ä¼¼":105790,"åľ¨ç¾İåĽ½":105791,"åı«åģļ":105792,"åĹİ":105793,"ç«ĭ足":105794,"çĶ¨éĢĶ":105795,"åħĨ":105796,"大æ°Ķ":105797,"åIJijä¸Ĭ":105798,"ä»ĸå°±":105799,"é¡¹çĽ®å»ºè®¾":105800,"èĭ¥å¹²":105801,"æĺ¯æľī":105802,"æ¿Ģæĥħ":105803,"çļĦæĦıä¹ī":105804,"æĺŃ":105805,"严éĩįçļĦ":105806,"å¯ĨéĽĨ":105807,"èĪŀè¹Ī":105808,"èį£èİ·":105809,"èİ·æĤī":105810,"æ±ŁåįĹ":105811,"åģĩå¦Ĥ":105812,"æĪ·å¤ĸ":105813,"线索":105814,"ç§ģ人":105815,"转åŀĭåįĩ级":105816,"çļĦä»·åĢ¼":105817,"åįķçĭ¬":105818,"èĢģçĻ¾å§ĵ":105819,"å°įæĸ¼":105820,"åĽ½éĻħåĮĸ":105821,"ä¼°åĢ¼":105822,"æľįåĬ¡ä¸ļ":105823,"èĩŃ":105824,"æİīäºĨ":105825,"解åĨ³äºĨ":105826,"ä¹Łä¸įèĥ½":105827,"åħ¹":105828,"æĸ¯çī¹":105829,"æķħæĦı":105830,"è¿ĩ度":105831,"èĬĤæĹ¥":105832,"çĻ½çĻľ":105833,"çĻ½çĻľé£İ":105834,"继æī¿":105835,"äºĨä¸įå°ij":105836,"äºĮ人":105837,"è§ģéĿ¢":105838,"æĥ³æĥ³":105839,"å¤įåIJĪ":105840,"康å¤į":105841,"åİ¿åŁİ":105842,"åľ¨åĽ½åĨħ":105843,"åľºåľ°":105844,"éĻ¶çĵ·":105845,"è¿Ļ项":105846,"çľ¼ä¸Ń":105847,"糸":105848,"æĦŁè§īåĪ°":105849,"æŀľçĦ¶":105850,"æĶ¾åħ¥":105851,"约æĿŁ":105852,"æİĴæŁ¥":105853,"车主":105854,"çļĦæĦıæĢĿ":105855,"æĸ°åŁİ":105856,"æĥ³çĿĢ":105857,"éģĤ":105858,"èĮ¶åı¶":105859,"ä¹°æĪ¿":105860,"åĨľæĪ·":105861,"é«ĺæīĭ":105862,"çİīç±³":105863,"æĸ°åĨłèĤºçĤİ":105864,"çħ§æĺİ":105865,"æĮĩåįĹ":105866,"踢":105867,"æķijæı´":105868,"æĻ¯çĤ¹":105869,"ç¨İæĶ¶":105870,"çļĦæīĭ":105871,"æŃ£å¥½":105872,"è¦ģæĬĬ":105873,"éļıæĦı":105874,"åħ¶å®ŀæĺ¯":105875,"ç»Ļèĩªå·±":105876,"è°ĪåĪ¤":105877,"æ¯ı天éĥ½":105878,"æĢģåĬ¿":105879,"é¢Ħ约":105880,"åİĨåı²ä¸Ĭ":105881,"å®Ŀè´Ŀ":105882,"åīįè¿Ľ":105883,"ä¹Łå°±æĺ¯è¯´":105884,"çļĦæĦıè§ģ":105885,"åı£ç½©":105886,"åİĺç±³":105887,"èĬ±è´¹":105888,"ä½ĵèĤ²æĬķ注":105889,"åħ¬ä¼Ĺåı·":105890,"èijĹåIJįçļĦ":105891,"å¼ĢæĪ·":105892,"æĭįåįĸ":105893,"å²ģæľĪ":105894,"åĨħ涵":105895,"å®Įæķ´çļĦ":105896,"é«ĺåİĭ":105897,"åħ¬åĬ¡åijĺ":105898,"使çĶ¨çļĦ":105899,"çĶŁäº§çº¿":105900,"妹妹":105901,"走访":105902,"æĺ¯åı¯ä»¥":105903,"åľ¨å®¶":105904,"æļ´åĬĽ":105905,"æ³°åĽ½":105906,"è´¨çĸij":105907,"ä¸įéģİ":105908,"天çĦ¶æ°Ķ":105909,"缺çĤ¹":105910,"å°ıåŀĭ":105911,"ä¸įä»ħæĺ¯":105912,"é»ijæļĹ":105913,"梨":105914,"æĸĩæĹħ":105915,"è¦ģæľī":105916,"ä¸Ńå±±":105917,"çļĦæķ°æį®":105918,"å¾Ĺå¾Ī":105919,"以便":105920,"对ä»ĸ":105921,"åĬłä»¥":105922,"çĻ¼çı¾":105923,"设å®ļ":105924,"èĤļåŃIJ":105925,"éĿĸ":105926,"å¥īçĮ®":105927,"ä¸įåıĺ":105928,"åı£ç¢ij":105929,"åľ¨åĵªéĩĮ":105930,"ä½IJ":105931,"è¿Ļ两个":105932,"çļĦæĸ¹åIJij":105933,"æŀ«":105934,"äºĮ次":105935,"çīĩåĮº":105936,"éłIJ":105937,"ç£Ĭ":105938,"æĭ¿çĿĢ":105939,"å·²ç»ıæĪIJ为":105940,"ä¹ĭä¸Ĭ":105941,"å®ĹæŨ":105942,"奶奶":105943,"é«ĺæĸ°åĮº":105944,"社æľĥ":105945,"è·Łè¸ª":105946,"æľįåĬ¡ä¸Ńå¿ĥ":105947,"æī¯":105948,"æīĭæĮĩ":105949,"礼çī©":105950,"宿èĪį":105951,"çĶ¨å¿ĥ":105952,"æıIJé«ĺäºĨ":105953,"亮çĤ¹":105954,"ä¸įæĦ¿æĦı":105955,"æĴŃæĶ¾":105956,"å¤ļå°ijéĴ±":105957,"没ä»Ģä¹Ī":105958,"æķ°åįģ":105959,"æĢ»çĽij":105960,"çļĦåŁİå¸Ĥ":105961,"æī¾åĪ°äºĨ":105962,"åĨħåľ°":105963,"åĪ°çİ°åľ¨":105964,"æĪĺæĸĹåĬĽ":105965,"åİŁå§ĭ":105966,"åĥ§":105967,"åĢĴæĺ¯":105968,"æľĢåħ·":105969,"è´«åĽ°æĪ·":105970,"éĢģåĪ°":105971,"级åĪ«":105972,"åĩºèµĦ":105973,"æĪªæŃ¢":105974,"ç§įåŃIJ":105975,"èĥ½ä¸įèĥ½":105976,"幸è¿IJ":105977,"èĸĩ":105978,"项éĵ¾":105979,"æĮĤçīĮ":105980,"ä¸Ģ樣":105981,"ä¹ĺ客":105982,"èIJ½åIJİ":105983,"ä½ĨæĪij":105984,"æĹ©åľ¨":105985,"åĬ¨æ¼«":105986,"å¹³çŃī":105987,"å¯¹ä½ł":105988,"ä¸įæĢķ":105989,"å¤ĸçķĮ":105990,"å¤ļå¹´æĿ¥":105991,"é¦ĸ个":105992,"æ²³åįĹçľģ":105993,"æĪĸåħ¶ä»ĸ":105994,"éķľå¤´":105995,"åįĹæĺĮ":105996,"ä¸ĢéĿ¢":105997,"éĢłæĪIJçļĦ":105998,"å´Ķ":105999,"çŃĴ":106000,"æķĻèĤ²éĥ¨":106001,"åľ°åŁŁ":106002,"æĺĨæĺİ":106003,"å·´é»İ":106004,"æīĭ游":106005,"ä¸ĢæŶ":106006,"çłį":106007,"顶级":106008,"åħ±è®¡":106009,"åİŁæ²¹":106010,"è¾īçħĮ":106011,"说æĺ¯":106012,"æĸ°åįİ社":106013,"ç»ıåİĨäºĨ":106014,"ä¸įæŃ¢":106015,"è¦ģä¹Ī":106016,"èĢħçļĦ":106017,"æĢ»æĬķèµĦ":106018,"è¡Į驶":106019,"ä¸Ĭå¸Ŀ":106020,"年纪":106021,"çIJ¼":106022,"ä¼łè¯´":106023,"ç²¾èĭ±":106024,"æĸ¹éĴĪ":106025,"æ±Łæ¹ĸ":106026,"æĪIJçĤº":106027,"æĢ»éĩı":106028,"æĬķæĶ¾":106029,"åĬ¨çĶ»":106030,"èŤ":106031,"çĶµæºIJ":106032,"éĴĻ":106033,"åIJĮè¡Į":106034,"æĻ®éĢļçļĦ":106035,"åĽ¾ä¹¦é¦Ĩ":106036,"è¯ĪéªĹ":106037,"æħĪåĸĦ":106038,"è¿Ļ份":106039,"主æĮģ人":106040,"å°±è¿Ļæł·":106041,"èĢĮæĪIJ":106042,"èĩªè¡Į车":106043,"ä¸ŃåĽ½çī¹èī²":106044,"èĤ¿çĺ¤":106045,"åIJ¾":106046,"å¼Łå¼Ł":106047,"åıĹçĽĬ":106048,"éĢīæĭ©äºĨ":106049,"æĺİæĺ¾çļĦ":106050,"æĬ¥èĢĥ":106051,"ç¬ijéģĵ":106052,"éĽĸçĦ¶":106053,"温å·ŀ":106054,"éĿŀæ´²":106055,"ç§įç§į":106056,"åıĤåĬłäºĨ":106057,"è´§è¿IJ":106058,"éļı便":106059,"就没æľī":106060,"縣":106061,"央è§Ĩ":106062,"ç©¿è¶Ĭ":106063,"çļĦçݰ象":106064,"åĩłæ¬¡":106065,"çļĦé£İéĻ©":106066,"æŃĮæĽ²":106067,"æľ¬å±Ĭ":106068,"å¹´åĨħ":106069,"ä¸įè¶ħè¿ĩ":106070,"è¿ĩå¤ļ":106071,"å¿ħé¡»è¦ģ":106072,"ç»ĵ论":106073,"åĢŁéī´":106074,"ç¥ŀå¥ĩ":106075,"æľŁæľĽ":106076,"ä¸ĵ享":106077,"éĿŀ常éĩįè¦ģ":106078,"æĦıè¯ĨåĪ°":106079,"åIJĪ并":106080,"æĬĬèĩªå·±":106081,"å¥Ĺè£ħ":106082,"éŃĶæ³ķ":106083,"å¤ıåŃ£":106084,"ä¸įåĥı":106085,"å¢ĥçķĮ":106086,"æĥĬåĸľ":106087,"æľīä¸Ģ天":106088,"çĦ¦çĤ¹":106089,"æĪij认为":106090,"åħ°å·ŀ":106091,"çĶµæ°Ķ":106092,"èģĶç³»æĪij们":106093,"ç§ijæĻ®":106094,"她说":106095,"çļĦæĸĩç«ł":106096,"å¥ĩæĢª":106097,"åıĭ好":106098,"饮æĸĻ":106099,"çļĦæĶ¯æĮģ":106100,"çŃĶåºĶ":106101,"éĩįéĩı":106102,"çij¶":106103,"åĩıè½»":106104,"ç§ijåŃ¦å®¶":106105,"巴西":106106,"éĩijèŀįæľºæŀĦ":106107,"åħļå§Ķ书记":106108,"貸款":106109,"ç²¾èĩ´":106110,"ä»İæľª":106111,"åį°åĪ·":106112,"åĽŀ顾":106113,"é¦ĸéĥ½":106114,"åıijèĤ²":106115,"éĹ®éģĵ":106116,"è¾¾åĪ°äºĨ":106117,"å¿įä¸įä½ı":106118,"æīįæľī":106119,"æįIJèµł":106120,"ä½ĽæķĻ":106121,"ä¸įæ¸ħ":106122,"éĺŁéķ¿":106123,"缸åıį":106124,"æĬ¥èѦ":106125,"大åħ¨":106126,"æ¬§çĽŁ":106127,"帮å¿Ļ":106128,"çļĦæĻĤåĢĻ":106129,"缮å½ķ":106130,"足以":106131,"èī°éļ¾":106132,"ä»ĸä¹Ł":106133,"å·¥ä½ľèĢħ":106134,"头èĦij":106135,"缺éĻ·":106136,"æĪIJç«ĭäºĨ":106137,"å°±å¼Ģå§ĭ":106138,"认åIJĮ":106139,"é»Ħèī²":106140,"çĹħæĥħ":106141,"覺å¾Ĺ":106142,"è¿Ļ两":106143,"ä¿¡ä»°":106144,"åľĭ家":106145,"ä¸įä»ħä»ħæĺ¯":106146,"çĭ¬å®¶":106147,"èĪ¬çļĦ":106148,"æĿIJè´¨":106149,"æµ·ä¸Ĭ":106150,"çĤºäºĨ":106151,"æľºåĬ¨è½¦":106152,"缸å½ĵäºİ":106153,"å¤ļåħĥåĮĸ":106154,"æĽ´å¤§çļĦ":106155,"èĽ®":106156,"åģĩæľŁ":106157,"å¼ıçļĦ":106158,"交éĢļè¿IJè¾ĵ":106159,"çľģå§Ķ":106160,"ä¸įç®Ĺ":106161,"æĶ¾ä¸ĭ":106162,"éĹ¯":106163,"äººåľ¨":106164,"港åı£":106165,"æĹ¨åľ¨":106166,"åij½ä»¤":106167,"æŁIJ个":106168,"平稳":106169,"åıªå¥½":106170,"人人":106171,"äºŀ":106172,"äºĮç»´":106173,"äºĮç»´çłģ":106174,"æŀģ为":106175,"åĪ«å¢ħ":106176,"åħ¶ä½Ļ":106177,"大äºĭ":106178,"主管éĥ¨éŨ":106179,"æĹłéĶ¡":106180,"éŵ":106181,"éģŃåĪ°":106182,"说è¿ĩ":106183,"ä¸ºä½ł":106184,"解çŃĶ":106185,"éªĮæĶ¶":106186,"çļĦç»ıéªĮ":106187,"åĮ¹éħį":106188,"çģ«ç®Ń":106189,"豪åįİ":106190,"æŁIJæŁIJ":106191,"çļĦæĹ¶ä»£":106192,"书éĿ¢":106193,"æģĴ大":106194,"延éķ¿":106195,"ä¸ĢåIJĮ":106196,"æľªèĥ½":106197,"交æį¢":106198,"çĶ¢åĵģ":106199,"çŃīåĪ°":106200,"åĪĨ离":106201,"æīĵçĶµè¯Ŀ":106202,"å¹²çĩ¥":106203,"è¾ĥå¤ļ":106204,"å¤ļå¹´çļĦ":106205,"èĥĮæĻ¯ä¸ĭ":106206,"为ä¾ĭ":106207,"æijĺè¦ģ":106208,"å´Ľèµ·":106209,"æѤåĪ»":106210,"æľīæľºä¼ļ":106211,"æĿ¡æ¬¾":106212,"é¢Ĩ导å°ıç»Ħ":106213,"çļĦ身ä½ĵ":106214,"åįķä¸Ģ":106215,"央è¡Į":106216,"ä¸įæĸŃæıIJé«ĺ":106217,"ä»·åĢ¼è§Ĥ":106218,"èĬ½":106219,"èIJį":106220,"æ³ķå¾ĭæ³ķè§Ħ":106221,"ä¸įéĶĪ":106222,"ä¸įéĶĪéĴ¢":106223,"åĩºäºİ":106224,"èĻļæĭŁ":106225,"æį®æĤī":106226,"çĥ¦æģ¼":106227,"åħ¨æĸ°çļĦ":106228,"æī«æıı":106229,"çĻ»éĻĨ":106230,"èīºæľ¯å®¶":106231,"çļĦé£Łçī©":106232,"çļĦåŃĺåľ¨":106233,"客åİħ":106234,"æĪij们就":106235,"æŁ¥çľĭæĽ´å¤ļ":106236,"è¯Ħ审":106237,"å¸Ĥåł´":106238,"è¬Ľ":106239,"巨头":106240,"ä¸ŃåĽ½ç»ıæµİ":106241,"äºĨèĩªå·±çļĦ":106242,"åĨ³è®®":106243,"çĽijçĿ£ç®¡çIJĨ":106244,"æĬķ票":106245,"åĨį度":106246,"è¡ĮçĤº":106247,"注åħ¥":106248,"ä½ľä¸ºä¸Ģ个":106249,"æ¯ı个人éĥ½":106250,"åįķåħĥ":106251,"è¦ģçŁ¥éģĵ":106252,"被称为":106253,"ä¹ĭéĻħ":106254,"解éĻ¤":106255,"丸":106256,"溫":106257,"ä¸īæĺŁ":106258,"é²ľæĺİ":106259,"ä¹Łéĥ½":106260,"æĹ¶æľº":106261,"åĩºæīĭ":106262,"æĥħå½¢":106263,"åķĨè´¸":106264,"éĢī举":106265,"对èĩªå·±":106266,"çĶŁåĬ¨":106267,"åħĭæľį":106268,"个ä½ĵ":106269,"èĭij":106270,"稱":106271,"大åݦ":106272,"æĺ¯å¯¹":106273,"åĪ©æģ¯":106274,"è¿IJåĬ¨åijĺ":106275,"åĮĸ解":106276,"åīį沿":106277,"æĦŁæģ©":106278,"æĢ»ä¹ĭ":106279,"é«ĺæĸ°æĬĢæľ¯":106280,"åĿĩ为":106281,"åħ¨åĮº":106282,"æ°Ķæ°Ľ":106283,"åı¯ä»¥è¯´æĺ¯":106284,"ä½ı宿":106285,"åħļåijĺå¹²éĥ¨":106286,"åĹ¯":106287,"è·µè¡Į":106288,"çļĦä¸ĵä¸ļ":106289,"èĢĥéªĮ":106290,"èķ¾":106291,"åħ¬åŃIJ":106292,"çļĦçĬ¶æĢģ":106293,"æ½®æµģ":106294,"ä¿¡æīĺ":106295,"è´¼":106296,"åIJĦæĸ¹":106297,"æķijåĬ©":106298,"éĿŀ常çļĦ":106299,"æ¡¥æ¢ģ":106300,"åħ¬æĸ¤":106301,"ä¼¼çļĦ":106302,"çľĭ好":106303,"å±Ģéĥ¨":106304,"å®īéĿĻ":106305,"éħį件":106306,"常è§Ħ":106307,"å¼Ģ车":106308,"第äºĮ次":106309,"ä¸Ĭ级":106310,"åıĤèµĽ":106311,"家å±ŀ":106312,"强åĬ¿":106313,"åľ¨ä»ĸ":106314,"åIJijåīį":106315,"ä¹ĭåľ°":106316,"éĥ¡":106317,"è¡Įç¨ĭ":106318,"èѦåijĬ":106319,"è§Ħå®ļçļĦ":106320,"åķĨåŁİ":106321,"äºĶ大":106322,"æķĻ室":106323,"åįģ足":106324,"æīĢä»¥åľ¨":106325,"å°Ĩ继ç»Ń":106326,"çŃīæĸ¹å¼ı":106327,"家ä¼ģä¸ļ":106328,"交ä»ĺ":106329,"çĤ¹è¯Ħ":106330,"ç»ĵç®Ĺ":106331,"ä¹Łåı¯":106332,"å¤ĸæ±ĩ":106333,"è¿Ļç§įæĥħåĨµ":106334,"æİĪäºĪ":106335,"å¸ĥç½®":106336,"æĪIJç«ĭäºİ":106337,"é¢ĦèѦ":106338,"管çIJĨ人åijĺ":106339,"å©ļ礼":106340,"ç»ĵæĿŁåIJİ":106341,"åħ¥éĢī":106342,"æĹłæ¯Ķ":106343,"åĴĮåıijå±ķ":106344,"çĻ½éħĴ":106345,"çİ©åħ·":106346,"ä¸ĩç¾İåħĥ":106347,"çļĦæĪIJ绩":106348,"æĭįçħ§":106349,"èĢĥèĻijåĪ°":106350,"ä¼ģä¸ļåıijå±ķ":106351,"äºĨ个":106352,"çĶŁæ°Ķ":106353,"çļĦ女人":106354,"äºĶåįģ":106355,"çĪ·çĪ·":106356,"纽约":106357,"éĥ½è¢«":106358,"ä¸Ĭ课":106359,"çĽ¡":106360,"ä¼łç»ŁæĸĩåĮĸ":106361,"æ½ľåľ¨":106362,"åıijå°Ħ":106363,"ä¸Ģ身":106364,"éĺ²å®Ī":106365,"åĪ®":106366,"é¢ĺ缮":106367,"åľ¨åĨħçļĦ":106368,"ç¾İ好çļĦ":106369,"è¿ĻéĩĮçļĦ":106370,"ä¸Ģä¸Ŀ":106371,"人åĿĩ":106372,"åĢ¡å¯¼":106373,"身åIJİ":106374,"æī©å±ķ":106375,"大éŨ":106376,"就被":106377,"è¯¥é¡¹çĽ®":106378,"æŀ¶æŀĦ":106379,"ä¸Ģåı£":106380,"ä¿¡æģ¯æĬĢæľ¯":106381,"å¼Ģä¸ļ":106382,"æĶ¶åıĸ":106383,"ç½ij页":106384,"æĶ¯æı´":106385,"å°ģéĹŃ":106386,"å¡ijéĢł":106387,"大èĥĨ":106388,"å¿«éĢŁåıijå±ķ":106389,"çľĭä¼¼":106390,"æ¸Ŀ":106391,"è¿Ļæł·ä¸Ģ个":106392,"模åĿĹ":106393,"注æĦıåĪ°":106394,"çł´è§£":106395,"èĩªä»İ":106396,"åijµåijµ":106397,"ä¹ĭå¾Į":106398,"ä¹ĭæĹħ":106399,"è·ŁæĪij":106400,"æ³ķ人":106401,"æİĴè¡Įæ¦ľ":106402,"åĿļå®Ī":106403,"好å¤Ħ":106404,"çŁ³å¤´":106405,"并å°Ĩ":106406,"èĪ±":106407,"æŃĩ":106408,"两岸":106409,"å¤ļä¹ħ":106410,"象å¾ģ":106411,"个æĢ§åĮĸ":106412,"çļĦè§Ĵ度":106413,"å¸Ĩ":106414,"ç¦ıå·ŀ":106415,"æŁ¥å¤Ħ":106416,"ä¸¤åĽ½":106417,"åIJ¸å¼ķäºĨ":106418,"é¦ĸå¸Ń":106419,"大åĵ¥":106420,"é¤Ĭ":106421,"涨å¹ħ":106422,"éĢīçĶ¨":106423,"許å¤ļ":106424,"èIJ½æĪ·":106425,"åĵĪå°Ķ":106426,"åĵĪå°Ķ滨":106427,"åģļä»Ģä¹Ī":106428,"以åħį":106429,"é¾į":106430,"æĹłéľĢ":106431,"åĪ°åºķæĺ¯":106432,"æĢ¡":106433,"åijĬè¯īä½ł":106434,"éĺ²æ°´":106435,"è¿ĻæŶåĢĻ":106436,"欢ä¹IJ":106437,"转åIJij":106438,"è¿Ļä¸ªåľ°åĽ¾":106439,"åħ¥é©»":106440,"èįīåİŁ":106441,"æĹ¶ä»£çļĦ":106442,"åıĺåĬ¨":106443,"åĬłå¼ºå¯¹":106444,"åģ¶å°Ķ":106445,"å®ĪæĬ¤":106446,"æ°Ķ温":106447,"人éĹ´":106448,"æľĿé²ľ":106449,"ç»ıè´¹":106450,"åĽŃæŀĹ":106451,"å·¥åľ°":106452,"è§Ħæł¼":106453,"åĩłåįģ":106454,"è¯ķåĽ¾":106455,"å¦ĥ":106456,"éĤ£æŶåĢĻ":106457,"å¼ĺæī¬":106458,"ä¸ļçķĮ":106459,"çļĦéĢŁåº¦":106460,"ä¼ļä¸įä¼ļ":106461,"èIJ¥æĶ¶":106462,"å°ıå¾®ä¼ģä¸ļ":106463,"çľĭè¿ĩ":106464,"æĬĬä»ĸ":106465,"éģµå¾ª":106466,"è¿Ļè¾¹":106467,"没æľī人":106468,"壶":106469,"æ¹ĸåįĹçľģ":106470,"æŀģåħ¶":106471,"çļĦ人çĶŁ":106472,"ä»ĸè¿ĺ":106473,"转åĮĸ为":106474,"èµ°è¿ĩ":106475,"æĬ±çĿĢ":106476,"çīĽå¥¶":106477,"ä¸ĩ亩":106478,"å¿ĥæĢģ":106479,"æĹ¥å¸¸çĶŁæ´»":106480,"ä½ĵæ£Ģ":106481,"æĻĥ":106482,"çŃīé¢ĨåŁŁ":106483,"æĩī該":106484,"åı¯ä»¥çľĭåĪ°":106485,"æī¾ä¸įåĪ°":106486,"èĢģå¹´":106487,"æĬĬæĪij":106488,"积åĪĨ":106489,"梳çIJĨ":106490,"绳":106491,"çļĦæĶ¿æ²»":106492,"å¸ĿåĽ½":106493,"éĻªä¼´":106494,"æ´Ľéĺ³":106495,"åħ¬æŃ£":106496,"å¼Ģåı£":106497,"çī¹èī²çļĦ":106498,"åĽ°å¢ĥ":106499,"ä¸Ĭæľī":106500,"ç«ĭä½ĵ":106501,"æīĵå·¥":106502,"åķ¤éħĴ":106503,"åľ¨éĤ£éĩĮ":106504,"éĤ£è¾¹":106505,"个åĪ«":106506,"ä¸Ģå®ļæĺ¯":106507,"çļĦéĩįè¦ģæĢ§":106508,"ä¸»å¼ł":106509,"åĴĮæľįåĬ¡":106510,"ä¸Ĭç½ij":106511,"è¡¥åĬ©":106512,"åıªéľĢ":106513,"弦":106514,"éģ®":106515,"åĬĽäºī":106516,"度è¿ĩ":106517,"èij¬":106518,"é¡¿æŶ":106519,"éĦī":106520,"纺ç»ĩ":106521,"åľ°åĿĹ":106522,"ä¿¡çĶ¨åį¡":106523,"ç½ļ款":106524,"åijĬè¯īæĪij":106525,"éĽĻ":106526,"书çĶ»":106527,"è¨Ńè¨Ī":106528,"æĢ»ä¼ļ":106529,"åĪ¤åĨ³":106530,"ä¿¡èªī":106531,"个èĤ¡":106532,"平常":106533,"æĢİ麼":106534,"ä½ĵçİ°åľ¨":106535,"é»Ħæ²³":106536,"åĽĽå·Ŀçľģ":106537,"羣缸":106538,"åIJĦé¡¹å·¥ä½ľ":106539,"åĬ¨åijĺ":106540,"å³°ä¼ļ":106541,"ä¸ĢæľŁ":106542,"æľīä¸Ģå®ļçļĦ":106543,"é«ĺ度éĩįè§Ĩ":106544,"ç¹ģèį£":106545,"åıijçİ°äºĨ":106546,"ç½ij红":106547,"æīĭæ³ķ":106548,"家åĽŃ":106549,"仪åĻ¨":106550,"è¾ĥä½İ":106551,"çļĦå®īåħ¨":106552,"æ¡IJ":106553,"ä»ĺ款":106554,"æĬijåĪ¶":106555,"åįĵè¶Ĭ":106556,"æŃ£éĿ¢":106557,"åĵij":106558,"强åĪ¶":106559,"ä»Ĭ天çļĦ":106560,"æĪĺèĥľ":106561,"楼å¸Ĥ":106562,"æĭ¿ä¸ĭ":106563,"é¢ľåĢ¼":106564,"举éĥ¨":106565,"çłĶåĪ¶":106566,"çļĦæĪĺçķ¥":106567,"åľ¨ä¸Ģ个":106568,"ä¸ī人":106569,"å®ĮäºĨ":106570,"æĸ°æĬĢæľ¯":106571,"ç»ıæµİæķĪçĽĬ":106572,"å¯Įæľī":106573,"澳洲":106574,"åĬ©çIJĨ":106575,"é¢Ĩåıĸ":106576,"è°Ń":106577,"çĩĥçĥ§":106578,"ç´łåħ»":106579,"éĤĦæľī":106580,"è¿ĽèĢĮ":106581,"ä»Ģä¹Īæĺ¯":106582,"çłĶ究ä¸Ńå¿ĥ":106583,"éĢĤçĶ¨äºİ":106584,"æİ¥æĶ¶":106585,"å¤±æľĽ":106586,"äºĮ级":106587,"éĹ´çļĦ":106588,"åİŁæłĩé¢ĺ":106589,"èªįçĤº":106590,"æį¡":106591,"对çĿĢ":106592,"对éĿ¢":106593,"ä¸ŃåİŁ":106594,"éĵĥ":106595,"çĶŁäº§çļĦ":106596,"åıijå¸ĥä¼ļ":106597,"士åħµ":106598,"è¿Ļåı¥è¯Ŀ":106599,"缴纳":106600,"ä¸Ģ个个":106601,"åѸçĶŁ":106602,"çĸijéĹ®":106603,"交èѦ":106604,"示èĮĥåĮº":106605,"天使":106606,"åľ¨ä¸Ĭæµ·":106607,"åIJĮæĻĤ":106608,"è½»æĺĵ":106609,"åĶ¯ä¸ĢçļĦ":106610,"çĥŃéĹ¹":106611,"ä¹IJè§Ĥ":106612,"çļĦ身份":106613,"åĸĦäºİ":106614,"大åİħ":106615,"èĤ¯å®ļæĺ¯":106616,"éĺ²çģ«":106617,"å¤ĸåĩº":106618,"æį®è¯´":106619,"é¡¹çĽ®çļĦ":106620,"ä¸Ģåı°":106621,"èĻļåģĩ":106622,"ä¸Ģç¬Ķ":106623,"ç«ĭæ³ķ":106624,"严èĤĥ":106625,"æī¿åĬŀ":106626,"åįģåĩł":106627,"çļĦ空éĹ´":106628,"æľ¬ç½ijç«Ļ":106629,"åģļå¾Ĺ":106630,"ä¿Ŀ温":106631,"æľĪåĪĿ":106632,"åľ¨ç½ijä¸Ĭ":106633,"åIJĦæĸ¹éĿ¢":106634,"ä¸ī天":106635,"交æĺĵæīĢ":106636,"解æŀIJ":106637,"åħļä¸Ń央":106638,"è¿Ľåĩºåı£":106639,"åĴĮ社ä¼ļ":106640,"次æķ°":106641,"ä¹ĭ家":106642,"维度":106643,"æ´¾åĩºæīĢ":106644,"产çĶŁäºĨ":106645,"带æľī":106646,"å¾Ī强":106647,"æľīäºĽäºº":106648,"å¹´åIJİ":106649,"äºĨ许å¤ļ":106650,"å¯Ĩ度":106651,"åŃ¦æľŁ":106652,"çıłæµ·":106653,"æľĢå¤ļçļĦ":106654,"è¾¹ç¼ĺ":106655,"容éĩı":106656,"第äºĮ个":106657,"ä¸Ģ缴æĺ¯":106658,"ä¸įç¦ģ":106659,"æŃ²":106660,"ä»ĭç»įäºĨ":106661,"ä¼ĺéĽħ":106662,"æ¯Ķè¼ĥ":106663,"èģĮä½į":106664,"温æŁĶ":106665,"æľīéĴ±":106666,"æľĢé«ĺçļĦ":106667,"åįļè§Īä¼ļ":106668,"ä¸įæĪIJ":106669,"éĶĻäºĨ":106670,"è¯ģçĽij":106671,"è¯ģçĽijä¼ļ":106672,"æĪIJ人":106673,"åĿĩåĮĢ":106674,"æľīåĪ©":106675,"è¶ĬåįĹ":106676,"æīĵäºĨ":106677,"好åIJĥ":106678,"系統":106679,"è·Łéļı":106680,"çļĦåľ°ä½į":106681,"æŃ£å¦Ĥ":106682,"ç¨įå¾®":106683,"åį°åıij":106684,"åĪĽç«ĭ":106685,"é£İåħī":106686,"å°ĨæĪIJ为":106687,"ä¸įé«ĺ":106688,"é¢ijç¹ģ":106689,"设æľī":106690,"ä¼ŀ":106691,"æĭĨéĻ¤":106692,"å½±åĥı":106693,"æ¸ĹéĢı":106694,"å¹´å¼Ģå§ĭ":106695,"ç½ijæĺĵ":106696,"è¦ģåģļ":106697,"çĶµåĬ¨è½¦":106698,"羣å¿ĥ":106699,"æµ·åĨĽ":106700,"ä¼łæĿ¥":106701,"å·®åĪ«":106702,"è°¨æħİ":106703,"çĥŁåı°":106704,"åįĥå¹´":106705,"è¯ģå®ŀ":106706,"çIJª":106707,"çļĦåħ·ä½ĵ":106708,"åĪ°å¤Ħ":106709,"ä¸įå®ľ":106710,"èľĢ":106711,"èĥ½åĬĽåĴĮ":106712,"çīºçī²":106713,"çļĦéĴ±":106714,"大éĺŁ":106715,"é¦ĸè¦ģ":106716,"ä¸įæĦ¿":106717,"çİ«çij°":106718,"人æ°ijç½ij":106719,"è¿ĺæĺ¯è¦ģ":106720,"åĽĽå¹´":106721,"æįŁä¼¤":106722,"çļĦåģļæ³ķ":106723,"éĿĪ":106724,"è¡Ķæİ¥":106725,"åIJĪæĪIJ":106726,"没人":106727,"éĹ¨æ§Ľ":106728,"ä¿¡è´·":106729,"çļĦ缸åħ³":106730,"举é£İ":106731,"社ä¿Ŀ":106732,"ä¸ĭ游":106733,"åĿĹéĴ±":106734,"è¿ĩåIJİ":106735,"çļĦåºĶçĶ¨":106736,"饶":106737,"é¢ģåıij":106738,"ä¸Ģå¤Ħ":106739,"åįİå¤ı":106740,"为ä¼ģä¸ļ":106741,"åıªä¼ļ":106742,"侵害":106743,"çļĦåĬŁèĥ½":106744,"åѸç¿Ĵ":106745,"ä¸Ńåįİæ°ijæĹı":106746,"åıijå¸ĥäºĨ":106747,"è¿İæİ¥":106748,"æĪijèĩªå·±":106749,"è¿ĺéľĢè¦ģ":106750,"太éĺ³èĥ½":106751,"åİ»ä¸ĸ":106752,"æĺ¯ä½ł":106753,"åIJĪåĬĽ":106754,"ç»ĺçĶ»":106755,"åı°åĮĹ":106756,"çĿ£ä¿ĥ":106757,"åĮĹéĥ¨":106758,"æľīå¤ļå°ij":106759,"å¾Īéĩįè¦ģ":106760,"åĪĴåĪĨ":106761,"åı·çº¿":106762,"æĶ¾å¤§":106763,"ä¼ļ被":106764,"èİ·å¥ĸ":106765,"ä¹ĭåĨħ":106766,"失åİ»äºĨ":106767,"çݩ家们":106768,"éĩĩéĽĨ":106769,"壹":106770,"家ä¼Ļ":106771,"çĻ½å¤©":106772,"åĽłä¸ºä»ĸ":106773,"社ä¼ļæ²»çIJĨ":106774,"å¼ĢåĪĽ":106775,"çĶµç¼Ĩ":106776,"æĸ°ä¸Ģ代":106777,"并è´Ń":106778,"就已ç»ı":106779,"çļĦ社ä¼ļ":106780,"éĻ¤éĿŀ":106781,"åı¯ä»¥çĶ¨":106782,"å©ī":106783,"æ¯Ķè¾ĥ好":106784,"å®ŀä¸ļ":106785,"åĪĽåĬŀ":106786,"æıIJèµ·":106787,"é»ĥ":106788,"ä½ıåľ¨":106789,"å¸ĤæĶ¿":106790,"éĿ¢ä¸´çļĦ":106791,"èĥ½åľ¨":106792,"çŁŃçŁŃ":106793,"çľŁäºº":106794,"æĺİæĺİ":106795,"èµĦåĬ©":106796,"çļĦä¸įåIJĮ":106797,"å°ıæľĭåıĭ":106798,"é¢ĺæĿIJ":106799,"ç¾İåij³":106800,"æĺŁåº§":106801,"ä¸įä¸Ģæł·çļĦ":106802,"çľĭä¸Ĭåİ»":106803,"ä¸Ģæł¹":106804,"广å·ŀå¸Ĥ":106805,"åıijçĶŁçļĦ":106806,"é«ĺç§ijæĬĢ":106807,"ä¸Ģè¾ĪåŃIJ":106808,"交åıī":106809,"ä½ĵ系建设":106810,"åĽłä¸ºæĪij":106811,"çıįæĥľ":106812,"ä¸ĬåѦ":106813,"æĪĺæľ¯":106814,"æѤ类":106815,"交å¾Ģ":106816,"æĮīæij©":106817,"人们çļĦ":106818,"åħ¶å¯¦":106819,"åİŁæĿIJæĸĻ":106820,"æ¸´æľĽ":106821,"缸å¤Ħ":106822,"微微":106823,"æ®·":106824,"ä¹ĺåĿIJ":106825,"å¼Ģå±ķäºĨ":106826,"é«ĺåĵģè´¨":106827,"æĹłäººæľº":106828,"ä¸įæĺ¯å¾Ī":106829,"çļĦæĬķèµĦ":106830,"èĬĤçľģ":106831,"èĩī":106832,"ç²¾éĢī":106833,"çļĦæłĩåĩĨ":106834,"åįĹéĥ¨":106835,"认è¯ĨåĪ°":106836,"å¹³éĿĻ":106837,"èĹ¥":106838,"æī«é»ij":106839,"æī«é»ijéĻ¤":106840,"æī«é»ijéĻ¤æģ¶":106841,"éĢĻ種":106842,"建çŃijéĿ¢ç§¯":106843,"ç¡®ç«ĭ":106844,"管çIJĨåĬŀæ³ķ":106845,"æĦıå¿Ĺ":106846,"丨":106847,"让åŃ©åŃIJ":106848,"æķijçģ¾":106849,"å½ĵä»Ĭ":106850,"çģ«çģ¾":106851,"åIJĦéĥ¨éŨ":106852,"ä¾µçĬ¯":106853,"æ¯ıåij¨":106854,"æı½":106855,"ä¸Ģ次æĢ§":106856,"åħ¶ä»ĸ人":106857,"éĶĻè¿ĩ":106858,"ä¸İåħ¶":106859,"åĭĩæ°Ķ":106860,"çĩĥæ°Ķ":106861,"é¦ĸå±Ĭ":106862,"æľį饰":106863,"ç²¥":106864,"å®Įæ¯ķ":106865,"å°±æĬĬ":106866,"åĬŀäºĭå¤Ħ":106867,"ä¸Ģä¼ļåĦ¿":106868,"离ä¸įå¼Ģ":106869,"å¦ĤæŀľæĤ¨":106870,"ä»ĵåºĵ":106871,"导å¸Ī":106872,"åIJĪéĢĤçļĦ":106873,"毫米":106874,"å®īåħ¨æĢ§":106875,"ä¾Ŀçħ§":106876,"产ä¸ļåĮĸ":106877,"ä½łçľĭ":106878,"羣çļĦå¾Ī":106879,"åѤçĭ¬":106880,"éĺ²å¾¡":106881,"å¾Īç®Ģåįķ":106882,"é£İæ°´":106883,"ä½Ĩä¹Ł":106884,"æݨåĩºäºĨ":106885,"æ°ijèIJ¥ä¼ģä¸ļ":106886,"çłģ头":106887,"å¤įæĿĤçļĦ":106888,"ç»ĦæĪIJéĥ¨åĪĨ":106889,"åħħ满äºĨ":106890,"è¿ijåĩłå¹´":106891,"çľģæĶ¿åºľ":106892,"æľīå¿ħè¦ģ":106893,"éĻ³":106894,"ä¹ĭç±»":106895,"ä¹ĭç±»çļĦ":106896,"æĢ§ä»·":106897,"æĢ§ä»·æ¯Ķ":106898,"åķĨåºĹ":106899,"å¸ĤåĢ¼":106900,"人æīįåŁ¹åħ»":106901,"æ·±åıĹ":106902,"管çIJĨå±Ģ":106903,"æģIJæĥ§":106904,"ä»ħæľī":106905,"æĬµè¾¾":106906,"æµ·åħ³":106907,"èµĭäºĪ":106908,"äºĭåĦ¿":106909,"ä»·éĴ±":106910,"æīĭä¸Ĭ":106911,"èĩªå¾ĭ":106912,"åħ³çĪ±":106913,"享æľī":106914,"éģĹæĨ¾":106915,"å¾Īå¿«å°±":106916,"æĽ´å¿«":106917,"æłĩè¯Ĩ":106918,"åºĨç¥Ŀ":106919,"ä¹Łå¥½":106920,"ä¸įæĺĵ":106921,"æĪijå¾Ī":106922,"æĶ¹éĿ©åıijå±ķ":106923,"å¤ĸåľ°":106924,"æĬµæĬ¼":106925,"è¯Ĺ人":106926,"åİķæīĢ":106927,"æĸ°åªĴä½ĵ":106928,"èĸĽ":106929,"è°Īè¯Ŀ":106930,"ä¸Ģå®ļç¨ĭ度":106931,"èµ°åľ¨":106932,"æľĢ强":106933,"åĬŁçİĩ":106934,"åħ±è¯Ĩ":106935,"大桥":106936,"ä¸ĭæĸ¹":106937,"å¤ĸèµĦ":106938,"碱":106939,"å·¡è§Ĩ":106940,"æ¹ĸåĮĹçľģ":106941,"个çĻ¾åĪĨ":106942,"个çĻ¾åĪĨçĤ¹":106943,"çļĦ责任":106944,"çļĦåĵģçīĮ":106945,"åĬ©æݨ":106946,"åĪĽéĢłäºĨ":106947,"ä»»èģĮ":106948,"å¿«æį·":106949,"æĿijåºĦ":106950,"åİ»çľĭ":106951,"æīįèĥ½å¤Ł":106952,"層":106953,"æĪij家":106954,"æĺ¯ä¸Ģ款":106955,"ç¾ħ":106956,"åĨ°éĽª":106957,"æŀģ大":106958,"çģ¯åħī":106959,"éĨĭ":106960,"ä¸İåħ¶ä»ĸ":106961,"æıIJåĩºçļĦ":106962,"éĿłè¿ij":106963,"è°ĥåĬ¨":106964,"å°½åı¯èĥ½":106965,"åıijåĬĽ":106966,"ç»Ļ她":106967,"éĢĤéĩı":106968,"è·¨åĽ½":106969,"åħĪè¡Į":106970,"æĸ°æĿIJæĸĻ":106971,"ä½ľäºĨ":106972,"满äºĨ":106973,"ä¸į满":106974,"çļĦçľ¼çĿĽ":106975,"çľĭå¾Ĺ":106976,"è¿Ļä¸Ģ次":106977,"é½IJåħ¨":106978,"çļĦä¸Ģéĥ¨åĪĨ":106979,"ä¸Ļ":106980,"æ¸ħæĸ°":106981,"說æĺİ":106982,"身边çļĦ":106983,"æīĢæľī人":106984,"å½°æĺ¾":106985,"è±¹":106986,"åį¿":106987,"è¿IJ转":106988,"æĮĩå¼ķ":106989,"å¸Ĥåħ¬å®īå±Ģ":106990,"åıĤå±ķ":106991,"ä¹ĭæŶ":106992,"éĩijèŀįæľįåĬ¡":106993,"èµĦæľ¬å¸Ĥåľº":106994,"èĥ½è®©":106995,"å¿ĺäºĨ":106996,"天åłĤ":106997,"æ¯Ķå¦Ĥ说":106998,"éĬĢè¡Į":106999,"èĽĭç³ķ":107000,"çĶ©":107001,"æł¸å®ŀ":107002,"æĻ®äº¬":107003,"ä¼ĺç¾İ":107004,"åı£èħĶ":107005,"漫çĶ»":107006,"çľ¼éĩĮ":107007,"äºĨä¸ĭæĿ¥":107008,"æĪijä»¬ä¹Ł":107009,"ä¾į":107010,"为ä¸Ńå¿ĥ":107011,"å¥ĩ迹":107012,"éĿĴçĿIJ":107013,"æĪªèĩ³çĽ®åīį":107014,"åĩºä¾Ĩ":107015,"æĢ»åħ¬åı¸":107016,"弥补":107017,"ç®Ĺæ³ķ":107018,"å·¥ä½ľå®¤":107019,"æīĢ以æĪij":107020,"æ°´åĪĨ":107021,"æīĢå±ŀ":107022,"ä¸į说":107023,"ä½Ĩæĺ¯åľ¨":107024,"è¦ģåİ»":107025,"åĪĽä¸ļèĢħ":107026,"ä¸įæ¸ħæ¥ļ":107027,"åĽĽåij¨":107028,"æĺ¯ä»İ":107029,"çļĦæł¹æľ¬":107030,"çģ¶":107031,"æ¯Ľæ³½":107032,"æ¯Ľæ³½ä¸ľ":107033,"æµ·åı£":107034,"åĽĽåįģ":107035,"ä¹Łè¢«":107036,"èģ·":107037,"ä¸Ģæīĭ":107038,"绩æķĪ":107039,"çļĦçĶ·äºº":107040,"书ç±į":107041,"ä¸ĢèĦ¸":107042,"大äºİ":107043,"鼶éĥ¨ä»¶":107044,"åħ³æĢĢ":107045,"平米":107046,"æļ´éľ²":107047,"å¾Ĺå¤ļ":107048,"ä¸ī级":107049,"æľ¬åij¨":107050,"两èĢħ":107051,"对ä¸ŃåĽ½":107052,"åıªè§ģ":107053,"欧ç¾İ":107054,"å¦Ĥæŀľæľī":107055,"å·²ç»ıæĺ¯":107056,"çľĭå®Į":107057,"çģ«éĶħ":107058,"èµIJ":107059,"ä¸Ģéģį":107060,"æĦŁåĨĴ":107061,"ç»ĵå±Ģ":107062,"ä»ĵåĤ¨":107063,"å®ŀåľ°":107064,"åī¯æĢ»ç»ıçIJĨ":107065,"ä¹Łä¸įçŁ¥éģĵ":107066,"碰åĪ°":107067,"åIJĪ计":107068,"客æĪ·çļĦ":107069,"ç½Ĺ马":107070,"æĦīå¿«":107071,"é£Ľ":107072,"çĥŃçĥĪ":107073,"伦æķ¦":107074,"åĮ»ä¿Ŀ":107075,"éĺ¿éĩĮå·´å·´":107076,"åĨį说":107077,"ä¸ºåŁºç¡Ģ":107078,"çĶŁäº§ç»ıèIJ¥":107079,"è¿ĻäºĽäºº":107080,"åĪĹ车":107081,"æ²³åĮĹçľģ":107082,"è¿Ļ段":107083,"æ´»åĬ¨ä¸Ń":107084,"å©·":107085,"çĶŁçIJĨ":107086,"ä¸ŃåĽ½äººæ°ij":107087,"éĦĤ":107088,"åIJ¬åıĸ":107089,"å¤įä¹ł":107090,"æľīçĽĬ":107091,"æĶ¶æĭ¾":107092,"å¾Īåı¯èĥ½":107093,"ç½ijç»ľæ¸¸æĪı":107094,"们çļĦ":107095,"èµĭèĥ½":107096,"éļ¾å¾Ĺ":107097,"åĪĨæīĭ":107098,"羣è¯ļ":107099,"åħ¬åı¸åľ¨":107100,"åĿĩè¡¡":107101,"åı£åij³":107102,"çīµå¤´":107103,"ä¸ĢèĪ¬çļĦ":107104,"轿车":107105,"çŃīäºİ":107106,"æ²īé»ĺ":107107,"æĪijéĥ½":107108,"å°ıç¨ĭåºı":107109,"ä¸Ģåī¯":107110,"æī¿è½½":107111,"åľ°è´¨":107112,"çķĮéĿ¢":107113,"çĶµæľº":107114,"çĦ¦èĻij":107115,"éĶĢåĶ®é¢Ŀ":107116,"æĸ°è½¦":107117,"ä¸Ĭ游":107118,"主æ¼Ķ":107119,"éļIJç§ģ":107120,"åıijå±ķæĪĺçķ¥":107121,"çļĦåĬªåĬĽ":107122,"å¼Ģåħ³":107123,"解åĨ³éĹ®é¢ĺ":107124,"çĿ£å¯¼":107125,"对æĬĹ":107126,"å¾Īå¤ļ人éĥ½":107127,"æĹłæķĪ":107128,"产åĵģè´¨éĩı":107129,"å®īå¿ĥ":107130,"åįİ人":107131,"ä¸į符åIJĪ":107132,"èĩªå®¶":107133,"éĺµå®¹":107134,"çļĦåIJĦç§į":107135,"çļĦçIJĨ念":107136,"çļĦæĸĩåĮĸ":107137,"为èĩªå·±":107138,"山水":107139,"游泳":107140,"éľĩèį¡":107141,"çĶŁæ´»æĸ¹å¼ı":107142,"è¿ľç¦»":107143,"çŁ³åĮĸ":107144,"æѤäºĭ":107145,"æĺ¯çľŁçļĦ":107146,"çļĦæ¯Ķä¾ĭ":107147,"çĶ¨çĶµ":107148,"奥è¿IJä¼ļ":107149,"ä¿Ŀå®ī":107150,"èĽĭçĻ½è´¨":107151,"çļĦå¿ĥçIJĨ":107152,"å·«":107153,"åı·çłģ":107154,"æ°Ķä½ĵ":107155,"åıijæĶ¹":107156,"åıijæĶ¹å§Ķ":107157,"åĮ»å¸Ī":107158,"æ¶ĤæĸĻ":107159,"æĺĬ":107160,"å¸Ĥ级":107161,"ä¸ĸçķĮçļĦ":107162,"åĪĨåĪ«æĺ¯":107163,"çł´äº§":107164,"ä¸ĢæĿ¯":107165,"æĭīå¼Ģ":107166,"å¹³åĩ¡":107167,"çļĦåıijçĶŁ":107168,"åĬ¨æīĭ":107169,"ä¸ĢçĽ´ä»¥æĿ¥":107170,"æīĭå·¥":107171,"éĩĮéĿ¢çļĦ":107172,"æĹłåħ³":107173,"ä»ĭåħ¥":107174,"èµ°ä¸Ĭ":107175,"å°±æĺ¯è¦ģ":107176,"å¹´éĹ´":107177,"åĩºçı¾":107178,"å½±éŁ¿":107179,"å¹ħ度":107180,"éĽģ":107181,"éģĵåħ·":107182,"缮çļĦåľ°":107183,"åIJİèĢħ":107184,"ä¸Ĭæ¼Ķ":107185,"äºĨåĩł":107186,"æ®ĭçĸ¾äºº":107187,"å¿Ļç¢Į":107188,"æĺ¯åIJ¦æľī":107189,"并对":107190,"ä¼ļ导èĩ´":107191,"æ°´åºĵ":107192,"ç»Ĩèĩ´":107193,"åIJİæĤĶ":107194,"å¿ĥæĢĿ":107195,"åģļäºĭ":107196,"åİĤæĪ¿":107197,"çĿ¿":107198,"è¿IJèIJ¥åķĨ":107199,"头éĥ¨":107200,"çļĦè§Ĵèī²":107201,"æĺ¯ä»ĸ":107202,"æĹ¢æľī":107203,"å°ıæŶåĢĻ":107204,"强åĬ²":107205,"主æĴŃ":107206,"åħ¨åĽ½åIJĦåľ°":107207,"æįı":107208,"æįŁåĿı":107209,"åķĨä¼ļ":107210,"ä¿Ŀç½Ĺ":107211,"çľģå¸Ĥ":107212,"éļ§éģĵ":107213,"æľīä¸įå°ij":107214,"è¦ģåľ¨":107215,"å»ºè®¾é¡¹çĽ®":107216,"ç³ĸå°¿":107217,"ç³ĸå°¿çĹħ":107218,"æĿ¡ä»¶ä¸ĭ":107219,"ä¼ĺè´¨çļĦ":107220,"é¦ĸåıij":107221,"å½ĵæŶçļĦ":107222,"丰çĶ°":107223,"大çĽĺ":107224,"缸继":107225,"å®ģå¤ı":107226,"åħ¥ä½ı":107227,"æĪijè¿ĺ":107228,"åħĭæĸ¯":107229,"å®ļä»·":107230,"å¹³æĸ¹åħ¬éĩĮ":107231,"çļĦçŁ¥è¯Ĩ":107232,"æĪij们ä¼ļ":107233,"åħĥå®Ŀ":107234,"ä½ĵéĩį":107235,"è³£":107236,"对æĪij们":107237,"çŁ³å®¶":107238,"çŁ³å®¶åºĦ":107239,"ç²¾åįİ":107240,"å½¢çĬ¶":107241,"åıĹåĪ°äºĨ":107242,"修订":107243,"ç¾İåľĭ":107244,"é«ĺæ¸ħ":107245,"çľ¼éķľ":107246,"è§īå¾Ĺèĩªå·±":107247,"带ç»Ļ":107248,"åĶ®ä»·":107249,"éĹ¨ç¥¨":107250,"åŃķå¦ĩ":107251,"çĶµè§Ĩåı°":107252,"åıijä½ľ":107253,"çļĦåij³éģĵ":107254,"éķ¿è¿ľ":107255,"åħ¬åħ±æľįåĬ¡":107256,"æŃ£å¸¸çļĦ":107257,"æľīè¿ĩ":107258,"é£İæĥħ":107259,"æ¯Ķéĩį":107260,"åIJ»":107261,"管çIJĨå·¥ä½ľ":107262,"综åIJĪæĢ§":107263,"已被":107264,"说起":107265,"æİĴæ°´":107266,"ä¸įæĸŃåľ°":107267,"æĥħæĢĢ":107268,"è¾ĵéĢģ":107269,"è¿ĩæķı":107270,"çļĦåı¯èĥ½æĢ§":107271,"æľįçĶ¨":107272,"æľī许å¤ļ":107273,"å§Ķåī¯ä¹¦è®°":107274,"åĮĸå¦Ĩåĵģ":107275,"æļĤåģľ":107276,"æĬķèµĦ人":107277,"çıŃ级":107278,"说çĿĢ":107279,"åįĹåĮĹ":107280,"åĪĨè¡Į":107281,"çıłå®Ŀ":107282,"寶":107283,"å¢ŀå¤ļ":107284,"被åĬ¨":107285,"çī¹æ®ĬçļĦ":107286,"éĹľä¿Ĥ":107287,"çļĦèĦ¸":107288,"æĥŁ":107289,"ä¸įä¸Ģå®ļ":107290,"ç¶Ń":107291,"çģ«çĪĨ":107292,"秣éĩij":107293,"çŀ§":107294,"éĩį建":107295,"è·ª":107296,"ä¸Ģ種":107297,"çļĦåIJĪä½ľ":107298,"å®īæħ°":107299,"ä»įæĺ¯":107300,"ä¸ĵä¸ļåĮĸ":107301,"è°ĥ解":107302,"ä¸į妨":107303,"éĢĻæĺ¯":107304,"å¿ħéłĪ":107305,"ä¼ĬæľĹ":107306,"å¾ĹäºĨ":107307,"æľįåĬ¡å¹³åı°":107308,"姬":107309,"åħĪéĶĭ":107310,"çİĭåŃIJ":107311,"çļĦä¸ĢåĪĩ":107312,"æĢ»çIJĨ":107313,"åĵ¼":107314,"çªij":107315,"çļĦå¿ĥæĥħ":107316,"çļĦéĩį大":107317,"çijŁ":107318,"ä¸Ģç¬ij":107319,"åıijå±ķä¸Ń":107320,"åģ¥åº·åıijå±ķ":107321,"åĵģçīĮçļĦ":107322,"禮":107323,"ä½Ļ人":107324,"ä»Ĭ年以æĿ¥":107325,"æķ°çłģ":107326,"çŃ¾è¯ģ":107327,"åİ»æī¾":107328,"åŁºéĩijä¼ļ":107329,"æĬ±æĢ¨":107330,"æŃ£å½ĵ":107331,"çıŃåŃIJæĪIJåijĺ":107332,"ä¸įåIJĪæł¼":107333,"åĪ¶å®ļäºĨ":107334,"ç¼ĵæħ¢":107335,"åĪ¶çº¦":107336,"æłı缮":107337,"å¸Ĥåľºç»ıæµİ":107338,"ç»ĦæĪIJçļĦ":107339,"严峻":107340,"æĹ¥è®¯":107341,"ä¸ĢçĤ¹çĤ¹":107342,"æĺ¯æĢİä¹Ī":107343,"çļĦçħ§çīĩ":107344,"éĺ»æŃ¢":107345,"模ç³Ĭ":107346,"缸":107347,"éģķåıį":107348,"æIJ¬è¿ģ":107349,"éĩijéĴ±":107350,"彬":107351,"ä¸įå®ī":107352,"æĪĺçķ¥åIJĪä½ľ":107353,"å¡«åĨĻ":107354,"讲究":107355,"åħħåĪĨåĪ©çĶ¨":107356,"èĥ½å¤ł":107357,"èij¡èIJĦéħĴ":107358,"éĩĩçĶ¨äºĨ":107359,"åľ¨ä»Ĭå¹´":107360,"ä¸Ńå°ıåѦ":107361,"åľ¨æĦı":107362,"çļĦåİĭåĬĽ":107363,"ä¸į幸":107364,"åĪ¶èį¯":107365,"åı¯ä»¥è®©":107366,"被è¯Ħ为":107367,"ç»ĨèıĮ":107368,"æĪıåī§":107369,"åįĬ导":107370,"åįĬ导ä½ĵ":107371,"è§Ĩè§Ĵ":107372,"åĸľæŃ¡":107373,"å¾ģæĶ¶":107374,"è°ĭåĪĴ":107375,"æŀģ大çļĦ":107376,"çĤ¹èµŀ":107377,"è®°èĢħä»İ":107378,"两åIJį":107379,"èĩªåĬ©":107380,"èµ·æŃ¥":107381,"æĬ¤å£«":107382,"å®Ŀ马":107383,"太åŃIJ":107384,"å°ıå°ıçļĦ":107385,"温æ³ī":107386,"åĩºç§Łè½¦":107387,"秣æĪ¿":107388,"两家":107389,"éľĩæĴ¼":107390,"ç§īæī¿":107391,"ä¸Ģ件äºĭ":107392,"çĥĪ士":107393,"å®ĺåħµ":107394,"转身":107395,"ä¹IJåĽŃ":107396,"çĻĮçĹĩ":107397,"模èĮĥ":107398,"æĦ£":107399,"è¿ĩåİ»çļĦ":107400,"代价":107401,"çļĦæ¦Ĥ念":107402,"åĩłçĻ¾":107403,"è´µéĺ³":107404,"æĭħ忧":107405,"éĢĤå®ľ":107406,"çݯå¢ĥä¿ĿæĬ¤":107407,"çĥ«":107408,"ä½łæĥ³":107409,"æѤåIJİ":107410,"ä½łä¹Ł":107411,"çįİ":107412,"éĻ¤æѤ":107413,"éĻ¤æѤä¹ĭå¤ĸ":107414,"è°ĥ度":107415,"ç§ij缮":107416,"æīĢ说çļĦ":107417,"åĬĩ":107418,"忽è§Ĩ":107419,"ä¸ī次":107420,"ä¸ĢæĹ¥":107421,"åŀĤ缴":107422,"ç«ŀæĬĢ":107423,"éĿ¢åĮħ":107424,"大æĪĺ":107425,"æIJºå¸¦":107426,"å¦Ĥæŀľæ²¡æľī":107427,"åħ»æĪIJ":107428,"åĩºè¡Ģ":107429,"çĪ±å¥½èĢħ":107430,"æīĵéĢļ":107431,"èµ·è¯ī":107432,"åijĪçİ°åĩº":107433,"æŃĮæīĭ":107434,"åľ¨å¤ĸ":107435,"é¢Ĩ导干éĥ¨":107436,"åĨ¥":107437,"èĪĨ论":107438,"æıIJåıĸ":107439,"éĺ¿å°Ķ":107440,"æľĽçĿĢ":107441,"ä¸īäºļ":107442,"財":107443,"åĪ·æĸ°":107444,"æĻļæĬ¥":107445,"è¿ĺæľīä¸Ģ个":107446,"åĨ°ç®±":107447,"ç½ijçĤ¹":107448,"åĩºåħ·":107449,"强çĥĪçļĦ":107450,"æĪijçĽ¸ä¿¡":107451,"å¸ĮæľĽèĥ½":107452,"çīĻ齿":107453,"äºĭå®ľ":107454,"ä¸ļåĨħ人士":107455,"ä»£æĽ¿":107456,"åıĺå½¢":107457,"éĽ²":107458,"è°ĥæݧ":107459,"åĪĽæĸ°åĪĽä¸ļ":107460,"æĭĨè¿ģ":107461,"æł¸æŁ¥":107462,"éĢĹ":107463,"åħ¥åѦ":107464,"æĦıåIJij":107465,"æıĽ":107466,"ä¸ĭ次":107467,"ä¼łè¾ĵ":107468,"ä»ĸä»¬åľ¨":107469,"èĢĮä¸Ķè¿ĺ":107470,"æĹ¥åľ¨":107471,"æķĻè®Ń":107472,"æ´»çĿĢ":107473,"çļĦæľīæķĪ":107474,"å¤įå·¥å¤į":107475,"å¤įå·¥å¤į产":107476,"æĺ¯ä¸Ģ件":107477,"çŃīçĿĢ":107478,"復":107479,"åĭĩæķ¢":107480,"éģŃåıĹ":107481,"å¥Ķé©°":107482,"讲座":107483,"说å®Į":107484,"ç»Ļåĩº":107485,"è°¦":107486,"è¯ĬçĸĹ":107487,"çĽ²çĽ®":107488,"客è¿IJ":107489,"å°±è¿ŀ":107490,"å¼Ģåħĥ":107491,"å¼Ģåħĥæ£ĭçīĮ":107492,"ä¸įæĸŃæıIJåįĩ":107493,"çĶ¨æĪ·çļĦ":107494,"æĴķ":107495,"ä¾Ľæ°´":107496,"ç¶ĵæ¿Ł":107497,"ä¸ŃåĮ»èį¯":107498,"èģĶæĥ³":107499,"åħ¬äº¤è½¦":107500,"èĪªçıŃ":107501,"æĬĢè¡ĵ":107502,"å¼ķèµ·çļĦ":107503,"å°¹":107504,"èµĦæ·±":107505,"åĽ½èµĦå§Ķ":107506,"èĺŃ":107507,"é¼»åŃIJ":107508,"éĹ½":107509,"æİĴéĺŁ":107510,"è§Ĥåħī":107511,"éģĹåĿĢ":107512,"ä¸ľäº¬":107513,"é¥ŃåºĹ":107514,"ä¸įæĸŃçļĦ":107515,"å°±æĺ¯ä¸Ģ个":107516,"éķ¿ä¹ħ":107517,"çļĦè§ĤçĤ¹":107518,"娶":107519,"æĪijçİ°åľ¨":107520,"çķ°":107521,"å¾Ĺåĩº":107522,"å¿ħå®ļ":107523,"ä¸įåıĹ":107524,"åıªéľĢè¦ģ":107525,"åĽ°æī°":107526,"ç§ijåѦæĬĢæľ¯":107527,"çīĽèĤī":107528,"è¾ĥé«ĺçļĦ":107529,"è·ijæŃ¥":107530,"æ²¾":107531,"èı©èIJ¨":107532,"æľĢå¾Į":107533,"ä¿Ŀå¯Ĩ":107534,"æ²»å®ī":107535,"éĤ±":107536,"常è¯Ĩ":107537,"èĦ¸èī²":107538,"åĮĹ大":107539,"æ±ĩèģļ":107540,"æijĨèĦ±":107541,"é¾Ļ头ä¼ģä¸ļ":107542,"女åıĭ":107543,"çŃīå·¥ä½ľ":107544,"ä¸Ńç¾İ":107545,"èģĮåľº":107546,"èĦijè¢ĭ":107547,"åĨĻçļĦ":107548,"饲æĸĻ":107549,"åĬ³åĬ¨åĬĽ":107550,"屯":107551,"æĮģèĤ¡":107552,"åĽ¾åĥı":107553,"è¿ĩåİ»äºĨ":107554,"貨":107555,"è¾²":107556,"éĹ®æĪij":107557,"è·Łä½ł":107558,"çĶŁæŃ»":107559,"审ç¾İ":107560,"é¢Ĺç²Ĵ":107561,"ä¸Ńæĸ¹":107562,"åĬłçĥŃ":107563,"æĹħè¡Į社":107564,"çĻ¼çĶŁ":107565,"ä¸įåłª":107566,"åĤ·":107567,"æ¥ł":107568,"åĬŀæ¡Ī":107569,"æŁĦ":107570,"æĹ¢æĺ¯":107571,"å¤ĦåĪĨ":107572,"羣å®ŀçļĦ":107573,"æĬ¥çº¸":107574,"å¸ĪçĪ¶":107575,"å®īå¾½çľģ":107576,"åī¯ä¸»å¸Ń":107577,"ä¹ĭéģĵ":107578,"导弹":107579,"åŃ¦æł¡çļĦ":107580,"åŁİå¸ĤçļĦ":107581,"è°ĪåĪ°":107582,"æ¢Ĺ":107583,"å¹³éĿ¢":107584,"说ä»Ģä¹Ī":107585,"é¢ijçİĩ":107586,"éķ¿ä¸īè§Ĵ":107587,"çļĦåĪ©çĽĬ":107588,"黨":107589,"è±ĨèħIJ":107590,"å®ŀéĻħæĥħåĨµ":107591,"æŀĹä¸ļ":107592,"纪æ£ĢçĽijå¯Ł":107593,"ä½ıéĻ¢":107594,"çļĦæķ´ä½ĵ":107595,"åīįè¡Į":107596,"æĮ¨":107597,"çħ¤çŁ¿":107598,"åī¯æĢ»è£ģ":107599,"å°ıåIJĥ":107600,"æŀģ端":107601,"å©Ĩå©Ĩ":107602,"çİ°è´§":107603,"è¯ĹæŃĮ":107604,"éĴ¥åĮĻ":107605,"缩çŁŃ":107606,"ä½Ĩè¿Ļ":107607,"æĸ°åĵģ":107608,"è¿Ļ对":107609,"çŁ¥åIJį度":107610,"å¿ĹæĦ¿æľįåĬ¡":107611,"大å±Ģ":107612,"è¡¡éĩı":107613,"ä½ĵçİ°äºĨ":107614,"æ¡ĥèĬ±":107615,"åIJ¸å¼ķåĬĽ":107616,"åł¤":107617,"æĵħéķ¿":107618,"åĴĴ":107619,"çĽ¸æľº":107620,"ä¸Ģç«Ļ":107621,"ä¸Ģç«Ļå¼ı":107622,"æľĢç¾İ":107623,"æ°¸ä¹ħ":107624,"çļĦéĥ¨åĪĨ":107625,"åĪĨå·¥":107626,"å·¥ç¨ĭ建设":107627,"æIJŃè½½":107628,"æ°´ä¸Ń":107629,"èĮ¨":107630,"çļĦæĵįä½ľ":107631,"ç»Łæ²»":107632,"çķħéĢļ":107633,"åħļçļĦåįģ":107634,"輸":107635,"測":107636,"ç¾İè§Ĥ":107637,"ä¸įåĪ©":107638,"åıįæĢĿ":107639,"éªĦåĤ²":107640,"æłĩçļĦ":107641,"æĿĢ人":107642,"éĺ¿å§¨":107643,"é£ŁæĿIJ":107644,"åIJĥçļĦ":107645,"åIJİåĨį":107646,"çŁ£":107647,"两侧":107648,"æ¸ħæ°´":107649,"è¿ĽçIJĥ":107650,"å¼Ģå§ĭäºĨ":107651,"åIJ¬äºĨ":107652,"çĦĬæİ¥":107653,"磮":107654,"å¨Ł":107655,"为人":107656,"éĢģç»Ļ":107657,"åĨĴéĻ©":107658,"æķ·":107659,"ç»ĪæŃ¢":107660,"æīįçŁ¥éģĵ":107661,"è¿IJæ°Ķ":107662,"éĢļé£İ":107663,"æĥĬ讶":107664,"ç§ijåѦéĻ¢":107665,"æıIJéĹ®":107666,"太åİŁ":107667,"缸åIJĮçļĦ":107668,"ä»ķ":107669,"èģĸ":107670,"æĥħæ³ģ":107671,"é¢Ĩ导人":107672,"åĩºæĿ¥äºĨ":107673,"沿线":107674,"éĻ½":107675,"æĦŁè¦º":107676,"ä»įåľ¨":107677,"æ©Ļ":107678,"约为":107679,"åĸĿéħĴ":107680,"çĶ¨èį¯":107681,"ä¸ĭä¸Ģ":107682,"æ³ķå®ĺ":107683,"顺åºı":107684,"åģļä¸Ģ个":107685,"åĭ¢":107686,"æŃª":107687,"çĶµç«ŀ":107688,"ä¼´éļıçĿĢ":107689,"ä¹ĭåĬĽ":107690,"ä¹ĭ人":107691,"äºij计ç®Ĺ":107692,"åĪ«äººçļĦ":107693,"ç§ijåѦåıijå±ķ":107694,"第åħ«":107695,"å¹²æī°":107696,"女ç¥ŀ":107697,"è¿Ļæł·åģļ":107698,"å¤Ħåľ¨":107699,"æ°´è´¨":107700,"éķ¿æĺ¥":107701,"å¸ĤåľºéľĢæ±Ĥ":107702,"ç»´æĿĥ":107703,"èĢ³æľµ":107704,"æĸĩåĮĸçļĦ":107705,"奶ç²ī":107706,"ä¼łè¾¾":107707,"æīĭæľºçīĪ":107708,"æĽ¾åľ¨":107709,"äºĮæľŁ":107710,"åİŁåĽłæĺ¯":107711,"æºIJ头":107712,"åıĪèĥ½":107713,"裸":107714,"æĬĢæľ¯åĪĽæĸ°":107715,"æĸĩåĮĸæĹħ游":107716,"åıij票":107717,"年级":107718,"ä½łä¸į":107719,"ä¹ĭå¿ĥ":107720,"æķ°çĻ¾":107721,"åIJijå¾Ģ":107722,"èĢģ家":107723,"åľĭéļĽ":107724,"çļĦé«ĺ度":107725,"æľĿéĺ³":107726,"æ¸ħéĻ¤":107727,"èĩªæľī":107728,"书ä¸Ń":107729,"游æĪıè£ħå¤ĩ":107730,"ä¸ĩå¤ļ":107731,"驾驶åijĺ":107732,"ä½łçŁ¥éģĵ":107733,"åĽ½åºĨ":107734,"é£ŁåłĤ":107735,"æİ¥åı£":107736,"æĢ»æķ°":107737,"åħ¶ä»ĸçļĦ":107738,"çĶŁåij½çļĦ":107739,"ä½łåľ¨":107740,"çļĦ缮åħī":107741,"è¿Ļæĸ¹éĿ¢":107742,"éĥ½è¯´":107743,"çĸĹæ³ķ":107744,"åĭĩ士":107745,"åľ¨åħ¨çIJĥ":107746,"ä¿ĿéĻ©åħ¬åı¸":107747,"çĿ£æŁ¥":107748,"åĸĦèī¯":107749,"表彰":107750,"è¹²":107751,"路段":107752,"æľĥåĵ¡è¦ı":107753,"æľĥåĵ¡è¦ıç¯Ħ":107754,"æĪ·åŀĭ":107755,"ä¿ĥ使":107756,"修建":107757,"é«ĺæ°´å¹³":107758,"åģļåĩºäºĨ":107759,"ä¸»åľº":107760,"è¡Įèµ°":107761,"空çĻ½":107762,"æľī人说":107763,"è¿Ļ个ä¸ĸçķĮ":107764,"åIJįä¹ī":107765,"å®Įç¾İçļĦ":107766,"羡æħķ":107767,"åıĬåħ¶ä»ĸ":107768,"åı¯çĶ¨":107769,"æĭIJ":107770,"è¾ĥ大çļĦ":107771,"æĬĢæľ¯åĴĮ":107772,"å°¼äºļ":107773,"çĻ¾è´§":107774,"æıī":107775,"éĢīè´Ń":107776,"éĺŁåıĭ":107777,"ä¼łæĦŁ":107778,"ä¼łæĦŁåĻ¨":107779,"åıªè¦ģä½ł":107780,"为ä»Ģä¹Īè¦ģ":107781,"ä¸ĵ注äºİ":107782,"ä½Ļé¢Ŀ":107783,"åħ¸åŀĭçļĦ":107784,"缮åīįå·²":107785,"æ¬²æľĽ":107786,"èģĶ绾":107787,"æµģä¼ł":107788,"çļĦ家åºŃ":107789,"åı·åı¬":107790,"çıįè´µ":107791,"ä¼Łå¤§çļĦ":107792,"éī´äºİ":107793,"è·Łä»ĸ":107794,"产çī©":107795,"ä¸įå·²":107796,"è¿Ŀæ³ķè¡Į为":107797,"头ä¸Ĭ":107798,"åĪĨ解":107799,"åı¯ä»¥çľĭåĩº":107800,"æł¡åĮº":107801,"åŃĹä½ĵ":107802,"ä¿®çĤ¼":107803,"çĶļèĩ³æĺ¯":107804,"微信åħ¬ä¼Ĺ":107805,"åıĸ代":107806,"èIJ¥ä¸ļæĶ¶åħ¥":107807,"æ½įåĿĬ":107808,"ä½łèĥ½":107809,"社ä¼ļä¿Ŀéļľ":107810,"æ¯ĶèµĽä¸Ń":107811,"污水å¤ĦçIJĨ":107812,"夫å¦ĩ":107813,"ä¸Ģå¹ħ":107814,"沿海":107815,"åı£æĦŁ":107816,"ä½Ĩåį´":107817,"å½ĵæĹ¥":107818,"çļĦæľĢ大":107819,"æ¯ıä¸Ģä½į":107820,"没äºĭ":107821,"çī¹åĪ¥":107822,"å¼ĢåѦ":107823,"è·¯éĿ¢":107824,"å¿ĥçIJĨåѦ":107825,"æĶ¾ç½®":107826,"éĩįåºĨå¸Ĥ":107827,"ä½łèĩªå·±":107828,"æ¶Īè´¹èĢħçļĦ":107829,"ä¸Ģæ³¢":107830,"èѦæĥķ":107831,"åį§å®¤":107832,"注å°Ħ":107833,"é£İ鼨":107834,"沿çĿĢ":107835,"åijĬ訴":107836,"表çİ°åĩº":107837,"åĽĽæĺ¯":107838,"åı¤åħ¸":107839,"æĽ´éĩįè¦ģçļĦ":107840,"好äºĭ":107841,"çľ¼æ³ª":107842,"æ¨ĵ":107843,"审åĪ¤":107844,"碰æĴŀ":107845,"车ç«Ļ":107846,"è¿Ľåħ¥äºĨ":107847,"éĽĨåIJĪ":107848,"æł¼å¤ĸ":107849,"宾é¦Ĩ":107850,"æĶ¯ä»ĺå®Ŀ":107851,"她æĺ¯":107852,"æĺ¯å¦Ĥä½ķ":107853,"人次":107854,"çļĦæĪIJåĬŁ":107855,"æĹłåĬĽ":107856,"æµ·æĭĶ":107857,"æĺ¥åŃ£":107858,"éĥ½ä¸įä¼ļ":107859,"çŃīå¤ļç§į":107860,"ä¸Ģ个å°ı":107861,"åģľè½¦åľº":107862,"è®©æĽ´å¤ļ":107863,"è¿ĻçĤ¹":107864,"æĪIJåĵģ":107865,"éĴī":107866,"éģĩè§ģ":107867,"çıŃ主任":107868,"æĦıæĦ¿":107869,"çļĦåIJĮåѦ":107870,"游è§Ī":107871,"åİĭ缩":107872,"åľ¨ä¼łå¥ĩ":107873,"å¼¹æĢ§":107874,"æĹ¥åĨħ":107875,"ç¦ı建çľģ":107876,"è§ĴèIJ½":107877,"åĪĨå¼Ģ":107878,"ä¼ļ让":107879,"å¤ĸåĽ´":107880,"çĨŁæĤīçļĦ":107881,"çĨĶ":107882,"ä¸ĩè¾Ĩ":107883,"å¤ľéĹ´":107884,"车身":107885,"ä¸ŃæľŁ":107886,"å®ĮåĸĦçļĦ":107887,"åĵģç±»":107888,"åıĭè°Ĭ":107889,"éĢīæĭĶ":107890,"éªij士":107891,"彦":107892,"çļĦçľĭæ³ķ":107893,"åĽ½çİĭ":107894,"è¾£æ¤Ĵ":107895,"åıijå¸ĥæŶéĹ´":107896,"åı¤åŁİ":107897,"éļıæľº":107898,"ç«ĸ":107899,"å¼Ģè¾Ł":107900,"ä¼ĹçĶŁ":107901,"没åĬŀæ³ķ":107902,"åįĥéĩĮ":107903,"æĿ¥æºIJäºİ":107904,"çļĦæĿĥåĪ©":107905,"æ¯ĶåĪĨ":107906,"满æĦıçļĦ":107907,"ä¿®è¡Į":107908,"åĿł":107909,"大海":107910,"èݹ":107911,"åĩºèº«":107912,"è«ĩ":107913,"åħ³èĬĤ":107914,"åIJį人":107915,"éľĢè¦ģ注æĦı":107916,"æĹ©æĻ¨":107917,"å¤ĸåįĸ":107918,"åıĪè¦ģ":107919,"æ¶īæ¡Ī":107920,"çĶ³è¯·äºº":107921,"éĻĦè¿ijçļĦ":107922,"åĬłå¿«æİ¨è¿Ľ":107923,"æĸ°å¹´":107924,"大è¡Ĺ":107925,"ä¸Ģé»ŀ":107926,"èĭıå®ģ":107927,"æĤĦæĤĦ":107928,"èĦ¾æ°Ķ":107929,"å¸ĮèħĬ":107930,"éļıåį³":107931,"æķ¢äºİ":107932,"å®ŀè·µä¸Ń":107933,"æĺ¯æ²¡æľī":107934,"æľī趣çļĦ":107935,"æĿ¥èĩªäºİ":107936,"è£ģåĪ¤":107937,"女åŃ©åŃIJ":107938,"èĩ³åħ³":107939,"èĩ³åħ³éĩįè¦ģ":107940,"æĻºåĬĽ":107941,"èµ°åĩºåİ»":107942,"çŁŃæĿ¿":107943,"å¤§åĽ½":107944,"çļĦ认è¯Ĩ":107945,"å¹´å¤ľ":107946,"åĨįåĪ°":107947,"åIJĮæł·çļĦ":107948,"å¯Ĩå°ģ":107949,"å¤ĸ交éĥ¨":107950,"çĶŁæķĪ":107951,"æĤ¨åı¯ä»¥":107952,"ä½łåĢij":107953,"è¿ĩå¹´":107954,"å¼ĵ":107955,"è¡ĮæĿİ":107956,"æ¯Ķèµ·":107957,"身é«ĺ":107958,"è¿Ļ个人":107959,"ä¸Ńå¤ĸ":107960,"éģĵæŃī":107961,"çĽ¯çĿĢ":107962,"亲åŃIJ":107963,"éŸ":107964,"çĻ½äºij":107965,"èĦĸåŃIJ":107966,"ä¸ĢåĪĩéĥ½":107967,"æ·ij":107968,"è°ľ":107969,"åģ¶çĦ¶":107970,"éĿłè°±":107971,"é«ĺ管":107972,"ä¸ĭåıij":107973,"æĶ¾åĪ°":107974,"ç±»åĪ«":107975,"ä¸ĭåĪĹ":107976,"æ··ä¹±":107977,"åIJĪæ³ķæĿĥçĽĬ":107978,"çݯçIJĥ":107979,"æľīæķĪåľ°":107980,"åķĨæĪ·":107981,"æ¹ĸ人":107982,"海岸":107983,"æĬķ产":107984,"两个æľĪ":107985,"éĥ½éĿŀ常":107986,"å¢ŀ强äºĨ":107987,"æĿ¥åĪ°äºĨ":107988,"åī©ä½Ļ":107989,"æĤ¨çļĦåŃ©åŃIJ":107990,"æµģæ°´":107991,"æŃ£ä¹ī":107992,"天çĮ«":107993,"åģļè¿ĩ":107994,"ä½ķæŶ":107995,"æĪijåİ»":107996,"çľģ份":107997,"å¥ĸéĩij":107998,"该å¦Ĥä½ķ":107999,"ä¸ĭçıŃ":108000,"åģ¶åĥı":108001,"æijĨæĶ¾":108002,"æĸ°æ¨¡å¼ı":108003,"æĬķè³ĩ":108004,"è·¯åı£":108005,"åĨľæ°ijå·¥":108006,"大åѸ":108007,"件äºĭ":108008,"æł¹æľ¬ä¸į":108009,"æµĵ度":108010,"æµĵåİļ":108011,"è½®èĥİ":108012,"æĪ¿ä¼ģ":108013,"éĿŀ常好":108014,"ä»İä¸Ń":108015,"äººæł¼":108016,"ç¿ģ":108017,"æŶéĹ´åĴĮ":108018,"è¿Ļä¸įæĺ¯":108019,"åĪ¸åķĨ":108020,"æĥĬ人":108021,"åĻ¨å®ĺ":108022,"åĩĨåĪĻ":108023,"æĥħæĻ¯":108024,"æĽ´é«ĺçļĦ":108025,"åŃ¦å®¶":108026,"泡沫":108027,"åľ°æĸ¹æĶ¿åºľ":108028,"å°±çŁ¥éģĵ":108029,"åij¼åIJģ":108030,"ç»ıè´¸":108031,"èĬ±éĴ±":108032,"æľīä¸Ģ次":108033,"æĦŁæħ¨":108034,"ä¸Ģåįĥ":108035,"å¤ľæĻļ":108036,"詹å§Ĩ":108037,"詹å§Ĩæĸ¯":108038,"è¦ģéĹ»":108039,"ç»Ĵ":108040,"æºIJäºİ":108041,"çļĦè´¨éĩı":108042,"注æĦıäºĭ项":108043,"æħ¢æĢ§":108044,"稳å®ļçļĦ":108045,"建设åĴĮ":108046,"æĻ¯è±¡":108047,"éĩıåĮĸ":108048,"çļĦ話":108049,"è¯Ħ级":108050,"æºľ":108051,"红åĮħ":108052,"éĢļéģİ":108053,"社ä¼ļ责任":108054,"æĸ°äº§åĵģ":108055,"åĨ·éĿĻ":108056,"çľĭä¸įåĪ°":108057,"èģĶéĤ¦":108058,"éŃĦ":108059,"çļĦåīįæıIJ":108060,"çļĦåīįæıIJä¸ĭ":108061,"è¾ĥ好":108062,"çļĦæĦŁæĥħ":108063,"客æĪ·æıIJä¾Ľ":108064,"çĭ¬èĩª":108065,"å¢ŀæĶ¶":108066,"æĸĩçĮ®":108067,"æĭ¼åij½":108068,"管çIJĨåĴĮ":108069,"æµģåĬ¨æĢ§":108070,"åħ¨å®¶":108071,"ä¸Ĭæĸ¹":108072,"æݨåĩºçļĦ":108073,"ä¸īåĽ½":108074,"ä¸Ģ个æĺ¯":108075,"æĸ°ä¸Ģè½®":108076,"æĸĩåĮĸéģĹ产":108077,"殺":108078,"大湾åĮº":108079,"éĥ½éľĢè¦ģ":108080,"çļĦå®ŀéĻħ":108081,"ç·Ĭ":108082,"大å¥ĸ":108083,"åħīèĬĴ":108084,"便äºİ":108085,"çļĦ表æĥħ":108086,"æ¼Ķç»İ":108087,"红åĨĽ":108088,"å½ĵæĪij":108089,"æ²»æĦĪ":108090,"é¢Ŀ度":108091,"éĿľ":108092,"ä»»ä½ķ人":108093,"è¡Ĺ头":108094,"çī¹æĸ¯":108095,"çī¹æĸ¯æĭī":108096,"åĮ»çĸĹæľºæŀĦ":108097,"ç»ĻåŃ©åŃIJ":108098,"è§Ħ磩":108099,"è£ľ":108100,"çļĦ身影":108101,"ä¸ĵæłı":108102,"æĿ¥ä¸´":108103,"童年":108104,"å¤įèĭı":108105,"è¨Ĥ":108106,"åŀĭåı·":108107,"åĽ¾æ¡Ī":108108,"ç®ĢåİĨ":108109,"æĭ±":108110,"èį·åħ°":108111,"ä»»æĦı":108112,"æī¿æİ¥":108113,"è¿Ļæīį":108114,"客车":108115,"æľĿçĿĢ":108116,"éłħ缮":108117,"åı°é£İ":108118,"çļĦæĪ¿åŃIJ":108119,"éªı":108120,"æĿ±è¥¿":108121,"éģĹä¼ł":108122,"è¶Ĭå¤ļ":108123,"äºĨä»ĸçļĦ":108124,"ä¸Ĭåij¨":108125,"管çIJĨåĪ¶åº¦":108126,"失ä¸ļ":108127,"çĶ·åıĭ":108128,"æİ¥ç§į":108129,"å¨ģåIJį":108130,"çĴ°å¢ĥ":108131,"åıijçĶŁåľ¨":108132,"ä¸ªåĽ½å®¶":108133,"åĪĽæĸ°åıijå±ķ":108134,"æĶ¹åıĺäºĨ":108135,"åģ¥åº·çļĦ":108136,"åĢ¼å¾Ĺä¸Ģ":108137,"åĢ¼å¾Ĺä¸ĢæıIJ":108138,"åĽ¢ä¼Ļ":108139,"åģĩ设":108140,"åı°ä¸Ĭ":108141,"è§ĦèĮĥåĮĸ":108142,"éĻªåIJĮ":108143,"座æ¤ħ":108144,"åı¯æĢľ":108145,"åħĭæĢĿ主ä¹ī":108146,"æ³ķå¾ĭ责任":108147,"ä¸Ģé¡¿":108148,"æĬ¬å¤´":108149,"为éĩįçĤ¹":108150,"è¿ľæ´ĭ":108151,"éĢıè¿ĩ":108152,"åħ¨çIJĥåĮĸ":108153,"趣åij³":108154,"票æĪ¿":108155,"æ¯ı人":108156,"åIJĦç§įåIJĦæł·":108157,"äºĨåĩºæĿ¥":108158,"ç»Ŀ对æĺ¯":108159,"ä¸ĭå±ŀ":108160,"ä¸ĢåıĮ":108161,"è¿ĻåĿĹ":108162,"æĬĹçĸ«":108163,"è¦ģçĤ¹":108164,"å½¢æĪIJçļĦ":108165,"æĪijçľĭ":108166,"ä¸ĩéĩĮ":108167,"èĢĥçłĶ":108168,"为åħ¶":108169,"æ°ij宿":108170,"å¤ļä½į":108171,"大èĩ´":108172,"ä»ĺè´¹":108173,"åħ¥æīĭ":108174,"å±ħ家":108175,"æīĢåľ¨åľ°":108176,"人身":108177,"è¿ĩå¾Ĺ":108178,"è¯ķè¯ķ":108179,"访è°Ī":108180,"åĬłéĩį":108181,"å°±ä¸įä¼ļ":108182,"çĶŁäº§ä¼ģä¸ļ":108183,"åĽŀåĽ½":108184,"åºķ线":108185,"赶åĪ°":108186,"æĶ¯éĺŁ":108187,"æĪij们éĥ½":108188,"éĤ®æĶ¿":108189,"缴èĩ³":108190,"éĴ¢çIJ´":108191,"åħľ":108192,"çłĶ讨ä¼ļ":108193,"æľĪ亮":108194,"åĿļæĮģ以":108195,"åħ¬å®īéĥ¨":108196,"éĴ¢ç®¡":108197,"å°ıçĻ½":108198,"ç½®ä¸ļ":108199,"èģĭ":108200,"书åĨĻ":108201,"æĿı":108202,"éħįæĸ¹":108203,"èĢĮåıĪ":108204,"çijŀ士":108205,"çķĮçļĦ":108206,"èĢģ大":108207,"æĪIJçĨŁçļĦ":108208,"å¹²ä»Ģä¹Ī":108209,"ä¸ĵ项æĸĹäºī":108210,"çŃīå¤ļ个":108211,"èĦ±ç¦»":108212,"ä¸ī个æľĪ":108213,"çłĶ究åijĺ":108214,"æĹĭ转":108215,"æŀģèĩ´":108216,"åħįè´£":108217,"åħį责声æĺİ":108218,"å¾Īå¤ļçݩ家":108219,"车ä¸Ĭ":108220,"交äºĴ":108221,"å·²æĺ¯":108222,"ä¸Ģå°ı":108223,"çļĦéĩįçĤ¹":108224,"èĬ±äºĨ":108225,"ä¸įæĺİ":108226,"æľīåħ³è§Ħå®ļ":108227,"çĬ¹å¦Ĥ":108228,"羸":108229,"寡":108230,"çļĦè¡£æľį":108231,"åĮħ裹":108232,"身åŃIJ":108233,"å¸ĪèĮĥ大åѦ":108234,"äºĭåħĪ":108235,"线æĿ¡":108236,"æ³ķåĪ¶":108237,"åħ»æĬ¤":108238,"稳å®ļæĢ§":108239,"éĤµ":108240,"åŀĦæĸŃ":108241,"é¡į":108242,"èĢĥåı¤":108243,"æĿłæĿĨ":108244,"èĭıèģĶ":108245,"æ°´çĶµ":108246,"åħ·ä½ĵçļĦ":108247,"æ¿Ģæ´»":108248,"æĪijæł¡":108249,"åĪļå¼Ģå§ĭ":108250,"åĩ¸æĺ¾":108251,"禾":108252,"åħ¼èģĮ":108253,"éĢıéģİ":108254,"åľ¨æ¸¸æĪıä¸Ń":108255,"社ä¼ļåıijå±ķ":108256,"好çİ©":108257,"å¹»æĥ³":108258,"ä¸į代表":108259,"注æĦıåĬĽ":108260,"æ£į":108261,"çĶ¨æīĭ":108262,"ç¾İ人":108263,"许å¤ļ人":108264,"å¾Īæĺ¯":108265,"çļĦçłĶåıij":108266,"æīĵåĩº":108267,"åIJĪä¼Ļ人":108268,"ä¸Ģå¤ľ":108269,"ç¼ĵç¼ĵ":108270,"ä¿®æŃ£":108271,"æĦŁçŁ¥":108272,"ç»Ī身":108273,"æ¿Ģç´ł":108274,"çݯå¢ĥä¸ĭ":108275,"次ä¼ļè®®":108276,"ç»ıæµİå¢ŀéķ¿":108277,"æīĽ":108278,"åıijéħµ":108279,"åĪĨæŀIJå¸Ī":108280,"åľ¨æľªæĿ¥":108281,"主è¦ģæľī":108282,"ä¸ĢåŃ£åº¦":108283,"çļĦ说æ³ķ":108284,"ä»İæĿ¥æ²¡æľī":108285,"货车":108286,"缩å°ı":108287,"太è¿ĩ":108288,"æķĪåĬĽ":108289,"ä¸įä¸ĭ":108290,"æĬķ稿":108291,"èį¯ä¸ļ":108292,"ç»Ħéķ¿":108293,"ç«ĻçĤ¹":108294,"å¾Īåĸľæ¬¢":108295,"éIJµ":108296,"åĬ¿å¤´":108297,"æ¼ıæ´ŀ":108298,"æĦ¤æĢĴ":108299,"åħħå®ŀ":108300,"åĪĽä¸ļæĿ¿":108301,"çĪª":108302,"æľªå¿ħ":108303,"åºķéĥ¨":108304,"å¾ĹåĪĨ":108305,"人æ°ijåĮ»éĻ¢":108306,"äºĮæīĭæĪ¿":108307,"å·²ç»ı被":108308,"大楼":108309,"æĸ°æĪ¿":108310,"辦æ³ķ":108311,"çĶ¨åĬĽ":108312,"æĭĵ宽":108313,"åĨħåľ¨":108314,"æĴŃåĩº":108315,"饰æ¼Ķ":108316,"ä¹Łè®©":108317,"ä½ľçĤº":108318,"çī©ä¸ļ管çIJĨ":108319,"åį´ä¸į":108320,"为ä¸ŃåĽ½":108321,"å±ĢåĬ¿":108322,"ä¸įèĤ¯":108323,"æľĢæĸ°çļĦ":108324,"åı¯ä»¥éĢīæĭ©":108325,"æĺ¾çİ°":108326,"å°±ç®Ĺæĺ¯":108327,"åľ¨æł¡":108328,"é¾Ł":108329,"两æĿ¡":108330,"çļĦå®ŀåĬĽ":108331,"è¶Ĭ好":108332,"å¥¹åľ¨":108333,"å¿łè¯ļ":108334,"ä¹ŁéľĢè¦ģ":108335,"游æĪıæĵįä½ľ":108336,"è¶ħåĩº":108337,"å¦Ĥæŀľä¸į":108338,"æīĢåľ¨çļĦ":108339,"ä½łè¿ĺ":108340,"以åĨħ":108341,"æľīä¸Ģå®ļ":108342,"åı¯è¾¾":108343,"è·ijåĪ°":108344,"åīĽ":108345,"建ç«ĭåģ¥åħ¨":108346,"æķ´è½¦":108347,"åīįæĸ¹":108348,"éĹ´æİ¥":108349,"çŃ¹å¤ĩ":108350,"çĸ²åĬ³":108351,"离å¼ĢäºĨ":108352,"æ±Ŀ":108353,"éĿ¢éĥ¨":108354,"ä¹ĭåīįçļĦ":108355,"åıĺ为":108356,"å¦Ĥæŀľè¯´":108357,"对ä»ĺ":108358,"åĿĩåı¯":108359,"被åijĬ人":108360,"ç²¾ç¾İ":108361,"èģļä¼ļ":108362,"çĿĢæĢ¥":108363,"è°·æŃĮ":108364,"ä¸Ģåı·":108365,"红åĪ©":108366,"ä¼łå¥ĩ游æĪı":108367,"å»ĸ":108368,"è´ŀ":108369,"ä¹°åĪ°":108370,"éŃļ":108371,"ä½ĵè´¨":108372,"å°ijäºĨ":108373,"æ³īå·ŀ":108374,"åIJŁ":108375,"ç»Ŀä¸į":108376,"é»ijæģ¶":108377,"é»ijæģ¶åĬ¿åĬĽ":108378,"ä¸Ĭæĺł":108379,"çļĦè¯Ŀé¢ĺ":108380,"ä¸ĩ人次":108381,"ä¸ĸéĹ´":108382,"çĶ¨å·¥":108383,"贯穿":108384,"å®ĿçŁ³":108385,"ä½łå¥½":108386,"åĪĩåī²":108387,"å¼ºåĽ½":108388,"åĽŀèIJ½":108389,"æ°´æĻ¶":108390,"模仿":108391,"洪水":108392,"éĢĻ麼":108393,"åįģä¸īäºĶ":108394,"ä½ij":108395,"éĻĦ件":108396,"çļĦå¢ŀéķ¿":108397,"éĻĦå±ŀ":108398,"çİ°å·²":108399,"å¸®ä½ł":108400,"éĩijçīĮ":108401,"é«ĺåİŁ":108402,"åľ¨å®¶éĩĮ":108403,"éĺ²èħIJ":108404,"ç¡®å®ŀæĺ¯":108405,"宣讲":108406,"天æīį":108407,"ç»ıèIJ¥ç®¡çIJĨ":108408,"éĶħçĤī":108409,"åIJĪä¸Ģ":108410,"è§Ĥèµı":108411,"éķ¿è¾¾":108412,"主ä¹īæĢĿæĥ³":108413,"éĤ£éº¼":108414,"é£İäºij":108415,"为主çļĦ":108416,"æļijåģĩ":108417,"æĮģä¹ħ":108418,"å¼Ĥåľ°":108419,"å¼ĢéŨ":108420,"模æĿ¿":108421,"æī¹æ¬¡":108422,"ä¸į便":108423,"天çĶŁ":108424,"åĩłä¸ªæľĪ":108425,"ä¸ĵç§ij":108426,"åı¦æľī":108427,"åħ¬å¸ĥçļĦ":108428,"æĩ·":108429,"åľºåIJĪ":108430,"çļĦå¿ĥæĢģ":108431,"è¿ĺ好":108432,"å®ŀæĪĺ":108433,"èĢģå¸ĪçļĦ":108434,"åħ©åĢĭ":108435,"åı¯åľ¨":108436,"éĤ£ä½į":108437,"å¥łå®ļäºĨ":108438,"ä¿ĥéĶĢ":108439,"æı´åĬ©":108440,"ä¸ĩçī©":108441,"æĥħæĬ¥":108442,"é¦ĸåħĪè¦ģ":108443,"æĸĩåĮĸåĴĮ":108444,"éĥ½å·²ç»ı":108445,"ä¸Ĭä¸ĸ纪":108446,"åĨľåľº":108447,"大æī¹":108448,"æĺİçĻ½äºĨ":108449,"çļĦæĪIJéķ¿":108450,"çļĦæ¯ĶèµĽ":108451,"失误":108452,"åģļæĪIJ":108453,"ä»Ĭ天å°ıç¼ĸ":108454,"é¢Ĩè¢ĸ":108455,"æıIJåįĩäºĨ":108456,"å¾IJå·ŀ":108457,"ä»įæľī":108458,"è¿ĩ滤":108459,"å¹½é»ĺ":108460,"çĥŃéĩı":108461,"ä¸Ģé¦ĸ":108462,"æ¼Ĥ亮çļĦ":108463,"åĩłç§į":108464,"åĢ¡è®®":108465,"å°±åı¯ä»¥äºĨ":108466,"æİĴåĪĹ":108467,"éĩįéĩį":108468,"ä¼ģä¸ļåĴĮ":108469,"ä¸ĵå±ŀ":108470,"çħİ":108471,"亲æĪļ":108472,"çĻ¾åĪĨä¹ĭ":108473,"稿件":108474,"è¿ĺå¾Ĺ":108475,"人åĵ¡":108476,"äºī夺":108477,"æĽ´å®¹æĺĵ":108478,"大èĩªçĦ¶":108479,"鼻èħ¦":108480,"太空":108481,"åľ°å¤Ħ":108482,"夢":108483,"ä»ĸ对":108484,"å¿ħå°Ĩ":108485,"ä¸įå½ĵ":108486,"严谨":108487,"åĩºåľº":108488,"å·²ç»ıæľī":108489,"é¢ĨåĨĽ":108490,"é«ĺæ¡£":108491,"ä¸ĢæīĢ":108492,"æłĹ":108493,"让åѦçĶŁ":108494,"æĽ¹æĵį":108495,"æŁIJä¸Ģ":108496,"伸åĩº":108497,"èĬ±åįī":108498,"æ¸ħéĨĴ":108499,"èģĶç³»æĸ¹å¼ı":108500,"åĪĨå±Ģ":108501,"èħ³":108502,"æ©¡èĥ¶":108503,"éķ¿å¾Ĺ":108504,"ç»¿åľ°":108505,"è¢į":108506,"çļĦèīºæľ¯":108507,"女æľĭåıĭ":108508,"ä¸Ńè¶ħ":108509,"离åŃIJ":108510,"å¤ļæł·åĮĸ":108511,"éĺ³åı°":108512,"ä½İ碳":108513,"ä¸Ģç±»":108514,"çŃīæĸ¹éĿ¢çļĦ":108515,"å¾Ĺ好":108516,"模åħ·":108517,"ä¸ĩ亿":108518,"çķĻæĦı":108519,"临æ²Ĥ":108520,"å°ijéĩı":108521,"çľĭåIJij":108522,"ç»ıèIJ¥èĢħ":108523,"çķĻä¸ĭäºĨ":108524,"åĿıäºĨ":108525,"åijĬåĪ«":108526,"羣çIJĨ":108527,"ç¼´è´¹":108528,"æĬĬä½ł":108529,"çļĦä»»åĬ¡":108530,"æĪij对":108531,"ä¹°åħ¥":108532,"çĻ»ä¸Ĭ":108533,"æľī两个":108534,"ä¸Ģ头":108535,"æĵįæݧ":108536,"åħ¨è¦ĨçĽĸ":108537,"çĿĢæīĭ":108538,"å¢ĻéĿ¢":108539,"å¤ļæĸ¹":108540,"åı¯çĪ±çļĦ":108541,"ä¹Łåı¯èĥ½":108542,"æľĢæľī":108543,"è¿ĻäºĽéĥ½æĺ¯":108544,"æĥ¡":108545,"å®®":108546,"å¾Īå°ı":108547,"éĹ®é¢ĺæĺ¯":108548,"åĿĩæľī":108549,"å¾ģéĽĨ":108550,"说åĩº":108551,"æľīæĦı":108552,"é¢Ĥ":108553,"æī¬å·ŀ":108554,"åķĨä¸ļ模å¼ı":108555,"çĶŁèĤĸ":108556,"æįIJ款":108557,"å²Ĥ":108558,"ç¾İæĻ¯":108559,"è¿ĺ羣":108560,"æĭ¥æĬ±":108561,"身ä½ĵåģ¥åº·":108562,"æ·±å¤Ħ":108563,"çľ¼ç¥ŀ":108564,"çļĦ形象":108565,"ä¼ĺè¶Ĭ":108566,"å½ĵæĪIJ":108567,"åĮºåĪĨ":108568,"åİ»éĻ¤":108569,"注å®ļ":108570,"å§IJ妹":108571,"åĮºåĨħ":108572,"é©ļ":108573,"æļĹ示":108574,"æĺİ亮":108575,"æħ°éĹ®":108576,"å¸Ĥåľºä»½é¢Ŀ":108577,"çĮªèĤī":108578,"çļĦèµĦéĩij":108579,"åİĨç»ı":108580,"å§ĭç»ĪåĿļæĮģ":108581,"çĶŁæľº":108582,"ä¸į顾":108583,"éĩijåĪļ":108584,"大声":108585,"éĻķ西çľģ":108586,"é²į":108587,"åĨľä¸ļåĨľæĿij":108588,"æľī害":108589,"éŨè¯Ĭ":108590,"æ¯ıä¸Ģ次":108591,"çļĦåĽłç´ł":108592,"é¢Ŀå¤ĸ":108593,"åݿ级":108594,"çļĩåIJİ":108595,"åĽ½ä¼ģ":108596,"é¦ĸéĢī":108597,"ç¼ĸåĨĻ":108598,"æĭ¿èµ·":108599,"åģ·åģ·":108600,"ä¸İä¸ŃåĽ½":108601,"åįĸ家":108602,"ç»Ļä»ĸ们":108603,"ç¥ŀè¯Ŀ":108604,"åŃ¸æł¡":108605,"æĪijä¸Ģ缴":108606,"çŁ¥éģĵäºĨ":108607,"åįĴ":108608,"åĴĮåľ°åĮº":108609,"ä»Ģä¹Īéĥ½":108610,"çĶ»å®¶":108611,"æľ¬çĿĢ":108612,"ä½ĻåIJį":108613,"审çIJĨ":108614,"ä¸ĢåIJij":108615,"åıijå±ķè¶ĭåĬ¿":108616,"åĮºéĹ´":108617,"注åĨĮèµĦæľ¬":108618,"çIJ¦":108619,"ä¸įåı¯ä»¥":108620,"çļĦåĦ¿åŃIJ":108621,"åĢ¼çıŃ":108622,"ä¸¥æł¼çļĦ":108623,"å®ŀä½ĵç»ıæµİ":108624,"æľīæĿĥ":108625,"æĪijåıĪ":108626,"éĵ¶æ²³":108627,"ç«ĭ马":108628,"æĿĢäºĨ":108629,"åĮħ容":108630,"管家":108631,"身é«Ķ":108632,"éĵħ":108633,"å°ıåŃIJ":108634,"管çIJĨç³»ç»Ł":108635,"æľīçļĦ人":108636,"é£İçĶµ":108637,"æĻºèĥ½åĪ¶éĢł":108638,"精确":108639,"æĭĽåķĨå¼ķ":108640,"æĭĽåķĨå¼ķèµĦ":108641,"äºĮæīĭ车":108642,"åİ¿å§Ķ":108643,"èīºäºº":108644,"å¥ķ":108645,"è¿İæĿ¥äºĨ":108646,"ç»ĵæĿŁäºĨ":108647,"çļĦä¼łç»Ł":108648,"æĭ¼æIJı":108649,"奥迪":108650,"çĸijæĥij":108651,"ä¹ĭæĹ¥èµ·":108652,"æłĩå¿ĹçĿĢ":108653,"åľ°åįĢ":108654,"è¯łéĩĬ":108655,"åĪ°æľŁ":108656,"åħ¨éĥ½":108657,"çŁŃæļĤ":108658,"æĺ¯æĪijåĽ½":108659,"æĪijå·²ç»ı":108660,"æ»´æ»´":108661,"天èµĭ":108662,"对她":108663,"åį«çĶŁéĹ´":108664,"çĶŁäº§åŁºåľ°":108665,"æĹ¥è®°":108666,"çļĦæķĻåѦ":108667,"åĵĩ":108668,"æ°ijäºĭ":108669,"è¿ĺåİŁ":108670,"æīĭä¸ŃçļĦ":108671,"çļĦèī¯å¥½":108672,"æ·«":108673,"ä¸Ńåħ±ä¸Ń央":108674,"åĪĥ":108675,"åĵĦ":108676,"åľ¨ä»ĸçļĦ":108677,"å°Īæ¥Ń":108678,"åľºéĿ¢":108679,"éĤ»å±ħ":108680,"çĹĴ":108681,"å¦Ħ":108682,"å¤ĸç§ij":108683,"ä¸įéĢĤ":108684,"举åĬŀçļĦ":108685,"éĤ¹":108686,"åħļçļĦ建设":108687,"çĻ¼è¡¨":108688,"è·¨çķĮ":108689,"æ²īæ·Ģ":108690,"大çīĩ":108691,"è¶Ĭé«ĺ":108692,"å°Ĩæĺ¯":108693,"è§īéĨĴ":108694,"åĤ¨åŃĺ":108695,"å¢ŀ大":108696,"ä¸į让":108697,"æķ´å½¢":108698,"å¹³åı°ä¸Ĭ":108699,"åĩłä½į":108700,"è¯īæ±Ĥ":108701,"好ä¸į好":108702,"åľį":108703,"æĸĩæľ¬":108704,"éĢ²åħ¥":108705,"ç´į":108706,"æł¹æĵļ":108707,"èįīæ¡Ī":108708,"åħŃ个":108709,"åĭ¿":108710,"åĪ¶æĪIJ":108711,"饮水":108712,"æ°¸æģĴ":108713,"èĩªæĿĢ":108714,"åı¸é©¬":108715,"éļ¾çĤ¹":108716,"为æĪij们":108717,"弧":108718,"åī©ä¸ĭçļĦ":108719,"åĩĨå¤ĩ好":108720,"çļĦæľĢä½³":108721,"èģĶåIJĪä¼ļ":108722,"æĤ£èĢħçļĦ":108723,"æĪijä¸įçŁ¥éģĵ":108724,"ä¸ĭä¸Ģ个":108725,"åıijå±ķæĸ¹åIJij":108726,"笨":108727,"æīĢ以æĪij们":108728,"åĨĻäºĨ":108729,"éĢłæĪIJäºĨ":108730,"æ²Ļæ¼ł":108731,"çŃĽéĢī":108732,"çģ¾åĮº":108733,"ä¸Ĭçľĭ":108734,"éħ¶":108735,"æ»ļåĬ¨":108736,"éļ¾åħį":108737,"åIJīåĪ©":108738,"ä¸Ģä¸Ģ":108739,"ç²¾å¯Ĩ":108740,"伸æīĭ":108741,"礼仪":108742,"åħ¨æĺ¯":108743,"è¶Ĭ大":108744,"ä¸Ńæłĩ":108745,"åıĸåĨ³":108746,"åıĸåĨ³äºİ":108747,"éĢĶä¸Ń":108748,"讨åİĮ":108749,"æīĭåĨĮ":108750,"第ä¹Ŀ":108751,"åŃĶåŃIJ":108752,"çĦ¶å¾Į":108753,"ä¸Ģåħ±":108754,"æµ·æĬ¥":108755,"款å¼ı":108756,"æķ´å¤©":108757,"è¾¹çķĮ":108758,"路边":108759,"æĻĭ级":108760,"åIJIJ槽":108761,"çļĦåħ³æ³¨":108762,"æĪij没æľī":108763,"å°±æĺ¯åľ¨":108764,"缮çļĦæĺ¯":108765,"åį³ä½¿æĺ¯":108766,"顶å°ĸ":108767,"å·²ç»ıåľ¨":108768,"å®īåħ¨éļIJæĤ£":108769,"æłĩæĿĨ":108770,"åįĹéĢļ":108771,"ä¼ļ对":108772,"座ä½į":108773,"èµ¢å¾ĹäºĨ":108774,"åİŁæĿ¥çļĦ":108775,"身为":108776,"书åºĹ":108777,"è¢Ńåĩ»":108778,"ä»ĬæĻļ":108779,"以èī²":108780,"以èī²åĪĹ":108781,"æĬĸéŁ³":108782,"åį´æ²¡æľī":108783,"丧失":108784,"çļĦå±ĢéĿ¢":108785,"åįģåĽĽäºĶ":108786,"çŃī缸åħ³":108787,"æ±ĩæĢ»":108788,"å¤ĸ表":108789,"为æ°ij":108790,"éľĩæĥĬ":108791,"å¥Ĺè·¯":108792,"çĬ¯ç½ªå«Įçĸij":108793,"å°Ĩ以":108794,"çİĩé¢Ĩ":108795,"éħĴåIJ§":108796,"è¡Įä¸ļåıijå±ķ":108797,"å¹´èĩ³":108798,"åĻ¨æĿIJ":108799,"åĴĮæĬĢæľ¯":108800,"æľĢå°ı":108801,"è¿Ļä¸ĢåĪĩ":108802,"èģĮ称":108803,"å½ĵä½ľ":108804,"æİĢèµ·":108805,"åĴĭ":108806,"ä¸Ńéĥ¨":108807,"æīĭèĩĤ":108808,"ç½¢äºĨ":108809,"媳å¦ĩ":108810,"æ´½è°Ī":108811,"æĹ¶ä»£ä¸ŃåĽ½":108812,"人çĶŁçļĦ":108813,"æŀģéĻIJ":108814,"ç¦Ħ":108815,"åĮºæĶ¿åºľ":108816,"æľ¬éĴ±":108817,"礼åĵģ":108818,"çļĦéĤ£ä¸ª":108819,"ä¾¦æŁ¥":108820,"太å¤ļçļĦ":108821,"å®ŀæĸ½æĸ¹æ¡Ī":108822,"é«ĺæłĩåĩĨ":108823,"æĮĩæĮ¥éĥ¨":108824,"åĢ¾æĸľ":108825,"çī¹èī²ç¤¾ä¼ļ":108826,"çµIJæŀľ":108827,"éĴ»çŁ³":108828,"移æ¤į":108829,"çī¹ç§į":108830,"èĩªæĦ¿":108831,"æĭľçĻ»":108832,"åįķ身":108833,"åį´åıĪ":108834,"åĪ¥äºº":108835,"åIJĪè§Ħ":108836,"æľºçĶµ":108837,"çī¹æĦı":108838,"å½ĵåīįä½įç½®":108839,"买家":108840,"åIJĪ约":108841,"èĤ©èĨĢ":108842,"为åĩĨ":108843,"家è£ħ":108844,"çļĦçĥŃæĥħ":108845,"éĿŀéģĹ":108846,"çļĦéŃħåĬĽ":108847,"åİŁåijĬ":108848,"社ä¼ļåIJĦçķĮ":108849,"ä¹°çļĦ":108850,"å¤ļåIJĥ":108851,"éĽķå¡ij":108852,"èµ·ä¹ī":108853,"åĬłåī§":108854,"éĤ£ä¸ĢåĪ»":108855,"å°Ĩè¿Ľä¸ĢæŃ¥":108856,"æ¡ĤæŀĹ":108857,"æĽ´å¼º":108858,"对ä¼ģä¸ļ":108859,"æĹłæĦı":108860,"ä¹łè¿ijå¹³æĸ°":108861,"æµģ失":108862,"微软":108863,"çĽ¸å¯¹äºİ":108864,"座è°Īä¼ļ":108865,"主èIJ¥ä¸ļ":108866,"主èIJ¥ä¸ļåĬ¡":108867,"ç§ģåĭŁ":108868,"å±ķ示äºĨ":108869,"常æĢģåĮĸ":108870,"è²´":108871,"符åı·":108872,"å¹´è½»çļĦ":108873,"å°±éľĢè¦ģ":108874,"ä¹ŁæĽ¾":108875,"çļĦæĥħ绪":108876,"è¾¾æłĩ":108877,"èĩ¨":108878,"ä½įå±ħ":108879,"ä»ħ为":108880,"é¦ĸ家":108881,"éĺ´éĺ³":108882,"ä¸įåĨįæĺ¯":108883,"åĽłä¸ºå®ĥ":108884,"ä¼ģä¸ļåľ¨":108885,"çĺ¾":108886,"åIJ¬è§ģ":108887,"åİŁæľī":108888,"åĪ¶è£ģ":108889,"å¯Ĥå¯ŀ":108890,"éĢļè¿ĩ对":108891,"æ»ijéĽª":108892,"è¿Ļå¼ł":108893,"çļĦçIJĨ解":108894,"æĸ°ä¸ŃåĽ½":108895,"è¿ĻåĦ¿":108896,"ä½İä»·":108897,"æĥ³è¿ĩ":108898,"çļĦä¿¡å¿ĥ":108899,"建çŃijçī©":108900,"çļĦé¢ľèī²":108901,"ä¸įåºĶ该":108902,"æĹłçĸijæĺ¯":108903,"å¼ķèµ·äºĨ":108904,"åħ¨åijĺ":108905,"æĿ°åĩº":108906,"è¿Ļæĺ¯æĪij":108907,"誰":108908,"èĺĩ":108909,"éĺµåľ°":108910,"åħħåĢ¼":108911,"çŁ¿ä¸ļ":108912,"çĿĢä»ĸ":108913,"信访":108914,"ä¸ĩè¾¾":108915,"æij©æĵ¦":108916,"å¼Ģ端":108917,"èı²å¾ĭ":108918,"èı²å¾ĭ宾":108919,"车åŃIJ":108920,"æľ¬èº«çļĦ":108921,"çģ«è½¦ç«Ļ":108922,"常å·ŀ":108923,"为代表":108924,"为代表çļĦ":108925,"广çĶµ":108926,"亲人":108927,"åı³æīĭ":108928,"éĽĨè£ħ":108929,"éĽĨè£ħç®±":108930,"çļĦåį°è±¡":108931,"æ©Łæľĥ":108932,"åĮĨåĮĨ":108933,"åħīçĶµ":108934,"大æĸ¹":108935,"è¿ĺæľª":108936,"åĪ©å¥½":108937,"ç»Ŀ大å¤ļæķ°":108938,"åľ¨è¿Ļç§į":108939,"ä¸Ģç»Ħ":108940,"æĸ°èĤ¡":108941,"转åıij":108942,"æ³ķåºŃ":108943,"æĹłæīĢ":108944,"éģĵè·¯ä¸Ĭ":108945,"çŁ¿å±±":108946,"èijī":108947,"æĶ¶åĽŀ":108948,"称ä¹ĭ":108949,"称ä¹ĭ为":108950,"æıŃéľ²":108951,"åı£å²¸":108952,"åIJ¼":108953,"å¿ĥæĥ³":108954,"çļĦ梦æĥ³":108955,"éĽ¯":108956,"ä¹ĭåĪĿ":108957,"å¥ĸ项":108958,"订éĺħ":108959,"èĵĿ天":108960,"åĿ¦åħĭ":108961,"ç«ĭæ¡Ī":108962,"èģĶæīĭ":108963,"ä½Ĩæĺ¯æĪij":108964,"帮æĪij":108965,"ä»ħ代表":108966,"说æĪij":108967,"çļĦè¶ĭåĬ¿":108968,"æ¯Ķè¾ĥ大":108969,"èµ°å»Ĭ":108970,"éĩįçĤ¹é¡¹çĽ®":108971,"èµĮåľº":108972,"åIJįçīĩ":108973,"æĦŁåı¹":108974,"åľ¨åľ°ä¸Ĭ":108975,"åıijçĥŃ":108976,"èĮĥçķ´":108977,"çļĦéģĵè·¯":108978,"éĩijèī²":108979,"ä»ĸåıĪ":108980,"ä¼ļ产çĶŁ":108981,"æ°ijåĽ½":108982,"å®ĺæĸ¹ç½ijç«Ļ":108983,"æĶ¶çĽĬçİĩ":108984,"çļĦåĪ°æĿ¥":108985,"çļĦåĬŀæ³ķ":108986,"æĶ¹åĪ¶":108987,"ä¸ĩç§ij":108988,"ä¸įäºĪ":108989,"è¿ĻäºĽéĹ®é¢ĺ":108990,"çĪ±ä¸Ĭ":108991,"çIJĥåľº":108992,"责令":108993,"æİĪ课":108994,"åľ¨é¦Ļ港":108995,"ç»Ĩèħ»":108996,"å¤ļä¸ĩ":108997,"åIJĮå¹´":108998,"大使":108999,"æĸĭ":109000,"ä¹Łä¸º":109001,"æĥłå·ŀ":109002,"åIJī祥":109003,"çĶ°åĽŃ":109004,"åĽ½å®¶éĺŁ":109005,"éĩįçĶŁ":109006,"åľ¨åħ¶":109007,"é¦Ļåij³":109008,"è´Łèį·":109009,"亲åĪĩ":109010,"èĩªè±ª":109011,"没éĶĻ":109012,"åĽłä¸ºåľ¨":109013,"æĺŁæĺŁ":109014,"éĤij":109015,"è¿ĺæľīå¾Īå¤ļ":109016,"æij©æīĺ":109017,"æij©æīĺ车":109018,"æŃ¥è¡Į":109019,"管çIJĨä½ĵç³»":109020,"èĦļä¸ĭ":109021,"éģİåİ»":109022,"æ±īè¯Ń":109023,"对ä¸įèµ·":109024,"çļĦç»ıåİĨ":109025,"åıĬ缸åħ³":109026,"ä¸įå°ij人":109027,"éĩįç£ħ":109028,"åĬ³åĬ¨èĢħ":109029,"大åĬĽåıijå±ķ":109030,"æĢİä¹Īåģļ":109031,"çĭĹçĭĹ":109032,"举åįĹäºļ":109033,"åĭĩäºİ":109034,"åħ¬éĸĭ":109035,"çĵ·çłĸ":109036,"åıĤçħ§":109037,"广æĴŃçĶµè§Ĩ":109038,"举åĬ¨":109039,"æ±Łè¥¿çľģ":109040,"æķĪèĥ½":109041,"åĶ¯æľī":109042,"éĿ¢è²Į":109043,"èĩªåĬ¨é©¾é©¶":109044,"æ¦ľåįķ":109045,"å½ĵæĪij们":109046,"仲è£ģ":109047,"æľ¨æĿIJ":109048,"ç±³åħ°":109049,"çĻ½éĵ¶":109050,"çļĦ人éĥ½":109051,"å°±åĥıæĺ¯":109052,"æŃ¥åħ¥":109053,"åįłçĶ¨":109054,"åĩ»è´¥":109055,"让大家":109056,"ä¼ļè®©ä½ł":109057,"åİ¿æĶ¿åºľ":109058,"è¦ģçĶ¨":109059,"çŃīå½¢å¼ı":109060,"åįĩé«ĺ":109061,"责任æĦŁ":109062,"å¤ĩçĶ¨":109063,"ä»ĸ认为":109064,"æ¸ħåįİ大åѦ":109065,"ä»ĸèĩªå·±":109066,"éĸ±è®Ģ":109067,"太平æ´ĭ":109068,"éĶģå®ļ":109069,"çŃĨ":109070,"è¿Ļçīĩ":109071,"æī§æĶ¿":109072,"è¿ĶåĽŀæIJľçĭIJ":109073,"å°±æѤ":109074,"éģĩåĪ°äºĨ":109075,"å¼Ģå¹ķå¼ı":109076,"管çIJĨéĥ¨éŨ":109077,"姿åĬ¿":109078,"设æĥ³":109079,"åĽĽåŃ£":109080,"æĬĢæľ¯äººåijĺ":109081,"å·®çĤ¹":109082,"è¾ŀèģĮ":109083,"èĢģ師":109084,"çļĦæĦŁåıĹ":109085,"ä¹ŁéĿŀ常":109086,"å¹´ä¸ĬåįĬå¹´":109087,"æĢªçī©":109088,"èĮĥæĸĩ":109089,"æĪĺå½¹":109090,"åIJ«ä¹ī":109091,"åħ¨è¿ĩç¨ĭ":109092,"èĢĮéĿŀ":109093,"éĢļ讯åijĺ":109094,"è¿Ļæł·æīįèĥ½":109095,"æľºç»Ħ":109096,"è£ı":109097,"çķ¶çĦ¶":109098,"èµĮåįļ":109099,"åIJĦæľī":109100,"å·¥ä½ľæľºåĪ¶":109101,"äºĭåIJİ":109102,"åī§éĻ¢":109103,"å±ĬæŶ":109104,"åĺ´éĩĮ":109105,"主线":109106,"ä¸ĢåľĪ":109107,"主è¦ģåİŁåĽł":109108,"å°¸ä½ĵ":109109,"åĮ»çĸĹåĻ¨æ¢°":109110,"ä½łæĢİä¹Ī":109111,"ä½ĨçĶ±äºİ":109112,"æĹ¶ç©º":109113,"çĶ·æľĭåıĭ":109114,"çĶľèľľ":109115,"é«ĺåľ°":109116,"æĻĸ":109117,"èĴIJéĽĨ":109118,"åĩĿèģļåĬĽ":109119,"å¤ĩåıĹ":109120,"æĸĩåĪĽ":109121,"马æĿ¥":109122,"马æĿ¥è¥¿äºļ":109123,"æŁ´æ²¹":109124,"使人":109125,"æķĻä¼ļ":109126,"ç§ĭ天":109127,"æĺİçıł":109128,"åħŃåįģ":109129,"çݯå¢ĥä¸Ń":109130,"æ¸ħæĻ¨":109131,"积æŀģåıĤä¸İ":109132,"å·ħå³°":109133,"ä¸ºæľŁ":109134,"çŃ¾åŃĹ":109135,"æĦŁæ¿Ģ":109136,"ç§ĭåŃ£":109137,"æĿijåŃIJ":109138,"æ¢ħ西":109139,"æļ´éĽ¨":109140,"çĶŁæ´»åľ¨":109141,"çªĹæĪ·":109142,"æģ¶åĬ£":109143,"纯粹":109144,"åľ¨æİ¥åıĹ":109145,"没èĥ½":109146,"è¡Į人":109147,"åĭº":109148,"æĭ¨æīĵ":109149,"ä½ľåĩºäºĨ":109150,"çļĦ主é¢ĺ":109151,"æľªä¾Ĩ":109152,"ä¸ŃæľĢ":109153,"æ¾ľ":109154,"é«ĺè¡Ģåİĭ":109155,"åħ´èµ·":109156,"æŃ£èĥ½éĩı":109157,"åŁ¹è®ŃçıŃ":109158,"æİ¥åħ¥":109159,"çĦ¶åIJİåĨį":109160,"åѦçĶŁä»¬":109161,"é¢ĨåħĪçļĦ":109162,"çģ«çĥŃ":109163,"ä¸ĵèģĮ":109164,"æĪĸèĢħ说":109165,"建è¨Ń":109166,"é»ı":109167,"对åħ¬åı¸":109168,"çī¹æľīçļĦ":109169,"åħīèį£":109170,"å½ĵåľº":109171,"éĿ¢åŃIJ":109172,"èµĦ产管çIJĨ":109173,"æĹ¶æľŁçļĦ":109174,"çŀİ":109175,"åįİ举":109176,"åıĪä¸Ģ次":109177,"èĥİåĦ¿":109178,"å®ļçĤ¹":109179,"头çĹĽ":109180,"液ä½ĵ":109181,"æĺ¯ä¸Ģä½į":109182,"帽åŃIJ":109183,"å¹´èµ·":109184,"ä¸įä½İäºİ":109185,"è¾ĥå°ij":109186,"éĿ¢ä¸´çĿĢ":109187,"å±Ĥå±Ĥ":109188,"èĿ´èĿ¶":109189,"èī°èĭ¦":109190,"éĺ¿æł¹":109191,"éĺ¿æł¹å»·":109192,"æ¦Ĥæĭ¬":109193,"请éĹ®":109194,"èµ·åºĬ":109195,"å±Ģå±Ģéķ¿":109196,"稳åģ¥":109197,"å¦ĤæŀľæĪij们":109198,"éħĴç²¾":109199,"æĪ·åı£":109200,"æĦŁæĤŁ":109201,"æĪij们éľĢè¦ģ":109202,"æĬĢèīº":109203,"èĩªåªĴä½ĵ":109204,"è¿ĽåĮĸ":109205,"æ¿ĢçĥĪçļĦ":109206,"ä½ĵ温":109207,"èļķ":109208,"èĩ´è¾ŀ":109209,"宪æ³ķ":109210,"ä¸ĢçŃīå¥ĸ":109211,"çĵ¶é¢Ī":109212,"æĥłæ°ij":109213,"èµ°è·¯":109214,"çİ°ä»»":109215,"åķĨéĩı":109216,"ä¸ĭ车":109217,"åĪł":109218,"責任":109219,"èŀįåIJĪåıijå±ķ":109220,"ç´łæĿIJ":109221,"油价":109222,"åģļ人":109223,"çŀª":109224,"æĶ¹éĿ©åĪĽæĸ°":109225,"çļĦåĮºåĪ«":109226,"è·¨å¢ĥçĶµåķĨ":109227,"æ¶īåıĬåĪ°":109228,"æīĺ管":109229,"æĪijè¿ĺæĺ¯":109230,"åĿIJæłĩ":109231,"ç½ij讯":109232,"å½ĵåľ°çļĦ":109233,"追溯":109234,"åľŁèĢ³":109235,"åľŁèĢ³åħ¶":109236,"åºķä¸ĭ":109237,"åĩłåįģå¹´":109238,"ç©¿è¿ĩ":109239,"çĶŁæĢģæĸĩæĺİ":109240,"æݨèĸ":109241,"æݨèĸ¦":109242,"éłĨ":109243,"åĴ³åĹ½":109244,"åĪĨæĪIJ":109245,"çĹķ迹":109246,"æĪ·ç±į":109247,"éĥ½ä¸įèĥ½":109248,"æĻļä¼ļ":109249,"åĢ©":109250,"ä½ĵåĬĽ":109251,"è¿Ļ个èģĮä¸ļ":109252,"æĹłå½¢":109253,"åıªæĥ³":109254,"è¿Ľåıĸ":109255,"æĿĢæŃ»":109256,"èĦĬ":109257,"äºijåįĹçľģ":109258,"æľªçŁ¥":109259,"ç¾İèģĶ":109260,"ç¾İèģĶåĤ¨":109261,"å¤ĸå½¢":109262,"诱æĥij":109263,"çĽ£":109264,"è¡Į使":109265,"åłĨ积":109266,"çĨŁç»ĥ":109267,"éĺIJè¿°":109268,"æľĢ大éĻIJ度":109269,"å·¡æŁ¥":109270,"夺åĨł":109271,"ä¼ģä¸ļæĸĩåĮĸ":109272,"çĭ®åŃIJ":109273,"ä¿Ŀå®Ī":109274,"ä¸ºæł¸å¿ĥçļĦ":109275,"æī©æķ£":109276,"åĪ¶éĢłåķĨ":109277,"æŁĶ软":109278,"为ä¸Ģä½ĵçļĦ":109279,"游çİ©":109280,"çĶŁçĹħ":109281,"幫åĬ©":109282,"åĶ±æŃĮ":109283,"æīįåı¯ä»¥":109284,"宽æĿ¾":109285,"è¦ģæ¯Ķ":109286,"æĺ¯æĢİæł·":109287,"çģ°èī²":109288,"çİĭåĽ½":109289,"æIJħæĭĮ":109290,"计éĩı":109291,"åij¨åĽ´çļĦ":109292,"æĻºèĥ½æīĭæľº":109293,"常åĬ¡":109294,"常åĬ¡åī¯":109295,"é©´":109296,"å°Ĩè¿ij":109297,"寻常":109298,"ä¸ŃåĽ½å¸Ĥåľº":109299,"容åĻ¨":109300,"å±±ä¸Ĭ":109301,"èĥĮåIJİçļĦ":109302,"亲å¯Ĩ":109303,"æīĢ以说":109304,"éİ®":109305,"çļĦçIJĨçĶ±":109306,"大åŁİå¸Ĥ":109307,"常年":109308,"æĹħ游ä¸ļ":109309,"å°±æĺ¯è¿Ļæł·":109310,"åĨįæĿ¥":109311,"é«ĺä½į":109312,"åĨħ饰":109313,"æŀĦéĢł":109314,"ä¸Ģèµ·æĿ¥":109315,"çĶ³è«ĭ":109316,"å·²ç»ıå¼Ģå§ĭ":109317,"çļĦåĬ¨ä½ľ":109318,"被迫":109319,"éģįå¸ĥ":109320,"åīĸæŀIJ":109321,"å°ıäºĭ":109322,"å¿ĥä¸ŃçļĦ":109323,"ä½ĵåĪ¶æĶ¹éĿ©":109324,"çļĩ家":109325,"æķĻåłĤ":109326,"åIJĥå®Į":109327,"åĽ½æ°ijåħļ":109328,"æĺİç¡®äºĨ":109329,"åıijå±ķè§ĦåĪĴ":109330,"第ä¸ĢæŃ¥":109331,"å¾Ĺèµ·":109332,"åľ¨åĵª":109333,"çļĦè·¯ä¸Ĭ":109334,"é»Ķ":109335,"çķ¶æĻĤ":109336,"大åĬĽæĶ¯æĮģ":109337,"åıĮéĩį":109338,"çŁ¥éģĵèĩªå·±":109339,"åIJĪä½ľåįıè®®":109340,"æ°ĶåĬ¿":109341,"éķ¿æķĪæľºåĪ¶":109342,"ç½ķè§ģ":109343,"åĽŀæĿ¥äºĨ":109344,"ä»ĸä¼ļ":109345,"ä¸Ńæĸ°":109346,"ä¸Ńæĸ°ç½ij":109347,"çļĦåķĨåĵģ":109348,"èµłéĢģ":109349,"決å®ļ":109350,"å¸ĤåľºçĽij管":109351,"çķĻåѦçĶŁ":109352,"çĶµåİĭ":109353,"äºļ马":109354,"äºļ马éĢĬ":109355,"è¿ĺæĺ¯æ¯Ķè¾ĥ":109356,"ä¿ĥè¿ĽäºĨ":109357,"æµģåħ¥":109358,"æijĦåĥı":109359,"æijĦåĥı头":109360,"æıIJåıĬ":109361,"åıijæİĺ":109362,"æī¾åĩº":109363,"æ¢Ŀ件":109364,"ç¹¼çºĮ":109365,"æĪijåĸľæ¬¢":109366,"å¥İ":109367,"æ¦ľæł·":109368,"å¼ĢèĬ±":109369,"æ²īéĩį":109370,"åŁºåĩĨ":109371,"ä»ħä»ħæĺ¯":109372,"轨éģĵ交éĢļ":109373,"åĶIJå±±":109374,"çŃīä¸Ģç³»åĪĹ":109375,"ä¸įè¿ĩæĺ¯":109376,"åŃĺåľ¨çĿĢ":109377,"èĬ±çĶŁ":109378,"夷":109379,"ç»Ī究":109380,"ä¹Łæĺ¯ä¸Ģ个":109381,"åįģåŃĹ":109382,"èĸªéħ¬":109383,"伤å¿ĥ":109384,"æĺ¥ç§ĭ":109385,"åĨ·åį´":109386,"ç²¾çģµ":109387,"çļĦåľ°åĽ¾":109388,"æ¯Ķçī¹":109389,"æ¯Ķçī¹å¸ģ":109390,"æĢ§åĪ«":109391,"ä½Ļä¸ĩåħĥ":109392,"ä¸įå¿ĺåĪĿå¿ĥ":109393,"å¿ĥçĸ¼":109394,"æĽ²çº¿":109395,"é«ĺä½İ":109396,"è¦ıå®ļ":109397,"æĻ¯èī²":109398,"è¦ģ说":109399,"åħ¬åı¸å°Ĩ":109400,"液åİĭ":109401,"è¿Ŀ约":109402,"åİļ度":109403,"åºŀ大çļĦ":109404,"è¿ĺæĺ¯å¾Ī":109405,"é¦ĸåħĪæĺ¯":109406,"çµ²":109407,"åĬ¡å®ŀ":109408,"並ä¸Ķ":109409,"å¢ŀè¿Ľ":109410,"ç»Ħç»ĩå¼Ģå±ķ":109411,"èµ·æĿ¥äºĨ":109412,"è¾ĥå°ı":109413,"导游":109414,"ä¸¤åľ°":109415,"ç¿ĺ":109416,"çģ¿çĥĤ":109417,"é£İéĩĩ":109418,"æĶ¯çº¿":109419,"æĶ¯çº¿ä»»åĬ¡":109420,"娱ä¹IJåľĪ":109421,"天津å¸Ĥ":109422,"åĮħåĽ´":109423,"æľ¬èµĽåŃ£":109424,"éĩįè¦ģ讲è¯Ŀ":109425,"åıĮåIJij":109426,"åįİ丽":109427,"éĶ¤":109428,"åĦ¿å¥³":109429,"åįĸåĩº":109430,"ä¾Ĩ說":109431,"ä»ĭç»įä¸Ģä¸ĭ":109432,"åIJ¦è®¤":109433,"åĭĿ":109434,"æĻ®éĢļ人":109435,"çļĦåĬ¨åĬĽ":109436,"涨åģľ":109437,"åŁºéĩij管çIJĨ":109438,"ä¸Ģ个éĩįè¦ģ":109439,"è¿IJæ²³":109440,"çħŀ":109441,"è´¢æĶ¿éĥ¨":109442,"è¡Įä¸ļåįıä¼ļ":109443,"éĥ½å°Ĩ":109444,"è¨Ģ论":109445,"ä¸ĭä¾Ĩ":109446,"墨西":109447,"墨西åĵ¥":109448,"åĽłä¸ºä»ĸ们":109449,"æĢİä¹ĪåĽŀäºĭ":109450,"åĬłå¤§å¯¹":109451,"èĬŃ":109452,"çīĮåŃIJ":109453,"ä¼ļ使":109454,"妹åŃIJ":109455,"ç«Ļéķ¿":109456,"å¿ħå¤ĩ":109457,"æłijæľ¨":109458,"æģ¶æĦı":109459,"æ²³éģĵ":109460,"å¯Įè£ķ":109461,"ç¹ģåįİ":109462,"ä»£è¡¨åĽ¢":109463,"æµij身":109464,"é¦ĸä½į":109465,"èĪªç©ºåħ¬åı¸":109466,"éĽ»å½±":109467,"ä¸ĵè¾ij":109468,"æ°´æºIJ":109469,"ä¸Ńæ¯Ĵ":109470,"並ä¸į":109471,"èĢĮåİ»":109472,"éĥĿ":109473,"äºİæѤ":109474,"æĸĩåĮĸ建设":109475,"èĤ¯å®ļä¼ļ":109476,"å¸ĮæľĽå¤§å®¶":109477,"æııåĨĻ":109478,"ä½İè°ĥ":109479,"æĸ°åħ´äº§ä¸ļ":109480,"æ·Ħåįļ":109481,"æĶ¾å¼Ģ":109482,"çļĦæĢ§æł¼":109483,"çĸ¾çĹħçļĦ":109484,"æķ´é¡¿":109485,"线ä¸Ĭ线ä¸ĭ":109486,"éĢī项":109487,"çļĦ认åı¯":109488,"æķ´é½IJ":109489,"çĶļä¹Ī":109490,"çľģåĨħ":109491,"åı¤äºº":109492,"æ°ijä¿Ĺ":109493,"çī¡ä¸¹":109494,"éŨçªĹ":109495,"éĤ£æł·çļĦ":109496,"çĽijäºĭä¼ļ":109497,"ç¿¡ç¿ł":109498,"禹":109499,"åįĥä¸ĩä¸įè¦ģ":109500,"æĶ¶ç¼©":109501,"çļĦæĸĩåŃĹ":109502,"åĴĮå°ļ":109503,"æĮĩ令":109504,"åħ±äº§åħļåijĺ":109505,"çļĦçĪ¶äº²":109506,"å®Įå·¥":109507,"åĬ¡å·¥":109508,"马æĭī":109509,"马æĭīæĿ¾":109510,"æµĭè¯Ħ":109511,"å²ļ":109512,"ä¸įåģļ":109513,"ä¸ĥå¹´":109514,"åĿĩä»·":109515,"主è§Ĥ":109516,"å¾Īä¸įéĶĻ":109517,"èĤ¡ä¸ľå¤§ä¼ļ":109518,"äºĶä¸Ģ":109519,"é£İåIJ¹":109520,"å¼Ģéĩĩ":109521,"è¿Ļä¹Ī大":109522,"èĥ½çľĭåĪ°":109523,"èĢĥè¯Ħ":109524,"åį³ä¾¿æĺ¯":109525,"çݰ代åĨľä¸ļ":109526,"æ¯Ķè¾ĥé«ĺ":109527,"è¦ģçľĭ":109528,"没äºĨ":109529,"解決":109530,"çݯæ¯Ķ":109531,"åĨ²åĬ¨":109532,"æ·±å¤ľ":109533,"åĩłåįĥ":109534,"ä¿ı":109535,"ç½ijæ°ij":109536,"就没":109537,"ä»ĸ表示":109538,"éĩıåŃIJ":109539,"æĹ©é¤IJåĬłçĽŁ":109540,"åįĬå²Ľ":109541,"æIJŀç¬ij":109542,"ä¸ĬæĬ¥":109543,"審":109544,"é¢Ħ订":109545,"èľĤèľľ":109546,"æŁ¥æī¾":109547,"ä¼ĹæīĢ":109548,"ä¼ĹæīĢåij¨":109549,"ä¼ĹæīĢåij¨çŁ¥":109550,"æĹ©æĹ¥":109551,"åıijæī¬":109552,"åĴĮ个人":109553,"åĬłåħ¥äºĨ":109554,"åĸ®ä½į":109555,"åĪĨæĺİ":109556,"第ä¸Ģæī¹":109557,"ç¾İåĨĽ":109558,"æĿĢæīĭ":109559,"éŨå¤ĸ":109560,"åķĨåľĪ":109561,"ä¸ĢåĪ»":109562,"çļĦçľ¼ç¥ŀ":109563,"éľĦ":109564,"äºĽä»Ģä¹Ī":109565,"åĬłæ·±":109566,"æ¯ıä½į":109567,"å¸ĤéĿ¢ä¸Ĭ":109568,"åıĶåıĶ":109569,"çļĦéĤ£ç§į":109570,"粤港澳":109571,"è´´å¿ĥ":109572,"æĸĩåĮĸ产ä¸ļ":109573,"红æĹĹ":109574,"åĺīåħ´":109575,"æĶ¶çĽĺ":109576,"å®ĮæĪIJåIJİ":109577,"ä¼ģä¸ļ管çIJĨ":109578,"纵横":109579,"ä¸įä¿¡":109580,"æĪIJéĥ½å¸Ĥ":109581,"æ´Ĺ澡":109582,"举è¡ĮçļĦ":109583,"çĶ¢çĶŁ":109584,"ç©¿ä¸Ĭ":109585,"åĪļ好":109586,"åħī线":109587,"æīĵæŀ¶":109588,"è¿Ļæľ¬ä¹¦":109589,"åĶ®åIJİæľįåĬ¡":109590,"åĩłåĪĨ":109591,"ä¸Ĭ次":109592,"ä¸įåĪĨ":109593,"产åIJİ":109594,"éģ¿å¼Ģ":109595,"ç»Īæŀģ":109596,"代表大ä¼ļ":109597,"æ¼ĶæĬĢ":109598,"åĽŀè´Ń":109599,"åŃ¦è´¹":109600,"éĺ»ç¢į":109601,"ä¸Ģ大æī¹":109602,"竣工":109603,"åĨ³å®ļäºĨ":109604,"ä½Ĩå¦Ĥæŀľ":109605,"çĶµæµģ":109606,"ä¸Ŀ毫":109607,"èĥ½å¤Łåľ¨":109608,"éĶĢåĶ®æĶ¶åħ¥":109609,"åľ¨åŃ¦æł¡":109610,"æ°´åĩĨ":109611,"è§Ĩ线":109612,"èĩªåľ¨":109613,"åķĨä¸ļéĵ¶è¡Į":109614,"为äºĨ让":109615,"çį²å¾Ĺ":109616,"çݩ家æľĭåıĭ":109617,"éĿ¢èĨľ":109618,"åĪĨåī²":109619,"åī§æľ¬":109620,"ç«Ń":109621,"说å¾Ĺ":109622,"æĥ³çŁ¥éģĵ":109623,"çļĦ人çī©":109624,"èĮħåı°":109625,"åIJĮä¸Ģ个":109626,"æķ°æį®ä¸Ńå¿ĥ":109627,"çĶĦ":109628,"åĸľæĤ¦":109629,"ä¸ĭæĿ¥çļĦ":109630,"å®ļåIJij":109631,"æŀģåħ·":109632,"çļĦåľŁåľ°":109633,"éĤ£åĢĭ":109634,"æijĦåħ¥":109635,"äºĨæĪijçļĦ":109636,"马路":109637,"åħ¨ç¤¾ä¼ļ":109638,"è®®æ¡Ī":109639,"å±ĭåŃIJ":109640,"åIJįåı«":109641,"åĮª":109642,"åľ¨å¤ĸéĿ¢":109643,"åįİåįĹ":109644,"åıijè´§":109645,"å¯ĴåĨ·":109646,"é«ĺçŃīæķĻèĤ²":109647,"详ç»ĨçļĦ":109648,"ä¸ªé¡¹çĽ®":109649,"çĶŁäº§åĬĽ":109650,"æĹ¶å¸¸":109651,"å°±æľĥ":109652,"ä¸ĩèĤ¡":109653,"éĻĮçĶŁäºº":109654,"æııç»ĺ":109655,"å½ĵçĦ¶æĺ¯":109656,"æĭīåĬ¨":109657,"éĵ¾æĿ¡":109658,"æī£éĻ¤":109659,"ä¸Ģ缴éĥ½":109660,"å°ıåŃ©åŃIJ":109661,"伤åı£":109662,"第äºĮå±Ĭ":109663,"è´Ńç½®":109664,"çļĩ马":109665,"æĹłèģĬ":109666,"表åĨ³":109667,"诸å¦Ĥ":109668,"åĵįèµ·":109669,"é£İæļ´":109670,"ä¸ĢæµģçļĦ":109671,"ç·¨":109672,"解æĶ¾åĨĽ":109673,"室å¤ĸ":109674,"å°±è¿Ļä¹Ī":109675,"島":109676,"æīĢæľī人éĥ½":109677,"æIJľç´¢å¼ķæĵİ":109678,"çļĦæĪIJæľ¬":109679,"åħļæĶ¿":109680,"åıijè¡Į人":109681,"çļĦäºĭå®ŀ":109682,"对该":109683,"åıĹæįŁ":109684,"ä¿Ħä¹Į":109685,"é²ľèĬ±":109686,"åĨľèį¯":109687,"æŀģéĢŁ":109688,"æĢ¥æĢ§":109689,"两ä¼ļ":109690,"ä¸ĢèĪ¬æĿ¥è¯´":109691,"æµ·é²ľ":109692,"åĨĪ":109693,"çĶ¨äºº":109694,"çĶ¨äººåįķä½į":109695,"åĢª":109696,"åĦªæĥł":109697,"æł¹æºIJ":109698,"åĽ¢è´Ń":109699,"ç¾İæ´²":109700,"ä¸ĭè¡Į":109701,"å¹´æľ«":109702,"èľ¡":109703,"è¯ģ件":109704,"åľ¨æĪijåĽ½":109705,"ä¸įåºĶ":109706,"æĮīæŶ":109707,"åłªç§°":109708,"åľºä¸Ĭ":109709,"å¹²éĥ¨èģĮå·¥":109710,"æľīå¾Ī大çļĦ":109711,"æķ°åŃĹç»ıæµİ":109712,"æ¼Ķç»ĥ":109713,"æį®ç»Łè®¡":109714,"å¾ĢæĿ¥":109715,"广åijĬæľįåĬ¡":109716,"çļĦè·Ŀ离":109717,"æѸ":109718,"è¨Ģè¯Ń":109719,"被èªī":109720,"被èªī为":109721,"åĭī强":109722,"å°Ĭæķ¬":109723,"ä¸ĩ亿åħĥ":109724,"ä¸ŃåĽ½åĽ½éĻħ":109725,"å¹²é¢Ħ":109726,"年产":109727,"èĢķåľ°":109728,"èĮİ":109729,"åį³æĺ¯":109730,"æĺ¨æĻļ":109731,"æĪIJ为ä¸Ģ个":109732,"çºłæŃ£":109733,"åij½åIJį":109734,"é¢ģå¸ĥ":109735,"çĮľæµĭ":109736,"ä¿ĿèŃ·æĶ¿çŃĸ":109737,"æĭ¢":109738,"活泼":109739,"çŃīéĥ¨éŨ":109740,"åѦåĪ°":109741,"å¢ŀåĢ¼ç¨İ":109742,"èĪªçº¿":109743,"åĨ¤":109744,"åįģåĩłå¹´":109745,"æݧèĤ¡èĤ¡ä¸ľ":109746,"ä¸ĢéŨ":109747,"ä¸ªå·¥ä½ľ":109748,"ä¸ªå·¥ä½ľæĹ¥":109749,"æĸ°è¥¿":109750,"æĸ°è¥¿åħ°":109751,"论è¯ģ":109752,"ä»Ĩ":109753,"åı¦å¤ĸä¸Ģ个":109754,"æĶ¹ç¼ĸ":109755,"严ç¦ģ":109756,"åĸľå¥½":109757,"个人信æģ¯":109758,"满æĦı度":109759,"åĵ¨":109760,"å¸ĪèµĦ":109761,"æĶ¹ä¸º":109762,"ç«ŀäºī对æīĭ":109763,"åĩºçĤī":109764,"åķĨ人":109765,"大æ£ļ":109766,"æĮĩ导ä¸ĭ":109767,"å¦ĩç§ij":109768,"輪":109769,"æīģ":109770,"åIJĮæŶè¿ĺ":109771,"并éĢļè¿ĩ":109772,"æĪĺéĺŁ":109773,"èĶĵ延":109774,"ä¿ŀ":109775,"éĢĤå½ĵçļĦ":109776,"åīįè¾Ī":109777,"åĵģåij³":109778,"æ¹¿åľ°":109779,"æĪIJåŀĭ":109780,"ä¸įåıªæĺ¯":109781,"æĥ©ç½ļ":109782,"åĩºåı°äºĨ":109783,"çݩ游æĪı":109784,"æīįåıijçİ°":109785,"åºĶèģĺ":109786,"å¤ĸæĿ¥":109787,"åįłé¢Ĩ":109788,"å±ķæľĽ":109789,"å«Ĥ":109790,"港èĤ¡":109791,"æ¡Įä¸Ĭ":109792,"æĶ¯æŁ±":109793,"çļĦæĥħå½¢":109794,"广éĺĶçļĦ":109795,"æĶ¯è¡Į":109796,"å´©æºĥ":109797,"æľĪä¸Ń":109798,"æľĪä¸ŃæŬ":109799,"ç»įåħ´":109800,"临è¿ij":109801,"æĬ¤æłı":109802,"æļ®":109803,"åįķèģĮä¸ļ":109804,"è¾¹å¢ĥ":109805,"æĹ¥çħ§":109806,"ä¸ĢåłĨ":109807,"缴å¾Ħ":109808,"åħ±åIJĮä½ĵ":109809,"æĸ°åįİç½ij":109810,"æīĵ好":109811,"çĶµåĬ¨æ±½è½¦":109812,"ä¸įæĺİçĻ½":109813,"éĢĻ裡":109814,"çĽĽå¤§":109815,"çİĭæľĿ":109816,"åĨįä¸Ģ次":109817,"åĬŀåħ¬åİħ":109818,"è´¨æĬ¼":109819,"åIJĪåĩ»":109820,"人们对":109821,"éĽ¶é£Ł":109822,"éĥ½ä¸įçŁ¥éģĵ":109823,"çļĦè¯Ńè¨Ģ":109824,"åĭŁéĽĨèµĦéĩij":109825,"åĬ¨èĦī":109826,"彤":109827,"è¿Ļåĩłå¹´":109828,"çŁŃè§Ĩé¢ij":109829,"太é«ĺ":109830,"常å§Ķä¼ļ":109831,"åĬłçıŃ":109832,"éĩįå¿ĥ":109833,"åªĴä½ĵæĬ¥éģĵ":109834,"没æ³ķ":109835,"éĹ»åIJį":109836,"çĥŃ度":109837,"å¹¿æ³ĽçļĦ":109838,"åħŃ大":109839,"çī©ä½ĵ":109840,"ä¸į该":109841,"é¢ĺ主":109842,"精彩çļĦ":109843,"ä¸ºè¿Ľä¸ĢæŃ¥":109844,"èĻŀ":109845,"åĽºçĦ¶":109846,"è´µå·ŀçľģ":109847,"çºłç»ĵ":109848,"代çIJĨ人":109849,"æ³ķå®ļ代表":109850,"åı¦ä¸Ģç§į":109851,"ä¸įåIJ«":109852,"æĭ¯æķij":109853,"ä¼ļç»Ļ":109854,"è¯Ĺè¯į":109855,"åIJĮç±»":109856,"å¾Ĺä¸įåĪ°":109857,"æĬĵç´§":109858,"以åħ¶":109859,"åħ¥åħļ":109860,"è¿ĺåı¯":109861,"æľŁåĪĬ":109862,"å¾Īå¤ļæŶåĢĻ":109863,"æĹ¥åIJİ":109864,"åħ¬çº¦":109865,"ä¸Ģ举":109866,"æ¯Ķè¾ĥå¤ļ":109867,"éĩijæ²Ļ":109868,"æįŀ":109869,"æİĴåĩº":109870,"æŃ¦æľ¯":109871,"ä¸įæĸ·":109872,"ä¸ŃèĢĥ":109873,"ä¿¡èµĸ":109874,"ä»İä¸ļ人åijĺ":109875,"çģ«çĦ°":109876,"éĨĴæĿ¥":109877,"ä½İ温":109878,"éĢ¾æľŁ":109879,"åĬ±å¿Ĺ":109880,"éħ¥":109881,"åı¯è°ĵæĺ¯":109882,"è¿ĻæĦıåij³çĿĢ":109883,"é¢łè¦Ĩ":109884,"åĮĹ京大åѦ":109885,"ä¸ĵ线":109886,"åıĬ以ä¸Ĭ":109887,"訪":109888,"èĢĮåIJİ":109889,"çŁ¥ä¹İ":109890,"ä¸Ģ对ä¸Ģ":109891,"å¨ĥå¨ĥ":109892,"çģ¾éļ¾":109893,"åħ¨å±Ģ":109894,"æīĢå¾Ĺç¨İ":109895,"å®ŀæĥł":109896,"èļĤèļģ":109897,"ä¹ŁçŁ¥éģĵ":109898,"温åĴĮ":109899,"èIJ½ä¸ĭ":109900,"åŀĭä¼ģä¸ļ":109901,"åĨįä¹Ł":109902,"ä¾ĽçĥŃ":109903,"é«ĺæ½®":109904,"çĢı覽åĻ¨":109905,"çļĦ巨大":109906,"åħĪ天":109907,"å¹´ä¸ŃåĽ½":109908,"类似çļĦ":109909,"çIJĨäºĭä¼ļ":109910,"空éĸĵ":109911,"çģµæĦŁ":109912,"åĬĽæ°Ķ":109913,"带ä¸Ĭ":109914,"ä¸į好æĦıæĢĿ":109915,"æľīä½ķ":109916,"å·²åľ¨":109917,"åıĸåĩº":109918,"è¿Ŀæ³ķçĬ¯ç½ª":109919,"åŃ¦ä¹łè´¯å½»":109920,"åľ°å¸¦":109921,"楼梯":109922,"çŃīæĥħåĨµ":109923,"ä»İåīį":109924,"çļĦä¹łæĥ¯":109925,"ç³Łç³ķ":109926,"å°±èĥ½å¤Ł":109927,"è©ķ":109928,"ä¸Ģå¾ĭ":109929,"æĮ«æĬĺ":109930,"åİŁæĸĩåľ°åĿĢ":109931,"å½ĵå±Ģ":109932,"ä¸įéĢļ":109933,"æķ°åįĥ":109934,"éĺŁä¼į建设":109935,"æŶèĬĤ":109936,"åģļèµ·":109937,"çļĦè®°å¿Ĩ":109938,"ç½ij绾å®īåħ¨":109939,"åĩ¡æĺ¯":109940,"æ°¯":109941,"éĽķåĪ»":109942,"åŁĥåıĬ":109943,"æĪijåı¯ä»¥":109944,"çĽijçIJĨ":109945,"æĽ´åħ·":109946,"åŁİ管":109947,"èĭ¯":109948,"åı¥åŃIJ":109949,"èĭ¥æľī":109950,"ä»İæĿ¥ä¸į":109951,"缸åħ³è´Łè´£":109952,"å®īåħ¨æĦŁ":109953,"æĽ´è¦ģ":109954,"çļĦæĥħæĦŁ":109955,"çī¢çī¢":109956,"è¾ĥ好çļĦ":109957,"æ°®":109958,"ç¬ijè¯Ŀ":109959,"车å±ķ":109960,"ä¹ĭç¾İ":109961,"ç®Ģ约":109962,"ç±»åŀĭçļĦ":109963,"èĢģåĮĸ":109964,"çľĭä½ł":109965,"è¿ĩåĪĨ":109966,"éŨåīį":109967,"ä¸ĢéĹ´":109968,"æĥ³åİ»":109969,"åªĽ":109970,"åľŁè±Ĩ":109971,"åıĪ称":109972,"ä¸Ńä¿¡":109973,"åŃĺéĩı":109974,"马äºij":109975,"èĩ´ä½¿":109976,"åħĪåīį":109977,"èĢģåŃIJ":109978,"æīĵæī®":109979,"æ¯ķä¸ļäºİ":109980,"æ¯ķä¸ļåIJİ":109981,"ç¾İ好çĶŁæ´»":109982,"å·¥ä¸ļä¼ģä¸ļ":109983,"就好äºĨ":109984,"èħIJèļĢ":109985,"çıįçıł":109986,"åĪ°è¿ĻéĩĮ":109987,"æīĢéľĢçļĦ":109988,"è¿Ļæĺ¯åĽłä¸º":109989,"çIJĨæĥ³çļĦ":109990,"å·®å¼ĤåĮĸ":109991,"é®":109992,"é®®":109993,"äºļ太":109994,"æĹłç©·":109995,"æıIJçİ°":109996,"ä¸ĵä¸ļæĬĢæľ¯":109997,"çĶ¢æ¥Ń":109998,"åѦåŃIJ":109999,"ç§ijå¹»":110000,"åįłåľ°éĿ¢ç§¯":110001,"ä¸įåĩĨ":110002,"æľªæĪIJ年人":110003,"æĶ¶å½ķ":110004,"è¿ĺ款":110005,"éĴ¢çŃĭ":110006,"æ¼¢":110007,"å¾ĹæĦı":110008,"综åIJĪä½ĵ":110009,"æŀģé«ĺ":110010,"åįķè¯į":110011,"é«ĺæķĪçļĦ":110012,"骨头":110013,"æī§çĿĢ":110014,"缼ä¸ĸ":110015,"模çī¹":110016,"æĽ´èĥ½":110017,"ç»ĿæľĽ":110018,"对åºĶçļĦ":110019,"æ¨Ĭ":110020,"æĸ°ä¸ī":110021,"æĸ°ä¸īæĿ¿":110022,"æģ°æģ°":110023,"åIJį家":110024,"æł¸å¿ĥæĬĢæľ¯":110025,"个å°ı":110026,"æĢİä¹Īä¼ļ":110027,"说ä¸įå®ļ":110028,"西çĵľ":110029,"åĵİ":110030,"ç¢Ł":110031,"å¿ħä¸įåı¯":110032,"å¿ħä¸įåı¯å°ij":110033,"ä¹ĭéĸĵ":110034,"åĪĨ管":110035,"交éĢļäºĭæķħ":110036,"å¼ĢåĬŀ":110037,"å¾ģæ±ĤæĦıè§ģ":110038,"亨":110039,"鼻åŃIJéĥµ":110040,"鼻åŃIJéĥµä»¶":110041,"ä¿¡æģ¯æľįåĬ¡":110042,"ä½łè§īå¾Ĺ":110043,"缴è§Ĥ":110044,"å·²å®ĮæĪIJ":110045,"åĪĨä¼ļ":110046,"åĽŀåįĩ":110047,"éļ»":110048,"好人":110049,"äºĨ解ä¸Ģä¸ĭ":110050,"åį«æµ´":110051,"æľĢçĪ±":110052,"åºŀ大":110053,"客æĪ¿":110054,"çijŀåħ¸":110055,"éĥ½ä¸įæĺ¯":110056,"館":110057,"èĹī":110058,"çļĦåIJĦ项":110059,"ä¸ºçĽ®æłĩ":110060,"çļĦè®¤çŁ¥":110061,"å½±åĵįåĬĽçļĦ":110062,"å¤¸å¼ł":110063,"佩æĪ´":110064,"æ±ĩçİĩ":110065,"çļĦçĪ±æĥħ":110066,"æĺ¥é£İ":110067,"æĺ¯æĪijçļĦ":110068,"樹":110069,"åįĬå°ıæŶ":110070,"å±±åİ¿":110071,"山西çľģ":110072,"èĢĮè¿Ļ":110073,"æĽ´å¤ļä¿¡æģ¯":110074,"è¿ĺæľīä¸ĢäºĽ":110075,"ç²¾ç»ĨåĮĸ":110076,"ç¾İåѦ":110077,"çĶ±æĸ¼":110078,"ä»ħä¾ĽåıĤèĢĥ":110079,"å¾Īé«ĺçļĦ":110080,"åıłåĬł":110081,"è¿Ļä¹Ī说":110082,"å±ķåĩº":110083,"åĽĽå¤Ħ":110084,"ä¸ĩ家":110085,"æĭĽåĭŁ":110086,"çļĦ强大":110087,"æĤ£æľī":110088,"å°ıäºİ":110089,"ä¹Łè®¸æĺ¯":110090,"对èĩªå·±çļĦ":110091,"èģĮä¸ļæķĻèĤ²":110092,"æĿ¥è¿Ľè¡Į":110093,"档次":110094,"æīĵèµ¢":110095,"éĥ½æľīçĿĢ":110096,"庸":110097,"è¯Ńæ°Ķ":110098,"çĶ²éĨĽ":110099,"空åĨĽ":110100,"车åĨħ":110101,"åĽłä¸ºä½ł":110102,"å®ŀæķĪ":110103,"æĥħä¾£":110104,"åıijè¾¾åĽ½å®¶":110105,"éķľåŃIJ":110106,"æ¯įå©´":110107,"ä½Ĩæĺ¯ä»ĸ":110108,"积æŀģæİ¨è¿Ľ":110109,"大å¹ħ度":110110,"çļĦ女åĦ¿":110111,"é¤IJæ¡Į":110112,"åIJ¬å¾Ĺ":110113,"çļĦ积æŀģæĢ§":110114,"好åIJ§":110115,"æĹ¥æ¶Īæģ¯":110116,"æľīä»»ä½ķ":110117,"æ¯Ĵåĵģ":110118,"æĹ©çĤ¹åĬłçĽŁ":110119,"第ä¸Ģ天":110120,"å°½åĬĽ":110121,"æłĸ":110122,"主æīĵ":110123,"æĺ¯ä¸ĢåIJį":110124,"çĪĨæĸĻ":110125,"äºĭä¸ļåıijå±ķ":110126,"å¾®åķĨ":110127,"äºİä¸Ģä½ĵçļĦ":110128,"çĶŁçĮª":110129,"èĩªçĦ¶èµĦæºIJ":110130,"çŀĦåĩĨ":110131,"è§Ħ模åĮĸ":110132,"并ä¸İ":110133,"èĤ¥èĥĸ":110134,"家çĶ¨":110135,"大çĪ·":110136,"é¢ĦåijĬ":110137,"æĿ¥åģļ":110138,"éĺ³åİ¿":110139,"æŀĦçŃij":110140,"é¢ģå¥ĸ":110141,"åİĨåı²æĸĩåĮĸ":110142,"æľįåĭĻæĪĸ":110143,"æĢ»åĨ³èµĽ":110144,"åıijåŀĭ":110145,"æĪij羣çļĦ":110146,"æĽ¦":110147,"åıĤä¼ļ":110148,"èĦĨå¼±":110149,"åĩĨåħ¥":110150,"èħ¹éĥ¨":110151,"åı¸ä»¤":110152,"æĤ²åī§":110153,"天ä¸Ĭ":110154,"åı£ä¸Ń":110155,"ä¸ĩ个":110156,"åѦä¸ļ":110157,"æıIJåĢ¡":110158,"两边":110159,"大èĤ¡ä¸ľ":110160,"åı¤éķĩ":110161,"è¡Ģç³ĸ":110162,"çļĦç¨ĭ度":110163,"æ£īèĬ±":110164,"åIJİåı°":110165,"å°±åĮ»":110166,"æķ´æķ´":110167,"èĴ²":110168,"çĽĪåĪ©èĥ½åĬĽ":110169,"ç±½":110170,"èĦ«":110171,"çľĭéĩį":110172,"家éķ·":110173,"èģĺçĶ¨":110174,"èµĽéģĵ":110175,"åīįèĢħ":110176,"建èŃ°":110177,"å¾ĭå¸ĪäºĭåĬ¡":110178,"èīºæľ¯åĵģ":110179,"æľīèĩªå·±çļĦ":110180,"åIJ¦å®ļ":110181,"ç¤¾åĽ¢":110182,"åij¨äºĶ":110183,"带åĪ°":110184,"å·¥ä½ľä¼ļè®®":110185,"èĤ¡æľ¬":110186,"å¤ĸåĮħ":110187,"家åħ¬åı¸":110188,"çĽijçĭ±":110189,"èĪĬ":110190,"åIJįæł¡":110191,"西æ¹ĸ":110192,"è¶ħè¿ĩäºĨ":110193,"åįĹå±±":110194,"ç»Ħ件":110195,"åĢ¼å¾Ĺ注æĦı":110196,"æĮ£æīİ":110197,"äºĭ迹":110198,"ç¶ĵçĩŁ":110199,"ç§ij室":110200,"好åIJĹ":110201,"æ¤ħåŃIJ":110202,"åľĪåŃIJ":110203,"ä½Ĩ她":110204,"æµģçķħ":110205,"åIJĦèĩªçļĦ":110206,"èģĮåijĺ":110207,"è¡įçĶŁ":110208,"åħ¨åľº":110209,"æĴ¤éĶĢ":110210,"åį´è¢«":110211,"å®ģéĿĻ":110212,"åīįæīĢ":110213,"åīįæīĢæľª":110214,"åīįæīĢæľªæľī":110215,"主ä¸ļ":110216,"åĮĹç¾İ":110217,"è¯Ħå®ļ":110218,"åĵģå°Ŀ":110219,"大家éĥ½åľ¨":110220,"主å¸ħ":110221,"ç»Ĩå¿ĥ":110222,"ä¿¡æģ¯æĬ«éľ²":110223,"çļĦç«ŀäºī":110224,"éĢĻ樣çļĦ":110225,"ç§ijåĪĽæĿ¿":110226,"éĩĩæijĺ":110227,"票æį®":110228,"éĢIJå¹´":110229,"èĭ±è¶ħ":110230,"è¡Įä¸ļåĨħ":110231,"人寿":110232,"åIJİåĭ¤":110233,"å¦ĤæĦı":110234,"ç¬Ķè¯ķ":110235,"æ·¡æ·¡çļĦ":110236,"ä¸įèĪĴæľį":110237,"ä½ĵ积":110238,"ä¹Łä¸įè¦ģ":110239,"éĿ¢æĸĻ":110240,"æł·æľ¬":110241,"ç¥ģ":110242,"æĮīè§Ħå®ļ":110243,"大æ¦Ĥæĺ¯":110244,"æĥħåĨµè¿Ľè¡Į":110245,"åIJĦåįķä½į":110246,"çļĦç¬ij容":110247,"åĩºèī²çļĦ":110248,"代表æĢ§":110249,"çļĦç¾İ好":110250,"éĴ¦":110251,"å¾®çĶŁçī©":110252,"è¶Ĭæĺ¯":110253,"æĸ¹åı¯":110254,"å¹²èĦĨ":110255,"éģĬæĪ²":110256,"çļĦåħ´è¶£":110257,"éĹ®è´£":110258,"åĽłä¸ºæĪij们":110259,"èĢĥéĩı":110260,"çĶŁçĶŁ":110261,"éĺ»åĬĽ":110262,"ä¸įåħģ许":110263,"æıIJè®®":110264,"åĩıæĮģ":110265,"åıªæĺ¯ä¸Ģ个":110266,"æĪijæĬĬ":110267,"åıijçİ°èĩªå·±":110268,"å¢ŀå¹ħ":110269,"å¦į":110270,"èĹĿè¡ĵ":110271,"ä¸Ģ家人":110272,"åĪĨ级":110273,"çļĦæķ°éĩı":110274,"è½®èŀįèµĦ":110275,"çŃīåĽłç´ł":110276,"大夫":110277,"èģĺ请":110278,"é£İæľº":110279,"绽æĶ¾":110280,"ä»»ä½ķä¸Ģ个":110281,"éłĤ":110282,"éĺ¶çº§":110283,"æĬĬ她":110284,"è¿ĽåĨĽ":110285,"èĥ½åģļåĪ°":110286,"åŁ¹è®ŃæľºæŀĦ":110287,"çī©æĸĻ":110288,"ç«¥è¯Ŀ":110289,"æĮĩ导æĦıè§ģ":110290,"éĺ®":110291,"æ·±åħ¥æİ¨è¿Ľ":110292,"ä¸»æľº":110293,"æ¸Ķä¸ļ":110294,"ä¸įæľį":110295,"æµĵéĥģ":110296,"è¡Ĺä¸Ĭ":110297,"ä¾Ŀ次":110298,"æĹ¶æ®µ":110299,"梵":110300,"çļĦåĸľçĪ±":110301,"å¾Īéķ¿":110302,"åĪĿ级":110303,"æŀľæĸŃ":110304,"æĬ¢æķij":110305,"é¼ĵèĪŀ":110306,"ä¾ĽéľĢ":110307,"æ·±åħ¥å¼Ģå±ķ":110308,"产ä¸ļéĽĨ群":110309,"åĻªéŁ³":110310,"åIJ¬çĿĢ":110311,"æ·±åĪ»çļĦ":110312,"å¿įåıĹ":110313,"çĶµç£ģ":110314,"强èĢħ":110315,"æ»ĭåij³":110316,"æĽ¼èģĶ":110317,"åı¯ä»¥çĽ´æİ¥":110318,"大米":110319,"æŃ·åı²":110320,"æĶ¿åĬ¡æľįåĬ¡":110321,"åħ¬å¼ı":110322,"社群":110323,"éģĵ士èģĮä¸ļ":110324,"ä¹ĭæĥħ":110325,"æµ·æ°´":110326,"æ¼Ķå¥ı":110327,"åºĹéĩĮ":110328,"迹象":110329,"åıijå±ķçIJĨ念":110330,"é«ĺ空":110331,"åij¨åĪĬ":110332,"åĽŀåĪ°äºĨ":110333,"ä¸įéĢĤåIJĪ":110334,"åłµå¡ŀ":110335,"åĬĪ":110336,"æ°´ä¸Ĭ":110337,"çĢijå¸ĥ":110338,"纳ç¨İ人":110339,"çĩĥæ²¹":110340,"å·¥ç¨ĭé¡¹çĽ®":110341,"峡谷":110342,"æľīéĴĪ对æĢ§":110343,"åľĨå½¢":110344,"æľ¬å¸Ĥ":110345,"è¿Ļè¯Ŀ":110346,"管çIJĨèĢħ":110347,"ç¡®è¯ĬçĹħä¾ĭ":110348,"æĬĬæīĭ":110349,"彩èī²":110350,"ä¸Ĭåīį":110351,"夯å®ŀ":110352,"ç¾ĬèĤī":110353,"å¾Ģå¹´":110354,"æĵħèĩª":110355,"迷人":110356,"èĪªæ¯į":110357,"ç²¾ç»Ĩ":110358,"åľ¨æĪijçļĦ":110359,"åĪĽæĬķ":110360,"麦åħĭ":110361,"æľĪç»ı":110362,"åĮĹæµ·":110363,"ä¹ĭæĺŁ":110364,"åı¶åŃIJ":110365,"å¸Ĥåľºç«ŀäºī":110366,"è¿Ļäºĭ":110367,"åıĥèĪĩ":110368,"äº§åľ°":110369,"åĶī":110370,"åķĨåĵģæĪ¿":110371,"èĪªè¿IJ":110372,"ä¼ĺå¼Ĥ":110373,"ä»ĸ们æĺ¯":110374,"éĽ¨æ°´":110375,"è¯įæ±ĩ":110376,"åĨľçĶ°":110377,"欧éĺ³":110378,"çŁŃ线":110379,"管ç½ij":110380,"æł¹åŁº":110381,"åıªæľīä¸Ģ个":110382,"éŀĭåŃIJ":110383,"å¸Ĥå§Ķ书记":110384,"åĪ»æĦı":110385,"è¡Į车":110386,"åıĪ被":110387,"åı¯éĿłæĢ§":110388,"è´±":110389,"ä»»åij½":110390,"åºĶåľ¨":110391,"å°±å¾Ĺ":110392,"æľįåĬ¡ä½ĵç³»":110393,"æĶ¿æĿĥ":110394,"åıijè¨Ģ人":110395,"è¿ĩå¾Ģ":110396,"两åıª":110397,"èĻ½è¯´":110398,"éĢģä¸Ĭ":110399,"ä»Ģä¹Īäºĭ":110400,"æķ£æĸĩ":110401,"æİĮæݧ":110402,"èĸĦå¼±":110403,"ä¸ĭéĿ¢å°±":110404,"主è¦ģåĨħ容":110405,"å¾Īéĩįè¦ģçļĦ":110406,"就说":110407,"çĻ½èī²çļĦ":110408,"éĤ£ä¸ªæŶåĢĻ":110409,"ç»ı纪人":110410,"çļĦæ¯į亲":110411,"ç¬Ķè®°æľ¬":110412,"åºķå±Ĥ":110413,"è¿ij代":110414,"解说":110415,"è²łè²¬":110416,"æľĢ大åĮĸ":110417,"åķĨéĵº":110418,"æł¡åıĭ":110419,"æ²ģ":110420,"ä¸įåĩºæĿ¥":110421,"éĻ·éĺ±":110422,"ç¨ħ":110423,"åħ¬å¸ĥäºĨ":110424,"åĩĢåĢ¼":110425,"çĽ¸å¯¹è¾ĥ":110426,"笼":110427,"æł¸ç®Ĺ":110428,"åįİ侨":110429,"æĢ¥æķij":110430,"æĮºå¥½":110431,"åħĴç«¥":110432,"äºĮèĥİ":110433,"åĩºèĩª":110434,"åĿŁ":110435,"æīĭä¸ĭ":110436,"屡":110437,"åĪĽéĢłæĢ§":110438,"ä¸¥æł¼æĮīçħ§":110439,"åĨįåİ»":110440,"举缣":110441,"人æµģ":110442,"äºĨä¸Ģ声":110443,"å°ıæŶåīį":110444,"è´µæĹı":110445,"éľĸ":110446,"ä¹Łæĺ¯éĿŀ常":110447,"éĢ±":110448,"çľĭäºĨçľĭ":110449,"ç¹ģæ®ĸ":110450,"èĩ³æѤ":110451,"é¢Ħå¤ĩ":110452,"å¾Īæĺİæĺ¾":110453,"æ¼Ķèīº":110454,"åĿIJçĿĢ":110455,"ä¿ĦåĨĽ":110456,"åľ¨è¿ĩåİ»":110457,"ä¹ĭäºĭ":110458,"æĬĵèİ·":110459,"åĿIJä¸ĭ":110460,"çĶ±ä¸ŃåĽ½":110461,"ä¹Łå¼Ģå§ĭ":110462,"çŃĶå¤į":110463,"åŀĥåľ¾åĪĨç±»":110464,"éĴĵé±¼":110465,"åIJĦ種":110466,"缸éģĩ":110467,"ä¸įåģľçļĦ":110468,"æī¹éĩı":110469,"éĩįè¦ģä½ľçĶ¨":110470,"å§Ķå±Ī":110471,"åħŃå¹´":110472,"ä¸ĥåįģ":110473,"ä¹ĭæĪĺ":110474,"é£İéĻ©ç®¡çIJĨ":110475,"éŁ³æ¨Ĥ":110476,"è¡ĮæĶ¿å¤Ħç½ļ":110477,"æľ¬äºĭ":110478,"æĴ°åĨĻ":110479,"èģļåIJĪ":110480,"éĢĤæŶ":110481,"æIJ¬å®¶":110482,"ç¢İçīĩ":110483,"çĽĽå®´":110484,"ç®Ģæ´ģ":110485,"åı¬éĽĨ":110486,"ç®ĢåĮĸ":110487,"åĮĹ京æŶéĹ´":110488,"第ä¸īå±Ĭ":110489,"æĿ¥åĽŀ":110490,"常çĶ¨çļĦ":110491,"京津":110492,"京津åĨĢ":110493,"梦幻":110494,"è¯ķè¡Į":110495,"æľºåºĬ":110496,"åĪ°æľĢåIJİ":110497,"åĬ©æīĭ":110498,"åĪĨ彩":110499,"åĩºåĵģ":110500,"åĪ¹è½¦":110501,"åIJ¯åıij":110502,"侧éĿ¢":110503,"æ¯ıå½ĵ":110504,"缸åħ³è§Ħå®ļ":110505,"ä¸ĸ人":110506,"è´Ń车":110507,"å¿ĥ缮":110508,"å¿ĥ缮ä¸Ń":110509,"äºĶéĩij":110510,"è¿ĺè®°å¾Ĺ":110511,"ä¾ĿçĦ¶æĺ¯":110512,"æıIJæ¡Ī":110513,"çĶµåķĨå¹³åı°":110514,"åģļåĪ°äºĨ":110515,"æĿľç»Ŀ":110516,"å®īåįĵ":110517,"ä¸ĸçķĮåIJĦåľ°":110518,"åīįéĢĶ":110519,"æ´ĹåĩĢ":110520,"å¥ĭåĬĽ":110521,"åŁİå¸Ĥ建设":110522,"å¤ļåĬŁèĥ½":110523,"ä¼ļéĢłæĪIJ":110524,"åıijå¸ĥä¼ļä¸Ĭ":110525,"究竣æĺ¯":110526,"åĪĨ红":110527,"çŁ¥èŃĺ":110528,"éĿ¢æĿ¿":110529,"æĹłå£°":110530,"æĢ¥éľĢ":110531,"å¤±çľł":110532,"çĪ¸å¦Ī":110533,"äºĤ":110534,"åħ¨æĻ¯":110535,"ç»ıåħ¸çļĦ":110536,"åī§ä¸Ń":110537,"é¢Ĩ导ä¸ĭ":110538,"åħļåĨħ":110539,"åħ¥ä¾µ":110540,"æĭīæĸ¯":110541,"ä¸Ģå¹ķ":110542,"åĬłä¹ĭ":110543,"èĤĨ":110544,"èĭ±æł¼":110545,"èĭ±æł¼åħ°":110546,"å·§åħĭ":110547,"å·§åħĭåĬĽ":110548,"ä¸Ģå¿ĥ":110549,"èģĤ":110550,"å¾Ģå¾Ģæĺ¯":110551,"管çIJĨå±Ĥ":110552,"çĻ»åħ¥":110553,"建ç«ĭèµ·":110554,"å»ºåĽ½":110555,"åŃIJ宫":110556,"åºĶä»ĺ":110557,"æݢ究":110558,"第ä¸Ģä½į":110559,"ä½Ļ家":110560,"çŃīæ´»åĬ¨":110561,"æīĢèĩ´":110562,"è¾ĥå¿«":110563,"æĺ¯éĿŀ":110564,"æıIJåIJį":110565,"äºĮèĢħ":110566,"åıªåī©ä¸ĭ":110567,"åħ¶ä¸ŃåĮħæĭ¬":110568,"ç¼ĸç¨ĭ":110569,"çł´ç¢İ":110570,"ä¸Ń举":110571,"å·¥ä½ľæĬ¥åijĬ":110572,"çŃ¾åIJį":110573,"éħĴä¸ļ":110574,"çŁ¥æĻĵ":110575,"çĥŃå¿ĥ":110576,"éĿŀåĩ¡":110577,"èIJ¥ä¸ļæī§":110578,"èIJ¥ä¸ļæī§çħ§":110579,"人大代表":110580,"ä¸Ģ个æĸ°çļĦ":110581,"å¨ģæµ·":110582,"éĤ£äºº":110583,"涨价":110584,"æ¶ĪçģŃ":110585,"éļ¾å¿ĺ":110586,"ç¶ĵé©Ĺ":110587,"åı£è¢ĭ":110588,"ç³»æķ°":110589,"æĸĩä¸Ń":110590,"好转":110591,"æĸ°éĽ¶åĶ®":110592,"讲述äºĨ":110593,"å¼ĢçĽĺ":110594,"çķĻç»Ļ":110595,"æħ¢æħ¢çļĦ":110596,"æĤ²ä¼¤":110597,"æľ¬æľŁ":110598,"äºĨå¤ļå°ij":110599,"è¿Ļ让":110600,"åIJĮçŃī":110601,"æ¸ħæĺİ":110602,"个åŁİå¸Ĥ":110603,"æºĸåĤĻ":110604,"åĩłä¹İæĺ¯":110605,"强åĬĽ":110606,"俯":110607,"水稻":110608,"åĽºå®ļçļĦ":110609,"æł¸åĩĨ":110610,"说æľį":110611,"顯示":110612,"è¿Ļå¥Ĺ":110613,"æĻºæħ§åŁİå¸Ĥ":110614,"å±ĭ顶":110615,"ä¸įæĿ¥":110616,"çĶŁé²ľ":110617,"çŁ¥æĥħ":110618,"æĬķ身":110619,"åijĬè¯īæĪij们":110620,"ä¸īåĽĽ":110621,"ä¸ĩä¸Ģ":110622,"è¾Ĩ车":110623,"为ä¹ĭ":110624,"åĪ°æŶåĢĻ":110625,"è¿Ļæīįæĺ¯":110626,"åIJįçīĮ":110627,"åºŁæ°´":110628,"åݻ年åIJĮæľŁ":110629,"å¹´éĻIJ":110630,"éģĭåĭķ":110631,"åıĮçľ¼":110632,"è¦ģç´§":110633,"对çŃĸ":110634,"åľºé¦Ĩ":110635,"çĻ¾ç§ij":110636,"è¶Ĭéĩİ":110637,"å¯ĮåIJ«":110638,"大å¤ļæķ°äºº":110639,"æľĢå°ij":110640,"åı¬åĶ¤":110641,"åħ¸èĮĥ":110642,"åĨľæľº":110643,"æŃ£æĸĩ":110644,"åºĶçĶ¨äºİ":110645,"æ·±èĢķ":110646,"ä¿Ń":110647,"ä»Ģä¹Īä¸ľè¥¿":110648,"å¥Ĺé¤IJ":110649,"å½ĵéĢī":110650,"å·¦æīĭ":110651,"è°ĥçIJĨ":110652,"æĻļé¤IJ":110653,"éļ¾åħ³":110654,"åĩŃè¯ģ":110655,"çĪ±äºº":110656,"æĮĩè´£":110657,"è´£ç¼ĸ":110658,"çļĦä¸Ģ款":110659,"éĵ²":110660,"åįģ个":110661,"èĢ»":110662,"æľįåĬ¡åķĨ":110663,"åľ°çĭ±":110664,"è¿ŀå¿Ļ":110665,"åĽ°æĥij":110666,"çļĵ":110667,"ä¸įåIJĥ":110668,"çİ°åľ¨å·²ç»ı":110669,"çĽĺçĤ¹":110670,"ä¸įåģľåľ°":110671,"管çIJĨ模å¼ı":110672,"è¿Ļ段æŶéĹ´":110673,"椰":110674,"礼åĮħ":110675,"æµģ转":110676,"æī«çłģ":110677,"éĽĨä¸Ńåľ¨":110678,"æ±ĤåĬ©":110679,"åįĬ个":110680,"å¿«éĢŁå¢ŀéķ¿":110681,"å¾Ģä¸ĭ":110682,"è¯ĦåĪĨ":110683,"å°±æĥ³":110684,"åķĨåĬ¡éĥ¨":110685,"æľīéĹ®é¢ĺ":110686,"èİ·åĪ©":110687,"æ¯ĽçĹħ":110688,"æĦŁåºĶ":110689,"èī¯æĢ§":110690,"åĪĨæѧ":110691,"åĨī":110692,"æĪij们çİ°åľ¨":110693,"è¦ģåĬłå¼º":110694,"å·§å¦Ļ":110695,"èŀºæĹĭ":110696,"åĪĩæį¢":110697,"çĭĦ":110698,"顺çķħ":110699,"å°¤åħ¶æĺ¯åľ¨":110700,"èĬĿ麻":110701,"éļ¾è¿ĩ":110702,"æĹĹå¸ľ":110703,"å¤įåį°":110704,"å¤įåį°ä»¶":110705,"å¿ħéľĢ":110706,"对å¤ĸå¼ĢæĶ¾":110707,"éļ¾åıĹ":110708,"åİŁæĿ¥æĺ¯":110709,"ç®ĹäºĨ":110710,"é«ĺå±±":110711,"离èģĮ":110712,"çµĦç¹":110713,"çµĦç¹Ķ":110714,"å±ģèĤ¡":110715,"çĻ¾å®¶":110716,"éģĩä¸Ĭ":110717,"æĺĶæĹ¥":110718,"ä¸į容":110719,"çĽij管éĥ¨éŨ":110720,"主æĦı":110721,"æµģåŁŁ":110722,"è·Įå¹ħ":110723,"èĩ³ä¸Ĭ":110724,"åĪ«è¯´":110725,"æĺ¯æ¯Ķè¾ĥ":110726,"å®ıè§Ĥç»ıæµİ":110727,"å¸Ĥåľºä¸»ä½ĵ":110728,"污æŁĵçī©":110729,"æķijæ²»":110730,"丰æĶ¶":110731,"åŃĺæĶ¾":110732,"åĩĦ":110733,"éĩijå±±":110734,"æį¢äºĨ":110735,"ä¸ĵ人":110736,"éĹľæĸ¼":110737,"æĹ¢è¦ģ":110738,"åĽ½è¶³":110739,"éļĭ":110740,"åıįåĩ»":110741,"起身":110742,"åħĪæĺ¯":110743,"å¸ĮæľĽèĥ½å¤Ł":110744,"åĪ¶è®¢":110745,"åºĹéĿ¢":110746,"åĸĢ":110747,"æķĻä½ł":110748,"éĻį温":110749,"åĬĽæ±Ĥ":110750,"ä¸īçĻ¾":110751,"çī©ä»·":110752,"丢失":110753,"å¢Ļä¸Ĭ":110754,"éĥ¨ä»½":110755,"æł·æĿ¿":110756,"ä¹ĭæĦı":110757,"ç½ijå°ıç¼ĸ":110758,"ä¸ĸä¸Ĭ":110759,"è°ĥè¯ķ":110760,"污æŁĵéĺ²æ²»":110761,"å½±éĻ¢":110762,"å®Įåħ¨åı¯ä»¥":110763,"éĢļåħ³":110764,"ä¹īåĬ¡æķĻèĤ²":110765,"没æľīåĬŀæ³ķ":110766,"èĢ¿":110767,"妳":110768,"æĹłæĥħ":110769,"å¾ĹçĽĬ":110770,"å¾ĹçĽĬäºİ":110771,"æľŁçĽ¼":110772,"娱ä¹IJåľº":110773,"çĶ²æĸ¹":110774,"ä¸Ģæ±½":110775,"çĹ°":110776,"çĸijä¼¼":110777,"æĸ°æµªå¾®åįļ":110778,"强è¡Į":110779,"å½ĵä»ĸ":110780,"èĥº":110781,"çĶ¨æĪ·æıIJä¾Ľ":110782,"åĮºå§Ķ":110783,"æĦ¿æĻ¯":110784,"æĬĺæī£":110785,"失踪":110786,"è¿«åĪĩ":110787,"åŃĹæ¯į":110788,"åĴ¯":110789,"èªįèŃĺ":110790,"ä»Ģä¹ĪæĦıæĢĿ":110791,"çĽĴåŃIJ":110792,"å½ķéŁ³":110793,"建设工ç¨ĭ":110794,"ä¸ļä½Ļ":110795,"å®ŀ践活åĬ¨":110796,"çľŁç©º":110797,"çĤĸ":110798,"åľ¨è·¯ä¸Ĭ":110799,"主è¦ģåĮħæĭ¬":110800,"该æĢİä¹Ī":110801,"æĢ»æľī":110802,"æĢ§æĦŁ":110803,"æ°ijèĪª":110804,"å¼ĢåºĹ":110805,"欺éªĹ":110806,"çªģåĩ»":110807,"缺失":110808,"æī§ä¸ļ":110809,"åľ°éģĵ":110810,"并æĹł":110811,"æ°ijåĬŀ":110812,"ç»Ħç»ĩçĶŁæ´»":110813,"æĪijå¦Ī":110814,"è¨ĺèĢħ":110815,"管åĪ¶":110816,"æī¾ä¸ª":110817,"èĹ»":110818,"çĤİçĹĩ":110819,"äºĴåĬ©":110820,"æµıè§ĪåĻ¨":110821,"çݩ家æĿ¥è¯´":110822,"éĻįä½İäºĨ":110823,"è£Ķ":110824,"æĮ£éĴ±":110825,"åķĨæľº":110826,"æĶ¹è£ħ":110827,"æµģ浪":110828,"æĶ¿æ³ķ":110829,"èĢģ头":110830,"çĶŁäº§åĴĮ":110831,"ç©Ĺ":110832,"亲çĪ±":110833,"亲çĪ±çļĦ":110834,"å±¥èģĮ":110835,"åŁİéĩĮ":110836,"ç»ĨåĪĨ":110837,"åĬ³åĬ¨åIJĪåIJĮ":110838,"åľ¨æĹ¥æľ¬":110839,"å¨ģå°Ķ":110840,"åį«è§Ĩ":110841,"éĢ£çµIJ":110842,"çĿĢéĩį":110843,"æĬĺ磨":110844,"åĽ¾ä¸º":110845,"çľ·":110846,"å·¥åºı":110847,"æĵģ":110848,"æĵģæľī":110849,"ç½ijç«Ļåľ°åĽ¾":110850,"çļĦä¸Ģ大":110851,"ç»Ħç»ĩå®ŀæĸ½":110852,"æĬĽå¼ĥ":110853,"åĴĮæĶ¯æĮģ":110854,"æ³ķåĪĻ":110855,"浪潮":110856,"çİ°æľīçļĦ":110857,"åĩłçİĩ":110858,"为客æĪ·":110859,"åįģä¸ĩ":110860,"è¹Ħ":110861,"çªģåĩºéĹ®é¢ĺ":110862,"åıĥåĬł":110863,"éĥ½ä¼ļæľī":110864,"缤":110865,"è°ģéĥ½":110866,"æīĭåĬ¨":110867,"çĽ´è¾¾":110868,"çĤ¹å¤ļ":110869,"éĺ¶å±Ĥ":110870,"ä¸įä½³":110871,"éĤ£æ®µ":110872,"滨海":110873,"æĺ¯åĽ½åĨħ":110874,"æĪijå¸ĮæľĽ":110875,"åIJĽåŃIJ":110876,"è§ĤéŁ³":110877,"åģļé¥Ń":110878,"æ±½è»Ĭ":110879,"åħ³ç¨İ":110880,"çľ¼åīįçļĦ":110881,"æ°´éĿ¢":110882,"èĢ³æľº":110883,"追踪":110884,"æݨéĢģ":110885,"éĴ±åĮħ":110886,"æģ¶å¿ĥ":110887,"æµ·åŁŁ":110888,"å·į":110889,"å¼ĢæĿ¥":110890,"表æĢģ":110891,"仪表":110892,"å¹³åİŁ":110893,"åįģå¤ļå¹´":110894,"ä¹ŁæĹłæ³ķ":110895,"åħ¼é¡¾":110896,"è¡£æŁľ":110897,"æł½åŁ¹":110898,"æĪ¿æºIJ":110899,"设ç«ĭäºĨ":110900,"ä¸ĩåIJį":110901,"æķ°é¢Ŀ":110902,"è¦ģåĿļæĮģ":110903,"åIJīæŀĹçľģ":110904,"请èģĶç³»":110905,"ç»ıåİĨè¿ĩ":110906,"çļĦæľ¬è´¨":110907,"åħ¥éŨ":110908,"æľ¬æ¡Ī":110909,"çİĩè¾¾åĪ°":110910,"åı°éĺ¶":110911,"éĴŀ":110912,"æĪijèĥ½":110913,"èݲèĬ±":110914,"éĴł":110915,"ä¸Ģäºĭ":110916,"åİŁæľīçļĦ":110917,"æ¯ıåĢĭ":110918,"æ¯Ķäºļ迪":110919,"æ£ĭçīĮ游æĪı":110920,"ä¸įä¼ļæľī":110921,"å½ĴæĿ¥":110922,"äºĶçĻ¾":110923,"è¿ĩé«ĺ":110924,"éĽ·è¾¾":110925,"ä¸Ģèµ·åİ»":110926,"æķĻ导":110927,"å°±è¯Ĭ":110928,"å°±å¾Ī":110929,"ä¸įåIJĮäºİ":110930,"俺":110931,"å¸ĸåŃIJ":110932,"æĶ¿åįıå§Ķåijĺ":110933,"çĸ«æĥħå½±åĵį":110934,"åĪĨè£Ĥ":110935,"为ä»Ģä¹Īä¼ļ":110936,"äºĶæĺŁ":110937,"å°ijåĦ¿":110938,"æĬ¢éĻ©":110939,"梦è§ģ":110940,"è®°èĢħéĩĩ访":110941,"山路":110942,"æĪij个人":110943,"æ²Ļ滩":110944,"è¹Ń":110945,"æĶ¹è®Ĭ":110946,"æĸ°åŀĭåĨł":110947,"æĸ°åŀĭåĨłçĬ¶":110948,"åĮ»æĬ¤":110949,"åĮ»æĬ¤äººåijĺ":110950,"æµ·å°Ķ":110951,"åħ³äºİæĪij们":110952,"éĻ¤å¤ĸ":110953,"åºļ":110954,"宣åijĬ":110955,"ä¸īåįĥ":110956,"榨":110957,"ç§ijæĬĢ大åѦ":110958,"ä¸ĥåħ«":110959,"顺åºĶ":110960,"çĪ¸çĪ¸å¦Īå¦Ī":110961,"éĢīåıĸ":110962,"åī§çĥĪ":110963,"乡æĿijæĹħ游":110964,"积æŀģæݢ索":110965,"表çݰ为":110966,"å¾Īæ¸ħæ¥ļ":110967,"大åĨĽ":110968,"æĿ¥çĶµ":110969,"å¥ĹæĪ¿":110970,"çİ°è¡Į":110971,"享åıĹåĪ°":110972,"çľĭçĤ¹":110973,"åĽºå®ļèµĦ产":110974,"以人为":110975,"ä»¥äººä¸ºæľ¬":110976,"ä¸įå®Į":110977,"éĻį鼨":110978,"åģļçļĦäºĭæĥħ":110979,"并äºİ":110980,"顽强":110981,"èĢ¸":110982,"åĺ´å·´":110983,"缸åħ³ä¿¡æģ¯":110984,"æĪij没":110985,"æĪĺçķ¥æĢ§":110986,"æĢĿ念":110987,"åĪĺå¤ĩ":110988,"åĬ©æĶ»":110989,"é£İè²Į":110990,"éĿ¢å¯¹éĿ¢":110991,"积æŀģå¼Ģå±ķ":110992,"çĸĹæķĪ":110993,"çľĭ书":110994,"缺åı£":110995,"åĽ½æ°ijç»ıæµİ":110996,"使çĶ¨æĿĥ":110997,"éģ¥è¿ľ":110998,"å¡«è¡¥":110999,"第ä¸ī人":111000,"åįĬå¤ľ":111001,"æѦæ±īå¸Ĥ":111002,"æĪijåıijçİ°":111003,"ä¼ĺæĥłæĶ¿çŃĸ":111004,"é£İåı£":111005,"å°±ä¸įèĥ½":111006,"为主è¦ģ":111007,"æµģåĩº":111008,"å´ĩæĭľ":111009,"并ä¸įèĥ½":111010,"é«ĺä¸ī":111011,"ä¸ĸçķĮä¸ĬæľĢ":111012,"æĥ³å¿ħ":111013,"åħ¶æīĢ":111014,"åĢĻéĢī":111015,"åĢĻéĢī人":111016,"ä¸įçĪ±":111017,"åī¯ä½ľçĶ¨":111018,"人æ°ijæĹ¥æĬ¥":111019,"æĪijä¸įæĺ¯":111020,"å®ŀçī©":111021,"çĶµåİĤ":111022,"ä¹Łç®Ĺæĺ¯":111023,"æľīéĹľ":111024,"æľīèĥ½åĬĽ":111025,"æĮĤåľ¨":111026,"çľ¼ä¸ĭ":111027,"约翰":111028,"å°ıåѦçĶŁ":111029,"èµ·åĪ°äºĨ":111030,"工夫":111031,"åIJĮå¿ĥ":111032,"åĿ¦è¨Ģ":111033,"çłĮ":111034,"åıijæĮ¥äºĨ":111035,"èģĮä¸ļéģĵå¾·":111036,"è¿ĻäºĽå¹´":111037,"念头":111038,"èĢģé¼ł":111039,"åħ¨èµĦ":111040,"åħ¨èµĦåŃIJ":111041,"ä¸Ģåij³":111042,"å¤ļä¸ĩåħĥ":111043,"æł¼æľĥ":111044,"éķ¿éĢĶ":111045,"带走":111046,"èĭ±å¯¸":111047,"æĸĩä½ĵ":111048,"对ä»ĸ们":111049,"åĵŃäºĨ":111050,"å¡«æĬ¥":111051,"çīĪæĿĥ声æĺİ":111052,"çĶµçº¿":111053,"è´Ńçī©ä¸Ńå¿ĥ":111054,"饱满":111055,"ä½İ头":111056,"强迫":111057,"ä¿Ŀæ´ģ":111058,"欧åĨł":111059,"缸è¿ŀ":111060,"认è´Ń":111061,"çģ«æĺŁ":111062,"é«ĺå°Ķ":111063,"é«ĺå°Ķ夫":111064,"èij«èĬ¦":111065,"æłĩ注":111066,"çļĦçIJĨæĥ³":111067,"æł¸éħ¸":111068,"æł¸éħ¸æ£Ģæµĭ":111069,"åĬī":111070,"ä¸ĢèĪ¬æĺ¯":111071,"æĢĿç´¢":111072,"轨迹":111073,"çĥŃ带":111074,"éĻ£":111075,"åĩĨç¡®æĢ§":111076,"æĪ´çĿĢ":111077,"åľ¨çĶŁæ´»ä¸Ń":111078,"æīĢèĥ½":111079,"æľ¯åIJİ":111080,"å¸¦ä½ł":111081,"ç¥ł":111082,"æ®ĭéħ·":111083,"ä¹Łåıªæĺ¯":111084,"çĶ³è´Ń":111085,"举åĬŀäºĨ":111086,"æľīæĦıä¹ī":111087,"æĹºçĽĽ":111088,"åľ¨ç¶²":111089,"åľ¨ç¶²è·¯ä¸Ĭ":111090,"å¾Ī大ç¨ĭ度":111091,"管è¾ĸ":111092,"çĸ«æĥħæľŁéĹ´":111093,"触æij¸":111094,"éĺ¶æ®µæĢ§":111095,"ä¼ļè§īå¾Ĺ":111096,"çļĦçĶ»éĿ¢":111097,"æİ¥åıĹäºĨ":111098,"表达äºĨ":111099,"éĤĵå°ı":111100,"éĤĵå°ıå¹³":111101,"åħļé£İ":111102,"åħļé£İå»īæĶ¿":111103,"åķĨåѦéĻ¢":111104,"åħijæį¢":111105,"é£Łåĵģèį¯åĵģ":111106,"éĿŀ常好çļĦ":111107,"çľ¯":111108,"纳米":111109,"åĬ¨æijĩ":111110,"åĽŀéģ¿":111111,"çľĭèijĹ":111112,"款项":111113,"åħ«å¹´":111114,"åģļ个":111115,"æĸĩæ¡£":111116,"éĩijèŀįç§ijæĬĢ":111117,"åħ¶ä¸Ńæľī":111118,"äºĨä¸Ģç³»åĪĹ":111119,"æĹĹèĪ°åºĹ":111120,"称èµŀ":111121,"éĽ¢éĸĭ":111122,"åĪ¶åĨ·":111123,"家éŨåı£":111124,"åįģå¤ļ":111125,"ä¼´ä¾£":111126,"çľĭçĹħ":111127,"æĭīçĿĢ":111128,"æīĴ":111129,"çĸ²æĥ«":111130,"å°ijæķ°æ°ijæĹı":111131,"åĽ¾å½¢":111132,"轧":111133,"å¢ŀéĩı":111134,"饲åħ»":111135,"çģ«å±±":111136,"æ¯ı个æľĪ":111137,"ä½ľä¸ºä¸ĢåIJį":111138,"è½´æī¿":111139,"æĸĩ书":111140,"ç¼ķ":111141,"åħ·ä½ĵæĥħåĨµ":111142,"çĹĽçĤ¹":111143,"缴éĶĢ":111144,"å¡Ĭ":111145,"ä¹Łæľĥ":111146,"çĥŃæ½®":111147,"å¹³æ°ij":111148,"æ¼ĶåĶ±ä¼ļ":111149,"æķĻçłĶ":111150,"éĢĥéģ¿":111151,"ä¸Ģè´¯":111152,"å°±è¶Ĭ":111153,"å®ŀå®ŀåľ¨":111154,"å®ŀå®ŀåľ¨åľ¨":111155,"ä¹łè¿ijå¹³æĢ»":111156,"溺":111157,"å¿ĥåºķ":111158,"éķ¿å¾ģ":111159,"媽媽":111160,"第ä¸ī次":111161,"åĩºæ¼Ķ":111162,"çĭĢæ³ģ":111163,"å°Ķæĸ¯":111164,"代çIJĨåķĨ":111165,"çĨı":111166,"çļĦ对象":111167,"çĶµéĩı":111168,"è¡ĮåĪĹ":111169,"åĽ½äºº":111170,"è·ijäºĨ":111171,"åįĶåĬ©":111172,"èIJ¥è¿IJ":111173,"å¸ĪåħĦ":111174,"榮":111175,"æĥ³åĥı":111176,"æĢ§å¼º":111177,"ç§ijåѦçłĶ究":111178,"延å®ī":111179,"ä¸¥æł¼èIJ½å®ŀ":111180,"é¢Ĩä¼ļ":111181,"çĽ¸å·®":111182,"路人":111183,"çĶ«":111184,"æľīä»·åĢ¼":111185,"æľīä»·åĢ¼çļĦ":111186,"ç¾İåĽ¢":111187,"æ°ij主çĶŁæ´»":111188,"æĪijæīį":111189,"ç¾İåĽ½äºº":111190,"æ°Ķåij³":111191,"åıįå°Ħ":111192,"çļĦåĨ³å¿ĥ":111193,"大è±Ĩ":111194,"交代":111195,"è¿Ľåĩº":111196,"åıįæĬĹ":111197,"æĮĩçļĦæĺ¯":111198,"ä»·ä½į":111199,"è¿Ľé©»":111200,"ä¸ĬçĻ¾":111201,"ä½įåĪĹ":111202,"ä¸ŃåĽ½ä¼ģä¸ļ":111203,"çļĦ好å¤Ħ":111204,"主ç¼ĸ":111205,"汽油":111206,"ä½ĨæĪij们":111207,"æĢİä¹Īçľĭ":111208,"é»Ħå±±":111209,"å¤ļåªĴä½ĵ":111210,"åIJİåį«":111211,"èİ·å¾ĹæĽ´å¤ļ":111212,"åĬ¡å¿ħ":111213,"为å¥ijæľº":111214,"é¦ĸ饰":111215,"ä¸ĩåįļ":111216,"è¶ĬæĿ¥è¶Ĭ大":111217,"ä¸ĵ项è¡ĮåĬ¨":111218,"å¥ĭè¿Ľ":111219,"ä»įçĦ¶æĺ¯":111220,"è´¨æĦŁ":111221,"å¦Ĥæŀľä¸įæĺ¯":111222,"ç«Ļèµ·æĿ¥":111223,"ä¹¾éļĨ":111224,"åı¯æĢķçļĦ":111225,"å¯Įè´µ":111226,"æ¸ħç®Ĺ":111227,"åIJijä¸ĭ":111228,"åĢļ":111229,"çļĦçŃĶæ¡Ī":111230,"èĪ¹ä¸Ĭ":111231,"çļĦ羣å®ŀæĢ§":111232,"çŃīåĬŁèĥ½":111233,"åĸľåī§":111234,"å¨ģåĬĽ":111235,"æĸ°é¢ĸ":111236,"æł¸çĶµ":111237,"æĬ¥éĶĢ":111238,"æķħ乡":111239,"ä¼´éļı":111240,"éŀŃ":111241,"å¦Ĭå¨ł":111242,"åĪĨåĮĸ":111243,"æľīå¾Ī大":111244,"æĢİä¹Ī说":111245,"æĻĤ代":111246,"产åĩº":111247,"ä»ĭç»į说":111248,"å¤ĦçIJĨåĻ¨":111249,"èĨ¨èĥĢ":111250,"åī¯å¸Ĥéķ¿":111251,"çļĦ妻åŃIJ":111252,"æł·åĵģ":111253,"åIJĮæ¯Ķä¸ĭéĻį":111254,"åħĥå·¦åı³":111255,"çĶ¨èĩªå·±çļĦ":111256,"é«ĺéĽĦ":111257,"æĺ¥æĻļ":111258,"ä¹Łæľīå¾Īå¤ļ":111259,"çľ¼çIJĥ":111260,"æķ£æŃ¥":111261,"ä»ĸ们éĥ½":111262,"第ä¸Ģ家":111263,"åĬŀ好":111264,"å®īéĺ²":111265,"ä¸Ģä¸ĩ":111266,"åľ¨éĩĮéĿ¢":111267,"éŁ³é¢ij":111268,"åı£åı·":111269,"ä¸Ģè¶Ł":111270,"ç¦ıçī¹":111271,"é³ŀ":111272,"æĥĬèī³":111273,"æĸ°å¨ĺ":111274,"绿èī²åıijå±ķ":111275,"ä¸Ńå¼ı":111276,"ä¹Łåıªæľī":111277,"çݰ身":111278,"åı¯ä¾Ľ":111279,"æ¯ıä¸Ģ个人":111280,"第ä¸īèĢħ":111281,"åľ°å½¢":111282,"éĴ¢ç»ĵæŀĦ":111283,"çĽijçĿ£æ£ĢæŁ¥":111284,"åı«æĪij":111285,"èĩ´æķ¬":111286,"æ´Ĺæīĭ":111287,"ä¸ĭè°ĥ":111288,"康çĨĻ":111289,"æĪIJ交éĩı":111290,"ä¹ŁæĪIJ为":111291,"åħīæ»ij":111292,"å®Įæķ´æĢ§":111293,"çģ¼":111294,"網éłģ":111295,"éķ¿å¯¿":111296,"éģ©çĶ¨":111297,"çļĦä¸Ģ项":111298,"çŀ©çĽ®":111299,"æĬĬèĩªå·±çļĦ":111300,"éĵ¶è¡Įåį¡":111301,"å°±å¿ħé¡»":111302,"ç¾İçĻ½":111303,"éŀįå±±":111304,"æľ¬é¢Ĩ":111305,"ä¸Ģç¢Ĺ":111306,"æīĵæ³ķ":111307,"æĤ¨å¥½":111308,"对åŃ©åŃIJ":111309,"æĬ¥éģĵ称":111310,"ä¼łåĩº":111311,"大èĩ£":111312,"ç¬ĭ":111313,"çĽı":111314,"é¾ļ":111315,"çĽ´çº¿":111316,"æĻºåºĵ":111317,"ç§Łè½¦":111318,"é£İåij³":111319,"çľĭä¸Ģä¸ĭ":111320,"æݨéĶĢ":111321,"éĥ¨éĥ¨éķ¿":111322,"è´¨éĩıåĴĮ":111323,"åĪĬçĻ»":111324,"å·¥ä¸ļåĮĸ":111325,"çİĩ为":111326,"鼶件":111327,"硬åĮĸ":111328,"ä¸Ĭåįĥ":111329,"ç»ıéªĮåĢ¼":111330,"å¹³è¡Į":111331,"声éģĵ":111332,"æľįåĬ¡è´¨éĩı":111333,"çĶŁçĶ¢":111334,"æľĢ容æĺĵ":111335,"ä¸Ģæŀļ":111336,"å¹´æĬ¥":111337,"åħ¬ç½ij":111338,"åħ¬ç½ijå®ī":111339,"åħ¬ç½ijå®īå¤ĩ":111340,"çļĦèĥ½éĩı":111341,"å®ŀéĻħè¡ĮåĬ¨":111342,"è¦ģä¸įè¦ģ":111343,"æĹ¥æľ¬äºº":111344,"èĢ¶ç¨£":111345,"ç¼ĸåī§":111346,"涩":111347,"åį°å°¼":111348,"ä¸Ĭä¸ĭ游":111349,"åĩłåı¥":111350,"ä¸Ńéĵģ":111351,"ç°¡åĸ®":111352,"èĩªå¸¦":111353,"çĶŁäºİ":111354,"ä¸Ģåı£æ°Ķ":111355,"åĭ¤å¥ĭ":111356,"éĻįä»·":111357,"å±ķçİ°äºĨ":111358,"å¸ĥæĭī":111359,"ä¼ļéĢīæĭ©":111360,"çļĦç»ıåħ¸":111361,"好æľĭåıĭ":111362,"车éģĵ":111363,"æķ´åĢĭ":111364,"åľĵ":111365,"éķ¿æľŁä»¥æĿ¥":111366,"æĬķå½±":111367,"çļĩåĨł":111368,"è¿ĩ大":111369,"åijĬè¯īä»ĸ":111370,"ä¼ģä¸ļæıIJä¾Ľ":111371,"æĬ½è±¡":111372,"éĢĤ度":111373,"çļĦ女åŃ©":111374,"èµ·ä¼ı":111375,"çļĦåĬŁæķĪ":111376,"ä¸ĵ项æķ´æ²»":111377,"åı¯éĢļè¿ĩ":111378,"ä¸įåIJĮç¨ĭ度":111379,"å¼Ĥè®®":111380,"åĩĢèµĦ产":111381,"åijĹ":111382,"ä»Ģä¹Īåij¢":111383,"å·¡éĢ»":111384,"è¸ıä¸Ĭ":111385,"ä½Ĩå®ĥ":111386,"精度":111387,"管å±Ģ":111388,"第ä¸ĢåIJį":111389,"åĨħåŃĺ":111390,"æijĨåľ¨":111391,"åī©ä¸ĭ":111392,"主ä½ĵ责任":111393,"çĤ¹åįĬ":111394,"以èĩ³äºİ":111395,"åħ»èĢģä¿ĿéĻ©":111396,"æĦŁåıĹåĪ°äºĨ":111397,"çŁ¥åIJįçļĦ":111398,"å¯Į豪":111399,"妥åĸĦ":111400,"åŃĻåŃIJ":111401,"éĵĤ":111402,"说èĩªå·±":111403,"让æĤ¨":111404,"æķ°æݧ":111405,"çļĦçľ¼åħī":111406,"注éĶĢ":111407,"çļĦçģµéŃĤ":111408,"è¿ĺä¸įéĶĻ":111409,"éĹ®ä»ĸ":111410,"èĩªä¸»çłĶåıij":111411,"èĵĭ":111412,"ç´«èī²":111413,"åĽ½å®¶å®īåħ¨":111414,"è¾½å®ģçľģ":111415,"ä¹Łæ¯Ķè¾ĥ":111416,"ç¾İèĤ¡":111417,"ä¸įç¡®å®ļæĢ§":111418,"å¿ĥ头":111419,"æĪ³":111420,"级åĪ«çļĦ":111421,"论述":111422,"çļĦåĽŀçŃĶ":111423,"ä¿Ŀè¯ģéĩij":111424,"çŃīè¡Įä¸ļ":111425,"幸ç¦ıæĦŁ":111426,"æѧè§Ĩ":111427,"æľºç¥¨":111428,"派人":111429,"èĩ´åij½":111430,"åĺ´è§Ĵ":111431,"æĸ°éĹ»ä¸Ńå¿ĥ":111432,"æĶ¾å¼ĥäºĨ":111433,"å®ľå±ħ":111434,"åĨĻä¸ĭ":111435,"éĹ®çŃĶ":111436,"è¿ĻéĩĮæĺ¯":111437,"å¤ļåľ°":111438,"åĮºåŁŁåĨħ":111439,"åīµæĸ°":111440,"çľĭä»ĸ":111441,"æī§æ³ķ人åijĺ":111442,"åĬ¨æľº":111443,"éŁ³åĵį":111444,"çļĦåij½è¿IJ":111445,"顶éĥ¨":111446,"åĵŁ":111447,"éĥ½æľĥ":111448,"æīĵéĢłæĪIJ":111449,"æĦıåĽ¾":111450,"çļĸ":111451,"åĢĴåħ¥":111452,"å·´èIJ¨":111453,"åĬ©åѦ":111454,"å¤įåı¤":111455,"åIJ¯çĶ¨":111456,"åĽ½éĻħå¸Ĥåľº":111457,"åĤ¨èĥ½":111458,"é»ijé¾Ļæ±Łçľģ":111459,"ä¹ĺ车":111460,"è¿IJåĬ¨ä¼ļ":111461,"ä¿ĿåĪ©":111462,"çŁ³æĿIJ":111463,"çµ®":111464,"çĤĴä½ľ":111465,"çļĦä¿¡ä»»":111466,"å°±æĪIJäºĨ":111467,"åı¯è§Ĥ":111468,"çļĩä¸Ĭ":111469,"è¿Ļåĩłå¤©":111470,"ä¸ĢéĶ®":111471,"åĨ·åĨ»":111472,"ä¿Ŀåį«":111473,"æł¸æ¡ĥ":111474,"åIJĪä½ľåħ³ç³»":111475,"éĢģåĩº":111476,"æĹĹä¸ĭçļĦ":111477,"åľ¨ä¹İ":111478,"为广大":111479,"åįĪé¤IJ":111480,"ä¸ĵ访":111481,"æĪĸå°Ĩ":111482,"éĿĴå²Ľå¸Ĥ":111483,"å¥Ķè·ij":111484,"æĹ¥æĬ¥éģĵ":111485,"å¥ijåIJĪ":111486,"æĸ°æĺ¥":111487,"ä¸įå°ıå¿ĥ":111488,"两ä¸ī":111489,"æĦıæĢĿæĺ¯":111490,"åĨ·èĹı":111491,"çļĦçĹĩçĬ¶":111492,"æĢ§åij½":111493,"è¶ħæłĩ":111494,"å¯Ĩ碼":111495,"ç§ijæĬĢèĤ¡ä»½":111496,"äºĨä¸Ģæī¹":111497,"çĿ£å¯Ł":111498,"åªĴä»ĭ":111499,"å°Ħæīĭ":111500,"ä¿®åħ»":111501,"çīĩåĪ»":111502,"éĢĤåIJĪèĩªå·±":111503,"åıªè¦ģæĺ¯":111504,"åIJĥè¿ĩ":111505,"éĩijéĵ¶":111506,"缴å±ŀ":111507,"åѦéĹ®":111508,"åİĭåĪ¶":111509,"çªĹå¤ĸ":111510,"æĶ¶åĪ°äºĨ":111511,"åħ¨åĽ½äººå¤§":111512,"ä½Ĩæĺ¯å¯¹äºİ":111513,"åľ¨æķ´ä¸ª":111514,"çļĦèĥĮåIJİ":111515,"åĩıå°ijäºĨ":111516,"åıįèħIJ":111517,"åıįèħIJåĢ¡":111518,"åıįèħIJåĢ¡å»ī":111519,"æĹ·":111520,"åĪĨæľŁ":111521,"åľ¨æ·±åľ³":111522,"æīĵçĿĢ":111523,"æī«ä¸Ģ":111524,"æī«ä¸Ģæī«":111525,"æĶ¿åºľéĥ¨éŨ":111526,"æİ¥è¿ŀ":111527,"å±ŀäºİèĩªå·±":111528,"åŃIJå¼¹":111529,"åIJĮæł·æĺ¯":111530,"æĢ»åħ±":111531,"车ä¼ģ":111532,"æ¢ĵ":111533,"åħ¬é¡·":111534,"åıij声":111535,"éĴĽ":111536,"èµ°åĬ¿åĽ¾":111537,"主èIJ¥":111538,"åĸĶ":111539,"æķ°æį®åĪĨæŀIJ":111540,"ä¸įè¿ľ":111541,"æľīåIJį":111542,"æľīåIJįçļĦ":111543,"åģ¿è¿ĺ":111544,"å¾Īä½İ":111545,"è®ĵ人":111546,"èĿī":111547,"é«ĺè´µ":111548,"å°ij许":111549,"æ°Ł":111550,"å¹¢":111551,"亲æĥħ":111552,"è¿Ļ件äºĭæĥħ":111553,"çĶ¨é¤IJ":111554,"缸åħ³æĸ°éĹ»":111555,"å°±åºĶ该":111556,"ç»ĪçĤ¹":111557,"æĺ¯å¤ļå°ij":111558,"çĻ»åľº":111559,"è¯ķ管":111560,"è¯ķ管婴åĦ¿":111561,"åģļ大":111562,"åģļ大åģļ强":111563,"çļĦä¾ĭåŃIJ":111564,"åħ«ä¸ª":111565,"æĺİæĹ¥":111566,"çĤ³":111567,"èµ°åİ»":111568,"éģº":111569,"墩":111570,"ä½ĵä¼ļåĪ°":111571,"åĴı":111572,"ä¸ĭè¾¾":111573,"å¤įåıij":111574,"追éĢIJ":111575,"æīĵåĵį":111576,"çļĦéļ±ç§ģæ¬Ĭ":111577,"åħ·æľīä¸Ģå®ļ":111578,"è¿Ļä¹Īå¤ļå¹´":111579,"æłijæŀĹ":111580,"æľĢéķ¿":111581,"åIJĮèĥŀ":111582,"åħīæ³½":111583,"åŁŁåIJį":111584,"æĮĩåIJij":111585,"åıĹ害èĢħ":111586,"æłijèĦĤ":111587,"æľīå¤ļ大":111588,"大éĿ¢ç§¯":111589,"æĹłç¼Ŀ":111590,"æĶ¹æŃ£":111591,"æĽ´å¤ļçļĦæĺ¯":111592,"æľŁæľ«":111593,"æŃ¼":111594,"ä¹īä¹Į":111595,"éĤ£ä½ł":111596,"çļĦ第ä¸Ģ个":111597,"èĮµ":111598,"å°§":111599,"èį«":111600,"ä¸įä»ħåı¯ä»¥":111601,"æ¶Įçİ°":111602,"æĢ»éĿ¢ç§¯":111603,"æĸ°éĹ»åıijå¸ĥ":111604,"æ°ijçĶ¨":111605,"就读":111606,"æīĵè´¥":111607,"å¤ĸè¯Ń":111608,"æĪij们ä¸Ģèµ·":111609,"é¢Ħå®ļ":111610,"çĥ¹é¥ª":111611,"æľĢ主è¦ģ":111612,"æľĢ主è¦ģçļĦ":111613,"çīĮçħ§":111614,"åĽłåħ¶":111615,"ä½İä¸ĭ":111616,"ä¼ļåIJĮ":111617,"è§ģ解":111618,"éĹ´éļĶ":111619,"æķĻç¨ĭ":111620,"å°ī":111621,"å¸Ĥä¸Ńå¿ĥ":111622,"åħ³éĶ®æĺ¯":111623,"æµ·åįĹçľģ":111624,"çī¹åĪ«æĺ¯åľ¨":111625,"ä¸ŃåĽ½å¤§éĻĨ":111626,"åħħ足çļĦ":111627,"æĹ¢èĥ½":111628,"åĤ³çµ±":111629,"çijľä¼½":111630,"åħ¥åĽ´":111631,"æħ¢æħ¢åľ°":111632,"æĬ¥éħ¬":111633,"æī¹å¤į":111634,"å·¥ä¸ļåĽŃåĮº":111635,"ä¸İåıijå±ķ":111636,"èĥ¸éĥ¨":111637,"åľ¨ç½ij绾":111638,"åľ¨ç½ij绾ä¸Ĭ":111639,"交è°Ī":111640,"æĽ´æĶ¹":111641,"åįłæľīçİĩ":111642,"ä¸Ŀ绸ä¹ĭè·¯":111643,"è¡Ľ":111644,"çłĶåĪ¤":111645,"åĪª":111646,"åĪªéĻ¤":111647,"è¿Ļåıª":111648,"çļĦæ°Ķæģ¯":111649,"åĬłå·ŀ":111650,"éĴ§":111651,"çIJĨäºĭéķ¿":111652,"ä¸ĸ家":111653,"æµģè¡ĮçļĦ":111654,"å¾Īæľīåı¯èĥ½":111655,"们éĥ½":111656,"ç»ıèIJ¥æ¨¡å¼ı":111657,"è¡Įä¸ļä¸Ń":111658,"éĢļçŁ¥ä¹¦":111659,"åij½é¢ĺ":111660,"æľ¬ç¶²ç«Ļ":111661,"æ²Ļçī¹":111662,"åıijåħī":111663,"é«ĺä»·":111664,"å·²çĦ¶":111665,"åıĮåįģä¸Ģ":111666,"ä¸Ĭè¯ī":111667,"ç¿ħèĨĢ":111668,"è¿Ļä¸Ģå¹´":111669,"大ä¼ļä¸Ĭ":111670,"éĩī":111671,"å®Įåħ¨æĺ¯":111672,"å¾Ĺ太":111673,"ä¸ĢèĪ¬äºº":111674,"è¿ĺç®Ĺ":111675,"æĬĺåıł":111676,"æĬķæľº":111677,"çĤ¹çĩĥ":111678,"çİ°éĩijæµģ":111679,"åħĶåŃIJ":111680,"ç½ijæł¼":111681,"æİ¥è¿ĩ":111682,"ä¾Ľè´§":111683,"éĺ´å½±":111684,"åİŁåħĪ":111685,"æį£":111686,"左侧":111687,"åħĭæĭī":111688,"æīĵåį¡":111689,"ç§ijæ¯Ķ":111690,"æ±ĩéĽĨ":111691,"åľ°çIJĨä½įç½®":111692,"è¯Ħå§Ķ":111693,"ç»ĵåIJĪèµ·æĿ¥":111694,"è¿Ľåħ¥åĪ°":111695,"åı¯è¡Į":111696,"åı¯è¡ĮæĢ§":111697,"让å®ĥ":111698,"åĪ¶åº¦æĶ¹éĿ©":111699,"çĶĺèĤĥçľģ":111700,"åĵĹ":111701,"åģıåģı":111702,"è¡£çī©":111703,"ç¥Ŀè´º":111704,"æºIJèĩª":111705,"并ä¸į代表":111706,"åĽ½åº¦":111707,"好åĿı":111708,"æĿĸ":111709,"æĿŃå·ŀå¸Ĥ":111710,"湿度":111711,"鲸":111712,"åįļ彩":111713,"æ³°å±±":111714,"æĿijèIJ½":111715,"æĸ°èģŀ":111716,"èĤĭ":111717,"åı¤èĢģçļĦ":111718,"çļĦç§ĺå¯Ĩ":111719,"ä¸Ģ个éĹ®é¢ĺ":111720,"éģıåĪ¶":111721,"åįĥ亿":111722,"è¿ĩ硬":111723,"å°Ħåĩ»":111724,"èĩªçĦ¶æĺ¯":111725,"产åĮº":111726,"çĤ¹çĤ¹å¤´":111727,"åı¯ä»¥å¸®åĬ©":111728,"说å®ŀ":111729,"说å®ŀè¯Ŀ":111730,"æĪijåıªæĺ¯":111731,"ä¹ĭä½Ļ":111732,"åIJĮæĹ¶ä¹Łæĺ¯":111733,"ä¸ŃåĽ½éĺŁ":111734,"建æĪIJåIJİ":111735,"ä¹IJè§Ĩ":111736,"åij¨å²ģ":111737,"èį¯åºĹ":111738,"éĩijåįİ":111739,"严éĩįå½±åĵį":111740,"è´¨åľ°":111741,"æĹħéģĬ":111742,"åħµåĻ¨":111743,"æķĻèĤ²æķĻåѦ":111744,"离åİ»":111745,"åIJĦå¼ıåIJĦæł·":111746,"ä»ĭç´":111747,"ä»ĭç´¹":111748,"å¼Ģ头":111749,"å°Ĩèĩªå·±çļĦ":111750,"åIJ¬åĬĽ":111751,"ä¿¡æģ¯ç³»ç»Ł":111752,"ä»İæł¹æľ¬":111753,"ä»İæł¹æľ¬ä¸Ĭ":111754,"æİĮ声":111755,"欢åĸľ":111756,"å±ķåĮº":111757,"åķ¸":111758,"太å¤ļäºĨ":111759,"éĹ²ç½®":111760,"èĥ¡èIJĿåįľ":111761,"å§Ķå®£ä¼ł":111762,"å§Ķå®£ä¼łéĥ¨":111763,"åįĹéĺ³":111764,"å·ŀåĮº":111765,"ä¸İæŶ":111766,"ä¸İæĹ¶ä¿±":111767,"ä¸İæĹ¶ä¿±è¿Ľ":111768,"å«Įçĸij人":111769,"èī¯å¿ĥ":111770,"头顶":111771,"è´¢æĬ¥":111772,"ä½Ľæ³ķ":111773,"å¾µ":111774,"åİŁä»¶":111775,"åĭŀ":111776,"çĶ·ç¯®":111777,"å¤ĸåĽ½äºº":111778,"è¿Ŀ纪":111779,"æī¾äºĨ":111780,"æįķæįī":111781,"缸è¯Ĩ":111782,"æIJľéĽĨ":111783,"çļĦä¼Łå¤§":111784,"ä¸īç»´":111785,"å°±è¡ĮäºĨ":111786,"çĭIJæľĪ":111787,"çĭIJæľĪå±±":111788,"å¸ĮæľĽéĢļè¿ĩ":111789,"èĢĮ对äºİ":111790,"éĿ¢å°į":111791,"åĨĽåĽ¢":111792,"è¡ĹåĮº":111793,"æĤ¬æĮĤ":111794,"便ç§ĺ":111795,"æľīä¸ĢçĤ¹":111796,"ä¼ļè®®ä¸Ĭ":111797,"ä¸ĭæīĭ":111798,"廣åijĬ":111799,"äºĶè¡Į":111800,"çŃīåĢĻ":111801,"ç´§ç´§åĽ´ç»ķ":111802,"æĭ¿äºĨ":111803,"æ¡ĮéĿ¢":111804,"ç¥ŀæĥħ":111805,"éĽĦåİļ":111806,"çŀ³":111807,"楼ä¸ĭ":111808,"彪":111809,"äºĭåıij":111810,"åĨįè§ģ":111811,"é¤ĺ":111812,"é¢ĦåĶ®":111813,"åİ»çľĭçľĭ":111814,"æĪij们åºĶ该":111815,"ä¸ī家":111816,"æµĬ":111817,"ä¹IJéĺŁ":111818,"çľĭä¸įè§ģ":111819,"èĦijåŃIJ":111820,"æĮģæľīçļĦ":111821,"çĻ½èıľ":111822,"éĹªçĥģ":111823,"åĸĿæ°´":111824,"æݧåĪ¶ç³»ç»Ł":111825,"ä¸ĵåĮº":111826,"æľĿå»·":111827,"æĪijå¿ĥéĩĮ":111828,"å±ķåİħ":111829,"èľĺèĽĽ":111830,"åĨ»ç»ĵ":111831,"粪":111832,"åºIJ":111833,"åIJij社ä¼ļ":111834,"åĨ³çŃĸéĥ¨ç½²":111835,"çŁŃæľŁåĨħ":111836,"æĸ°ä¸ļæĢģ":111837,"æľĶ":111838,"æŶæĬ¥":111839,"使ä¹ĭ":111840,"åĽłåŃIJ":111841,"åıĤä¸İèĢħ":111842,"çļĦ年轻人":111843,"æīĭ表":111844,"å°ģéĶģ":111845,"为ä»Ģä¹Īä¸į":111846,"åIJ¸çĥŁ":111847,"æ¯Ĵç´ł":111848,"åĪijæ³ķ":111849,"磫æŃ£":111850,"身æĹģ":111851,"åİŁè°ħ":111852,"çĽijæĬ¤":111853,"æѤå¤Ħ":111854,"éļ¨æĻĤ":111855,"æŀľå®ŀ":111856,"åĮ»çĸĹæľįåĬ¡":111857,"ä¸įåIJĪçIJĨ":111858,"æIJŀ好":111859,"çļĦèĦļæŃ¥":111860,"å¤ĸå¥Ĺ":111861,"ç¶ĵéģİ":111862,"æĶ¾ç¼ĵ":111863,"åģľçķĻ":111864,"æĺŁçIJĥ":111865,"çļĦä¸ĢéĿ¢":111866,"åĩłä½ķ":111867,"è½®åĽŀ":111868,"æ¯Ľå·¾":111869,"ä¿®çIJĨ":111870,"ä¸įçŁ¥ä¸į":111871,"ä¸įçŁ¥ä¸įè§ī":111872,"æķ´ä¸ªäºº":111873,"æ¯ģçģŃ":111874,"åı°å·ŀ":111875,"使çĶ¨å¯¿åij½":111876,"é»ijçĻ½":111877,"æij¸ç´¢":111878,"é¼łæłĩ":111879,"éĿ©æĸ°":111880,"麵":111881,"ä¸ĵéĹ¨ä¸º":111882,"å¾Īå¤ļæľĭåıĭ":111883,"å·¥ä½ľç»Ħ":111884,"åIJĪå½±":111885,"çĤºä»Ģ麼":111886,"æŀģ度":111887,"çļĦè¿ĽæŃ¥":111888,"å½ĵä¹ĭ":111889,"å½ĵä¹ĭæĹł":111890,"å½ĵä¹ĭæĹłæĦ§":111891,"è´´è¿ij":111892,"尺度":111893,"åľ¨çİ°åľº":111894,"éĻį临":111895,"åħ»èĢģéĩij":111896,"ç£ķ":111897,"åı¯ä»¥ä½¿":111898,"管çIJĨæ°´å¹³":111899,"æľ¬æĬ¥è®°èĢħ":111900,"æ³ķ令":111901,"åį¡è½¦":111902,"ä¸ľæµ·":111903,"å¤ļéĩį":111904,"åħ¶éĹ´":111905,"ç´Ļ":111906,"éĩįå¤§é¡¹çĽ®":111907,"æ±Ĺæ°´":111908,"ç»Ħå§Ķä¼ļ":111909,"ä¿¡æģ¯åħ¬å¼Ģ":111910,"ä¸į论æĺ¯":111911,"ä¸ĢåIJ¬":111912,"èĴ¸æ±½":111913,"æıŃç§ĺ":111914,"è¶ħéģİ":111915,"触åıij":111916,"婦":111917,"åħ³èģĶ交æĺĵ":111918,"å°±ç»Ļ大家":111919,"好ä¹ħ":111920,"åĢŁè´·":111921,"游æĪıè§Ĵèī²":111922,"å¼ĢåIJ¯äºĨ":111923,"æİł":111924,"åħļçļĦåįģä¹Ŀ":111925,"ä¸ĭ鼨":111926,"çŁŃæŶéĹ´åĨħ":111927,"å¯ħ":111928,"导åħ¥":111929,"å·¥ä½ľç»ıéªĮ":111930,"ä¹Łåıªèĥ½":111931,"鼷éľĨ":111932,"è·Łè¿Ľ":111933,"åį¡éĢļ":111934,"é¢ĩæľī":111935,"æľºä½ĵ":111936,"æĪĺ士èģĮä¸ļ":111937,"女主":111938,"ä½ĵåĪ¶æľºåĪ¶":111939,"足åįı":111940,"èĪĴéĢĤçļĦ":111941,"åĢŁåı£":111942,"æī¹åĪ¤":111943,"æķ°åĢ¼":111944,"諾":111945,"éĺ¿æĭī伯":111946,"åĺİ":111947,"æħ¶":111948,"达人":111949,"å¼Ģæ°´":111950,"å¤§éĽ¨":111951,"温室":111952,"ä½İè¿·":111953,"ä»įæŧ":111954,"éªĹåŃIJ":111955,"亲å±ŀ":111956,"çIJĨæĻº":111957,"æľ¬åŁºéĩij":111958,"å¨ħ":111959,"åĨĻåŃĹ楼":111960,"å¢Ļå£ģ":111961,"宵":111962,"èĻ½çĦ¶æĺ¯":111963,"顺çĿĢ":111964,"åħ«åį¦":111965,"åķĨçĶ¨":111966,"ä¸į失":111967,"è¿·èĮ«":111968,"顺便":111969,"æļijæľŁ":111970,"æ¬ºè´Ł":111971,"é¢ijé¢ij":111972,"è¯¥æł¡":111973,"æĸĻçIJĨ":111974,"æ·±æĥħ":111975,"åīįéĶĭ":111976,"ä¿ĿèŃī":111977,"èģĮä¸ļçĶŁæ¶¯":111978,"åħ¬å¼Ģåıij":111979,"åħ¬å¼Ģåıijè¡Į":111980,"åħ¥æĪ·":111981,"éłĵ":111982,"åĢ¾åIJ¬":111983,"éŃģ":111984,"æĦīæĤ¦":111985,"åĽŀåIJĪ":111986,"åħ¨åĬĽä»¥":111987,"åħ¨åĬĽä»¥èµ´":111988,"åĥ¹åĢ¼":111989,"èĥ½åĬĽå¼º":111990,"ç»ıå¼Ģ":111991,"ç»ıå¼ĢåĮº":111992,"è¿ľæĸ¹":111993,"çļĦéģĵçIJĨ":111994,"缴åįĩ":111995,"缴åįĩæľº":111996,"为主é¢ĺçļĦ":111997,"ç»ĻæĤ¨":111998,"è¿ĺæĥ³":111999,"æ¯ĶæĪij":112000,"åĨľçī§":112001,"æµ·åºķ":112002,"çŃ¾è®¢äºĨ":112003,"对äºİæĪij们":112004,"æĹ¶è®¸":112005,"éĶ®çĽĺ":112006,"å®ŀéĻħæݧåĪ¶":112007,"çļĦæ¨¡æł·":112008,"åıįæĺłäºĨ":112009,"代åĬŀ":112010,"åĮ»çĶ¨":112011,"éĽĨç»ĵ":112012,"åıijå±ķåīįæĻ¯":112013,"æĮĩçĿĢ":112014,"åįİåĮĹ":112015,"è¿Ļåĩłä¸ª":112016,"åIJįæ°Ķ":112017,"åĤįæĻļ":112018,"èĩªåıij":112019,"æ³¢åħ°":112020,"大åĬĽæİ¨è¿Ľ":112021,"èĩªç§°":112022,"èįĨå·ŀ":112023,"æIJį害":112024,"äºĨä¸Ģåı¥":112025,"æľĢåĪĿçļĦ":112026,"éĩijèŀįåį±æľº":112027,"æĢĢ念":112028,"è¡Įåĭķ":112029,"女æİĴ":112030,"ä¸į解":112031,"ä¼łéĶĢ":112032,"转载请":112033,"饰åĵģ":112034,"åıªä¸º":112035,"ä¸İä¼Ĺ":112036,"ä¸İä¼Ĺä¸įåIJĮ":112037,"èĥ½èĢĹ":112038,"èı©æıIJ":112039,"è¿ij两年":112040,"è¿Ķ乡":112041,"马ä¸Ĭå°±":112042,"äºĮçŃīå¥ĸ":112043,"水管":112044,"æ³ķåѦ":112045,"çģŃçģ«":112046,"大å§IJ":112047,"åij¨è½¬":112048,"æľīæľŁ":112049,"æľīæľŁå¾Ĵ":112050,"æľīæľŁå¾ĴåĪij":112051,"å°įæĸ¹":112052,"ç¥ŀèī²":112053,"æ²¹èĦĤ":112054,"ä¸īçĤ¹":112055,"ä¸įåĪ©äºİ":112056,"äºĭä¸ļéĥ¨":112057,"å°±è·Ł":112058,"å¼ĢæĶ¯":112059,"å°ı女åŃ©":112060,"åħ±åIJĮåĬªåĬĽ":112061,"çĶļèĩ³è¿ĺ":112062,"è¿ĻåIJį":112063,"è¿Ļç¬Ķ":112064,"çݯåį«":112065,"æľīç§į":112066,"è§ĨåĬĽ":112067,"çĨŁçŁ¥":112068,"åħ¬ç§¯éĩij":112069,"æ¶Īéĺ²å®īåħ¨":112070,"é¢ĩ为":112071,"大èħ¿":112072,"éĿ¶":112073,"çī¹æķĪ":112074,"æľįåĬ¡åĮº":112075,"å¼Ģåĩº":112076,"深度èŀįåIJĪ":112077,"æĹłå¿§":112078,"æŁ¥éĺħ":112079,"ç»Īç»ĵ":112080,"ä¿Ŀç¨İ":112081,"è¨İè«ĸ":112082,"å½ĵåģļ":112083,"è·³èĪŀ":112084,"寧":112085,"女çİĭ":112086,"è®°èĢħåľ¨":112087,"åħ¨äº§ä¸ļéĵ¾":112088,"è´¯éĢļ":112089,"åħ´ä¸ļ":112090,"éĻįåĪ°":112091,"å°ģéĿ¢":112092,"åħ¨éĿ¢æİ¨è¿Ľ":112093,"奶èĮ¶":112094,"éĢīåĿĢ":112095,"äºĨä¸Ģåľº":112096,"åIJĮä¼´":112097,"议论":112098,"æIJĵ":112099,"诸èijĽ":112100,"诸èijĽäº®":112101,"å¹²åĺĽ":112102,"æµģæĦŁ":112103,"ä¸ĵä¸ļçŁ¥è¯Ĩ":112104,"çĶµç«Ļ":112105,"åĩıå¼±":112106,"åĩºåħ¥":112107,"åIJĦçľģ":112108,"éĿŀ常é«ĺ":112109,"åľ°æ¯¯":112110,"åıijæĸĩ":112111,"çĦī":112112,"çĥ§çĥ¤":112113,"å£ģ纸":112114,"æģ¶åĮĸ":112115,"èĬ¸":112116,"èĥĸåŃIJ":112117,"çĩĴ":112118,"çľģéĴ±":112119,"çĻ¾å¼º":112120,"çIJĨ工大åѦ":112121,"éĴ¢æĿIJ":112122,"åĽ½æľīèµĦ产":112123,"æĪĺæľº":112124,"æ³Ħéľ²":112125,"åIJİéĿ¢çļĦ":112126,"æ°´èµĦæºIJ":112127,"æ¢ħèĬ±":112128,"åĨĻçĿĢ":112129,"ä¹ĭ声":112130,"æĹłåı¯":112131,"æĺİæľĿ":112132,"ç«ĭæĸ¹ç±³":112133,"ç·£":112134,"æĶ¾è¿ĩ":112135,"ç¦ıçĶ°":112136,"å¾Ĺä½ı":112137,"åıĹä¼Ĺ":112138,"ä¸Ń级":112139,"çĹħåıĺ":112140,"ä¸Ģçŀ¬éĹ´":112141,"æĿĥéĩį":112142,"人æĢ§åĮĸ":112143,"åĮ»çĸĹåį«çĶŁ":112144,"ä¸įåĪ°ä½į":112145,"æĻºèĥ½å®¶å±ħ":112146,"饮çĶ¨":112147,"æ¼Ķåıĺ":112148,"é«ĺç´łè´¨":112149,"ä¹Ļæĸ¹":112150,"åģľçķĻåľ¨":112151,"èİ·æī¹":112152,"ç©¿æ¢Ń":112153,"å®¢åľº":112154,"æĮ½åĽŀ":112155,"京åŁİ":112156,"çĶŁåij½åĬĽ":112157,"實éļĽ":112158,"çĩĪ":112159,"åĨįçİ°":112160,"çİ°å®ŀä¸Ń":112161,"æľīä¿¡å¿ĥ":112162,"çĸıéĢļ":112163,"åĺ´åĶĩ":112164,"鼷éĶĭ":112165,"èıľåįķ":112166,"éħ¯":112167,"è¶ħé«ĺ":112168,"å¾Īé«ĺåħ´":112169,"çĶŁæ®ĸ":112170,"éĢłä»·":112171,"误åĮº":112172,"æĨĭ":112173,"好æ¶Īæģ¯":112174,"å´Ń":112175,"以èĩ´":112176,"å¼Ģçİ©ç¬ij":112177,"çĽijè§Ĩ":112178,"å·¡å¯Ł":112179,"å¾·å·ŀ":112180,"æĹ©æĹ©":112181,"éĹªçĶµ":112182,"æĪªåĽ¾":112183,"åı¯ä»¥æł¹æį®":112184,"æīĭèīº":112185,"æݥ轨":112186,"ç§įæĹı":112187,"æĢĢéĩĮ":112188,"åİ»åĮ»éĻ¢":112189,"ä¸ĢäºĮ":112190,"å¼ĢéĺĶ":112191,"åĩıéĢŁ":112192,"ä½Ĩä»İ":112193,"éĢĻä¸Ģ":112194,"åĩıåħį":112195,"主é¢ĺæķĻèĤ²":112196,"å¼Ģ工建设":112197,"蹦":112198,"æľĪ饼":112199,"ä¸ĭæ²ī":112200,"å°Ĭ严":112201,"éĻĩ":112202,"å®ŀæľ¨":112203,"å»łåķĨ":112204,"声称":112205,"èĢĥåľº":112206,"å¸ĥé²ģ":112207,"èĩªæĿ¥":112208,"èĩªæĿ¥æ°´":112209,"éĴ¾":112210,"年以ä¸Ĭ":112211,"大åıĶ":112212,"ä»ĸå·²ç»ı":112213,"åħ¨æĿij":112214,"èģĶç³»çĶµè¯Ŀ":112215,"为导åIJij":112216,"åĪ¤å¤Ħ":112217,"对éĺµ":112218,"缮æ¨Ļ":112219,"åIJįé¢Ŀ":112220,"客æ°Ķ":112221,"横åIJij":112222,"çŃīåĨħ容":112223,"åĩłçĤ¹":112224,"è°Ī论":112225,"ä¸įä¹ı":112226,"å±ķçİ°åĩº":112227,"è¾ĥéķ¿":112228,"éĢĨ转":112229,"å°ıæĻĤ":112230,"æĺ¯å¤ļä¹Ī":112231,"æľ¬æľĪ":112232,"è¿ijè§Ĩ":112233,"æĪIJç«ĭ以æĿ¥":112234,"代表çĿĢ":112235,"æĬ¥å¤į":112236,"æĪıæĽ²":112237,"è¨ŃåĤĻ":112238,"åħ¥èĤ¡":112239,"å¾ģæľį":112240,"é«ĺåĩº":112241,"èĪŀåı°ä¸Ĭ":112242,"å¿ĥåĬ¨":112243,"两çĤ¹":112244,"缸çķ¶":112245,"èĻĽ":112246,"主页":112247,"åĩłå®¶":112248,"æĹłä¸į":112249,"åįıå®ļ":112250,"æĸIJ":112251,"å¯ĵæĦı":112252,"åħ¨çº¿":112253,"æįķé±¼":112254,"åı¯ä»¥ä»İ":112255,"æľīè¿Ļæł·çļĦ":112256,"æģ¶éŃĶ":112257,"åĮħåŃIJ":112258,"æģ¤":112259,"å¼Ģå¥ĸç»ĵæŀľ":112260,"ä¸įæŃ»":112261,"èĹį":112262,"å¼¯æĽ²":112263,"海峡":112264,"éĶĢæ¯ģ":112265,"çļĦçĭ¬çī¹":112266,"示æĦı":112267,"ä¸įèĥ½åĨį":112268,"èĥ½æĬĬ":112269,"éĺ²çº¿":112270,"ä¸įå°ijäºİ":112271,"æ±Ģ":112272,"çļĦéĤ£ä¸Ģ":112273,"羣æĥħ":112274,"åŀ®":112275,"被æīĵ":112276,"åĽ½å®ī":112277,"ç¾İå¦Ļ":112278,"è¿Ļåĩł":112279,"åĩºéģĵ":112280,"æľįåĬ¡äºİ":112281,"æĪIJæŀľè½¬åĮĸ":112282,"æīįåįİ":112283,"天é¹ħ":112284,"åĩłä¸ªäºº":112285,"åĢĺèĭ¥":112286,"èĢ½è¯¯":112287,"æĬĹæĪĺ":112288,"è¡ĮéĬ·":112289,"æĿ¥è¢Ń":112290,"åĢŁéĮ¢":112291,"èįīèİĵ":112292,"ä¸¥æł¼æī§è¡Į":112293,"举è¡ĮäºĨ":112294,"å¤ĸç±į":112295,"已达":112296,"æĿijåħļæĶ¯éĥ¨":112297,"è¡Ŀ":112298,"éĻįèĩ³":112299,"æµ·éĩı":112300,"é¤IJé¦Ĩ":112301,"æĢ¥å¿Ļ":112302,"æ·±è¿ľ":112303,"å¾Ģè¿Ķ":112304,"ç¨İåĬ¡å±Ģ":112305,"å¹¿æ³ĽåºĶçĶ¨":112306,"è®®åijĺ":112307,"æĹłæķĮ":112308,"çľ¼åħī":112309,"çĥŃè¡Ģä¼łå¥ĩ":112310,"æŃIJ":112311,"äºĨäºĽ":112312,"è¿ĿèĥĮ":112313,"è¿Ļæĺ¯ä¸Ģç§į":112314,"ä¸į稳å®ļ":112315,"大家åĪĨ享":112316,"表çı¾":112317,"åīįåįģ":112318,"è·¯è¿ĩ":112319,"æĴ©":112320,"åIJĮæĥħ":112321,"ä¹łä¿Ĺ":112322,"åıijè´¢":112323,"åºĶæľīçļĦ":112324,"æĿİæŁIJ":112325,"èĤĽ":112326,"马åħĭ":112327,"éĢļåijĬ":112328,"巨人":112329,"ä¸ĢåĽ¢":112330,"éĢĻ次":112331,"ä¸įäºĨ解":112332,"æĸ½è¡Į":112333,"èij¡èIJĦçīĻ":112334,"åıĺå¾ĹæĽ´åĬł":112335,"æı£":112336,"åĪĽæĸ°èĥ½åĬĽ":112337,"çķħéĶĢ":112338,"表æī¬":112339,"æ¯ĶåĪ©":112340,"æ¯ĶåĪ©æŶ":112341,"åĮ»çĸĹä¿ĿéĻ©":112342,"æĵį纵":112343,"伤亡":112344,"æµİå®ģ":112345,"åıĺäºĨ":112346,"æľ¬æ¬¡æ´»åĬ¨":112347,"åľŁè±ª":112348,"æĥ³åĬŀæ³ķ":112349,"æĺķ":112350,"å½ĵæĻļ":112351,"åĩºå±Ģ":112352,"çĥŃè®®":112353,"è°Īè°Ī":112354,"æĻĭåįĩ":112355,"åĬ¿å¿ħ":112356,"çĻ»å±±":112357,"éĤ£åĦ¿":112358,"åIJĥåĪ°":112359,"ä¹ĭåŁİ":112360,"å¿«æĿ¥":112361,"æ¹Ľæ±Ł":112362,"第ä¸ī个":112363,"åħ¨éĿ¢æıIJåįĩ":112364,"å¥ĸåѦ":112365,"å¥ĸåѦéĩij":112366,"æĬķåħ¥ä½¿çĶ¨":112367,"é½IJé²ģ":112368,"åı¯ä»¥æĬĬ":112369,"åĴĮä»ĸçļĦ":112370,"è´ŃæĪ¿èĢħ":112371,"æŃ£å¼ıåIJ¯åĬ¨":112372,"åįİ润":112373,"ä¸įæĸŃå®ĮåĸĦ":112374,"éĴ¢æĿ¿":112375,"累积":112376,"满èĦ¸":112377,"åĽĽæĸ¹":112378,"è´¢çī©":112379,"ä»ĸ们ä¼ļ":112380,"å¤ıæĹ¥":112381,"éĤ£ä¸ªäºº":112382,"éĿłçĿĢ":112383,"çĤ¹äºĨ":112384,"çĤ¹äºĨçĤ¹å¤´":112385,"æ©ĭ":112386,"åıĪ好":112387,"åıĪ好åıĪ":112388,"åıĪ好åıĪå¿«":112389,"éĺµéĺµ":112390,"å°ģ建":112391,"æľ¬çĶ°":112392,"çī©ä¸ļæľįåĬ¡":112393,"èĩªè´¸åĮº":112394,"åIJı":112395,"便åĪ©åºĹ":112396,"åĽ½å®¶æłĩåĩĨ":112397,"éĿ¢ç²ī":112398,"èī°è¾Ľ":112399,"æĶ»åħ³":112400,"æīĵåĮħ":112401,"车éĺŁ":112402,"人éĢī":112403,"åı¯ä¸įæĺ¯":112404,"äºĮåįģå¹´":112405,"åIJįå¸Ī":112406,"æµ¦ä¸ľ":112407,"åħ¬è¯ģ":112408,"è¿IJéĢģ":112409,"æĺ¯æľĢ好çļĦ":112410,"æŁĶåĴĮ":112411,"çİĭæŁIJ":112412,"çĹħæĪ¿":112413,"åĨ¶éĩij":112414,"ä¸Ģ件äºĭæĥħ":112415,"åį¤":112416,"åı¯æݧ":112417,"çīŁ":112418,"æĭĤ":112419,"å·²äºİ":112420,"人éĢł":112421,"çĶŁçī©åĮ»èį¯":112422,"ä½ĵçİ°åĩº":112423,"èĤ²åĦ¿":112424,"èĢģå®ŀ":112425,"åľĸçīĩ":112426,"諸":112427,"ç´¯äºĨ":112428,"æĦŁåħ´è¶£çļĦ":112429,"åĽ¾çīĩæĿ¥æºIJ":112430,"ä¹Łæĺ¯ä¸Ģç§į":112431,"æ¾İæ¹ĥæĸ°éĹ»":112432,"æĹ¶è¡¨ç¤º":112433,"åħīè¾ī":112434,"æĬ¥åºŁ":112435,"å²ģæŶ":112436,"éħ®":112437,"æ£Ģä¿®":112438,"åıĺéĢŁ":112439,"åıĺéĢŁç®±":112440,"åľ¨èģĮ":112441,"éı¡":112442,"æįĤ":112443,"çĿ£åĬŀ":112444,"æ°¸ä¸į":112445,"åģļä¸ĢäºĽ":112446,"åİĨæŶ":112447,"å·¥ç¨ĭæľºæ¢°":112448,"æģ°å½ĵ":112449,"å°±åľ¨äºİ":112450,"称åij¼":112451,"éĢļ常æĺ¯":112452,"æł·å¼ı":112453,"åij¨ä¸Ģ":112454,"èĭ±éķij":112455,"åĿĩ线":112456,"ä¼łéĹ»":112457,"çĶ¨æĪ·ä½ĵéªĮ":112458,"èµŀåIJĮ":112459,"骨æĬĺ":112460,"为主ä½ĵ":112461,"æ±Łå±±":112462,"æ¸ħæľĿ":112463,"æĶĢåįĩ":112464,"ä¸įçĽ¸ä¿¡":112465,"éĿ´":112466,"æѦåĬŁ":112467,"åĭ¤åĬ³":112468,"æĿ¥æī¾":112469,"å°ĨæĮģç»Ń":112470,"丫头":112471,"æ¨Ļæºĸ":112472,"裴":112473,"深深çļĦ":112474,"åŃķèĤ²":112475,"è§ĦåĪĴ建设":112476,"æ¸ħçĪ½":112477,"ç²¾åĩĨæī¶è´«":112478,"æīĵçł´äºĨ":112479,"è¿Ļä¸Ģ天":112480,"å·¥ä½ľæĢ»ç»ĵ":112481,"æĹħç¨ĭ":112482,"举èIJ¥":112483,"æĶ¾å°Ħ":112484,"æľīåĩłä¸ª":112485,"éĿŀçī©è´¨":112486,"åIJĥå¾Ĺ":112487,"åŨ":112488,"ä¼ļåıijçĶŁ":112489,"篮æĿ¿":112490,"å¼Ģå°ģ":112491,"麻å°Ĩ":112492,"èııæ³½":112493,"ä¸įåIJĪ":112494,"ç³»åĪĹ产åĵģ":112495,"èѬå¦Ĥ":112496,"ç¾İèªī":112497,"èĩªå·±åĸľæ¬¢":112498,"交æĺĵä¸Ńå¿ĥ":112499,"åIJĪåĶ±":112500,"使æĪij":112501,"åĥıç´ł":112502,"带éĺŁ":112503,"ä½Ĩ对äºİ":112504,"æĬĬè¿Ļ个":112505,"èĤĿèĦı":112506,"åįķ纯çļĦ":112507,"æĶ»åĿļæĪĺ":112508,"缼ä¼ļ":112509,"åijµæĬ¤":112510,"æªĢ":112511,"赶ä¸Ĭ":112512,"æ¥Ĭ":112513,"ä¹ħäºĨ":112514,"ç¡Ŀ":112515,"çŃĶé¢ĺ":112516,"ä¿ĿæĮģçĿĢ":112517,"è§ģè¯Ĩ":112518,"çĤ¹åĦ¿":112519,"åįĬ个æľĪ":112520,"æ»ĩ":112521,"浸泡":112522,"ä¼łéĢģ":112523,"åľ¨å¸Ĥåľºä¸Ĭ":112524,"ä¹ĭ乡":112525,"çī¹éķ¿":112526,"éĽŀ":112527,"èªł":112528,"身å¤Ħ":112529,"æŁłæª¬":112530,"身穿":112531,"çľģåħ¬å®ī":112532,"çľģåħ¬å®īåİħ":112533,"åıĻåĪ©äºļ":112534,"åĩłåĪĨéĴŁ":112535,"人åĢij":112536,"åľ°æ®µ":112537,"èĩªåѦ":112538,"ä¹Łè¶ĬæĿ¥è¶Ĭ":112539,"èģĮæĿĥ":112540,"æĸ§":112541,"èĩ»":112542,"å½Ĵ纳":112543,"驾é©Ń":112544,"éĥ¨åĪĨåľ°åĮº":112545,"没æľīæĥ³åĪ°":112546,"æĴĩ":112547,"ä¹Įé²ģ":112548,"ä¹Įé²ģæľ¨":112549,"ä¹Įé²ģæľ¨é½IJ":112550,"èĤ²äºº":112551,"çļĦæŃ¥ä¼IJ":112552,"å»¶æľŁ":112553,"æ²¹æ°Ķ":112554,"åģļå®Į":112555,"åľ£åľ°":112556,"丰åİļ":112557,"宽带":112558,"åı¯éĿłçļĦ":112559,"åºŃéĻ¢":112560,"åŃľ":112561,"å°ı康社ä¼ļ":112562,"å®īåħ¨ç®¡çIJĨ":112563,"年第":112564,"æİĴ污":112565,"èĥĮåĮħ":112566,"家ä½ı":112567,"åħ¶å®ŀå°±æĺ¯":112568,"ä¼ļè§ģ":112569,"帮åĬ©ä¼ģä¸ļ":112570,"ç½ijè´Ń":112571,"æĺ¯ä¸įä¼ļ":112572,"飯åºĹ":112573,"æŃ»åİ»":112574,"åħįçĸ«åĬĽ":112575,"æľķ":112576,"åĸĿäºĨ":112577,"轻微":112578,"个æľĪåĨħ":112579,"ç»ĦåĽ¢":112580,"åĴĮå®ĮåĸĦ":112581,"鸽":112582,"æıIJéĢŁ":112583,"西å®īå¸Ĥ":112584,"ä¸Ńå¿ĥ主任":112585,"æŶéĹ´ä¸º":112586,"æľŁæĿĥ":112587,"è¶ķ":112588,"ä¸įä»ħè¦ģ":112589,"æľįä»İ":112590,"é¡ĺæĦı":112591,"ä¸įå°ı":112592,"ä¸įå°ıçļĦ":112593,"ç°ĩ":112594,"窦":112595,"åĪĩæĪIJ":112596,"åĵĪåĪ©":112597,"å¤©çľŁ":112598,"ä¸Ģ次次":112599,"éĩijå¸ģ":112600,"æĢİä¹Īèĥ½":112601,"ç½ijè´·":112602,"ä¼ļ计å¸Ī":112603,"çŁŃ缺":112604,"对æłĩ":112605,"åıĺå¾ĹæĽ´":112606,"åīįåĩłå¤©":112607,"éĺ²æ±Ľ":112608,"彩èĻ¹":112609,"åĵģä½į":112610,"è¡¨æł¼":112611,"严å¯Ĩ":112612,"æ¯ĽåĪ©çİĩ":112613,"çļĦåį±å®³":112614,"å½ķåĪ¶":112615,"æ°´åĬ¡":112616,"èĥ½å¤Łè®©":112617,"å¹³æĿ¿":112618,"ä¹³æĪ¿":112619,"è¸ıå®ŀ":112620,"é¦ĸåĪĽ":112621,"é¦Ļèķī":112622,"æĬ¥è¡¨":112623,"ä¸ĢæĬ¹":112624,"åĩºçĶŁäºİ":112625,"è²»çĶ¨":112626,"åĩºè®©":112627,"åIJĪæ³ķæĢ§":112628,"å°¼åħĭ":112629,"åĨ°åĨ·":112630,"é¦Ļæ°Ķ":112631,"åı·ç§°":112632,"èµ·çłģ":112633,"åŁİåİ¿":112634,"çİ©èĢį":112635,"ä¸ĬéĻIJ":112636,"ä¼ļ议精ç¥ŀ":112637,"æĹģè¾¹çļĦ":112638,"便ä¼ļ":112639,"æıŃæĻĵ":112640,"çİ©æĦı":112641,"éĽªå±±":112642,"åIJijçĿĢ":112643,"ä½ĵèĤ²åľ¨çº¿":112644,"说æĺİ书":112645,"åĮĸèĤ¥":112646,"åħļç»Ħ书记":112647,"åĬ¨äºº":112648,"ä¹ĭæīĢ":112649,"æľĪèĩ³":112650,"æľĢå¿«çļĦ":112651,"èĬĤåģĩæĹ¥":112652,"ä¸ĵåľº":112653,"èĢĥä¸Ĭ":112654,"çªŁ":112655,"é²ľè¡Ģ":112656,"è¾ĥ强çļĦ":112657,"æĤĦçĦ¶":112658,"å¤ļä¸ªåĽ½å®¶":112659,"çªĹå¸ĺ":112660,"æŀģå¤§åľ°":112661,"ä¸įçĶ¨æĭħå¿ĥ":112662,"è¿Ļä¹Īåģļ":112663,"åĥ¹æł¼":112664,"ç¾İ丽乡æĿij":112665,"å°ıæŶåĨħ":112666,"紧迫":112667,"大çģ«":112668,"èĥ³èĨĬ":112669,"æĵįä½ľç³»ç»Ł":112670,"æ®ĭçķĻ":112671,"åĨĻåĩº":112672,"ç¦ģå¿Į":112673,"åĬłçĽŁåºĹ":112674,"è¿ijçĻ¾":112675,"便åı¯":112676,"æķ´æĶ¹æݪæĸ½":112677,"éĩĩ访æŶ":112678,"åĶIJ代":112679,"æ·±åĮĸæĶ¹éĿ©":112680,"çŁ¢":112681,"éĥ½åĸľæ¬¢":112682,"è¶ĬæĿ¥è¶Ĭé«ĺ":112683,"èĬ±æľµ":112684,"头çĸ¼":112685,"å®ī康":112686,"å¢ŀéķ¿çİĩ":112687,"çľ¼çľĭ":112688,"å°±æĺ¯ä¸ºäºĨ":112689,"èĢĮ导èĩ´":112690,"åĬłå¿«å»ºè®¾":112691,"èĬ±æł·":112692,"åĨħå¿ĥçļĦ":112693,"æĺĨå±±":112694,"è³ĩæºIJ":112695,"åĽŀåĪ°å®¶":112696,"èıĬèĬ±":112697,"æ°´éĩı":112698,"å¾ģä¿¡":112699,"è¡ĮæĶ¿åĮº":112700,"ä¹ĥæĺ¯":112701,"æĬķèµĦé¡¹çĽ®":112702,"å«ģç»Ļ":112703,"ç¥ŀåľ£":112704,"稳":112705,"æľ¬æĿ¥å°±":112706,"éĢIJä¸Ģ":112707,"èģĮä¸ļæĬĢæľ¯":112708,"ä¸įèī¯ä¿¡æģ¯":112709,"æīĺè¿IJ":112710,"åIJ¯ç¤º":112711,"ä¹ĭåħ§å®¹":112712,"飶":112713,"奢åįİ":112714,"æıŃ示":112715,"æĪIJ为ä¸ŃåĽ½":112716,"æ¶Īè´¹åĵģ":112717,"åħ¬çĶ¨":112718,"æIJŀå®ļ":112719,"è¯·ä½ł":112720,"æŁļ":112721,"åĨħè¡£":112722,"ä½Ĩä»ĸ们":112723,"ä¿Ŀ湿":112724,"该åİ¿":112725,"饱åĴĮ":112726,"æݨåIJij":112727,"èµĦæĸĻæĺ¾ç¤º":112728,"ä¸įå½±åĵį":112729,"人人éĥ½":112730,"åıijå±ķ壮大":112731,"åħ»èĢģæľįåĬ¡":112732,"çĶŁæ´»æ°´å¹³":112733,"åIJĦåİ¿":112734,"ä½łéľĢè¦ģ":112735,"说çļĦæĺ¯":112736,"å¤ĸåªĴ":112737,"æŃ¤äºº":112738,"次è¦ģ":112739,"追赶":112740,"åºĶ该å¦Ĥä½ķ":112741,"æĹ¥åĩĮæĻ¨":112742,"çķ¥æľī":112743,"éĥ½æĥ³":112744,"游ä¹IJ":112745,"è¿Ļ款游æĪı":112746,"平淡":112747,"æĺ¯ä¸ĢåĢĭ":112748,"å¤ĩèĢĥ":112749,"åĪ¶æŃ¢":112750,"ä¸Ģå®ļèĥ½":112751,"å¾Ĵå¼Ł":112752,"以çĤº":112753,"åįĥåħĥ":112754,"äºĶåħŃ":112755,"迪士":112756,"迪士尼":112757,"éĺ³æĢ§":112758,"åĨ¬å¥¥ä¼ļ":112759,"å°±æĺ¯åĽłä¸º":112760,"æĮĤéĴ©":112761,"æ¦ĤåĨµ":112762,"åıªè¦ģæľī":112763,"æ²¹çĶ»":112764,"åľ°æłĩ":112765,"ä¸Ĭè°ĥ":112766,"产ä¸ļåĽŃåĮº":112767,"åħ«åįģ":112768,"棱":112769,"液æĻ¶":112770,"æĿijå§Ķä¼ļ":112771,"çŃ¾çº¦ä»ªå¼ı":112772,"è¿Ļåħ¶ä¸Ń":112773,"åĨĻéģĵ":112774,"示èĮĥåŁºåľ°":112775,"éĩİçĶŁåĬ¨çī©":112776,"鼻åŃIJä¿¡ç®±":112777,"åĽ½éĻħè´¸æĺĵ":112778,"人æĿĥ":112779,"ä¿Ŀ管":112780,"èĭ¥æĤ¨":112781,"åİĭæĬij":112782,"黼":112783,"åľ°çľĭçĿĢ":112784,"éĻ°":112785,"ä¸Ģå¹´å¤ļ":112786,"ä»İ容":112787,"ä¸ŃæĸŃ":112788,"å¯Łè§ī":112789,"移交":112790,"éĶ¯":112791,"æĪĸ许æĺ¯":112792,"綳":112793,"两项":112794,"æľĢåĸľæ¬¢":112795,"æľĢåĸľæ¬¢çļĦ":112796,"å¤ľéĩĮ":112797,"åIJĮä»ģ":112798,"åĪĽæĸ°é©±åĬ¨":112799,"è°ģèĥ½":112800,"飾":112801,"åħīåѦ":112802,"åİĦ":112803,"èĦ±é¢ĸ":112804,"èĦ±é¢ĸèĢĮåĩº":112805,"迦":112806,"æĺ¯ä¸įåı¯èĥ½":112807,"窥":112808,"èĥ½æ»¡è¶³":112809,"宽度":112810,"伦çIJĨ":112811,"åı¯ä»¥èİ·å¾Ĺ":112812,"转ä¼ļ":112813,"å±±æĿij":112814,"éĵºè®¾":112815,"åĩºåĩ»":112816,"æĸĩåĮĸèīºæľ¯":112817,"ä¼ļ议室":112818,"æŃĮ声":112819,"æ»Ķ":112820,"èIJİ缩":112821,"æľįåĬ¡åijĺ":112822,"åıij表äºĨ":112823,"æĸ¼æĺ¯":112824,"æĺİç¡®è§Ħå®ļ":112825,"ç»´å¥ĩ":112826,"水产":112827,"æĬķä¿Ŀ":112828,"éĺ´éģĵ":112829,"赶快":112830,"夺å¾Ĺ":112831,"ä¸ĭåįķ":112832,"çī©æµģåħ¬åı¸":112833,"çݯç»ķ":112834,"å½Ī":112835,"ä½ľé£İ建设":112836,"æĹħ游æĻ¯åĮº":112837,"æľīæĽ´å¤ļçļĦ":112838,"丰å¯Įå¤ļ彩":112839,"çIJĨ财产åĵģ":112840,"åĩºå·®":112841,"ä»İ严治":112842,"ä»İ严治åħļ":112843,"çĽ¸å¹²":112844,"æ»ĭ润":112845,"主åĬŀæĸ¹":112846,"åī§åľº":112847,"æ»ļçIJĥ":112848,"æ©Ħæ¦Ħ":112849,"èĩªä¸»åĪĽæĸ°":112850,"éĢļå¾Ģ":112851,"æł¼å°Ķ":112852,"çļĦä¼ĺçĤ¹":112853,"èĥĮä¸Ĭ":112854,"çªľ":112855,"çĪĨåĩº":112856,"å¹³æķ´":112857,"ä¸ĢèĦļ":112858,"åħ¨ä½ĵåijĺå·¥":112859,"éĻIJå®ļ":112860,"åŁİéķĩåĮĸ":112861,"æ·³":112862,"éĢ®æįķ":112863,"è¡ĮåĬ¨è®¡åĪĴ":112864,"æīĵå¾Ĺ":112865,"åİļéĩį":112866,"纪å½ķçīĩ":112867,"åĿļä¿¡":112868,"央ä¼ģ":112869,"åĨįä¹Łä¸į":112870,"天涯":112871,"åıĤèĢĥèµĦæĸĻ":112872,"æľīæ¯Ĵ":112873,"åIJ¸çº³":112874,"è¶Ĭåıij":112875,"éĩįè¦ģæĦıä¹ī":112876,"åĽ½éĺ²éĥ¨":112877,"è¿Ļ个è¡Įä¸ļ":112878,"æĻ®æŁ¥":112879,"å¼ĤæĢ§":112880,"å»¶è¿Ł":112881,"å°ıå¹ħ":112882,"èī²æĥħ":112883,"综åIJĪæ²»çIJĨ":112884,"æŃ£æĺ¯åĽłä¸º":112885,"产ä¸ļç»ĵæŀĦ":112886,"çłĶ究æĬ¥åijĬ":112887,"åģľä¸ĭ":112888,"éķ¿èĢģ":112889,"éĩĿå°į":112890,"åįĹ京å¸Ĥ":112891,"çģĮæºī":112892,"转è¿IJ":112893,"欺è¯Ī":112894,"éĢłåģĩ":112895,"åĪĨå¸ĥå¼ı":112896,"æĦŁè§¦":112897,"æĪijå½ĵæŶ":112898,"åıijè§ī":112899,"åĽ¾çº¸":112900,"æĶ¹èī¯":112901,"çĭłçĭł":112902,"åĨ²åĪº":112903,"æĸ°äº¬":112904,"æĸ°äº¬æĬ¥":112905,"ç¥ŀåĻ¨":112906,"秸ç§Ĩ":112907,"çĪº":112908,"å°Ĩè¿İæĿ¥":112909,"工信":112910,"工信éĥ¨":112911,"éĻIJéĩı":112912,"æŃ¢æįŁ":112913,"åѦä¼ļäºĨ":112914,"åįİ缼":112915,"åįİçĽĽé¡¿":112916,"å¾Įä¾Ĩ":112917,"ä¸ĭéĿ¢æĺ¯":112918,"ä¸ĭéĿ¢æĺ¯å°ı":112919,"æIJ¬è¿IJ":112920,"ç¾İæľ¯é¦Ĩ":112921,"æ¸ħåĩī":112922,"å¤ļå¹´åīį":112923,"è©ŀ":112924,"åįĥç±³":112925,"表述":112926,"æ±ŁéŨ":112927,"åĬłæ²¹ç«Ļ":112928,"æľ¬èĥ½":112929,"导读":112930,"åĽ´è§Ĥ":112931,"并åIJij":112932,"åŁºæľ¬æĥħåĨµ":112933,"æīĵå¼ĢäºĨ":112934,"è¿Ļä¸ī个":112935,"æ±ķ头":112936,"强æľīåĬĽ":112937,"强æľīåĬĽçļĦ":112938,"è¿Ľåľº":112939,"ä¹Ŀæ±Ł":112940,"çIJĥæĺŁ":112941,"好çľĭçļĦ":112942,"大æĪ·":112943,"湯":112944,"å¥ĩå¦Ļ":112945,"ä¹IJåĻ¨":112946,"æĪijçļĦå¿ĥ":112947,"çľī头":112948,"åĨľä¸ļçĶŁäº§":112949,"ç¼ĸçłģ":112950,"åŁºç¤":112951,"åŁºç¤İ":112952,"天æĸĩ":112953,"åĢĭ人è³ĩè¨Ĭ":112954,"åİ»è¿ĩ":112955,"èģĨåIJ¬":112956,"æĶ¾åģĩ":112957,"ä¸įåħ·å¤ĩ":112958,"æ·Ģç²ī":112959,"大佬":112960,"åħ¨å¤©":112961,"åħ¨éĿ¢å»ºæĪIJ":112962,"éļIJå½¢":112963,"ç¼ħçĶ¸":112964,"åIJ³":112965,"è¡ĮæĶ¿æī§æ³ķ":112966,"åŁİåł¡":112967,"èİ«æĸ¯":112968,"èİ«æĸ¯ç§ij":112969,"æīĢæľīæĿĥ":112970,"éĽĨåľĺ":112971,"å±Ģåī¯å±Ģéķ¿":112972,"åĩłä¹İ没æľī":112973,"æ´ģåĩĢ":112974,"çĶµå½±èĬĤ":112975,"åŃ©ç«¥":112976,"æīĢåģļçļĦ":112977,"æ¸ħ代":112978,"æĸ°çīĪ":112979,"éĵĿåIJĪéĩij":112980,"为æĬĵ":112981,"为æĬĵæīĭ":112982,"åĪ¤å®ļ":112983,"çī¹äº§":112984,"æīĭæ©Ł":112985,"ä¸įåı¯æĪĸ":112986,"ä¸įåı¯æĪĸ缺":112987,"å¸Ĥåľºè§Ħ模":112988,"åĿ¯":112989,"åĮ»åѦéĻ¢":112990,"å¿«è¦ģ":112991,"èĮľ":112992,"æĬĺèħ¾":112993,"äºĨè¿ĩæĿ¥":112994,"æĬ¥åijĬæľŁåĨħ":112995,"çī©ç§į":112996,"ç»Łè®¡å±Ģ":112997,"æī©å»º":112998,"æ¶ħ":112999,"责任人":113000,"éĺİ":113001,"è¯Ħè®®":113002,"å¾Ģäºĭ":113003,"æīĢ示":113004,"æķ´æ´ģ":113005,"éĹºèľľ":113006,"æĹħéĢĶ":113007,"å®ŀè®Ń":113008,"ä¹ĭ称":113009,"巴士":113010,"éĢŁåº¦å¿«":113011,"ä¸įä»ħå¦ĤæѤ":113012,"å®Ŀè´µçļĦ":113013,"åºŁçī©":113014,"河水":113015,"æݥ纳":113016,"ç²¾æ¹Ľ":113017,"åħ¶æ¬¡æĺ¯":113018,"顺德":113019,"åħ¬åħ±åį«çĶŁ":113020,"è¤IJèī²":113021,"ä¸įæĥľ":113022,"æĬĢæľ¯æľįåĬ¡":113023,"æİ·":113024,"æ±ĤèģĮ":113025,"ä¸ī峡":113026,"æĬķåħ¥åĪ°":113027,"太åIJİ":113028,"åIJ¯åĬ¨ä»ªå¼ı":113029,"缴æݥ影åĵį":113030,"æĸ°æ¬¾":113031,"个乡éķĩ":113032,"çĻ¾äº¿":113033,"庫":113034,"ä¹ŁæŃ£æĺ¯":113035,"åı¶çīĩ":113036,"æľĢæĹ©çļĦ":113037,"æĪĺ绩":113038,"å·¥æľŁ":113039,"æĻļæľŁ":113040,"è¿Ļæł·è¯´":113041,"è¯įè¯Ń":113042,"ä¾Ħ":113043,"æķ£çĥŃ":113044,"éĽĨæĪIJçĶµè·¯":113045,"åIJįè¯į":113046,"æĻºåķĨ":113047,"æĭ¥åłµ":113048,"çĭĤ欢":113049,"è¿ĻèĪ¬":113050,"浴室":113051,"åijķåIJIJ":113052,"æľªæĿ¥åıijå±ķ":113053,"ä¸īä½įä¸Ģä½ĵ":113054,"åªĴé«Ķ":113055,"ä¸įå¾Ĺ转载":113056,"åĽłä¸ºå¥¹":113057,"æĺ¾ç¤ºå±ı":113058,"ä¾Ľæļĸ":113059,"éĨ«éĻ¢":113060,"æľīæĦıæĢĿ":113061,"æľīæĦıæĢĿçļĦ":113062,"娱ä¹IJåŁİ":113063,"åįµå·¢":113064,"åĪĽéĢłåĬĽ":113065,"ç«łèĬĤ":113066,"人大常å§Ķ":113067,"èĢĮçİ°åľ¨":113068,"å¤ĸå©Ĩ":113069,"å¢ŀæĮģ":113070,"äºĶåįĥ":113071,"èĢģå¸Ī们":113072,"æ´ĽæĿī":113073,"æ´ĽæĿī磶":113074,"æİĮæı¡äºĨ":113075,"ä¸ŃåĽ½æĸĩåĮĸ":113076,"æĸ°æĶ¿":113077,"主è¦ģçĶ¨äºİ":113078,"åıijçĥ§":113079,"类似äºİ":113080,"åĮĹæŀģ":113081,"æĪij们认为":113082,"弥漫":113083,"åħ¨çIJĥç»ıæµİ":113084,"é¢IJ":113085,"ä¸Ģèµ·è£ħä¿®":113086,"æĶĴ":113087,"æĭīèIJ¨":113088,"帶ä¾Ĩ":113089,"åĨ·æ°´":113090,"ä¸īåĨľ":113091,"æĿ¿æĿIJ":113092,"è¿ŀè¿ŀ":113093,"éĵ®":113094,"ç»ıèIJ¥çIJĨ念":113095,"山顶":113096,"å¾Īæĥ³":113097,"çĺ«":113098,"å§ĭç»Īä¿ĿæĮģ":113099,"åľ¨å¹¿å·ŀ":113100,"ä¸įåIJĮæĦı":113101,"åıĺåİĭ":113102,"åıĺåİĭåĻ¨":113103,"产éĶĢ":113104,"表éĿ¢ä¸Ĭ":113105,"æīĢ以ä»ĸ":113106,"ç»ıéªĮ丰å¯Į":113107,"éĥ¨å§Ķ":113108,"åħµåĽ¢":113109,"æīĢè¿°":113110,"æķ¦çħĮ":113111,"ç»ıèIJ¥èĮĥåĽ´":113112,"åı£è¯Ń":113113,"失信":113114,"æ¯ı个人çļĦ":113115,"æīĭæĮģ":113116,"æģIJæħĮ":113117,"åł¡åŀĴ":113118,"é¦ħ":113119,"éĵ¸éĢł":113120,"æĭ¿åĩºæĿ¥":113121,"æİ¢æµĭ":113122,"大家ä¸Ģèµ·":113123,"奧":113124,"å®ŀè´¨æĢ§":113125,"å°ıåĦ¿":113126,"èĩºåįĹ":113127,"èĩºåįĹå¸Ĥ":113128,"å¼ĢåıijèĢħ":113129,"åı¯æł¹æį®":113130,"ç®±åŃIJ":113131,"饺åŃIJ":113132,"å¿ĻçĿĢ":113133,"æĿ¥ä¸įåıĬ":113134,"çĽ¸ä¼ł":113135,"åĽ½ç½ij":113136,"èħ¹æ³»":113137,"è¿ĻéĩĮæľī":113138,"é£İæĻ¯åĮº":113139,"åıĤä¿Ŀ":113140,"æŃ»èĢħ":113141,"æĪ´ä¸Ĭ":113142,"æ©Łæ§ĭ":113143,"è¯ķéªĮåĮº":113144,"ä¼łæİĪ":113145,"æµ·è¾¹":113146,"泪水":113147,"缸åħ³åĨħ容":113148,"éĥijå·ŀå¸Ĥ":113149,"åħijçİ°":113150,"两åij¨":113151,"èĬľæ¹ĸ":113152,"çĶµåŃIJä¿¡æģ¯":113153,"红å¤ĸ":113154,"æĹħ游å±Ģ":113155,"å¾Ģå¾Ģä¼ļ":113156,"è¿ħçĮĽ":113157,"ä¼łçľŁ":113158,"æ¸ħæ¾Ī":113159,"å°±è¿ij":113160,"微信群":113161,"ç³»åĪĹæ´»åĬ¨":113162,"ç»ı常ä¼ļ":113163,"è§Ĥæµĭ":113164,"å¿ĥå¾Ĺä½ĵä¼ļ":113165,"éĻĪåĪĹ":113166,"åĮĹæĸĹ":113167,"è«®":113168,"諮詢":113169,"è¿ĺæĺ¯ä¼ļ":113170,"æµĭç®Ĺ":113171,"æĺŁç©º":113172,"宽容":113173,"çī©ä¸ļåħ¬åı¸":113174,"æĪĴæĮĩ":113175,"å¸ħæ°Ķ":113176,"ä¸ĢæŃ¥æŃ¥":113177,"åħ±é¸£":113178,"åĨ³ä¸į":113179,"æݥ管":113180,"å¦ĩèģĶ":113181,"æ¯Ķåĸ»":113182,"é²ģè¿ħ":113183,"æĮģçºĮ":113184,"çĽ¸äº²":113185,"å¨ģå°¼æĸ¯äºº":113186,"ç«ĭ项":113187,"åĪĿå§ĭ":113188,"èĩªåĪ¶":113189,"è¿Īè¿Ľ":113190,"ä¸Ĭæ±½":113191,"å®ıä¼Ł":113192,"æł¹æľ¬æ²¡æľī":113193,"æĸ°åĨłçĹħæ¯Ĵ":113194,"åĵªç§į":113195,"康åħ»":113196,"è¡°èĢģ":113197,"å½ķåĥı":113198,"é«Ķé©Ĺ":113199,"ç»ijå®ļ":113200,"é¢Ŀ头":113201,"äºĶæľĪ":113202,"èĬ±å¼Ģ":113203,"ä¸Ģ线åŁİå¸Ĥ":113204,"åĪ°åľº":113205,"æĬķéĻį":113206,"çĹĺçĹĺ":113207,"åıĹä¸įäºĨ":113208,"æīİæł¹":113209,"æĽ´ä½ķåĨµ":113210,"æĬ½æŁ¥":113211,"åĩºè·¯":113212,"审议éĢļè¿ĩ":113213,"ä¸įåĥħ":113214,"èī²è°ĥ":113215,"çĻ¾ä½Ļ":113216,"èĤłéģĵ":113217,"æ·±åİļçļĦ":113218,"马åĬĽ":113219,"æĹ©æĻļ":113220,"æŃĮèĪŀ":113221,"éĺ²æĻĴ":113222,"æľĢåIJİä¸Ģ个":113223,"樱èĬ±":113224,"å°ıä¼ĻåŃIJ":113225,"åľ¨å½ĵåľ°":113226,"å°ıä¼Ļ伴们":113227,"èµ·æºIJ":113228,"åħ¨åªĴä½ĵ":113229,"ç°½":113230,"éħ±æ²¹":113231,"æĹłè®ºå¦Ĥä½ķ":113232,"裤åŃIJ":113233,"åģľäº§":113234,"ä¸įçĶ±å¾Ĺ":113235,"çīµå¼ķ":113236,"ä¼łåĬ¨":113237,"ä¹Ŀé¾Ļ":113238,"åĬłåĽº":113239,"ä¹Łä¸įæķ¢":113240,"æĬĢæľ¯æĶ¯æĮģ":113241,"ä¸Ĭå²Ĺ":113242,"ç»ıéªĮåĴĮ":113243,"æł¼æŀĹ":113244,"åIJ¸éĻĦ":113245,"æľªæĪIJå¹´":113246,"奢ä¾Īåĵģ":113247,"追æį§":113248,"好ä¸į容æĺĵ":113249,"èķ´åIJ«":113250,"ä¿Ŀå®ļ":113251,"æĬ¥ä¸ļ":113252,"æµ·åĨħå¤ĸ":113253,"ä½łçİ°åľ¨":113254,"æ²¹èĢĹ":113255,"è´¨éĩı管çIJĨ":113256,"æ½ľæ°´":113257,"ä¸½æ±Ł":113258,"转åħ¥":113259,"è¿Ļä¹Īä¹ħ":113260,"æĺİ代":113261,"责任åĪ¶":113262,"éĩįå·¥":113263,"大巴":113264,"触åıĬ":113265,"èµ·åĪĿ":113266,"大å¦Ī":113267,"æĸ¯å¡Ķ":113268,"åĨĽå·¥":113269,"书éĻ¢":113270,"峨":113271,"æݨçIJĨ":113272,"è¿Ļç¯ĩæĸĩç«ł":113273,"è¿ģ移":113274,"åľ¨åIJĮä¸Ģ":113275,"ç»Ĩç»Ĩ":113276,"åīĬå¼±":113277,"书æĪ¿":113278,"ç¶ĵ常":113279,"è¯ķé¢ĺ":113280,"æĤ£ä¸Ĭ":113281,"çĻ«çĹ«çĹħ":113282,"åĨ²æ´Ĺ":113283,"å¤ĸæı´":113284,"åħĭåĪ¶":113285,"åįģæľĪ":113286,"åģļä¸įåĪ°":113287,"ç¾İåĮĸ":113288,"å¦ĤæľŁ":113289,"è¿ĺéľĢ":113290,"å¤©åºľ":113291,"å°±æĦıåij³çĿĢ":113292,"çļĦç¡®æĺ¯":113293,"éªĹå±Ģ":113294,"å°ıç»ĦèµĽ":113295,"è©©":113296,"ä¹Ŀå¹´":113297,"æĻĵå¾Ĺ":113298,"çłĶ究人åijĺ":113299,"大éħĴåºĹ":113300,"ç§ijåѸ":113301,"åħŃåIJĪ":113302,"çķĮå®ļ":113303,"车载":113304,"å¼ĢçĿĢ":113305,"毫æĹłçĸij":113306,"毫æĹłçĸijéĹ®":113307,"è¿IJç»´":113308,"ç¦ģåĮº":113309,"èĦ±èIJ½":113310,"讲å¸Ī":113311,"产ä¸ļåŁºåľ°":113312,"é«ĺæĢ§èĥ½":113313,"åħī彩":113314,"çİ°éĺ¶æ®µ":113315,"åĩ¿":113316,"è¾ĥå·®":113317,"饮çĶ¨æ°´":113318,"éĸĭçĻ¼":113319,"ç½ijåIJ§":113320,"çĮ´åŃIJ":113321,"æѦæŀĹ":113322,"å®īåİ¿":113323,"ä¸įåı¯æĢĿ":113324,"ä¸įåı¯æĢĿè®®":113325,"éĬ·åĶ®":113326,"è´«ç©·":113327,"为åķ¥":113328,"éºĵ":113329,"å¹¾åĢĭ":113330,"è§Ħ模以ä¸Ĭ":113331,"æıļ":113332,"è¢«åĽ°":113333,"缺å¸Ń":113334,"å¿«é¤IJ":113335,"æĬ¢åįł":113336,"æĻŁ":113337,"å¤įæ´»":113338,"æľ¬æĬ¥è®¯":113339,"åĪĽä¸ĭ":113340,"海滩":113341,"éĩı产":113342,"å¦Ĥä½ķåİ»":113343,"车ä½į":113344,"å¯ĩ":113345,"äºĮåįģåĽĽ":113346,"ç»ıæµİæįŁå¤±":113347,"éħįå¥Ĺ设æĸ½":113348,"åŁºæľ¬éĿ¢":113349,"äºī论":113350,"就好åĥı":113351,"çłĶ究æĪIJæŀľ":113352,"éĻĪè¿°":113353,"æīĵåĬ¨":113354,"ä¸ĭå·´":113355,"ç§ĴéĴŁ":113356,"对人ä½ĵ":113357,"æĬĢæľ¯çłĶåıij":113358,"åİŁåŃIJ":113359,"æĺ¯ä¸Ģ项":113360,"äºĨä¸Ģ份":113361,"æĮĩçĶ²":113362,"çĶ¨éĩı":113363,"è¿ĺä¸įå¤Ł":113364,"æĶ¿åºľéĩĩè´Ń":113365,"çŁ¥è¯ĨçĤ¹":113366,"ä¸ŃåĽ½æ¢¦":113367,"å¾Īå¼Ģå¿ĥ":113368,"礼è²Į":113369,"éĿŀ常å¤ļ":113370,"éĿŀ常å¤ļçļĦ":113371,"åĽļ":113372,"æĹħé¦Ĩ":113373,"å°½æĥħ":113374,"æŃĮåĶ±":113375,"æ²Ļé¾Ļ":113376,"车åİ¢":113377,"客æµģ":113378,"åģıå·®":113379,"积累äºĨ":113380,"æ¡Ķ":113381,"çĶ»çĶ»":113382,"ä¹ŁåºĶ该":113383,"åºĶçĶ¨ç¨ĭåºı":113384,"èĥĥèĤł":113385,"以å¾Į":113386,"豪å®ħ":113387,"æ·±åĬłå·¥":113388,"缴è¨Ģ":113389,"åĮĸçŁ³":113390,"åĽ½éģĵ":113391,"ä¸ĥ个":113392,"ä»İèĢĮ使":113393,"èĤłèĥĥ":113394,"æĹ¥è¶ĭ":113395,"çĪ¶åŃIJ":113396,"ç·©":113397,"æĭĽçīĮ":113398,"产å¦ĩ":113399,"çķªèĮĦ":113400,"æĪijéĻ¢":113401,"建çŃijå·¥ç¨ĭ":113402,"å±ķè§Īä¼ļ":113403,"家éķ¿ä»¬":113404,"åĨľä½ľçī©":113405,"æĹ¥å¤ľ":113406,"æĶ»æĵĬ":113407,"è§Ħéģ¿":113408,"èĪŁå±±":113409,"便æ°ij":113410,"åħ«åŃĹ":113411,"ä¸įæĽ¾":113412,"æĶ¯éħį":113413,"çĨ¬å¤ľ":113414,"人é¡ŀ":113415,"ç´ĢéĮĦ":113416,"ç»ıèIJ¥æ´»åĬ¨":113417,"大涨":113418,"å¸Ĥå§Ķ常å§Ķ":113419,"åĪĨéIJĺ":113420,"ä¸Ģ个èģĮä¸ļ":113421,"çĹħåĽł":113422,"è¿Ļ对äºİ":113423,"ä¸įå¾Ĺä¸į说":113424,"åıijçĶµæľº":113425,"æľīæīĢ帮åĬ©":113426,"缮æłĩä»»åĬ¡":113427,"åĽłåľ°":113428,"åĽłåľ°åĪ¶":113429,"åĽłåľ°åĪ¶å®ľ":113430,"å°Ĩè¾¾åĪ°":113431,"ç²Ĺç³Ļ":113432,"ç¨³åĽº":113433,"å«£":113434,"çİ°åľ¨å¾Īå¤ļ":113435,"ä¸ĸçķĮ级":113436,"å¼łæŁIJ":113437,"çĤ¹ç¼Ģ":113438,"èijµ":113439,"社ä¼ļç»Ħç»ĩ":113440,"å¾ĢåIJİ":113441,"åĬłæģ¯":113442,"åĻªå£°":113443,"æľīåħ´è¶£":113444,"为æĤ¨æıIJä¾Ľ":113445,"æ²¹æ¼Ĩ":113446,"ç¬¬åĽĽå±Ĭ":113447,"çļĩ宫":113448,"ä¹Ĵä¹ĵ":113449,"ä¹Ĵä¹ĵçIJĥ":113450,"éļ¨èijĹ":113451,"éģ©åIJĪ":113452,"åįĹéĿŀ":113453,"æĵ´":113454,"西æ´ĭ":113455,"åĬłå¯Ĩ":113456,"æĪIJåĬŁä¸¾åĬŀ":113457,"åı£æ°´":113458,"æĪIJ年人":113459,"æīĢæıIJä¾ĽçļĦ":113460,"éļĶå£ģ":113461,"åľ¨äº¬":113462,"å½ĵåľ°æŶéĹ´":113463,"çŃīåIJĦç§į":113464,"é£İæ°Ķ":113465,"å±ĭéĩĮ":113466,"ä¸ĢåŃĹ":113467,"çļĦæŶéĹ´éĩĮ":113468,"åĺ¿åĺ¿":113469,"快讯":113470,"ä¸Ńåľº":113471,"ä¸Ģçĵ¶":113472,"æ»ķ":113473,"é¢Ĩè·ij":113474,"好èݱ":113475,"好èݱåĿŀ":113476,"没åħ³ç³»":113477,"åĩºå¢ĥ":113478,"ä¸įæĺ¯ä¸Ģ个":113479,"éĥ½æĺ¯éĿŀ常":113480,"éľĩåĬ¨":113481,"èİ·èĥľ":113482,"åįļå¼Ī":113483,"æĬļåħ»":113484,"对ç«ĭ":113485,"æľįåĬ¡æľºæŀĦ":113486,"è°£è¨Ģ":113487,"社ä¼ļç§ijåѦ":113488,"åIJ¬è¯´è¿ĩ":113489,"æī³":113490,"æīĵ磨":113491,"åı£æľį":113492,"好åĥıæĺ¯":113493,"以åıĬåħ¶ä»ĸ":113494,"çī¹è´¨":113495,"亲è¿ij":113496,"ä¸Ģç»ı":113497,"æ¶Ŀ":113498,"éŃĶæľ¯":113499,"éģĵ路交éĢļ":113500,"è§Ħ模æľĢ大":113501,"å®ŀæĸ½æĦıè§ģ":113502,"ä¹ŀ":113503,"ä¸Ģä¸ĸ":113504,"åŁ·è¡Į":113505,"è±Ĩçĵ£":113506,"åĪĹ为":113507,"æķħ宫":113508,"çĶŁåij½åij¨æľŁ":113509,"ä¸īç§įèģĮä¸ļ":113510,"详ç»Ĩä»ĭç»į":113511,"å®Įå¤ĩ":113512,"å²©çŁ³":113513,"éļıæīĭ":113514,"飲":113515,"æķĪæŀľåĽ¾":113516,"ç§ĭåĨ¬":113517,"åĬŁå¾·":113518,"è§Ħç«łåĪ¶åº¦":113519,"æĹ¥æ¸IJ":113520,"æīĢéľĢè¦ģ":113521,"æīĢéľĢè¦ģçļĦ":113522,"å²Ľä¸Ĭ":113523,"åĩºåľŁ":113524,"åĽ¾æĸĩ":113525,"ç§ijæĬĢè¿ĽæŃ¥":113526,"éĢļèĥĢ":113527,"èĢģ太太":113528,"èĭĹæľ¨":113529,"éĵ¶å·Ŀ":113530,"å¸IJ篷":113531,"éĿŀè¦ģ":113532,"éħįçĶµ":113533,"å¤Ħå¢ĥ":113534,"èĤ¡æĿĥæĬķèµĦ":113535,"ä¸Ģ缴åĪ°":113536,"åĿĩçĶ±":113537,"æĬĹæĹ¥":113538,"æį®ä»ĭç»į":113539,"ä½łåĸľæ¬¢":113540,"åĪĽæĸ°åŀĭ":113541,"åıĺè¿ģ":113542,"è§Ĩå¯Ł":113543,"å®Įåħ¨æ²¡æľī":113544,"åħĥæŦ":113545,"åı¯ä¿¡":113546,"åı¦è¡Į":113547,"æĿij级":113548,"åħ¥åľº":113549,"æIJŃæ¡£":113550,"ä¹ŁåĽłæѤ":113551,"æį¢æĪIJ":113552,"ä¸įè´Ł":113553,"äºĨ大éĩıçļĦ":113554,"éģĶåĪ°":113555,"å¸Ĥåİ¿":113556,"å¹´è¼ķ":113557,"å¿«æīĭ":113558,"å¸Įå°Ķ":113559,"èĩªèIJ¥":113560,"éĽªèĬ±":113561,"æIJģ":113562,"çľ¼ç§ij":113563,"æŃ£ç¢º":113564,"çļĦ姿æĢģ":113565,"åĿļå®ŀçļĦ":113566,"æĮĩ纹":113567,"æªĶæ¡Ī":113568,"ç½®äºİ":113569,"佩æľį":113570,"豪éŨ":113571,"åĵĴ":113572,"æģ°å¥½":113573,"æª¢æŁ¥":113574,"åĪĿè¡·":113575,"大åĶIJ":113576,"约ä¼ļ":113577,"èĴ¸åıij":113578,"çŃ¹åĪĴ":113579,"å¹´ç»Ī":113580,"è¡Įæ¥Ń":113581,"åħ±éĿĴ":113582,"åħ±éĿĴåĽ¢":113583,"ä¼ļå¼ķèµ·":113584,"ä¸Ńç§ij":113585,"ä¸Ńç§ijéĻ¢":113586,"æĮ¯åĬ¨":113587,"åį´åıijçİ°":113588,"ä¸įåĬ¨äº§":113589,"èĮ¹":113590,"æĪ¿éĹ´éĩĮ":113591,"è´§å¸ģæĶ¿çŃĸ":113592,"æ²»çĻĤ":113593,"æħİéĩį":113594,"å¡ŀå°Ķ":113595,"åĽ½ç±į":113596,"åĽłæŀľ":113597,"çŃīçī¹çĤ¹":113598,"山谷":113599,"ä¸ĭè¼ī":113600,"è®ĵæĪij":113601,"饮éħĴ":113602,"è¿Ļ个游æĪı":113603,"ç»Ŀ大éĥ¨åĪĨ":113604,"åĴ¨è¯¢æľįåĬ¡":113605,"干活":113606,"è®®ä¼ļ":113607,"æ¦Ĥè¿°":113608,"åĪĨåĮº":113609,"æŃ»åIJİ":113610,"ç«ĻçĿĢ":113611,"主è¦ģé¢Ĩ导":113612,"åIJĮåŁİ":113613,"大æłij":113614,"对åѦçĶŁ":113615,"社ä¼ļä¿ĿéĻ©":113616,"å¢ŀèµĦ":113617,"主人åħ¬":113618,"å®£ä¼łæķĻèĤ²":113619,"æĸĩåĮĸ交æµģ":113620,"客æĪ¶":113621,"çŁ¥åIJįåĵģçīĮ":113622,"æ»ŀåIJİ":113623,"äºĴè¡¥":113624,"æĦŁäºº":113625,"åī¿":113626,"åIJİ代":113627,"äºī龸":113628,"æķĻèĤ²åŁ¹è®Ń":113629,"éĿĻèĦī":113630,"ä¹ıåĬĽ":113631,"说åĩºæĿ¥":113632,"çİĭèĢħèį£èĢĢ":113633,"åĢ«":113634,"åįĩèµ·":113635,"éķģ":113636,"åĩºæ¸¸":113637,"éĢļè¡Įè¯ģ":113638,"å·¥ä½ľå²Ĺä½į":113639,"åĮłå¿ĥ":113640,"æĭ¿æĿ¥":113641,"æ´Ĺè¡£æľº":113642,"æĪijä¸įæĥ³":113643,"é¢Ħè§ģ":113644,"æ¼Ķ示":113645,"ä¸ĢçĽ´æ²¡æľī":113646,"è·Łå¥¹":113647,"对çħ§æ£ĢæŁ¥":113648,"ç°¿":113649,"ä¸ĵå¿ĥ":113650,"è®®äºĭ":113651,"åīį端":113652,"åį¡å°Ķ":113653,"è¨Ńå®ļ":113654,"设置äºĨ":113655,"å©ļ纱":113656,"åľ¨åĽ½å¤ĸ":113657,"åı³ä¾§":113658,"è³¼çī©":113659,"å¥ĩèij©":113660,"å¢ŀåĬłåĢ¼":113661,"好è¿IJ":113662,"åĽ½éĻħæľºåľº":113663,"ä¸ĭ称":113664,"缮åīį为æŃ¢":113665,"ç¥ŀä»Ļ":113666,"å®ĥåı¯ä»¥":113667,"æ¾Ħæ¸ħ":113668,"èĥ½ä½¿":113669,"游åĩ»":113670,"游åĩ»éĺŁ":113671,"åĩ¹":113672,"ä¸įè¦ģåĨį":113673,"åĨ³èĥľ":113674,"åĨ³æĪĺ":113675,"æĭ½":113676,"缼åħ¸":113677,"å¾Īå¥½åľ°":113678,"æľĢç¾İçļĦ":113679,"åĥļ":113680,"å·´åŁº":113681,"å·´åŁºæĸ¯åĿ¦":113682,"æľĢéĢĤåIJĪ":113683,"é«ĺèģĮ":113684,"ä¿Ŀå§Ĩ":113685,"æİĪæ¬Ĭ":113686,"说åĪ°è¿ĻéĩĮ":113687,"æݨå¼Ģ":113688,"çİĩè¾¾":113689,"ä¸īåĪĨä¹ĭä¸Ģ":113690,"管çIJĨä¸Ńå¿ĥ":113691,"交æ±ĩ":113692,"森æŀĹåħ¬åĽŃ":113693,"å¾Ģä¸Ĭ":113694,"éªijè¡Į":113695,"æį®æѤ":113696,"纽带":113697,"ç»ŀ":113698,"ä¸īæĸ¹":113699,"æĦıä¹īä¸ĬçļĦ":113700,"æİ¨è¿Ł":113701,"å¤ļæł·æĢ§":113702,"æĥ³èµ·äºĨ":113703,"æİĴåIJį第":113704,"å·¨é¢Ŀ":113705,"æĿŁç¼ļ":113706,"å®īå®ļ":113707,"äºĭ實":113708,"çļĦæĦ¿æľĽ":113709,"è£ħå¤ĩåĪ¶éĢł":113710,"人å±ħ":113711,"人å±ħçݯå¢ĥ":113712,"å¿ĺè®°äºĨ":113713,"该游æĪı":113714,"楼ä¸Ĭ":113715,"å¼Ģä¼ļ":113716,"æģ³":113717,"åıĭæĥħéĵ¾æİ¥":113718,"ç¡Ĵ":113719,"ç»ĻäºĪäºĨ":113720,"åģı好":113721,"åĵī":113722,"交éĢļå®īåħ¨":113723,"éĽĮ":113724,"æ²»çĹħ":113725,"è§īå¾Ĺå¾Ī":113726,"衬衫":113727,"å¿ĥæĦ¿":113728,"æ´ŀå¯Ł":113729,"æ°ijæ£Ģå¯ŁéĻ¢":113730,"æıIJçĤ¼":113731,"è¦ģè¿Ľä¸ĢæŃ¥":113732,"驾车":113733,"æĻ®æĥł":113734,"æķĸ":113735,"ç¦ıéŁ³":113736,"éĢģè¾¾":113737,"è§ĦåĪĴ设计":113738,"æīĭå¥Ĺ":113739,"å®īä¿Ŀ":113740,"è¿ĺä¸įå¦Ĥ":113741,"åīįè¿°":113742,"æłĩè®°":113743,"ç´§æİ¥çĿĢ":113744,"æ§IJ":113745,"æ·±æ·±åľ°":113746,"满满çļĦ":113747,"æĺ¥è¿IJ":113748,"æĹ¥äº§":113749,"çĪ±æĬ¤":113750,"åħ¨æĹ¥":113751,"åħ¨æĹ¥åĪ¶":113752,"转åĬ¨":113753,"ç¥Ńç¥Ģ":113754,"ä¹°ä¸ľè¥¿":113755,"å¯¹æľªæĿ¥":113756,"æ¶Ī失äºĨ":113757,"åļ´éĩį":113758,"ä¸īæĿ¡":113759,"éħ¸å¥¶":113760,"éĽĨåĽ¢èĤ¡ä»½":113761,"西路":113762,"åıªå¾Ĺ":113763,"éĢģåİ»":113764,"çĭłæĬĵ":113765,"åĪ©çĶ¨çİĩ":113766,"ä¸ĭåij¨":113767,"å¥ĭæĪĺ":113768,"æĺ¥èĬĤæľŁéĹ´":113769,"è´Łè´£ä»»":113770,"æĺĤè´µ":113771,"尾巴":113772,"ç¯ĩæĸĩç«ł":113773,"åħ®":113774,"è®ĬæĪIJ":113775,"å¹¹":113776,"çĻ»éĮĦ":113777,"ä½Ī":113778,"å·¥åĮł":113779,"åĵªæĢķæĺ¯":113780,"åıįåĵį":113781,"ç§ĥ":113782,"åĩºè½¨":113783,"æĹ¥åĨĽ":113784,"åIJįèªī":113785,"æķıéĶIJ":113786,"æľįåĬ¡æ°´å¹³":113787,"çħ§å°Ħ":113788,"ä¼Ĭæĭī":113789,"ä¼Ĭæĭīåħĭ":113790,"åĨħéĺģ":113791,"èĬĴæŀľ":113792,"ä¸ĩåĪĨ":113793,"éĢĢ款":113794,"缴æĴŃéĹ´":113795,"æĭ¿åĪ°äºĨ":113796,"å°İèĩ´":113797,"空æ°Ķä¸Ń":113798,"客æĪ·æľįåĬ¡":113799,"è¿IJåĬ¿":113800,"ç»ĵçŁ³":113801,"ä¸įå¿ħè¦ģçļĦ":113802,"èĥ¶åĽĬ":113803,"çIJĨä¼ļ":113804,"æĬ½åĩº":113805,"空æ°Ķè´¨éĩı":113806,"æ¯ķç«Łæĺ¯":113807,"åĨ·æ¼ł":113808,"ä¸Ģå¦Ĥ":113809,"ä¸Ģå¦ĤæĹ¢":113810,"ä¸Ģå¦ĤæĹ¢å¾Ģ":113811,"æĤ£çĹħ":113812,"åĬłæĮģ":113813,"èµŀåĬ©":113814,"é«®":113815,"åij½ä¸Ń":113816,"æĦıä¹īä¸Ĭ":113817,"ä¸įèĪį":113818,"åģļ梦":113819,"æīĵæī«":113820,"æĺŁåħī":113821,"æĸŃè£Ĥ":113822,"åħ¨å¥Ĺ":113823,"è£ģå®ļ":113824,"马åħĭæĢĿ":113825,"骨骼":113826,"ä¸Ģè·¯ä¸Ĭ":113827,"å®ļæŶ":113828,"å·¥ç¨ĭæĬĢæľ¯":113829,"å½¼å¾Ĺ":113830,"æ±²åıĸ":113831,"ä¸Ģè§Ī":113832,"åIJµæŀ¶":113833,"ä¿Ĺ称":113834,"æłªæ´²":113835,"åºŁæŧ":113836,"è¡ĮæĺŁ":113837,"åıijçĶŁåıĺåĮĸ":113838,"é¦ĸä»ĺ":113839,"åįģåĪĨéĩįè¦ģ":113840,"æĬĬè¿ĻäºĽ":113841,"ç¥ŀå·ŀ":113842,"æıIJä¾ĽåķĨ":113843,"楷":113844,"å±İ":113845,"çĬ¶åħĥ":113846,"åŁİå¢Ļ":113847,"çľĭä¸Ģçľĭ":113848,"çĶŁäº§èĥ½åĬĽ":113849,"åŁºæľ¬ä¸Ĭéĥ½":113850,"æīĵæī°":113851,"åĪĿ次":113852,"åĩºç¤º":113853,"åħ¶ä¸Ńä¸Ģ个":113854,"çĶŁæĢģç³»ç»Ł":113855,"æīĭæİĮ":113856,"æµİåįĹå¸Ĥ":113857,"åľĭåħ§":113858,"æŃ£åĢ¼":113859,"å¹¾ä¹İ":113860,"æݨèįIJéĺħ读":113861,"è¿Ń代":113862,"è°ĥä¾ĥ":113863,"饮åĵģ":113864,"å¢Ļä½ĵ":113865,"åıĺçİ°":113866,"äºĨ好":113867,"äºĨ好åĩł":113868,"ä¸įçķĻ":113869,"çĪ²":113870,"å°½æĹ©":113871,"æŃ£åľ¨è¿Ľè¡Į":113872,"åĩºéĻ¢":113873,"æĿĢ害":113874,"æıIJ款":113875,"åıijå±ķ空éĹ´":113876,"åīį身":113877,"ä¸įæĸŃå¢ŀ强":113878,"æ·±å±Ĥ次":113879,"容纳":113880,"éĤ£ä»½":113881,"å·¥ä½ľæķĪçİĩ":113882,"æľ¬åĽ½":113883,"失èIJ½":113884,"æŃ£åĽłä¸º":113885,"èĬĤæ°´":113886,"ä¸ĭä¸Ģ代":113887,"çłĶåıijä¸Ńå¿ĥ":113888,"ä¸įçIJĨ":113889,"å®Į好":113890,"ä¿ĿæĬ¤åĮº":113891,"ç»ĵæŀĦè°ĥæķ´":113892,"å¥łå®ļ":113893,"宣称":113894,"éĺ»æĮ¡":113895,"æĴ¤ç¦»":113896,"ä¸įæĸ¹ä¾¿":113897,"åĴķ":113898,"ç¬ijäºĨç¬ij":113899,"çݯå¢ĥ污æŁĵ":113900,"ä½ıæĪ·":113901,"ç»Ŀç¼ĺ":113902,"éĻ¤å°ĺ":113903,"é«ĺå°ļ":113904,"æĢİä¹Īåı¯èĥ½":113905,"éĿ¢èī²":113906,"åķĨæ¥Ń":113907,"çĸ¹":113908,"èµĦæºIJä¼ĺåĬ¿":113909,"è¾ĸåĮºåĨħ":113910,"èĢĢçľ¼":113911,"æij§æ¯ģ":113912,"ä¸ĸçķĮç»ıæµİ":113913,"å¼ķæĿ¥":113914,"ä¸ĢåĪĻ":113915,"æĭĩæĮĩ":113916,"æĬµå¾¡":113917,"éĽį":113918,"åĩĨå¤ĩå·¥ä½ľ":113919,"çıłä¸īè§Ĵ":113920,"ç¨ĢåľŁ":113921,"èİ·å¾ĹæĦŁ":113922,"æĪIJåĬŁçİĩ":113923,"ç½ij约":113924,"ç½ij约车":113925,"èĦIJ":113926,"æķ¬ä¸ļ":113927,"éĩijä»·":113928,"ç²¾é«ĵ":113929,"买车":113930,"åħ³åı£":113931,"åĨįå¤ļ":113932,"æŀģåĵģ":113933,"åIJĦ家":113934,"举æĬ¥çĶµè¯Ŀ":113935,"èļĬ":113936,"æĸ¹å½¢":113937,"ç§ijæĬĢæĪIJæŀľ":113938,"æľĢ好æĺ¯":113939,"éĹ®åĢĻ":113940,"红éħĴ":113941,"åĽĽç§į":113942,"ç¿Ĵæħ":113943,"ç¿Ĵæħ£":113944,"åŀ¦":113945,"éĤ£åıª":113946,"é¢ĨæĤŁ":113947,"çľ¼éĥ¨":113948,"æ³°å®ī":113949,"ä»»æľŁ":113950,"磨æįŁ":113951,"æĽ¿æį¢":113952,"åħ¸ç¤¼":113953,"符åIJĪæĿ¡ä»¶":113954,"è¿ĺæľīä»Ģä¹Ī":113955,"åħ±äº«åįķ车":113956,"åı¯åĪĨ为":113957,"åŃ£åIJİ":113958,"åŃ£åIJİèµĽ":113959,"举èİŀå¸Ĥ":113960,"å¿ĥæĦı":113961,"æīŃæĽ²":113962,"ä½ľä¸ºä¸Ģç§į":113963,"è¿Ļéĥ¨åĪĨ":113964,"åıĤä¸İåĪ°":113965,"ç½ijçIJĥ":113966,"實çı¾":113967,"ç»Ħè£ħ":113968,"åIJijå¤ĸ":113969,"å·¥ä½ľæĸ¹æ¡Ī":113970,"åįģæĿ¡":113971,"課ç¨ĭ":113972,"颤æĬĸ":113973,"åĵ©":113974,"éĤ®å¯Ħ":113975,"亢":113976,"åħįè²»":113977,"秤":113978,"åºĶæĢ¥ç®¡çIJĨ":113979,"åĽĽäºĶ":113980,"éºĴéºŁ":113981,"å¾ĴæŃ¥":113982,"è¨ĺå¾Ĺ":113983,"çĴIJ":113984,"æĺ¯åIJ¦ä¼ļ":113985,"æĦıè§ģåıįé¦Ī":113986,"éļ¾æĢª":113987,"çªį":113988,"交æİ¥":113989,"两åįĥ":113990,"æĩīçĶ¨":113991,"æľŁéĸĵ":113992,"æIJ¬åĪ°":113993,"è®®é¢ĺ":113994,"碧æ¡Ĥ":113995,"碧æ¡ĤåĽŃ":113996,"åģļçĶŁæĦı":113997,"éĻĽä¸ĭ":113998,"è·ĭ":113999,"èĢģ人家":114000,"带åĽŀ":114001,"æŀ¸æĿŀ":114002,"è¡Įéķ¿":114003,"åĨħ容ç®Ģä»ĭ":114004,"梢":114005,"æĮĩæݧ":114006,"éĩįçĹĩ":114007,"ç½ijåıĭ们":114008,"çı¾ä»£":114009,"类产åĵģ":114010,"å¥Ķæ³¢":114011,"渺":114012,"ç²īç¢İ":114013,"è¿Ļåıªæĺ¯":114014,"æ£Ģå¯Łæľºåħ³":114015,"é½Ĭ":114016,"æĪ¿ç§Ł":114017,"å¾·æĭī":114018,"å²ģ以ä¸Ĭ":114019,"纯åĩĢ":114020,"åĪĨå¸ĥåľ¨":114021,"èĥ½å¾ĹåĪ°":114022,"ä¸įå°½":114023,"ç«ŀä»·":114024,"çļĦ带é¢Ĩ":114025,"çļĦ带é¢Ĩä¸ĭ":114026,"ä¸Ńèį¯æĿIJ":114027,"æĿijéķĩ":114028,"ä¸įåı¯éģ¿åħį":114029,"éľ²å¤©":114030,"å°ıå§ijå¨ĺ":114031,"çī©ä»¶":114032,"èijĹä½ľæĿĥ":114033,"æĭĺçķĻ":114034,"éĥ½è§īå¾Ĺ":114035,"æĽ²æĬĺ":114036,"æ·»åĬłåīĤ":114037,"åı¬åĽŀ":114038,"æīİå®ŀæİ¨è¿Ľ":114039,"æĬĦè¢Ń":114040,"åĮĸ身":114041,"缴èIJ¥":114042,"ä¹Łå¸ĮæľĽ":114043,"èį£èªī称åı·":114044,"åįĸç»Ļ":114045,"æľīä¸įåIJĮçļĦ":114046,"å¥ĩçī¹":114047,"éĥ½è®¤ä¸º":114048,"å¦ŀ":114049,"æĪIJéķ¿ä¸º":114050,"辩æĬ¤":114051,"主æķĻç»ĥ":114052,"æ³ķå¸ĪèģĮä¸ļ":114053,"æ¤įåħ¥":114054,"索尼":114055,"åIJ¬è¿ĩ":114056,"ä¹łæĥ¯äºĨ":114057,"夺åıĸ":114058,"éŁĵ":114059,"æľ¬è´¨ä¸Ĭ":114060,"æİ¥åĬĽ":114061,"äºij端":114062,"è¦ģåģļ好":114063,"è·¯çģ¯":114064,"åįıåIJĮåıijå±ķ":114065,"æľīå¾ħ":114066,"æ°´åŁŁ":114067,"æIJľçĭIJé¦ĸ页":114068,"è´¨éĩıå®īåħ¨":114069,"åįģäºĮäºĶ":114070,"åĵ®åĸĺ":114071,"èĵ¬åĭĥåıijå±ķ":114072,"åIJį声":114073,"身亡":114074,"çİĭåºľ":114075,"åİŁåĪĻä¸Ĭ":114076,"çĥĺå¹²":114077,"éģĹæ¼ı":114078,"éĿ¢çĽ®":114079,"åĽ½ä¼ļ":114080,"ä¸Ģ缴éĥ½æĺ¯":114081,"æľīä¸Ģä½į":114082,"éħįæľī":114083,"éĻªçĿĢ":114084,"ä¼ģåĽ¾":114085,"æĮīä¸ĭ":114086,"èĵĿåĽ¾":114087,"æ©ĺ":114088,"大å¤ļæĺ¯":114089,"辩论":114090,"æĹĭå¾ĭ":114091,"æĬ¥éĢģ":114092,"æĿ¡è§Ħå®ļ":114093,"åĬ¨éĿĻ":114094,"åĮĪ奴":114095,"æĭľè®¿":114096,"ä¸ĢåĪĢ":114097,"ä»ĸçŁ¥éģĵ":114098,"主æĿĥ":114099,"ä»ĸæĽ¾":114100,"æĴŃç§į":114101,"å£ģåŀĴ":114102,"çī¢è®°ä½¿åij½":114103,"åľ¨è¿Ļæĸ¹éĿ¢":114104,"æīĭèħķ":114105,"æĶ¯æŀ¶":114106,"ä¾Ĩèĩª":114107,"éĩįå¡ij":114108,"å¤ļå±Ĥ次":114109,"ä»ĭè´¨":114110,"éĿ¢åŃĶ":114111,"潮湿":114112,"åİ¿åŁŁ":114113,"游æĪıå½ĵä¸Ń":114114,"å£ŀ":114115,"åĪĹåĩº":114116,"èµĽåĮº":114117,"å¤ļåįĬ":114118,"éĩįçĤ¹å·¥ä½ľ":114119,"æĪij们å¿ħé¡»":114120,"æŁıæŀĹ":114121,"é²ģèĥ½":114122,"æĸ½å±ķ":114123,"åIJĦåĮº":114124,"åħįç¨İ":114125,"èµĽåIJİ":114126,"æľĢéĩįè¦ģ":114127,"ä¸Ģ个好çļĦ":114128,"è¿Ŀæ³ķè¿Ŀè§Ħ":114129,"äºĨè§£æĽ´å¤ļ":114130,"æķ¬è¯·":114131,"ç¬ijçĿĢ说":114132,"ä¸įæĸŃåıijå±ķ":114133,"æijĦå½±å¸Ī":114134,"以éĺ²":114135,"çĤ¸å¼¹":114136,"声åĵį":114137,"ç¤ģ":114138,"æĩ¿":114139,"èĪĨæĥħ":114140,"èĩªçĶ±è´¸æĺĵ":114141,"æķıæį·":114142,"ä¸ī大éĺ¶æ®µ":114143,"èĭĶ":114144,"æĹºåŃ£":114145,"ä¸į满æĦı":114146,"微信åı·":114147,"修为":114148,"çł´è£Ĥ":114149,"éĢĥ离":114150,"æ¯ıèĤ¡":114151,"è¾¾ä¸įåĪ°":114152,"æ¯ıå¹´éĥ½":114153,"çģ¯ç¬¼":114154,"æŃ¤åŁºç¡Ģä¸Ĭ":114155,"åĥı个":114156,"åĪĨ娩":114157,"æĻ¾":114158,"ä¸įèĩ³äºİ":114159,"红线":114160,"误解":114161,"ä¸ľè·¯":114162,"æ·®å®ī":114163,"产åѦ":114164,"产åѦçłĶ":114165,"èī¾æ»ĭ":114166,"èī¾æ»ĭçĹħ":114167,"åīįæıIJæĺ¯":114168,"æ¯ıä¸Ģ天":114169,"ä¸ĥ大":114170,"æłijåı¶":114171,"èµ°å¾Ĺ":114172,"è¿Ļ两ç§į":114173,"æİıåĩº":114174,"æİIJ":114175,"é¢Ĩ导èĢħ":114176,"ä¸Ģæľµ":114177,"个å¤ļæľĪ":114178,"ä¸Ńåħ³":114179,"ä¸Ńåħ³æĿij":114180,"课åłĤæķĻåѦ":114181,"大åĴĸ":114182,"éģĭçĶ¨":114183,"è¯ļæĦı":114184,"ç»ĦåĽ¾":114185,"è¯ķçĿĢ":114186,"ä¹Ķæ²»":114187,"è¿ĺä¸įæĺ¯":114188,"æľīæĽ´å¥½çļĦ":114189,"åIJİå¤ĩ":114190,"æĸ°çĶŁåĦ¿":114191,"æ°Ķè¡Ģ":114192,"æ²¥éĿĴ":114193,"å±ıéļľ":114194,"æ¥ŃåĭĻ":114195,"æĪij以为":114196,"éķ¿çĽ¸":114197,"èĢģçĪ¸":114198,"éķĩæ±Ł":114199,"æľºæ¢°è®¾å¤ĩ":114200,"ä½Ĩæĺ¯å¦Ĥæŀľ":114201,"åĿļå®ļä¸į":114202,"åĿļå®ļä¸į移":114203,"åĨ²éĶĭ":114204,"ç®Ģ缴æĺ¯":114205,"åĤ¨èĵĦ":114206,"纯çĶµåĬ¨":114207,"漫æŃ¥":114208,"举起":114209,"æģ¶æĢ§":114210,"è¨ĺéĮĦ":114211,"èģĮèĥ½éĥ¨éŨ":114212,"åħ¨éķ¿":114213,"鼻è¦ĸ":114214,"ä¹³èħº":114215,"ä½ķå¤Ħ":114216,"æ¶Īæŀģ":114217,"æŃ£å¤Ħäºİ":114218,"å®īå®ģ":114219,"æĪIJéķ·":114220,"åıĻè¿°":114221,"æºĥçĸ¡":114222,"ä½Ĩçİ°åľ¨":114223,"女æĺŁ":114224,"å©´å¹¼åĦ¿":114225,"æĬķèŀįèµĦ":114226,"éĹ®éĹ®":114227,"æıŃå¼Ģ":114228,"è¯ı":114229,"åIJįå½ķ":114230,"èĺijèıĩ":114231,"åIJĬ顶":114232,"æ¹ĸåĮº":114233,"åįĸåľº":114234,"建ç¯":114235,"建ç¯ī":114236,"èݽ":114237,"åIJ¬åIJ¬":114238,"ç«ŀäºīä¼ĺåĬ¿":114239,"åĩºä»»":114240,"æľī两ç§į":114241,"æ©±æŁľ":114242,"褪":114243,"è¯ķåį·":114244,"ç»ıæµİæĬĢæľ¯":114245,"æ·±å±Ĥ":114246,"éĩįè¦ģåĨħ容":114247,"é£İæݧ":114248,"çĬ¶æĢģä¸ĭ":114249,"éĥ¨éĸĢ":114250,"广汽":114251,"è§Ĥæij©":114252,"éģĹçķĻ":114253,"转账":114254,"æĮģä»ĵ":114255,"æĢ»è®¡":114256,"åľĺéļĬ":114257,"æĪ¿ä¸ľ":114258,"éĺĢéŨ":114259,"åħ¬åħ³":114260,"åħ³åĪĩ":114261,"èĤĺ":114262,"æķ¸æĵļ":114263,"ä¸īåįģå¹´":114264,"è§ģè¯ģäºĨ":114265,"å±Ĩ":114266,"çģ°å°ĺ":114267,"æ¦ľé¦ĸ":114268,"è¦ĨçĽĸçİĩ":114269,"ä»Ļ女":114270,"çĶŁäº§æĢ»":114271,"çĶŁäº§æĢ»åĢ¼":114272,"æĪ¿è´·":114273,"æ±ŁåĮº":114274,"åħħçĶµæ¡©":114275,"çĻ¾åIJĪ":114276,"確èªį":114277,"转移åĪ°":114278,"éĥ½æĹłæ³ķ":114279,"纪念é¦Ĩ":114280,"çŃ¾ç½²äºĨ":114281,"并ä¸įå¤ļ":114282,"æĮł":114283,"ä¸į太好":114284,"ä¸ĸ代":114285,"误导":114286,"é«ĺ峰论åĿĽ":114287,"åħ¼å®¹":114288,"龸æ°Ķ":114289,"æĿ¥è®¿":114290,"æīĢ带æĿ¥çļĦ":114291,"æĺ¯ä¸Ģéĥ¨":114292,"æĻļé¥Ń":114293,"åİĨ代":114294,"åIJ¦åīĩ":114295,"ä¹ħä¹ħ":114296,"æľīæķĪæľŁ":114297,"诱åıij":114298,"æĢ»èµĦ产":114299,"æľ¬èº«å°±æĺ¯":114300,"çĶŁäº§åİĤ家":114301,"æŶ髦":114302,"èĢIJçĶ¨":114303,"ä»İå°ıå°±":114304,"æĿ¡çº¦":114305,"èĭ±åĭĩ":114306,"ä¿Ĺè¯Ŀ说":114307,"寺åºĻ":114308,"å¿ĥçIJĨåģ¥åº·":114309,"ä»Ģä¹Īäºĭæĥħ":114310,"æ±īåŃĹ":114311,"çķĻä½ı":114312,"åįĹè·¯":114313,"ä¸ī项":114314,"丢äºĨ":114315,"æĥ³åĪ°äºĨ":114316,"çŃ¹éĽĨ":114317,"éĻĦåĬłåĢ¼":114318,"西è£ħ":114319,"ä¹ĭä½ľ":114320,"åģļçļĦäºĭ":114321,"çķ¶æĤ¨":114322,"çķ¶æĤ¨åľ¨":114323,"é¦ĸ款":114324,"ä¸įåľ¨ä¹İ":114325,"å·¥ç¨ĭæĸ½å·¥":114326,"éļIJéļIJ":114327,"åıĺ身":114328,"沿éĢĶ":114329,"æĤłæĤł":114330,"ä¿Ŀæļĸ":114331,"çĶŁæ´»åŀĥåľ¾":114332,"渤海":114333,"æŃ¦ä¾ł":114334,"女主è§Ĵ":114335,"举ä¾ĭ":114336,"æ·¨":114337,"çĻ½é¢Ĩ":114338,"è£ĻåŃIJ":114339,"è¿Ķè¿ĺ":114340,"è¿Īåĩº":114341,"é¾ĻéŨ":114342,"ç»ıæµİä½ĵ":114343,"æĶ¶å®ĺ":114344,"çķĮéĻIJ":114345,"è·³åĩº":114346,"åįĩåĢ¼":114347,"绵éĺ³":114348,"çĸ¤çĹķ":114349,"çľĭæ¸ħ":114350,"æĭĴçµķ":114351,"è¥Ħéĺ³":114352,"课å¤ĸ":114353,"åŃIJåŃĻ":114354,"æŃĮè¯į":114355,"æĪIJåIJį":114356,"溶液":114357,"åĦĴ家":114358,"åķĨä¸ļåĮĸ":114359,"辨åĪ«":114360,"å¤ļè¾¾":114361,"ç½ijåºĹ":114362,"ä¹Ŀ大":114363,"ä¹Ŀ大精ç¥ŀ":114364,"æŃ¤ä¸¾":114365,"è¿ŀè½½":114366,"ä¸ĢåĢĭ人":114367,"èī²æ³½":114368,"涵çĽĸäºĨ":114369,"è¦ıåĬĥ":114370,"åĽ½æĥħ":114371,"åį«çĶŁåģ¥åº·":114372,"积æŀģåĵįåºĶ":114373,"æĭĻ":114374,"åĪ¶åĬ¨":114375,"æĥ³è±¡åĬĽ":114376,"çļĦä¹IJ趣":114377,"å¼łå®¶çķĮ":114378,"å´İ":114379,"éĩįåŀĭ":114380,"å¤ĸå¢Ļ":114381,"æĶ¾åѦ":114382,"è®¤çľŁåŃ¦ä¹ł":114383,"è´¬åĢ¼":114384,"æ³ķæ¡Ī":114385,"æĬ¤èĤ¤åĵģ":114386,"éĻ·åħ¥äºĨ":114387,"请æĤ¨":114388,"åŀ¢":114389,"æķĻèĤ²èµĦæºIJ":114390,"交æĺĵå¹³åı°":114391,"æŶè£ħ":114392,"ä¼łæŁĵçĹħ":114393,"æ¹ĸæ³Ĭ":114394,"èµĦ管":114395,"åݨå¸Ī":114396,"éĹľéį":114397,"éĹľéįµ":114398,"åĵĪåĵĪåĵĪ":114399,"çĽĹçªĥ":114400,"çĶľç¾İ":114401,"åºĦåĽŃ":114402,"缮åīįå·²ç»ı":114403,"è¾¹ä¸Ĭ":114404,"çģ«èĬ±":114405,"æĬ¥è®°èĢħ":114406,"æģĭæĥħ":114407,"ç´§åĩij":114408,"æ°´æµģ":114409,"è¿Ļæĺ¯æĪij们":114410,"æ³¥åľŁ":114411,"æĽ¾ä»»":114412,"æĸ¹è¨Ģ":114413,"åij¨åħŃ":114414,"åı·æ¥¼":114415,"ä¼ijåģĩ":114416,"误ä¼ļ":114417,"åĽ½åĢº":114418,"åīįå¤ķ":114419,"ä¸¤å¼ł":114420,"éĹ«":114421,"éŃĶ鬼":114422,"æĬĬæĮģ":114423,"èĬĤèĥ½çݯä¿Ŀ":114424,"æ¸ħæ´ģèĥ½æºIJ":114425,"èĤ¥æĸĻ":114426,"é«ĺé¢ij":114427,"å°±æľīäºĨ":114428,"交ä¼ļ":114429,"没éĴ±":114430,"éĽħæĢĿ":114431,"è¦ģåıĬæŶ":114432,"åŁ¹åħ»åѦçĶŁ":114433,"欣åĸľ":114434,"çĥŃæ°´åĻ¨":114435,"é¾Ļæ¹ĸ":114436,"äºĮ楼":114437,"æĸ°æµªè´¢ç»ı":114438,"æĸ°åĬ¨èĥ½":114439,"èµ£å·ŀ":114440,"æĭ³å¤´":114441,"æµģåIJij":114442,"ä¹Łæĺ¯å¾Ī":114443,"åıijåĶ®":114444,"ä¸ŃåIJ«æľī":114445,"åIJĵå¾Ĺ":114446,"å·¨æĺŁ":114447,"æĹłæīĢè°ĵ":114448,"æ¯ĽåŃĶ":114449,"åħ¬åħ±äº¤éĢļ":114450,"çĤİçĥŃ":114451,"èµ·èįī":114452,"åĬłçĽŁåķĨ":114453,"说ä¸įåĩº":114454,"大åѦæ¯ķä¸ļ":114455,"å·¥ä¸ļåĽŃ":114456,"éłĺåŁŁ":114457,"åºĨåħ¸":114458,"æµģ产":114459,"èģ²éŁ³":114460,"ä¼¼ä¹İæĺ¯":114461,"è´§æºIJ":114462,"æ·±åĪĩ":114463,"æ²»çĸĹæĸ¹æ³ķ":114464,"èµĦæºIJéħįç½®":114465,"網åıĭ":114466,"çĶ£":114467,"亥":114468,"èº²åľ¨":114469,"社ç§ij":114470,"è»Łé«Ķ":114471,"女è£ħ":114472,"æŃ¡è¿İ":114473,"综åIJĪå®ŀåĬĽ":114474,"æł¼å°ĩ":114475,"åħļåı²åŃ¦ä¹ł":114476,"æľĢåŁºæľ¬":114477,"æľĢåŁºæľ¬çļĦ":114478,"çľĭæľĽ":114479,"åıĹè´¿":114480,"ä¸įä»ħèĥ½":114481,"ä½ķå¿ħ":114482,"ä¸Ģ个å°ıæŶ":114483,"ç¾Į":114484,"æĭĽæĶ¶":114485,"çĤĴèĤ¡":114486,"æĿijå¹²éĥ¨":114487,"缸çĪ±":114488,"æ½ľèĥ½":114489,"ä¹į":114490,"æĹ¶è¾°":114491,"欣æħ°":114492,"éĵ¶è¡Įä¸ļ":114493,"çĭŃçªĦ":114494,"éĩįçĤ¹é¢ĨåŁŁ":114495,"çİ°å®ŀçĶŁæ´»":114496,"éĮ¯èª¤":114497,"æĸ°è§Ħ":114498,"滥çĶ¨":114499,"æŶä¸į":114500,"æŶä¸įæŶ":114501,"帳èĻŁ":114502,"ç¨Ģ缺":114503,"åIJij举":114504,"ä¿Ŀåģ¥åĵģ":114505,"çıŃéķ¿":114506,"äºĴåĭķ":114507,"笼罩":114508,"æ½Ľ":114509,"æļĸå¿ĥ":114510,"è½°çĤ¸":114511,"åºĨ幸":114512,"è²Įä¼¼":114513,"æĵº":114514,"èĢIJ磨":114515,"ä¸ĵä¸ļ人士":114516,"ä¸ĢèĪ¬éĥ½æĺ¯":114517,"æ¼³å·ŀ":114518,"åħ¨èĩªåĬ¨":114519,"å½ķçĶ¨":114520,"大è·Į":114521,"æľīæķĪæĢ§":114522,"èĩªåĭķ":114523,"ä¸ī个æĸ¹éĿ¢":114524,"港åĮº":114525,"信貸":114526,"éĢļè¯Ŀ":114527,"é«ĺ涨":114528,"æ³Ħæ¼ı":114529,"éħįä¸Ĭ":114530,"åħļå·¥å§Ķ":114531,"被认为":114532,"被认为æĺ¯":114533,"ä¸įä¼ļåĨį":114534,"è°ĥåīĤ":114535,"åıĤèĤ¡":114536,"èĦ±åıij":114537,"å¿łå®ŀ":114538,"åĨħåĪĨæ³Į":114539,"ç¹ģå¿Ļ":114540,"åıĮåĪĽ":114541,"é©»æĿij":114542,"åĪĴç®Ĺ":114543,"éģİä¾Ĩ":114544,"åľ£ç»ı":114545,"èıľé¸Ł":114546,"æĭ¼å¤ļå¤ļ":114547,"ä¸ŃåĽ½æ±½è½¦":114548,"çĥŁèįī":114549,"缴æµģ":114550,"äºĨä¸Ģåı£æ°Ķ":114551,"ä½İæĪIJæľ¬":114552,"æī¾åĽŀ":114553,"èĩªåįij":114554,"總æĺ¯":114555,"æĸĩåĮĸåĪĽæĦı":114556,"天河":114557,"樱æ¡ĥ":114558,"éªijåħµ":114559,"éĩĮéĿ¢æľī":114560,"çİ®":114561,"èĥ½æī¾åĪ°":114562,"éĢĥè·ij":114563,"åĪĩå°Ķ":114564,"åĪĩå°Ķ西":114565,"以ä¸ĭæĺ¯":114566,"å²³éĺ³":114567,"çļĦæ¦Ĥçİĩ":114568,"æĬµåĪ¶":114569,"å¸ĪäºĭåĬ¡":114570,"å¸ĪäºĭåĬ¡æīĢ":114571,"åĩĨæŶ":114572,"屬æĸ¼":114573,"订è´Ń":114574,"åįłæį®äºĨ":114575,"ä¸ŃéĢĶ":114576,"å°ĭ":114577,"é»ij马":114578,"åİ¿åħ¬å®īå±Ģ":114579,"ä¸ĥæľĪ":114580,"èī²ç´ł":114581,"å¿ĥèĦıçĹħ":114582,"æŶéĻIJ":114583,"æ¯įåħ¬åı¸":114584,"å¹ķåIJİ":114585,"ä¸Ĭæ¦ľ":114586,"åĢ¾åIJijäºİ":114587,"纸ä¸Ĭ":114588,"æ¡ĵ":114589,"éĽĨä½ĵç»ıæµİ":114590,"æĥħå¢ĥ":114591,"è¦ģåģļåĪ°":114592,"ç©į極":114593,"åıªæĢķ":114594,"æ¹ĺ西":114595,"çļ±çº¹":114596,"åħ¨åľĭ":114597,"çĦ¡è«ĸ":114598,"好æĦŁ":114599,"åįķä»·":114600,"è¿Ľç¨ĭä¸Ń":114601,"æĺĨä»ij":114602,"åĪĽå®¢":114603,"åħħæĸ¥":114604,"åħĪæĬĬ":114605,"该æĢİä¹ĪåĬŀ":114606,"åĵģå¾·":114607,"åħ¨éĿ¢åıijå±ķ":114608,"è¨ĪåĬĥ":114609,"æĢ»å·¥ä¼ļ":114610,"ä½Ľå±±å¸Ĥ":114611,"æĬĹè¡¡":114612,"å¼Ģåľº":114613,"éĴ±å¸ģ":114614,"åıĭ们":114615,"å«īå¦Ĵ":114616,"ç´¢èµĶ":114617,"è®ĬåĮĸ":114618,"æĮ¤åİĭ":114619,"æĮijè¡ħ":114620,"çŃīä¸Ģæī¹":114621,"æĿ¨æ¬¢":114622,"ä¸ĵ家åѦèĢħ":114623,"èĥ½è¾¾åĪ°":114624,"èµ°è¿ij":114625,"è´«åĽ°åľ°åĮº":114626,"éĻIJæľŁ":114627,"ä¸į平衡":114628,"åĽ½åĨħå¸Ĥåľº":114629,"èµĽåľº":114630,"éħįèµĦ":114631,"è¦ģèĢĥèĻij":114632,"ä¸ĩåı°":114633,"æľĪæľ«":114634,"éĶ¥":114635,"åŃ«":114636,"æݥ触åĪ°":114637,"åĩºäº§":114638,"æķĻåѸ":114639,"ä½ľå¼Ĭ":114640,"çļĦæľĢåIJİä¸Ģ":114641,"ä¿ĥæĪIJ":114642,"åIJ¸åıĸ":114643,"æ½ľèīĩ":114644,"被éªĹ":114645,"è¾ĵäºĨ":114646,"çĭIJçĭ¸":114647,"åįĩéĻį":114648,"è¿ĻäºĽä¸ľè¥¿":114649,"æĬķèµĦåŁºéĩij":114650,"çĶŁçī©åѦ":114651,"ç½ij绾èIJ¥éĶĢ":114652,"åIJijè®°èĢħ":114653,"èįīåľ°":114654,"æĢ¯":114655,"æľįåĬ¡èĥ½åĬĽ":114656,"éĥģéĹ·":114657,"åįķåĵģ":114658,"å¾Ĺ罪":114659,"æĺĵäºİ":114660,"个å¤ļå°ıæŶ":114661,"éĩįä»»":114662,"ä¸Ĭå®ĺ":114663,"æľ¬éĩij":114664,"çı¾åł´":114665,"溢价":114666,"æĺŁè¾°":114667,"æ´»åĬ¨çİ°åľº":114668,"丹麦":114669,"å¸Ŀçİĭ":114670,"æŁ¥æĺİ":114671,"åŃĺåľ¨äºİ":114672,"é¦Ļæ°´":114673,"æĬ½æ£Ģ":114674,"å®ŀéĻħä¸Ĭæĺ¯":114675,"æĸ°å¾ģç¨ĭ":114676,"è´¢åĬ¡ç®¡çIJĨ":114677,"æİĽ":114678,"åĨľåİĨ":114679,"éĥ½èĥ½å¤Ł":114680,"éĤ¯éĥ¸":114681,"çľŁå¯¦":114682,"ç»Ĭ":114683,"åĨµä¸Ķ":114684,"置身":114685,"ç¥Ī祷":114686,"çĿģå¼Ģ":114687,"æĮĩçĤ¹":114688,"å¼Ģæľº":114689,"西å®ģ":114690,"åĮĹ约":114691,"积水":114692,"åĩºåĬ¨":114693,"åıijå±ķ模å¼ı":114694,"转æĬĺ":114695,"èĢĥçĤ¹":114696,"æľīç½ijåıĭ":114697,"è´«åĽ°æĿij":114698,"æĪijä»¬çŁ¥éģĵ":114699,"åĪĨéĶĢ":114700,"å±±èĦī":114701,"æ¯ĶæĭŁ":114702,"ä¼°ç®Ĺ":114703,"æĶ¹å»º":114704,"壮è§Ĥ":114705,"ç§īæĮģ":114706,"æıª":114707,"ç¦Ģ":114708,"åĮĸåѦåĵģ":114709,"ä¸ŃåĽ½åĪ¶éĢł":114710,"ä¸Ģæŀ¶":114711,"æīįè¡Į":114712,"æĭĽå¾ħ":114713,"åıĺæį¢":114714,"åīį线":114715,"幸好":114716,"è¿Ļæł·çļĦè¯Ŀ":114717,"å¿ĥè¡Ģ管":114718,"æĢ§çĸ¾çĹħ":114719,"åħ¨èĥ½":114720,"åĪij侦":114721,"ä¿¡æģ¯åıijå¸ĥ":114722,"æĺ¾çĦ¶æĺ¯":114723,"éĿĴéĵľ":114724,"åIJĥä»Ģä¹Ī":114725,"çĶµä»·":114726,"æ³ķå¾ĭè§Ħå®ļ":114727,"çħ²":114728,"çĵ·åĻ¨":114729,"èĤīç±»":114730,"æıĴåħ¥":114731,"åĹľ":114732,"è¿Łè¿Ł":114733,"ä¸ĢçĤ¹éĥ½ä¸į":114734,"è¿ĺåĮħæĭ¬":114735,"èĪįä¸įå¾Ĺ":114736,"æłĩå¿ĹæĢ§":114737,"æľĪ以æĿ¥":114738,"ç³ĸæŀľ":114739,"éĥ½åºĶ该":114740,"çݯå¢ĥåį«çĶŁ":114741,"èĪªè¡Į":114742,"éĥijéĩį":114743,"ç½ijæĬķ":114744,"åįģä½³":114745,"ç§ģä¸ĭ":114746,"æļ´è·Į":114747,"åĬłå¿«åıijå±ķ":114748,"产åĵģçłĶåıij":114749,"åĪĽéĢłåĩº":114750,"æĢ»è§īå¾Ĺ":114751,"åºķçĽĺ":114752,"èķĬ":114753,"åĩºå¸Ńä¼ļè®®":114754,"主æĿ¿":114755,"æĹ¥æĻļéĹ´":114756,"å®ĺæĸ¹å¾®åįļ":114757,"å¼ķçĶ¨æĹ¥æľŁ":114758,"åī¯æķĻæİĪ":114759,"çĶµåŃIJ产åĵģ":114760,"è¡°éĢĢ":114761,"çķĻåŃĺ":114762,"çģ«åĬĽ":114763,"çĴ§":114764,"çļĤ":114765,"åħ¼åħ·":114766,"éĩįè¿Ķ":114767,"é¢Ĩçķ¥":114768,"åĪĩéĻ¤":114769,"åĨįçĶŁèĥ½æºIJ":114770,"å®ŀåľ¨å¤ª":114771,"çIJĨ论ä¸Ĭ":114772,"ä¸īå±Ĥ":114773,"ä¸ĸçķĮåIJĦåĽ½":114774,"å®ľæĺĮ":114775,"èĢ³è¾¹":114776,"宽æķŀ":114777,"æ±īæĹı":114778,"çĻ½çĻ½":114779,"è¿ĻéĩĮéĿ¢":114780,"çĶŁæ´»ä¹łæĥ¯":114781,"èµŀèµı":114782,"çĶ·å£«":114783,"ä¸Ńä¿Ħ":114784,"车祸":114785,"åīĤéĩı":114786,"éĻ¤åİ»":114787,"左边":114788,"çŃijçī¢":114789,"çīĽå¸Ĥ":114790,"家åĬ¡":114791,"åķĥ":114792,"ç½®æį¢":114793,"ç´«å¤ĸ":114794,"ç´«å¤ĸ线":114795,"å¾Ģåīį":114796,"åĬĽåѦ":114797,"ç´§è·Ł":114798,"缮çļĦåľ¨äºİ":114799,"ç»®":114800,"ç¥Ĥ":114801,"宣è¨Ģ":114802,"äºĮæ°§åĮĸ":114803,"äºĮæ°§åĮĸ碳":114804,"æĹłç¼ĺ":114805,"ç²¾éĢļ":114806,"診":114807,"å¼ķåıijäºĨ":114808,"æľĢåħĪ":114809,"派驻":114810,"ä¸įå¿į":114811,"æĪijçĪ¸":114812,"å¹´ä¸ĭåįĬå¹´":114813,"æ·ĭå·´":114814,"没éĹ®é¢ĺ":114815,"åºĹåĨħ":114816,"è·ŁæĪij说":114817,"çĶŁäº§çĶŁæ´»":114818,"è§ĤæľĽ":114819,"æ¸į":114820,"被æī§è¡Į":114821,"被æī§è¡Į人":114822,"èĪľ":114823,"æݺ":114824,"ä¸Ģç§Ĵ":114825,"èįīåĿª":114826,"åij¼åĴĮ":114827,"åij¼åĴĮ浩":114828,"åij¼åĴĮ浩çī¹":114829,"人æ°ijéĵ¶è¡Į":114830,"çĦķåıij":114831,"è¯ģåĪ¸äº¤æĺĵ":114832,"çķĶ":114833,"æľºèĥ½":114834,"妾":114835,"æĻļå¹´":114836,"å·¥åķĨèģĶ":114837,"åİŁåŀĭ":114838,"è§Ĵ度çľĭ":114839,"æĬ¥ç¤¾":114840,"è¯įæĿ¡":114841,"躲éģ¿":114842,"éĩįåIJ¯":114843,"å¤ķéĺ³":114844,"èĤ¡æĿĥ转让":114845,"åľ¨ä¸Ģ":114846,"åľ¨ä¸ĢæĹģ":114847,"社ä¼ļåĮĸ":114848,"åıijå±ķåİĨç¨ĭ":114849,"æĭĸæ¬ł":114850,"使èĢħ":114851,"ä¸İåIJ¦":114852,"æĸ°å±ĢéĿ¢":114853,"ä»Ĭ天æĪij们":114854,"é½IJèģļ":114855,"对æĪij说":114856,"éĢĴ交":114857,"æľªæĽ¾":114858,"èİĬ":114859,"éĸī":114860,"亲æīĭ":114861,"è§ĴéĢIJ":114862,"æľīé»ŀ":114863,"ç¨İçİĩ":114864,"ä½İ声":114865,"é»ĺå¥ij":114866,"æĻ®æ³ķ":114867,"大ä¸ĵ":114868,"第äºĮ大":114869,"ä½ıåĿĢ":114870,"æĶ¾è¿Ľ":114871,"äºĮæĪĺ":114872,"亲身":114873,"åĽºåĮĸ":114874,"ä¸ĭ乡":114875,"åħ³éĶ®æĬĢæľ¯":114876,"åĽŀæĥ³":114877,"æĬ¥åĪĬ":114878,"æ¶ĤæĬ¹":114879,"èĹıçĿĢ":114880,"ç¥ĿæĦ¿":114881,"åįĩ温":114882,"çĶļèĩ³è¿ŀ":114883,"åħ¬åħĥåīį":114884,"ç¾İæĸ¹":114885,"è¯ļå®ŀ":114886,"æĹłåģ¿":114887,"åīµæ¥Ń":114888,"å°ıå¿ĥ翼":114889,"å°ıå¿ĥ翼翼":114890,"两æīĭ":114891,"温馨æıIJ示":114892,"ä»¿çľŁ":114893,"æĥ¶":114894,"èĥ¡åŃIJ":114895,"å·¥ä½ľç«Ļ":114896,"硬çĽĺ":114897,"ç«¿":114898,"åĤ³éĢģ":114899,"åħ¨æł¡":114900,"é²ľæ´»":114901,"çĴĢçĴ¨":114902,"ç»ĵå°¾":114903,"æį¢æĿ¥":114904,"æĪĢ":114905,"ä½İä½į":114906,"ä¸ĩåħĥ以ä¸Ĭ":114907,"åĬłåĪĨ":114908,"æݨä»ĭä¼ļ":114909,"çIJĨèµĶ":114910,"å¾·å°Ķ":114911,"æĬĹè®®":114912,"æ´¼":114913,"åĸ§":114914,"åŁİéĻħ":114915,"å¾Īæ£Ĵ":114916,"人æŃ»äº¡":114917,"ä¼ļå±ķä¸Ńå¿ĥ":114918,"äºĴèģĶäºĴéĢļ":114919,"èĸĦèĨľ":114920,"éĩįé»ŀ":114921,"ç¦ģæ¯Ĵ":114922,"åĨ·ç¬ij":114923,"大家åı¯ä»¥":114924,"é¦ĸ缸":114925,"è¿ijè·Ŀ离":114926,"æµ®çİ°":114927,"ç§ĺè¯Ģ":114928,"èµ·é£ŀ":114929,"æIJ¶":114930,"羣åģĩ":114931,"æģķ":114932,"å°ıåºĹ":114933,"æ°ijçľ¾":114934,"åıijå¸ĥåħ¬åijĬ":114935,"侧éĩį":114936,"å¾ĺå¾Ĭ":114937,"æĢĶ":114938,"æªIJ":114939,"æķ°çĽ®":114940,"åī¯ç§ĺ书éķ¿":114941,"两åı¥":114942,"éļIJçŀĴ":114943,"åıĮåıĮ":114944,"æīĭæĦŁ":114945,"èij¡äº¬":114946,"éģĹå¿ĺ":114947,"鬥":114948,"è¿Ļä¸ªåľ°æĸ¹":114949,"说çļĦè¯Ŀ":114950,"å·¡åĽŀ":114951,"è¿Ŀç«ł":114952,"æī¾å·¥ä½ľ":114953,"æĶ¯çIJĥéĺŁ":114954,"裡éĿ¢":114955,"æĺ¾ç¤ºåĩº":114956,"èĩ³å°Ĭ":114957,"两级":114958,"åīį段æŶéĹ´":114959,"çĺ¦èº«":114960,"èĤ¢ä½ĵ":114961,"æ¯į親":114962,"æīĭç»Ńè´¹":114963,"汽车è¡Įä¸ļ":114964,"æİ©çĽĸ":114965,"æݧèĤ¡éĽĨåĽ¢":114966,"åı£å¾Ħ":114967,"æĶ¿çŃĸæݪæĸ½":114968,"海绵":114969,"åħ¨éķĩ":114970,"äºĭåħ³":114971,"å¸Ńæī§è¡Į":114972,"å¸Ńæī§è¡Įå®ĺ":114973,"éĤ£æ¬¡":114974,"åı¯èĥ½åĩºçİ°":114975,"ä¸Ńå¿ĥåŁİå¸Ĥ":114976,"翻身":114977,"ä¹Łç®Ĺ":114978,"ä¾µçķ¥":114979,"åĸĩåıŃ":114980,"æ¯ı次éĥ½":114981,"è§ħ":114982,"éĻ¢éĻ¢éķ¿":114983,"å§ĭäºİ":114984,"èѦåĬ¡":114985,"èį¯æĿIJ":114986,"å±łæĿĢ":114987,"æľ¬èº«å°±":114988,"éļıæŶéļı":114989,"éļıæŶéļıåľ°":114990,"åĶ®åįĸ":114991,"æĹłäººé©¾é©¶":114992,"é¢ħ":114993,"åĵģ質":114994,"åĺ²ç¬ij":114995,"è·ijåİ»":114996,"åħĭéĩĮæĸ¯":114997,"çķ¸å½¢":114998,"修饰":114999,"磩éĺµ":115000,"éŁ³ä¹IJä¼ļ":115001,"æŁ³å·ŀ":115002,"齡":115003,"ä¼ļè°Ī":115004,"æŃ£çīĪ":115005,"ä¹ŁåIJĮæł·":115006,"æļ§æĺ§":115007,"è¡ĮæĶ¿éĥ¨éŨ":115008,"ä¹ĸä¹ĸ":115009,"èĤ¤èī²":115010,"æŶ任":115011,"羣åĪĩ":115012,"æľĪä¸ĭ":115013,"æľĪä¸ĭæŬ":115014,"举æĸ¹è´¢å¯Į":115015,"è£ħä¿®åħ¬åı¸":115016,"éĢĢè¿ĺ":115017,"åĭĺå¯Ł":115018,"åĵ¥ä¼¦":115019,"åĵ¥ä¼¦æ¯Ķäºļ":115020,"çĭ¬ä¸Ģ":115021,"çĭ¬ä¸ĢæĹł":115022,"çĭ¬ä¸ĢæĹłäºĮ":115023,"è°ĥåij³":115024,"åİĭè¿«":115025,"åħ¨çIJĥæľĢ大":115026,"åī¯æł¡éķ¿":115027,"æĽ´ä½İ":115028,"åĪĨéĴŁåIJİ":115029,"åĽŀä¾Ĩ":115030,"åĪ¶åīĤ":115031,"åijĬè¯ī大家":115032,"çĤ¹éĴŁ":115033,"åįģä¸īå±Ĭ":115034,"åij¨åĽĽ":115035,"è¿Ļæł·ä¸Ģ":115036,"è¿Ļæł·ä¸ĢæĿ¥":115037,"èĭŁ":115038,"æľĽåİ»":115039,"æĪIJè¯Ń":115040,"å½ĵåį³":115041,"ç¬ij声":115042,"ä¹ĭåĬ¿":115043,"åĪijäºĭæ¡Ī件":115044,"æĮĤçĿĢ":115045,"ä½ķç§į":115046,"å°ı游æĪı":115047,"åĽ½å®¶æĪĺçķ¥":115048,"åĨ·åĨ·":115049,"å®ľå®¾":115050,"æIJºç¨ĭ":115051,"è¶ĭäºİ":115052,"åıįçľģ":115053,"常说":115054,"ä¸ĩæĪ·":115055,"åĥµå°¸":115056,"åįĥä¸ĩåĪ«":115057,"åıijçİ°éĹ®é¢ĺ":115058,"åı¯çŁ¥":115059,"éŨæĪ·ç½ijç«Ļ":115060,"åģ¥åº·äº§ä¸ļ":115061,"åı³è¾¹":115062,"æµ·è¿IJ":115063,"è¿ijä¹İ":115064,"åĮ»æ²»":115065,"æĢ»ç®Ĺ":115066,"ä¸ĢåĪĨéĴŁ":115067,"æĭ§":115068,"ä¹Łæľīä¸ĢäºĽ":115069,"ä¾ĽçĶµåħ¬åı¸":115070,"å»īä»·":115071,"帮ä»ĸ":115072,"æŃ¤æ¬¡æ´»åĬ¨":115073,"åıªèĥ½è¯´":115074,"èĬĭ":115075,"çīĩ段":115076,"åŃĺåľ¨éĹ®é¢ĺ":115077,"ä½łä¼ļåıijçİ°":115078,"è½®å»ĵ":115079,"ç½ijéĢļ":115080,"æ»¨æ±Ł":115081,"æİĪä¿¡":115082,"é»İæĺİ":115083,"ä¸įå±ŀäºİ":115084,"约åįł":115085,"éķ¿æ²Ļå¸Ĥ":115086,"èĥļèĥİ":115087,"åħĥ件":115088,"éĻĨåĨĽ":115089,"購買":115090,"æĮĩæľĽ":115091,"å®ŀä¹łçĶŁ":115092,"çī¹çĤ¹æĺ¯":115093,"çıłæ±Ł":115094,"çľĭä¸įåĩº":115095,"ä¸įè§ģäºĨ":115096,"ç¼ī":115097,"éĺµèIJ¥":115098,"åĶIJæľĿ":115099,"没å¿ħè¦ģ":115100,"åĽ½åľŁèµĦæºIJ":115101,"ç»ıæµİåŃ¦å®¶":115102,"åIJĪèĤ¥å¸Ĥ":115103,"çIJ¢ç£¨":115104,"ç¡®åĪĩ":115105,"åŁİå¸Ĥåıijå±ķ":115106,"çŃ·åŃIJ":115107,"人æ°ijæľįåĬ¡":115108,"满åĪĨ":115109,"è¿·ä¿¡":115110,"ä½ľèĢħæľ¬äºº":115111,"æĸĩç«łæĿ¥æºIJ":115112,"ç«Ļç«ĭ":115113,"æŀĦæĪIJäºĨ":115114,"è¾Ľåĭ¤":115115,"è¶ħ强":115116,"éĶļ":115117,"åīįä¸īåŃ£åº¦":115118,"å°±è§īå¾Ĺ":115119,"å´ĩé«ĺ":115120,"è¶Ĭä¾Ĩ":115121,"è¶Ĭä¾Ĩè¶Ĭ":115122,"å¸ĤåľºèIJ¥éĶĢ":115123,"综åIJĪç´łè´¨":115124,"åŃļ":115125,"侮辱":115126,"äºĮåŃĹ":115127,"å·¥ä½ľä»»åĬ¡":115128,"åı²ä¸ĬæľĢ":115129,"æľĢä¼ĺ":115130,"åIJ©åĴIJ":115131,"表çĻ½":115132,"èİ«åIJį":115133,"èİ«åIJįåħ¶":115134,"èİ«åIJįåħ¶å¦Ļ":115135,"å¹£":115136,"åIJĮå¿Ĺ们":115137,"建设çĶ¨åľ°":115138,"åĦĢ":115139,"éħįåģ¶":115140,"弩":115141,"åĶ±çīĩ":115142,"æīĭèĦļ":115143,"åħ¼ä»»":115144,"åģľæĶ¾":115145,"æŃ£å®Ĺ":115146,"æĸ°åĨľæĿij":115147,"åĤ¬çĶŁ":115148,"æīĢåŃ¦æł¡":115149,"å¿µä½Ľ":115150,"åĶ¤éĨĴ":115151,"åħ±åĪĽ":115152,"æĭīä¸ģ":115153,"èĥĮçĿĢ":115154,"çĶŁæĢģä¿ĿæĬ¤":115155,"åı£å¤´":115156,"æĸ¹åIJijçĽĺ":115157,"調æķ´":115158,"æĭĽèģĺä¿¡æģ¯":115159,"åħ¶ä»ĸåĽ½å®¶":115160,"ç®Ģæĺĵ":115161,"åĮ¿åIJį":115162,"è¯Ħæµĭ":115163,"æĺ¯ä¸Ģ座":115164,"çīµæīĭ":115165,"足迹":115166,"çIJĨ解åĴĮ":115167,"æľĢåıĹ":115168,"å¿ĥè·³":115169,"çĪ¶è¦ª":115170,"éĿŀ常åĸľæ¬¢":115171,"èĭ¦éļ¾":115172,"æĬĢå¸Ī":115173,"æ°ijæĦı":115174,"æĪĺåĽ½":115175,"æĽ¿è¡¥":115176,"津贴":115177,"ä¸ŃåĽ½ä¼łç»Ł":115178,"åIJĦè¡Į":115179,"åIJĦè¡ĮåIJĦ":115180,"åIJĦè¡ĮåIJĦä¸ļ":115181,"第äºĶå±Ĭ":115182,"èį·èĬ±":115183,"æĦıèŃĺ":115184,"票价":115185,"åĪĨæµģ":115186,"æĿİçĻ½":115187,"æ±ŁåĮĹ":115188,"æİĴæĸ¥":115189,"ä½ĵéĩı":115190,"åĮħåIJ«äºĨ":115191,"åĪĺæŁIJ":115192,"çİ°å¦Ĥä»Ĭ":115193,"å·¥èīºåĵģ":115194,"è¿Ļç§įæĸ¹æ³ķ":115195,"åĬŀåħ¬æ¥¼":115196,"çĶµå·¥":115197,"çħĻ":115198,"åį¡çīĩ":115199,"å¹´å¹´åºķ":115200,"ä¸ĵ项èµĦéĩij":115201,"åĮ»ç§ij":115202,"åĮ»ç§ij大åѦ":115203,"åĽŀ头çľĭ":115204,"ä¸įå±ij":115205,"èĩªé©¾":115206,"没æĶ¶":115207,"æīĵçĮİ":115208,"èĦ¸éĥ¨":115209,"åıĥèĢĥ":115210,"å°Ĩ士":115211,"è´«åĽ°äººåı£":115212,"çIJĨæĥ³ä¿¡å¿µ":115213,"é£İå°ļ":115214,"人æīįéĺŁä¼į":115215,"çij¾":115216,"æĿ¥è¿ĻéĩĮ":115217,"æ´Ĺ涤":115218,"å¹´èĸª":115219,"èĭįçĻ½":115220,"ä¸ĩäºĭ":115221,"è¯¾æľ¬":115222,"åºĵéĩĮ":115223,"çī¹æ´¾":115224,"çī¹æ´¾åijĺ":115225,"èµŀç¾İ":115226,"ç©¿æĪ´":115227,"è£½ä½ľ":115228,"èµŀæĪIJ":115229,"ä¸Ģ侧":115230,"å½ĵåľ°äºº":115231,"æĭİ":115232,"纸质":115233,"ä½Ļ个":115234,"éĶĤçĶµæ±ł":115235,"æľºåŀĭ":115236,"éĻ¢éĻ¢å£«":115237,"åģļå·¥":115238,"å¼łè´´":115239,"ç¥Ľæĸij":115240,"æ®ĸæ°ij":115241,"å¥ij约":115242,"æ¹ĺæ½Ń":115243,"æIJĸ":115244,"åŃĺè´§":115245,"交éĢļ大åѦ":115246,"è¶ģçĿĢ":115247,"æĸĩçī©ä¿ĿæĬ¤":115248,"å¤ĩæĪĺ":115249,"éĩĩ纳":115250,"åįĬæľĪ":115251,"æľĢåħ³éĶ®":115252,"æľĢåħ³éĶ®çļĦ":115253,"æİ¥éĢģ":115254,"æĶ¶åī²":115255,"åıįåĢĴ":115256,"çĥĽ":115257,"æ½Ķ":115258,"ä¼Łå¤§å¤įåħ´":115259,"çļĦè¯Ŀè¯Ń":115260,"容å¿į":115261,"å®ļéĩı":115262,"æķĹ":115263,"åĵģçīĮ形象":115264,"æīŃ转":115265,"åĽ½å®¶éĩįçĤ¹":115266,"èĨĿçĽĸ":115267,"ä¸Ģ楼":115268,"大éĻ¸":115269,"éĤªæģ¶":115270,"åĽŀåij³":115271,"çĮ¿":115272,"çĿ¡åīį":115273,"æĹłè¾ľ":115274,"çĹħæ¯ĴæĦŁæŁĵ":115275,"æľºæ¢°åĮĸ":115276,"çĤ¹äº®":115277,"溶解":115278,"åĩłä¹İæīĢæľī":115279,"è·ijéģĵ":115280,"çĶµè§Ĩæľº":115281,"åı¨":115282,"æijĩäºĨ":115283,"æijĩäºĨæijĩ头":115284,"èĩªè´Ł":115285,"综åIJĪåĪ©çĶ¨":115286,"èĩªå¦Ĥ":115287,"åİŁä¾Ĩ":115288,"ä¹Łä¸įæĥ³":115289,"èĬĤ课":115290,"è¿ĩåī©":115291,"çĶ²çĬ¶":115292,"çĶ²çĬ¶èħº":115293,"æĸ°ä¸ĸ纪":115294,"èĩªä¸»åĵģçīĮ":115295,"é«ĺå±Ĥ次":115296,"ä¸Ģè§Ĵ":115297,"è¡Įäºĭ":115298,"ç¥ĸåħĪ":115299,"å©ļåIJİ":115300,"éĹ´éļĻ":115301,"ç¼ĿéļĻ":115302,"è¿ĻæĶ¯":115303,"ä¸įæĸŃåĪĽæĸ°":115304,"å¾®åŀĭ":115305,"æĽĻåħī":115306,"享çĶ¨":115307,"ä¸ŃåĽ½ç§»åĬ¨":115308,"éĹŃçݯ":115309,"æī§æĦı":115310,"åıijå±ķæł¼å±Ģ":115311,"æł¸å¿ĥåĮº":115312,"éªļæī°":115313,"åħļåĴĮåĽ½å®¶":115314,"ä¸ŃåĽ½æĶ¿åºľ":115315,"帶èijĹ":115316,"ä¸ĩåįĥçĵ¦":115317,"åħ©äºº":115318,"äºİæĺ¯æĪij":115319,"åĽºä½ĵ":115320,"çªģå¦Ĥ":115321,"çªģå¦Ĥåħ¶":115322,"çªģå¦Ĥåħ¶æĿ¥":115323,"éĩĮç¨ĭç¢ij":115324,"çĪ±ç¾İ":115325,"æŁ¥éªĮ":115326,"åıĮèµ¢":115327,"éĹªåħī":115328,"楼å®ĩ":115329,"æĻı":115330,"æľīè¶³å¤ŁçļĦ":115331,"æŁĶæĢ§":115332,"ä¿¡æģ¯å®īåħ¨":115333,"管线":115334,"并ä¸įä¼ļ":115335,"åĻ¨ä»¶":115336,"ä½łåºĶ该":115337,"çĿĢå®ŀ":115338,"æĺİæ¸ħ":115339,"æĬĹçĶŁç´ł":115340,"æīĵæŃ»":115341,"å®Įåħ¨ä¸įåIJĮ":115342,"èĬ±æ¤Ĵ":115343,"æĶ¾å®½":115344,"ä½İ端":115345,"åĽĽèĤ¢":115346,"åĮĹäº¬èµĽè½¦":115347,"éĽĨå¸Ĥ":115348,"æľªå©ļ":115349,"大å¹ħæıIJåįĩ":115350,"建çŃij设计":115351,"çĭ¬æľīçļĦ":115352,"æİ¢éĻ©":115353,"æ²³æµģåŁŁ":115354,"æħķ容":115355,"被çĽĹ":115356,"åĵºä¹³":115357,"èıģ":115358,"æĥ¬æĦı":115359,"è¶ĬæĿ¥è¶Ĭ好":115360,"广大群ä¼Ĺ":115361,"å¾·èĤ²":115362,"å¸Ĥåľºä»·æł¼":115363,"奥巴":115364,"奥巴马":115365,"èĬĤ缮ä¸Ń":115366,"两款":115367,"ä¸ĩä½Ļåħĥ":115368,"ç»´å°Ķ":115369,"çĶŁçī©ç§ijæĬĢ":115370,"åIJ¬èµ·æĿ¥":115371,"çłļ":115372,"æĭŁå®ļ":115373,"æ²¹çĶ°":115374,"声èªī":115375,"建çŃijä¸ļ":115376,"éĻIJè´Ń":115377,"çīĩåŃIJ":115378,"çķľç¦½":115379,"ç½ijé¦ĸ页":115380,"ä¼ĹçŃ¹":115381,"æĴŀåĩ»":115382,"åīįä¸įä¹ħ":115383,"åīįä¸ĸ":115384,"åĽĽä¸ªæĦıè¯Ĩ":115385,"æµĭç»ĺ":115386,"éĺ²ç©º":115387,"漫éķ¿çļĦ":115388,"æ²IJæµ´":115389,"æ¯Ķè¾ĥç®Ģåįķ":115390,"æµĭå®ļ":115391,"åĽŀè°ĥ":115392,"让人们":115393,"èĴĭä»ĭ":115394,"èĴĭä»ĭçŁ³":115395,"ç»ĵæĻ¶":115396,"å¢ŀæ·»äºĨ":115397,"æĿ¡è¯Ħ论":115398,"åī¯ä¼ļéķ¿":115399,"ä½ıæīĢ":115400,"ç»ĻåĩºäºĨ":115401,"è°ĥéħį":115402,"æ²ĸ":115403,"æľīçĶ¨":115404,"æľīçĶ¨çļĦ":115405,"ä¸ĢæĿ¡é¾Ļ":115406,"éĩİå¤ĸ":115407,"ç¼ĺåĪĨ":115408,"æ°¸è¿ľä¸įä¼ļ":115409,"æŀľæłij":115410,"大åıijå¿«ä¸ī":115411,"麻éĨī":115412,"äºijéĽĨ":115413,"åİ»åĵªéĩĮ":115414,"åħ¥å¸Ĥ":115415,"ä»»æĢ§":115416,"建档":115417,"建档ç«ĭ":115418,"建档ç«ĭåį¡":115419,"ä¸Ģ棵":115420,"社åįĢ":115421,"çĽ¸ä¼´":115422,"åļ·":115423,"å¡«åħħ":115424,"ä¸ĢæĹı":115425,"ç¾ģ":115426,"åıĸè¯ģ":115427,"èĪ°éĺŁ":115428,"åİĤåĮº":115429,"è¡·å¿ĥ":115430,"åıijå±ķéĺ¶æ®µ":115431,"é«ĺ强度":115432,"åĹĵåŃIJ":115433,"é¢Ĩè¡Ķ":115434,"楼主":115435,"大èĴľ":115436,"æŀķ头":115437,"粮油":115438,"é»Ħçĵľ":115439,"æĵĴ":115440,"å°ıçĭĹ":115441,"æĶ¹éĿ©å§Ķ":115442,"åįģåĪĨéĴŁ":115443,"é²ľèī³":115444,"åħ³ç¾½":115445,"çĭĢæħĭ":115446,"å®ŀçĶ¨æĢ§":115447,"å°ijè§ģ":115448,"é£ŀæī¬":115449,"çĶ°éĩİ":115450,"æIJĤ":115451,"è¿Ļ个è¯į":115452,"åºĶæĢ¥é¢Ħæ¡Ī":115453,"è§Ĵ度æĿ¥çľĭ":115454,"æķ¬çķı":115455,"æ³ķå®Ŀ":115456,"åĸĦæĦı":115457,"æīĵæĸŃ":115458,"对åĨ³":115459,"çµķå°į":115460,"åĢŁæѤ":115461,"å¼ĢæºIJ":115462,"å°ı說":115463,"祺":115464,"å²ģ以ä¸ĭ":115465,"éĢĢå½¹åĨĽäºº":115466,"ä¸įä¹ħåīį":115467,"åĩºåİĤ":115468,"讽åĪº":115469,"æĿ¥çľĭçľĭåIJ§":115470,"éŃĶåħ½":115471,"çķĻä¸ĭæĿ¥":115472,"å±ħ室":115473,"åłħæĮģ":115474,"çľĭäºĨä¸Ģ":115475,"çľĭäºĨä¸Ģçľ¼":115476,"éĽĨåĽ¢æĹĹä¸ĭ":115477,"æĪĺæĪĺç»ĦåIJĪ":115478,"è®¤çľŁèIJ½å®ŀ":115479,"汽车产ä¸ļ":115480,"çī©çIJĨåѦ":115481,"æķµ":115482,"éĴĿ":115483,"åĽ¢éķ¿":115484,"ä¸įæĸŃæī©å¤§":115485,"èĤ©è´Ł":115486,"åıijå±ķ缮æłĩ":115487,"è³ĩéĩij":115488,"åīįç½®":115489,"ä¸ŃåĽ½åı¤ä»£":115490,"æŃ»åĪij":115491,"åħħåĪĨä½ĵçİ°":115492,"åħ³éŨ":115493,"ç¾İæĦŁ":115494,"æīĵåħ¥":115495,"æĬijéĥģçĹĩ":115496,"å°ijçĪ·":115497,"æłijæŀĿ":115498,"æ¶Īæģ¯ç§°":115499,"æ´Ľåħĭ":115500,"åį¯":115501,"è¿ĪåIJij":115502,"æݨåĭķ":115503,"ä»İä¸ļèĢħ":115504,"åݻ买":115505,"欢快":115506,"æĭ¥æĮ¤":115507,"马桶":115508,"æĬĬæݧ":115509,"æĶ¿åħļ":115510,"å¼łæī¬":115511,"客æłĪ":115512,"红æĺŁ":115513,"éĢģæĿ¥":115514,"åħ¨åŁŁæĹħ游":115515,"èĩªç§ģ":115516,"åįģäºĮæĿ¡":115517,"åı¹æģ¯":115518,"ä¸Ģèīĺ":115519,"ä¿Ŀè´¹":115520,"æĸ½å·¥çİ°åľº":115521,"æľī幸":115522,"ç»ŃèĪª":115523,"åı¯èĥ½æľĥ":115524,"èĥĮåıĽ":115525,"ä½£éĩij":115526,"ä¸īçŃīå¥ĸ":115527,"å¾Ī满æĦı":115528,"游æĪıåī¯æľ¬":115529,"群éĩĮ":115530,"æŀĦ件":115531,"åºıå¹ķ":115532,"太æ¹ĸ":115533,"æľ¨è´¨":115534,"æĻĭæ±Ł":115535,"çµĤæĸ¼":115536,"è·³è·ĥ":115537,"åĢºæĿĥ人":115538,"çŃī诸å¤ļ":115539,"æĶ¾åĩº":115540,"åħ³éĶ®æŶåĪ»":115541,"æĦŁæŁĵèĢħ":115542,"é£ŀè¡Įåijĺ":115543,"èĥĨåĽº":115544,"èĥĨåĽºéĨĩ":115545,"æĬ±æŃī":115546,"åij¨äºĮ":115547,"æĸ°æĹ¶æľŁ":115548,"åĨ·éĵ¾çī©æµģ":115549,"è¿Ļç§įæĸ¹å¼ı":115550,"该æĿij":115551,"åĽŀé¦Ī":115552,"åŁºçĿ£æķĻ":115553,"人åıĤ":115554,"æŀ¯çĩ¥":115555,"æī¹åıijå¸Ĥåľº":115556,"åħħåĪĨèĤ¯å®ļ":115557,"å¸ĤæĶ¿åįı":115558,"äºĭæ¥Ń":115559,"龸çİĭ":115560,"çĥŃæIJľ":115561,"åįģä¹Ŀ大":115562,"ä¼´æľī":115563,"ç¾İåĽ½æĢ»ç»Ł":115564,"åŁİå¸Ĥ管çIJĨ":115565,"ä¸ĭ令":115566,"èĥ¸åı£":115567,"åıªçŁ¥éģĵ":115568,"åij¨ä¸ī":115569,"çĶ¨æĪ¶":115570,"éŃ¯":115571,"å¿ĥè¡Ģ":115572,"带头人":115573,"åĮ»åĬ¡":115574,"åĮ»åĬ¡äººåijĺ":115575,"æݧåĪ¶åĻ¨":115576,"ä½ľåĵģåĨħ容":115577,"æĪĺåıĭ":115578,"åİĨå¹´":115579,"ä¸įåħĭ":115580,"ä¸įåħĭä¸įåıĬ":115581,"æĹ¥æŃ£å¼ı":115582,"è±IJå¯Į":115583,"ç¨İè´¹":115584,"æŶæķĪ":115585,"å±ķä½į":115586,"è¡¡éĺ³":115587,"æĪ¿è²¸":115588,"çĪĨ款":115589,"ä¹IJæĦı":115590,"çĶ·ä¸»":115591,"寬":115592,"æľĥèŃ°":115593,"ä¹ĭå¤ľ":115594,"åIJĮ樣":115595,"ä¸įè¦ģ太":115596,"ä¼Ĭæĸ¯":115597,"ä¼Ĭæĸ¯åħ°":115598,"åŁºæľ¬åİŁåĪĻ":115599,"åİ»æİī":115600,"ä½İä¿Ŀ":115601,"个交æĺĵ":115602,"个交æĺĵæĹ¥":115603,"èģĬèģĬ":115604,"åĽĽä½į":115605,"åħļç»ĦæĪIJåijĺ":115606,"主è¦ģä»İäºĭ":115607,"å½±éŁ³":115608,"åĨĴåĩº":115609,"åij¼åIJ¸éģĵ":115610,"è¾¾å°Ķ":115611,"æľ¨åľ°æĿ¿":115612,"诡å¼Ĥ":115613,"çģ¯åħ·":115614,"çģ«çĥ§":115615,"解èĦ±":115616,"æĦĪåıij":115617,"æ¹ĸå·ŀ":115618,"é£İä¿Ĺ":115619,"æĸ°å½¢åĬ¿":115620,"æĸ°å½¢åĬ¿ä¸ĭ":115621,"è²Ŀ":115622,"èĦĵ":115623,"åĬ¨åĬĽçĶµæ±ł":115624,"é£ŀèĪ¹":115625,"飧æĢ§":115626,"åĪ©çī©":115627,"åĪ©çī©æµ¦":115628,"ä¸į认è¯Ĩ":115629,"ç¼ĸç»ĩ":115630,"ä½ľåĿĬ":115631,"èģĮä¸ļæĬĢèĥ½":115632,"çľĭè¦ĭ":115633,"åĽ´æ£ĭ":115634,"æĺıè¿·":115635,"å½Ĵå±ŀäºİ":115636,"æĤ¬å´ĸ":115637,"éĨ«çĻĤ":115638,"å®ĭ代":115639,"åºĦæĿij":115640,"èĹķ":115641,"çĮĽçĦ¶":115642,"çĩĥæĸĻçĶµæ±ł":115643,"å®ŀä½ĵåºĹ":115644,"ä¸į足以":115645,"æĥħç·":115646,"æĥħç·Ĵ":115647,"å»ĬåĿĬ":115648,"çĶµåı°":115649,"åºĶåĬĽ":115650,"ä¸Ńå°ıåѦçĶŁ":115651,"èĥ¡åIJĮ":115652,"éī´åĪ«":115653,"åĨħç½®":115654,"乱象":115655,"æ¬ĬçĽĬ":115656,"å¼ĢæĶ¾å¼ı":115657,"åįļæĸĩ":115658,"讲课":115659,"çŃīåİŁåĽł":115660,"穷人":115661,"äº¤æĽ¿":115662,"æĬ¤çħ§":115663,"åıijå±ķæľºéģĩ":115664,"客åķĨ":115665,"åıįä¹ĭ":115666,"ç±³é¥Ń":115667,"并åıij":115668,"并åıijçĹĩ":115669,"æ±īåŃIJ":115670,"æŀľåĽŃ":115671,"对æĪijæĿ¥è¯´":115672,"åģıåIJij":115673,"æī¹ç¤º":115674,"读åIJİ":115675,"读åIJİæĦŁ":115676,"æĺİæĻº":115677,"åĽ´çĿĢ":115678,"åıį转":115679,"æĿ¨å¹Ĥ":115680,"ä¸ĵåįĸ":115681,"ä¸ĵåįĸåºĹ":115682,"åıĹéĻIJ":115683,"åºŁè¯Ŀ":115684,"æŀģå°ij":115685,"åįĪåIJİ":115686,"è¿Ľä¿®":115687,"åīĬåĩı":115688,"æľ¬ç§ijçĶŁ":115689,"ä¼ĺéĢī":115690,"åħīçħ§":115691,"åıĻäºĭ":115692,"åıĸæļĸ":115693,"åĮĹè·¯":115694,"æ¦ķ":115695,"èİĨçĶ°":115696,"楼å±Ĥ":115697,"天èĬ±":115698,"天èĬ±æĿ¿":115699,"çĤľ":115700,"å·²ç»ıæľīäºĨ":115701,"趾":115702,"çĶ³åįļ":115703,"çĶµéĺ»":115704,"åĬŁè¯¾":115705,"æŃ¥æŃ¥":115706,"éĤ£ä¹Ī容æĺĵ":115707,"æѤæĸĩ":115708,"ä½°":115709,"计è¾ĥ":115710,"çīĩéĿ¢":115711,"çĶµå½±éĻ¢":115712,"ä¸įåħ¬å¹³":115713,"ä¸īæľŁ":115714,"æĹħ游èµĦæºIJ":115715,"å¤ļç§įå½¢å¼ı":115716,"è£Ĥç¼Ŀ":115717,"åIJİæİĴ":115718,"硬度":115719,"åĽŀæļĸ":115720,"éģĵæķĻ":115721,"è´«è¡Ģ":115722,"æ¸ħé¦Ļ":115723,"伤çĹħ":115724,"æĦı義":115725,"çļĦç¼ĺ":115726,"çļĦç¼ĺæķħ":115727,"åºĦ严":115728,"åıªæĺ¯ä¸ºäºĨ":115729,"æīĵæĬĺ":115730,"以ä¾Ĩ":115731,"滿足":115732,"çİĽä¸½":115733,"風éļª":115734,"æĸĩç§ij":115735,"éħįå¤ĩäºĨ":115736,"è¿Ľé£Ł":115737,"涡":115738,"è·¯ç¨ĭ":115739,"åı«å£°":115740,"ä¸Ńå¿ĥåŁİåĮº":115741,"æľīæīĢä¸įåIJĮ":115742,"張貼":115743,"é¢ĦæĬ¥":115744,"æľīå¤ļä¹Ī":115745,"è¿Ľè¡Įåħ¨éĿ¢":115746,"æĽ¾ç¶ĵ":115747,"ä¸ī代":115748,"å®ı大":115749,"æ¸ħæī«":115750,"éĢīåĩº":115751,"åĵªä¸Ģ个":115752,"主義":115753,"ä¾Ŀæĵļ":115754,"çļ®éĿ©":115755,"赶æĿ¥":115756,"çŃĽæŁ¥":115757,"æ¨Ł":115758,"ä¿ĿèįIJ":115759,"åIJĥæĥĬ":115760,"æľĭåıĭ们对":115761,"ä»ĸæĺ¯ä¸Ģ个":115762,"åºŁæ°Ķ":115763,"æ»ħ":115764,"è´¢ç¨İ":115765,"æĿijæĿijæ°ij":115766,"èµĦäº§è´ŁåĢº":115767,"å®īå¨ľ":115768,"缮åīįåĽ½åĨħ":115769,"æĦŁè§īèĩªå·±":115770,"çµIJåIJĪ":115771,"éĶ¦æłĩ":115772,"éĶ¦æłĩèµĽ":115773,"æĽ´æ·±":115774,"åŁºæķ°":115775,"éħ¿éħĴ":115776,"çī¹èī²äº§ä¸ļ":115777,"åİĭå®ŀ":115778,"ä¾Ŀæ³ķ追究":115779,"æ·¡å®ļ":115780,"ç®ĢçĽ´å°±æĺ¯":115781,"å£ĵåĬĽ":115782,"æ°ijå¿ĥ":115783,"ä¸įåIJĪéĢĤ":115784,"çĶ±æѤåı¯è§ģ":115785,"èµŀèªī":115786,"澤":115787,"åĩłå¹´åīį":115788,"åIJīä»ĸ":115789,"çł´æįŁ":115790,"è½»è½»åľ°":115791,"å²Ľå±¿":115792,"æĦıå¢ĥ":115793,"ä»Ģä¹Īåı«":115794,"åģĩè£ħ":115795,"éĢģè´§":115796,"å¹ķå¢Ļ":115797,"妥åįı":115798,"åĽ½æĹĹ":115799,"äºĨå¾Īä¹ħ":115800,"åĪĨ辨çİĩ":115801,"ç´Ķ":115802,"éĺ³åĮº":115803,"åĩŃçĿĢ":115804,"åģľè½¦ä½į":115805,"京éĥ½":115806,"éĶ£":115807,"æĵ¾":115808,"è¿ĽéŨ":115809,"åĪĺæµ·":115810,"åĽĽçº§":115811,"女足":115812,"è¡ĮæĶ¿å®¡æī¹":115813,"éģ¥æݧ":115814,"ä¸įéĮ¯":115815,"å¾Ĺå¾Ī好":115816,"ä¸ºçĽ®çļĦ":115817,"ä»įæľª":115818,"ç²¾è£ħ":115819,"éĢįéģ¥":115820,"尽头":115821,"çºłç¼ł":115822,"éłĺå°İ":115823,"æĭħè´Ł":115824,"æĪĸèĢħåħ¶ä»ĸ":115825,"åıªä¸įè¿ĩæĺ¯":115826,"åı®åĺ±":115827,"åģĩåĨĴ":115828,"æļĸæ°Ķ":115829,"çĽIJåŁİ":115830,"被è§Ĩ为":115831,"诺è´Ŀå°Ķ":115832,"ç»ĻäºĨæĪij":115833,"è¿ijåįĥ":115834,"éĩįåĽŀ":115835,"éĨĴäºĨ":115836,"çĶµè§£":115837,"忽çķ¥äºĨ":115838,"èĥĮéĥ¨":115839,"æĸĩæĺİåŁİå¸Ĥ":115840,"æºħ":115841,"è²ĵ":115842,"æĬµæĮ¡":115843,"åĸľæ¬¢åIJĥ":115844,"éĿĻéĿĻåľ°":115845,"å¾Īæ·±":115846,"åŁºç¡ĢçŁ¥è¯Ĩ":115847,"è¿ĩéĶĻ":115848,"çIJĨç§ij":115849,"交æµģåIJĪä½ľ":115850,"èĪĶ":115851,"èª¿æŁ¥":115852,"æħĪæĤ²":115853,"éĴ°":115854,"èĩ´çĶµ":115855,"å®£ä¼łæ´»åĬ¨":115856,"åıĺéĩı":115857,"çļĦ人æĿ¥è¯´":115858,"æŶéļĶ":115859,"ä¸įç®¡ä½ł":115860,"缸è¿ij":115861,"è´µéĩijå±ŀ":115862,"ä¹Łä¸įåı¯èĥ½":115863,"ç²īæľ«":115864,"åįĹçĵľ":115865,"çĻ½é©¬":115866,"åħīæºIJ":115867,"éĩijå¥ĸ":115868,"çĭ¬è§Ĵ":115869,"çĭ¬è§Ĵåħ½":115870,"妨ç¢į":115871,"ç»ĻåĬĽ":115872,"ä½Ĩä»į":115873,"å¼łå®¶åı£":115874,"èIJ¬åħĥ":115875,"渲æŁĵ":115876,"éķ¿å¤§äºĨ":115877,"è®°èĢħäºĨ解":115878,"æĢĢçĿĢ":115879,"è¦ģåѦä¼ļ":115880,"游æĪı代":115881,"游æĪı代ç»ĥ":115882,"äºĮçĻ¾":115883,"æĦıè¯Ĩå½¢æĢģ":115884,"çݺ":115885,"计åĪĴçĶŁèĤ²":115886,"æī¾åĩĨ":115887,"åħ°èĬ±":115888,"è¿Ļ座åŁİå¸Ĥ":115889,"污泥":115890,"å®ĺæĸ¹å¾®ä¿¡":115891,"å½Ĵå±ŀ":115892,"æ°§æ°Ķ":115893,"éģİç¨ĭä¸Ń":115894,"åį°è±¡æ·±åĪ»":115895,"稳妥":115896,"çµIJæĿŁ":115897,"åŃķæľŁ":115898,"çī¹æĿĥ":115899,"åĿļåĽº":115900,"顺åĬ¿":115901,"æŀľèĶ¬":115902,"éĨ«å¸«":115903,"åİ®":115904,"ä¹Łæĺ¯å¦ĤæѤ":115905,"é¦Ĵ头":115906,"缸åĬ©":115907,"干线":115908,"ä¸Ģæľ¬ä¹¦":115909,"绥":115910,"æĮ¯å¥ĭ":115911,"èĤ¾èĦı":115912,"åĭķçī©":115913,"é£ŀè·ĥ":115914,"èıľåĵģ":115915,"å¤ļä½Ļ":115916,"å¤ļä½ĻçļĦ":115917,"éĢĿä¸ĸ":115918,"æģĭ人":115919,"å¼ĢåıijåĪ©çĶ¨":115920,"顺丰":115921,"éĩİå¿ĥ":115922,"æł¡å¤ĸ":115923,"æģIJé¾Ļ":115924,"éĿ¢åħ·":115925,"éķ¿è¾Ī":115926,"éļıå¤Ħ":115927,"éļıå¤Ħåı¯è§ģ":115928,"紧缺":115929,"éĩįä¸Ń":115930,"éĩįä¸Ńä¹ĭ":115931,"éĩįä¸Ńä¹ĭéĩį":115932,"奥æĸ¯":115933,"奥æĸ¯åį¡":115934,"ä¸Ģ个å¤ļ":115935,"ä¸Ģ个å¤ļæľĪ":115936,"ä¸įåı¯ç¼ºå°ij":115937,"æĸ°æł¼å±Ģ":115938,"æıIJæĮ¯":115939,"è¡Įè´¿":115940,"æ¼Ĥæµģ":115941,"èģĬåŁİ":115942,"åħ´å»º":115943,"è´¨æ£Ģ":115944,"ç§ģæľį游æĪı":115945,"æĽ´éĩįè¦ģ":115946,"è´®":115947,"çħľ":115948,"转åıĺ为":115949,"è¿Ļ两年":115950,"ä¿Ŀé²ľ":115951,"æī§æķĻ":115952,"çĥ¨":115953,"å¼Ģåıij建设":115954,"è¿IJèIJ¥ç®¡çIJĨ":115955,"误差":115956,"京åī§":115957,"å¸IJåı·":115958,"å·¥ä½ľä½ľé£İ":115959,"ä¸ĸä¿Ĺ":115960,"çĻ½å®«":115961,"å¤©åĽ½":115962,"å¤©åĽ½ç»§ç»Ń":115963,"å·´æĸ¯":115964,"èIJ¥åĪ©":115965,"åĵģæł¼":115966,"æĿijæ°ij们":115967,"æĪ¿è½¦":115968,"çŃīçĹĩçĬ¶":115969,"å¦Ĥå®ŀ":115970,"宸":115971,"å±Ĥ级":115972,"éĶĻè¿ĩäºĨ":115973,"ç»ĵå®ŀ":115974,"ç¬ijèĦ¸":115975,"羣å®ŀæĢ§":115976,"éĥ½å¸ĤæĬ¥":115977,"é¥Ńèıľ":115978,"åºĶ注æĦı":115979,"æĬ½çĥŁ":115980,"伪éĢł":115981,"åīįä¸Ģ天":115982,"éŃĶé¾Ļ":115983,"éŃĶé¾Ļ令çīĮ":115984,"约è°Ī":115985,"绣çŃ¹æİ¨è¿Ľ":115986,"让çĶ¨æĪ·":115987,"åħ¨éĿ¢èIJ½å®ŀ":115988,"å¼Ħå¾Ĺ":115989,"è°ĪæģĭçĪ±":115990,"鸣æĪIJéķ¿":115991,"鸣æĪIJéķ¿è®°":115992,"æ´ĭæ´ĭ":115993,"çĸıæķ£":115994,"éĿ¢ç§¯çº¦":115995,"æµĵ缩":115996,"æĸ¯é¡¿":115997,"çĶŁæĢģåľĪ":115998,"æī§å¯¼":115999,"移éĢģ":116000,"齿轮":116001,"æł¹æľ¬å°±ä¸į":116002,"缩åĩı":116003,"èµ°ä¸ĭåİ»":116004,"çĿ«æ¯Ľ":116005,"ä¹Łä¸įéĶĻ":116006,"åıįæĺłåĩº":116007,"èĭ¦æģ¼":116008,"缸åħ³æĶ¿çŃĸ":116009,"é«ĺ楼":116010,"ç²īèī²":116011,"æĬķèµĦé¢Ŀ":116012,"ä¸įç»ı":116013,"ä¸įç»ıæĦı":116014,"å®ģæĦ¿":116015,"èĪĮ头":116016,"æ»ĭçĶŁ":116017,"å®ģåİ¿":116018,"åīįåĪĹèħº":116019,"åĩ³":116020,"é£Łæ¬²":116021,"åıĸèĥľ":116022,"éĻ¢åŃIJ":116023,"ç´łè´¨æķĻèĤ²":116024,"滨å·ŀ":116025,"æĬ¢æĬĵ":116026,"å¼Ĥåij³":116027,"åĴļ":116028,"åĬį":116029,"宽éĺĶ":116030,"æļ´æ¶¨":116031,"æĥłåıĬ":116032,"è§Ħç¨ĭ":116033,"ä¾Ľåħ»":116034,"éĢģå¾Ģ":116035,"å±±åºĦ":116036,"举äºļ":116037,"å±ķé¦Ĩ":116038,"解éĶģ":116039,"æĹłè§Ĩ":116040,"éĻįèIJ½":116041,"è¿ŀäºij":116042,"è¿ŀäºij港":116043,"åıĤè°ĭ":116044,"çİĸ":116045,"ç¬ĥ":116046,"èĢĹè´¹":116047,"æī¿å¾·":116048,"社ä¼ļæķĪçĽĬ":116049,"åįĹæµ·ç½ij":116050,"åĪĽä¼¤":116051,"èIJ±":116052,"åħħæ²Ľ":116053,"ç½ijç«Ļ建设":116054,"大åºĨ":116055,"åĨįéĢł":116056,"åŃĹæł·":116057,"åħ¨æ°ijåģ¥èº«":116058,"èĮ«èĮ«":116059,"æµ®åĬ¨":116060,"åīįåı°":116061,"å¢ŀ设":116062,"éĢĽè¡Ĺ":116063,"åĢĴéĹŃ":116064,"æ³ķå¾ĭ顾éĹ®":116065,"çĸ®":116066,"çĹħçĹĩ":116067,"空åīį":116068,"请æķĻ":116069,"èĥľä»»":116070,"æĿĢèıĮ":116071,"æĪĺæĸĹæľº":116072,"ç»ĺåĪ¶":116073,"å¤Ħæĸ¹":116074,"çªģåĽ´":116075,"çĮ«åĴª":116076,"æĬ¥åijĬæĺ¾ç¤º":116077,"ç¿Ł":116078,"çķ¶åľ°":116079,"æľĢéļ¾":116080,"纪å§Ķ书记":116081,"ä½İåİĭ":116082,"èĻļ空":116083,"è¿Ļéĥ¨çĶµå½±":116084,"产ä¸ļåįĩ级":116085,"è°·çĪ±":116086,"è°·çĪ±åĩĮ":116087,"æĬ¼éĩij":116088,"女æĸ¹":116089,"éĴ»çłĶ":116090,"æļĹæļĹ":116091,"è¿·ä½ł":116092,"æīĢè¬Ĥ":116093,"å¨ģå»ī":116094,"å¼ĢæľĹ":116095,"å²Ķ":116096,"çģ«çĤ¬":116097,"åIJĪçIJĨæĢ§":116098,"åħ¬åĬŀ":116099,"ä¼ļä¼ļéķ¿":116100,"éĺ´è°ĭ":116101,"å¼Ģå±Ģ":116102,"æĻ®éĢļè¯Ŀ":116103,"åį¡æĭī":116104,"å°ijåIJĥ":116105,"éĹªèĢĢ":116106,"æŀľæ±ģ":116107,"æī§è¡ĮåĬĽ":116108,"è°Ľ":116109,"æĬ¢åĬ«":116110,"é«ĺéĢŁåıijå±ķ":116111,"飬":116112,"åįĹæ²Ļ":116113,"é«ĺçŃīåŃ¦æł¡":116114,"æį¢ä¸ª":116115,"åı¯èĥ½åŃĺåľ¨":116116,"æĬĴ":116117,"è°±åĨĻ":116118,"被æĬĵ":116119,"æĿ¯åŃIJ":116120,"èĬĤèĥ½åĩıæİĴ":116121,"æ°ĶåĢĻåıĺåĮĸ":116122,"åĪĨåĪ¥":116123,"ä¸Ńæŀ¢":116124,"欢åij¼":116125,"åħī纤":116126,"è¿Ļ群":116127,"çľ¼çķĮ":116128,"åħ±åIJĮåıijå±ķ":116129,"çİ°ä»Ĭ":116130,"éĹ»è¨Ģ":116131,"çī¹èī²å°ıéķĩ":116132,"æķij人":116133,"éĻįæ°´":116134,"ä¸ĸçķĮä¸Ģæµģ":116135,"å°±é¤IJ":116136,"çŀ¥":116137,"å¤įä»ĩ":116138,"ç¾½æ¯Ľ":116139,"ç¾½æ¯ĽçIJĥ":116140,"è´©åįĸ":116141,"æºIJæ³ī":116142,"æĢ»ä½ĵè§ĦåĪĴ":116143,"åĬ¨æĦŁ":116144,"ä¸Ģ审":116145,"åĢŁéĴ±":116146,"è§ģæķĪ":116147,"èĬ±èįī":116148,"åIJĮä¸ļ":116149,"æŁ¥è©¢":116150,"åĽ½éĻħåIJĪä½ľ":116151,"ä¾ĽåĽ¾":116152,"åģ´":116153,"æłĵ":116154,"缸éĢļ":116155,"è°ĪåıĬ":116156,"è¿ĩç¨ĭå½ĵä¸Ń":116157,"é¦Ļèıĩ":116158,"åįģåĽĽæĿ¡":116159,"ä¸Ģå¼Ģå§ĭå°±":116160,"ä¸ĵåijĺ":116161,"æĺİ顯":116162,"æīĵéĢłåĩº":116163,"ä¸ĭéĿ¢æĪij们":116164,"æľºæ²¹":116165,"åı°è¯į":116166,"åŃIJå¼Ł":116167,"æľĢ常è§ģçļĦ":116168,"æĪijè®°å¾Ĺ":116169,"ç»°":116170,"æĤ¬æµ®":116171,"è¿ĺ羣æĺ¯":116172,"æĮĤåı·":116173,"åıĭåĸĦ":116174,"éĩį伤":116175,"çħ§äº®":116176,"æѦèѦ":116177,"åĩºçİ°éĹ®é¢ĺ":116178,"è¸Ĭè·ĥ":116179,"åľ°çIJĥä¸Ĭ":116180,"å¸Ĥ人大":116181,"åıĹ害人":116182,"å²IJ":116183,"åIJĮåѸ":116184,"éĩijèŀįå¸Ĥåľº":116185,"æľīçļĦçݩ家":116186,"å¸ĤæķĻèĤ²":116187,"å¸ĤæķĻèĤ²å±Ģ":116188,"åIJĦå¼Ĥ":116189,"ç·ļä¸Ĭ":116190,"æģº":116191,"æľī大éĩıçļĦ":116192,"åķĨæĬ¥":116193,"åįķåįķ":116194,"åħ¨é¢Ŀ":116195,"ä¾Ŀæŧæĺ¯":116196,"好åĩłä¸ª":116197,"åĸµ":116198,"éĩįæķ´":116199,"çĶŁæ´»è´¨éĩı":116200,"æݢ访":116201,"åį°èĬ±":116202,"缼è¡Į":116203,"å¾®è§Ĥ":116204,"èĪįå¾Ĺ":116205,"åºŁå¼ĥçī©":116206,"积èĵĦ":116207,"å®ļå±ħ":116208,"æĤ¼":116209,"èĮ¸":116210,"çļĦ帮åĬ©":116211,"çļĦ帮åĬ©ä¸ĭ":116212,"亿åIJ¨":116213,"åŃĶéĽĢ":116214,"è¿ĻæĿ¡è·¯":116215,"饵":116216,"æĦĪåĬł":116217,"éķį":116218,"ä½ľæ¡Ī":116219,"èįĶæŀĿ":116220,"太å°ij":116221,"跻身":116222,"åħ¬çĽĬæ´»åĬ¨":116223,"çĻ½æĸij":116224,"æĬĢæľ¯æ°´å¹³":116225,"帧":116226,"æĹłçŁ¥":116227,"åºĶ该æĢİä¹Ī":116228,"éĢĢå¸Ĥ":116229,"æ¸Ń":116230,"åħ»çĮª":116231,"驼":116232,"ç¾¤å²Ľ":116233,"大åį«":116234,"ä¹ĺçĶ¨è½¦":116235,"èı²å°Ķ":116236,"è´´åIJ§":116237,"åģľä¸ĭæĿ¥":116238,"æľīæľºç»ĵåIJĪ":116239,"åĪ»èĭ¦":116240,"çļĦåľ°":116241,"çļĦåľ°æŃ¥":116242,"è¯ĬæīĢ":116243,"å¼ĢæĪĺ":116244,"èĢģçīĮ":116245,"çŃ¹çłģ":116246,"åħ«å¤§ä»¥æĿ¥":116247,"楼æĪ¿":116248,"åŃĻæĤŁ":116249,"åŃĻæĤŁç©º":116250,"åħĴåŃIJ":116251,"第ä¸ĢæĿ¡":116252,"社交åªĴä½ĵ":116253,"æĥ³èµ·æĿ¥":116254,"大æ´ĭ":116255,"æĭ¼éŁ³":116256,"è¿Ľåįļä¼ļ":116257,"è¿ĩåħ³":116258,"æ²¼":116259,"ç©¿æIJŃ":116260,"éĤ£ä¸Ģ天":116261,"çł´éŨ":116262,"æĬķæłĩ人":116263,"赢家":116264,"èĻļå¼±":116265,"æ¿ĥ":116266,"å®īæ£Ģ":116267,"客家":116268,"çĭ¬ç«ĭèij£äºĭ":116269,"æīĭåĬ¿":116270,"åīµéĢł":116271,"åľĨ满å®ĮæĪIJ":116272,"为主线":116273,"好å¥ĩå¿ĥ":116274,"é¢ĨåľŁ":116275,"çªĸ":116276,"åħ¸åŀĭæ¡Īä¾ĭ":116277,"çªģåıijäºĭ件":116278,"åºķæ°Ķ":116279,"头æĻķ":116280,"å®Ľå¦Ĥ":116281,"觸":116282,"æ¸ħæ·¡":116283,"åļ¼":116284,"åģľçĶµ":116285,"ç²īå°ĺ":116286,"éĻįä½İæĪIJæľ¬":116287,"æĶ¾æīĭ":116288,"è®°èĢħ表示":116289,"æĭĸ延":116290,"éªĩ":116291,"æ®ĭå¿į":116292,"çľģæķĻèĤ²":116293,"çľģæķĻèĤ²åİħ":116294,"é«ĺé¢Ŀ":116295,"éĦĻ":116296,"æ¥ŀ":116297,"åĨħç§ij":116298,"èIJ¥ä¸ļé¢Ŀ":116299,"åŁºçŁ³":116300,"æµģæ·Į":116301,"主æŨ":116302,"éĺIJéĩĬ":116303,"建åįİ":116304,"æĥĬåı¹":116305,"çī¢åĽºæłijç«ĭ":116306,"æĺ¯åIJ¦åŃĺåľ¨":116307,"建åĨĽ":116308,"éĽ¾éľ¾":116309,"åħ¬è®¤":116310,"åħ¬è®¤çļĦ":116311,"æ°¨åŁº":116312,"æ°¨åŁºéħ¸":116313,"åīįåĩłå¹´":116314,"åĪ¹éĤ£":116315,"æ±Łä¸ľ":116316,"å·¥æ¥Ń":116317,"ä¸ĢçĤ¹ä¹Łä¸į":116318,"修士":116319,"äºĨä¸Ģéģį":116320,"åĪģ":116321,"æ»ļæ»ļ":116322,"åĪĨæł¡":116323,"羣çĪ±":116324,"è¡ĢèĦī":116325,"æĢ¥åī§":116326,"ä¸Ģ群人":116327,"羯":116328,"æĪIJé¾Ļ":116329,"ç²¾ç¥ŀçĹħ":116330,"缸åħ³äººåijĺ":116331,"éĿĵ丽":116332,"ä¸īåŃ£åº¦":116333,"åĪĴå®ļ":116334,"ä¸ĸçķĮ第ä¸Ģ":116335,"éĢļä¿Ĺ":116336,"åķĨä¸ļåľ°äº§":116337,"åĬŁèĥ½æĢ§":116338,"èµĦæľ¬ä¸»ä¹ī":116339,"详è§ģ":116340,"æĬĵæįķ":116341,"æĸĩæĺĮ":116342,"å®Ŀå®ī":116343,"è£ħéħįå¼ı":116344,"æºIJæºIJ":116345,"æºIJæºIJä¸įæĸŃ":116346,"çĶŁæĢķ":116347,"纵åIJij":116348,"壽":116349,"çľ¼è¢ĭ":116350,"èĤīä½ĵ":116351,"åı¤ä»Ĭ":116352,"èŀįåªĴä½ĵ":116353,"åģī":116354,"æł¼æľĥåĵ¡":116355,"çĥ·":116356,"åĬŁçĶ¨":116357,"æīŃ磩":116358,"绿èī²éĢļéģĵ":116359,"åī§ç»Ħ":116360,"å¼±åĬ¿":116361,"è´¨éĩıéĹ®é¢ĺ":116362,"éĻIJé¢Ŀ":116363,"éªĨ":116364,"éģµä¹ī":116365,"å¯Ŀ室":116366,"æĥ³å¿µ":116367,"åł±åijĬ":116368,"ä»ħ次":116369,"ä»ħ次äºİ":116370,"èŀįåĪĽ":116371,"æĭĽèģĺä¼ļ":116372,"åºĬåŀ«":116373,"转åŀĭåıijå±ķ":116374,"ä¸ŃåĽ½çĶµä¿¡":116375,"åIJ¬è¯Ŀ":116376,"è«ĭæ±Ĥ":116377,"大éĥ¨åĪĨ人":116378,"æ´»å¾Ĺ":116379,"åĵŃæ³£":116380,"è¶Ļ":116381,"åıijçĹħçİĩ":116382,"ä¸į符":116383,"åĨĽå®ĺ":116384,"é¢Īæ¤İ":116385,"æĸ°åĨłçĸ«æĥħ":116386,"æŁ¬åŁĶ":116387,"æŁ¬åŁĶ寨":116388,"ä»»ä½ķå½¢å¼ı":116389,"人éĻħ":116390,"人éĻħåħ³ç³»":116391,"æĢ»æī¿åĮħ":116392,"å¹³åĿĩæ¯ı":116393,"æģŃåĸľ":116394,"åĦĺ":116395,"åħµé©¬":116396,"è¿ŁåĪ°":116397,"工伤":116398,"çīĪæĿĥå½Ĵ":116399,"çīĪæĿĥå½ĴåİŁ":116400,"æĭ¥æĬ¤":116401,"ç³Ĭæ¶Ĥ":116402,"å¹²æ¶ī":116403,"å°ijä¸įäºĨ":116404,"æĥ³æī¾":116405,"è´¹çİĩ":116406,"该éĻ¢":116407,"èŀįåĮĸ":116408,"è¿İåIJĪ":116409,"è§ĨåIJ¬èĬĤ缮":116410,"æł¼ç¶²ç«Ļ":116411,"çľīæ¯Ľ":116412,"欢è¿İ大家":116413,"家åºŃæķĻèĤ²":116414,"ä¾µèļĢ":116415,"ç»Ļä½łä»¬":116416,"è¡Ģ液循çݯ":116417,"å¯Ħæīĺ":116418,"å°ĸåı«":116419,"以ä¸ĭåĩłä¸ª":116420,"è¿ĺ以为":116421,"åħ¶ä»ĸçݩ家":116422,"ç¬ijç¬ij":116423,"æīĵåIJ¬":116424,"èĩªçĦ¶ç§ijåѦ":116425,"åŁºç«Ļ":116426,"ä¹Ŀå·ŀ":116427,"ä¿Ŀ驾":116428,"ä¿Ŀ驾æĬ¤":116429,"ä¿Ŀ驾æĬ¤èĪª":116430,"æĶ¾çľ¼":116431,"çŁ¥åIJįä¼ģä¸ļ":116432,"縮":116433,"稽":116434,"æļĩ":116435,"使çĶ¨ç¶²è·¯":116436,"é¢ĦçķĻ":116437,"大象":116438,"åıijæĺİä¸ĵåĪ©":116439,"æĸĩ娱":116440,"éĢłç¦ı":116441,"湿润":116442,"éĿ¢æĿ¡":116443,"æ¶Īè´¹åįĩ级":116444,"è®Ĭå¾Ĺ":116445,"åĩłåIJį":116446,"ä»Ħ":116447,"认æ¸ħ":116448,"è¿ľæĻ¯":116449,"æıĴ座":116450,"诸侯":116451,"åıĺæĢģ":116452,"ç¦ı彩":116453,"è´§æŀ¶":116454,"失æݧ":116455,"移åĬ¨ç«¯":116456,"ä¸Ĭåı¸":116457,"éĢłçº¸":116458,"å¸ĥæľĹ":116459,"çĴĩ":116460,"åı°åįĹ":116461,"åĮĹ京åĨ¬å¥¥":116462,"èĵĿçīĻ":116463,"éķ¿çŁŃ":116464,"æĬĺå°Ħ":116465,"ç»ijæŀ¶":116466,"å¯Ĵåģĩ":116467,"è½¬åŁºåĽł":116468,"æĢ¥äºİ":116469,"æŃ£åĵģ":116470,"åħħ滿":116471,"大纲":116472,"æĬĹä½ĵ":116473,"è¨ĵç·´":116474,"æĶ¶ç´§":116475,"æ¯Ķè³½":116476,"åħµåĬĽ":116477,"æľ¬æĽ¸":116478,"äºĮ代":116479,"æĢ¥è¯Ĭ":116480,"æĸĩæ¡Ī":116481,"ç»ıåķĨ":116482,"æĻ¨æĬ¥":116483,"æ£ĺ":116484,"æĢ»ä¹¦è®°åľ¨":116485,"åıĹéĤĢ":116486,"äºĶåĽĽ":116487,"å²ŃåįĹ":116488,"çĪ±åIJĥ":116489,"åŁĥå°Ķ":116490,"å¿ĥå¢ĥ":116491,"è¦ĨçĽĸéĿ¢":116492,"å®ŀåľ¨æĺ¯å¤ª":116493,"æł¹åºķ":116494,"纷纷表示":116495,"åĹħ":116496,"éļıçĿĢæŶéĹ´":116497,"åİĨåı²æĤłä¹ħ":116498,"éħī":116499,"æĢ»éĺŁ":116500,"主é¢ĺæ´»åĬ¨":116501,"éĹ®åį·":116502,"é©¿ç«Ļ":116503,"æı¡ä½ı":116504,"åı¯èĥ½å¯¼èĩ´":116505,"æ°ijéĸĵ":116506,"éĸĭåķŁ":116507,"ä½Ĩä¸įéĻIJ":116508,"ä½Ĩä¸įéĻIJäºİ":116509,"åįģéĩĮ":116510,"娥":116511,"æįŁèĢĹ":116512,"çĸı导":116513,"çݯ氧":116514,"ç¥ŀéĢļ":116515,"çĪ±å°Ķ":116516,"çĪ±å°Ķåħ°":116517,"æľ´å®ŀ":116518,"å¿«æĬ¥":116519,"æĶ¶åıĹ":116520,"æĪĸ許":116521,"èĥĮéĿ¢":116522,"æĸĩåĮĸä¼łåªĴ":116523,"ä¸īåĢĭ":116524,"æĶ»åĬ¿":116525,"å®ī举":116526,"å®īä¸ľå°¼":116527,"åĿĩå·²":116528,"顾èĻij":116529,"éĦŃ":116530,"è¿Ļ家åħ¬åı¸":116531,"åħ¬åijĬ称":116532,"æıIJä¾Ľä¼ĺè´¨":116533,"稳æŃ¥æİ¨è¿Ľ":116534,"å¤įè¯ķ":116535,"å°Ĩé¢Ĩ":116536,"è°Īèµ·":116537,"å¨Ħ":116538,"è¿ŀ线":116539,"æ©ŁéĹľ":116540,"åºĶçĶ¨åľºæĻ¯":116541,"çĶ»åĥı":116542,"è´¢è¿IJ":116543,"ä¿Ŀéļª":116544,"çĹħçIJĨ":116545,"æ¯Ľä¸»å¸Ń":116546,"ä¸Ŀ毫ä¸į":116547,"çĪ±å¥ĩ":116548,"çĪ±å¥ĩèīº":116549,"ä¸ĵ家ç»Ħ":116550,"åij¼åĶ¤":116551,"éĭ¼":116552,"çģ¸":116553,"é¢ĨåħĪåľ°ä½į":116554,"æıIJæĭĶ":116555,"龸éģĵ":116556,"å±±åĿ¡":116557,"èĿİ":116558,"沸èħ¾":116559,"该项":116560,"ä»ĬçĶŁ":116561,"ä¸Ģç¯ĩæĸĩç«ł":116562,"æĸ¹å¼ıè¿Ľè¡Į":116563,"é»ij客":116564,"æĶ¹åĬ¨":116565,"主é¡Į":116566,"æķ£å¸ĥ":116567,"ä»Ģä¹Īåľ°æĸ¹":116568,"åĮĸåIJĪ":116569,"åĮĸåIJĪçī©":116570,"éĿĻçĶµ":116571,"æĢ»æĶ¶åħ¥":116572,"å§Ķç»Ħç»ĩ":116573,"å§Ķç»Ħç»ĩéĥ¨":116574,"éĿĻæĢģ":116575,"èĢģåŃĹåı·":116576,"室åıĭ":116577,"éĥ½ä¸įæķ¢":116578,"æŀ¶åŃIJ":116579,"çģµæķı":116580,"审è§Ĩ":116581,"æĤ£åĦ¿":116582,"山寨":116583,"èĸªèµĦ":116584,"é©°æı´":116585,"éĥ¨åĪĨåĨħ容":116586,"好似":116587,"æĪIJåijĺåĽ½":116588,"åľ¨æĪijçľĭæĿ¥":116589,"åħ³æ³¨åº¦":116590,"éĻĪæŁIJ":116591,"è¿Ļç§įäºĭæĥħ":116592,"éĢīå®ļ":116593,"ç²¾åŃIJ":116594,"å£ģçĶ»":116595,"æ±Łæ·®":116596,"é«ĺæĺĤ":116597,"æł¼åĬĽ":116598,"輩":116599,"åѦåłĤ":116600,"æĤ¨åIJĮæĦı":116601,"ä¸ĢåĪĩéĥ½æĺ¯":116602,"潤":116603,"éĸĥ":116604,"å¸ĮæľĽèĩªå·±":116605,"ä¿ĺ":116606,"æ±Łåİ¿":116607,"æ³¾":116608,"ç§ijæķĻ":116609,"æīĵè¿Ľ":116610,"ä¸įæħİ":116611,"å¯ĴåĨ¬":116612,"æ¸Ķæ°ij":116613,"鼷æĸ¯":116614,"主宰":116615,"æĹħ游度åģĩ":116616,"çĶµåŃIJéĤ®ä»¶":116617,"æ±Ĥå©ļ":116618,"éļİ段":116619,"åģ¥èº«æĪ¿":116620,"注æĺİåĩºå¤Ħ":116621,"äºĭæķħåıijçĶŁ":116622,"级以ä¸Ĭ":116623,"åŃĺæ´»":116624,"æĸ½èĤ¥":116625,"èľľèľĤ":116626,"嵩":116627,"æĮĸæİĺæľº":116628,"æĬĹæĭĴ":116629,"ä¼łå¯¼":116630,"æĺ¯ä»Ģä¹Īåij¢":116631,"ä¸Ĭå¹´åIJĮæľŁ":116632,"建åħļ":116633,"çĶŁæħĭ":116634,"ä¿Ŀä½ı":116635,"款车åŀĭ":116636,"人èĦī":116637,"éļIJèĶ½":116638,"失æķĪ":116639,"éģ¿åŃķ":116640,"ç®Ģ便":116641,"è°¢è°¢ä½ł":116642,"å®Īä½ı":116643,"æĶ¾æĺł":116644,"è¨Īçķ«":116645,"çݰ代çī©æµģ":116646,"é¤IJ廳":116647,"æķħå±ħ":116648,"大大å°ı":116649,"大大å°ıå°ı":116650,"çī¹åĪ«å£°æĺİ":116651,"éģįåıĬ":116652,"å¿ĥçIJĨåĴ¨è¯¢":116653,"è³´":116654,"çĮ®è¡Ģ":116655,"å·²ç»ıè¾¾åĪ°":116656,"æīĵæĭĽåij¼":116657,"åıĮè¾¹":116658,"ä¸Ģæĸ¹éĿ¢æĺ¯":116659,"å´ĩå°ļ":116660,"éĺ¿å¯Į":116661,"éĺ¿å¯Įæ±Ĺ":116662,"æĮģæľī人":116663,"è±ģ":116664,"é£İçŃĿ":116665,"åĬ¨èį¡":116666,"äºĨä¸Ģä¼ļ":116667,"äºĨä¸Ģä¼ļåĦ¿":116668,"ä¸ĩ象":116669,"çľĭçĶµè§Ĩ":116670,"åįģä¸īæĿ¡":116671,"çĮĽçĥĪ":116672,"è¦ģä¸įçĦ¶":116673,"太æŀģæĭ³":116674,"å¼ķçĪĨ":116675,"ç»ıè¿ĩå¤ļå¹´":116676,"游æĪıéĩĮçļĦ":116677,"é¾Ļæ³ī":116678,"æłĩéħį":116679,"è®ĵä»ĸåĢij":116680,"éĢłæŀĹ":116681,"åĮºåŁŁæĢ§":116682,"亿ä¸ĩ":116683,"æĪĺçķ¥å¸ĥå±Ģ":116684,"éķĩæĶ¿åºľ":116685,"åĶ®ç¥¨":116686,"çĶŁäº§å·¥èīº":116687,"éķĩåħļå§Ķ":116688,"ä¸Ńå°ıåŀĭ":116689,"æľ¨èĢ³":116690,"河边":116691,"èĦ¾èĥĥ":116692,"欢è¿İæĤ¨":116693,"åıĺå¼Ĥ":116694,"缤纷":116695,"åŀĥåľ¾æ¡¶":116696,"辩è¯ģ":116697,"车åºĵ":116698,"æ¯Ķçİĩ":116699,"åħ´æĹº":116700,"详ç»ĨäºĨ解":116701,"å®īå±ħ":116702,"çħ§æĸĻ":116703,"æĸ¹æīį":116704,"赦":116705,"åĨķ":116706,"å¥Ķèµ´":116707,"å®Ŀ鸡":116708,"åľºåĿĩ":116709,"缮åīįæŃ£åľ¨":116710,"åIJŀåĻ¬":116711,"è¿°èģĮ":116712,"æĩµ":116713,"å¥ĩçijŀ":116714,"ä»įå°Ĩ":116715,"èĪī辦":116716,"å·¥åķĨå±Ģ":116717,"å¡ijèĥ¶":116718,"åĬŀå®ŀäºĭ":116719,"æĸ¹æĸ¹éĿ¢":116720,"æĸ¹æĸ¹éĿ¢éĿ¢":116721,"æĸĩåĮĸèĬĤ":116722,"åħ¥èģĮ":116723,"鸥":116724,"ç©¿éĢı":116725,"ä»¥ä¹łè¿ijå¹³":116726,"åį±éļª":116727,"æľ¦èĥ§":116728,"åİĨåı²æĢ§":116729,"æķŀå¼Ģ":116730,"ä¼Ļä¼´åħ³ç³»":116731,"çŁ¿åĮº":116732,"åĽ½éĻħåľ¨çº¿":116733,"ä¼łå¥ĩéĩĮéĿ¢":116734,"è¿ijäºĽ":116735,"è¿ijäºĽå¹´":116736,"åĬ£åĬ¿":116737,"æĶ»åĩ»åĬĽ":116738,"æĻºéĢł":116739,"禧":116740,"çİĭåħĪçĶŁ":116741,"éĨ«çĶŁ":116742,"åĽĽé¡¹":116743,"å®ŀæĻ¯":116744,"åĪĿåĪĽ":116745,"å¿ĥ裡":116746,"æĻ¶ä½ĵ":116747,"交éĻħ":116748,"让æ¶Īè´¹èĢħ":116749,"课æĸĩ":116750,"æİĴæ°Ķ":116751,"并ä¸įæĦıåij³":116752,"çĽ¸å£°":116753,"第ä¸Ģå±Ĭ":116754,"åİŁèijĹ":116755,"鼾":116756,"没æľī太大":116757,"补水":116758,"çī©æµģä¼ģä¸ļ":116759,"第äºĮæī¹":116760,"åħ¶å®ĥéĹ®é¢ĺ":116761,"æİĮéŨ":116762,"责任å¿ĥ":116763,"é¤IJåħ·":116764,"ç¾Ĭæ¯Ľ":116765,"没æľīå¿ħè¦ģ":116766,"ä¹IJåĽ¢":116767,"è¿ĽåŁİ":116768,"ä¸ĢçĤ¹åĦ¿":116769,"身形":116770,"çļ®èĤ¤çĹħ":116771,"æĺ±":116772,"å¢ŀèĩ³":116773,"èģ²æĺİ":116774,"æıIJè´¨":116775,"ä½ĵèĤ²åľº":116776,"çŃ¹å»º":116777,"é¬Ĩ":116778,"车çīĮ":116779,"éļĶéŁ³":116780,"è´Łè´£åIJĮå¿Ĺ":116781,"丰ç¡ķ":116782,"ä½ĽéĻĢ":116783,"äºīåIJµ":116784,"庶":116785,"æ·¡æ°´":116786,"å°ıçĶ·åŃ©":116787,"ç§ģèĩª":116788,"åĮĸè¿Ľç¨ĭ":116789,"æĪĺ士æĿ¥è¯´":116790,"æ²¹èħ»":116791,"èĦ±è´«èĩ´å¯Į":116792,"æĹ¥å¸¸å·¥ä½ľ":116793,"交èŀį":116794,"åĨľè´¸":116795,"åĨľè´¸å¸Ĥåľº":116796,"åĵĪçĻ»":116797,"çĶµè´¹":116798,"èµĺ":116799,"åıĮèħ¿":116800,"æĵĶå¿ĥ":116801,"æĿ¥å½¢å®¹":116802,"使åij½æĦŁ":116803,"éĤ£ä¹Īç®Ģåįķ":116804,"èĬĻèĵī":116805,"åĢŁæ¬¾äºº":116806,"ç§Ģ丽":116807,"è®ĵä»ĸ":116808,"严åİīæīĵåĩ»":116809,"è³ŀ":116810,"æļ«":116811,"çħ¤æ°Ķ":116812,"çĪ¬ä¸Ĭ":116813,"æ½ĩæ´Ĵ":116814,"太ä¹ħ":116815,"åij½åIJį为":116816,"è·¯çĶ±":116817,"è·¯çĶ±åĻ¨":116818,"驯":116819,"æıIJæĹ©":116820,"æĬĹåĩ»çĸ«æĥħ":116821,"åĩĽ":116822,"交åıĭ":116823,"éĶĢåĶ®æ¸łéģĵ":116824,"毫ä¸įçĬ¹è±«":116825,"èIJ¥åľ°":116826,"çłĶ究表æĺİ":116827,"鱼类":116828,"æį¢å±Ĭ":116829,"æİ¡åıĸ":116830,"çīĨ":116831,"缼å¼Ģ":116832,"沧æ¡ij":116833,"åºŃ审":116834,"ç»ıæŁ¥":116835,"åĬłå¼·":116836,"缸æ¯Ķäºİ":116837,"ä¸ĵçıŃ":116838,"ä½ĵåŀĭ":116839,"被害":116840,"被害人":116841,"æĶ¶æ¬¾":116842,"åħ·æľīèī¯å¥½":116843,"é«ĺå³°æľŁ":116844,"åģıä½İ":116845,"åĦŁ":116846,"åĨľä¸ļç§ijæĬĢ":116847,"çī¹æ®ĬæĥħåĨµ":116848,"å¦Ĥæŀľçݩ家":116849,"éķ¿çº¦":116850,"第åħŃå±Ĭ":116851,"åħ¬å¼ĢæĭĽèģĺ":116852,"åĪĩæĸŃ":116853,"迫使":116854,"çĸĹç¨ĭ":116855,"第äºĮç§į":116856,"ä¸įåħį":116857,"å¹²èѦ":116858,"çŁ³æ¦´":116859,"åĹ£":116860,"两类":116861,"çĪµå£«":116862,"åŁİ乡å±ħæ°ij":116863,"æŃ¤é¡¹":116864,"缴è¾ĸ":116865,"缴è¾ĸå¸Ĥ":116866,"åij¼åºĶ":116867,"éĴ¯":116868,"ç¦ıå¾·":116869,"æľºèº«":116870,"æĵįåľº":116871,"æ¿Ĵ临":116872,"人群ä¸Ń":116873,"èĤ¡æ°ij":116874,"åŃ½":116875,"æ³ķåħ°":116876,"é¨İ":116877,"糯米":116878,"æĢ»çļĦ":116879,"æĢ»çļĦæĿ¥è¯´":116880,"åħ¸éĽħ":116881,"æĸ°éĻĪ":116882,"æĸ°éĻĪ代谢":116883,"缮çĿ¹":116884,"é¢Ħè¨Ģ":116885,"è·Įçł´":116886,"æĸ°ç¯ĩç«ł":116887,"æ¯ĴæĢ§":116888,"åĸĿèĮ¶":116889,"æŁ¥èİ·":116890,"亮丽":116891,"çĶŁäº§åķĨ":116892,"æĶ¹æĪIJ":116893,"为äºĨæĽ´å¥½":116894,"深交":116895,"深交æīĢ":116896,"æİĥ":116897,"ä¹ĻèĤĿ":116898,"泸å·ŀ":116899,"åħĪè¿ĽæĬĢæľ¯":116900,"è¾ĵç»Ļ":116901,"æķ£æĪ·":116902,"æĢĿç»´æĸ¹å¼ı":116903,"åºĹ主":116904,"è°ĭæ±Ĥ":116905,"游æĪıæĬĢå·§":116906,"ä¸Ģ年级":116907,"çľ¼è§Ĵ":116908,"ä¸Ńä»ĭæľºæŀĦ":116909,"å·§åIJĪ":116910,"éĺ²çĽĹ":116911,"导è´Ń":116912,"æĪĬ":116913,"æĽ´éĢĤåIJĪ":116914,"åŁºæľ¬ä¿¡æģ¯":116915,"马ä¸ģ":116916,"åħ»æ®ĸåľº":116917,"åıįè¿ĩæĿ¥":116918,"æݨå´ĩ":116919,"å¯ĨåĪĩåħ³æ³¨":116920,"åŁºéĩijç»ıçIJĨ":116921,"æĮīéĶ®":116922,"åĨħéĥ¨æݧåĪ¶":116923,"æĪIJåijĺåįķä½į":116924,"æľ¯è¯Ń":116925,"åĪ¶æľį":116926,"åĪļéľĢ":116927,"æ£Ģç´¢":116928,"大大æıIJé«ĺ":116929,"åģ¥åº·ç®¡çIJĨ":116930,"èĩªæѤ":116931,"客æĪ·éľĢæ±Ĥ":116932,"丰èĥ¸":116933,"èµ·éĩį":116934,"èµ·éĩįæľº":116935,"æ¬łç¼º":116936,"æ¡ĪåŃIJ":116937,"æĥħ人èĬĤ":116938,"åħļæł¡":116939,"è¢ľ":116940,"该åī§":116941,"è¿·å¤±ä¼łå¥ĩ":116942,"ç»ļ丽":116943,"åķª":116944,"æĹłç§ģ":116945,"éĢ²ä¸ĢæŃ¥":116946,"第ä¸Ģç«ł":116947,"åĻ¨åħ·":116948,"åĨľèµĦ":116949,"確實":116950,"åºıåĪĹ":116951,"娱ä¹IJå¹³åı°":116952,"èŀįèµĦ秣èµģ":116953,"èµĦæºIJåħ±äº«":116954,"èģ½åĪ°":116955,"æIJŀå¾Ĺ":116956,"继ç»Ńä¿ĿæĮģ":116957,"åIJ¯èĴĻ":116958,"çľº":116959,"ä¸Ŀè·¯":116960,"设æĸ½å»ºè®¾":116961,"æİ¥åľ°":116962,"æİ¥åľ°æ°Ķ":116963,"第ä¸īåŃ£åº¦":116964,"åŁºè°ĥ":116965,"åıijéŁ³":116966,"社ä¼ļèµĦæľ¬":116967,"éĽĩ主":116968,"è¿ŀèĥľ":116969,"没åķ¥":116970,"廢":116971,"赶赴":116972,"æ¼ĶåĮĸ":116973,"åı¤æĢª":116974,"çİĭçĪ·":116975,"é¢ĦåħĪ":116976,"å¼Ģåħ·":116977,"åĽŀé¦ĸ":116978,"åľ°ä¸ĭæ°´":116979,"å°ıç¼ĸä¸Ģèµ·":116980,"èµİåĽŀ":116981,"åľ°è²Į":116982,"åĪĿä¸ī":116983,"åı¯çĶ¨äºİ":116984,"éģĹ迹":116985,"è¿Ļæī¹":116986,"èĸªæ°´":116987,"å¿ħçĦ¶ä¼ļ":116988,"æ²½":116989,"éįĭ":116990,"第ä¸Ģéĥ¨":116991,"åĪĬçī©":116992,"å®ŀä¾ĭ":116993,"æ¸ħåĩĢ":116994,"ä¸ĬèµĽåŃ£":116995,"åĽ¾è¡¨":116996,"éĤ®è½®":116997,"åĵªè£¡":116998,"缸è§ģ":116999,"æī°ä¹±":117000,"æ¯ıæ¯ı":117001,"è¿Ļè¾ĪåŃIJ":117002,"ç¡«éħ¸":117003,"äºī缸":117004,"溯æºIJ":117005,"åĩºä¼Ĺ":117006,"çİīçŁ³":117007,"åħ±çĶŁ":117008,"æŶéĹ´æ®µ":117009,"éĩįè¦ģæĮĩ示":117010,"æ¶Īè´¹éľĢæ±Ĥ":117011,"éķ¿éķ¿":117012,"éķ¿éķ¿çļĦ":117013,"å®īæĬļ":117014,"å¢ŀé«ĺ":117015,"æľ¬è½®":117016,"äº²çľ¼":117017,"é£İæ³¢":117018,"èĢģå¦Ī":117019,"æĶ¶è´¹æłĩåĩĨ":117020,"åĨħéĻĨ":117021,"æĮ¥åıij":117022,"åįĩåѦ":117023,"èĥ¸åīį":117024,"åģıè¿ľ":117025,"纯æ´ģ":117026,"æĸ½å·¥åįķä½į":117027,"身价":117028,"è´¢åĬĽ":117029,"纶":117030,"è£ħçĶ²":117031,"æĺ¾ç¤ºåĻ¨":117032,"毫åįĩ":117033,"æ·±çŁ¥":117034,"èĢ¶ç©":117035,"èĢ¶ç©Į":117036,"è¾ĥéĩı":117037,"åľ¨è¿ĩ渡":117038,"åľ¨è¿ĩæ¸¡æľŁ":117039,"èĮĹ":117040,"ä¸Ģ个æĺŁæľŁ":117041,"èĬ·":117042,"è´¿èµĤ":117043,"æ¿ķ":117044,"æĩĤäºĭ":117045,"秧":117046,"åħħå½ĵ":117047,"åĽ½ç«ĭ":117048,"èĬ±çĵ£":117049,"éĤĦè¦ģ":117050,"åħ¬åľĴ":117051,"触åĬ¨":117052,"æ³°å·ŀ":117053,"ä»Ģä¹Īæł·":117054,"æ»ĭåħ»":117055,"è¯ĦåĪ¤":117056,"æĮ¥æīĭ":117057,"èĦĪ":117058,"姥姥":117059,"è¿IJè´¹":117060,"æ¯ħåĬĽ":117061,"å¿ĥæĻº":117062,"ä¸įæİĴéĻ¤":117063,"第ä¸ī代":117064,"éĢĢè´§":117065,"æĺŁéĻħ":117066,"æ°¸åĪ©":117067,"æĬ¤åį«":117068,"çıŃ车":117069,"è¨Ģè¡Į":117070,"繪":117071,"主åĬ¨æĢ§":117072,"å·¥ç¨ĭè´¨éĩı":117073,"éĥĬåĮº":117074,"ä¸Ģæłĭ":117075,"ä½Ĩå®ŀéĻħä¸Ĭ":117076,"ä¸ī大èģĮä¸ļ":117077,"åij¼åı«":117078,"女åħĴ":117079,"è¯ģåĪ¸æĬķèµĦ":117080,"èĢĥæħ®":117081,"çĤ«èĢĢ":117082,"治好":117083,"åĺ¶":117084,"èĥ¤":117085,"åħīä¼ıåıijçĶµ":117086,"åĩłæŃ¥":117087,"æīĢæīĢ":117088,"æīĢæīĢéķ¿":117089,"çħ§æł·":117090,"åĵ¥ä»¬":117091,"è¯Ľ":117092,"è¿Ļä¸ĢåĪ»":117093,"çŁ¿çī©è´¨":117094,"ä¸įå¾Ĺå·²":117095,"åIJĮ缣":117096,"ç»Ĩå¾®":117097,"è·¯èĻİ":117098,"çĻ¾èĬ±":117099,"æ··æ²Į":117100,"ä¸Ĭæµ·è¯ģåĪ¸":117101,"éĢĢç¨İ":117102,"èµŀåı¹":117103,"æī®æ¼Ķ游æĪı":117104,"åIJįåĪĹ":117105,"åIJįåĪĹåīį":117106,"åIJįåĪĹåīįèĮħ":117107,"ç±³å°Ķ":117108,"ä»Ģä¹ĪåİŁåĽł":117109,"å®īåħ¨ä¿Ŀéļľ":117110,"ä¸Ģåıªæīĭ":117111,"ä¹³ä¸ļ":117112,"ä¸įçĶĺ":117113,"æĥħåķĨ":117114,"æĮ¡ä½ı":117115,"åİŁåĽłä¹ĭä¸Ģ":117116,"è¿Ļ两天":117117,"çĥĺçĦĻ":117118,"豬":117119,"ä½łä»¥ä¸º":117120,"没è§ģè¿ĩ":117121,"åĵªå®¶å¥½":117122,"åīįä»»":117123,"è¿Ľè´§":117124,"éĢĢåĽŀ":117125,"串èģĶ":117126,"èĩ³æĸ¼":117127,"åĨ°æ·ĩ":117128,"åĨ°æ·ĩæ·ĭ":117129,"æŁ¥çľĭ详æĥħ":117130,"çı¾å¯¦":117131,"æݨæµĭ":117132,"æİ¥æīĭ":117133,"éļ¶å±ŀäºİ":117134,"åŁİå¸Ĥ群":117135,"æĿİåħĪçĶŁ":117136,"çŁ¿æ³īæ°´":117137,"çī¹ä»·":117138,"æĽ´å¤ļ精彩":117139,"ç¨ĭå¼ı":117140,"读æĩĤ":117141,"å±ıèĶ½":117142,"奥æŀĹ":117143,"奥æŀĹåĮ¹":117144,"奥æŀĹåĮ¹åħĭ":117145,"红èĸ¯":117146,"奮":117147,"å®Ŀçİī":117148,"網絡":117149,"貧":117150,"欧å¼ı":117151,"çĻ½ç³ĸ":117152,"èĩªçĦ¶çģ¾å®³":117153,"åijĬè¯ī她":117154,"å»ļ":117155,"çĤ¹åĩ»æŁ¥çľĭ":117156,"é£İ湿":117157,"èµĦ产éĩįç»Ħ":117158,"ä¹Łä¸įä¾ĭå¤ĸ":117159,"åįĬ个å°ıæŶ":117160,"åIJ¸å¼ķæĽ´å¤ļ":117161,"æŶéĹ´èĬĤçĤ¹":117162,"æĶ¶çº³":117163,"åIJ¸æ¯Ĵ":117164,"èĢģ乡":117165,"çIJħ":117166,"æľĢçµĤ":117167,"åıįæĦŁ":117168,"çĶ¨å¾®ä¿¡":117169,"çĶ¨å¾®ä¿¡æī«":117170,"éĢŁçİĩ":117171,"大çĨĬçĮ«":117172,"åı¯æĥ³":117173,"åı¯æĥ³èĢĮ":117174,"åı¯æĥ³èĢĮçŁ¥":117175,"åĴ§":117176,"èµ°åħ¥":117177,"碳éħ¸":117178,"èĮĥåĨ°":117179,"èĮĥåĨ°åĨ°":117180,"被åĪ¤":117181,"积æŀģæݨåĬ¨":117182,"足足":117183,"ç²ĴåŃIJ":117184,"大å®Ĺ":117185,"大å®ĹåķĨåĵģ":117186,"ç½ij绾ç§ijæĬĢ":117187,"æĽ¼åŁİ":117188,"å·²ä¹ħ":117189,"å·²ä¹ħçļĦ":117190,"秦çļĩ":117191,"秦çļĩå²Ľ":117192,"ä»»æķĻ":117193,"åĶ¯ç¾İ":117194,"æ·¡åĮĸ":117195,"æ¡ĤèĬ±":117196,"çŁ¥è¯ĨåĪĨåŃIJ":117197,"æĩĴå¾Ĺ":117198,"主åħ¬":117199,"设计çIJĨ念":117200,"賺":117201,"æīĢæıIJä¾Ľ":117202,"æīĢæıIJä¾Ľä¹ĭ":117203,"æĶ»åħĭ":117204,"åĤ¾":117205,"è¯Ńæ³ķ":117206,"åįĥåı¤":117207,"éĸĭæĶ¾":117208,"第ä¸ĢèĬĤ":117209,"éĤĦæ²Ĵ":117210,"éĢĥçĶŁ":117211,"æ³Ĺ":117212,"åİ¿å§Ķ书记":117213,"ä½ľèĢħæīĢæľī":117214,"çħ½":117215,"ç»ħ":117216,"æłħ":117217,"æľ´ç´ł":117218,"çijķçĸµ":117219,"åĮħåĮħ":117220,"æ°ij主åħļ":117221,"ä¸įè¿ľå¤Ħ":117222,"å¥ĩå¼Ĥ":117223,"åĺ»åĺ»":117224,"æī¼":117225,"ç¿»å¼Ģ":117226,"æĢİèĥ½":117227,"éģ´éĢī":117228,"解éĩĭ":117229,"å¹¼ç¨ļ":117230,"è¦ģ好好":117231,"è¶´åľ¨":117232,"ç´¢åıĸ":117233,"ç»ĪçĶŁ":117234,"åħ¨æµģç¨ĭ":117235,"éģ©çķ¶":117236,"åįıè°ĥåıijå±ķ":117237,"æĬ¥ä»ĩ":117238,"ç§ijæĬĢåĽŃ":117239,"ä»Ģä¹Īéĥ½ä¸į":117240,"æľĢåIJİä¸Ģ次":117241,"ç»Ļ人ä¸Ģç§į":117242,"æł¸å®ļ":117243,"被åĪĹåħ¥":117244,"æĦıæĥ³ä¸įåĪ°":117245,"èĢĥæŁ¥":117246,"åľ¨æѤä¹ĭåīį":117247,"æīĵçIJĥ":117248,"è¶ĬæĿ¥è¶Ĭå°ij":117249,"å®ļå¾ĭ":117250,"è¡ĮæĶ¿æľºåħ³":117251,"ä½ıæĪ¿åħ¬ç§¯":117252,"å°ıå§IJå§IJ":117253,"ä¸īèı±":117254,"修补":117255,"èŀĥèŁ¹":117256,"西çĶ²":117257,"æĢł":117258,"çŃīå¤ļ项":117259,"产ä¸ļéĽĨèģļ":117260,"ä»·æł¼ä¸Ĭ涨":117261,"åħ¬åħ±åľºæīĢ":117262,"è¢ĭåŃIJ":117263,"æĨ§æĨ¬":117264,"çļĦæĸ¹å¼ıæĿ¥":117265,"åĪ°è´¦":117266,"çģ½":117267,"å·´èı²":117268,"å·´èı²çī¹":117269,"æ¼Ķä¹ł":117270,"èŃ¦ç¤ºæķĻèĤ²":117271,"çķıæĥ§":117272,"å¼ķæµģ":117273,"æĶ¶æĶ¯":117274,"å±Ĥåĩº":117275,"å±Ĥåĩºä¸į":117276,"å±Ĥåĩºä¸įç©·":117277,"æijĩæ»ļ":117278,"辦çIJĨ":117279,"纵è§Ĥ":117280,"æķijæµİ":117281,"家éĥ½çŁ¥éģĵ":117282,"åĮ¯":117283,"å°ı鸣":117284,"ä»»åĭĻ":117285,"计åħ¥":117286,"ç«ŀéĢī":117287,"å¼ĢèįĴæĹ¶æľŁ":117288,"åij¨æģ©":117289,"åij¨æģ©æĿ¥":117290,"交ç»ĩ":117291,"çķ¢æ¥Ń":117292,"æł¹æį®èĩªå·±":117293,"æĸ°äººçݩ家":117294,"åѵåĮĸåĻ¨":117295,"éĩĩæļĸ":117296,"å¹³åĿĩæ°´å¹³":117297,"åħ¬å¼Ģ课":117298,"失åĪ©":117299,"伺æľį":117300,"çĬģ":117301,"忽æĤł":117302,"主è¦ģéĽĨä¸Ń":117303,"æ¤įæłij":117304,"æ¯ĹéĤ»":117305,"èĩºçģ£":117306,"åĩºåĽ½çķĻåѦ":117307,"æĬĹéľĩ":117308,"æĥ©æĪĴ":117309,"å¹´åºķåīį":117310,"åĴ¸éĺ³":117311,"æ°ijå±ħ":117312,"大çIJĨçŁ³":117313,"éĿ³":117314,"éķĸ":117315,"æ¸ħè¿ľ":117316,"è£ħè½½":117317,"èĩĢ":117318,"å½±ä¸ļ":117319,"å¼ŁåħĦ":117320,"æĤ²è§Ĥ":117321,"çĿĢçľ¼äºİ":117322,"æįįåį«":117323,"åī¥å¤º":117324,"ç¯Ĩ":117325,"å¾Īéķ¿æŶéĹ´":117326,"è¥Ł":117327,"第ä¸ĢçĻ¾":117328,"ä¸ĢåĪĨéĴ±":117329,"æĸ°éĹ»è®°èĢħ":117330,"éķ·æľŁ":117331,"æ³ķæĪĺç»ĦåIJĪ":117332,"è°ģçŁ¥éģĵ":117333,"èħ°éĥ¨":117334,"æ±īåł¡":117335,"åħ¥çĿ¡":117336,"åįĸæİī":117337,"æ¶Īè²»èĢħ":117338,"æĥ¯ä¾ĭ":117339,"æĥ³äºĨ":117340,"æĥ³äºĨæĥ³":117341,"èĢģæŧå°ıåĮº":117342,"ä¼łè¨Ģ":117343,"åĪĨæķ°çº¿":117344,"æµģ泪":117345,"ç»Ħç»ĩé¢Ĩ导":117346,"äºļåĨĽ":117347,"å¢ŀåĢ¼æľįåĬ¡":117348,"å¾¹":117349,"伶":117350,"äºĽè®¸":117351,"å¸ĥèݱ":117352,"强æĤį":117353,"宫廷":117354,"绿èĮ¶":117355,"åĮ¡":117356,"å¾ĪæŃ£å¸¸":117357,"æĺ¥å¤ı":117358,"æ¯Ļ":117359,"è¯Ħæ¯Ķ":117360,"åĩ¡äºĭ":117361,"æĬīæĭ©":117362,"åĢĴéľī":117363,"éĩį度":117364,"åįıä¼ļä¼ļéķ¿":117365,"忧èĻij":117366,"ä¸ĭä¸Ģç¯ĩ":117367,"沪深":117368,"æĪİ":117369,"æīĵä»Ĺ":117370,"åįĪé¥Ń":117371,"å¹´é¾Ħ段":117372,"ä¸ŃåĽ½è¶³çIJĥ":117373,"设计æĸ¹æ¡Ī":117374,"åºĶçĶ¨æŁ¥çľĭ":117375,"é¢ĦæĸĻ":117376,"åĹ¡":117377,"ç¥ĸçĪ¶":117378,"çļĦä¸Ģåijĺ":117379,"æ´Ĺå¹²åĩĢ":117380,"åİĨåı²æĸ°":117381,"åİĨåı²æĸ°é«ĺ":117382,"çĭ¬åħ·":117383,"æħĭ度":117384,"æīĵ交":117385,"æīĵ交éģĵ":117386,"é»ĦçŁ³":117387,"çĽ¼æľĽ":117388,"çī§åľº":117389,"转弯":117390,"åįĩåįİ":117391,"åĨįä¹Łæ²¡æľī":117392,"èĭ±æīį":117393,"æĽ´åIJį为":117394,"åĢŁçĶ¨":117395,"çºłéĶĻ":117396,"ç»Ŀ对ä¸įä¼ļ":117397,"çİĭçīĮ":117398,"çĽĨåľ°":117399,"失è°ĥ":117400,"好象":117401,"é³¥":117402,"ä¿Ŀä¿®":117403,"åĽĽä¸ªèĩªä¿¡":117404,"头çļ®":117405,"åİŁåīĩ":117406,"æĬ¥æ¡Ī":117407,"奴éļ¶":117408,"å³Ļ":117409,"è°ĥæĸĻ":117410,"ä¹Łè¨±":117411,"èIJ½åĪ°":117412,"èIJ½åĪ°å®ŀ":117413,"èIJ½åĪ°å®ŀå¤Ħ":117414,"çĦļçĥ§":117415,"çĶŁæ´»çݯå¢ĥ":117416,"åºĶåıĬæŶ":117417,"è¶Ĭè¿ĩ":117418,"æĦŁè¬Ŀ":117419,"æĻ¯å¾·":117420,"æĻ¯å¾·éķĩ":117421,"çĬĢ":117422,"身éĤĬ":117423,"ç¨İåĬ¡æĢ»å±Ģ":117424,"åĩĢåľŁ":117425,"ä¾µåįł":117426,"åĬ¨å·¥":117427,"å¹´ä¹ĭ":117428,"å¹´ä¹ĭä¹ħ":117429,"第äºĮèĬĤ":117430,"åĬ¨çī©åĽŃ":117431,"第ä¸Ģ书记":117432,"éħļ":117433,"çĶŁäº§è®¾å¤ĩ":117434,"æŁIJç§įç¨ĭ度":117435,"åľŃ":117436,"åĩŃåĢŁçĿĢ":117437,"éĺħè§Ī":117438,"çĻ½æ²Ļ":117439,"æ²¹çĥŁ":117440,"çªģçł´åı£":117441,"åıĹå½±åĵį":117442,"åı¯ä»¥æĽ´å¥½":117443,"å³°åĢ¼":117444,"æĿĤè´¨":117445,"宿è¿ģ":117446,"çĽĺæ´»":117447,"æ¿Ģèµ·":117448,"åĦ¿ç§ij":117449,"åĿIJèIJ½åľ¨":117450,"æĮªå¨ģ":117451,"æµ·å²Ľ":117452,"绣绣":117453,"éĻ¨":117454,"ä¼ĺäºİ":117455,"å°Ī家":117456,"ä¸ĢéĤĬ":117457,"èIJĬ":117458,"äºĨä¸Ģåı£":117459,"æ²ĥå°Ķæ²ĥ":117460,"æŃ£å¸¸ä½¿çĶ¨":117461,"æĻ®éģįåŃĺåľ¨":117462,"丰满":117463,"çĶ»åį·":117464,"åºĶæĶ¶":117465,"åºĶæĶ¶è´¦":117466,"åºĶæĶ¶è´¦æ¬¾":117467,"å®Įæķ´çĥŃ":117468,"å®Įæķ´çĥŃæ¦ľ":117469,"注è§Ĩ":117470,"çĨĦ":117471,"躬":117472,"éĶĢåĶ®äººåijĺ":117473,"è¶ĭåIJij":117474,"çĦ¦æĢ¥":117475,"åįģå¹´åīį":117476,"ä¼łç»Łäº§ä¸ļ":117477,"質éĩı":117478,"åĩ¤åĩ°ç½ij":117479,"èµĦæºIJæķ´åIJĪ":117480,"æ¶Įåħ¥":117481,"æĸĩåĮĸä¼łæĴŃ":117482,"çķĮ第ä¸Ģ":117483,"æ°´æ³µ":117484,"宫殿":117485,"æݢ寻":117486,"ä¿®åīª":117487,"æĦıè¦ĭ":117488,"ç´Ĭä¹±":117489,"æĽī":117490,"çĻ½è¡£":117491,"èĻİåį«":117492,"ç´§æī£":117493,"å¤Ħå¤Ħéķ¿":117494,"åĪĽå»ºå·¥ä½ľ":117495,"红æŀ£":117496,"饼干":117497,"äºĨåįĬ天":117498,"ä¼ļå½±åĵįåĪ°":117499,"çĽ¸ä¿¡å¤§å®¶":117500,"èħ¾é£ŀ":117501,"å°±å¦ĤåIJĮ":117502,"ä¸ĭéĿ¢å°ıç¼ĸ":117503,"æ°ijèIJ¥ç»ıæµİ":117504,"æĻ¦":117505,"è£ħæī®":117506,"é»ijå¤ľ":117507,"常德":117508,"å·¥ä¸ļ大åѦ":117509,"æĺİçŁ¥":117510,"éĺŁåijĺ们":117511,"åIJ¬è¯¾":117512,"æ¯ıéļĶ":117513,"羣æĺ¯å¤ª":117514,"åIJĪä½ľåħ±èµ¢":117515,"çIJĨåıij":117516,"æīįå¹²":117517,"çľĭèµ·ä¾Ĩ":117518,"殿ä¸ĭ":117519,"å®īéĺ³":117520,"æīĢ产çĶŁçļĦ":117521,"éĽĩä½£":117522,"æĬ¬èµ·å¤´":117523,"æį®æĬ¥éģĵ":117524,"éļĨéĩį举è¡Į":117525,"交éĶĻ":117526,"è¶ħé¢Ŀ":117527,"åĮĸçĸĹ":117528,"é¡Ĩ":117529,"纵深":117530,"çĪ±åĽ½ä¸»ä¹ī":117531,"éĻ¢åī¯éĻ¢éķ¿":117532,"讳":117533,"羣æŃ£åģļåĪ°":117534,"åѤåįķ":117535,"èĩªçĦ¶èĢĮ":117536,"èĩªçĦ¶èĢĮçĦ¶":117537,"修身":117538,"èĬ¹":117539,"æģ¯æģ¯":117540,"æģ¯æģ¯çĽ¸åħ³":117541,"é©¾æł¡":117542,"æݩ饰":117543,"æ³½è¿ŀ":117544,"æ³½è¿ŀæĸ¯åŁº":117545,"举æŃ¢":117546,"管çIJĨä½ĵåĪ¶":117547,"åħ¶ä¸Ńä¹ĭä¸Ģ":117548,"æĿ¾å¼Ľ":117549,"æĭ¦æĪª":117550,"åį«åģ¥":117551,"åį«åģ¥å§Ķ":117552,"ä»İåݻ年":117553,"åĤ¢":117554,"è´Ń票":117555,"åĽ¾æłĩ":117556,"河西":117557,"æ°ijæĶ¿å±Ģ":117558,"ç§ģèIJ¥":117559,"å¤ĸåĽ½è¯Ń":117560,"干货":117561,"æĵ¦æĭŃ":117562,"åľ°ä¸Ń":117563,"åľ°ä¸Ńæµ·":117564,"æµĵæµĵ":117565,"æµĵæµĵçļĦ":117566,"å§ĭ建":117567,"å§ĭ建äºİ":117568,"ç¶ĵæŃ·":117569,"è·¯æ¼Ķ":117570,"æļ´é£İ":117571,"åŁºè¾ħ":117572,"æī¶è´«å·¥ä½ľ":117573,"ä¸Ģ缴å¤Ħäºİ":117574,"æĥħ趣":117575,"äºĮåŃ£åº¦":117576,"åİĮæģ¶":117577,"顺åĪ©å®ĮæĪIJ":117578,"æŁ¥å°ģ":117579,"顶端":117580,"ä¸įåŃķ":117581,"ä¸Ģ大åłĨ":117582,"被æ·ĺæ±°":117583,"æĺ¯çĶ¨æĿ¥":117584,"æľĢåIJĪéĢĤ":117585,"äº®çľ¼":117586,"并ä¸įæĺ¯å¾Ī":117587,"ç§ijçłĶéĻ¢":117588,"ç§ijçłĶéĻ¢æīĢ":117589,"ç²Ł":117590,"é¢Īéĥ¨":117591,"é»ĺé»ĺåľ°":117592,"é«ĺä¸ŃçĶŁ":117593,"æĹıèĩªæ²»åİ¿":117594,"æķĻåŃ¦è´¨éĩı":117595,"æĪĺçģ«":117596,"åĿİåĿ·":117597,"æIJŃä¹ĺ":117598,"è¯ĹæĦı":117599,"åĪijèѦ":117600,"åĩºæ±Ĺ":117601,"åįģåħŃæĿ¡":117602,"请åıĬæŶ":117603,"åĨľä¸ļ大åѦ":117604,"èIJ½åı¶":117605,"æĢ»èĢĮè¨Ģ":117606,"æĢ»èĢĮè¨Ģä¹ĭ":117607,"æĿľåħ°":117608,"æĿľåħ°çī¹":117609,"éĻªä½ł":117610,"åħ¬æĬ¥":117611,"çķĻè¨ĢæĿ¿":117612,"éĺħåİĨ":117613,"競çĪŃ":117614,"ç»ĻåĪ«äºº":117615,"æĹ¥æĬ¥ç¤¾":117616,"åĿIJèIJ½":117617,"åĿIJèIJ½äºİ":117618,"éĩijåŃĹ":117619,"éĩijåŃĹå¡Ķ":117620,"åĽ¤":117621,"è¯Ŀåī§":117622,"æĮģç»Ńæİ¨è¿Ľ":117623,"æ¼ıæ°´":117624,"詳細":117625,"æĢĢæĬ±":117626,"åıĺå¹»":117627,"饥饿":117628,"éļIJ身":117629,"ä¸ªèµĽåŃ£":117630,"åĵ¡å·¥":117631,"æģ¢å¤įæŃ£å¸¸":117632,"äºĨ好å¤ļ":117633,"æĺŁå·´":117634,"æĺŁå·´åħĭ":117635,"åħīçݯ":117636,"å¸ħåĵ¥":117637,"çĻ½éĽª":117638,"ç¨įç¨į":117639,"计æıIJ":117640,"æĦĽæĥħ":117641,"éİĸ":117642,"ä¿¡éĺ³":117643,"è§Ģå¯Ł":117644,"å¦Ĥæŀľä½łæĥ³":117645,"缸æ¯Ķä¹ĭä¸ĭ":117646,"解å¼Ģ":117647,"æīĵåį°æľº":117648,"身躯":117649,"ç²¾ç¥ŀæĸĩæĺİ":117650,"èĤ¡æĮĩ":117651,"å¾®åĪĽ":117652,"红èĮ¶":117653,"èĩ´çĻĮ":117654,"æģ©æĸ½":117655,"èħ¿éĥ¨":117656,"大åŀĭå¤ļ人":117657,"å®īåĢį":117658,"è¾ħ导åijĺ":117659,"èĪªéģĵ":117660,"å¸ĥå°Ķ":117661,"åįĹå®ģå¸Ĥ":117662,"ä¸ĬçıŃæĹı":117663,"侧ç»ĵæŀĦæĢ§":117664,"追éļı":117665,"å½ĵåľ°æĶ¿åºľ":117666,"èµ°åĩºæĿ¥":117667,"éĩijèŀįä¸ļ":117668,"ä¸Ľä¹¦":117669,"é¡¹çĽ®ç»ıçIJĨ":117670,"è¿ĩæĪ·":117671,"骨æŀ¶":117672,"è¡Ļ":117673,"ä»Ģ麽":117674,"èħĭ":117675,"è¦ģ害":117676,"åľ¨åºĬä¸Ĭ":117677,"代è¨Ģ人":117678,"並å°ĩ":117679,"åIJĦ个æĸ¹éĿ¢":117680,"è°´è´£":117681,"åħ±æĮ¯":117682,"åį³å°ĨåĪ°æĿ¥":117683,"èĤºçĻĮ":117684,"ä¾ĽéĶĢ":117685,"丼æŀĹ":117686,"èµĥ":117687,"åįģä½Ļå¹´":117688,"åĭĺæİ¢":117689,"飵åij³":117690,"èĭ¦ç¬ij":117691,"æľĢ大ç¨ĭ度":117692,"éĩįçĤ¹åħ³æ³¨":117693,"ä¹ĭ举":117694,"满æĢĢ":117695,"åıĹåĪ°å½±åĵį":117696,"æĭĽæĬķæłĩ":117697,"è¡¥é½IJ":117698,"西红":117699,"è¥¿çº¢æŁ¿":117700,"鬧":117701,"è£ħåį¸":117702,"éĤ»éĩĮ":117703,"èĤĩäºĭ":117704,"æİĴæ¯Ĵ":117705,"åѤåĦ¿":117706,"鼶è·Ŀ离":117707,"å®ŀå¹²":117708,"çľĭæŁ¥çľĭ":117709,"æĶ¶è´¹ç«Ļ":117710,"ç»·":117711,"åħ¬çĽĬæĢ§":117712,"éĢĴç»Ļ":117713,"æĶ»æīĵ":117714,"æĺŁçº§éħĴåºĹ":117715,"æĺİåªļ":117716,"çį¨ç«ĭ":117717,"è¯Ŀè¯ŃæĿĥ":117718,"ä¸ĢæŃ¥ä¸ĢæŃ¥":117719,"书æ³ķ家":117720,"æľªç»ıæİĪæĿĥ":117721,"çŁ³èĨı":117722,"åĩŃä»Ģä¹Ī":117723,"çļĦæĹ¥":117724,"çļĦæĹ¥åŃIJéĩĮ":117725,"诱人":117726,"çĻ¾åĪĨçĻ¾":117727,"èĪĪ趣":117728,"å¼łåħĪçĶŁ":117729,"èĢģçĪ·åŃIJ":117730,"æ³¢çī¹":117731,"åŁºéĩij份é¢Ŀ":117732,"æ²Ļåıijä¸Ĭ":117733,"å¥ĭæĸĹ缮æłĩ":117734,"æ°¢èĥ½":117735,"æ²ĥå°ĶçİĽ":117736,"義åĭĻ":117737,"éŁ³ç®±":117738,"æ²ī浸":117739,"æ²īæµ¸åľ¨":117740,"èĭ±åľĭ":117741,"çģ¯çģ«":117742,"è¿Ľé¡¹":117743,"两端":117744,"ä¹Ķ丹":117745,"èĦ¸é¢Ĭ":117746,"åıijå±ķæ½ľåĬĽ":117747,"åĭķä½ľ":117748,"åĵĪä½Ľ":117749,"å®´ä¼ļ":117750,"æ§į":117751,"ç«ĭå¿Ĺ":117752,"ç¡ķ士åѦä½į":117753,"åĭĭç«ł":117754,"è¿Ļåľºæ¯ĶèµĽ":117755,"æĮģå¹³":117756,"éķĢéĶĮ":117757,"èĭ±çī¹":117758,"èĭ±çī¹å°Ķ":117759,"æķĻèģĮå·¥":117760,"åĬŁåĬĽ":117761,"该æ¡Ī":117762,"ä¸Ģæ¢Ŀ":117763,"åĺīå¹´":117764,"åĺīå¹´åįİ":117765,"è¿«ä¸įåıĬ":117766,"è¿«ä¸įåıĬå¾ħ":117767,"è¿Ļ个æĹ¶ä»£":117768,"精彩æĴŃæĬ¥":117769,"人èĦ¸":117770,"人èĦ¸è¯ĨåĪ«":117771,"æ£Ģå¯Łå®ĺ":117772,"å°ıèħ¿":117773,"éĨĴ缮":117774,"åħļæĢ»":117775,"åħļæĢ»æĶ¯":117776,"æĪŁ":117777,"èĮ«çĦ¶":117778,"è±ĨæµĨ":117779,"主治":117780,"éĿĴæµ·çľģ":117781,"åĪijäºĭ责任":117782,"çł°":117783,"ä¹ĭæ¬ĬåĪ©":117784,"äºĶå®ĺ":117785,"è¿·æĥij":117786,"åħ¥åºĵ":117787,"家纺":117788,"弹簧":117789,"åįģäºĶæĿ¡":117790,"ç»Ļå®Ŀå®Ŀ":117791,"èĪªç©ºèĪªå¤©":117792,"å¾Ģå¤ĸ":117793,"å¼ķåĬĽ":117794,"çľ¼çļ®":117795,"æ¶ī足":117796,"æĿ¥å®¾":117797,"åľ¨çº¿è§Ĵèī²":117798,"çĥŃéĶĢ":117799,"æµģéĢĿ":117800,"泡泡":117801,"éĻįå¹ħ":117802,"è´ŁéĿ¢å½±åĵį":117803,"红楼":117804,"红楼梦":117805,"éļĶçĿĢ":117806,"侥幸":117807,"许ä¹ħ":117808,"åĴĮçĿ¦":117809,"èŃ½":117810,"使çĶ¨èĢħæĪĸ":117811,"ä¹°åįķ":117812,"è¿´":117813,"é£İæīĩ":117814,"æķĻ師":117815,"æ¡ĮåŃIJä¸Ĭ":117816,"å¾Īæ¼Ĥ亮":117817,"åł±å°İ":117818,"第ä¸ĢåŃ£åº¦":117819,"ç©©å®ļ":117820,"æĤ²åĵĢ":117821,"çĿĢåĬĽæīĵéĢł":117822,"æĮŁ":117823,"路桥":117824,"åijIJ":117825,"åľ£è¯ŀèĬĤ":117826,"çļĩåŃIJ":117827,"ä»ĩæģ¨":117828,"éħĿéħ¿":117829,"ä¸įéĹ´":117830,"ä¸įéĹ´æĸŃ":117831,"æĮĩå°ĸ":117832,"ä¸ŃåĽ½ç½ij游":117833,"åŀ£":117834,"æĦıè§ģ建议":117835,"æ¯ħçĦ¶":117836,"亮度":117837,"èģĶè°Ĭ":117838,"å½ķåħ¥":117839,"åĦ²":117840,"å¨ĺ家":117841,"ç§ijå°Ķ":117842,"ä¹Łæ²¡ä»Ģä¹Ī":117843,"æł¹æį®ä¸įåIJĮ":117844,"åı¶ä¿®":117845,"åĢ¼å®Ī":117846,"æľ«ç«¯":117847,"åĪ¨":117848,"åĤµåĭĻ":117849,"èģ¯åIJĪ":117850,"å¥ĩå¹»":117851,"èĻļæŀĦ":117852,"é»Ħæĺı":117853,"å¹³åĿ¦":117854,"æµģæ°ĵ":117855,"æĸ°åŁºå»º":117856,"æĮ½æķij":117857,"åįİå°Ķ":117858,"åįİå°Ķè¡Ĺ":117859,"æľĢåıĹ欢è¿İ":117860,"ç»Ń约":117861,"å¼Ĭ端":117862,"éŃĶæ³ķå¸Ī":117863,"éŃĶæ³ķå¸ĪåĴĮ":117864,"åħ·ä½ĵåĨħ容":117865,"çIJīçĴĥ":117866,"æī©å®¹":117867,"èĮ¶åĽŃ":117868,"主ä¹īèĢħ":117869,"ç«ĭéĿ¢":117870,"æİ¥åıĹéĩĩ访":117871,"åĩºåħ¥å¢ĥ":117872,"ç§ijåįı":117873,"éĴ³":117874,"çµIJæ§ĭ":117875,"ç»ĵæŀľæĺ¾ç¤º":117876,"åı°è´¦":117877,"å°±æĿ¥çľĭçľĭ":117878,"èĩªæķij":117879,"åıįæĩī":117880,"åİ»åĵªåĦ¿":117881,"è¿Ļé¦ĸ":117882,"è¿Ļé¦ĸæŃĮ":117883,"åIJ¬ä¼Ĺ":117884,"å¤ĸ壳":117885,"ä½ĵèĤ²é¦Ĩ":117886,"實æĸ½":117887,"èŀºä¸Ŀ":117888,"æĭīåįĩ":117889,"çĮĽåľ°":117890,"åħ¨åĽ½äººæ°ij":117891,"æĤīå°¼":117892,"æĹı群":117893,"åĽ¢åijĺ":117894,"两个å°ıæŶ":117895,"åľ¨çݩ家":117896,"åľ¨çݩ家ä¸Ń":117897,"çĶľçĶľ":117898,"æĬķè¡Į":117899,"åįĶæľĥ":117900,"éĻ¡":117901,"åĬłå·¥åİĤ":117902,"æ¦ĨæŀĹ":117903,"æŃ»è§Ĵ":117904,"åĨħå¹ķ":117905,"æīĢæľīæĥħèĬĤ":117906,"åĪ·åį¡":117907,"æ°´èĤ¿":117908,"èĥĥåı£":117909,"å«Įå¼ĥ":117910,"沮丧":117911,"ä¸ī年级":117912,"æ¶Ĥå±Ĥ":117913,"å¿ĥ仪":117914,"å¿ĥ仪çļĦ":117915,"å¤Ń":117916,"é¦ĸè½®":117917,"æĹłè®ºæĺ¯åħ¶":117918,"éĢıæ°Ķ":117919,"äºĮåįģäºĶ":117920,"箫":117921,"åĬŁåĬ³":117922,"çŃ¾ä¸ĭ":117923,"æ²īè¿·":117924,"æķijåij½":117925,"éĹªéĹª":117926,"åIJĥäºı":117927,"å±ķåĵģ":117928,"åį³æŶåıijçĶŁ":117929,"綾":117930,"綾åIJĪ":117931,"æłĩæĺİ":117932,"çľĭçĶµå½±":117933,"åħ¬ç«ł":117934,"éĺ¿æ£®":117935,"éĺ¿æ£®çº³":117936,"身åĪĽéĢł":117937,"身åĪĽéĢłçļĦ":117938,"æ¸Ľå°ij":117939,"åĢ¼å¾Ĺåħ³æ³¨":117940,"鼶åĶ®åķĨ":117941,"æįĨç»ij":117942,"è¸ıåħ¥":117943,"èĽŁ":117944,"æŁ´çº³":117945,"èĢģåħµ":117946,"绿èī²çݯä¿Ŀ":117947,"é¹Ń":117948,"éº»æľ¨":117949,"æıŃçīĮ":117950,"è¿Ļ款车":117951,"ç¾İå¾·":117952,"ç¾İå¾·åħ¬åı¸":117953,"涧":117954,"è°ģçŁ¥":117955,"æ´ĭèij±":117956,"æ¯įæł¡":117957,"ä¸ĢéĹª":117958,"çĶ·ä¸»è§Ĵ":117959,"æĹłçº¿çĶµ":117960,"å±łå®°":117961,"æĺ¯éŁ©åĽ½":117962,"æĺ¯éŁ©åĽ½å¨±":117963,"容è²Į":117964,"åĿĩ使åħ¶":117965,"太快":117966,"å¹´çĶ±":117967,"å¹´çĶ±çĽĽ":117968,"èĭ¦èĭ¦":117969,"åĬĽè¿ĺæĺ¯":117970,"åĬĽè¿ĺæĺ¯èĩª":117971,"æĨ©":117972,"èģ¯çµ¡":117973,"åĶ¾":117974,"åħ·æľīæĪĺ士":117975,"追éĹ®":117976,"åłĨæĶ¾":117977,"åıį驳":117978,"å®ŀäºĭæ±Ĥ":117979,"å®ŀäºĭæ±Ĥæĺ¯":117980,"åѸéĻ¢":117981,"åįģåĩłä¸ª":117982,"æķijæĬ¤":117983,"æķijæĬ¤è½¦":117984,"ç½ijç»ľä¼łæĴŃ":117985,"åįģåħ«å±Ĭ":117986,"éĥ¨åī¯":117987,"éĥ¨åī¯éĥ¨éķ¿":117988,"çĹ´è¿·":117989,"管çIJĨæĿ¡ä¾ĭ":117990,"èŀį为ä¸Ģä½ĵ":117991,"æĢ»äº§åĢ¼":117992,"è³ĵ":117993,"ä¸ĥæĺŁ":117994,"çıŃç»Ħ":117995,"绣é¢Ĩ":117996,"请大家":117997,"éĩijéĻµ":117998,"èĪħèĪħ":117999,"æµ·æ¹¾":118000,"æĸ½çŃĸ":118001,"享èªī":118002,"麥":118003,"端åįĪ":118004,"绿åŁİ":118005,"確ä¿Ŀ":118006,"å·´æĭī":118007,"åĨĴçĿĢ":118008,"æħ·æħ¨":118009,"个人è§ĤçĤ¹":118010,"ä¹Ļçĥ¯":118011,"ç¡ħè°·":118012,"éĸĭå±ķ":118013,"å°ļ书":118014,"åĿļ飧":118015,"庵":118016,"èĢģé¾Ħ":118017,"èĢģé¾ĦåĮĸ":118018,"çľ¨çľ¼":118019,"绿水":118020,"绿水éĿĴå±±":118021,"书é¦Ļ":118022,"主åĬĽåĨĽ":118023,"æīįæĺ¯çľŁæŃ£":118024,"æĬ¢åħĪ":118025,"æĪIJå°±æĦŁ":118026,"éĩįæŀĦ":118027,"éĴ¢åİĤ":118028,"æĪIJ份":118029,"èĬ±çº¹":118030,"ä¹ĭäºī":118031,"å¹²ç»Ĩèĥŀ":118032,"æĹ¢åı¯ä»¥":118033,"ç¹ģçIJIJ":118034,"æĦļèł¢":118035,"éĿŀ常æĺİæĺ¾":118036,"ä½ĵ彩":118037,"æĬĢæ³ķ":118038,"æĿĨèıĮ":118039,"å¹¿æ³Ľåħ³æ³¨":118040,"åĮĹå®ĭ":118041,"å§Ĭ妹":118042,"åįıåĬŀ":118043,"æ·®åįĹ":118044,"çĥı":118045,"æ´ĹèĦ¸":118046,"åıĹ访":118047,"åıĹ访èĢħ":118048,"éĩįè¦ģåĽłç´ł":118049,"å½±è§Ĩåī§":118050,"综èīºèĬĤ缮":118051,"èľķåıĺ":118052,"äºĮ线":118053,"äºĮ线åŁİå¸Ĥ":118054,"ä¼Ĭå§ĭ":118055,"çıĬçijļ":118056,"èĩªæŁ¥":118057,"åħ¥åĽŃ":118058,"åĩ¶æīĭ":118059,"åħ¬è¯ī":118060,"éģĩéļ¾":118061,"éĩĩçŁ¿çŃī":118062,"èĩªçIJĨ":118063,"åĸ·æ¶Ĥ":118064,"æī©åħħ":118065,"éĢıè§Ĩ":118066,"é«ĺéĢŁå¢ŀéķ¿":118067,"åĽ¾çĶ»":118068,"ç¾¹":118069,"èĤĩåºĨ":118070,"è¾ľè´Ł":118071,"èµĶä»ĺ":118072,"è·¡":118073,"åģ¥åº·æĪIJéķ¿":118074,"以ä¸ĬåѦåİĨ":118075,"åıĸå¾Ĺ以åıĬ":118076,"æ²ī积":118077,"åįģä¹Ŀå±Ĭ":118078,"缸éĹľæľįåĭĻ":118079,"æī§åĭ¤":118080,"åī¯åİ¿éķ¿":118081,"寰":118082,"åģľæ»ŀ":118083,"淹没":118084,"çŁ³çģ°":118085,"çį¸":118086,"åĢ¦":118087,"ç¾İåªĴ":118088,"æķĻæ¡Ī":118089,"åĬłçĽĸ":118090,"åħ¬å¼ĢèµĽ":118091,"å¥łåŁº":118092,"æĺĨèĻ«":118093,"çŀħ":118094,"磷éħ¸":118095,"äºīåĪĽ":118096,"çİĭæĻĵ":118097,"ç¼ĵåĨ²":118098,"åİļåİļ":118099,"åİļåİļçļĦ":118100,"æŀ£åºĦ":118101,"ç²¾çĽĬ":118102,"ç²¾çĽĬæ±Ĥ":118103,"ç²¾çĽĬæ±Ĥç²¾":118104,"åĪĨæĶ¯æľºæŀĦ":118105,"å®ŀæĸ½ç»ĨåĪĻ":118106,"æĸ°èµĽåŃ£":118107,"總統":118108,"éĢłè¡Ģ":118109,"é¢ĩåħ·":118110,"é»ĦåŁĶ":118111,"è¡ĢèĦĤ":118112,"交éĢļå·¥åħ·":118113,"å³¥":118114,"æĹıèĩªæ²»å·ŀ":118115,"寺éĻ¢":118116,"確å®ļ":118117,"æ¦Ĥ念èĤ¡":118118,"æĦŁå®ĺ":118119,"æŁľåı°":118120,"åĶĶ":118121,"çŀŃ解並":118122,"æĢ»ä»·":118123,"åIJ¸åħ¥":118124,"æĢ¼":118125,"æĻļéĹ´":118126,"å±Ĭæ¯ķä¸ļçĶŁ":118127,"çĶŁå§ľ":118128,"éĺħ读åħ¨æĸĩ":118129,"å¾ĹåĪ°æľīæķĪ":118130,"æIJľæķij":118131,"åİĨæĿ¥":118132,"èŃīæĺİ":118133,"åĥ»":118134,"èĨ³é£Ł":118135,"åĦĦåħĥ":118136,"æīĵåİĭ":118137,"宾客":118138,"åķ¼":118139,"ä¸ĢçĻ¾å¤ļ":118140,"æ·±åħ¥äººå¿ĥ":118141,"æ¢ħå·ŀ":118142,"çłĶåѦ":118143,"åħ³ä¹İ":118144,"è¼Ľ":118145,"亲åıĭ":118146,"éħįæĸĻ":118147,"æĪijçĪ±ä½ł":118148,"è´¸æĺĵæĪĺ":118149,"æľīèī²":118150,"æľīèī²éĩijå±ŀ":118151,"æįIJåĬ©":118152,"为é¦ĸ":118153,"为é¦ĸçļĦ":118154,"å¯ĮåĬĽ":118155,"çĶ·ç¥ŀ":118156,"é³³":118157,"æµĩæ°´":118158,"åIJ±":118159,"æĺİç¡®æıIJåĩº":118160,"åı¹äºĨ":118161,"åı¹äºĨåı£æ°Ķ":118162,"礼æĭľ":118163,"è¿Ļ个åIJįåŃĹ":118164,"ä¿¡å¾Ĵ":118165,"å¿Ĺ强":118166,"éĻIJæŶ":118167,"æĶ¶è²»":118168,"åĨľå®¶ä¹IJ":118169,"å°ıé¾ĻèĻ¾":118170,"èIJ½å¹ķ":118171,"æ§Ł":118172,"åѦ龸":118173,"æĪĸå¤ļ":118174,"æĪĸå¤ļæĪĸ":118175,"æĪĸå¤ļæĪĸå°ij":118176,"座è°Īä¼ļä¸Ĭ":118177,"涼":118178,"éŃĶçİĭ":118179,"å²±":118180,"顶å±Ĥ":118181,"顶å±Ĥ设计":118182,"èĦijåŃIJéĩĮ":118183,"éĻ¢åŃIJéĩĮ":118184,"轩è¾ķ":118185,"身å¿ĥåģ¥åº·":118186,"èħij":118187,"éĹľæ³¨":118188,"åıĤåĬłä¼ļè®®":118189,"ä¸ŃåįİæĸĩåĮĸ":118190,"追寻":118191,"å®īçĦ¶":118192,"é£Ļåįĩ":118193,"éŁŃèıľ":118194,"鸦":118195,"åĤ¨éĩı":118196,"çĶ·æĸ¹":118197,"å¤ĩ份":118198,"æijĶåĢĴ":118199,"润æ»ijæ²¹":118200,"éĢ¼è¿ij":118201,"çĶ³è¯ī":118202,"鸣类":118203,"çŁ³æ²¹åĮĸå·¥":118204,"åĿļæŀľ":118205,"è¿Ļ家ä¼Ļ":118206,"æĭĴä¸į":118207,"羣çļ®":118208,"è·ĿéĽ¢":118209,"è¿ĺæĮº":118210,"éĽķåĥı":118211,"åĪĿæģĭ":118212,"æıIJä¾ĽæĽ´å¤ļ":118213,"æŁ¥çľĭåħ¨æĸĩ":118214,"æķ°åŃĹè´§å¸ģ":118215,"åĸīåĴĻ":118216,"åı¦ä¸Ģä½į":118217,"åĤ¬åĮĸ":118218,"åĤ¬åĮĸåīĤ":118219,"ä»İæĿ¥æ²¡":118220,"å¯ĨåĪĩ缸åħ³":118221,"éĥ¨ä¸»ä»»":118222,"产åĵģç»ıçIJĨ":118223,"並åIJĮæĦı":118224,"èIJ½åħ¥":118225,"å±ıå¹ķä¸Ĭ":118226,"åħ¬åı¸ç«łç¨ĭ":118227,"æį¢åı¥è¯Ŀ":118228,"æį¢åı¥è¯Ŀ说":118229,"ä½įæĸ¼":118230,"ä½Ķ":118231,"åĩ»æĿĢ":118232,"缸è¾ĥ":118233,"缸è¾ĥäºİ":118234,"ç²½åŃIJ":118235,"åįĹæŀģ":118236,"宫é¢Ī":118237,"è£ģåijĺ":118238,"æĺİç»Ĩ":118239,"ä»·åĢ¼éĵ¾":118240,"åĽĽä¸ªæĸ¹éĿ¢":118241,"æĥħåĨµæĿ¥çľĭ":118242,"æĮijåīĶ":118243,"æ®ĺ":118244,"æŀģåĬĽ":118245,"çĸijéļ¾":118246,"æĬµæĬĹåĬĽ":118247,"æĢ¥éĢŁ":118248,"æĪĮ":118249,"ä½İä¼°":118250,"éĹªè¿ĩ":118251,"æģ¬":118252,"èµŀæī¬":118253,"ä»ĸå¦Ī":118254,"æĪIJ为ä¸ĢåIJį":118255,"æ´Ĺ礼":118256,"é¢Ħ计å°Ĩ":118257,"åħĪè¿Ľåįķä½į":118258,"è¼Ķ":118259,"éĢĥèĦ±":118260,"çİ°åŃĺ":118261,"èĢģèĻİæľº":118262,"åįģä¸ĥæĿ¡":118263,"åı¦ä¸ĢåįĬ":118264,"温æĥħ":118265,"åī¥ç¦»":118266,"ä¸ĸè´¸":118267,"å®ĺåı¸":118268,"å¾Īå·®":118269,"éĹ´è·Ŀ":118270,"请注æĦı":118271,"åı²è¯Ĺ":118272,"åĪ©åĻ¨":118273,"è¿IJç®Ĺ":118274,"沦为":118275,"該使çĶ¨èĢħ":118276,"èĮ¬":118277,"éĶ¦ç»£":118278,"åı²æĸĻ":118279,"çģµæ´»æĢ§":118280,"èģĶ社":118281,"æĹłåĬ©":118282,"æĬĹæ°§åĮĸ":118283,"èıľèĤ´":118284,"éĢłèĪ¹":118285,"æİīèIJ½":118286,"å¤įæŁ¥":118287,"åĭĥåĭĥ":118288,"åij¼å£°":118289,"給äºĪ":118290,"åIJĮäºĭ们":118291,"ç½°":118292,"è¯ķæİ¢":118293,"åħ³éĶ®åŃĹ":118294,"æįIJçĮ®":118295,"ç»Łè®¡æķ°æį®":118296,"åĪĽä½ľèĢħ":118297,"ä¸ĭåįĬ":118298,"ä¸ĭåįĬåľº":118299,"æī¿æĭħ责任":118300,"端æŃ£":118301,"ç©¿è¡£":118302,"ä¼łçIJĥ":118303,"åĬ©éķ¿":118304,"åĩ±":118305,"éķ¶åµĮ":118306,"é£ŀç¿Ķ":118307,"è¾ĵåįµ":118308,"è¾ĵåįµç®¡":118309,"ä¸ĩåħ¬éĩĮ":118310,"æݨ广åºĶçĶ¨":118311,"å¿«æ¨Ĥ":118312,"秽":118313,"èī°å·¨":118314,"åIJ¬å®Į":118315,"åĿļ硬":118316,"å¥¥åľ°":118317,"å¥¥åľ°åĪ©":118318,"é¢ĵ":118319,"èĻIJå¾ħ":118320,"ä¾Ľæ±Ĥ":118321,"éľīç´ł":118322,"伪è£ħ":118323,"ä¹¡åľŁ":118324,"åĩ¡æľ¬ç½ij":118325,"åĩ¡æľ¬ç½ij注":118326,"ä¼ĬåĪ©":118327,"è¡¡æ°´":118328,"æĽ´åĥıæĺ¯":118329,"åĪĨéĴŁå·¦åı³":118330,"è¦ı模":118331,"äºĶåĪĨéĴŁ":118332,"åºĹåĬłçĽŁ":118333,"åĽ°éĽ£":118334,"åħ³åģľ":118335,"æĢĿ绪":118336,"åĴ½åĸī":118337,"缸符":118338,"çĥ¦èºģ":118339,"æĻĤæľŁ":118340,"åijĪçı¾":118341,"解æķ£":118342,"诱导":118343,"éļĶçĥŃ":118344,"çĮ¶":118345,"åįĹå®ĭ":118346,"æ·±åħ¥äºĨ解":118347,"çŃĶçĸij":118348,"æĺ¼å¤ľ":118349,"åįĥä¼ı":118350,"åĬ³åĬ¡æ´¾éģ£":118351,"红è±Ĩ":118352,"åĿıäºĭ":118353,"çĤ¹æ»´":118354,"å°±ä¸ļå²Ĺä½į":118355,"约åIJĪ":118356,"åħįéĻ¤":118357,"éĢĨåĬ¿":118358,"éĩįéĩijå±ŀ":118359,"å®ĺ宣":118360,"ä½İå»ī":118361,"æģ¨ä¸įå¾Ĺ":118362,"å¾Ĺ天":118363,"å¾Ĺ天çĭ¬":118364,"å¾Ĺ天çĭ¬åİļ":118365,"ä¸Ģå°ģä¿¡":118366,"æĬ½å¥ĸ":118367,"è¾Ĺ转":118368,"çķĻå®Ī":118369,"çķĻå®ĪåĦ¿ç«¥":118370,"çŃĶåį·":118371,"å·¨åŀĭ":118372,"æľĢ好ä¸įè¦ģ":118373,"æµĻæ±Łå¤§åѦ":118374,"æĨ¨":118375,"æı¡æīĭ":118376,"éĴĪç»ĩ":118377,"æİĴ骨":118378,"çĤ½":118379,"å°ģè£ħ":118380,"åįĢåŁŁ":118381,"空æ°ĶåĩĢåĮĸ":118382,"åħīå½±":118383,"åĢĴå¡Į":118384,"å§ļæĺİ":118385,"æ¤į被":118386,"åѦåīį":118387,"åѦåīįæķĻèĤ²":118388,"èĬĿåĬł":118389,"èĬĿåĬłåĵ¥":118390,"缩水":118391,"ä½Ł":118392,"åľ¨çº¿åĴ¨è¯¢":118393,"èµıæŀIJ":118394,"éĿĴèĽĻ":118395,"æĬ±ä½ı":118396,"èĮĤåIJį":118397,"åħ¨åĬĽæīĵéĢł":118398,"åįļ士åѦä½į":118399,"沧å·ŀ":118400,"åĻ¢":118401,"æĿĤçī©":118402,"åĪ»çĶ»":118403,"æįħ":118404,"å¾®éĩı":118405,"å¾®éĩıåħĥç´ł":118406,"ä¸ĢåĽŀäºĭ":118407,"鸡èĤī":118408,"åĪ©æ¶¦çİĩ":118409,"æīįç®Ĺ":118410,"å¾®å¦Ļ":118411,"棵æłij":118412,"贪婪":118413,"åĩıåĢ¼":118414,"梦å¢ĥ":118415,"åı¯è§Ĩ":118416,"åı¯è§ĨåĮĸ":118417,"广大å¸Ĥæ°ij":118418,"ä¸ĵä¸ļä»İäºĭ":118419,"ç»ı纬":118420,"ç´§çĽ¯":118421,"çŁ¥å·±":118422,"è¤ļ":118423,"æĸĩåĮĸåºķèķ´":118424,"åݦéŨå¸Ĥ":118425,"临港":118426,"对åħ¶çľŁå®ŀ":118427,"岸边":118428,"è¦ĸçĤº":118429,"æĬĹçĻĮ":118430,"åĶIJå®ĩ":118431,"ä¸įå¾Ĺè¶ħè¿ĩ":118432,"å¨ģæħij":118433,"æ¡Ĩæŀ¶åįıè®®":118434,"èµ°ç§ģ":118435,"åĽ¢å§Ķ":118436,"夸大":118437,"æ¬Ħ":118438,"ç¥ŀç»ıç³»ç»Ł":118439,"æijĦå½±ä½ľåĵģ":118440,"èĬ¥":118441,"å®īåºĨ":118442,"海滨":118443,"æŀĦæĢĿ":118444,"çīµæĮĤ":118445,"åı©":118446,"éĺIJæĺİ":118447,"éģģ":118448,"精油":118449,"ç©´ä½į":118450,"æĬ¤èº«":118451,"æĬ¤èº«ç¬¦":118452,"æĮĩå°İ":118453,"åŃĺåľ¨ä¸Ģå®ļ":118454,"å¯ĤéĿĻ":118455,"æµ·å¤ĸå¸Ĥåľº":118456,"éĿ¡":118457,"综åIJĪå¾ģ":118458,"ä¿IJ":118459,"è¨Īç®Ĺ":118460,"æĺİæľĹ":118461,"äºļè¿IJ":118462,"äºļè¿IJä¼ļ":118463,"åīįçŀ»æĢ§":118464,"åĮ®ä¹ı":118465,"产ä¸ļæī¶è´«":118466,"èĦijæµ·":118467,"èĦijæµ·ä¸Ń":118468,"åħļçļĦé¢Ĩ导":118469,"åĪĺéĤ¦":118470,"æµģæĺŁ":118471,"æĵĤ":118472,"æĶĢçĻ»":118473,"åĴĶ":118474,"ä¸Ģä¸ĭåŃIJå°±":118475,"è¯Ĭæ²»":118476,"使åĬ²":118477,"åīµä½ľ":118478,"éĵŃè®°":118479,"éĴ±è´¢":118480,"æĹ¥æĬ¥è®°èĢħ":118481,"çĥŁçģ«":118482,"èĥľè´Ł":118483,"åįļ主":118484,"ä¸ŃåĽ½èģĶéĢļ":118485,"ç½ijç«Ļé¦ĸ页":118486,"å°±å¤Ł":118487,"å°±å¤ŁäºĨ":118488,"æīijåħĭ":118489,"å±ħå§Ķä¼ļ":118490,"è°¬":118491,"å®īåħ¨äºĭæķħ":118492,"åķĨçĶ¨è½¦":118493,"循çݯç»ıæµİ":118494,"æ·¤":118495,"èĢĥè¯ģ":118496,"å®ĿèĹı":118497,"å®Įç»ĵ":118498,"çłĶåıijæĬķåħ¥":118499,"å²ij":118500,"æģŃæķ¬":118501,"离éĢĢä¼ij":118502,"水墨":118503,"婶":118504,"è¯Ĺåı¥":118505,"å®ģæ³¢å¸Ĥ":118506,"å¼±çĤ¹":118507,"åģľçīĮ":118508,"奶油":118509,"å¥ĩ纳河":118510,"æĨĤ":118511,"社ä¼ļå®ŀè·µ":118512,"è´Ŀ壳":118513,"çłĤæµĨ":118514,"èĪ¹åıª":118515,"宣æī¬":118516,"综åIJĪæķ´æ²»":118517,"åĤij":118518,"æ°ijæĹıæĸĩåĮĸ":118519,"éĩįçİ°":118520,"积æ·Ģ":118521,"åħ¬çĦ¶":118522,"çħī":118523,"缸èģļ":118524,"æ±¾":118525,"纹çIJĨ":118526,"çĩĥçħ¤":118527,"æѤç§į":118528,"ç¾İå¦Ĩ":118529,"åįĥçĵ¦":118530,"çIJĽ":118531,"驾驶è¯ģ":118532,"éĺ¶æ¢¯":118533,"ä¸Ŀä¸Ŀ":118534,"å¾Īå¤ļäºĭæĥħ":118535,"åħīéĺ´":118536,"èijĹä½ľæ¬Ĭ":118537,"åħ§éĥ¨":118538,"çĽ¸å¯¹æĿ¥è¯´":118539,"éĸĴ":118540,"éľĩæħij":118541,"說話":118542,"æĨij":118543,"ç«¥è£ħ":118544,"ä½ıæĪ¿åĴĮ":118545,"ä½ıæĪ¿åĴĮåŁİ":118546,"å·²ç»ıè¶ħè¿ĩ":118547,"ä¾¦å¯Ł":118548,"çŁ¿çī©":118549,"ä¾Ľå¤§å®¶":118550,"çī¹éĤĢ":118551,"ç¨ĭåºıåijĺ":118552,"çķľçī§ä¸ļ":118553,"æ°ª":118554,"çijª":118555,"åĢĴåľ¨":118556,"åĢĴåľ¨åľ°":118557,"æ¯Ģ":118558,"梯éĺŁ":118559,"æİ¥èijĹ":118560,"æĬĹèıĮ":118561,"è¤ĩ":118562,"ç¬Ļ":118563,"æ¯Ķä¸Ĭå¹´":118564,"鸡汤":118565,"åŃ¦ä¹łæĪIJ绩":118566,"æĸijæĸĵ":118567,"åħĪ导":118568,"åĪĹ举":118569,"è°ĥæŁ¥æĺ¾ç¤º":118570,"æ©«":118571,"ä¹Ŀåįģ":118572,"è°¢éŁµ":118573,"è·¨è¶Ĭå¼ı":118574,"女æĢ§æľĭåıĭ":118575,"èIJ¥åħ»ä»·åĢ¼":118576,"å®ŀè·µç»ıéªĮ":118577,"èĭıå·ŀå¸Ĥ":118578,"çĵ¶åŃIJ":118579,"æĸ°çļĦä¸Ģ":118580,"æĸ°çļĦä¸Ģå¹´":118581,"æĺİæĻ°":118582,"å®łçĪ±":118583,"åŃĹ第":118584,"æľĹ诵":118585,"纳æĸ¯":118586,"éĢĨè¡Į":118587,"è«ĭæĤ¨":118588,"è«ĭæĤ¨æıIJä¾Ľ":118589,"èĥ¸æĢĢ":118590,"第ä¸ĥå±Ĭ":118591,"强壮":118592,"代åŃķ":118593,"汶å·Ŀ":118594,"家åĸ»":118595,"家åĸ»æĪ·":118596,"家åĸ»æĪ·æĻĵ":118597,"èħ®":118598,"åIJ¯è¿ª":118599,"æĹłéļľç¢į":118600,"èĻķçIJĨåıĬ":118601,"æĿ¥åİĨ":118602,"å®ŀåĬ¡":118603,"ä¹Łéļıä¹ĭ":118604,"æĬĢèĥ½åŁ¹è®Ń":118605,"åѤç«ĭ":118606,"åīģ":118607,"éĥ´å·ŀ":118608,"æĶ¶æķĽ":118609,"éł»éģĵ":118610,"èį£å¹¸":118611,"èİ«è¿ĩäºİ":118612,"æѤæĻĤ":118613,"纪å§ĶçĽij":118614,"纪å§ĶçĽijå§Ķ":118615,"缸éĤ»":118616,"åı¦ä¸Ģè¾¹":118617,"çªĴæģ¯":118618,"æľīå¾Īå¤ļç§į":118619,"æ¯ıéĢ¢":118620,"éĹ®ä¸ĸ":118621,"累累":118622,"éĿĴæĺ¥æľŁ":118623,"è·¯åĨµ":118624,"åħĭèݱ":118625,"è¿Ħä»Ĭ为æŃ¢":118626,"æĥĬå¥ĩ":118627,"跨度":118628,"éħ¿éĢł":118629,"åĩĭ":118630,"è¿ijä¸īå¹´":118631,"åĨħ马":118632,"åĨħ马å°Ķ":118633,"æıį":118634,"è¿Ľå±ķæĥħåĨµ":118635,"èĮ§":118636,"æľīåºıæİ¨è¿Ľ":118637,"æĢ»åĨłåĨĽ":118638,"æĪIJ绩åįķ":118639,"éĽ»è©±åıĬ":118640,"ç´§å¯Ĩç»ĵåIJĪ":118641,"åºĬä½į":118642,"é¹Ĭ":118643,"æķ£åıijçĿĢ":118644,"åĭŁèµĦ":118645,"æ°¨éħ¸":118646,"彩ç¥ŀ":118647,"è®Ģåıĸ":118648,"éĩį温":118649,"ä¸ŃåŃĺåľ¨çļĦ":118650,"ç¾İéºĹ":118651,"ä¸įæĸŃå¢ŀåĬł":118652,"è½®æµģ":118653,"æİ¥åIJ¬":118654,"年产åĢ¼":118655,"åįĥåħĭ":118656,"æĪĺåľºä¸Ĭ":118657,"çħ§é¡§":118658,"å¹²éĥ¨éĺŁä¼į":118659,"åį°ç«ł":118660,"ä¸Ģèĩ´æĢ§":118661,"è¿ŀå¤ľ":118662,"åħħè£ķ":118663,"é»ijåIJįåįķ":118664,"åĩĢæ°´":118665,"ä¸Ģ大æĹ©":118666,"åĮħ袱":118667,"çĬ¯è§Ħ":118668,"çIJĨè«ĸ":118669,"æŀģæĺĵ":118670,"骸":118671,"å¨ĺå¨ĺ":118672,"åĽ¢åľĨ":118673,"亿åħĥ以ä¸Ĭ":118674,"åĪ©çĶ¨æĤ¨çļĦ":118675,"带æĿ¥æĽ´å¤ļ":118676,"ä¸Ń央空è°ĥ":118677,"æľĪèĸª":118678,"çĮľæĥ³":118679,"åĪºå®¢":118680,"ä½ľæģ¯":118681,"åįķè°ĥ":118682,"äºĴåĪ©":118683,"å¦Ĥæľīä¾µæĿĥ":118684,"å°ıå·§":118685,"åįģåł°":118686,"åĵĪåĵĪåĵĪåĵĪ":118687,"è¾¹éĻħ":118688,"æłĩè¯Ń":118689,"åĪĩåħ¥çĤ¹":118690,"éĢĨè¢Ń":118691,"è¯ķåīĤ":118692,"绿è±Ĩ":118693,"è®ļ":118694,"åŁºçĿ£å¾Ĵ":118695,"壬":118696,"åħ¨æĺİæĺŁ":118697,"éĢīç§Ģ":118698,"èĪĮå°ĸ":118699,"ä¸įåIJĮç±»åŀĭ":118700,"çĥŁåĽ±":118701,"çģµæ°Ķ":118702,"åĮºç®¡å§Ķä¼ļ":118703,"åĨľåī¯":118704,"åĨľåī¯äº§åĵģ":118705,"èĶļæĿ¥":118706,"沪æĮĩ":118707,"åħ»æ®ĸæĪ·":118708,"æĸĹå¿Ĺ":118709,"é¦ĸé¢Ĩ":118710,"è¡Ģèħ¥":118711,"åĬłç´§":118712,"ä¸Ģèĩ´å¥½è¯Ħ":118713,"第ä¸īèĬĤ":118714,"æī¬å°ĺ":118715,"交éĢļæŀ¢çº½":118716,"鼶ç¢İ":118717,"é»ijæ´ŀ":118718,"çľĭä¸įæĩĤ":118719,"å±ŀå®ŀ":118720,"主åŁİåĮº":118721,"å¨Ľ":118722,"å¨Ľæ¨Ĥ":118723,"ç¬ijæĦı":118724,"èĻ¹æ¡¥":118725,"åIJĦ个çݯèĬĤ":118726,"çķ¥å¾®":118727,"èĢķèĢĺ":118728,"æľ¬åľºæ¯ĶèµĽ":118729,"æĪIJè´¥":118730,"éĢīèĤ¡":118731,"èªŀè¨Ģ":118732,"çŃĶ辩":118733,"èĩªä¹ł":118734,"棺":118735,"ä¸ĩ欧åħĥ":118736,"åģľå·¥":118737,"对åħ¶è¿Ľè¡Į":118738,"积æŀģéħįåIJĪ":118739,"ä¹¾åĿ¤":118740,"å¦ĸæĢª":118741,"èļĮåŁł":118742,"èµĦ产è¯Ħä¼°":118743,"è°ĥçļ®":118744,"éĻ¤å¤ķ":118745,"åĽ´å¢Ļ":118746,"æľįå½¹":118747,"æ·±æ¸Ĭ":118748,"é¢ĦåĪ¶":118749,"çĥ½":118750,"å®ī稳":118751,"建æŀĦ":118752,"çĭĻåĩ»":118753,"主åĭķ註åĨĬ":118754,"éĥ½æľīèĩªå·±":118755,"æİĴåIJį第ä¸Ģ":118756,"麻辣":118757,"çĢļ":118758,"çĥŁèĬ±çĪĨ":118759,"çĥŁèĬ±çĪĨ竹":118760,"èĩªçĦ¶ä¿ĿæĬ¤":118761,"ä»Ļå¢ĥ":118762,"为äºĨéģ¿åħį":118763,"åĨ·åºĵ":118764,"解æĶ¾æĢĿæĥ³":118765,"åĪĿäºĮ":118766,"ä½ĵè´´":118767,"é¦ĸå¯Į":118768,"迪æĭľ":118769,"æļĤç¼ĵ":118770,"æĶ¯æĮģåĬĽåº¦":118771,"侦æİ¢":118772,"马åĪº":118773,"åĮĹæ±½":118774,"ç¹ŀ":118775,"è°İè¨Ģ":118776,"éĢ£çºĮ":118777,"å·³":118778,"ä»»ä½ķæŶåĢĻ":118779,"车èģĶç½ij":118780,"åįķ项":118781,"å¸Ńåį·":118782,"建çŃijæĿIJæĸĻ":118783,"ä¸Ńç§ĭèĬĤ":118784,"ç¡ķ士çłĶ究":118785,"ç§ģç«ĭ":118786,"åħļåĴĮæĶ¿åºľ":118787,"æľ¬æ¬¡äº¤æĺĵ":118788,"èººåľ¨åºĬä¸Ĭ":118789,"ç½ijåıĭè¯Ħ论":118790,"å¦Ŀ":118791,"害ç¾ŀ":118792,"åħ¬ç«ĭåĮ»éĻ¢":118793,"ä¸ŀ":118794,"çĶŁçī©è´¨":118795,"åºĶéĤĢ":118796,"æĬ½åıĸ":118797,"åĩłå¼ł":118798,"æijĺç¼ĸ":118799,"ç»ĺæľ¬":118800,"详解":118801,"强硬":118802,"æľĢåħĪè¿ĽçļĦ":118803,"æĭĽèĤ¡":118804,"æĭĽèĤ¡ä¹¦":118805,"åįĥæĸ¹":118806,"åįĥæĸ¹çĻ¾":118807,"åįĥæĸ¹çĻ¾è®¡":118808,"éħįéŁ³":118809,"驾çħ§":118810,"å¾ģæĪĺ":118811,"èªĵè¨Ģ":118812,"æĭľå¸Ī":118813,"æĭľå¸ĪåѦ":118814,"æĭľå¸ĪåѦèīº":118815,"æĬ±åĽ¢":118816,"ç±³ç²ī":118817,"éĿŀ常éĢĤåIJĪ":118818,"èĪªæµ·":118819,"履约":118820,"åįģåħ«æĿ¡":118821,"éĶ»éĢł":118822,"éĩįè¦ģ举æݪ":118823,"åıijæĮ¥ä½ľçĶ¨":118824,"æ·ļ":118825,"人社":118826,"人社å±Ģ":118827,"è¯ķçĤ¹å·¥ä½ľ":118828,"éĺľéĺ³":118829,"æ¡ĥåľĴ":118830,"æ°ijä¼ģ":118831,"æ´ģçĻ½":118832,"贵宾":118833,"åħ¬ç¤¾":118834,"è§īæĤŁ":118835,"è®°å¿ĨåĬĽ":118836,"æľĥåĵ¡è¨»åĨĬ":118837,"æѤæ¡Ī":118838,"麻çĹ¹":118839,"çıĢ":118840,"æĸ©èİ·":118841,"çĶ·åŃ©åŃIJ":118842,"å±ĢéĻIJäºİ":118843,"åĭĺæŁ¥":118844,"åIJĥ饱":118845,"èĬ¬åħ°":118846,"æ£ķèī²":118847,"ç¦ıç¥ī":118848,"çĶ³èĬ±":118849,"æµ·çĽĹ":118850,"èĶij":118851,"æĸĩåѸ":118852,"æ´»æĢ§çĤŃ":118853,"缴éĢļ车":118854,"è°¢éĤĢ":118855,"躺çĿĢ":118856,"åľĥ":118857,"æ¯ıæĹ¥ç»ıæµİ":118858,"åħ¬åħ±æĸĩåĮĸ":118859,"讲æķħäºĭ":118860,"å¯Łçľĭ":118861,"æĤłéĹ²":118862,"åľ°åĿª":118863,"æ¶Įçİ°åĩº":118864,"é«ĺçŃīéĻ¢æł¡":118865,"èĮĦåŃIJ":118866,"éĺ²åį«":118867,"ä¾ĭè¡Į":118868,"æĺ¾éľ²":118869,"æĸ°å¸¸æĢģ":118870,"ç»Ŀä½³":118871,"å¯Įæ°ij":118872,"以人æ°ij":118873,"以人æ°ij为":118874,"éĤ¢åı°":118875,"å±ķæ¼Ķ":118876,"çĻ¼å¸ĥ":118877,"è´Łè½½":118878,"åģı离":118879,"æ°¸éģł":118880,"éĩįè¦ģåİŁåĽł":118881,"åįıä¼ļä¼ļåijĺ":118882,"éļ¾æ°ij":118883,"çĶŁäº§è½¦éĹ´":118884,"çģµåĬ¨":118885,"两年åīį":118886,"æĸ¹åľĨ":118887,"æ´»ä¸ĭåİ»":118888,"ä¸ĸçķĮè§Ĥ":118889,"éªĹåıĸ":118890,"ç¾İè²Į":118891,"èĥ½çľĭåĩº":118892,"çĻ¼æı®":118893,"è§Ĥå½±":118894,"åīĥ":118895,"åIJĪèµĦåħ¬åı¸":118896,"婧":118897,"å¹²æű":118898,"åħŃ个æľĪ":118899,"尤为éĩįè¦ģ":118900,"èĤ½":118901,"ç§¦åĽ½":118902,"æīĺç¦ı":118903,"建çŃijå¸Ī":118904,"åįĩ级æĶ¹éĢł":118905,"å°ıé¢Ŀ":118906,"å°ıé¢Ŀ贷款":118907,"两个维æĬ¤":118908,"æĭįæĭį":118909,"åı¯çĸij":118910,"æį¢åıĸ":118911,"æŃ¦å£«":118912,"èµĸ以":118913,"èµĸ以çĶŁåŃĺ":118914,"æĮļ":118915,"殿åłĤ":118916,"èĩªçĦ¶çķĮ":118917,"ç£ģåľº":118918,"å¦Ĥä½ķçľĭå¾ħ":118919,"ä»ĬæĹ¥å¤´æĿ¡":118920,"è¥¿åŁŁ":118921,"èİ·è¯Ħ":118922,"é¢¨æł¼":118923,"ä¿ĦåĽ½":118924,"æīĵæĭ¼":118925,"å®£ä¼łçīĩ":118926,"å¾Īæĸ¹ä¾¿":118927,"ä¾Ľç»Ļ侧":118928,"纪念ç¢ij":118929,"毫åħĭ":118930,"èĬ³é¦Ļ":118931,"å·¥åķĨéĵ¶è¡Į":118932,"请çĤ¹åĩ»":118933,"缪":118934,"æĹłæķ°æ¬¡":118935,"èį¯å¸Ī":118936,"èħ¸":118937,"游èīĩ":118938,"åĮ¾":118939,"å·¡èĪª":118940,"æ²»çIJĨä½ĵç³»":118941,"èIJ¥éĢłèī¯å¥½":118942,"æ··æ·Ĩ":118943,"éĢļçķħ":118944,"åĬ³ç´¯":118945,"ä»ĵä½į":118946,"å¢ŀéķ·":118947,"éļIJ约":118948,"æĿĤå¿Ĺ社":118949,"åħ»èĤ²":118950,"åı¯èĥ½åıijçĶŁ":118951,"èĢĥ試":118952,"西侧":118953,"åĬłåĢį":118954,"主æĮģåı¬å¼Ģ":118955,"çķ¢ç«Ł":118956,"éĹ®è¯¢":118957,"æµ·æ£ł":118958,"èĹ©":118959,"注æĺİæĿ¥æºIJ":118960,"æ£Ģçĸ«":118961,"请åģĩ":118962,"æĬļæij¸":118963,"èĵĦçĶµæ±ł":118964,"è·Łä¸įä¸Ĭ":118965,"çݰ代社ä¼ļ":118966,"çŃ¹èµĦ":118967,"ä½ĵèĤ²å½©ç¥¨":118968,"延误":118969,"è¾Ľè¾£":118970,"éĿ¢å®¹":118971,"åį°è®°":118972,"çģŃ亡":118973,"ç´łé£Ł":118974,"åħ´èĩ´":118975,"éľĢè¦ģçĶ¨":118976,"éľĢè¦ģçĶ¨åĪ°":118977,"å®Ŀå¦Ī":118978,"ç£ĭåķĨ":118979,"éļ¶å±ŀ":118980,"è´¡çĮ®åĬĽéĩı":118981,"åħ¬åħ±èµĦæºIJ":118982,"大éĺª":118983,"åĨĽè®Ń":118984,"æĤ¬å¿µ":118985,"社ä¼ļ稳å®ļ":118986,"å¹²äºĭåĪĽä¸ļ":118987,"æľīæĿ¡ä»¶":118988,"æľīæĿ¡ä»¶çļĦ":118989,"ä¸Ģå¹´ä¸Ģ度":118990,"åİ¥":118991,"强奸":118992,"豪车":118993,"æİĮæŁľ":118994,"æ°´åĪ©å·¥ç¨ĭ":118995,"峪":118996,"积æŀģä½ľçĶ¨":118997,"æµ·æ·Ģ":118998,"æµ·æ·ĢåĮº":118999,"çĥŃæĴŃ":119000,"åĿļæĮģä¸įæĩĪ":119001,"åıĮèĦļ":119002,"绣æĪĺ":119003,"ä»»ä½ķ人éĥ½":119004,"åľ°ä¸ĭ室":119005,"åĨ¶çĤ¼":119006,"è°ħ解":119007,"æ¸ĶèĪ¹":119008,"太éĺ³åŁİ":119009,"被æįķ":119010,"计ç®ĹåĻ¨":119011,"西åĮ»":119012,"èĪĴå¿ĥ":119013,"桦":119014,"éģ²":119015,"åĬij":119016,"è¨Ĺ":119017,"èݺ":119018,"åĸ¬":119019,"çĵ¯":119020,"åĺĺ":119021,"åłķ":119022,"æķĿ":119023,"åij¦":119024,"èĭŀ":119025,"æŃ¹":119026,"æĵ¬":119027,"æ£Ħ":119028,"èĪµ":119029,"奪":119030,"çļĭ":119031,"æĶ¸":119032,"åľ©":119033,"ç¤Ļ":119034,"ç¢ĺ":119035,"éıĪ":119036,"æĦķ":119037,"ç¹³":119038,"èĺ¸":119039,"è²Ĥ":119040,"æ¼²":119041,"æij¹":119042,"æĶĿ":119043,"åŃ¢":119044,"èķŃ":119045,"騰":119046,"æ½¼":119047,"éħ°":119048,"æĴ¥":119049,"蹬":119050,"é¨Ļ":119051,"踹":119052,"éģIJ":119053,"çĺĢ":119054,"èĽ¤":119055,"æĤĸ":119056,"çĴŀ":119057,"ç£IJ":119058,"æİ°":119059,"è¾Ĭ":119060,"å¾ij":119061,"æİĸ":119062,"éģŀ":119063,"éĤ¸":119064,"éĽı":119065,"æĨİ":119066,"æľ½":119067,"çį»":119068,"ç®Ķ":119069,"褶":119070,"æļ¢":119071,"æĺµ":119072,"çıĤ":119073,"æĤ¸":119074,"åģµ":119075,"åĻľ":119076,"壯":119077,"æĴ®":119078,"æģį":119079,"å©ķ":119080,"篱":119081,"éĺĻ":119082,"çīł":119083,"è£ĺ":119084,"è³¢":119085,"éĩľ":119086,"éĵł":119087,"èİĺ":119088,"æ®Ĩ":119089,"çĻ¸":119090,"è´ı":119091,"ç²±":119092,"å«¡":119093,"åĨ¢":119094,"è¤Ĵ":119095,"æĩĬ":119096,"éľĵ":119097,"塵":119098,"æĭ£":119099,"å»Ł":119100,"飽":119101,"é¢Į":119102,"åļİ":119103,"æ·º":119104,"èĨł":119105,"åİŃ":119106,"åļĩ":119107,"åijĥ":119108,"çĴĭ":119109,"çѱ":119110,"æĭ·":119111,"èį§":119112,"éĶ°":119113,"åŃ°":119114,"èĵĵ":119115,"èĨ½":119116,"æŀī":119117,"åĸ½":119118,"çĽĶ":119119,"çŃIJ":119120,"ç¾ļ":119121,"èħĮ":119122,"辫":119123,"æ³ĵ":119124,"çĶ¬":119125,"èŁ²":119126,"åĸª":119127,"å¦ĵ":119128,"è¬Ģ":119129,"çĤĬ":119130,"æĽľ":119131,"æ±IJ":119132,"è´Ī":119133,"èįĢ":119134,"æĬł":119135,"碾":119136,"æ«ĥ":119137,"éŀł":119138,"èijĨ":119139,"祯":119140,"å½Ŀ":119141,"é¦į":119142,"åĮ£":119143,"æľŃ":119144,"åĿĤ":119145,"ä¿ij":119146,"èĵ®":119147,"çijĽ":119148,"æīī":119149,"èĩŁ":119150,"貫":119151,"çİ¥":119152,"æ·¼":119153,"åݲ":119154,"é³Į":119155,"å³Ń":119156,"åijĽ":119157,"é§":119158,"é§IJ":119159,"éģ·":119160,"俪":119161,"æĢĤ":119162,"è¾į":119163,"å±į":119164,"åĭģ":119165,"å¥ļ":119166,"éļħ":119167,"éĴ´":119168,"è¼Ŀ":119169,"宦":119170,"èIJĥ":119171,"çĺĭ":119172,"æĨ¶":119173,"æĤħ":119174,"è¾Ļ":119175,"åijľ":119176,"çłº":119177,"éĢŀ":119178,"æµļ":119179,"éĸ£":119180,"èĸ©":119181,"éĻĭ":119182,"çĤĻ":119183,"èªķ":119184,"丣":119185,"é¹½":119186,"ç±Į":119187,"è´°":119188,"éĭª":119189,"çľ©":119190,"æĴIJ":119191,"èĨº":119192,"éŀĺ":119193,"ç¾²":119194,"窮":119195,"ç´IJ":119196,"æ®´":119197,"纾":119198,"èºį":119199,"ç´ĭ":119200,"çĦĸ":119201,"çĶº":119202,"çī½":119203,"çĤ¯":119204,"ç¼Ķ":119205,"æ¯ĵ":119206,"嬰":119207,"梧":119208,"äºŁ":119209,"è¢ħ":119210,"çįĦ":119211,"è¿¥":119212,"æ¼¾":119213,"çĿij":119214,"績":119215,"é¦ĭ":119216,"é¤ħ":119217,"æ¹Ħ":119218,"æĺĩ":119219,"æŀŃ":119220,"èĸ°":119221,"æŁij":119222,"榻":119223,"åĻĹ":119224,"åĻ´":119225,"棣":119226,"åĶ§":119227,"çĨ¹":119228,"輯":119229,"å¢Ł":119230,"é²²":119231,"æĪĽ":119232,"èī¦":119233,"èĬ®":119234,"åĺŁ":119235,"帥":119236,"å¿»":119237,"çĮĿ":119238,"寵":119239,"賦":119240,"èĽ¾":119241,"滾":119242,"çĤķ":119243,"éĵ¬":119244,"èĴ¿":119245,"éĴ¨":119246,"çĥĻ":119247,"ç²ķ":119248,"æĥ¦":119249,"溧":119250,"é¢į":119251,"éħ£":119252,"峦":119253,"ç±ģ":119254,"çĥĥ":119255,"åĨĹ":119256,"åıģ":119257,"缧":119258,"ç½µ":119259,"éĴĹ":119260,"å¬ī":119261,"è°ı":119262,"糧":119263,"è¾Ń":119264,"æ·¬":119265,"èŁĴ":119266,"诩":119267,"è¦ĥ":119268,"çĻĸ":119269,"é½Ĵ":119270,"çĪIJ":119271,"ç®į":119272,"ç¼İ":119273,"磺":119274,"诫":119275,"褲":119276,"æĵł":119277,"èIJ¦":119278,"çĿ¬":119279,"è°į":119280,"éĦ°":119281,"æł¾":119282,"é¡ı":119283,"縱":119284,"桨":119285,"éĨ¬":119286,"襲":119287,"讪":119288,"婺":119289,"èįŁ":119290,"åĮĿ":119291,"çĨł":119292,"èĽĬ":119293,"æ¸ļ":119294,"å´½":119295,"鲤":119296,"åķ°":119297,"åĮķ":119298,"ä¸IJ":119299,"讥":119300,"åı½":119301,"åı¼":119302,"çļ¿":119303,"è¿Ĥ":119304,"åIJĨ":119305,"å±¹":119306,"èĩ¼":119307,"讹":119308,"é©®":119309,"纫":119310,"æ±ŀ":119311,"æĬ¡":119312,"èĭĩ":119313,"åIJł":119314,"åIJŃ":119315,"åIJ®":119316,"å²ĸ":119317,"ä½ĥ":119318,"çĭĪ":119319,"åºĩ":119320,"åIJĿ":119321,"éĹ°":119322,"æ±¹":119323,"忱":119324,"æĭĦ":119325,"æĭĹ":119326,"èĮī":119327,"èĭĽ":119328,"èĮģ":119329,"çŁ¾":119330,"èĻı":119331,"åij»":119332,"åĴĦ":119333,"å¿¿":119334,"èĤ®":119335,"çĭŀ":119336,"çĸŁ":119337,"çĸĻ":119338,"çĸļ":119339,"æ³ŀ":119340,"å¸ļ":119341,"å±ī":119342,"è¿¢":119343,"驹":119344,"çİ·":119345,"çıĬó":119346,"çıĬół":119347,"çıĬółĦ":119348,"çıĬółĦģ":119349,"æĮİ":119350,"æĭ´":119351,"åŀĽ":119352,"èį¤":119353,"æ®ĥ":119354,"çĽ¹":119355,"åĵĨ":119356,"è´»":119357,"毡":119358,"çĭ°":119359,"çĭ¡":119360,"æŁĴ":119361,"æģĥ":119362,"诬":119363,"è¢Ħ":119364,"诲":119365,"èļ¤":119366,"èĢĻ":119367,"åŁĤ":119368,"æįİ":119369,"æįĮ":119370,"æ¢Ĩ":119371,"éħĮ":119372,"çł¾":119373,"æ®ī":119374,"åĶł":119375,"æĻĮ":119376,"èļ£":119377,"èļª":119378,"èļĵ":119379,"鸯":119380,"åĶģ":119381,"åĶĨ":119382,"åĢĶ":119383,"èĪĢ":119384,"豺":119385,"èĥ°":119386,"鸵":119387,"鸳":119388,"é¦ģ":119389,"ç¾Ķ":119390,"涣":119391,"æ¶ķ":119392,"æĤ¯":119393,"诽":119394,"è°Ĩ":119395,"ç¥Ł":119396,"绢":119397,"æįº":119398,"æį¶":119399,"æį»":119400,"æİĤ":119401,"èıł":119402,"èIJ¤":119403,"éħĹ":119404,"羶":119405,"åķĦ":119406,"èļ¯":119407,"èĽĢ":119408,"åĶ¬":119409,"帷":119410,"éĵIJ":119411,"éĵĽ":119412,"åģİ":119413,"å¾Ļ":119414,"èĦ¯":119415,"è±ļ":119416,"çĮĸ":119417,"çĹĬ":119418,"涮":119419,"æĥŃ":119420,"æĤ´":119421,"æĥĭ":119422,"è°ļ":119423,"æı©":119424,"æIJĢ":119425,"æIJĶ":119426,"æ¦Ķ":119427,"æ¤Ń":119428,"éĽ³":119429,"åĸ³":119430,"è·Ľ":119431,"èľĵ":119432,"èľĴ":119433,"é¹ĥ":119434,"éĶĦ":119435,"çĶ¥":119436,"çŃı":119437,"çĮ©":119438,"çĮ¬":119439,"çĮ¾":119440,"çĹ¢":119441,"çĹª":119442,"æĥ°":119443,"çªĺ":119444,"è°¤":119445,"éļĺ":119446,"å©¿":119447,"é¹ī":119448,"çijĻ":119449,"æĸŁ":119450,"椿":119451,"éħª":119452,"éĽ¹":119453,"åŦ":119454,"è··":119455,"è·º":119456,"è·¤":119457,"èľĪ":119458,"èľĹ":119459,"å¹Į":119460,"é¦ı":119461,"èªĬ":119462,"æ¼ĵ":119463,"è¤Ĥ":119464,"èĶĹ":119465,"èĶ¼":119466,"åħ¢":119467,"裳":119468,"èľ»":119469,"èĿĩ":119470,"åĺĢ":119471,"éĶ¹":119472,"ç®ķ":119473,"箩":119474,"çĺ©":119475,"çĺŁ":119476,"æ¼±":119477,"寥":119478,"骡":119479,"æĴµ":119480,"æĴ¬":119481,"è±Į":119482,"åĺ¹":119483,"èĿł":119484,"èĿĮ":119485,"èĿĹ":119486,"èĿĻ":119487,"éķIJ":119488,"稼":119489,"ç¯ĵ":119490,"èĨĽ":119491,"鲫":119492,"çĺª":119493,"鲨":119494,"æĨĶ":119495,"ç¿©":119496,"褥":119497,"ç¼Ń":119498,"åĻ©":119499,"çĵ¢":119500,"éľİ":119501,"踱":119502,"è¹Ĥ":119503,"èŁĨ":119504,"鹦":119505,"篡":119506,"çĺ¸":119507,"窿":119508,"ç¼°":119509,"èĹIJ":119510,"è¹ĭ":119511,"èŁĭ":119512,"èŁĢ":119513,"赡":119514,"èĩĬ":119515,"é³Ħ":119516,"ç³ł":119517,"æĩ¦":119518,"åļ£":119519,"éķ°":119520,"é³į":119521,"ç°¸":119522,"çĻ£":119523,"é³ĸ":119524,"é¬ĵ":119525,"èłķ":119526,"éľ¹":119527,"èºı":119528,"黯":119529,"çĵ¤":119530,"çŁĹ":119531,"ä¹Ĥ":119532,"ä¹ľ":119533,"åħĢ":119534,"å¼ĭ":119535,"åŃij":119536,"åŃĵ":119537,"幺":119538,"äºĵ":119539,"廿":119540,"ä¸ı":119541,"åįħ":119542,"ä»ĥ":119543,"ä»ī":119544,"ä»Ĥ":119545,"åĪĪ":119546,"çĪ»":119547,"åįŀ":119548,"éĹ©":119549,"讣":119550,"夬":119551,"çĪ¿":119552,"æ¯ĭ":119553,"éĤĹ":119554,"éĤĽ":119555,"èī½":119556,"èī¿":119557,"åıµ":119558,"ä¸ķ":119559,"åĮľ":119560,"åĬ¢":119561,"åįŁ":119562,"åı±":119563,"åı»":119564,"仨":119565,"代":119566,"仡":119567,"仫":119568,"ä»ŀ":119569,"åį®":119570,"æ°IJ":119571,"çĬ°":119572,"åĪį":119573,"éĤĿ":119574,"éĤĻ":119575,"讦":119576,"讧":119577,"讫":119578,"å°»":119579,"éĺ¡":119580,"å°ķ":119581,"å¼ģ":119582,"èĢĴ":119583,"çİİ":119584,"çİij":119585,"åľ¬":119586,"æī¦":119587,"åľª":119588,"åľ¹":119589,"æīª":119590,"åľ®":119591,"åľ¯":119592,"èĬĬ":119593,"èĬį":119594,"èĬĦ":119595,"èĬ¨":119596,"èĬij":119597,"èĬİ":119598,"èĬĹ":119599,"äºĺ":119600,"åİį":119601,"夼":119602,"æĪį":119603,"å°¥":119604,"乩":119605,"æĹ¯":119606,"æĽ³":119607,"å²Į":119608,"屺":119609,"åĩ¼":119610,"åĽ¡":119611,"éĴĩ":119612,"缶":119613,"æ°ĺ":119614,"æ°ĸ":119615,"çīĿ":119616,"ä¼İ":119617,"ä¼Ľ":119618,"ä¼¢":119619,"佤":119620,"仵":119621,"ä¼¥":119622,"伧":119623,"ä¼ī":119624,"伫":119625,"åĽŁ":119626,"æ±Ĩ":119627,"åĪĸ":119628,"å¤Ļ":119629,"æĹ®":119630,"åĪİ":119631,"çĬ·":119632,"çĬ¸":119633,"èĪĽ":119634,"åĩ«":119635,"éĤ¬":119636,"饧":119637,"æ±Ķ":119638,"æ±ľ":119639,"æ±Ĭ":119640,"å¿ĸ":119641,"å¿ı":119642,"è®´":119643,"讵":119644,"è®·":119645,"èģ¿":119646,"èī®":119647,"åݾ":119648,"å¦ģ":119649,"纡":119650,"纣":119651,"纥":119652,"纨":119653,"çİķ":119654,"çİĻ":119655,"æĬŁ":119656,"æĬĶ":119657,"åľ»":119658,"åĿį":119659,"æĬĥ":119660,"ã§IJ":119661,"èĬ«":119662,"èĬ¾":119663,"èĭĪ":119664,"èĭ£":119665,"èĭĭ":119666,"èĬ¼":119667,"èĭĮ":119668,"èĭģ":119669,"èĬ©":119670,"èĬª":119671,"èĬ¡":119672,"èĬŁ":119673,"èĭĦ":119674,"èĭİ":119675,"èĭ¡":119676,"æĿĮ":119677,"æĿĵ":119678,"æĿĪ":119679,"å¿ij":119680,"åŃĽ":119681,"éĤ´":119682,"éĤ³":119683,"å¥ģ":119684,"è±ķ":119685,"å¿Ĵ":119686,"欤":119687,"轫":119688,"è¿ĵ":119689,"éĤ¶":119690,"å¿IJ":119691,"åį£":119692,"éĤº":119693,"æĹ°":119694,"åijĭ":119695,"åijĴ":119696,"åijĵ":119697,"åijĶ":119698,"åijĸ":119699,"æŸ":119700,"åIJ¡":119701,"èĻ¬":119702,"åIJ½":119703,"åIJ£":119704,"åIJ²":119705,"å¸ı":119706,"å²Ī":119707,"å²ĺ":119708,"åħķ":119709,"åĽµ":119710,"åĽ«":119711,"éĴĬ":119712,"éĴĭ":119713,"éĴĮ":119714,"è¿ķ":119715,"æ°Ļ":119716,"æ°ļ":119717,"çī¤":119718,"ä½ŀ":119719,"ä½ļ":119720,"ä½Ŀ":119721,"ä½Ĺ":119722,"å½·":119723,"ä½ĺ":119724,"ä½¥":119725,"豸":119726,"åĿĮ":119727,"èĤŁ":119728,"å¥Ĥ":119729,"åĬ¬":119730,"çĭģ":119731,"鸳":119732,"饨":119733,"饩":119734,"饫":119735,"饬":119736,"åºij":119737,"åºĭ":119738,"çĸĶ":119739,"çĸĸ":119740,"èĤĵ":119741,"éű":119742,"éĹ³":119743,"çĤĢ":119744,"æ²£":119745,"æ²ħ":119746,"æ²Ķ":119747,"沤":119748,"æ²ı":119749,"æ²ļ":119750,"汩":119751,"汨":119752,"沨":119753,"æ±´":119754,"æ²Ĩ":119755,"沩":119756,"æ³IJ":119757,"æĢĥ":119758,"æĢĦ":119759,"å¿¡":119760,"忤":119761,"忾":119762,"æĢħ":119763,"忪":119764,"æĢĨ":119765,"å¿Ń":119766,"忸":119767,"è¯Ĥ":119768,"è¯ĥ":119769,"è¯ħ":119770,"è¯ĭ":119771,"è¯Į":119772,"è¯Ĵ":119773,"éĻĤ":119774,"éĻī":119775,"妩":119776,"妪":119777,"妣":119778,"å¦Ĺ":119779,"妫":119780,"å§Ĵ":119781,"妤":119782,"åĬŃ":119783,"åĪŃ":119784,"éĤ°":119785,"çºŃ":119786,"纰":119787,"纴":119788,"çİ¡":119789,"çİŃ":119790,"çİł":119791,"çİ¢":119792,"çݦ":119793,"çĽĤ":119794,"å¿Ŀ":119795,"åĮ¦":119796,"åĿ©":119797,"æĬ¨":119798,"æĭ¤":119799,"åĿ«":119800,"æĭĪ":119801,"åŀĨ":119802,"æĬ»":119803,"åĬ¼":119804,"æĭĥ":119805,"æĭĬ":119806,"åĿ¼":119807,"åĿ»":119808,"ã§Ł":119809,"åĿ¨":119810,"åĿŃ":119811,"æĬ¿":119812,"åĿ³":119813,"èĭ·":119814,"èĭ¤":119815,"èĮı":119816,"èĭ«":119817,"èĭľ":119818,"èĭ´":119819,"èĭĴ":119820,"èĭĺ":119821,"èĮĮ":119822,"èĭ»":119823,"èĭĵ":119824,"èĮļ":119825,"èĮĨ":119826,"èĮij":119827,"èĮĵ":119828,"èĮĶ":119829,"èĮķ":119830,"èĮĢ":119831,"èĭķ":119832,"æŀ¥":119833,"æŀĩ":119834,"æĿª":119835,"æĿ³":119836,"æŀ§":119837,"æĿµ":119838,"æŀ¨":119839,"æŀŀ":119840,"æŀĭ":119841,"æĿ»":119842,"æĿ·":119843,"æĿ¼":119844,"磸":119845,"çłĢ":119846,"åĪ³":119847,"å¥Ħ":119848,"æ®ģ":119849,"éĥı":119850,"è½Ń":119851,"éĥħ":119852,"鸢":119853,"缱":119854,"æĺĻ":119855,"æĿ²":119856,"æĺĥ":119857,"åĴĤ":119858,"åij¸":119859,"æĺĢ":119860,"æĹ»":119861,"æĺī":119862,"çĤħ":119863,"çķĢ":119864,"èĻ®":119865,"åĴĢ":119866,"åij·":119867,"黾":119868,"åij±":119869,"åij¤":119870,"åĴĨ":119871,"åĴĽ":119872,"åij¶":119873,"åij£":119874,"åĴĿ":119875,"å²¢":119876,"岿":119877,"岬":119878,"岫":119879,"å¸Ļ":119880,"å²£":119881,"å³ģ":119882,"åĪ¿":119883,"å²·":119884,"åīĢ":119885,"å¸Ķ":119886,"å³Ħ":119887,"æ²ĵ":119888,"åĽ¹":119889,"ç½Ķ":119890,"éĴį":119891,"éĴİ":119892,"éĴı":119893,"éĴĴ":119894,"éĴķ":119895,"éĤ¾":119896,"è¿®":119897,"çī¦":119898,"竺":119899,"迤":119900,"佶":119901,"ä¾ij":119902,"ä¾ī":119903,"èĩ¾":119904,"ä¾Ĺ":119905,"ä¾ı":119906,"侩":119907,"ä½»":119908,"ä½¾":119909,"侪":119910,"ä½¼":119911,"佯":119912,"侬":119913,"å¸Ľ":119914,"ä¾Ķ":119915,"å¾Ĥ":119916,"åĪ½":119917,"éĥĦ":119918,"ç±´":119919,"çĵ®":119920,"æĪĹ":119921,"èĤ¼":119922,"äıĿ":119923,"èĤ±":119924,"èĤ«":119925,"è¿©":119926,"éĥĩ":119927,"çĭİ":119928,"çĭį":119929,"çĭĴ":119930,"åĴİ":119931,"饯":119932,"饴":119933,"åĨ½":119934,"åĨ¼":119935,"åºĸ":119936,"çĸł":119937,"çĸĿ":119938,"åħĸ":119939,"åĬ¾":119940,"ð¬ī":119941,"ð¬ī¼":119942,"çĤĺ":119943,"çĤĿ":119944,"çĤĶ":119945,"æ³Ķ":119946,"æ²Ń":119947,"æ³·":119948,"æ³±":119949,"æ³ħ":119950,"æ³ł":119951,"泺":119952,"æ³ĸ":119953,"泫":119954,"æ³®":119955,"æ²±":119956,"泯":119957,"æĢĻ":119958,"æĢµ":119959,"æĢ¦":119960,"æĢĽ":119961,"æĢı":119962,"æĢį":119963,"ã¤":119964,"ã¤ĺ":119965,"æĢ©":119966,"æĢ«":119967,"æĢ¿":119968,"å®ķ":119969,"穹":119970,"å®ĵ":119971,"è¯ĵ":119972,"è¯Ķ":119973,"è¯ĸ":119974,"è¯ĺ":119975,"æĪ¾":119976,"è¯Ļ":119977,"æĪ½":119978,"éĥĵ":119979,"è¡©":119980,"ç¥Ĩ":119981,"ç¥İ":119982,"ç¥ĩ":119983,"è¯ľ":119984,"è¯Ł":119985,"诣":119986,"诤":119987,"诧":119988,"诨":119989,"æĪķ":119990,"éĻĶ":119991,"妲":119992,"妯":119993,"å§Ĺ":119994,"å¸ij":119995,"åŃ¥":119996,"驽":119997,"èĻ±":119998,"迨":119999,"ç»Ģ":120000,"ç»ģ":120001,"ç»Ĥ":120002,"é©·":120003,"驸":120004,"ç»ī":120005,"ç»Į":120006,"éªĢ":120007,"çĶ¾":120008,"çıı":120009,"çıIJ":120010,"çıij":120011,"çݳ":120012,"顸":120013,"çıī":120014,"çıĪ":120015,"æĭ®":120016,"åŀŃ":120017,"æĮĿ":120018,"æĮŀ":120019,"åŀ¤":120020,"èµ³":120021,"è´²":120022,"åŀ±":120023,"åŀĮ":120024,"åŀ§":120025,"åŀĵ":120026,"æĮ¦":120027,"åŀł":120028,"èįļ":120029,"èįij":120030,"è´³":120031,"èįľ":120032,"èİĴ":120033,"èĮ¼":120034,"èĮ´":120035,"èĮ±":120036,"èİĽ":120037,"èįŀ":120038,"èĮ¯":120039,"èįı":120040,"èįĩ":120041,"èįĥ":120042,"èįł":120043,"èĮŃ":120044,"åŀ©":120045,"èį¥":120046,"èį¦":120047,"èį¨":120048,"èį©":120049,"åīĭ":120050,"èįª":120051,"èį¬":120052,"èį®":120053,"æŁ°":120054,"æłī":120055,"æŁĺ":120056,"æłĬ":120057,"æŁ©":120058,"æŀ°":120059,"æłĮ":120060,"æŁĻ":120061,"æŀµ":120062,"æŀ³":120063,"æŁŀ":120064,"æŁĿ":120065,"æłĢ":120066,"æŁ¢":120067,"æłİ":120068,"æŁĪ":120069,"æŁģ":120070,"æŀ·":120071,"æŁ½":120072,"åīĮ":120073,"éħĬ":120074,"éĥ¦":120075,"çĶŃ":120076,"çłĹ":120077,"çłĺ":120078,"çłĴ":120079,"æĸ«":120080,"çłŃ":120081,"çłľ":120082,"èĢ·":120083,"èĻº":120084,"æ®Ĥ":120085,"æ®ĩ":120086,"æ®Ħ":120087,"è½±":120088,"è½²":120089,"è½³":120090,"轶":120091,"轸":120092,"èĻ¿":120093,"æ¯ĸ":120094,"è§ĩ":120095,"å°ľ":120096,"åĵIJ":120097,"çľĦ":120098,"çľį":120099,"ðł³":120100,"ðł³IJ":120101,"éĥ¢":120102,"çľĩ":120103,"çľĬ":120104,"çľĪ":120105,"禺":120106,"åĵĤ":120107,"åĴ´":120108,"æĽ·":120109,"æĺ´":120110,"åĴ¦":120111,"åĵĵ":120112,"åĵĶ":120113,"çķİ":120114,"åij²":120115,"èĥĦ":120116,"çķĭ":120117,"çķĪ":120118,"èĻ¼":120119,"èĻ»":120120,"çĽħ":120121,"åĴ£":120122,"åĵķ":120123,"åīIJ":120124,"éĥ§":120125,"åĴ»":120126,"åĽ¿":120127,"åĴ¿":120128,"åĵĮ":120129,"åĵĻ":120130,"åĵļ":120131,"åĴ©":120132,"åĴ¤":120133,"åĵĿ":120134,"åĵı":120135,"åĵŀ":120136,"å³£":120137,"ç½ĺ":120138,"å³Ĵ":120139,"峤":120140,"å³ĭ":120141,"è´¶":120142,"éĴļ":120143,"éĴ¡":120144,"éĴ£":120145,"éĴ¤":120146,"éĴ«":120147,"æ°¡":120148,"çī¯":120149,"éĥľ":120150,"ç§ķ":120151,"ç§Ń":120152,"竽":120153,"ç¬Ī":120154,"俦":120155,"俨":120156,"ä¿ħ":120157,"åıŁ":120158,"åŀ¡":120159,"çī®":120160,"ä¿£":120161,"ä¿ļ":120162,"çļĪ":120163,"ä¿Ł":120164,"éĢħ":120165,"å¾ĩ":120166,"å¾ī":120167,"èĪ¢":120168,"éĥĹ":120169,"ä¿İ":120170,"éĥ¤":120171,"çĪ°":120172,"éĥĽ":120173,"çĵ´":120174,"èĥ¨":120175,"èĥª":120176,"èĥĽ":120177,"èĥĤ":120178,"èĥĻ":120179,"èĥį":120180,"èĥĹ":120181,"èĥĿ":120182,"æľIJ":120183,"èĥ«":120184,"鸨":120185,"åĮį":120186,"çĭ¨":120187,"çĭ¯":120188,"é£ij":120189,"çĭ©":120190,"çĭ²":120191,"è¨ĩ":120192,"éĢĦ":120193,"æĺĿ":120194,"饷":120195,"饸":120196,"饹":120197,"åŃª":120198,"å¨Ī":120199,"庥":120200,"çĸ¬":120201,"çĸ£":120202,"çĸ¥":120203,"çĸŃ":120204,"åºł":120205,"ç«ij":120206,"é£Ĵ":120207,"éĹ¼":120208,"éĹ¾":120209,"éĹ¿":120210,"éĺĤ":120211,"ç¾ij":120212,"迸":120213,"ç±¼":120214,"éħĭ":120215,"çĤ»":120216,"çĥĢ":120217,"çĤ·":120218,"æ´±":120219,"æ´¹":120220,"æ´§":120221,"æ´Į":120222,"æµĥ":120223,"æ´ĩ":120224,"æ´Ħ":120225,"æ´Ļ":120226,"æ¶İ":120227,"æ´İ":120228,"æ´«":120229,"æµį":120230,"æ´®":120231,"æ´µ":120232,"æµĴ":120233,"æµĶ":120234,"æµķ":120235,"æ´³":120236,"æģ¸":120237,"æģĵ":120238,"æģ¹":120239,"æģ«":120240,"æģ»":120241,"æģĤ":120242,"æģª":120243,"æģ½":120244,"宥":120245,"æīĥ":120246,"衲":120247,"衽":120248,"è¡¿":120249,"è¢Ĥ":120250,"ç¥ľ":120251,"ç¥ĵ":120252,"ç¥ļ":120253,"诮":120254,"ç¥Ĺ":120255,"祢":120256,"诰":120257,"诳":120258,"鸩":120259,"æĺ¶":120260,"åĴ«":120261,"å¼Ń":120262,"çīģ":120263,"èĥ¥":120264,"éĻŁ":120265,"姮":120266,"å¨Ĩ":120267,"å§Ŀ":120268,"姣":120269,"å§ĺ":120270,"姹":120271,"羿":120272,"çĤ±":120273,"磾":120274,"ç»Ķ":120275,"éªģ":120276,"éªħ":120277,"ç»Ĺ":120278,"综":120279,"éªĪ":120280,"èĢĸ":120281,"æĮĪ":120282,"çı¥":120283,"çıĻ":120284,"顼":120285,"çı°":120286,"çı©":120287,"çı§":120288,"çı£":120289,"çıŀ":120290,"çIJ¤":120291,"çı²":120292,"æģļ":120293,"åŁķ":120294,"åŁĺ":120295,"åŁĻ":120296,"åŁļ":120297,"æĮ¹":120298,"èĢĨ":120299,"èĢĦ":120300,"åŁĴ":120301,"æįĭ":120302,"è´½":120303,"åŀ¸":120304,"æįĥ":120305,"çĽį":120306,"èį¸":120307,"èݳ":120308,"èİ´":120309,"èݪ":120310,"èİł":120311,"èİľ":120312,"èİħ":120313,"èį¼":120314,"èİ©":120315,"èį½":120316,"èݸ":120317,"èį»":120318,"èݨ":120319,"鸪":120320,"èݼ":120321,"æł²":120322,"æł³":120323,"æ¡¡":120324,"æ¡İ":120325,"æ¡¢":120326,"桤":120327,"æ¢ĥ":120328,"æłĿ":120329,"æ¡ķ":120330,"æ¡ģ":120331,"桧":120332,"æ¡ħ":120333,"æłŁ":120334,"æ¡ī":120335,"æł©":120336,"éĢij":120337,"éĢĭ":120338,"彧":120339,"鬲":120340,"è±ĩ":120341,"éħIJ":120342,"éĢ¦":120343,"åİĿ":120344,"åѬ":120345,"çłĿ":120346,"çł¹":120347,"糧":120348,"çł·":120349,"糣":120350,"çł¼":120351,"çł¥":120352,"çł£":120353,"åīŀ":120354,"çł»":120355,"è½¼":120356,"è½¾":120357,"è¾Ĥ":120358,"鸫":120359,"趸":120360,"é¾Ģ":120361,"鸬":120362,"èĻĶ":120363,"羬":120364,"åĶĽ":120365,"çľĻ":120366,"åĵ§":120367,"åĵ½":120368,"æĻģ":120369,"鸮":120370,"趵":120371,"趿":120372,"çķĽ":120373,"èļ¨":120374,"èļľ":120375,"èļį":120376,"èļĭ":120377,"èļ¬":120378,"èļĿ":120379,"èļ§":120380,"åĶ¢":120381,"åľĦ":120382,"åĶ£":120383,"åĶı":120384,"çĽİ":120385,"åĶij":120386,"å´Ĥ":120387,"å´ĥ":120388,"罡":120389,"ç½Ł":120390,"è§Ĭ":120391,"èµħ":120392,"éĴ²":120393,"éĴµ":120394,"éĴ¹":120395,"éĴº":120396,"éĴ½":120397,"éĴ¼":120398,"éĴ¿":120399,"éĵĢ":120400,"éĵĦ":120401,"éĵĨ":120402,"éĵĪ":120403,"éĵī":120404,"éĵĬ":120405,"éĵĭ":120406,"éĵĮ":120407,"éĵį":120408,"ä¥":120409,"䥽":120410,"éĵİ":120411,"æ°©":120412,"æ°¤":120413,"æ°¦":120414,"毪":120415,"èĪIJ":120416,"秣":120417,"秫":120418,"çĽī":120419,"ç¬Ħ":120420,"ç¬ķ":120421,"ç¬Ĭ":120422,"ç¬ı":120423,"ç¬Ĩ":120424,"俸":120425,"俵":120426,"åģĮ":120427,"俳":120428,"俶":120429,"åĢ¬":120430,"åĢı":120431,"æģģ":120432,"åĢŃ":120433,"俾":120434,"åĢľ":120435,"éļ¼":120436,"éļ½":120437,"åĢĮ":120438,"åĢ¥":120439,"èĩ¬":120440,"éĥ«":120441,"åĢ¨":120442,"è¡Ħ":120443,"é¢Ģ":120444,"å¾ķ":120445,"èĪ«":120446,"衾":120447,"èĥ¯":120448,"èĥ±":120449,"èĥ´":120450,"èĥŃ":120451,"èĦį":120452,"èĥ¼":120453,"èĦĴ":120454,"鸱":120455,"鸲":120456,"çĭ·":120457,"çĮģ":120458,"çĭ³":120459,"çĮĥ":120460,"çĭº":120461,"éĢĸ":120462,"æ¡Ģ":120463,"饽":120464,"åĩĩ":120465,"æĮĽ":120466,"亳":120467,"çĸ³":120468,"çĸ´":120469,"çĸ¸":120470,"çĸ½":120471,"çĹĪ":120472,"çĸ±":120473,"çĹĤ":120474,"çĹī":120475,"è¡®":120476,"é¢ĥ":120477,"æģ£":120478,"æĹĨ":120479,"æĹĦ":120480,"æĹĥ":120481,"éĺĥ":120482,"éĺĦ":120483,"è¨ļ":120484,"éĺĨ":120485,"æģĻ":120486,"ç²ij":120487,"çĥľ":120488,"çĥ©":120489,"çĥĬ":120490,"åī¡":120491,"éĥ¯":120492,"çĥ¬":120493,"æ¶ij":120494,"浯":120495,"æ¶ŀ":120496,"æ¶Ł":120497,"å¨ij":120498,"æ¶ł":120499,"æµŀ":120500,"æ¶ĵ":120501,"æµ¥":120502,"æ¶Ķ":120503,"æµľ":120504,"æµł":120505,"æµ£":120506,"æĤļ":120507,"æĤŃ":120508,"æĤĿ":120509,"æĤĴ":120510,"æĤĮ":120511,"æĤĽ":120512,"çªĪ":120513,"åīľ":120514,"诹":120515,"诼":120516,"è¢Ĵ":120517,"袢":120518,"诿":120519,"è°Ģ":120520,"è°Ĥ":120521,"è°Ħ":120522,"è°ĩ":120523,"å±IJ":120524,"å±Ļ":120525,"éĻ¬":120526,"åĭIJ":120527,"å¥ĺ":120528,"çīĤ":120529,"èļ©":120530,"éĻ²":120531,"å¨Į":120532,"å¨ī":120533,"娲":120534,"娴":120535,"娣":120536,"å¨ĵ":120537,"å©Ģ":120538,"çķļ":120539,"éĢ¡":120540,"绳":120541,"éªĬ":120542,"绡":120543,"éªĭ":120544,"绦":120545,"绨":120546,"éªİ":120547,"éĤķ":120548,"鸶":120549,"å½Ĺ":120550,"èĢľ":120551,"çĦĺ":120552,"èĪĤ":120553,"çIJı":120554,"çIJĩ":120555,"麸":120556,"æı¶":120557,"åŁ´":120558,"åŁ¯":120559,"æį¯":120560,"æݳ":120561,"æİ´":120562,"åŁ¸":120563,"åŁµ":120564,"赧":120565,"åŁ¤":120566,"æįŃ":120567,"éĢµ":120568,"åŁĿ":120569,"åłĭ":120570,"åłį":120571,"æݬ":120572,"鸷":120573,"æį½":120574,"æİĬ":120575,"åłī":120576,"æݸ":120577,"æį©":120578,"æİ®":120579,"æĤ«":120580,"åŁŃ":120581,"åŁ½":120582,"æİĩ":120583,"æݼ":120584,"èģĥ":120585,"èIJģ":120586,"èıĺ":120587,"åłĩ":120588,"èIJĺ":120589,"èIJĭ":120590,"èı½":120591,"èıĸ":120592,"èIJľ":120593,"èIJ¸":120594,"èIJij":120595,"棻":120596,"èıĶ":120597,"èıŁ":120598,"èIJı":120599,"èı¹":120600,"èıª":120601,"èıħ":120602,"èıĢ":120603,"èı°":120604,"èı¡":120605,"梿":120606,"æ¢ı":120607,"è§ĭ":120608,"æ¡´":120609,"æ¡·":120610,"æ£ģ":120611,"æ¡«":120612,"æ£Ĥ":120613,"åķ¬":120614,"éĥ¾":120615,"æķķ":120616,"è±ī":120617,"éĦĦ":120618,"éħŀ":120619,"ç¡İ":120620,"ç¡Ń":120621,"ç¡ĸ":120622,"ç¡Ĺ":120623,"ç¡IJ":120624,"ç¡ĩ":120625,"ç¡Į":120626,"鸸":120627,"çĵł":120628,"åĮı":120629,"åİ©":120630,"æ®Ĵ":120631,"æ®ĵ":120632,"æ®į":120633,"èµī":120634,"鼩":120635,"è¾Ħ":120636,"åłij":120637,"çľŃ":120638,"羦":120639,"åķ§":120640,"æĻ¡":120641,"æĻ¤":120642,"çľµ":120643,"åľĬ":120644,"åĸı":120645,"åķī":120646,"åĭĸ":120647,"æĻŀ":120648,"åĶµ":120649,"æĻĹ":120650,"åķŃ":120651,"çķ¦":120652,"趺":120653,"åķ®":120654,"è·Ħ":120655,"èļ¶":120656,"èĽĦ":120657,"èĽİ":120658,"èĽĨ":120659,"èļ°":120660,"åľī":120661,"èļ±":120662,"èĽī":120663,"èĽı":120664,"èļ´":120665,"åķģ":120666,"åķķ":120667,"åĶ¿":120668,"åķIJ":120669,"åĶ¼":120670,"åĶ·":120671,"åķĸ":120672,"åķµ":120673,"åķ¶":120674,"åķ·":120675,"åĶ³":120676,"åĶ°":120677,"åķľ":120678,"帻":120679,"å´ļ":120680,"å´¦":120681,"帼":120682,"å´®":120683,"å´¤":120684,"å´Ĩ":120685,"èµĩ":120686,"èµĪ":120687,"èµĬ":120688,"éĵij":120689,"éĵĴ":120690,"éĵĹ":120691,"éĵĻ":120692,"éĵŁ":120693,"éĵ¡":120694,"éĵ¢":120695,"éĵ£":120696,"éĵ¤":120697,"éĵ§":120698,"éĵ¨":120699,"éĵ©":120700,"éĵª":120701,"éĵ«":120702,"éĵ¯":120703,"éĵ°":120704,"éĵ±":120705,"éĵ³":120706,"éĵµ":120707,"éĵ·":120708,"çī¾":120709,"鸹":120710,"秾":120711,"éĢ¶":120712,"笺":120713,"çŃĩ":120714,"笸":120715,"笪":120716,"笮":120717,"笳":120718,"笥":120719,"笤":120720,"笳":120721,"笾":120722,"ç¬ŀ":120723,"åģ¾":120724,"åģĥ":120725,"åģķ":120726,"åģĪ":120727,"åĤĢ":120728,"åģ¬":120729,"åģ»":120730,"çļij":120731,"çļİ":120732,"鸻":120733,"å¾ľ":120734,"èĪ¸":120735,"èĪ»":120736,"èĪ´":120737,"èĪ·":120738,"é¾Ľ":120739,"ç¿İ":120740,"èĦ¬":120741,"èĦĺ":120742,"èĦ²":120743,"åĮIJ":120744,"çĮĹ":120745,"çĮ¡":120746,"çĮŀ":120747,"æĸĽ":120748,"çĮķ":120749,"é¦Ĺ":120750,"é¦ĥ":120751,"é¦Ħ":120752,"鸾":120753,"庹":120754,"庾":120755,"çĹĶ":120756,"çĹį":120757,"ç¿Ĭ":120758,"æĹĮ":120759,"æĹİ":120760,"袤":120761,"éĺĩ":120762,"éĺĪ":120763,"éĺī":120764,"éĺĬ":120765,"éĺĭ":120766,"éĺį":120767,"éĺı":120768,"ç¾Ł":120769,"ç²Ŀ":120770,"çĦIJ":120771,"çĦĵ":120772,"çĦĹ":120773,"æ·ħ":120774,"æ·ŀ":120775,"æ¸İ":120776,"涿":120777,"æ·ĸ":120778,"æĮ²":120779,"æ·ł":120780,"涸":120781,"æ¸ij":120782,"æ·¦":120783,"æ·Ŀ":120784,"涪":120785,"æ·Ļ":120786,"涫":120787,"æ¸Į":120788,"æĤ»":120789,"æĤ±":120790,"æĥĿ":120791,"æĥĺ":120792,"æĥĨ":120793,"æĥļ":120794,"æĥĩ":120795,"æĥ®":120796,"çªķ":120797,"è°Į":120798,"æīĪ":120799,"çļ²":120800,"è°ij":120801,"è£Ĩ":120802,"袷":120803,"è£ī":120804,"è°Ĵ":120805,"è°Ķ":120806,"è°ķ":120807,"è°ĸ":120808,"è°Ĺ":120809,"è°Ļ":120810,"è°Ŀ":120811,"éĢ¯":120812,"éĥ¿":120813,"éļĪ":120814,"ç²ľ":120815,"éļį":120816,"éļĹ":120817,"å©Ĭ":120818,"娼":120819,"å©¢":120820,"婵":120821,"èĥ¬":120822,"è¢Ī":120823,"ç¿Į":120824,"æģ¿":120825,"欸":120826,"绫":120827,"éªIJ":120828,"绯":120829,"ç»±":120830,"éªĴ":120831,"绲":120832,"éªĵ":120833,"绶":120834,"绺":120835,"ç»»":120836,"绾":120837,"éªĸ":120838,"ç¼ģ":120839,"èĢł":120840,"çIJ«":120841,"çIJµ":120842,"çIJ¶":120843,"çIJ¥":120844,"çIJ¨":120845,"çIJ°":120846,"çIJ®":120847,"çIJ¯":120848,"çIJ¬":120849,"çIJļ":120850,"è¾ĩ":120851,"é¼ĭ":120852,"æı³":120853,"åłŀ":120854,"æIJ½":120855,"æı¸":120856,"æıł":120857,"åłĻ":120858,"è¶Ħ":120859,"æıĸ":120860,"é¢ī":120861,"å¡Ħ":120862,"æı¿":120863,"èĢĭ":120864,"æıĦ":120865,"èĽ©":120866,"èĽ°":120867,"å¡Ĩ":120868,"æijĴ":120869,"æıĨ":120870,"æݾ":120871,"èģĴ":120872,"èijij":120873,"èijļ":120874,"éĿ°":120875,"éĿ¸":120876,"èij³":120877,"èijº":120878,"èij¸":120879,"èIJ¼":120880,"èij¶":120881,"èĴĮ":120882,"èijŃ":120883,"楮":120884,"棼":120885,"æ¤Ł":120886,"棹":120887,"椤":120888,"棰":120889,"èµį":120890,"æ¤ĭ":120891,"æ¤ģ":120892,"椪":120893,"æ¤IJ":120894,"é¹ģ":120895,"éħ¤":120896,"éħ¢":120897,"éħ¡":120898,"é¹Ĥ":120899,"æ®ļ":120900,"æ®Ľ":120901,"鼱":120902,"è¾ĭ":120903,"æ¤ł":120904,"è¾İ":120905,"çĿĦ":120906,"çĿĩ":120907,"çĿĥ":120908,"æĪ¢":120909,"åĸĭ":120910,"åĹĴ":120911,"åĸĥ":120912,"åĸ±":120913,"åĸ¹":120914,"æĻ·":120915,"åĸĪ":120916,"è·ĸ":120917,"è·Ĺ":120918,"è·ŀ":120919,"è·ļ":120920,"è·İ":120921,"è·ı":120922,"è·Ĩ":120923,"èĽ±":120924,"èĽ²":120925,"èĽŃ":120926,"èĽ³":120927,"èĽIJ":120928,"èĽĶ":120929,"èĽŀ":120930,"èĽ´":120931,"èĽĺ":120932,"åĸģ":120933,"åĸŁ":120934,"åķ¾":120935,"åĹĸ":120936,"åĸij":120937,"åĹŁ":120938,"åĹŀ":120939,"åĸĻ":120940,"åµĺ":120941,"åµĸ":120942,"å´´":120943,"éģĦ":120944,"è©Ī":120945,"åµİ":120946,"嵬":120947,"åµĽ":120948,"嵯":120949,"åµĿ":120950,"嵫":120951,"å¹Ħ":120952,"åµĭ":120953,"èµķ":120954,"éĵ»":120955,"éĵ¼":120956,"éĵ¿":120957,"éĶĥ":120958,"éĶĨ":120959,"éĶĩ":120960,"éĶī":120961,"éĶı":120962,"éĶij":120963,"éĶĴ":120964,"éĶĶ":120965,"éĶķ":120966,"æİ£":120967,"磬":120968,"æ°°":120969,"毳":120970,"毽":120971,"çĬĬ":120972,"çĬĦ":120973,"çĬĭ":120974,"é¹Ħ":120975,"çĬį":120976,"åµĩ":120977,"é»į":120978,"ç¨ĥ":120979,"ç¨Ĥ":120980,"çŃļ":120981,"çѵ":120982,"çŃĮ":120983,"åĤ£":120984,"åĤĪ":120985,"èĪĦ":120986,"çīį":120987,"åĤ¥":120988,"åĤ§":120989,"éģij":120990,"åĤ©":120991,"徨":120992,"åªŃ":120993,"çķ²":120994,"å¼ij":120995,"ç¿ķ":120996,"é¹Ĩ":120997,"èħĪ":120998,"èħĵ":120999,"èħĨ":121000,"èħ´":121001,"èħļ":121002,"èħ±":121003,"鱿":121004,"é²Ģ":121005,"é²Ĥ":121006,"çĮ¢":121007,"çĮ¹":121008,"çĮ¥":121009,"é£ĵ":121010,"è§ŀ":121011,"è§ļ":121012,"çĮ±":121013,"é¢İ":121014,"飧":121015,"é¦ĩ":121016,"é¦Ĭ":121017,"亵":121018,"èĦĶ":121019,"è£Ĵ":121020,"çĹ£":121021,"çŨ":121022,"çŦ":121023,"çĹŀ":121024,"çŤ":121025,"çŧ":121026,"èµĵ":121027,"竦":121028,"çĵ¿":121029,"åķ»":121030,"é¢ı":121031,"é¹ĩ":121032,"éĺij":121033,"éĺĴ":121034,"éĺķ":121035,"ç²ŀ":121036,"éģĴ":121037,"åŃ³":121038,"çĦ¯":121039,"çĦľ":121040,"çĦ±":121041,"é¹Ī":121042,"渫":121043,"æ¹®":121044,"æ¹İ":121045,"æ¹ľ":121046,"æ¹į":121047,"湫":121048,"溲":121049,"æ¹Ł":121050,"æºĨ":121051,"æ¹²":121052,"æ¹Ķ":121053,"æ¹ī":121054,"渥":121055,"æ»ģ":121056,"æĦł":121057,"æĥº":121058,"æĦ¦":121059,"æĥ´":121060,"æĦĢ":121061,"æĦİ":121062,"æĦĶ":121063,"åĸ¾":121064,"å¯IJ":121065,"è°Ł":121066,"裢":121067,"è£İ":121068,"裥":121069,"祾":121070,"è°ł":121071,"è°¡":121072,"è°¥":121073,"è°§":121074,"åѱ":121075,"å¼¼":121076,"å·½":121077,"éªĺ":121078,"媪":121079,"å·¯":121080,"ç¿ļ":121081,"çļ´":121082,"éªĽ":121083,"ç¼Ĥ":121084,"ç¼ĥ":121085,"ç¼Ħ":121086,"å½ĺ":121087,"ç¼ĩ":121088,"ç¼Ī":121089,"ç¼Į":121090,"ç¼ij":121091,"ç¼Ĵ":121092,"ç¼Ĺ":121093,"飨":121094,"èĢ¢":121095,"çijģ":121096,"çijĹ":121097,"çijĦ":121098,"éģ¨":121099,"éªľ":121100,"飫":121101,"é«¡":121102,"塬":121103,"éĦ¢":121104,"è¶Ķ":121105,"è¶ij":121106,"æijħ":121107,"æijģ":121108,"èľĩ":121109,"æIJĭ":121110,"æIJª":121111,"æIJIJ":121112,"æIJĽ":121113,"æIJł":121114,"æijĪ":121115,"å½Ģ":121116,"æ¯Ĥ":121117,"æIJ¦":121118,"æIJ¡":121119,"èĵģ":121120,"æĪ¡":121121,"èĵį":121122,"éĦŀ":121123,"èĵIJ":121124,"èĵ¦":121125,"é¹ĭ":121126,"èĴ½":121127,"èĵĸ":121128,"èĵĬ":121129,"èĴ¯":121130,"èĵŁ":121131,"èĵij":121132,"èĴº":121133,"èĵł":121134,"èĴŁ":121135,"èĴ¡":121136,"èĴ¹":121137,"èĴ´":121138,"èĴĹ":121139,"èĵ¥":121140,"æ¥Ķ":121141,"æ¥Ĥ":121142,"æ¥Ŀ":121143,"楫":121144,"楸":121145,"椴":121146,"æ§Į":121147,"楯":121148,"çļĻ":121149,"æ¦Ī":121150,"æ§İ":121151,"æ¦ī":121152,"楦":121153,"楣":121154,"楹":121155,"椽":121156,"åī½":121157,"éħ©":121158,"èľĥ":121159,"ç¢Ľ":121160,"ç¢ĵ":121161,"硼":121162,"ç¢ī":121163,"ç¢ļ":121164,"ç¢ĩ":121165,"ç¢ľ":121166,"é¹Į":121167,"è¾ı":121168,"é¾ĥ":121169,"é¾ħ":121170,"訾":121171,"ç²²":121172,"çĿļ":121173,"åĹª":121174,"éŁª":121175,"åĹ·":121176,"åĹī":121177,"çĿ¨":121178,"çĿ¢":121179,"éĽİ":121180,"çĿ¥":121181,"åĹij":121182,"åĹ«":121183,"åŬ":121184,"åĹĶ":121185,"åĹĿ":121186,"æĪ¥":121187,"åĹĦ":121188,"çħ¦":121189,"æļĦ":121190,"éģ¢":121191,"æļĮ":121192,"è·¬":121193,"è·¶":121194,"è·¸":121195,"è·IJ":121196,"è·£":121197,"è·¹":121198,"èĽ¸":121199,"èľĬ":121200,"èľį":121201,"èľī":121202,"èľ£":121203,"çķ¹":121204,"èĽ¹":121205,"åĹ¥":121206,"åĹ²":121207,"åĹ³":121208,"åĹĮ":121209,"åĹį":121210,"åĹIJ":121211,"åŤ":121212,"åŵ":121213,"罨":121214,"åµĬ":121215,"åµ´":121216,"骰":121217,"éĶĹ":121218,"éĶĽ":121219,"éĶľ":121220,"éĶĿ":121221,"éĶŀ":121222,"éĶŁ":121223,"éĶ¢":121224,"éĶ¨":121225,"éĶ©":121226,"éĶŃ":121227,"éĶ±":121228,"éĽī":121229,"æ°²":121230,"çĬı":121231,"æŃĥ":121232,"ç¨ŀ":121233,"ç¨Ĺ":121234,"ç¨Ķ":121235,"çŃł":121236,"çŃ¢":121237,"çŃ®":121238,"çŃ²":121239,"çīĴ":121240,"æķ«":121241,"å¾Ń":121242,"æĦĨ":121243,"èīĦ":121244,"è§İ":121245,"毹":121246,"è²Ĭ":121247,"è²ħ":121248,"è²ī":121249,"é¢Ķ":121250,"èħł":121251,"èħ©":121252,"èħ¼":121253,"èħŃ":121254,"èħ§":121255,"å¡į":121256,"媵":121257,"é²ħ":121258,"é²Ĩ":121259,"é²ĩ":121260,"é²Ī":121261,"é²ĭ":121262,"é²IJ":121263,"èĤĦ":121264,"é¹IJ":121265,"é£ķ":121266,"觥":121267,"éģĽ":121268,"é¦IJ":121269,"é¹ij":121270,"亶":121271,"çĺĥ":121272,"çű":121273,"çĹ¼":121274,"çĹ¿":121275,"çĺIJ":121276,"çĺģ":121277,"çĺĨ":121278,"éºĤ":121279,"æŃĨ":121280,"æĹĴ":121281,"éĺĸ":121282,"éĺĹ":121283,"羧":121284,"è±¢":121285,"ç²³":121286,"çĮ·":121287,"çħ³":121288,"çħ¨":121289,"çħħ":121290,"çħĬ":121291,"çħ¸":121292,"çħº":121293,"æ»Ł":121294,"溱":121295,"æºĺ":121296,"æ¼Ń":121297,"滢":121298,"溥":121299,"溽":121300,"è£Ł":121301,"溻":121302,"溷":121303,"æ»Ĺ":121304,"滫":121305,"溴":121306,"æ»ı":121307,"æ»ĥ":121308,"滦":121309,"æºı":121310,"æ»Ĥ":121311,"æ»ĵ":121312,"æºŁ":121313,"滪":121314,"æĦ«":121315,"æħĬ":121316,"é²İ":121317,"éªŀ":121318,"çªł":121319,"窣":121320,"裱":121321,"裨":121322,"裾":121323,"裰":121324,"ç¦Ĭ":121325,"è°©":121326,"è°ª":121327,"媾":121328,"å««":121329,"媲":121330,"å«Ĵ":121331,"å«Ķ":121332,"媸":121333,"ç¼Ļ":121334,"ç¼ľ":121335,"ç¼Ľ":121336,"è¾Ķ":121337,"éªĿ":121338,"ç¼Ł":121339,"缡":121340,"ç¼¢":121341,"ç¼£":121342,"éªŁ":121343,"èĢ¥":121344,"çĴĪ":121345,"çijŃ":121346,"çįĴ":121347,"è§ı":121348,"æħĿ":121349,"å«ł":121350,"åıĨ":121351,"æij½":121352,"å¢ģ":121353,"æĴĤ":121354,"æijŀ":121355,"æĴĦ":121356,"ç¿¥":121357,"è¸ħ":121358,"æijŃ":121359,"å¢ī":121360,"å¢Ĵ":121361,"æ¦ĸ":121362,"綦":121363,"èĶ«":121364,"èĶ·":121365,"éĿº":121366,"éĿ¼":121367,"éŀħ":121368,"éĿ¿":121369,"çĶį":121370,"èĶ¸":121371,"èĶŁ":121372,"èĶº":121373,"æĪ¬":121374,"èķĸ":121375,"èĶ»":121376,"èĵ¿":121377,"æĸ¡":121378,"é¹ķ":121379,"èĵ¼":121380,"æ¦Ľ":121381,"榧":121382,"榫":121383,"æ¦Ń":121384,"æ§Ķ":121385,"榱":121386,"æ§ģ":121387,"æ§ł":121388,"榷":121389,"åĥ°":121390,"éħ½":121391,"éħ¹":121392,"碡":121393,"碴":121394,"碣":121395,"碲":121396,"èĩ§":121397,"豨":121398,"殡":121399,"éľģ":121400,"èľļ":121401,"é¾ĩ":121402,"é¾Ī":121403,"äģ":121404,"äģĸ":121405,"çĿ½":121406,"åĺŀ":121407,"åĺĪ":121408,"åĺĮ":121409,"åĺģ":121410,"æļĿ":121411,"è¸Į":121412,"è¸ī":121413,"èľŀ":121414,"èľ¥":121415,"èľ®":121416,"èĿĪ":121417,"èľ´":121418,"èľ±":121419,"èľ©":121420,"èľ·":121421,"èľ¿":121422,"èŀĤ":121423,"èľ¢":121424,"åĺ¡":121425,"é¹Ĺ":121426,"åĺ£":121427,"åĺ¤":121428,"åĺļ":121429,"åĹ¾":121430,"åĺ§":121431,"ç½´":121432,"ç½±":121433,"å¹Ķ":121434,"å¶Ĥ":121435,"å¹Ľ":121436,"èµĻ":121437,"ç½Ĥ":121438,"骷":121439,"骶":121440,"é¹ĺ":121441,"éĶ²":121442,"éĶ´":121443,"éĶ¶":121444,"éĶ·":121445,"éĶ¸":121446,"éĶµ":121447,"éķĤ":121448,"çĬĴ":121449,"ç®IJ":121450,"箦":121451,"箧":121452,"箸":121453,"箬":121454,"ç®ħ":121455,"箪":121456,"箾":121457,"箢":121458,"ç®ĵ":121459,"åĥĸ":121460,"åĦĨ":121461,"åĥ³":121462,"åĥŃ":121463,"åĬģ":121464,"åĥ®":121465,"éŃĥ":121466,"éŃĨ":121467,"çĿ¾":121468,"èīĭ":121469,"éĦ±":121470,"èĨĪ":121471,"èĨij":121472,"é²ij":121473,"é²Ķ":121474,"é²ļ":121475,"é²Ľ":121476,"é²Ł":121477,"çįIJ":121478,"觫":121479,"éĽĴ":121480,"夤":121481,"é¦ij":121482,"éĬ®":121483,"塾":121484,"çĺĮ":121485,"çĺĬ":121486,"çĺĺ":121487,"çĺĻ":121488,"æĹĸ":121489,"èĨĤ":121490,"éĺļ":121491,"éĦ¯":121492,"é²ŀ":121493,"粿":121494,"ç²¼":121495,"ç³ģ":121496,"æ§Ĭ":121497,"é¹ļ":121498,"çĨĺ":121499,"çĨ¥":121500,"æ½¢":121501,"æ¼ķ":121502,"滹":121503,"漯":121504,"漶":121505,"æ½ĭ":121506,"æ½´":121507,"漪":121508,"æ¼ī":121509,"漩":121510,"æ¾ī":121511,"æħµ":121512,"æIJ´":121513,"窨":121514,"寤":121515,"綮":121516,"è°®":121517,"褡":121518,"è¤Ļ":121519,"è¤ĵ":121520,"è¤Ľ":121521,"è¤Ĭ":121522,"è°¯":121523,"è°°":121524,"è°²":121525,"å±£":121526,"é¹Ľ":121527,"嫱":121528,"å«ĸ":121529,"嫦":121530,"å«ļ":121531,"å«ĺ":121532,"é¼IJ":121533,"çŀĢ":121534,"é¹ľ":121535,"éªł":121536,"ç¼¥":121537,"缦":121538,"缧":121539,"缨":121540,"骢":121541,"缫":121542,"èĢ¦":121543,"èĢ§":121544,"çĴľ":121545,"çĴİ":121546,"çĴģ":121547,"å¥Ń":121548,"髯":121549,"é««":121550,"æĴ·":121551,"æĴħ":121552,"èµŃ":121553,"æĴ¸":121554,"éĭĨ":121555,"æĴĻ":121556,"æĴº":121557,"å¢Ģ":121558,"èģ©":121559,"è§IJ":121560,"éŀij":121561,"èķĻ":121562,"éŀĴ":121563,"èķĪ":121564,"èķ¨":121565,"èķ¤":121566,"èķŀ":121567,"èķº":121568,"çŀ¢":121569,"èķĥ":121570,"èķ²":121571,"èµľ":121572,"槿":121573,"樯":121574,"æ§Ń":121575,"æ¨Ĺ":121576,"æ¨ĺ":121577,"槲":121578,"éĨĮ":121579,"éĨħ":121580,"éĿ¥":121581,"éŃĩ":121582,"é¤į":121583,"ç£Ķ":121584,"ç£Ļ":121585,"éľĪ":121586,"è¾ĺ":121587,"é¾ī":121588,"é¾Ĭ":121589,"è§ij":121590,"çŀĮ":121591,"çŀĭ":121592,"çŀij":121593,"åĺŃ":121594,"åĻİ":121595,"åĻ¶":121596,"é¢Ļ":121597,"æļ¹":121598,"åĻĺ":121599,"è¸Ķ":121600,"è¸Ŀ":121601,"è¸Ł":121602,"è¸Ĵ":121603,"踬":121604,"踮":121605,"踯":121606,"踺":121607,"è¸ŀ":121608,"èĿ½":121609,"èĿ¾":121610,"èĿ»":121611,"èĿ°":121612,"èĿ®":121613,"èŀĭ":121614,"èĿĵ":121615,"èĿ£":121616,"èĿ¼":121617,"åĺ¬":121618,"é¢ļ":121619,"åĻį":121620,"åĻĻ":121621,"åĻĮ":121622,"åĻĶ":121623,"é¢Ľ":121624,"å¹ŀ":121625,"幡":121626,"å¶Ļ":121627,"å¶Ŀ":121628,"骺":121629,"éķĬ":121630,"éķī":121631,"éķĮ":121632,"éķı":121633,"éķĴ":121634,"éķĵ":121635,"éķĶ":121636,"稷":121637,"ç®´":121638,"ç¯ij":121639,"ç¯ģ":121640,"ç¯Į":121641,"çīĸ":121642,"åĦĭ":121643,"èĻ¢":121644,"é¹ŀ":121645,"èĨĺ":121646,"é²ł":121647,"鲡":121648,"é²¢":121649,"é²£":121650,"é²¥":121651,"鲧":121652,"鲩":121653,"çįĹ":121654,"çįł":121655,"觯":121656,"é¦ĵ":121657,"é¦Ķ":121658,"麾":121659,"å»Ľ":121660,"çĺĽ":121661,"çĺ¼":121662,"çĺ¢":121663,"çĺł":121664,"é½ij":121665,"ç¾°":121666,"ð¥»":121667,"ð¥»Ĺ":121668,"ç³Į":121669,"ç³į":121670,"ç³ħ":121671,"çĨľ":121672,"çĨµ":121673,"æ¾į":121674,"æ¾Į":121675,"潸":121676,"潦":121677,"æ½²":121678,"éĭĪ":121679,"æ½Ł":121680,"潺":121681,"寮":121682,"窳":121683,"è°³":121684,"褴":121685,"è¤Ł":121686,"褫":121687,"è°µ":121688,"çĨ¨":121689,"屦":121690,"åĭ°":121691,"æĪ®":121692,"èĿ¥":121693,"缬":121694,"ç¼®":121695,"缯":121696,"骣":121697,"çķ¿":121698,"èĢ©":121699,"èĢ¨":121700,"èĢª":121701,"çĴŁ":121702,"éĿĽ":121703,"çĴł":121704,"çĴĺ":121705,"èģ±":121706,"èŀ¯":121707,"é«»":121708,"é«Ń":121709,"髹":121710,"æĵĢ":121711,"çĶı":121712,"æĵŀ":121713,"縳":121714,"磬":121715,"é¢ŀ":121716,"èķ»":121717,"é¢Ł":121718,"èĸ¤":121719,"èĸ¨":121720,"æªł":121721,"èĸı":121722,"èĸ®":121723,"èĸľ":121724,"èĸħ":121725,"樾":121726,"æ©Ľ":121727,"æ©ĩ":121728,"樵":121729,"æªİ":121730,"橹":121731,"樽":121732,"樨":121733,"橼":121734,"墼":121735,"æ©IJ":121736,"ç¿®":121737,"éĨIJ":121738,"éĨį":121739,"éĨļ":121740,"磲":121741,"èµĿ":121742,"殪":121743,"éľı":121744,"éĮ¾":121745,"è¾ļ":121746,"éģ½":121747,"æ°ħ":121748,"çŀŁ":121749,"çŀł":121750,"çŀ°":121751,"åļĦ":121752,"åļĨ":121753,"åĻ¤":121754,"æļ¾":121755,"è¹Ģ":121756,"踵":121757,"踽":121758,"è¹ī":121759,"è¹ģ":121760,"èŀ¨":121761,"èŀĪ":121762,"èŀħ":121763,"èŀŃ":121764,"èŀł":121765,"èŀŁ":121766,"åĻ±":121767,"åĻ«":121768,"åĻ»":121769,"åĻ¼":121770,"ç½¹":121771,"åľľ":121772,"ä¦":121773,"ä¦ĥ":121774,"éķĹ":121775,"éķĺ":121776,"éķļ":121777,"éķĽ":121778,"éķĿ":121779,"éķŀ":121780,"éķł":121781,"æ°ĩ":121782,"æ°Ĩ":121783,"ç©ij":121784,"ç¯Ŀ":121785,"篥":121786,"篦":121787,"篪":121788,"ç¯Ļ":121789,"çĽ¥":121790,"åĬĵ":121791,"翱":121792,"éŃī":121793,"éŃĪ":121794,"å¾¼":121795,"æŃĻ":121796,"èĨ¦":121797,"èĨĻ":121798,"é²®":121799,"é²±":121800,"é²³":121801,"é²´":121802,"é²µ":121803,"é²·":121804,"é²»":121805,"çį´":121806,"çįŃ":121807,"çį¬":121808,"éĤĤ":121809,"鹧":121810,"廨":121811,"èµŁ":121812,"çĺ°":121813,"廪":121814,"çĺ¿":121815,"çĺµ":121816,"çĺ´":121817,"çĻĥ":121818,"çĺ³":121819,"éºĩ":121820,"éºĪ":121821,"嬴":121822,"å£ħ":121823,"ç³Ĺ":121824,"çĶij":121825,"çĩİ":121826,"çĩł":121827,"çĩĶ":121828,"çĩ§":121829,"æ¿ij":121830,"æ¿ī":121831,"æ½ŀ":121832,"澧":121833,"æ¾¹":121834,"æ¾¥":121835,"澶":121836,"æ¿Ĥ":121837,"褰":121838,"窸":121839,"å¬ĸ":121840,"çĬŁ":121841,"éļ°":121842,"å¬Ĺ":121843,"颡":121844,"ç¼±":121845,"ç¼²":121846,"ç¼³":121847,"çĴ©":121848,"çĴª":121849,"èŀ«":121850,"æĵ¤":121851,"å£ķ":121852,"觳":121853,"ç½Ħ":121854,"æĵ¢":121855,"èĸ¹":121856,"éŀ¡":121857,"éŀ¬":121858,"èĸ·":121859,"èĹĵ":121860,"èĹģ":121861,"æªĦ":121862,"檩":121863,"æĩĭ":121864,"éĨ¢":121865,"翳":121866,"ç¤ħ":121867,"磴":121868,"鹩":121869,"é¾ĭ":121870,"é¾Į":121871,"è±³":121872,"å£ij":121873,"é»»":121874,"åļı":121875,"åļħ":121876,"è¹ij":121877,"è¹Ĵ":121878,"è¹Ĭ":121879,"èŁ¥":121880,"èŀ¬":121881,"èŀµ":121882,"çĸĥ":121883,"èŀ³":121884,"èŁij":121885,"åļĵ":121886,"ç½½":121887,"ç½¾":121888,"嶷":121889,"黾":121890,"é»Ŀ":121891,"é«ģ":121892,"é«Ģ":121893,"éķ¡":121894,"éķ¢":121895,"éķ£":121896,"éķ¦":121897,"éķ§":121898,"éķ©":121899,"éķª":121900,"éķ«":121901,"ç½ħ":121902,"ç°Į":121903,"篾":121904,"篼":121905,"ç°ĸ":121906,"ç°ĭ":121907,"é¼¢":121908,"åĦ¡":121909,"鹪":121910,"é¼¾":121911,"çļ¤":121912,"éŃį":121913,"é¾ł":121914,"ç¹ĩ":121915,"è²ĺ":121916,"éĤĪ":121917,"è²Ķ":121918,"èĩĮ":121919,"èĨ»":121920,"èĩĨ":121921,"èĩĥ":121922,"é²¼":121923,"é²½":121924,"é³Ģ":121925,"é³ĥ":121926,"é³ħ":121927,"é³ĩ":121928,"é³Ĭ":121929,"èŀ½":121930,"çĩ®":121931,"鹫":121932,"ç³ľ":121933,"縻":121934,"çĻį":121935,"éºĭ":121936,"æĩij":121937,"æ¿¡":121938,"æ¿®":121939,"æ¿ŀ":121940,"æ¿ł":121941,"濯":121942,"è¹ĩ":121943,"è¬ĩ":121944,"éĤĥ":121945,"è¥ģ":121946,"æªĹ":121947,"æĵĺ":121948,"åŃº":121949,"éļ³":121950,"嬷":121951,"èŁĬ":121952,"鹬":121953,"éįª":121954,"éıĬ":121955,"é¬Ī":121956,"é¬ĥ":121957,"çŀ½":121958,"éŀ¯":121959,"éŀ¨":121960,"éŀ«":121961,"éŀ§":121962,"éŀ£":121963,"èĹľ":121964,"èĹł":121965,"éĨª":121966,"è¹Ļ":121967,"ç¤ĵ":121968,"çĩ¹":121969,"餮":121970,"çŀ¿":121971,"æĽĽ":121972,"颢":121973,"èºĩ":121974,"è¹ļ":121975,"èŁĽ":121976,"èŁª":121977,"èŁł":121978,"èŁ®":121979,"é¹®":121980,"黳":121981,"黣":121982,"é«ħ":121983,"é«Ĥ":121984,"éķ¬":121985,"éķŃ":121986,"éķ¯":121987,"馥":121988,"ç°Ł":121989,"ç°ª":121990,"鼬":121991,"鼳":121992,"èīŁ":121993,"é³İ":121994,"é³ı":121995,"é³IJ":121996,"çĻŀ":121997,"çĻĶ":121998,"糨":121999,"蹩":122000,"éİı":122001,"éĤĭ":122002,"é¬ı":122003,"æĶī":122004,"éŀ²":122005,"éŀ´":122006,"èĹ¿":122007,"èĺ§":122008,"èĺħ":122009,"éĨ®":122010,"éĨ¯":122011,"éħĥ":122012,"éľª":122013,"éľŃ":122014,"龨":122015,"黼":122016,"åļ¯":122017,"è¹°":122018,"蹶":122019,"è¹½":122020,"è¹¼":122021,"è¹´":122022,"è¹¾":122023,"蹿":122024,"èłĸ":122025,"èłĵ":122026,"èŁ¾":122027,"èłĬ":122028,"黢":122029,"é«ĭ":122030,"é«Į":122031,"éķ²":122032,"ç±Ģ":122033,"é½ģ":122034,"éŃij":122035,"èī¨":122036,"é³ĵ":122037,"é³Ķ":122038,"é³ķ":122039,"é³Ĺ":122040,"é³Ļ":122041,"éıĸ":122042,"羸":122043,"ã¸Ĩ":122044,"çĢ£":122045,"çĢĽ":122046,"襦":122047,"è°¶":122048,"è¥ŀ":122049,"骥":122050,"ç¼µ":122051,"çĵĴ":122052,"æĶĺ":122053,"èĺ©":122054,"èĺĸ":122055,"éĨ´":122056,"éľ°":122057,"éħĨ":122058,"çŁį":122059,"èºħ":122060,"é¼į":122061,"å·ī":122062,"黩":122063,"黥":122064,"黪":122065,"éķ³":122066,"éķ´":122067,"黧":122068,"çºĤ":122069,"çĴº":122070,"鼯":122071,"èĩľ":122072,"é³ľ":122073,"é³Ŀ":122074,"é³Ł":122075,"çį¾":122076,"åŃĢ":122077,"骧":122078,"çĵĺ":122079,"é¼Ļ":122080,"éĨº":122081,"礴":122082,"颦":122083,"æĽ©":122084,"é³¢":122085,"éºĿ":122086,"å¤Ķ":122087,"çĪĿ":122088,"çģı":122089,"禳":122090,"éIJ¾":122091,"ç¾¼":122092,"èł¡":122093,"èĢ±":122094,"é¹³":122095,"æ°į":122096,"é¥ķ":122097,"èºIJ":122098,"é«ij":122099,"éķµ":122100,"ç©°":122101,"é¥Ķ":122102,"鬻":122103,"鬣":122104,"趱":122105,"æĶ«":122106,"æĶ¥":122107,"颧":122108,"èºľ":122109,"é¼¹":122110,"çĻ¯":122111,"èł²":122112,"èł¹":122113,"èºŀ":122114,"è¡¢":122115,"çģŀ":122116,"襻":122117,"çºĽ":122118,"鬣":122119,"æĶ®":122120,"åĽĶ":122121,"é¦ķ":122122,"æĪĨ":122123,"çĪ¨":122124,"é½ī":122125,"äºį":122126,"å°¢":122127,"å½³":122128,"åį¬":122129,"殳":122130,"ðłĻ¶":122131,"æ¯Į":122132,"éĤĺ":122133,"æĪĭ":122134,"åľ¢":122135,"æ°ķ":122136,"ä¼ĭ":122137,"ä»Ŀ":122138,"åĨ®":122139,"æ°¿":122140,"æ±Ī":122141,"æ°¾":122142,"å¿ī":122143,"å®Ħ":122144,"ð¬£Ļ":122145,"è®±":122146,"æīŀ":122147,"åľ²":122148,"åľ«":122149,"èĬı":122150,"èĬĥ":122151,"æľ³":122152,"æľ¸":122153,"ð¨Ļ":122154,"ð¨Ļ¸":122155,"éĤ¨":122156,"åIJĴ":122157,"åIJĸ":122158,"å±¼":122159,"å±¾":122160,"辿":122161,"éĴĨ":122162,"仳":122163,"ä¼£":122164,"ä¼Ī":122165,"çĻ¿":122166,"çĶª":122167,"éĤł":122168,"çĬ´":122169,"åĨ±":122170,"éĤ¡":122171,"ð¬ĩķ":122172,"æ±ĭ":122173,"äľ":122174,"äľ£":122175,"è®»":122176,"ð¬£ŀ":122177,"åŃĸ":122178,"ð¬ĺĵ":122179,"纩":122180,"çİĴ":122181,"çİĵ":122182,"çİĺ":122183,"çİļ":122184,"åĪ¬":122185,"ð«ŃŁ":122186,"åĿľ":122187,"åĿī":122188,"æī½":122189,"ð«Ń¢":122190,"åĿĭ":122191,"æīº":122192,"ã§ij":122193,"æ¯IJ":122194,"èĬ°":122195,"èĬ£":122196,"èĭĬ":122197,"èĭī":122198,"èĬĺ":122199,"èĬ´":122200,"èĬł":122201,"ð«ĩ":122202,"ð«ĩŃ":122203,"èĬ¤":122204,"æĿķ":122205,"æĿĻ":122206,"æĿĦ":122207,"æĿ§":122208,"æĿ©":122209,"å°ª":122210,"å°¨":122211,"轪":122212,"ð«IJĦ":122213,"åĿĴ":122214,"èĬĪ":122215,"æĹ´":122216,"æŵ":122217,"åijĻ":122218,"ãķ":122219,"ãķ®":122220,"å²į":122221,"ð«µ":122222,"𫵷":122223,"å²ł":122224,"å²ľ":122225,"åijĩ":122226,"åĨı":122227,"è§ĥ":122228,"å²Ļ":122229,"ä¼¾":122230,"ãijĩ":122231,"ä¼Ń":122232,"ä½ĸ":122233,"ä¼²":122234,"ä½ģ":122235,"é£ı":122236,"çĭĥ":122237,"éŶ":122238,"汧":122239,"汫":122240,"ð£²ĺ":122241,"ð£²Ĺ":122242,"æ²Ħ":122243,"æ²ĺ":122244,"ð¬ĩĻ":122245,"æ±Ń":122246,"ã³ĩ":122247,"æ²ĩ":122248,"å¿®":122249,"忳":122250,"忺":122251,"𬣡":122252,"ç¥ĥ":122253,"è¯ĩ":122254,"éĤ²":122255,"è¯İ":122256,"è¯IJ":122257,"å±ĥ":122258,"ð«¸":122259,"𫸩":122260,"å²Ĭ":122261,"éĺ½":122262,"䢺":122263,"éĺ¼":122264,"妧":122265,"å¦ĺ":122266,"ð¨ļ":122267,"ð¨ļķ":122268,"纮":122269,"驲":122270,"ð«ĺľ":122271,"纻":122272,"ð¬ĺĺ":122273,"ð«ĺĿ":122274,"纼":122275,"çݤ":122276,"çİŀ":122277,"çݱ":122278,"çİŁ":122279,"éĤ½":122280,"éĤ¿":122281,"åĿ¥":122282,"åĿ°":122283,"åĿ¬":122284,"åĿ½":122285,"å¼Ĩ":122286,"èĢµ":122287,"䢼":122288,"ð¦Ń":122289,"ð¦Ńľ":122290,"èĮĭ":122291,"èĭ§":122292,"èĭ¾":122293,"èĭł":122294,"æŀħ":122295,"ãŃİ":122296,"æŀĺ":122297,"æŀį":122298,"çŁ¼":122299,"磻":122300,"åĮ¼":122301,"ð¬¨Ĥ":122302,"ð¬Ģ©":122303,"ð¬Ģª":122304,"æĹ¿":122305,"æĺĦ":122306,"æĺĴ":122307,"æĺĪ":122308,"åĴī":122309,"åĴĩ":122310,"åĴį":122311,"å²µ":122312,"å²½":122313,"岨":122314,"å²ŀ":122315,"å³Ĥ":122316,"ãŁ":122317,"ãŁĥ":122318,"åĽ·":122319,"𬬩":122320,"éĴIJ":122321,"éĴĶ":122322,"éĴĸ":122323,"çī¥":122324,"ä½´":122325,"åŀĪ":122326,"ä¾ģ":122327,"ä¾¹":122328,"佸":122329,"佺":122330,"éļ¹":122331,"ãijĬ":122332,"ä¾Ĥ":122333,"ä½½":122334,"ä¾ĺ":122335,"éĥĪ":122336,"èĪł":122337,"éĥIJ":122338,"éĥĥ":122339,"æĶ½":122340,"èĤŃ":122341,"èĤ¸":122342,"èĤ·":122343,"çĭī":122344,"çĭĿ":122345,"饳":122346,"å¿ŀ":122347,"çĤĮ":122348,"çĤĨ":122349,"æ³Ļ":122350,"沺":122351,"æ³Ĥ":122352,"æ³ľ":122353,"æ³ĥ":122354,"æ³ĩ":122355,"æĢĬ":122356,"å³ĥ":122357,"穸":122358,"ç¥ĭ":122359,"ç¥Ĭ":122360,"ð«į£":122361,"𬣳":122362,"𬩽":122363,"鸤":122364,"å¼¢":122365,"弨":122366,"éĻij":122367,"𬮿":122368,"éĻİ":122369,"ð¬¯Ģ":122370,"åįº":122371,"乸":122372,"å¦Ń":122373,"å§Ī":122374,"ð«°":122375,"ð«°Ľ":122376,"迳":122377,"åıķ":122378,"𬳵":122379,"驵":122380,"𬳶":122381,"äĮ":122382,"äĮ¹":122383,"驺":122384,"ð«łĬ":122385,"ç»ĭ":122386,"ç»IJ":122387,"çłī":122388,"èĢĶ":122389,"ãĽĥ":122390,"çݶ":122391,"çıĩ":122392,"çıħ":122393,"ð¬įĽ":122394,"çıĭ":122395,"çݹ":122396,"çıĮ":122397,"çİ¿":122398,"飨":122399,"åŀļ":122400,"åŀ¯":122401,"åŀĻ":122402,"åŀ²":122403,"åŁı":122404,"åŀį":122405,"èĢĩ":122406,"é¿į":122407,"åŀİ":122408,"åŀ´":122409,"åŀŁ":122410,"åŀŀ":122411,"æĮĵ":122412,"åŀµ":122413,"åŀı":122414,"æĭ¶":122415,"èįĸ":122416,"èįģ":122417,"èįĻ":122418,"èįĽ":122419,"èĮĪ":122420,"èĮ½":122421,"èįĦ":122422,"èĮº":122423,"ð¬ľ¬":122424,"èįĵ":122425,"èĮ³":122426,"ð¦°":122427,"𦰡":122428,"èĮĽ":122429,"èįŃ":122430,"ãŃķ":122431,"æŁ·":122432,"æŁĥ":122433,"æŁĬ":122434,"æŀ¹":122435,"æłIJ":122436,"æŁĸ":122437,"éĥļ":122438,"åīħ":122439,"ä´ĵ":122440,"迺":122441,"åİĸ":122442,"çłĨ":122443,"çłij":122444,"çłĦ":122445,"èĢı":122446,"å¥ĵ":122447,"ä¶":122448,"䶮":122449,"è½µ":122450,"è½·":122451,"è½¹":122452,"轺":122453,"æĺº":122454,"ðª¾":122455,"𪾢":122456,"æĺ½":122457,"缷":122458,"åĴ¡":122459,"åĴº":122460,"æĺ³":122461,"æĺ£":122462,"æĺ¤":122463,"æĺ«":122464,"æĺ¡":122465,"åĴ¥":122466,"æĺª":122467,"èĻ·":122468,"èĻ¸":122469,"åĵĥ":122470,"å³ĺ":122471,"èĢij":122472,"å³Ľ":122473,"𪨰":122474,"å³Ĺ":122475,"峧":122476,"帡":122477,"éĴĺ":122478,"ð«ĵ§":122479,"éĴľ":122480,"𬬮":122481,"𬬱":122482,"ð¬¬Ń":122483,"éĴª":122484,"éĴ¬":122485,"éĴŃ":122486,"磧":122487,"秬":122488,"ä¿«":122489,"èĪģ":122490,"ä¿ľ":122491,"ä¿Ļ":122492,"ä¿į":122493,"åŀķ":122494,"è¡İ":122495,"èĪ£":122496,"å¼ĩ":122497,"ä¾´":122498,"鸧":122499,"äı¡":122500,"èĥł":122501,"ð¦Ļ¶":122502,"èĥĪ":122503,"èĥ©":122504,"èĥ£":122505,"æľı":122506,"é£IJ":122507,"è¨Ħ":122508,"饻":122509,"庤":122510,"çĸ¢":122511,"çĤ£":122512,"çĤŁ":122513,"ã¶":122514,"㶲":122515,"æ´Ń":122516,"æ´ĺ":122517,"æ´ĵ":122518,"æ´¿":122519,"ã³ļ":122520,"æ³ļ":122521,"æµĪ":122522,"æµī":122523,"æ´¸":122524,"æ´ij":122525,"æ´¢":122526,"æ´Ī":122527,"æ´ļ":122528,"æ´º":122529,"æ´¨":122530,"æµIJ":122531,"ã³ĺ":122532,"æ´´":122533,"æ´£":122534,"æģĶ":122535,"宬":122536,"çªĢ":122537,"æīĤ":122538,"è¢Ĩ":122539,"ç¥ı":122540,"ç¥IJ":122541,"ç¥ķ":122542,"åıļ":122543,"éĻ§":122544,"éĻŀ":122545,"å¨Ģ":122546,"å§ŀ":122547,"姱":122548,"姤":122549,"姶":122550,"姽":122551,"æŀ²":122552,"ç»ĸ":122553,"éªĥ":122554,"ð¬ĺ¡":122555,"𬳽":122556,"ð¬ĺ©":122557,"ð«Ħ§":122558,"å½ĸ":122559,"éªī":122560,"æģĿ":122561,"çıª":122562,"çıĽ":122563,"çı¹":122564,"çIJĬ":122565,"çݼ":122566,"çıĸ":122567,"ðªŁ":122568,"ðªŁĿ":122569,"çı½":122570,"çı¦":122571,"çı«":122572,"çıĴ":122573,"ð¬į¤":122574,"çı¢":122575,"çıķ":122576,"çıĿ":122577,"ð«Ń¼":122578,"åŁĹ":122579,"åŀ¾":122580,"åŀº":122581,"åŁĨ":122582,"åŀ¿":122583,"åŁĮ":122584,"åŁĩ":122585,"èİ°":122586,"èĮĿ":122587,"ð¬ľ¯":122588,"éĦĢ":122589,"èݶ":122590,"èİĿ":122591,"äĵĸ":122592,"èİĻ":122593,"æł»":122594,"æ¡ł":122595,"ð¬Ĥ":122596,"ð¬Ĥ©":122597,"æ¡Ħ":122598,"æ¢ł":122599,"æł´":122600,"梴":122601,"æłĴ":122602,"éħİ":122603,"éħı":122604,"ð«łĨ":122605,"çłµ":122606,"çłł":122607,"çł«":122608,"糬":122609,"ç¡ģ":122610,"æģ§":122611,"ç¿ĥ":122612,"éĥª":122613,"ð¨IJ":122614,"ð¨IJĪ":122615,"è¾Ģ":122616,"è¾ģ":122617,"ð¬Į":122618,"ð¬ĮĹ":122619,"åīķ":122620,"èµĢ":122621,"åĵ¢":122622,"æĻħ":122623,"æĻĬ":122624,"åĶĿ":122625,"åĵ³":122626,"åĵ±":122627,"åĨĶ":122628,"æĻĶ":122629,"æĻIJ":122630,"çķĸ":122631,"èļĦ":122632,"èļĨ":122633,"ð«ij":122634,"ð«ij¡":122635,"帱":122636,"å´ģ":122637,"峿":122638,"𪨶":122639,"å´Ħ":122640,"帨":122641,"å´Ģ":122642,"èµĨ":122643,"𬬸":122644,"éĴ·":122645,"𬬻":122646,"𬬹":122647,"𬬿":122648,"ð¬Ńģ":122649,"çľļ":122650,"çĶ¡":122651,"笫":122652,"åĢ»":122653,"åĢ´":122654,"èĦ©":122655,"åĢ®":122656,"åĢķ":122657,"åĢŀ":122658,"ð«¢":122659,"𫢸":122660,"åĢĵ":122661,"åĢ§":122662,"è¡ĥ":122663,"èĻĴ":122664,"èĪŃ":122665,"èĪ¯":122666,"èĪ¥":122667,"çĵŀ":122668,"鬯":122669,"鸰":122670,"èĦİ":122671,"æľĵ":122672,"èĥ²":122673,"èĻĵ":122674,"é±½":122675,"çĭ´":122676,"å³±":122677,"çĭ»":122678,"çľ¢":122679,"ð«Ĺ§":122680,"åĭį":122681,"çĹĦ":122682,"çĸ°":122683,"çĹĥ":122684,"ç«ĺ":122685,"ç¾ĸ":122686,"ç¾ĵ":122687,"æ¡Ĭ":122688,"æķī":122689,"çĥł":122690,"çĥĶ":122691,"çĥ¶":122692,"çĥ»":122693,"ð¬ĬĪ":122694,"æ¶į":122695,"浡":122696,"æµŃ":122697,"浬":122698,"æ¶Ħ":122699,"涢":122700,"æ¶IJ":122701,"æµ°":122702,"æµŁ":122703,"æµĽ":122704,"æµ¼":122705,"æµ²":122706,"æ¶ĺ":122707,"æĤĪ":122708,"æĤĥ":122709,"æĤ¢":122710,"ð¬ĴĪ":122711,"宧":122712,"çªħ":122713,"çªĬ":122714,"çªİ":122715,"æīħ":122716,"æīĨ":122717,"袪":122718,"è¢Ĺ":122719,"袯":122720,"祧":122721,"éļº":122722,"åł²":122723,"çĸį":122724,"ð¨º":122725,"ð¨ºĻ":122726,"éĻ´":122727,"çĥĿ":122728,"çł®":122729,"ãĽļ":122730,"åĵ¿":122731,"ç¿Ģ":122732,"ç¿Ĥ":122733,"åīŁ":122734,"𬳿":122735,"ð«Ħ¨":122736,"绤":122737,"éªį":122738,"ð¬ĺ«":122739,"äĤ":122740,"äĤ®":122741,"çIJİ":122742,"çı¸":122743,"çıµ":122744,"çIJĦ":122745,"çIJĪ":122746,"çIJĢ":122747,"çıº":122748,"æİŃ":122749,"åłİ":122750,"åłIJ":122751,"åŁ¼":122752,"æİİ":122753,"åŁ«":122754,"åłĮ":122755,"æĻ¢":122756,"ð«®":122757,"ð«®ĥ":122758,"æİŀ":122759,"åŁª":122760,"壸":122761,"ãĻį":122762,"èģį":122763,"èıĿ":122764,"èIJļ":122765,"èı¥":122766,"èİ¿":122767,"äĵ«":122768,"åĭļ":122769,"äĵ¬":122770,"èIJĨ":122771,"èıĤ":122772,"èıį":122773,"èı¼":122774,"èIJ£":122775,"äĵ¨":122776,"èıī":122777,"äĵĽ":122778,"梼":122779,"梽":122780,"桲":122781,"梾":122782,"桯":122783,"梣":122784,"æ¢Į":122785,"桹":122786,"æķĶ":122787,"åİ£":122788,"ç¡Ķ":122789,"é¿İ":122790,"ç¡Ļ":122791,"ç¡ļ":122792,"ç¡Ĭ":122793,"ç¡į":122794,"åĭĶ":122795,"ä´ķ":122796,"é¾ģ":122797,"éĢ´":122798,"åĶª":122799,"åķ«":122800,"ç¿Ī":122801,"ã«":122802,"ã«°":122803,"æĻĻ":122804,"çķ¤":122805,"ð¬±ĸ":122806,"趼":122807,"è·Ĥ":122808,"èĽĥ":122809,"èļ²":122810,"ð¬Ł½":122811,"èļº":122812,"åķ´":122813,"äİĥ":122814,"å´§":122815,"å´Ł":122816,"å´ŀ":122817,"å´Ĵ":122818,"å´Į":122819,"å´¡":122820,"éĵı":122821,"ð«ĵ¯":122822,"ð«Ł¹":122823,"éĵķ":122824,"ð«Ł¼":122825,"éĵĸ":122826,"éĵĺ":122827,"éĵļ":122828,"éĵŀ":122829,"éĵ¥":122830,"éĵ´":122831,"çī»":122832,"çī¿":122833,"ç¨Ĩ":122834,"笱":122835,"笯":122836,"åģ°":122837,"åģ¡":122838,"鸺":122839,"åģŃ":122840,"åģ²":122841,"åģģ":122842,"ã¿":122843,"ã¿ł":122844,"éĦħ":122845,"åģĵ":122846,"å¾Ľ":122847,"è¡Ĵ":122848,"èĪ³":122849,"èĪ²":122850,"鸼":122851,"æĤĨ":122852,"éĦĥ":122853,"çĵ»":122854,"äĿ":122855,"äĿĻ":122856,"èĦ¶":122857,"èĦŀ":122858,"èĦŁ":122859,"äı²":122860,"é±¾":122861,"çĮĩ":122862,"çĮĬ":122863,"çĮĦ":122864,"è§ĸ":122865,"ðłħ":122866,"ðłħ¤":122867,"庱":122868,"庼":122869,"庳":122870,"çĹĵ":122871,"ä´Ķ":122872,"ç««":122873,"åłĥ":122874,"éĺĮ":122875,"ç¾Ŀ":122876,"ç¾ķ":122877,"çĦĨ":122878,"çĥº":122879,"çĦĮ":122880,"æ·ı":122881,"ð¬ĩ¹":122882,"æ·Ł":122883,"æ·ľ":122884,"æ·´":122885,"æ·¯":122886,"æ¹´":122887,"涴":122888,"ð¬į¡":122889,"ã¥":122890,"ã¥Ħ":122891,"æĥĽ":122892,"æĥĶ":122893,"æĤ°":122894,"æĥĻ":122895,"å¯ģ":122896,"éĢŃ":122897,"ð¬¤ĩ":122898,"ð«į¯":122899,"袼":122900,"è£Ī":122901,"祲":122902,"ð¬¤Ĭ":122903,"ð«į²":122904,"è°ŀ":122905,"èī´":122906,"弸":122907,"弶":122908,"ð¬¯İ":122909,"éļĥ":122910,"å©ŀ":122911,"娵":122912,"婼":122913,"åªĸ":122914,"婳":122915,"å©į":122916,"å©Į":122917,"å©«":122918,"婤":122919,"å©ĺ":122920,"å©ł":122921,"ð¬ĺ¬":122922,"ð¬ĺŃ":122923,"ð¬´Ĥ":122924,"ð«ĺ¦":122925,"绹":122926,"ð«Łħ":122927,"ð¬ĺ¯":122928,"éªķ":122929,"ð«ĺ§":122930,"絾":122931,"çı·":122932,"çIJ²":122933,"çIJ¡":122934,"çIJŁ":122935,"çIJĶ":122936,"çIJŃ":122937,"åł¾":122938,"åł¼":122939,"æıķ":122940,"ãĻĺ":122941,"åł§":122942,"åĸĨ":122943,"åł¨":122944,"å¡ħ":122945,"åłł":122946,"çµ·":122947,"ðª£":122948,"𪣻":122949,"ð¡İ":122950,"ð¡İļ":122951,"èijľ":122952,"æĥİ":122953,"èIJ³":122954,"èijĻ":122955,"éĿ¬":122956,"èij´":122957,"èĴĩ":122958,"èĴĪ":122959,"éĦļ":122960,"èĴī":122961,"èĵĩ":122962,"èIJ©":122963,"èij°":122964,"èijİ":122965,"éĦij":122966,"èĴİ":122967,"èijĸ":122968,"èĴĦ":122969,"èIJ¹":122970,"棤":122971,"棽":122972,"棫":122973,"æ¤ĵ":122974,"æ¤ij":122975,"ð¬ĥ":122976,"ð¬ĥĬ":122977,"é¹Ģ":122978,"æ¤Ĩ":122979,"æ£ĵ":122980,"棬":122981,"棪":122982,"æ¤Ģ":122983,"æ¥Ĺ":122984,"ð¬·":122985,"ð¬·ķ":122986,"çĶ¦":122987,"éħ¦":122988,"è§Į":122989,"奡":122990,"çļķ":122991,"硪":122992,"欹":122993,"è©Ł":122994,"ð«IJIJ":122995,"è¾Į":122996,"æ£IJ":122997,"é¾Ĥ":122998,"ð¬¹":122999,"𬹼":123000,"黹":123001,"çīļ":123002,"çĿİ":123003,"æĻ«":123004,"æĻª":123005,"æĻ±":123006,"ð§":123007,"ð§¿":123008,"𧿹":123009,"èĽij":123010,"çķ¯":123011,"æĸĿ":123012,"åĸ¤":123013,"å´¶":123014,"åµģ":123015,"ð«¶":123016,"ð«¶ĩ":123017,"å´¾":123018,"åµħ":123019,"å´¿":123020,"åµļ":123021,"ç¿Ļ":123022,"ð«ĸ®":123023,"åľĮ":123024,"åľIJ":123025,"èµij":123026,"èµĴ":123027,"é¿ı":123028,"éĵ¹":123029,"ð¬ŃĬ":123030,"éĵ½":123031,"ð¨±ĩ":123032,"ð«ĵ¶":123033,"éĶĬ":123034,"éĶį":123035,"éĶİ":123036,"ð¬Ńİ":123037,"éĶĵ":123038,"çĬĩ":123039,"é¢ĭ":123040,"ç¨Į":123041,"çŃĢ":123042,"çŃĺ":123043,"çŃľ":123044,"çŃ¥":123045,"çŃħ":123046,"åĤĥ":123047,"åĤī":123048,"ç¿Ľ":123049,"åĤĴ":123050,"åĤķ":123051,"èĪ¾":123052,"çķ¬":123053,"ð«ĸ¯":123054,"èĦ¿":123055,"èħĺ":123056,"äIJ":123057,"äIJĥ":123058,"èħĻ":123059,"èħĴ":123060,"ð¬±Ł":123061,"é²ĥ":123062,"çĮ°":123063,"ð«Ľ":123064,"ð«ĽŃ":123065,"çĮ¯":123066,"ãº":123067,"ãºĦ":123068,"é¦ī":123069,"åĩĵ":123070,"éĦĹ":123071,"ð«·":123072,"ð«··":123073,"å»ĭ":123074,"å»Ĩ":123075,"éĦĮ":123076,"ç²¢":123077,"éģĨ":123078,"æĹIJ":123079,"𬮱":123080,"çĦŀ":123081,"ð¬Ĭ¤":123082,"欻":123083,"ð£¸":123084,"𣸣":123085,"æºļ":123086,"æºģ":123087,"æ¹Ŀ":123088,"渰":123089,"æ¹ĵ":123090,"ã´":123091,"ã´Ķ":123092,"æ¸Ł":123093,"æºł":123094,"渼":123095,"æºĩ":123096,"æ¹£":123097,"æ¹ij":123098,"æºŀ":123099,"æĦIJ":123100,"æĦĥ":123101,"æķ©":123102,"çĶ¯":123103,"棨":123104,"æīĬ":123105,"裣":123106,"祼":123107,"å©»":123108,"åªĨ":123109,"åªŀ":123110,"ãĽ¹":123111,"åªĵ":123112,"åªĤ":123113,"åªĦ":123114,"毵":123115,"çŁŀ":123116,"ð¬´ĥ":123117,"ð«ĺ¨":123118,"ç¼Ĭ":123119,"ç¼IJ":123120,"éªĻ":123121,"çijĥ":123122,"çijĵ":123123,"çijħ":123124,"çijĨ":123125,"ä´ĸ":123126,"çijĸ":123127,"çijĿ":123128,"çijĶ":123129,"çijĢ":123130,"ð¤§":123131,"ð¤§Ľ":123132,"çij³":123133,"çijĤ":123134,"å¶ħ":123135,"çijij":123136,"éģĺ":123137,"é«¢":123138,"å¡¥":123139,"åł½":123140,"赪":123141,"æijĽ":123142,"å¡Ŀ":123143,"æIJĴ":123144,"æIJĮ":123145,"èĴ±":123146,"èĴ¨":123147,"èĵı":123148,"èĶĢ":123149,"èĵ¢":123150,"èĵĤ":123151,"èĴ»":123152,"èĵ£":123153,"椹":123154,"楪":123155,"æ¦ĥ":123156,"æ¦ħ":123157,"æ¥Ĵ":123158,"楩":123159,"æ¦ĩ":123160,"椸":123161,"æ¥Ļ":123162,"æŃħ":123163,"ð¬ª":123164,"𬪩":123165,"ç¢ĥ":123166,"ç¢ı":123167,"ð¬ĴĶ":123168,"ç¢Ī":123169,"äĥħ":123170,"ç¡¿":123171,"éĦł":123172,"è¾Ĵ":123173,"ð¬¨İ":123174,"ð«IJĵ":123175,"é¾Ĩ":123176,"è§ľ":123177,"ä£":123178,"ä£ĺ":123179,"æļķ":123180,"é¹į":123181,"ð««":123182,"ð««ĩ":123183,"ã¬Ĭ":123184,"æļħ":123185,"è·±":123186,"èľIJ":123187,"èľİ":123188,"åµ²":123189,"èµĹ":123190,"骱":123191,"éĶĸ":123192,"ð«ĵ¹":123193,"éĶĺ":123194,"éĶ³":123195,"éĶ§":123196,"éĶª":123197,"ð¬Ńļ":123198,"éĶ«":123199,"éĶ¬":123200,"ð¬ŃĽ":123201,"ç¨ij":123202,"ç¨Ļ":123203,"äħ":123204,"äħŁ":123205,"ð¬ķ":123206,"ð¬ķĤ":123207,"çŃ»":123208,"çŃ¼":123209,"çѶ":123210,"çѦ":123211,"çѤ":123212,"åĤº":123213,"é¹İ":123214,"åĥĩ":123215,"èīħ":123216,"èīī":123217,"è°¼":123218,"è²Ĩ":123219,"èħ½":123220,"èħ¨":123221,"èħ¯":123222,"é²ī":123223,"é²Ĭ":123224,"é²Į":123225,"ä²Ł":123226,"ð¬¶ĭ":123227,"ð¬¶į":123228,"é²ı":123229,"éĽĬ":123230,"çĮº":123231,"é£Ķ":123232,"è§Ł":123233,"ð¦Ŀ¼":123234,"é¦Į":123235,"è£Ľ":123236,"å»Ĵ":123237,"çĺħ":123238,"éĦĺ":123239,"é¹Ĵ":123240,"éĦľ":123241,"éºĢ":123242,"éĦ£":123243,"éĺĺ":123244,"ð«Ķ¶":123245,"çħģ":123246,"çħĥ":123247,"çħ´":123248,"çħĭ":123249,"çħŁ":123250,"çħĵ":123251,"æ»ł":123252,"æºį":123253,"溹":123254,"æ»Ĩ":123255,"æ»ī":123256,"溦":123257,"溵":123258,"æ¼·":123259,"滧":123260,"æ»ĺ":123261,"æ»į":123262,"æĦŃ":123263,"æħ¥":123264,"æħĨ":123265,"塱":123266,"ð«ĮĢ":123267,"裼":123268,"ç¦ĭ":123269,"ç¦Ķ":123270,"ç¦ĺ":123271,"ç¦Ĵ":123272,"è°«":123273,"é¹Ķ":123274,"ð«ĸ³":123275,"æĦį":123276,"å«Ħ":123277,"媱":123278,"æĪ¤":123279,"åĭł":123280,"æĪ£":123281,"ð«ĺª":123282,"ð«ĺ¬":123283,"ç¼ŀ":123284,"èĢ¤":123285,"çij§":123286,"ð«ŀ":123287,"ð«ŀ©":123288,"çij¨":123289,"çij±":123290,"çij·":123291,"çij¢":123292,"æĸł":123293,"æijı":123294,"å¢ķ":123295,"å¢Ī":123296,"å¢IJ":123297,"å¢ĺ":123298,"æij´":123299,"éĬİ":123300,"ð¡IJ":123301,"ð¡IJĵ":123302,"å¢ļ":123303,"æĴĸ":123304,"ðª¤":123305,"ðª¤Ĺ":123306,"éĿ½":123307,"éŀģ":123308,"èĶĮ":123309,"èĶĪ":123310,"èĵ°":123311,"èĶ¹":123312,"èĶĬ":123313,"åĺı":123314,"榰":123315,"æ¦ij":123316,"æ§ļ":123317,"ð£Ĺ":123318,"ð£Ĺĭ":123319,"æ§ľ":123320,"æ¦į":123321,"çĸIJ":123322,"ð¬¸ĺ":123323,"éħº":123324,"éħ¾":123325,"éħ²":123326,"éħ´":123327,"碶":123328,"äĥİ":123329,"ð¬ĴĹ":123330,"碨":123331,"ð¥Ķ":123332,"ð¥Ķ²":123333,"碹":123334,"碥":123335,"åĬĤ":123336,"ð«ļĸ":123337,"ä´Ĺ":123338,"夥":123339,"çŀį":123340,"é¹ĸ":123341,"ã¬İ":123342,"è·½":123343,"èľ¾":123344,"å¹ĸ":123345,"å¶į":123346,"åľĻ":123347,"ð¨±ı":123348,"éĶº":123349,"éĶ¼":123350,"éĶ½":123351,"ð¬Ń¤":123352,"éĶ¾":123353,"éĶ¿":123354,"éķĥ":123355,"éķĦ":123356,"éķħ":123357,"é¦Ŀ":123358,"é¹Ļ":123359,"箨":123360,"ç®ĸ":123361,"åĬĦ":123362,"åĥ¬":123363,"åĥ¦":123364,"åĥĶ":123365,"åĥİ":123366,"æ§ĥ":123367,"ãĻ¦":123368,"é²Ĵ":123369,"é²ķ":123370,"ð«ļķ":123371,"é²ĸ":123372,"é²Ĺ":123373,"é²ĺ":123374,"é²Ļ":123375,"ð¬¶IJ":123376,"ð¬¶ı":123377,"ð©½":123378,"𩽾":123379,"å¤IJ":123380,"çįį":123381,"é£Ĺ":123382,"ð¬¸ļ":123383,"åĩĺ":123384,"å»ij":123385,"å»Ļ":123386,"çĺĹ":123387,"çĺ¥":123388,"çĺķ":123389,"é²Ŀ":123390,"éĦ«":123391,"çĨĩ":123392,"æ¼¹":123393,"æ¼ĸ":123394,"æ½Ĩ":123395,"漤":123396,"潩":123397,"æ¼¼":123398,"æ¼´":123399,"ã½":123400,"ã½ı":123401,"æ¼Ī":123402,"æ¼ĭ":123403,"æ¼»":123404,"æħ¬":123405,"窬":123406,"çªŃ":123407,"ã®":123408,"㮾":123409,"ð¬¤Ŀ":123410,"è¤ķ":123411,"禼":123412,"ç¦ļ":123413,"éļ©":123414,"å«ķ":123415,"å«Ń":123416,"å«ľ":123417,"嫪":123418,"ð¬ĻĤ":123419,"ã»":123420,"㻬":123421,"麹":123422,"çĴĨ":123423,"漦":123424,"åıĩ":123425,"墣":123426,"墦":123427,"墡":123428,"åĬIJ":123429,"èĸģ":123430,"èķ°":123431,"èĶĥ":123432,"é¼Ĵ":123433,"槱":123434,"é¹Ŀ":123435,"ç£ı":123436,"ç£ī":123437,"殣":123438,"æħŃ":123439,"éľħ":123440,"æļµ":123441,"æļ²":123442,"æļ¶":123443,"踦":123444,"踣":123445,"äĹĸ":123446,"èĿĺ":123447,"èĿ²":123448,"èĿ¤":123449,"åĻĩ":123450,"åĻĤ":123451,"åĻĢ":123452,"罶":123453,"嶲":123454,"å¶ĵ":123455,"ãłĩ":123456,"å¶Ł":123457,"å¶Ĵ":123458,"éķĨ":123459,"éķĪ":123460,"éķĭ":123461,"éķİ":123462,"ð¬Ń©":123463,"éķķ":123464,"稹":123465,"åĦĩ":123466,"çļŀ":123467,"çļĽ":123468,"ä´ĺ":123469,"èīİ":123470,"èīı":123471,"é¹Ł":123472,"ð©¾ĥ":123473,"鲦":123474,"鲪":123475,"鲬":123476,"æ©¥":123477,"è§Ń":123478,"é¹ł":123479,"鹡":123480,"ç³ĩ":123481,"ç³Ī":123482,"翦":123483,"é¹¢":123484,"é¹£":123485,"çĨĽ":123486,"æ½ĸ":123487,"æ½µ":123488,"ãµ":123489,"ãµIJ":123490,"æ¾Ĥ":123491,"æ¾Ľ":123492,"çij¬":123493,"æ½½":123494,"æ½¾":123495,"æ½ı":123496,"æĨŃ":123497,"æĨķ":123498,"𬸣":123499,"æĪŃ":123500,"褯":123501,"禤":123502,"ð«į½":123503,"嫽":123504,"éģ¹":123505,"ð¬´Ĭ":123506,"çĴ¥":123507,"çĴ²":123508,"çĴĴ":123509,"æĨĻ":123510,"æĵIJ":123511,"éĦ¹":123512,"èĸ³":123513,"éŀĶ":123514,"é»ĩ":123515,"ð¬ŀ":123516,"ð¬ŀŁ":123517,"èķĹ":123518,"èĸ¢":123519,"èķ¹":123520,"æ©ŀ":123521,"æ©ij":123522,"橦":123523,"éĨij":123524,"觱":123525,"磡":123526,"ð¥ķ":123527,"ð¥ķ¢":123528,"ç£ľ":123529,"è±®":123530,"ð«Ł¦":123531,"ð¬ºĪ":123532,"ð«łľ":123533,"é¹¾":123534,"èĻ¤":123535,"æļ¿":123536,"æĽĮ":123537,"æĽĪ":123538,"ã¬ļ":123539,"è¹ħ":123540,"踶":123541,"äĹĽ":123542,"èŀĹ":123543,"çĸģ":123544,"ãłĵ":123545,"幪":123546,"ðª©":123547,"ðª©ĺ":123548,"嶦":123549,"ð¬Ń¬":123550,"ð¨±ij":123551,"ð¬Ń¯":123552,"é¦ŀ":123553,"ç©Ħ":123554,"ç¯ļ":123555,"篯":123556,"ç°ī":123557,"é¼½":123558,"è¡ł":123559,"缦":123560,"èŀ£":123561,"縢":123562,"é²Ń":123563,"鲯":123564,"é²°":123565,"鲺":123566,"é²¹":123567,"ð«Ĺ´":123568,"亸":123569,"çĻĢ":123570,"çĺŃ":123571,"𬸦":123572,"ç¾±":123573,"ç³Ĵ":123574,"çĩĭ":123575,"çĨ»":123576,"çĩĬ":123577,"çĩļ":123578,"çĩı":123579,"æ¿©":123580,"æ¿ĭ":123581,"澪":123582,"æ¾½":123583,"æ¾´":123584,"æ¾Ń":123585,"æ¾¼":123586,"æĨ·":123587,"æĨº":123588,"æĩĶ":123589,"é»ī":123590,"å¬Ľ":123591,"鹨":123592,"翯":123593,"ð«Ħ·":123594,"çĴ±":123595,"𤩽":123596,"çĴ¬":123597,"çĴ®":123598,"髽":123599,"æĵ¿":123600,"èĸ¿":123601,"èĸ¸":123602,"æªij":123603,"æ«Ĩ":123604,"æªŀ":123605,"éĨ¨":123606,"ç¹Ħ":123607,"磹":123608,"磻":123609,"çŀ«":123610,"çŀµ":123611,"è¹IJ":123612,"èŁı":123613,"ãĺ":123614,"ãĺİ":123615,"ð¬Ń³":123616,"éķ¤":123617,"ð¬Ń¶":123618,"ð«Ķį":123619,"éķ¥":123620,"éķ¨":123621,"ð¬Ń¸":123622,"ð¨±Ķ":123623,"ð¬Ń¼":123624,"ð«Ķİ":123625,"磰":123626,"ç©Ļ":123627,"ç©ľ":123628,"ç©Ł":123629,"ç°ķ":123630,"ç°ĥ":123631,"ç°ı":123632,"åĦ¦":123633,"éŃĭ":123634,"æĸ¶":123635,"èīļ":123636,"𬸪":123637,"è°¿":123638,"ä²ł":123639,"ð¬¶Ł":123640,"é²¾":123641,"ð¬¶ł":123642,"鲿":123643,"é³ģ":123644,"é³Ĥ":123645,"é³Ī":123646,"é³ī":123647,"çį¯":123648,"äĹª":123649,"é¦ĺ":123650,"è¥ķ":123651,"è¥ļ":123652,"𬶨":123653,"èŀ±":123654,"çĶĵ":123655,"嬬":123656,"嬥":123657,"ð¦Ī":123658,"ð¦Ī¡":123659,"ð«Ħ¸":123660,"çĵĢ":123661,"éĩIJ":123662,"鬶":123663,"çĪĩ":123664,"éŀ³":123665,"éŀ®":123666,"ð¬Łģ":123667,"èĹŁ":123668,"èŦ":123669,"èŨ":123670,"é¹²":123671,"檫":123672,"黡":123673,"ç¤ŀ":123674,"ç¤Į":123675,"ð¥ĸ":123676,"ð¥ĸ¨":123677,"è¹¢":123678,"è¹ľ":123679,"èŁ«":123680,"äĹ´":123681,"åļļ":123682,"é«ĥ":123683,"éķ®":123684,"éķ±":123685,"éħĤ":123686,"馧":123687,"ç°ł":123688,"ç°Ŀ":123689,"ç°°":123690,"鼫":123691,"鼩":123692,"çļ¦":123693,"èĩij":123694,"ä²¢":123695,"é³ij":123696,"é³Ĵ":123697,"é¹±":123698,"鹯":123699,"çĻĹ":123700,"ð¦Ĵ":123701,"ð¦Ĵį":123702,"æĹŀ":123703,"ç¿·":123704,"åĨģ":123705,"äİĸ":123706,"çĢĶ":123707,"çĢį":123708,"çĢĮ":123709,"è¥ľ":123710,"ä´Ļ":123711,"ð¬ĻĬ":123712,"åļŃ":123713,"ã°":123714,"ã°Ģ":123715,"鬷":123716,"éĨŃ":123717,"蹯":123718,"èłĭ":123719,"翾":123720,"é³ĺ":123721,"åĦ³":123722,"åĦ´":123723,"é¼Ĺ":123724,"ð¬¶Ń":123725,"ð©¾Į":123726,"é³ļ":123727,"é³Ľ":123728,"éºij":123729,"éºĸ":123730,"èłĥ":123731,"å½Ł":123732,"嬿":123733,"é¬Ĵ":123734,"èĺĺ":123735,"æ¬Ĥ":123736,"éĨµ":123737,"颥":123738,"çĶĹ":123739,"ð¨Ł":123740,"ð¨Łł":123741,"å·ĩ":123742,"éħħ":123743,"é«İ":123744,"çĬ¨":123745,"𬶮":123746,"ð¨Ń":123747,"ð¨Ńī":123748,"ã¸Į":123749,"çĪĶ":123750,"çĢ±":123751,"çĢ¹":123752,"çĢ¼":123753,"çĢµ":123754,"襫":123755,"åŃħ":123756,"骦":123757,"ð¬Ļĭ":123758,"èĢ°":123759,"ð¤«":123760,"ð¤«ī":123761,"çĵĸ":123762,"é¬ĺ":123763,"趯":123764,"ð¬ºĵ":123765,"ç½į":123766,"é¼±":123767,"é³ł":123768,"鳡":123769,"é³£":123770,"çĪŁ":123771,"çĪļ":123772,"çģĪ":123773,"éŁĤ":123774,"ç³µ":123775,"èĺ¼":123776,"礵":123777,"é¹´":123778,"èºĶ":123779,"çļŃ":123780,"é¾¢":123781,"鳤":123782,"亹":123783,"ç±¥":123784,"é¼·":123785,"ð«ļŃ":123786,"çİĥ":123787,"éĨ¾":123788,"é½ĩ":123789,"觿":123790,"èł¼":123791,"ק":123792,"פ":123793,"׼":123794,"×ķת":123795,"ס":123796,"×Ļ×Ŀ":123797,"צ":123798,"×Ĵ":123799,"×ĺ":123800,"×ķר":123801,"×Ŀ":123802,"×ķ׾":123803,"×ĸ":123804,"à¹Ĥ":123805,"ïº":123806,"ðŁį":123807,"ðŁIJ":123808,"×Ļר":123809,"ï»":123810,"ðŁij":123811,"ðĿIJ":123812,"ðŁı":123813,"ðŁĶ":123814,"ðŁĮ":123815,"ðŁİ":123816,"ðŁĵ":123817,"ף":123818,"ðĿij":123819,"×ķ×ĵ":123820,"ï¦":123821,"Ġ×ķ":123822,"×ķ×ij":123823,"à¸Ńà¸ĩ":123824,"ðĿĺ":123825,"×Ļת":123826,"ðĿķ":123827,"à¸Ĺีà¹Ī":123828,"ائ":123829,"ðŁ¤":123830,"×ķף":123831,"رÙĬ":123832,"×Ļ׾":123833,"ระ":123834,"าย":123835,"ï¯":123836,"ï®":123837,"าม":123838,"âĩ":123839,"ðŁ¥":123840,"ïŃ":123841,"ðĿĻ":123842,"×ķ׳":123843,"á½":123844,"Ġ׼":123845,"ðŁļ":123846,"âļ":123847,"ï§":123848,"×ijר":123849,"×Ļ׳":123850,"á´":123851,"Ġ×Ĺ":123852,"á¼":123853,"ðĿĹ":123854,"Ġ×¢":123855,"×Ļ×Ķ":123856,"ãģ£ãģŁ":123857,"ãģĵãģ¨":123858,"á¸":123859,"ÙĬÙĨ":123860,"ãģªãģĦ":123861,"اع":123862,"ศ":123863,"à¹Īà¸ĩ":123864,"×Ļ×ĵ":123865,"×ŀש":123866,"áĪ":123867,"׳×Ļ":123868,"×Ļ×ij":123869,"ï¥":123870,"ðĿĵ":123871,"Ġ×Ļ":123872,"×ļ":123873,"ัà¸ĩ":123874,"âĵ":123875,"ï¤":123876,"ĠاÙĦØ£":123877,"าà¸ģ":123878,"à¹īà¸Ļ":123879,"à¹Ģร":123880,"×ķ×Ŀ":123881,"á¹":123882,"ึ":123883,"×Ļק":123884,"à¸ĭ":123885,"à¸Ħร":123886,"à¸ĺ":123887,"ัà¸ģ":123888,"ðŁķ":123889,"ÙĪÙĨ":123890,"à¸Ńย":123891,"âĬ":123892,"ðĿĴ":123893,"ĠاÙĦع":123894,"าà¸Ļ":123895,"×Ļף":123896,"ÙĦÙĬ":123897,"×Ļש":123898,"à¸Ľà¸£à¸°":123899,"à¹Ģà¸Ľ":123900,"Ġ׳":123901,"×ķס":123902,"à¸ł":123903,"ÙħÙĨ":123904,"×ķ×¢":123905,"×ķ×ŀ":123906,"âĮ":123907,"ðŁ§":123908,"à¹ĩà¸Ļ":123909,"à¸į":123910,"ãİ":123911,"áµ":123912,"ĠاÙĦس":123913,"×ķק":123914,"หล":123915,"ðŁĩ":123916,"âı":123917,"ðŁ¦":123918,"Ġ×Ķ×ŀ":123919,"ÙĪا":123920,"Ġת":123921,"ר×IJ":123922,"à¸Ńà¸Ļ":123923,"ษ":123924,"à¹Īว":123925,"×ķצ":123926,"íĹ":123927,"ãĦ":123928,"ï¨":123929,"ï¹":123930,"âİ":123931,"ï²":123932,"ðĿļ":123933,"ðIJ":123934,"à¸Ħว":123935,"หà¸Ļ":123936,"Ġר":123937,"بÙĬ":123938,"รà¹Į":123939,"را":123940,"شر":123941,"×ķ×Ĺ":123942,"×ķפ":123943,"×ķש":123944,"×ķ×Ĵ":123945,"íĿ":123946,"âĽ":123947,"à¸ķิ":123948,"à¹Ģà¸ģ":123949,"ï³":123950,"ï±":123951,"à¸Ķà¹ī":123952,"ë¹":123953,"ï¬":123954,"á¿":123955,"ðŁĽ":123956,"ðĿĸ":123957,"à¹Īาà¸ĩ":123958,"ูà¹ī":123959,"Ġ×Ķ×IJ":123960,"ĠاÙĦØŃ":123961,"פר":123962,"ÙĪÙħ":123963,"à¹Ģล":123964,"íĸ":123965,"×Ļ×¢":123966,"ìĪ":123967,"íĵ":123968,"ðŁħ":123969,"áł":123970,"à¸Ħวาม":123971,"à¸Īะ":123972,"׳×Ķ":123973,"Ġק":123974,"à¸Ł":123975,"à¹īà¸ĩ":123976,"หม":123977,"تÙħ":123978,"׾×Ļ":123979,"ÙĬد":123980,"à¹Īà¸Ļ":123981,"×Ĺר":123982,"שר":123983,"à¹Ģà¸Ĺ":123984,"×ŀר":123985,"ëĸ":123986,"عÙĦ":123987,"×ŀ×¢":123988,"â²":123989,"׾×Ķ":123990,"Ġפ":123991,"à¸Ńà¸ģ":123992,"سÙĦ":123993,"×Ļ×ŀ":123994,"ÙĤÙĬ":123995,"íİ":123996,"تØŃ":123997,"×Ļס":123998,"×Ļ×Ĺ":123999,"íĽ":124000,"ï°":124001,"â½":124002,"áī":124003,"áĬ":124004,"á¨":124005,"Ùĩا":124006,"Ġ׾×Ķ":124007,"×ķ×IJ":124008,"Ùħا":124009,"à¹īà¸Ńà¸ĩ":124010,"رب":124011,"ĠاÙĦج":124012,"×ŀ×ĵ":124013,"ÙħÙĦ":124014,"تر":124015,"à¹Ģà¸Ķ":124016,"קר":124017,"íħ":124018,"ì¼":124019,"ê¿":124020,"ãĪ":124021,"áIJ":124022,"ðŁĹ":124023,"ê¦":124024,"áĭ":124025,"ðĿĶ":124026,"à¹Ģà¸Ľà¹ĩà¸Ļ":124027,"à¹ĥห":124028,"มา":124029,"วà¹Īา":124030,"มี":124031,"ีà¹ī":124032,"à¹Ħมà¹Ī":124033,"ÙĨÙĬ":124034,"ؤ":124035,"รา":124036,"×ķ×Ļ":124037,"ãĤĪãģĨ":124038,"ิà¸Ķ":124039,"×Ļפ":124040,"×Ĺ׾":124041,"ÙĤد":124042,"à¹Ģส":124043,"×Ļ×ĺ":124044,"à¸ģล":124045,"ר׼":124046,"×ķ׼":124047,"×Ļ׼":124048,"ëĪ":124049,"ëĥ":124050,"ðŁĸ":124051,"áħ":124052,"â¼":124053,"ãī":124054,"à¹Ħà¸Ķà¹ī":124055,"ת×Ļ":124056,"×Ļ×IJ":124057,"ĠاÙĦØ¥":124058,"à¸łà¸²":124059,"ริ":124060,"ÙĤØ©":124061,"ØŃد":124062,"ê»":124063,"ì±":124064,"ת×Ĺ":124065,"ìº":124066,"âĭ":124067,"áĦ":124068,"á¾":124069,"âµ":124070,"â¾":124071,"ĠÙĪاÙĦ":124072,"׳×ķ":124073,"ÙĢ":124074,"ÙĬا":124075,"à¸ģà¹ĩ":124076,"×ŀ×Ķ":124077,"ãģĦãĤĭ":124078,"عد":124079,"ĠاÙĦÙĨ":124080,"Ġ×Ķש":124081,"ئ":124082,"ัà¹īà¸ĩ":124083,"รัà¸ļ":124084,"ÙĪÙĤ":124085,"ãģ§ãģį":124086,"à¹Ģà¸ŀ":124087,"׼׾":124088,"×ĺר":124089,"ัà¸Ķ":124090,"à¸Ńา":124091,"ì¢":124092,"à¸Ńà¸ļ":124093,"à¸ķร":124094,"à¹Ģà¸Ĭ":124095,"ìĶ":124096,"ãģĹãģ¾":124097,"ëģ":124098,"ëķ":124099,"ðŁĻ":124100,"âĴ":124101,"á¶":124102,"à¹ģล":124103,"ÙĨا":124104,"à¹ĥหà¹ī":124105,"à¹Ħà¸Ľ":124106,"×£":124107,"ัว":124108,"าà¸ĩ":124109,"×ĵר":124110,"×ij׾":124111,"פ×Ļ":124112,"Ġ×ĵ":124113,"ĠاÙĦÙģ":124114,"à¹Ģà¸Ĥ":124115,"ש×Ķ":124116,"×IJר":124117,"ë¬":124118,"ãģ«ãģª":124119,"ÑĢо":124120,"วิ":124121,"Ùħر":124122,"×IJת":124123,"Ùĥر":124124,"سب":124125,"ÙĨت":124126,"ãģĹãģĦ":124127,"اج":124128,"à¸Ńรà¹Į":124129,"ÙĥÙĦ":124130,"سÙħ":124131,"สิ":124132,"×Ļצ":124133,"ëĿ":124134,"íľ":124135,"ìī":124136,"áĨ":124137,"ÙĩÙħ":124138,"à¸Ļีà¹ī":124139,"ãģĤãĤĭ":124140,"ãģĦãģ¦":124141,"سÙĬ":124142,"׾×IJ":124143,"در":124144,"ãģļ":124145,"ÙĪج":124146,"ĠاÙĦØ®":124147,"صر":124148,"íı":124149,"à¹īาà¸ĩ":124150,"ุà¸Ķ":124151,"×ķ×ĺ":124152,"×ij×¢":124153,"íĨ":124154,"à¸Ĭา":124155,"รม":124156,"ש×ŀ":124157,"×ŀס":124158,"ê´":124159,"ì´":124160,"ëľ":124161,"ì¿":124162,"ì©":124163,"ë»":124164,"â¤":124165,"ðŁĨ":124166,"áĮ":124167,"áķ":124168,"ذا":124169,"à¸Ĺำ":124170,"à¸ķà¹Ī":124171,"ĠاÙĦÙĤ":124172,"ÙĦÙĥ":124173,"ูà¹Ī":124174,"à¸Ħุ":124175,"ÙĬÙħ":124176,"׳×Ļ×Ŀ":124177,"ืà¹Īà¸Ń":124178,"ÙĪع":124179,"ãĤĩ":124180,"اÙĤ":124181,"Ġ×ij×¢":124182,"à¹Ģม":124183,"جÙħ":124184,"ừ":124185,"ãģĵãģ¨ãģĮ":124186,"بد":124187,"×ķ×Ķ":124188,"ש׾":124189,"Ùĩر":124190,"à¹Ģà¸Ļ":124191,"ãģ¹":124192,"íĭ":124193,"ì»":124194,"ì½":124195,"ëŃ":124196,"ìĮ":124197,"íĢ":124198,"ëĮ":124199,"ëº":124200,"ãĬ":124201,"à¹ĥà¸Ļ":124202,"Ġ×Ĵ":124203,"à¹Ĩ":124204,"à¸Īาà¸ģ":124205,"วย":124206,"à¹ĥà¸Ĭ":124207,"à¸ĩาà¸Ļ":124208,"ĠاÙĦØ´":124209,"اØŃ":124210,"à¹īาà¸Ļ":124211,"ืà¹Īà¸Ńà¸ĩ":124212,"×IJ×Ļ":124213,"بÙĦ":124214,"ãģ¨æĢĿ":124215,"×ł×¡":124216,"ãģ¾ãģĽ":124217,"ÙĥÙĨ":124218,"ער":124219,"ĠاÙĦد":124220,"שת":124221,"íŀ":124222,"Ùħس":124223,"صÙĦ":124224,"×ķ׳×Ķ":124225,"ارة":124226,"ÙĦÙħ":124227,"สม":124228,"Ø£ÙĨ":124229,"תר":124230,"×IJ×ŀ":124231,"عب":124232,"خت":124233,"ãĤĥ":124234,"ì¡":124235,"ì£":124236,"ива":124237,"สั":124238,"ึà¸ģ":124239,"ì¸":124240,"ëĨ":124241,"алÑĮн":124242,"ì³":124243,"ìį":124244,"ê¼":124245,"ê½":124246,"ìı":124247,"ãĮ":124248,"ãı":124249,"ï©":124250,"êª":124251,"áİ":124252,"Ġ×ĸ":124253,"à¸ģัà¸Ļ":124254,"×Ļ×ķ":124255,"à¸Ħà¸Ļ":124256,"׳×ķת":124257,"à¸ľà¸¹à¹ī":124258,"à¹ĥà¸Ī":124259,"ãģĦãģŁ":124260,"Ùģر":124261,"×ĺ×Ļ":124262,"צ×Ļ":124263,"ãĤĤãģ®":124264,"ĠاÙĦص":124265,"ãģ¾ãģĽãĤĵ":124266,"دة":124267,"×ij×Ļ":124268,"ĠاÙĦر":124269,"Ġ×ŀ×IJ":124270,"สำ":124271,"à¹Ģห":124272,"عر":124273,"ãģªãģı":124274,"à¸ģระ":124275,"×ij×ĵ":124276,"à¹Ģà¸Ī":124277,"×Ļ×ļ":124278,"×Ĺ×Ļ":124279,"ÙĬع":124280,"ש×ij":124281,"ÙĨØ©":124282,"ÙĪض":124283,"ÙĦÙģ":124284,"ÙĢÙĢ":124285,"פע":124286,"íĪ":124287,"×ŀק":124288,"à¸IJ":124289,"ØŃØ©":124290,"اص":124291,"Ñĭва":124292,"à¸Ħม":124293,"วั":124294,"à¸Ľà¸¥":124295,"ìŁ":124296,"íļ":124297,"ë´":124298,"ëij":124299,"ëī":124300,"ëĩ":124301,"ì¨":124302,"ë±":124303,"ëİ":124304,"â¬":124305,"á¥":124306,"áĹ":124307,"áĽ":124308,"áį":124309,"Å©":124310,"à¸Ķี":124311,"ôi":124312,"Ġס":124313,"׾×ķ":124314,"á»Ŀi":124315,"à¸Ħุà¸ĵ":124316,"ây":124317,"à¸Ļา":124318,"×Ĺ×ĵ":124319,"×ĵ×Ļ":124320,"หา":124321,"جÙĦ":124322,"à¹Ģว":124323,"ãĤĩãģĨ":124324,"ÙħØ©":124325,"ĠاÙĦÙĥ":124326,"Ġ×Ķ×¢":124327,"جر":124328,"×ĸר":124329,"اط":124330,"×Ľ×ª":124331,"×ķ׳×Ļ×Ŀ":124332,"ØŃÙħ":124333,"ê¶":124334,"رÙĥ":124335,"Ġ×ľ×¢":124336,"×ķ×ĸ":124337,"สร":124338,"צ׾":124339,"Ø¢":124340,"است":124341,"à¹Īม":124342,"خر":124343,"צע":124344,"×Ļר×ķת":124345,"ادة":124346,"شار":124347,"×ŀ×Ĺ":124348,"íĴ":124349,"à¹Ģรีย":124350,"×Ĺק":124351,"اث":124352,"รà¸ĩ":124353,"à¹Ģà¸ķ":124354,"à¸Īำ":124355,"à¸Ŀ":124356,"à¹Īาย":124357,"à¸Ħล":124358,"ÙĤÙĪ":124359,"иÑĩеÑģк":124360,"à¸ĵà¹Į":124361,"ัย":124362,"Ùħع":124363,"ë¨":124364,"ë¿":124365,"ë®":124366,"ï´":124367,"ì¥":124368,"ì«":124369,"ëµ":124370,"á¡":124371,"âį":124372,"ðĵ":124373,"â°":124374,"à¸Ĥà¸Ńà¸ĩ":124375,"Ùĭ":124376,"à¸ģัà¸ļ":124377,"ãģ®ãģ§":124378,"à¹īว":124379,"à¸Ńยà¹Īาà¸ĩ":124380,"ãģŃ":124381,"á»ĩt":124382,"à¸ķà¹īà¸Ńà¸ĩ":124383,"×ŀ×Ļ":124384,"à¹ģà¸ļ":124385,"×Ĵר":124386,"ÙĪÙģ":124387,"ÙĤÙĦ":124388,"à¸łà¸²à¸ŀ":124389,"ר×Ļ":124390,"ลา":124391,"ÙĬس":124392,"Ġצ":124393,"ÙĬÙģ":124394,"Ġ×ĺ":124395,"à¸ľà¸¥":124396,"áng":124397,"รว":124398,"Ġ×ŀש":124399,"×IJ×ķת":124400,"×ĸ×Ķ":124401,"ูà¸ģ":124402,"à¸Ļัà¸ģ":124403,"اÙĨÙĬ":124404,"دا":124405,"ãģ³":124406,"׼ף":124407,"ãĤīãĤĮ":124408,"ãĤĮãģ°":124409,"תק":124410,"úc":124411,"ÙĪز":124412,"×Ļר×Ķ":124413,"Ġngh":124414,"ánh":124415,"Ġ×ķ×IJ":124416,"á»ħ":124417,"สุà¸Ķ":124418,"ëį°":124419,"اض":124420,"اÙĦÙĬ":124421,"بار":124422,"عÙħ":124423,"à¸ļา":124424,"تج":124425,"à¸ŀร":124426,"×ķר×Ķ":124427,"ảng":124428,"Ø®ÙĦ":124429,"à¸ī":124430,"ắc":124431,"ש×Ļ×Ŀ":124432,"íĶ":124433,"Ùģس":124434,"×Ļ×Ĵ":124435,"пÑĢ":124436,"ĠاÙĦØ«":124437,"سط":124438,"รูà¹ī":124439,"ีà¹Īย":124440,"à¸Ńà¸Ķ":124441,"ãģªãĤĬ":124442,"×Ĵ×ĵ":124443,"ãģĦãģ¾ãģĹãģŁ":124444,"סק":124445,"خص":124446,"laÅŁ":124447,"енно":124448,"بØŃ":124449,"สà¸Ļ":124450,"ฮ":124451,"ר×IJש":124452,"ÙħÙĪ":124453,"دÙĬد":124454,"ษา":124455,"×ķ×ļ":124456,"ãĥ§ãĥ³":124457,"à¸ķุ":124458,"Ġêµ":124459,"ĠÑģво":124460,"צ×ij":124461,"à¸Ńม":124462,"à¸Ľà¸£":124463,"تع":124464,"×Ķת":124465,"اÙħÙĦ":124466,"×ŀ׳":124467,"ç¶ļ":124468,"ฤ":124469,"íį":124470,"ëĺ":124471,"ë¤":124472,"ìij":124473,"â´":124474,"ãĭ":124475,"ĠباÙĦ":124476,"á»ģu":124477,"ĠاÙĦÙĦ":124478,"à¸ķัว":124479,"Ø°Ùĩ":124480,"ึà¸ĩ":124481,"à¹ĥà¸Ĭà¹ī":124482,"á»ĵng":124483,"à¸Ļั":124484,"มาà¸ģ":124485,"ãĥŁ":124486,"×ŀ×ķ":124487,"à¸Ĺย":124488,"á»Ļi":124489,"ằ":124490,"ảo":124491,"à¹Ĥà¸Ķ":124492,"×IJ׾":124493,"สาม":124494,"ÙĪب":124495,"à¸Ĺุ":124496,"ยัà¸ĩ":124497,"עת":124498,"×ķ׳×ķת":124499,"à¸Ĥึ":124500,"à¸Ĥึà¹īà¸Ļ":124501,"à¸ģà¹Ī":124502,"ẫ":124503,"á»ijc":124504,"ãģĹãĤĩãģĨ":124505,"á»ĭch":124506,"Ġ×IJ×ķת":124507,"Ġש×IJ":124508,"׼×ķ׾":124509,"á»Ļc":124510,"عة":124511,"à¸Ĺี":124512,"à¹Ģà¸Ń":124513,"Ùĥت":124514,"ãģ»":124515,"ẻ":124516,"ìĹħ":124517,"à¸Ńà¸Ńà¸ģ":124518,"اÙĨت":124519,"à¹Ħร":124520,"Ġ×IJ×Ĺר":124521,"طر":124522,"ÙĨد":124523,"ืà¹īà¸Ń":124524,"Ø·ÙĦ":124525,"×IJ×Ķ":124526,"uyên":124527,"íĸī":124528,"×ij×Ķ":124529,"à¸Ħà¹Ī":124530,"à¸Ĭà¹Īว":124531,"ãģĤãĤĬãģ¾ãģĻ":124532,"ÙĬب":124533,"ק׾":124534,"ãĥĻ":124535,"Ä©":124536,"سر":124537,"าว":124538,"ãĤ±":124539,"à¸ļริ":124540,"ר×Ĵ":124541,"á»ĥu":124542,"ØŃت":124543,"×ķ×ŀ×Ļ":124544,"بÙĨ":124545,"êµIJ":124546,"ÄŁu":124547,"ãģªãĤĵ":124548,"×ijק":124549,"Ġפר":124550,"ắn":124551,"ØŃÙĦ":124552,"×ij×Ĺ":124553,"ấu":124554,"×ij×ķ×ĵ":124555,"ãĥ¯":124556,"Ġ׾ק":124557,"ัà¸į":124558,"à¸ŀิ":124559,"×Ĺ×Ķ":124560,"×ĸ׼":124561,"ãĥ¼ãĥł":124562,"ÑĤелÑĮ":124563,"×ŀ×Ļ×ĵ":124564,"ÙĬØ®":124565,"ẳ":124566,"تص":124567,"à¸ĺิ":124568,"è¾¼":124569,"ìĵ":124570,"ÙĥØ©":124571,"ÙĤب":124572,"à¸Ħà¹Į":124573,"à¹īาย":124574,"à¸ĵะ":124575,"าะ":124576,"ëĴ":124577,"ê¾":124578,"ë·":124579,"ìĩ":124580,"êº":124581,"ìģ":124582,"ëĢ":124583,"ì¾":124584,"ë½":124585,"ëļ":124586,"ìŃ":124587,"ìİ":124588,"áij":124589,"ëĹ":124590,"êĴ":124591,"à¡":124592,"à¬":124593,"ðIJĮ":124594,"ãĩ":124595,"ðĿĦ":124596,"Ġ׾×IJ":124597,"ãģ¨ãģĦãģĨ":124598,"Ġnhi":124599,"×Ļ×ķת":124600,"Ġש×Ķ":124601,"à¹ģลà¹īว":124602,"Æ°á»Ľc":124603,"à¸Ķà¹īวย":124604,"à¸Ĺาà¸ĩ":124605,"×ł×ª":124606,"פת":124607,"à¹ģà¸ķà¹Ī":124608,"Æ°ng":124609,"à¸Ńยูà¹Ī":124610,"à¹īำ":124611,"Ġ×IJ׾":124612,"ÙĥÙħ":124613,"ấp":124614,"ลà¸ĩ":124615,"ãģŁãĤģ":124616,"×Ĵ׾":124617,"หร":124618,"ĠÑĢе":124619,"à¹Ģà¸Ĥà¹īา":124620,"ÙĤر":124621,"Ġ×Ķס":124622,"ÙĪÙĬ":124623,"สามาร":124624,"สามารà¸ĸ":124625,"Äĥn":124626,"à¸Ńี":124627,"פ×ķ":124628,"×Ļ׳×ķ":124629,"วัà¸Ļ":124630,"ặc":124631,"íķĻ":124632,"×ŀת":124633,"êu":124634,"ẹ":124635,"ÙģÙĬ":124636,"×ŀצ":124637,"à¸Ħา":124638,"ãģĿãģĨ":124639,"ãĢħ":124640,"از":124641,"اÙĩ":124642,"ר×Ļ×Ŀ":124643,"ấn":124644,"หาร":124645,"ạt":124646,"ÙĨÙĩ":124647,"à¹Ģà¸Ħร":124648,"جÙĩ":124649,"׼×Ļ":124650,"ắt":124651,"à¸Ħà¹īา":124652,"رة":124653,"ãĥı":124654,"ÙĥÙĪÙĨ":124655,"ứng":124656,"Ġìļ°":124657,"ยà¹Į":124658,"à¹Īวà¸Ļ":124659,"à¸ģำ":124660,"ثر":124661,"Ñģи":124662,"ĠاÙĦØ·":124663,"Ġ×Ķצ":124664,"ĠØ·":124665,"ĠاÙĦÙĪ":124666,"ê¹Į":124667,"ØŃÙĬ":124668,"ارات":124669,"à¹Ģà¸ĭ":124670,"با":124671,"гÑĢ":124672,"รี":124673,"ืà¸Ńà¸Ļ":124674,"عت":124675,"ÙĤاÙĦ":124676,"دÙħ":124677,"Ø¡":124678,"Ġ×ŀק":124679,"×ĵ×Ļ×Ŀ":124680,"×¢×ľ":124681,"ãģĴ":124682,"ëĭĺ":124683,"×¢×Ķ":124684,"Ġìĸ´":124685,"ÑģÑĮ":124686,"ÙĤØ·":124687,"ãĥĽ":124688,"èĢĥãģĪ":124689,"à¹ģà¸Ļ":124690,"ÙĪات":124691,"âu":124692,"ĠìĤ¬ëŀ":124693,"หว":124694,"ĠاÙĦØ£Ùħ":124695,"Ġ×Ķ×ŀש":124696,"بÙĪ":124697,"à¸Ĭà¸Ļ":124698,"ãĤĵãģ§ãģĻ":124699,"วà¸Ļ":124700,"à¸ģรรม":124701,"×ŀ×ķ×ĵ":124702,"ÙĥاÙĨ":124703,"×ķ×£":124704,"олог":124705,"تÙĨ":124706,"à¸ķà¹Į":124707,"ê²ĥ":124708,"ר×ĺ":124709,"ừng":124710,"×ķ×ij×Ķ":124711,"ÙħØŃ":124712,"ĠЧ":124713,"פ×Ĵ":124714,"สà¸ĸ":124715,"ãģĭãĤĬ":124716,"ınız":124717,"à¹Ģย":124718,"ãĥ¼ãĥ³":124719,"ãģĬãĤĬ":124720,"פש":124721,"ิà¸ķ":124722,"Ø·ÙĨ":124723,"×Ļת×Ļ":124724,"×IJ׳":124725,"çek":124726,"ìª":124727,"×ŀ×ij":124728,"ศา":124729,"ãĤ¹ãĤ¿":124730,"à¸ļุ":124731,"×ĵ×ijר":124732,"ãģĦãģı":124733,"สะ":124734,"à¹Ģหล":124735,"ิà¸ĩ":124736,"à¸ŀัà¸Ļ":124737,"ãģĦãģŁãģł":124738,"ãĤĤãĤī":124739,"à¹īม":124740,"ãģĵãģ¨ãģĮãģ§ãģį":124741,"ารà¹Į":124742,"ุà¸ĩ":124743,"íij":124744,"ì¯":124745,"ë¼":124746,"íĤ":124747,"ì·":124748,"ê¡":124749,"áı":124750,"áĴ":124751,"ðĿľ":124752,"á©":124753,"ðŁĦ":124754,"ðIJ¤":124755,"Ġש׾":124756,"Ġ×ŀ×Ķ":124757,"à¹ģละ":124758,"Ġ׼׾":124759,"ẽ":124760,"á»Ļng":124761,"Ø°ÙĬ":124762,"ле":124763,"×¥":124764,"ãģªãģ©":124765,"ĠÙĪØ£":124766,"หà¸Ļà¹īา":124767,"ãģ¾ãģ§":124768,"à¸ķà¹Īà¸Ń":124769,"à¸Ĺัà¹īà¸ĩ":124770,"ãģłãģij":124771,"à¹ģà¸ļà¸ļ":124772,"à¹Ģรา":124773,"פ׾":124774,"ãģŁãģĦ":124775,"à¹Ģลย":124776,"ãģ£ãģ¦ãģĦãĤĭ":124777,"ếp":124778,"ึà¹Īà¸ĩ":124779,"ê´Ģ":124780,"ê³Ħ":124781,"׼×ķ":124782,"à¹Ģรืà¹Īà¸Ńà¸ĩ":124783,"ק×Ļ":124784,"êµŃ":124785,"פס":124786,"تÙĬ":124787,"ãĥĦ":124788,"Ġ×Ķ×Ĺ":124789,"ги":124790,"ר×IJ׾":124791,"×ŀ׾":124792,"ĠØ£ÙĬ":124793,"ĠعÙĦÙĬ":124794,"ãģĭãģ£ãģŁ":124795,"ש×Ļ":124796,"дÑĥ":124797,"×ŀף":124798,"׳×ĺ":124799,"׳×Ļת":124800,"miÅŁ":124801,"׼×Ŀ":124802,"Ġ×ijר":124803,"Ġ׾×ij":124804,"ĠÐĽ":124805,"çe":124806,"×ķ׳×Ļ":124807,"ãĤĪãģĨãģ«":124808,"פ×ķר":124809,"ãĥį":124810,"ÙĥÙĬ":124811,"×Ĺת":124812,"ÙģÙĦ":124813,"Ġ×Ķק":124814,"Ġ×Ķ×ij":124815,"Ġ×ŀס":124816,"à¹Īาà¸Ļ":124817,"пеÑĢ":124818,"à¹Īาว":124819,"Ġ×ij×IJ":124820,"ĠÙĪÙĩ":124821,"à¸Ļำ":124822,"Ġ×ijש":124823,"׳ק":124824,"ãģ©ãģĨ":124825,"ש×ķת":124826,"×ĵ×Ķ":124827,"à¹Ģà¸ļ":124828,"ÙĨس":124829,"Ġìļ°ë¦¬":124830,"สà¹Īวà¸Ļ":124831,"ลัà¸ĩ":124832,"جز":124833,"Ġ×Ĺ×Ļ":124834,"Ùĥثر":124835,"ละ":124836,"Ùĩد":124837,"ĠÙĪب":124838,"اÙĦÙħ":124839,"à¹ģม":124840,"Æ¡i":124841,"Ġ×ij×Ĺ":124842,"ữa":124843,"à¹Ģà¸Ĺศ":124844,"à¸ķัà¹īà¸ĩ":124845,"огда":124846,"׾ק":124847,"دد":124848,"สรà¹īาà¸ĩ":124849,"à¸Ĭี":124850,"Ùģض":124851,"à¹ģห":124852,"uyá»ĩn":124853,"รัà¸ģ":124854,"á»ĩm":124855,"สา":124856,"פק":124857,"ียà¸ĩ":124858,"à¸ķà¹Īาà¸ĩ":124859,"à¸Ħรัà¹īà¸ĩ":124860,"ØŃÙĤ":124861,"à¹Ģà¸Ńà¸ĩ":124862,"ائÙĬ":124863,"×ĺ×¢":124864,"اÙĦØ©":124865,"ิà¹Īม":124866,"ãĤ½":124867,"دÙī":124868,"Ġר×IJ":124869,"ãģ£ãģ¨":124870,"ãĥĥãĥĹ":124871,"ÙĬرة":124872,"ê±´":124873,"×ŀ×IJ":124874,"×ķ×ķ":124875,"بع":124876,"ãģ²":124877,"ราย":124878,"×ĵ×Ŀ":124879,"تÙģ":124880,"à¸ķà¸ģ":124881,"ạng":124882,"ãĤĴè¦ĭ":124883,"à¸Ĭั":124884,"Æ°á»Ł":124885,"Æ°á»Łng":124886,"جب":124887,"×ķ×ŀר":124888,"ĠìĤ¬ëŀĮ":124889,"óng":124890,"รั":124891,"Ġ×Ķ×ĸ":124892,"רצ":124893,"Ġ×Ĺ×ĵ":124894,"Ø°ÙĦÙĥ":124895,"×ķר×Ļ":124896,"ãģ¡ãĤĥ":124897,"Ùģع":124898,"Ġ׾צ":124899,"ái":124900,"à¹ĩà¸ļ":124901,"ãģİ":124902,"à¸ģิ":124903,"ạc":124904,"ë©°":124905,"ãģªãĤĭ":124906,"×ķ׾×Ŀ":124907,"à¹ģà¸Ĺ":124908,"×ķ×¥":124909,"меÑĤ":124910,"Ã¼ÅŁ":124911,"ÑĢÑı":124912,"à¸Ĵ":124913,"ÑģÑĤоÑı":124914,"عÙĪد":124915,"Ùħار":124916,"طة":124917,"à¸ŀื":124918,"кÑĢ":124919,"à¹ģà¸ģ":124920,"à¹Ĥรà¸ĩ":124921,"×ij×Ļ×ĺ":124922,"ê²ł":124923,"×ķ׾×Ķ":124924,"ØŃر":124925,"ืà¹Īà¸Ńà¸Ļ":124926,"×ķ×ijר":124927,"×Ĺש":124928,"ãĥķãĤ¡":124929,"×ŀ×ĺ":124930,"út":124931,"Ġdön":124932,"ắng":124933,"ëłĩ":124934,"ẳng":124935,"วà¸ģ":124936,"صد":124937,"خط":124938,"à¸Ńั":124939,"ãĤıãĤĮ":124940,"سÙĦاÙħ":124941,"à¹Ģรà¹ĩ":124942,"×Ļש×Ļ":124943,"جاÙĦ":124944,"ãģijãĤĭ":124945,"à¸Ĭาà¸ķิ":124946,"ÙĪاÙĤ":124947,"à¹Ĥà¸Ļ":124948,"ãģ¦ãģĹãģ¾":124949,"اعة":124950,"ãĤŃãĥ£":124951,"à¸įา":124952,"ÙĦاÙĤ":124953,"ิà¸ģ":124954,"ĠÑģов":124955,"ÑĢак":124956,"×Ļ׳×Ļ":124957,"Ã¼ÄŁ":124958,"Ã¼ÄŁÃ¼":124959,"ק×ij":124960,"à¹Īà¸Ńà¸ĩ":124961,"Ġgerçek":124962,"à¸Ĺั":124963,"ованиÑı":124964,"×ŀ׼":124965,"سة":124966,"×Ļ×£":124967,"leÅŁ":124968,"Ùħؤ":124969,"ĠìĿĺ":124970,"à¸IJาà¸Ļ":124971,"ĠÑģоб":124972,"ĠêµŃ":124973,"עצ":124974,"зв":124975,"สà¸ĩ":124976,"زÙĦ":124977,"ãģıãĤĮ":124978,"иÑĢÑĥ":124979,"تأ":124980,"полн":124981,"ìĺĢ":124982,"ÙĨØ´":124983,"׼×IJ":124984,"ÙħØ´":124985,"à¸Ķà¹Į":124986,"ÙĪÙĬÙĦ":124987,"à¹ģà¸Ĥ":124988,"ãģ£ãģ¦ãģĹãģ¾":124989,"ноÑģÑĤ":124990,"вл":124991,"ÙħÙĤ":124992,"راج":124993,"å¤ī":124994,"ëĽ":124995,"â¸":124996,"ìIJ":124997,"à»":124998,"áļ":124999,"â»":125000,"êĻ":125001,"â§":125002,"ðĴ":125003,"ðĿĩ":125004,"Ġ×IJת":125005,"ĠÙĦÙĦ":125006,"ĠØ£ÙĨ":125007,"Ġ×ķ×Ķ":125008,"ãģ«ãģ¯":125009,"Ġ×Ļש":125010,"تÙĩ":125011,"ÃŃnh":125012,"ÙĬات":125013,"Ġ×ij×ŀ":125014,"à¸Ļัà¹īà¸Ļ":125015,"à¸Ļà¹īำ":125016,"Ãło":125017,"à¸ķาม":125018,"ãģ®ãģ¯":125019,"dır":125020,"Ġnghi":125021,"ặt":125022,"×ŀ×Ļ×Ŀ":125023,"ãģ¦ãģĦãĤĭ":125024,"Ġ×ijת":125025,"หรืà¸Ń":125026,"ĠسÙĬ":125027,"ãģªãĤī":125028,"à¹Ĥà¸Ķย":125029,"ıyor":125030,"à¸Ńีà¸ģ":125031,"á»ĩnh":125032,"Ñĭм":125033,"à¸Ĺุà¸ģ":125034,"Ġ׾×Ĺ":125035,"Ġ×Ķר":125036,"Ġ×Ķ×Ļ":125037,"à¸ŀระ":125038,"à¹Ģวลา":125039,"Ġغ":125040,"ẫn":125041,"mÄ±ÅŁ":125042,"׼×Ķ":125043,"á»ijn":125044,"ãģ§ãģĹãĤĩãģĨ":125045,"ãĥ¢":125046,"à¸Ľà¸µ":125047,"ס×Ļ":125048,"ãģĵãĤį":125049,"Ġ׾פ":125050,"รà¸ĸ":125051,"ê¸Ī":125052,"à¸ģวà¹Īา":125053,"무":125054,"á»įng":125055,"ãĤĵãģ§":125056,"ãĤĪãģĨãģª":125057,"á»ĵi":125058,"ãĤ¬":125059,"สà¹Īà¸ĩ":125060,"×Ļ׳×Ķ":125061,"à¸ĸูà¸ģ":125062,"à¸Īัà¸Ķ":125063,"Ġ×Ķ×Ĵ":125064,"ãĥľ":125065,"×ŀ×ķת":125066,"ÙĪÙĥ":125067,"ëĭ¨":125068,"ĠØ«":125069,"ãģ®ãģĮ":125070,"à¹Ģหà¹ĩà¸Ļ":125071,"عا":125072,"à¸Ļิ":125073,"Åŀ":125074,"à¸Ńะ":125075,"ãģĪãĤĭ":125076,"Ø«ÙĦ":125077,"ØŃÙħد":125078,"à¹Ģà¸ģิà¸Ķ":125079,"פשר":125080,"פ×Ķ":125081,"มิ":125082,"ئÙĬس":125083,"à¸Ĺำà¹ĥหà¹ī":125084,"×¢×ĵ":125085,"ìĭ¤":125086,"à¸Ĭà¹Īวย":125087,"ĠاÙĦÙħÙĨ":125088,"زÙĬ":125089,"عÙĬ":125090,"Ġ׼×IJ":125091,"ạnh":125092,"ỹ":125093,"ãĤĵãģª":125094,"สู":125095,"צר":125096,"Æ°á»Ľng":125097,"×ķ×ķ×Ķ":125098,"à¹Ĥล":125099,"ĠاÙĦÙĩ":125100,"วา":125101,"หลาย":125102,"Ñīе":125103,"à¸Ĥà¹īà¸Ń":125104,"à¹īà¸Ńย":125105,"بط":125106,"каÑı":125107,"ĠØ¢":125108,"ĠиÑģ":125109,"ĠاÙĦغ":125110,"à¸ģา":125111,"à¸Ļà¹Īา":125112,"ÙĬÙĪ":125113,"×ij×ķר":125114,"á»ħn":125115,"วà¸ĩ":125116,"×Ļ×ĸ":125117,"ì²Ń":125118,"ним":125119,"룰":125120,"×Ĵ×ķר":125121,"صØŃ":125122,"ÙĦÙĪ":125123,"×Ĺ×ķת":125124,"สุ":125125,"رÙĬÙĤ":125126,"ס×ĺ":125127,"Ġ×ŀ×¢":125128,"ãĥĨãĤ£":125129,"à¸Ħิà¸Ķ":125130,"ãĤįãģĨ":125131,"à¹Ħล":125132,"à¸Ļà¹Į":125133,"á»ıi":125134,"ÑģÑĤÑĢо":125135,"สà¸Ķ":125136,"สาร":125137,"ÙĪÙĦØ©":125138,"ầm":125139,"รà¹Īว":125140,"รà¹Īวม":125141,"รุ":125142,"ĠاÙĦسÙĬ":125143,"ìĺģ":125144,"Ġ×ŀ×ij":125145,"פ×ĺ":125146,"à¸ķิà¸Ķ":125147,"×ĺ×Ļ×Ŀ":125148,"Ġ무":125149,"ÙĤدÙħ":125150,"ĠdÃ¼ÅŁ":125151,"ائÙĦ":125152,"мÑĭ":125153,"ØŃس":125154,"ÙĪص":125155,"×Ļק×Ķ":125156,"ãģ§ãģ¯ãģªãģĦ":125157,"à¹Ģหม":125158,"оÑĢÑĤ":125159,"íĨµ":125160,"ãģIJ":125161,"кÑĢа":125162,"ียว":125163,"عار":125164,"ئة":125165,"íĥĢ":125166,"ãģ«ãģªãĤĬ":125167,"جة":125168,"ÙĪÙĤع":125169,"ÑĮÑı":125170,"×ķצ×Ķ":125171,"ש×Ŀ":125172,"بÙĤ":125173,"Ġ×Ļ×Ķ":125174,"ÙĬØ·":125175,"ımız":125176,"деÑĢж":125177,"×Ļשר×IJ׾":125178,"غÙĬر":125179,"รà¸Ńà¸ĩ":125180,"à¹Ģรียà¸Ļ":125181,"Ġ×Ķ×ĺ":125182,"หมาย":125183,"ÙħÙĩ":125184,"اÙģØ©":125185,"ĠоÑĢг":125186,"ÙĪÙī":125187,"ãĥ©ãĤ¤":125188,"×ŀ׳×Ķ":125189,"ĠÄijo":125190,"ĠгоÑĢ":125191,"اÙħØ©":125192,"楽":125193,"Ø«ÙĬر":125194,"à¸ģิà¸Ī":125195,"á»ĵn":125196,"ÙĨب":125197,"ÑĢÑĥд":125198,"ìĹĪ":125199,"Ġ×Ĺ×ijר":125200,"ÑĢаж":125201,"ạch":125202,"تÙĪ":125203,"à¹Ĥม":125204,"×ij×Ļ×ij":125205,"ĠíĨµ":125206,"acaģı":125207,"جÙĦس":125208,"à¹Ģà¸Ľà¸¥":125209,"วà¸Ķ":125210,"à¸Ńล":125211,"ãģŁãĤĬ":125212,"à¸Ľà¸±à¸į":125213,"ĠìķĮ":125214,"عرÙģ":125215,"à¹Ħà¸Ł":125216,"أخ":125217,"å¤ļãģĦ":125218,"à¸Ķัà¸ĩ":125219,"Ø´Ùģ":125220,"ãģ£ãģ¦ãģĦãģ¾ãģĻ":125221,"×Ľ×ł×¡":125222,"ÑĨе":125223,"еÑģп":125224,"ÙħاÙħ":125225,"à¸ŀืà¹īà¸Ļ":125226,"иÑĩеÑģки":125227,"خد":125228,"ÙĥÙĪÙħ":125229,"Ġ×Ķר×IJש":125230,"تاب":125231,"é£Łãģ¹":125232,"ืà¸Ļ":125233,"оÑĢо":125234,"Ġböl":125235,"×ķ×Ĺ×ĵ":125236,"دÙĬر":125237,"ắm":125238,"دع":125239,"ãģķãģĽ":125240,"à¸ĺร":125241,"à¸ĺรรม":125242,"ãģĭãĤĤ":125243,"å¤ļãģı":125244,"rä":125245,"سع":125246,"×Ļ׾×Ķ":125247,"ضر":125248,"ĠاÙĦشر":125249,"×ĸ×ķר":125250,"×¢×ijר":125251,"ạm":125252,"алÑĮно":125253,"رÙĨ":125254,"اÙħج":125255,"׼×ļ":125256,"dıģ":125257,"ден":125258,"ضا":125259,"ÙĦÙĬÙħ":125260,"Ġê·¸ëŁ¬":125261,"تÙħاع":125262,"ارÙĬØ®":125263,"à¹Ĥà¸ķ":125264,"ĠÑģÑĢед":125265,"Ġ׳×ķס":125266,"ÙĤبÙĦ":125267,"оÑĤов":125268,"leÅŁtir":125269,"ĠмеÑģÑĤ":125270,"سÙĦÙħ":125271,"Ġעצ":125272,"ĠاÙĦسÙĦ":125273,"еÑĤÑĮ":125274,"ابة":125275,"нак":125276,"สà¸ĸาà¸Ļ":125277,"Ġ×ij׳":125278,"à¸ļัà¸Ļ":125279,"׼׳":125280,"ĠÃ¶ÄŁ":125281,"ãģ¨è¨Ģ":125282,"uyến":125283,"diÄŁ":125284,"áºŃu":125285,"ÑĢаÑģ":125286,"ãĤ·ãĥ§ãĥ³":125287,"nız":125288,"×ķ×ĵ×Ķ":125289,"تس":125290,"ÙħاÙĦ":125291,"à¹Ģหà¸ķุ":125292,"ยว":125293,"à¸ŀัà¸ģ":125294,"ãģĦãģªãģĦ":125295,"ĠкаÑĩ":125296,"ลà¹Į":125297,"×¨×Ľ×ª":125298,"ÅŁtur":125299,"×ŀ×ķס":125300,"ãģ¥":125301,"бол":125302,"عÙħاÙĦ":125303,"×ķרת":125304,"ÑĨион":125305,"ศึà¸ģ":125306,"à¸ı":125307,"ÑĢен":125308,"اسÙĬ":125309,"ائر":125310,"à¹Ĥà¸Ľà¸£":125311,"Ġseç":125312,"غÙĬ":125313,"ÑįÑĤ":125314,"енн":125315,"ãģªãģ®":125316,"×Ļש×Ķ":125317,"×Ļפ×ķר":125318,"ãģŁãĤģãģ«":125319,"زة":125320,"Ġçoc":125321,"ãĤ¯ãĥª":125322,"ÑĪен":125323,"ãĤıãģij":125324,"رÙĬد":125325,"ĠÑĢаÑģÑģ":125326,"Ùĥات":125327,"สà¸Ńà¸ļ":125328,"ceÄŁi":125329,"ãĤ¿ãĤ¤":125330,"à¸ļร":125331,"ĠاÙĦبر":125332,"׳×ķ×¢":125333,"rün":125334,"راض":125335,"ศาส":125336,"à¸ķรà¹Į":125337,"ãģįãģŁ":125338,"×ķ׾×ĵ":125339,"еÑĢи":125340,"íĹĺ":125341,"ắp":125342,"تعÙĦ":125343,"Ùĥد":125344,"иÑĤелÑĮно":125345,"Ø·Ùģ":125346,"ĠавÑĤом":125347,"Ġ×ŀצ":125348,"ÑĪиÑħ":125349,"اتÙģ":125350,"ĠÑħоÑĤ":125351,"Ùİا":125352,"ãģıãĤĭ":125353,"×Ķפ":125354,"à¹Ĥà¸Ĺ":125355,"à¹ģà¸ŀ":125356,"à¹Īà¸Ńย":125357,"ĠاÙĦÙħØ´":125358,"à¸ģารà¸ĵà¹Į":125359,"аниз":125360,"×Ķ׾":125361,"ظÙħ":125362,"ยุ":125363,"liÄŁ":125364,"à¹Ħà¸Ĥ":125365,"à¸ĸืà¸Ń":125366,"öz":125367,"ãģijãģ¦":125368,"à¹Ģà¸ľ":125369,"ุม":125370,"ãĥĹãĥ¬":125371,"Ġ×Ķ×IJ×Ĺר":125372,"ختÙĦÙģ":125373,"à¸İ":125374,"ÙĦاØŃ":125375,"Ġdüzen":125376,"צ×Ķ":125377,"ساء":125378,"×ķר×ļ":125379,"×ķ×ĵ×Ļ":125380,"ÑĢаÑĦ":125381,"ÅŁtır":125382,"ãģ«åħ¥":125383,"ãģĪãģ°":125384,"صÙĪÙĦ":125385,"ĠÐľÐ¾Ñģ":125386,"اÙĩر":125387,"ãģ£ãģ":125388,"ĠлÑİб":125389,"×Ļ×¢×Ķ":125390,"Ġ×Ķ×ŀק":125391,"สิà¸Ĺ":125392,"สิà¸Ĺà¸ĺิ":125393,"×Ļ׳×Ŀ":125394,"ÙĦاÙģ":125395,"à¸ŀัà¸Ļà¸ĺ":125396,"×ķ×IJ×Ķ":125397,"มั":125398,"à¸Ĥà¸ĵะ":125399,"доÑĢ":125400,"ãģ¨ãģª":125401,"à¸ģระà¸Ĺ":125402,"acı":125403,"×ķ׾×ķ×Ĵ":125404,"ÑĥÑĪ":125405,"ãĥ¥ãĥ¼":125406,"ãĥ¦":125407,"Ùħست":125408,"ĠaÅŁ":125409,"שק":125410,"פת×Ĺ":125411,"ายà¸Ļ":125412,"íĩ":125413,"ë¢":125414,"ï·":125415,"íī":125416,"ìµ":125417,"ì¬":125418,"ðĿĽ":125419,"ìĴ":125420,"ëĻ":125421,"ê§":125422,"áĸ":125423,"â¨":125424,"â±":125425,"áĺ":125426,"ðĸ":125427,"àł":125428,"áĶ":125429,"ðIJŃ":125430,"ững":125431,"Å©ng":125432,"Ġ×Ķת":125433,"ĠاÙĦا":125434,"Ġ×ŀת":125435,"à¸ĸึà¸ĩ":125436,"òn":125437,"á»ĭnh":125438,"нÑĭм":125439,"Ġcả":125440,"à¸Ķู":125441,"Ġà¹ģà¸ķà¹Ī":125442,"Ġ×ij×Ķ":125443,"ói":125444,"ãģ¨ãģĹãģ¦":125445,"úng":125446,"ĠØ°":125447,"Ġ×Ķ׳":125448,"ĠبÙĨ":125449,"ÙĦاÙĦ":125450,"à¹Ħà¸Ĺย":125451,"á»ĩp":125452,"tı":125453,"มัà¸Ļ":125454,"ằng":125455,"á»ijt":125456,"ком":125457,"à¸ĭึà¹Īà¸ĩ":125458,"à¸Ħรัà¸ļ":125459,"à¸ļà¹īาà¸Ļ":125460,"ĠاÙĦÙĬ":125461,"lü":125462,"ÙĪس":125463,"ãģłãģ£ãģŁ":125464,"à¹Ģà¸ĩ":125465,"Ġê³µ":125466,"нÑĥ":125467,"ãĤĪãĤĬ":125468,"мÑĥ":125469,"à¹Ģà¸Ĥา":125470,"ãĤĢ":125471,"ние":125472,"ãģ«ãģªãĤĭ":125473,"áºŃy":125474,"ĠÙĪا":125475,"볤":125476,"ש×ķ":125477,"áp":125478,"×ĵ×ķ":125479,"ãģ§ãģĹãģŁ":125480,"عض":125481,"Ñģкой":125482,"æĦŁãģĺ":125483,"ÑİÑĤÑģÑı":125484,"Ġ×Ļ׼×ķ׾":125485,"ãĤĵãģł":125486,"ви":125487,"à¹Ģลà¹Īà¸Ļ":125488,"ìĿ´ëĭ¤":125489,"ĠÙĦÙĩ":125490,"à¸Ħืà¸Ń":125491,"تÙĥ":125492,"ÙħÙĥÙĨ":125493,"aģı":125494,"׳×ĵ":125495,"민":125496,"à¹Ħว":125497,"สำห":125498,"สำหรัà¸ļ":125499,"Ñģлед":125500,"tır":125501,"ĠÙĦÙĬ":125502,"ĠاÙĦعÙħÙĦ":125503,"×ij×ķת":125504,"×ij×Ļ×Ŀ":125505,"à¸Ħำ":125506,"à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩ":125507,"lıģı":125508,"ืà¸Ńà¸ĩ":125509,"جد":125510,"íŀĪ":125511,"ìĭ¬":125512,"×¢×ķת":125513,"สิà¸Ļ":125514,"Ñĩи":125515,"رض":125516,"à¹Ģà¸Ľà¸´à¸Ķ":125517,"à¸Ħà¹Īา":125518,"ìĦł":125519,"ÙĪرة":125520,"ק×ĺ":125521,"ìľł":125522,"عÙħÙĦ":125523,"×IJ×Ļ×Ŀ":125524,"׾×Ļ×Ŀ":125525,"à¹ĥหà¸į":125526,"à¹ĥหà¸įà¹Ī":125527,"ừa":125528,"á»įi":125529,"ãģ¶":125530,"ÃŃch":125531,"ãĥĩãĤ£":125532,"×ķר×Ļ×Ŀ":125533,"Ñģо":125534,"ìķ½":125535,"ова":125536,"ÑĩаÑģÑĤ":125537,"à¹Ģà¸Īà¹īา":125538,"пÑĢо":125539,"Ġ×ŀ×Ĺ":125540,"ãĥİ":125541,"×ķ×Ļ×ķת":125542,"Ġде":125543,"ë§Ī":125544,"ì§ģ":125545,"×Ļפ×Ķ":125546,"ĠاÙĦعاÙĦÙħ":125547,"르":125548,"ר×IJ×Ķ":125549,"uyá»ĥn":125550,"×¢×Ļ":125551,"มืà¸Ń":125552,"Ø¥ÙĨ":125553,"รู":125554,"Ġز":125555,"×Ļ×ķ×Ŀ":125556,"à¸ķà¹īà¸Ļ":125557,"ãģ¦ãģĦãģ¾ãģĻ":125558,"ÙħاÙĨ":125559,"ĠÐ¥":125560,"à¸Ľà¸£à¸°à¹Ģà¸Ĺศ":125561,"ỳ":125562,"׾×ij":125563,"à¹Ģà¸Ķà¹ĩ":125564,"ãģŁãģ¡":125565,"à¸Ĺีม":125566,"à¸Ļะ":125567,"ìĹ°":125568,"ĠìłĢ":125569,"ÙĦÙĩ":125570,"ợi":125571,"ĠاÙĦز":125572,"دار":125573,"ãĤ³ãĥ³":125574,"мин":125575,"à¹ģหà¹Īà¸ĩ":125576,"à¸Ķัà¸ļ":125577,"׼ר":125578,"жа":125579,"íĸĪ":125580,"×ŀ×ĸ":125581,"ợi":125582,"à¸Ķา":125583,"Ġعبد":125584,"à¹ģร":125585,"×IJתר":125586,"×¢×ł×Ļ":125587,"à¹Ģà¸Ħ":125588,"×ķצר":125589,"ì§Ģë§Į":125590,"ائÙħ":125591,"أس":125592,"uyá»ģn":125593,"Ġ×IJ׳":125594,"×Ĺ׳×ķ":125595,"×ĸ×Ļ":125596,"รà¹īาà¸Ļ":125597,"ĠÐłÐ¾Ñģ":125598,"ĠÐłÐ¾ÑģÑģ":125599,"ربÙĬØ©":125600,"tür":125601,"ãĤĭãģĵãģ¨":125602,"ظر":125603,"бÑĭ":125604,"à¸Ĺีà¹Īสุà¸Ķ":125605,"Ġצר":125606,"èĩªåĪĨ":125607,"лаÑģ":125608,"ĠÑıв":125609,"ĠÑıвлÑı":125610,"à¸ŀรà¹īà¸Ńม":125611,"à¸Ńาà¸Ī":125612,"à¸ļริà¸ģาร":125613,"Ġçı":125614,"ëįĺ":125615,"ĠاÙĦÙħست":125616,"تش":125617,"ש×ķ×ij":125618,"ãĤ´":125619,"Ġyapıl":125620,"ĠاÙĦØ°":125621,"ุà¹Īม":125622,"à¸ĸà¹īา":125623,"ìĦ¤":125624,"ì°¨":125625,"ваÑĢ":125626,"à¹Ģà¸ŀิà¹Īม":125627,"Æ°á»Ľi":125628,"Ùĥس":125629,"à¸Ńยาà¸ģ":125630,"ãģ¦ãĤĤ":125631,"Ġгод":125632,"ÙĬار":125633,"à¸ķà¸Ńà¸Ļ":125634,"ĠигÑĢ":125635,"à¹Ħà¸Ķà¹īรัà¸ļ":125636,"ĠاÙĦÙħر":125637,"ÙĤت":125638,"Ġëĺ":125639,"ĠëĺIJ":125640,"ẩn":125641,"ãģĻãĤĭãģĵãģ¨":125642,"×Ĵ×Ŀ":125643,"Ġ×ij×ij":125644,"تد":125645,"ÙĪار":125646,"ãĤ®":125647,"пол":125648,"Ġмог":125649,"ترÙĥ":125650,"ÙĪØ«":125651,"Ġçık":125652,"اة":125653,"à¹Ģà¸Ķียว":125654,"มีà¸Ħวาม":125655,"Ġ×ŀ×Ĵ":125656,"صÙģ":125657,"ĠТак":125658,"Ġ×Ľ×ª":125659,"×Ļ×ĵ×Ļ":125660,"овоÑĢ":125661,"ầy":125662,"สิà¹Īà¸ĩ":125663,"بت":125664,"ürü":125665,"ÙĨج":125666,"หลัà¸ģ":125667,"×Ļ×Ķ×Ŀ":125668,"ÙĤص":125669,"зÑĭ":125670,"×Ľ×ª×ij":125671,"Æ°u":125672,"mız":125673,"ĠìĦ¸":125674,"лог":125675,"ÙħÙĬÙĦ":125676,"ÙĬج":125677,"íĴĪ":125678,"à¸ŀà¸ļ":125679,"หัว":125680,"зна":125681,"רק":125682,"à¹Ĥร":125683,"Ġ×ijס":125684,"ĠBaÅŁkan":125685,"ĠëĶ°":125686,"à¸Ńัà¸Ļ":125687,"ีà¹Īยว":125688,"неÑģ":125689,"à¹Ģà¸Ķิà¸Ļ":125690,"ÙĬاÙĨ":125691,"×ķ׾×Ļ":125692,"اخت":125693,"צ×ķת":125694,"ãģĵãģĵ":125695,"ĠاÙĦاÙĨ":125696,"ĠпÑĢоÑĨ":125697,"ãģ¾ãģł":125698,"×Ľ×¡":125699,"ĠاÙĦØ¢":125700,"ÙĬز":125701,"ĠاÙĦدÙĪÙĦ":125702,"ĠíķĺëĤĺ":125703,"ضع":125704,"ê»ĺ":125705,"ÅĽwi":125706,"ยิ":125707,"ãģ¡ãĤĥãĤĵ":125708,"ĠÙħØ´":125709,"à¸ĺี":125710,"ãģ¨ãģį":125711,"׳×Ļ×ķת":125712,"Ġë¯":125713,"Ġ미":125714,"Ġsı":125715,"ëĭĪê¹Į":125716,"Ġпл":125717,"غÙĦ":125718,"à¹ģรà¸ĩ":125719,"بÙĬر":125720,"ãģĤãĤĬãģ¾ãģĽãĤĵ":125721,"ê·¼":125722,"Ġyüz":125723,"ĠdeÄŁer":125724,"åł´åIJĪ":125725,"ỡ":125726,"маÑĤ":125727,"ราà¸Ĭ":125728,"ÙĪرÙĬ":125729,"жен":125730,"ãģ¾ãĤĬ":125731,"ãģ®ä¸Ń":125732,"×Ļ×ĵ×¢":125733,"à¸Ńุ":125734,"à¸ļà¸Ńล":125735,"à¸Ľà¸±à¸įหา":125736,"زÙħ":125737,"ÄŁa":125738,"à¸Ńืà¹Ī":125739,"à¸Ńืà¹Īà¸Ļ":125740,"пл":125741,"ĠнеобÑħодим":125742,"׼×ij":125743,"à¹Ģศ":125744,"קר×Ķ":125745,"ì²ĺ":125746,"볨":125747,"×ŀק×ķ×Ŀ":125748,"jÄħc":125749,"ÙĩÙĦ":125750,"Ġ×¢×ij×ķ×ĵ":125751,"à¹Ħมà¹ī":125752,"à¸ģลัà¸ļ":125753,"×ķ׼׾":125754,"ק×ĵ":125755,"اÙĦÙĬØ©":125756,"رÙĩ":125757,"ãģijãĤĮãģ°":125758,"ĠÙĨÙģس":125759,"ãĤ¢ãĥ«":125760,"ìĹĪëĭ¤":125761,"ק×ķר":125762,"неÑĢ":125763,"باب":125764,"ãĤ¶":125765,"سبب":125766,"ÙĦÙĬÙĦ":125767,"صÙĨ":125768,"صدر":125769,"ếm":125770,"à¸Ĭà¹Īวà¸ĩ":125771,"ØŃÙĨ":125772,"Ġ×ij×Ĵ":125773,"×ŀ×ķ×¢":125774,"׾×Ĺ":125775,"大ãģį":125776,"تب":125777,"неÑĤ":125778,"×Ļ×ij×Ķ":125779,"бл":125780,"ãĥĹãĥª":125781,"اصة":125782,"ãģ¤ãģij":125783,"×Ļ×ŀ×ķש":125784,"ãģĮãģĤ":125785,"ëĭ´":125786,"ãģĭãĤĤãģĹ":125787,"ãģĭãĤĤãģĹãĤĮ":125788,"ãģ¡ãĤī":125789,"×ij×ĺ":125790,"ĠbaÄŁ":125791,"×Ļ×Ĺס":125792,"×ij×ķ×¢":125793,"ลี":125794,"פע×Ļ׾":125795,"ими":125796,"gÅĤ":125797,"Ġиме":125798,"خداÙħ":125799,"×IJ×Ļר":125800,"Ġyapt":125801,"ãģ¨ãģĦ":125802,"à¸ĩà¹Īาย":125803,"׾×Ļ×ķ":125804,"ØŃدث":125805,"راÙĤ":125806,"ĠÄIJi":125807,"ادر":125808,"ãģĵãģ¨ãĤĤ":125809,"×ij×Ļר":125810,"Ġвз":125811,"ضاÙģ":125812,"ת×ķ׼":125813,"ÑĢом":125814,"رات":125815,"à¹Ģà¸Ĺà¹Īา":125816,"ãģĺãĤĥ":125817,"ãģĿãģĵ":125818,"اجتÙħاع":125819,"à¹īà¸Ńà¸Ļ":125820,"ÙĤÙħ":125821,"본":125822,"Äŀ":125823,"ש×Ļ×ķ":125824,"×ij׳×Ļ":125825,"ìľĦìĽIJ":125826,"à¹ģà¸Ī":125827,"×Ĺ×ķר":125828,"دÙĬÙĨØ©":125829,"تط":125830,"ằm":125831,"òa":125832,"ยà¸Ńà¸Ķ":125833,"Ġëĭ¹":125834,"สุà¸Ĥ":125835,"×ĵר×ļ":125836,"دÙĨ":125837,"سÙĬÙĨ":125838,"ÙĪÙĤÙģ":125839,"ÑĨÑĭ":125840,"гоÑĤов":125841,"еждÑĥ":125842,"à¸ŀวà¸ģ":125843,"اÙĤتص":125844,"اÙĤتصاد":125845,"czÄĻ":125846,"niÄĻ":125847,"ÑĢеб":125848,"ØŃÙĪ":125849,"à¸Ĺà¹Į":125850,"ãĤĪãģŃ":125851,"дж":125852,"à¸ģลà¹Īาว":125853,"دÙĬØ«":125854,"ãĤ³ãĥŁ":125855,"ÙĤÙĪÙħ":125856,"ĠتØŃ":125857,"à¹Ģà¸ķิ":125858,"اÙģظ":125859,"à¸Īุ":125860,"رÙĬاض":125861,"×ŀש×ļ":125862,"à¹Ĥย":125863,"еÑĢе":125864,"ãģ¿ãģŁãģĦ":125865,"ìĿ´ëĿ¼":125866,"ĠاÙĦÙħÙĪ":125867,"ĠÑģÑĤо":125868,"à¹Ģรà¹ĩว":125869,"ĠдеÑĤ":125870,"ĠÑģдел":125871,"à¹Ģà¸Ĭืà¹Īà¸Ń":125872,"פ׳×Ļ":125873,"ÙĪضÙĪع":125874,"×ijס":125875,"à¹ģà¸Ķ":125876,"óc":125877,"ริม":125878,"ÑĢад":125879,"ìĪł":125880,"ãĥ¼ãĤº":125881,"ãģ«ãģĬ":125882,"ино":125883,"פ×Ļ׾":125884,"à¸Ĭัà¹Īà¸Ļ":125885,"×Ĺ×ĵש":125886,"à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ":125887,"׳×Ļס":125888,"غرب":125889,"ãĤ¸ãĥ£":125890,"สัà¸ĩ":125891,"à¹Ģà¸Ĺีà¹Ī":125892,"à¹Ģà¸Ĺีà¹Īยว":125893,"ëŁ¼":125894,"à¹ģà¸Ł":125895,"ãĥ¼ãĤ·":125896,"ãĥ¼ãĤ·ãĥ§ãĥ³":125897,"Ġвозмож":125898,"جÙħÙĪع":125899,"×ijר×Ļ×Ŀ":125900,"ãĥĪãĥ©":125901,"ĠкаÑĩеÑģÑĤв":125902,"Ø·ÙĬ":125903,"ÑĤÑı":125904,"צ×ķ×¢":125905,"ģını":125906,"عÙĦÙī":125907,"اذ":125908,"ÙĪاÙĤع":125909,"ÙħÙĪا":125910,"ائÙĬÙĦ":125911,"кол":125912,"á»ģm":125913,"à¸ľà¸¥à¸´à¸ķ":125914,"×Ļ׳×ĺר":125915,"سÙĥ":125916,"ש×Ļר":125917,"ศึà¸ģษา":125918,"à¸ļั":125919,"ÑĩаÑģ":125920,"×ķפ×Ķ":125921,"×Ļפ×ķ׾":125922,"ĠاÙĦساب":125923,"رÙĬب":125924,"ĠاÙĦبÙĬ":125925,"ãĤ¹ãĥĨ":125926,"Ñĩен":125927,"à¹ģà¸ľ":125928,"Ġ׳ש":125929,"زÙĬد":125930,"ØŃاد":125931,"ëįĶ":125932,"رÙĪع":125933,"à¸Ĺุà¸Ļ":125934,"สมา":125935,"czeÅĦ":125936,"×Ļ×ĵ×Ķ":125937,"ãģ§ãģĤ":125938,"Ġçocuk":125939,"خب":125940,"à¸ļาย":125941,"à¸Ľà¸£à¸°à¸Ĭา":125942,"×ŀש׾":125943,"ãģªãģĭ":125944,"à¸ģาย":125945,"ãĥģãĥ£":125946,"аÑĢи":125947,"ĠÑĩа":125948,"à¸Ķำ":125949,"à¸Ĺัà¹Īว":125950,"ÑĥÑħ":125951,"Ġöz":125952,"Ġì¢ĭ":125953,"جرÙĬ":125954,"ائÙĤ":125955,"à¸łà¸±à¸¢":125956,"طار":125957,"دارة":125958,"Ä©nh":125959,"Ø«ÙĨ":125960,"zellik":125961,"اÙĦت":125962,"Ġgeli":125963,"ãĥķãĤ©":125964,"олод":125965,"ربع":125966,"שת×ŀש":125967,"à¸ļรร":125968,"íĿ¬":125969,"Ġürün":125970,"Ġê·¸ëłĩ":125971,"ศาสà¸ķรà¹Į":125972,"ãģľ":125973,"×Ļ×ij׾":125974,"ĠпÑĢедÑģÑĤав":125975,"سطÙĬÙĨ":125976,"ãĤĴ使":125977,"ĠпомоÑī":125978,"×ķקר":125979,"ãĥ¯ãĥ¼":125980,"Ġyönet":125981,"×Ļקר":125982,"à¸Ĥา":125983,"еÑĢиал":125984,"ØŃÙģ":125985,"Ġ×Ļצ":125986,"à¸Ĺิ":125987,"売":125988,"à¸Ļà¸Ńà¸ģ":125989,"×ķ׼ר":125990,"íĻľ":125991,"ủy":125992,"ĠاÙĦÙĤر":125993,"×Ļ×ij×ķת":125994,"ÅĽni":125995,"Ùħشار":125996,"ượt":125997,"ĠÙĦدÙĬ":125998,"ÑĤел":125999,"ĠØ¥ÙĦÙĬ":126000,"عÙĦÙĪÙħ":126001,"ìķĺ":126002,"виÑĤ":126003,"à¸Ħะ":126004,"yrı":126005,"ãģ¨ãģ£ãģ¦":126006,"à¹Ģà¸ī":126007,"à¸ĸาม":126008,"ÙĤار":126009,"عÙĦاÙħ":126010,"ặng":126011,"ÙħÙĴ":126012,"×Ļ×ŀת":126013,"سبة":126014,"ãĤ¯ãĥ©":126015,"×ķסף":126016,"ĠпÑĢин":126017,"ãģĦãĤį":126018,"ساس":126019,"عتبر":126020,"วิà¸Ĺย":126021,"วิà¸Ĺยา":126022,"سÙĥر":126023,"ãĤ·ãĥ§":126024,"ãģģ":126025,"ัà¸ģษ":126026,"×ij×ķ×Ķ":126027,"หย":126028,"ãģ¾ãĤĮ":126029,"ĠоÑĢганиз":126030,"казал":126031,"ĠÑģвÑıз":126032,"uyết":126033,"ĠпÑĢоиз":126034,"Ġק×ĺ":126035,"à¹ģà¸ģà¹ī":126036,"пÑĥÑģ":126037,"Ġê·¸ê²ĥ":126038,"ëĬIJ":126039,"лекÑģ":126040,"ãĥ¼ãĥĹ":126041,"à¸ķำ":126042,"ת×Ĺ×Ļ׾":126043,"à¸Ńà¸ĩà¸Ħà¹Į":126044,"ẵ":126045,"׳צ":126046,"أش":126047,"Ø´Ùĩ":126048,"ยะ":126049,"à¸ģà¸İ":126050,"ĠاÙĦإسÙĦاÙħ":126051,"едÑĮ":126052,"ãģ²ãģ¨":126053,"ëıĦë¡Ŀ":126054,"ãģ©ãģ®":126055,"Ñĥв":126056,"еÑĩение":126057,"ĠاÙĦتج":126058,"ãģ«è¡Į":126059,"Ġпозв":126060,"ãĤıãĤĬ":126061,"ÙĦاث":126062,"íķĺìĺĢ":126063,"ĠмаÑĢ":126064,"ĠkonuÅŁ":126065,"ãĥ¬ãĤ¹":126066,"ãĤĴæĮģ":126067,"ĠоÑģнов":126068,"×Ĺ×ij":126069,"ÙĪجÙĪد":126070,"פ×ķף":126071,"воÑĢ":126072,"Ġник":126073,"ãģĭãĤĭ":126074,"ÅŁtırma":126075,"×Ļס×ĺ":126076,"Ø£ÙĦ":126077,"หà¹Į":126078,"иона":126079,"лÑĮн":126080,"ĠгоÑģ":126081,"ĠÐľÐ¾Ñģк":126082,"ÑĢоб":126083,"×ķ×IJ×Ļ":126084,"ãģĬãĤĬãģ¾ãģĻ":126085,"ãģ£ãģ±":126086,"кл":126087,"à¸Ļà¸Ķà¹Į":126088,"رÙĬÙģ":126089,"اسب":126090,"ĠÑĢеÑĪ":126091,"Ġдол":126092,"ãģ¹ãģį":126093,"×Ļ×ij×ķר":126094,"меÑī":126095,"ĠнаÑĪ":126096,"à¹ģà¸Ľà¸¥":126097,"ÑĢиÑĤ":126098,"кÑĥÑģ":126099,"иÑĢа":126100,"аÑĤÑĥÑĢ":126101,"ÙĪاصÙĦ":126102,"à¹Ģà¸ľà¸¢":126103,"à¸Ńำ":126104,"à¹Ģà¸ģิà¸Ļ":126105,"غÙħ":126106,"ãģĻãģİ":126107,"lıkl":126108,"ÅĦsk":126109,"견":126110,"×Ļ׼×Ķ":126111,"×Ĺש×ij":126112,"ÙĪرÙĬØ©":126113,"ĠдейÑģÑĤв":126114,"×Ĺ׾×ĺ":126115,"Ġ׾×ŀ×¢":126116,"צ׾×Ļ×Ĺ":126117,"еÑĩа":126118,"Ùģاع":126119,"×Ĵ×Ļ×ĵ":126120,"áºŃm":126121,"ÄĻb":126122,"شع":126123,"ãģıãĤĬ":126124,"à¸ŀุ":126125,"едеÑĢ":126126,"à¸Ĥà¸Ļ":126127,"à¸Ħาร":126128,"ĠболÑĮÑĪ":126129,"ãģıãģªãĤĬ":126130,"à¸ĵา":126131,"×ĵ×ķ×Ĵ":126132,"Ġмн":126133,"ä¸ĬãģĮ":126134,"ç¶ļãģį":126135,"ฤษ":126136,"à¸Ĩ":126137,"Ø®ÙĬ":126138,"à¹Ģà¸Ĺà¸ŀ":126139,"สัม":126140,"à¹Ģสà¸Ļ":126141,"à¹Ģสà¸Ļà¸Ń":126142,"ãĥ´":126143,"ĠиÑģÑĤ":126144,"باشر":126145,"ĠÑĥÑĢов":126146,"×ŀ×ķ×ĸ":126147,"abı":126148,"waż":126149,"×ķצ×IJ×Ķ":126150,"ÑĤвеÑĢ":126151,"à¸ŀัà¸Ļà¸ĺà¹Į":126152,"׳×Ĵ×ĵ":126153,"ãĤĭãģĵãģ¨ãģĮãģ§ãģį":126154,"ĠÑĤÑĢеб":126155,"à¸ģรุà¸ĩ":126156,"ØŃتاج":126157,"à¹Ģà¸Ħล":126158,"ãĨ":126159,"ÄĻtr":126160,"Ġszczeg":126161,"Ġרש":126162,"à¸Ĺà¸ĺ":126163,"Ġнек":126164,"ĠнекоÑĤоÑĢ":126165,"вÑĪ":126166,"Ь":126167,"à¹Īวย":126168,"ลุ":126169,"бÑĢÑı":126170,"หมูà¹Ī":126171,"à¹ģà¸ķà¸ģ":126172,"ר׼×Ļ×Ŀ":126173,"Ġíĸī":126174,"ãi":126175,"Ùĥرة":126176,"âŃ":126177,"íIJ":126178,"ãį":126179,"áģ":126180,"â®":126181,"â¥":126182,"ì®":126183,"à¿":126184,"â¿":126185,"áĤ":126186,"á¤":126187,"âł":126188,"íŁ":126189,"ðIJį":126190,"ðIJ°":126191,"ðĿĨ":126192,"ðŁĪ":126193,"Ġ×¢×ľ":126194,"ĠعÙĨ":126195,"ĠÙħع":126196,"Ġ×ĸ×Ķ":126197,"ĠÙħا":126198,"ĠmÃł":126199,"Ġdụ":126200,"á»ĩc":126201,"аÑħ":126202,"sı":126203,"íķĺê³ł":126204,"Ġ×ķ×ij":126205,"ĠÐŁÐ¾":126206,"×ķתר":126207,"ĠÙĦÙħ":126208,"Ġ×ķ׾":126209,"ãģĹãģ¦ãģĦãĤĭ":126210,"Ġ×ŀ×Ļ":126211,"ĠبÙĬÙĨ":126212,"за":126213,"ĠÙĥاÙĨ":126214,"Ġ×Ķ×Ļ×Ķ":126215,"ëħĦ":126216,"×IJ×ķ":126217,"ди":126218,"ĠпеÑĢе":126219,"dı":126220,"Ġ׾ש":126221,"Ġש×ŀ":126222,"ãģĮãģĤãĤĭ":126223,"ãģĦãģĦ":126224,"ÑĢе":126225,"ק×ķ":126226,"или":126227,"ме":126228,"ÙĬت":126229,"ãģ§ãģĤãĤĭ":126230,"Ġво":126231,"à¹ĥหม":126232,"à¹ĥหมà¹Ī":126233,"Ġש×ij":126234,"Ġà¹Ĥà¸Ķย":126235,"ÙĬÙĩ":126236,"ãģ§ãģĻãģĮ":126237,"ãģ¨ãģ¯":126238,"ר×ķ":126239,"Ġà¸ĭึà¹Īà¸ĩ":126240,"ãģ§ãģįãĤĭ":126241,"мо":126242,"à¹Ģà¸ŀืà¹Īà¸Ń":126243,"צ×ķ":126244,"×ĺ×ķ":126245,"ìķĪ":126246,"Ġhá»į":126247,"à¹Ģà¸ĩิà¸Ļ":126248,"ĠاÙĦب":126249,"Ġมี":126250,"물":126251,"Ñģе":126252,"ëĵ¤ìĿ´":126253,"Ġë§IJ":126254,"ĠlỼ":126255,"aÅĤ":126256,"×Ĺ×ijר":126257,"Ġdá»±":126258,"ÙĬØ«":126259,"Ġthá»ĭ":126260,"à¸ģà¹Īà¸Ńà¸Ļ":126261,"Ġ×ij׼׾":126262,"ãģ¸":126263,"ãģ¨æĢĿãģĦãģ¾ãģĻ":126264,"ảnh":126265,"ยา":126266,"Ùģا":126267,"สี":126268,"à¸ķา":126269,"ë²ķ":126270,"ãĥªãĥ¼":126271,"ราà¸Ħา":126272,"Ġ×ķ׾×IJ":126273,"ãģ¨ãģĵãĤį":126274,"à¹Ģลืà¸Ń":126275,"diÄŁi":126276,"ÙĪاÙĨ":126277,"Ġ׾×Ķת":126278,"รวม":126279,"פ×Ļ×Ŀ":126280,"à¸ľà¸¡":126281,"жи":126282,"cı":126283,"ÑĢод":126284,"ĠkarÅŁÄ±":126285,"×Ĵ×ķ":126286,"ãģ«ãģ¤":126287,"ãģ«ãģ¤ãģĦãģ¦":126288,"rÃł":126289,"×Ļ×ķתר":126290,"ĠìĨĮ":126291,"ק×Ķ":126292,"ÑģÑĤво":126293,"ãģijãģ©":126294,"gé":126295,"à¸Ķà¹īาà¸Ļ":126296,"çļĦãģ«":126297,"ĠÙĬÙħÙĥÙĨ":126298,"ìĨį":126299,"ÙĬÙĥ":126300,"à¹Ħวà¹ī":126301,"Ñģкий":126302,"ìm":126303,"Ġ׾×IJ×Ĺר":126304,"à¸Ńาหาร":126305,"Ġà¹Ģà¸ŀ":126306,"ราะ":126307,"ลูà¸ģ":126308,"ÑģÑĤа":126309,"Ġìľł":126310,"ÙĤÙĪÙĦ":126311,"боÑĢ":126312,"Ñģкого":126313,"หลัà¸ĩ":126314,"à¸Ĥà¹Īาว":126315,"à¹Ģมืà¸Ńà¸ĩ":126316,"ê°ģ":126317,"tÃł":126318,"ÙĬÙĬÙĨ":126319,"عرض":126320,"ë°©":126321,"ĠëıĻ":126322,"Ġà¹Ģà¸Ľ":126323,"Ġà¹Ģà¸Ľà¹ĩà¸Ļ":126324,"çi":126325,"liÄŁi":126326,"ìĹIJê²Į":126327,"ãĤ¿ãĥ¼":126328,"Ġ×ľ×ª":126329,"פ×ķת":126330,"à¸Ĥà¸Ń":126331,"رس":126332,"ìłIJ":126333,"à¸ľà¹Īาà¸Ļ":126334,"ÑĦи":126335,"جÙĨ":126336,"ì¢ħ":126337,"Ġ×Ķפ":126338,"Ġngo":126339,"á»ĭa":126340,"Ġtá»ķ":126341,"Ġ그리":126342,"à¹Ģมืà¹Īà¸Ń":126343,"Ø°Ùĥر":126344,"ìĸij":126345,"ìĹŃ":126346,"×ĺ׾":126347,"kı":126348,"ĠعÙħÙĦ":126349,"ĠعÙĨد":126350,"à¸ĭืà¹īà¸Ń":126351,"Ġê±°":126352,"ве":126353,"rü":126354,"à¹Ģà¸Ńา":126355,"สà¹Į":126356,"à¸Īà¸Ļ":126357,"סת":126358,"Ġgiả":126359,"ãĤĭãģ¨":126360,"à¸ģำลัà¸ĩ":126361,"ней":126362,"à¸Īริ":126363,"à¸Īริà¸ĩ":126364,"Ġëį":126365,"ĠëįĶ":126366,"à¸Ħà¹Īะ":126367,"ìn":126368,"Ġsüre":126369,"Ġquy":126370,"à¸ļาà¸ĩ":126371,"åıĸãĤĬ":126372,"ר×Ĺ":126373,"×ijת":126374,"ãģĮãģĤãĤĬãģ¾ãģĻ":126375,"רש":126376,"ìĹIJëĬĶ":126377,"Ġ×IJפשר":126378,"ayı":126379,"ãģĮãĤī":126380,"ØŃب":126381,"анÑģ":126382,"سÙĪ":126383,"ĠпÑĢе":126384,"دÙĪ":126385,"ãģ«ãĤĪ":126386,"à¹Ģà¸ģม":126387,"สูà¸ĩ":126388,"makt":126389,"maktad":126390,"maktadır":126391,"Ġönem":126392,"×Ļ×ŀ×Ļ×Ŀ":126393,"бо":126394,"ÙĪÙĬØ©":126395,"à¸£à¸¹à¸Ľ":126396,"à¹Ĥลà¸ģ":126397,"ÙħÙĬع":126398,"ÑģÑĤÑĥп":126399,"à¹Ĥà¸Ń":126400,"دÙĬÙĨ":126401,"ì¤ij":126402,"ãģĹãģı":126403,"à¹Ģสีย":126404,"вÑĭ":126405,"Ùħت":126406,"íĺĦ":126407,"ãĥIJãĥ¼":126408,"اش":126409,"קס":126410,"Ġtụ":126411,"ลà¸Ķ":126412,"ÙģØ©":126413,"íijľ":126414,"رج":126415,"kÅĤad":126416,"ĠÅŁey":126417,"ĠØ£Ùħ":126418,"Ġà¹Ģม":126419,"ĠبÙĦ":126420,"ÑģкаÑı":126421,"ãģ¨ãģ®":126422,"Ġìĭ¤":126423,"ấm":126424,"หà¹īà¸Ńà¸ĩ":126425,"à¸Ĭม":126426,"dü":126427,"Ġçek":126428,"Ġê³ł":126429,"×Ĵ×ij":126430,"à¸Ĭีวิ":126431,"à¸Ĭีวิà¸ķ":126432,"ÙģضÙĦ":126433,"ฯ":126434,"çı":126435,"Ġبش":126436,"ĠÙĩÙĨا":126437,"ãģįãģ¾ãģĹãģŁ":126438,"tü":126439,"Ġìĺģ":126440,"ĠTürk":126441,"кÑĤ":126442,"פרס":126443,"ãģ¨ãģĦãģĨãģĵãģ¨":126444,"íĶĦ":126445,"à¹ģรà¸ģ":126446,"ר×ķף":126447,"Ġaras":126448,"×ŀצ×IJ":126449,"Ġtá»ī":126450,"سا":126451,"à¸ŀà¸Ń":126452,"ĠاÙĦÙħØŃ":126453,"ãĥ¤":126454,"ĠاÙĦاست":126455,"ÙģÙĨ":126456,"×Ļ×ŀ×Ķ":126457,"رت":126458,"ãģ¨ãĤĤ":126459,"ĠнаÑģ":126460,"пÑĢи":126461,"Ġ×Ĺ×ķ":126462,"ила":126463,"ÙĬØ´":126464,"Ġgöz":126465,"Ġ×ij׳×Ļ":126466,"ımı":126467,"ĠÑĤеÑħ":126468,"Ġhá»Ļ":126469,"غر":126470,"кон":126471,"اØŃت":126472,"Ġà¸ŀ":126473,"à¸Ńà¸Ńà¸Ļ":126474,"à¸Ńà¸Ńà¸Ļà¹Ħล":126475,"à¸Ńà¸Ńà¸Ļà¹Ħลà¸Ļà¹Į":126476,"Ñħо":126477,"Ñıв":126478,"à¹ģสà¸Ķ":126479,"à¹ģสà¸Ķà¸ĩ":126480,"à¹Ģà¸ŀียà¸ĩ":126481,"ÑĤов":126482,"اÙĬ":126483,"Ġ×Ķ×ĵ":126484,"Ġ×ķ׼":126485,"ãĤīãģĦ":126486,"×ķפף":126487,"Ġë¶Ī":126488,"ลà¸Ńà¸ĩ":126489,"طاÙĦ":126490,"Ġни":126491,"ĠÙħست":126492,"ếc":126493,"Ġש׼":126494,"ĠëķĮ문":126495,"วัà¸Ļà¸Ĺีà¹Ī":126496,"×Ļ׾×ĵ":126497,"ØŃا":126498,"еÑĨ":126499,"Ġcứ":126500,"×ĵ×ķר":126501,"ĠÙħØŃ":126502,"ר׼×ij":126503,"بÙĬع":126504,"нии":126505,"ĠاÙĦØ£ÙĪÙĦ":126506,"à¸Ħวร":126507,"ãģ¨æĢĿãģĨ":126508,"ĠСо":126509,"ائÙĬØ©":126510,"راء":126511,"оÑģоб":126512,"ĠبأÙĨ":126513,"×¢×ķ×ĵ":126514,"ĠÑĤе":126515,"ãģĵãģĨ":126516,"ÑģÑĤÑĢа":126517,"айн":126518,"Ġsöz":126519,"تÙĨا":126520,"à¸Ńิ":126521,"ặp":126522,"ĠìķĦëĭĪ":126523,"íķŃ":126524,"Ġר×IJש":126525,"Ġà¹Ħà¸Ķà¹ī":126526,"Ġ×Ĵ×ĵ":126527,"Ġספר":126528,"обÑīе":126529,"ĠÙĪØ¥":126530,"adaÅŁ":126531,"ãģ¡ãĤĩ":126532,"ק×ķ׾":126533,"ÑĢез":126534,"ĠdÃ¼ÅŁÃ¼n":126535,"Ġ×ij×IJ×ŀ":126536,"Ġìĸ´ëĸ":126537,"ער×ij":126538,"нее":126539,"ĠÑģÑĤÑĢан":126540,"ساÙĨ":126541,"ynı":126542,"ĠاÙĦرئÙĬس":126543,"ãģĹãģª":126544,"Ġ×ł×ª":126545,"ãģ«ãģªãģ£ãģŁ":126546,"gü":126547,"åıĹãģij":126548,"×ľ×ª":126549,"ìłĪ":126550,"ëĬĶëį°":126551,"Ø®ÙĬر":126552,"à¸ķà¹īà¸Ńà¸ĩà¸ģาร":126553,"ĠÙĦØ£ÙĨ":126554,"Ġchá»ĭ":126555,"ÙĪØ©":126556,"à¹ĥส":126557,"ë¶ĢíĦ°":126558,"íķĺë©´":126559,"ữu":126560,"à¹Ģหมืà¸Ńà¸Ļ":126561,"беÑĢ":126562,"ĠìĿ´ìļ©":126563,"ĠÑģеб":126564,"wiÄĻks":126565,"Ġ×ł×¢":126566,"ÑĤÑĥÑĢ":126567,"ĠnghÄ©":126568,"ש×ķ×ĺ":126569,"tiÄŁi":126570,"ĠdeÄŁi":126571,"×IJ×ij":126572,"Ġ×ŀ×ŀ":126573,"ãĥĹãĥŃ":126574,"waÅĤ":126575,"à¸Īึà¸ĩ":126576,"خدÙħ":126577,"×IJ×Ŀ":126578,"Ä±ÅŁÄ±":126579,"czÄħ":126580,"ר×ĵ":126581,"ĠÑĢÑĥб":126582,"خرÙī":126583,"ãģ®æĸ¹":126584,"ĠденÑĮ":126585,"×Ĺ×Ļ×Ŀ":126586,"еÑĤе":126587,"ëĤľ":126588,"×IJ×Ĵ":126589,"×¢×ķר":126590,"ë³Ħ":126591,"åIJĮãģĺ":126592,"ãĤ²":126593,"ר×ļ":126594,"×ķש×IJ":126595,"ìľ¡":126596,"اخ":126597,"צ×Ļ×Ķ":126598,"á»±a":126599,"ãģĪãģ¦":126600,"ש×Ķ×ķ":126601,"анÑĤ":126602,"ลาà¸Ķ":126603,"инг":126604,"ë¡ł":126605,"اعد":126606,"ÙĪسط":126607,"Ġвоп":126608,"ĠвопÑĢоÑģ":126609,"ÙħÙĬÙĨ":126610,"à¸Ħà¸ĩ":126611,"×Ļר×Ļ×Ŀ":126612,"ców":126613,"격":126614,"Ġê·¸ëŁ°":126615,"Ġì§Ħ":126616,"Ġש׾×Ķ":126617,"à¹Ģริà¹Īม":126618,"à¸Ĭà¸Ńà¸ļ":126619,"деÑĤ":126620,"ÑİÑīиÑħ":126621,"à¸ļà¸Ńà¸ģ":126622,"æĢĿãģĦ":126623,"عÙĬد":126624,"ס×ŀ":126625,"×Ĵ×Ļ×¢":126626,"צ×ĵ":126627,"بات":126628,"ĠëĶ°ëĿ¼":126629,"à¸Īัà¸ĩ":126630,"ãģłãģijãģ§":126631,"×¢×Ļר":126632,"ĠÑĩел":126633,"ĠÑĩелов":126634,"ĠÑĩеловек":126635,"ãĥĥãĥģ":126636,"à¹Ģà¸ģีà¹Īยว":126637,"à¸Ķิ":126638,"Ġפע":126639,"×Ļ×ŀ×Ļ":126640,"ë°ĺ":126641,"خار":126642,"×ij×Ļת":126643,"×¢×Ļ×Ŀ":126644,"üyor":126645,"ãĤģãģ¦":126646,"клад":126647,"Ġà¸Īาà¸ģ":126648,"à¹Ģà¸Ħย":126649,"สà¸Ńà¸ĩ":126650,"à¹ģà¸Ħà¹Ī":126651,"ẫu":126652,"หà¸Ļัà¸ĩ":126653,"ש׾×ķ×Ŀ":126654,"اÙĨÙĬØ©":126655,"åĩºä¼ļ":126656,"åĩºä¼ļãģĦ":126657,"à¸łà¸²à¸¢":126658,"à¸ļาà¸Ĺ":126659,"à¸Ĭาว":126660,"muÅŁ":126661,"Ġ׾ק×ij׾":126662,"ãĤ·ãĥ£":126663,"ĠÄ°ÅŁ":126664,"×Ĵ×ĵ×ķ׾":126665,"جعÙĦ":126666,"ë³Ģ":126667,"ยิà¹Īà¸ĩ":126668,"à¸Ļาย":126669,"à¸Ļีà¹Ī":126670,"วิà¸ĺี":126671,"ãĤīãģªãģĦ":126672,"ëłĪ":126673,"Ġë¬¸ìłľ":126674,"Ġà¸ģ":126675,"à¸Ĺำà¸ĩาà¸Ļ":126676,"à¹Ģวà¹ĩà¸ļ":126677,"ÑĦе":126678,"楽ãģĹ":126679,"สำà¸Ħ":126680,"สำà¸Ħัà¸į":126681,"رÙħ":126682,"ãģķãĤĮãģ¦":126683,"Ġобла":126684,"ר×IJ×Ļ":126685,"หมà¸Ķ":126686,"ÙĨÙĬØ©":126687,"лин":126688,"ĠeÄŁ":126689,"itim":126690,"ëł¹":126691,"صاÙĦ":126692,"ÅĽl":126693,"à¸ľà¸´à¸Ķ":126694,"ãĥŀãĥ³":126695,"åħ¥ãĤĮ":126696,"à¹Ģà¸ķà¸Ńรà¹Į":126697,"ارÙĬ":126698,"ĠЦ":126699,"dür":126700,"สวย":126701,"립":126702,"رÙĥØ©":126703,"Ġhã":126704,"×Ļת×Ķ":126705,"à¸Ĥà¸Ļา":126706,"à¸Ĥà¸Ļาà¸Ķ":126707,"à¸Īำà¸Ļ":126708,"à¸Īำà¸Ļวà¸Ļ":126709,"ש×ķק":126710,"Ġдом":126711,"ì±ħ":126712,"ãģĭãģij":126713,"פ×ķ׾":126714,"à¸Ĭาย":126715,"ÑģмоÑĤÑĢ":126716,"ÑģлÑĥж":126717,"ש×IJ׾":126718,"кÑĢÑĭÑĤ":126719,"Ġìŀĺ":126720,"é«ĺãģĦ":126721,"ĠÑĢÑĥк":126722,"ÙĨص":126723,"дав":126724,"ưỡ":126725,"ưỡng":126726,"راÙħ":126727,"×Ļ׳×Ļ×Ŀ":126728,"ãĥ©ãĥ¼":126729,"ëĦ¤":126730,"Ġتع":126731,"lke":126732,"好ãģį":126733,"æĮģãģ¡":126734,"Ġë§İ":126735,"Ġyük":126736,"ĠÑģоÑģÑĤав":126737,"енÑĤÑĢ":126738,"peÅĤ":126739,"à¹Ģà¸Ľà¸¥à¸µà¹Īย":126740,"à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļ":126741,"íıī":126742,"ãĤĦãģĻ":126743,"×Ĺ×ĸ":126744,"×ijר×Ķ":126745,"루":126746,"ìĶĢ":126747,"بØŃØ«":126748,"à¹Ģà¸ķà¹ĩ":126749,"ówi":126750,"بÙĩ":126751,"ãģįãģ¾ãģĻ":126752,"Ġ×¢×ŀ":126753,"×Ĵ×ķ׾":126754,"езд":126755,"ÙĬÙģØ©":126756,"สà¸Ļà¹ĥà¸Ī":126757,"Ġ×ª×ľ":126758,"ÑıÑī":126759,"ĠسÙĨ":126760,"ĠÙĪاØŃد":126761,"ĠÑģм":126762,"ladı":126763,"ıld":126764,"×Ļרת":126765,"ียà¸Ļ":126766,"ת×Ĺת":126767,"Ġжиз":126768,"à¸ŀั":126769,"à¸ŀัà¸Ĵ":126770,"à¸ŀัà¸Ĵà¸Ļา":126771,"à¸Ĭิ":126772,"اخÙĦ":126773,"ãģ£ãģ¦ãģĦãģŁ":126774,"รัà¸IJ":126775,"ãĤģãĤĭ":126776,"à¹Ĥà¸ģ":126777,"ĠTá»ķ":126778,"Ġhakk":126779,"رÙģ":126780,"ìłĢ":126781,"Ñģоб":126782,"ãģªãģijãĤĮãģ°":126783,"ÙĩÙĪ":126784,"Ġë²ķ":126785,"ãĤĨ":126786,"ĠاÙĦسعÙĪد":126787,"Ġ×IJתר":126788,"اغ":126789,"Ġ׾×ĵ":126790,"à¹ģà¸ķ":126791,"à¹ģà¸ķà¹Īà¸ĩ":126792,"íĮĮ":126793,"ÑĥпиÑĤÑĮ":126794,"à¸ŀืà¹īà¸Ļà¸Ĺีà¹Ī":126795,"×ijת×Ļ":126796,"à¹ĩà¸ģ":126797,"ÅĤat":126798,"Ġê°ľìĿ¸":126799,"ìłķë³´":126800,"ÑĤал":126801,"Ġgüven":126802,"ĠÄ°l":126803,"Ġê°ģ":126804,"Ġبت":126805,"×ŀ×ķ׳×Ķ":126806,"ĠاÙĦØŃÙĥÙĪÙħ":126807,"ÙĤات":126808,"à¹ģà¸ģà¹Ī":126809,"หาà¸ģ":126810,"нÑĮ":126811,"à¸Ľà¸£à¸±à¸ļ":126812,"มาà¸ĵ":126813,"ĠнеÑģк":126814,"Ġض":126815,"สมั":126816,"สมัà¸Ħร":126817,"ãģĮãģĤãĤĬ":126818,"меÑģÑĤ":126819,"Ġ×IJצ׾":126820,"Ġкомпани":126821,"סר":126822,"ÙĬÙħØ©":126823,"ĠÑħоÑĢо":126824,"ĠÑħоÑĢоÑĪ":126825,"Ġ×Ļ×ķ×ĵ":126826,"üs":126827,"×Ĵ×Ļש":126828,"à¸ļà¸Ĺ":126829,"تÙĨظ":126830,"วาà¸ĩ":126831,"มหา":126832,"Ġ׼×ķ׾":126833,"à¸Ĥà¹īาà¸ĩ":126834,"ë°ľ":126835,"год":126836,"дан":126837,"ãģĭãĤĤãģĹãĤĮãģ¾ãģĽãĤĵ":126838,"ãģĵãģ¡ãĤī":126839,"ãĥIJãĤ¤":126840,"eceÄŁi":126841,"دÙĬدة":126842,"ÙĨÙī":126843,"Ġëĭ¤ìĿĮ":126844,"วี":126845,"غا":126846,"лиз":126847,"à¹Ģà¸Ķิ":126848,"à¹Ģà¸Ķิม":126849,"ĠÙĬست":126850,"Ġyılı":126851,"koÅĦ":126852,"ãģ§ãģĹãĤĩãģĨãģĭ":126853,"ãģĤãģª":126854,"ãģĤãģªãģŁ":126855,"ÑĨен":126856,"ĠÙĪز":126857,"×IJ×Ļש":126858,"à¹Īà¸Ń":126859,"رØŃ":126860,"ê´ij":126861,"ÑĢаÑģÑĤ":126862,"Ġ×Ķ׾":126863,"ãģĹãģ¦ãĤĤ":126864,"×ŀר׼":126865,"×ŀר׼×ĸ":126866,"éģķãģĦ":126867,"ãģŁãģı":126868,"ĠÑģÑĥд":126869,"веÑģÑĤи":126870,"ĠíķĦìļĶ":126871,"ãĥķãĤ§":126872,"ÑĤелÑĮно":126873,"à¹Ģà¸ŀืà¹Īà¸Ńà¸Ļ":126874,"ÅĤuż":126875,"à¹Ģà¸Ķิà¸Ļà¸Ĺาà¸ĩ":126876,"ש×ķר":126877,"Ġ×ŀ×ĵ":126878,"×ķ×¢×ľ":126879,"ÙĦاÙħ":126880,"à¹Ħà¸ĭ":126881,"лей":126882,"кÑĥÑĢ":126883,"Ả":126884,"à¸Ĺาà¸Ļ":126885,"ì§ij":126886,"ĠгоÑĢод":126887,"רס":126888,"׾×ķ×Ĵ":126889,"masını":126890,"ĠлÑĥÑĩ":126891,"ลà¹Īา":126892,"ìļ¸":126893,"ש×ĺ":126894,"ĠÐĺн":126895,"íĤ¤":126896,"ÙĪÙĦا":126897,"ìķł":126898,"ĠØ£ÙĬضا":126899,"Ùĥار":126900,"ĠاÙĦتع":126901,"สูà¹Ī":126902,"ãĤ¼":126903,"×ij×Ļ×IJ":126904,"ยà¸ģ":126905,"ĠØŃÙĤ":126906,"ربÙĬ":126907,"ãģĺãĤĥãģªãģĦ":126908,"รัà¸ģษา":126909,"ÑħодиÑĤ":126910,"à¸ķà¸Ńà¸ļ":126911,"׳×ĺ×Ļ":126912,"ĠاÙĦÙħج":126913,"تÙħع":126914,"оваÑĤÑĮ":126915,"ÙĦÙĬÙĨ":126916,"×Ļ×ŀ×ķת":126917,"Ġmù":126918,"nÄĻ":126919,"ĠدÙĬ":126920,"׼ש×Ļ×ķ":126921,"Ġhiç":126922,"ëijIJ":126923,"ÙĪاء":126924,"ÙĪØ·":126925,"ĠاÙĦبÙĦ":126926,"à¹ģมà¹ī":126927,"ק×ķת":126928,"ÙĪجد":126929,"å§ĭãĤģ":126930,"ÙĬئة":126931,"Ġ매":126932,"صبØŃ":126933,"פ×IJ":126934,"гоÑĢ":126935,"ס×Ķ":126936,"بÙĬÙĤ":126937,"ยาà¸ģ":126938,"Ġнад":126939,"ÙĬÙij":126940,"ĠبÙĪ":126941,"ס×ķר":126942,"ÙħÙĥاÙĨ":126943,"ר×ij":126944,"×Ĵ×ĸ":126945,"צת":126946,"bilit":126947,"лаг":126948,"ĠNgo":126949,"×IJ×ķר":126950,"à¸ķà¸Ļ":126951,"íĬ¹":126952,"à¸Ĺีà¹Īà¸Ķี":126953,"à¸Ľà¸£à¸°à¸Īำ":126954,"ование":126955,"ãģĦãģ¤":126956,"ãĥĥãĤ¯ãĤ¹":126957,"åIJĪãĤı":126958,"åIJĪãĤıãģĽ":126959,"×Ļ׳×ķ×Ļ":126960,"ạy":126961,"Ø«ÙĤ":126962,"ĠпÑĢоб":126963,"ĠпÑĢоблем":126964,"ÅŁeh":126965,"ÅŁehir":126966,"عادة":126967,"اÙĨÙĪÙĨ":126968,"à¸ķัวà¹Ģà¸Ńà¸ĩ":126969,"ì¶ķ":126970,"ılan":126971,"бан":126972,"ãĥ³ãĥī":126973,"à¸Īี":126974,"Ġ×Ķש׳×Ļ":126975,"поÑĤ":126976,"×ķ׾×Ļ×Ŀ":126977,"ลัà¸ļ":126978,"ĠÑįÑĤи":126979,"×ijקש":126980,"ë¹ĦìĬ¤":126981,"à¸Ńยà¹Īาà¸ĩà¹Ħร":126982,"×Ļ׾×Ļ":126983,"à¹ĥà¸Ĭà¹Ī":126984,"ĠاÙĦÙĥÙĦ":126985,"ãĥļãĥ¼ãĤ¸":126986,"صة":126987,"ÑĤиÑĢ":126988,"ãĤĵãģ©":126989,"зÑĭк":126990,"wyż":126991,"ÙĩÙĬ":126992,"ĠÙħÙĦÙĬ":126993,"Ġвиде":126994,"ظاÙħ":126995,"داÙĪÙĦ":126996,"×ŀת×Ļ":126997,"Ġsık":126998,"à¹Ģà¸ķิม":126999,"ãĤ¢ãĤ¤":127000,"каÑħ":127001,"צ×Ļ׾":127002,"à¹Ģà¸Ĭà¹Īà¸Ļ":127003,"маг":127004,"магаз":127005,"магазин":127006,"à¸Ľà¸±":127007,"à¸Ľà¸±à¸Ī":127008,"Ġש×Ļר×ķת":127009,"ียม":127010,"ãĥĸãĥ«":127011,"ĠدÙĪÙĦ":127012,"קר×Ļ×Ŀ":127013,"ÙĩÙı":127014,"ово":127015,"Ġüret":127016,"دÙĪÙĨ":127017,"à¹ģà¸Ļว":127018,"à¹Ģà¸Ļืà¹īà¸Ń":127019,"ĠÑĦоÑĤ":127020,"ãĥĺ":127021,"ãģ¤ãģĭ":127022,"ÑıÑģ":127023,"ĠíķĺëĤĺëĭĺ":127024,"ائع":127025,"ĠплаÑĤ":127026,"ìĺĪ":127027,"ĠdostÄĻp":127028,"ÙĪجÙĩ":127029,"Ġ×Ķ×Ĺ×Ļ":127030,"׳×Ļק":127031,"дей":127032,"íĽĦ":127033,"ıy":127034,"بØŃر":127035,"à¹Ģสริม":127036,"Ġ׾×Ĵ":127037,"Ø°Ùĩب":127038,"جÙĬÙĦ":127039,"رÙĥز":127040,"Ġëħ":127041,"Ġëħ¸":127042,"פ×Ļ׾×ķ":127043,"ãģ¾ãģļ":127044,"iriÅŁ":127045,"ĠÙĥÙĬÙģ":127046,"Ġ×ijצ":127047,"ĠêµIJ":127048,"ÑĢоÑģÑģ":127049,"ĠØ´ÙĬ":127050,"Ġiçer":127051,"×Ĵ×ķ×ij×Ķ":127052,"менно":127053,"×¢×ij×Ļר":127054,"×ķ×ŀ×Ķ":127055,"ãĤīãģĹãģĦ":127056,"ãģ¼":127057,"Ñīин":127058,"è²·ãģĦ":127059,"جÙħÙĪعة":127060,"Ġdönem":127061,"Ġ×ij×IJר":127062,"веÑģÑĤ":127063,"×ķר×ķת":127064,"سÙģ":127065,"à¹ģà¸Ĺà¸Ļ":127066,"ĠдокÑĥменÑĤ":127067,"ĠاÙĬ":127068,"جاÙĨ":127069,"צ×ķ×¢×Ļ":127070,"ĠоÑģоб":127071,"ĠاÙĦÙħس":127072,"ÑĢаб":127073,"à¸łà¸¹":127074,"à¸Ķาว":127075,"лекÑĤ":127076,"عÙĤ":127077,"×ķ×ĵ×ķת":127078,"Ġolu":127079,"ĠoluÅŁtur":127080,"ãģ¾ãģ¾":127081,"един":127082,"à¹Ģà¸Ńà¸ģ":127083,"ãĤµãĤ¤":127084,"ëĦĪ":127085,"Ø·ÙĨÙĬ":127086,"Ø·ÙĤØ©":127087,"ĠÐłÐ°Ð·":127088,"ÙĦÙij":127089,"Ñĩем":127090,"Ġ׾×ĺ":127091,"สัà¹Īà¸ĩ":127092,"سرائÙĬÙĦ":127093,"Ġפר×ĺ×Ļ":127094,"деÑģÑĮ":127095,"Ġ׳׼":127096,"اÙĨب":127097,"ÙĬاة":127098,"Ùħبر":127099,"Ġkı":127100,"à¸Ľà¸ı":127101,"à¸Ľà¸ıิ":127102,"à¸ļัà¸ķิ":127103,"×ł×ª×Ļ":127104,"ìĨ¡":127105,"راب":127106,"à¹ĥà¸ķ":127107,"à¹ĥà¸ķà¹ī":127108,"×Ļ×ł×ª":127109,"ÙĪÙĬر":127110,"Ġ×Ķ×ŀ×Ļ":127111,"ейÑĩаÑģ":127112,"ק×ķ×ij":127113,"دراس":127114,"ĠÙħÙĤ":127115,"رÙĬÙĨ":127116,"خاص":127117,"ãģĬéĩij":127118,"Ġجدا":127119,"ãģĨãģ¡":127120,"ëħ¸":127121,"ırım":127122,"æ§ĺ":127123,"ãģ«å¯":127124,"ãģ«å¯¾":127125,"ÑĨев":127126,"Ġvard":127127,"ĠÐIJн":127128,"eÄŁ":127129,"ÑģÑĤвенно":127130,"Ш":127131,"سد":127132,"à¸ģุ":127133,"à¹ģà¸ľà¸Ļ":127134,"รูà¹īส":127135,"รูà¹īสึà¸ģ":127136,"اتØŃاد":127137,"ÑijÑĤ":127138,"×Ĺ×ķק":127139,"ãģĻãģIJ":127140,"Ø·ÙĦاÙĤ":127141,"Ġק×ķ×ĵ":127142,"à¹ĥà¸Ĭà¹īà¸ĩ":127143,"à¹ĥà¸Ĭà¹īà¸ĩาà¸Ļ":127144,"ãĥ¼ãĤ¿":127145,"Ġsür":127146,"ÑĢок":127147,"ë³ij":127148,"สมาà¸Ĭ":127149,"สมาà¸Ĭิà¸ģ":127150,"ãĥķãĥ¬":127151,"è¾¼ãģ¿":127152,"ãĤ»ãĥ³":127153,"Ġê°Ģì§Ģ":127154,"à¸ľà¹īา":127155,"ÑįÑĤомÑĥ":127156,"иÑĤел":127157,"à¸łà¸±":127158,"à¸ij":127159,"ãĥĸãĥ©":127160,"×Ľ×ª×ķ×ij":127161,"׳×Ŀ":127162,"еннÑĭе":127163,"×¢×¨×Ľ×ª":127164,"ĠìĤ":127165,"ĠìĤ´":127166,"à¸Ĥà¹īา":127167,"׳×ķס":127168,"ãĥ¬ãĥĵ":127169,"ÑĢеÑģ":127170,"à¹Ģลà¸Ĥ":127171,"ثاÙĦ":127172,"ìĹĨ":127173,"ĠÑĩаÑģÑĤ":127174,"าศ":127175,"ãĥªãĤ¢":127176,"uç":127177,"×Ļ׼×ķת":127178,"ลà¹īาà¸Ļ":127179,"ië":127180,"ãĤ¸ãĤ§":127181,"à¸Īà¸Ń":127182,"ÙĪØŃد":127183,"×Ļצ×ķ×ij":127184,"Ġ×ijש׾":127185,"око":127186,"ضة":127187,"ذر":127188,"ĠÑĥд":127189,"Ä°L":127190,"×ķצ×Ļ×Ŀ":127191,"×ĸ×ŀף":127192,"à¸Ľà¸ģ":127193,"íķĻêµIJ":127194,"ساÙħ":127195,"à¹Ħà¸Ķ":127196,"ละà¹Ģà¸Ń":127197,"ละà¹Ģà¸Ńีย":127198,"ละà¹Ģà¸Ńียà¸Ķ":127199,"ảy":127200,"аÑĨион":127201,"ãĤ¹ãĤ¯":127202,"פ×ķס":127203,"รà¹Īาà¸ĩ":127204,"еннÑĭй":127205,"عÙĨ":127206,"عÙĦÙĨ":127207,"ائÙģ":127208,"dÄĻ":127209,"ؤÙĪÙĦ":127210,"׾×ķ×ķ":127211,"Ġ×ijש×ij":127212,"ä»ĬåĽŀ":127213,"ĠاÙĦجÙĨ":127214,"داد":127215,"waÄĩ":127216,"ãĥªãĥ³":127217,"ĠìŀIJìĭł":127218,"اÙĨÙĬا":127219,"ãĥ¡ãĥª":127220,"ÙĦÙĪÙĨ":127221,"à¸Ĺà¹Īà¸Ńà¸ĩ":127222,"à¸Ĺà¹Īà¸Ńà¸ĩà¹Ģà¸Ĺีà¹Īยว":127223,"اÙģÙĬ":127224,"ĠлиÑĪ":127225,"ÙħÙĬØ©":127226,"оÑĤвеÑĤ":127227,"Ñĩин":127228,"ÃĬ":127229,"ãĥ¡ãĥ³":127230,"å®Ł":127231,"éļĽãģ«":127232,"ĠÑĢай":127233,"ãĤ¦ãĥ³":127234,"×Ļר×ķש":127235,"×Ļר×ķש׾×Ļ×Ŀ":127236,"มะ":127237,"Ġara":127238,"казаÑĤÑĮ":127239,"à¸ķัà¸Ķ":127240,"ÑĥÑİÑĤ":127241,"Ġüst":127242,"×Ĵ×ķ×ij":127243,"×Ĵ×ķ×ij×ķת":127244,"malı":127245,"егод":127246,"егоднÑı":127247,"اÙģÙĤ":127248,"à¸Ĭà¹Īà¸Ńà¸ĩ":127249,"Ġözellik":127250,"×Ļצ×ķר":127251,"ĠmiÄĻd":127252,"ĠiliÅŁ":127253,"ĠнаÑħод":127254,"×¢×ĸר":127255,"×ľ×Ľ×ª":127256,"ÙĨتاج":127257,"ĠÑģем":127258,"à¸Īà¹Īาย":127259,"à¸ķรว":127260,"à¸ķรวà¸Ī":127261,"פר×ķ":127262,"à¸Ĥัà¸ļ":127263,"ãģŀ":127264,"Ġпло":127265,"колÑĮ":127266,"×ŀ×¢×ĺ":127267,"íķĺìĭľ":127268,"jÄħce":127269,"ÙĨاÙĨ":127270,"ลีà¸ģ":127271,"нÑĥÑĤ":127272,"ĠобÑĢаз":127273,"Ùĥبر":127274,"ĠاÙĦÙĪØ·ÙĨ":127275,"ãģķãģĽãģ¦":127276,"ÙĤاء":127277,"×ŀ×ĵ×Ļ׳":127278,"yü":127279,"פ×Ļת":127280,"׳×ķף":127281,"ÙħÙĨظ":127282,"หà¸Ļัà¸ģ":127283,"ìŀĪ":127284,"ãĤ«ãĥ¼ãĥī":127285,"عÙĨÙĬ":127286,"под":127287,"ضاء":127288,"à¸Ļà¸ķà¹Į":127289,"×ŀשפ":127290,"วà¹Į":127291,"ר×ķק":127292,"สืà¹Īà¸Ń":127293,"פק×Ļ×ĵ":127294,"ãģªãĤīãģªãģĦ":127295,"ĠìŬ룬":127296,"ÙĦج":127297,"ÑīиÑĤ":127298,"ãĥĥãĤ·":127299,"ÙĦÙĬس":127300,"ĠÙĦÙħا":127301,"ìłij":127302,"×ij×Ļף":127303,"ãĥģãĤ§":127304,"Ġgüç":127305,"Ġchứ":127306,"×ķצ×IJ":127307,"קר×ij":127308,"à¹Ĥà¸ŀ":127309,"оÑĩно":127310,"סק×Ļ":127311,"ש׾×Ŀ":127312,"صرÙģ":127313,"ĠLÃł":127314,"×¢×Ļת":127315,"á»·":127316,"à¹Ĥà¸Ńà¸ģ":127317,"à¹Ĥà¸Ńà¸ģา":127318,"à¹Ĥà¸Ńà¸ģาส":127319,"Ġ×Ķ×ĵ×ijר":127320,"à¸Ļัà¹Īà¸Ļ":127321,"زر":127322,"нако":127323,"íļį":127324,"ãĤĤãģ¡":127325,"ãĤĤãģ¡ãĤį":127326,"ãĤĤãģ¡ãĤįãĤĵ":127327,"اÙħت":127328,"عداد":127329,"инÑĭ":127330,"ÅĤyw":127331,"à¸Ħà¸ĵะ":127332,"à¸Ĺะ":127333,"ktör":127334,"×Ļ×Ĺ×Ķ":127335,"Ġме":127336,"ĠмеÑģÑı":127337,"׳×Ķ×Ĵ":127338,"ĠÑģÑĥÑīеÑģÑĤв":127339,"à¸Ļัà¸Ļ":127340,"ÑĦÑĦ":127341,"екÑĤив":127342,"عÙĦÙĪÙħات":127343,"бÑĥд":127344,"à¸Ļัà¸ģà¸ĩาà¸Ļ":127345,"หà¸Ļà¹īาà¸Ĺีà¹Ī":127346,"ÙĤÙĬÙĤ":127347,"ãĤ·ãĥ³":127348,"ãģ«éĸ¢":127349,"×IJר×Ĵ":127350,"ĠпÑĢоÑĤ":127351,"ĠпÑĢоÑĤив":127352,"ĠìŀĪìĸ´":127353,"ÙĤÙĬÙĤØ©":127354,"ìĹĩ":127355,"kür":127356,"ãģ«ãģªãĤĬãģ¾ãģĹãģŁ":127357,"ĠдеÑıÑĤ":127358,"ĠдеÑıÑĤелÑĮ":127359,"פ×ķר×ĺ":127360,"à¸Łà¹īา":127361,"à¹Ģà¸ł":127362,"ĠавÑĤомаÑĤ":127363,"×ĸ×Ļק":127364,"Ġolduk":127365,"عاÙħ":127366,"ĠÑĤоÑĢ":127367,"yrıca":127368,"êÌ":127369,"ãĤŃãĥ³ãĤ°":127370,"ãģ«ãģ¨ãģ£ãģ¦":127371,"à¹Ģà¸īà¸ŀ":127372,"à¹Ģà¸īà¸ŀาะ":127373,"ãģ¯ãģļ":127374,"×ŀ×IJ×Ļ":127375,"สะà¸Ķ":127376,"สะà¸Ķวà¸ģ":127377,"ìľ¼ë©°":127378,"à¸ģี":127379,"ฬ":127380,"Ġ×¢×ķש":127381,"à¸łà¸²à¸©à¸²":127382,"à¸Ĺัà¸Ļ":127383,"acakt":127384,"acaktır":127385,"اعدة":127386,"ĠÑĥÑģлÑĥг":127387,"סר×ĺ":127388,"×ķ×ŀ×ķת":127389,"×Ķ×ķר":127390,"×ŀ×ķ×ij":127391,"×ŀ×ķ×ijף":127392,"سÙĬاس":127393,"اتÙģاÙĤ":127394,"×Ķצ׾":127395,"Ùħؤس":127396,"Ġpó":127397,"Ġкни":127398,"×Ļ׼×ķ׾":127399,"à¹Ģหลืà¸Ń":127400,"׼׾׼":127401,"׳×ĸ":127402,"ÑĪие":127403,"rès":127404,"ĠاÙĦØŃÙĤ":127405,"лÑıÑĢ":127406,"หà¸į":127407,"หà¸įิà¸ĩ":127408,"ר×Ĵ×Ļש":127409,"à¹Ģสà¹īà¸Ļ":127410,"ש×ij×ķף":127411,"ôtel":127412,"апÑĢ":127413,"апÑĢимеÑĢ":127414,"ابÙĦ":127415,"ĠÑĢазвиÑĤ":127416,"ĠполÑĮз":127417,"ĠСеÑĢ":127418,"×ķ×ij×Ļ":127419,"róż":127420,"ìĭŃ":127421,"ãĤ¯ãĥĪ":127422,"ãģĹãĤĪãģĨ":127423,"à¸ģรม":127424,"ØŃÙĥÙĪÙħ":127425,"à¹Ĥà¸ļ":127426,"à¸Ĺà¹īาย":127427,"ĠMá":127428,"ĠÑĤÑĭ":127429,"à¸Ħรัว":127430,"ÑĢÑĥб":127431,"ạp":127432,"ĠmÅĤ":127433,"ĠmÅĤod":127434,"ĠgörÃ¼ÅŁ":127435,"ĠgeliÅŁ":127436,"Æ°Æ¡i":127437,"×ŀשק":127438,"ÙĢÙĢÙĢÙĢ":127439,"ราว":127440,"ãģĹãģ£":127441,"ãģĹãģ£ãģĭãĤĬ":127442,"ĠÐļон":127443,"Ġkê":127444,"à¹Ĥà¸Ĺร":127445,"èIJ½ãģ¡":127446,"åĩºãģ¦":127447,"ลัà¸ģษ":127448,"Ġ×Ĵ×ij×ķ×Ķ":127449,"ãĥĻãĥ«":127450,"ê±°ëĤĺ":127451,"ë§IJ":127452,"×Ļ׾×ĵ×Ļ×Ŀ":127453,"ĠëĦĪ":127454,"×ŀר×Ļ":127455,"รส":127456,"ãĥŃãĥ³":127457,"ило":127458,"ноÑģÑĤÑĮÑİ":127459,"×ĸר×Ĺ":127460,"пон":127461,"Ġ×Ķש׾":127462,"ê²łìĬµëĭĪëĭ¤":127463,"ĠkiÅŁ":127464,"ĠÐļи":127465,"วร":127466,"داع":127467,"ÅŁim":127468,"ÙĨÙij":127469,"ваÑĤ":127470,"راÙĥ":127471,"باÙĦ":127472,"иде":127473,"Ġ×Ķ×ŀ×Ĺ":127474,"ìĸµ":127475,"تÙģاع":127476,"أت":127477,"ëĬĺ":127478,"ש×Ļת":127479,"ستÙħر":127480,"ĠÑĦак":127481,"ĠاÙĦØ£ÙħرÙĬ":127482,"ëŀ¨":127483,"اسÙħ":127484,"ĠaÄŁ":127485,"Ġçev":127486,"ÙĥÙĪر":127487,"ãģķãģ¾":127488,"Ġçöz":127489,"Ġرس":127490,"Äħda":127491,"สà¸Ļุ":127492,"ãģĹãģ¦ãģıãĤĮ":127493,"нÑİ":127494,"leÅŁme":127495,"ãĤªãĥ³":127496,"ãģ¨ãģªãĤĬ":127497,"avaÅŁ":127498,"×ĺ×Ļ×ij":127499,"ØŃض":127500,"×ķצ×IJ×ķת":127501,"ÙĨÙħÙĪ":127502,"ıt":127503,"ĠÑħа":127504,"ĠÑħаÑĢак":127505,"ĠÑħаÑĢакÑĤеÑĢ":127506,"ĠdÅĤ":127507,"ãĥĹãĥ©":127508,"à¸Ĭุม":127509,"à¹Īà¸Ńà¸Ļ":127510,"×ķ×ij׾":127511,"Ñģол":127512,"×ĵ×Ĵ":127513,"аÑĢаÑĤ":127514,"nivers":127515,"ĠgerçekleÅŁtir":127516,"ĠاÙĦÙĦÙĬ":127517,"ระยะ":127518,"ĠÙħختÙĦÙģ":127519,"Ġgönder":127520,"Ùģار":127521,"doÄŁ":127522,"doÄŁan":127523,"صÙĦاØŃ":127524,"Ġyayın":127525,"ãĥĨãĥ³":127526,"รวà¸Ī":127527,"×Ļ×Ĺ×Ļ×ĵ":127528,"ünkü":127529,"ÑĨиалÑĮн":127530,"à¸ļู":127531,"มุ":127532,"hä":127533,"Ø®Ùģ":127534,"å¢Ĺ":127535,"å¢ĹãģĪ":127536,"еÑĩно":127537,"ĠاÙĦسÙĨ":127538,"à¸Ĥาว":127539,"imdi":127540,"Ы":127541,"à¸Ļà¸Ńà¸ģà¸Īาà¸ģ":127542,"à¸ļาล":127543,"תש":127544,"Ġdüzenle":127545,"мÑĭÑģл":127546,"ãģıãģª":127547,"żu":127548,"ĠwspóÅĤ":127549,"Ġназ":127550,"ındaki":127551,"ترة":127552,"ÅŁek":127553,"Ġöd":127554,"ĠÙĪÙĥ":127555,"ĠпозволÑı":127556,"Ġת×ķ׼":127557,"ÙħÙĨتج":127558,"ë§ī":127559,"ĠاÙĦØ«ÙĦاث":127560,"аÑĨиÑİ":127561,"ÙĪرÙĪ":127562,"ÑĭваеÑĤ":127563,"خصص":127564,"ĠاÙĦÙģÙĦ":127565,"ĠاÙĦÙģÙĦسطÙĬÙĨ":127566,"إجر":127567,"إجراء":127568,"اÙĨتخ":127569,"اÙĨتخاب":127570,"ارÙĬØ©":127571,"×ķÖ":127572,"Ø¢ÙĨ":127573,"×ŀ×¢×ķת":127574,"Ġмал":127575,"Ġ×IJ×Ĺ":127576,"à¸Ĺà¹īà¸Ńà¸ĩ":127577,"zeÅĽ":127578,"Ġë§Įëĵ¤":127579,"رÙĬع":127580,"äºĭãĤĴ":127581,"à¸ļริหาร":127582,"׾×ŀ×Ļ×ĵ":127583,"ĠмÑĥж":127584,"ترÙĪ":127585,"ĠباÙĦØ¥":127586,"פ×Ļק":127587,"زÙħØ©":127588,"ĠÃ¶ÄŁrenc":127589,"ãĥ¶":127590,"اÙħعة":127591,"ק×ij×ķצ":127592,"×ŀ׳×ķת":127593,"رÙĬÙħ":127594,"Ġоказ":127595,"ãģłãģijãģ©":127596,"Ġhız":127597,"Ġש×IJת":127598,"ãĤ¢ãĥ¼":127599,"Ġmożliwo":127600,"ìĦ¼":127601,"ÙĪاب":127602,"огÑĢаÑĦ":127603,"ĠعبداÙĦ":127604,"ãĤĴè¡Į":127605,"بÙĬÙĦ":127606,"Ġİç":127607,"ยาย":127608,"ĠÑĥÑĩаÑģÑĤ":127609,"ÑĦеÑģÑģ":127610,"ÑĦеÑģÑģиона":127611,"Ấ":127612,"ÙĨÙĬÙĨ":127613,"عدÙĦ":127614,"สรร":127615,"دÙĬÙĦ":127616,"×ij×Ļק":127617,"czyÅĤ":127618,"ÑĢоме":127619,"Ġмед":127620,"ìĻĶ":127621,"ãĥ©ãĤ¤ãĥ³":127622,"ĠÑĤеп":127623,"еÑĢÑĮ":127624,"iÄŁi":127625,"вели":127626,"ÑĢиÑģÑĤ":127627,"ס×ķפ":127628,"×ŀ׾×Ĺ":127629,"ĠاÙĦØ¥ÙĨ":127630,"Ġ׾×Ķש":127631,"è¶ĬãģĹ":127632,"ĠÑĢÑĭ":127633,"×ķ×IJר":127634,"رÙĩاب":127635,"פ×ķ×IJ×Ļ":127636,"ĠгоÑģÑĥд":127637,"ĠгоÑģÑĥдаÑĢ":127638,"ĠгоÑģÑĥдаÑĢÑģÑĤв":127639,"ĠاÙĦØ£ÙħÙĬر":127640,"Ùħج":127641,"à¹Ģหมาะ":127642,"ÑĢев":127643,"à¸Ĭีà¸ŀ":127644,"ãĥķãĥĪ":127645,"иÑĩно":127646,"ĠاÙĦÙħؤ":127647,"Ġiht":127648,"íħľ":127649,"دÙĨÙĬ":127650,"رص":127651,"лаÑģÑĤ":127652,"à¹Ģหลà¹Īา":127653,"ılır":127654,"รà¸ĵà¹Į":127655,"×ŀש×Ļ×ļ":127656,"Ġdá»ĭ":127657,"Ø·ÙģاÙĦ":127658,"×ĺ×ķף":127659,"Ġ×ij×Ļ׳":127660,"ãģ¾ãģ£ãģŁ":127661,"ложениÑı":127662,"تØŃر":127663,"باØŃ":127664,"à¹Ģสืà¹īà¸Ń":127665,"ãģĻãģĶ":127666,"ltür":127667,"à¸ĩาม":127668,"Ġtü":127669,"ĠпÑĢим":127670,"ĠпÑĢимен":127671,"Ġhayat":127672,"ëĥIJ":127673,"ëĭĮ":127674,"׳×Ļ×ķ":127675,"веден":127676,"ìħ¨":127677,"à¸Īัย":127678,"à¸ģà¹Īà¸Ń":127679,"Ġвод":127680,"оÑģÑĤоÑı":127681,"наÑĤ":127682,"à¹ģหล":127683,"سÙħÙĬ":127684,"à¸Ķำà¹Ģà¸Ļ":127685,"à¸Ķำà¹Ģà¸Ļิà¸Ļ":127686,"wód":127687,"öyle":127688,"ãĥĢãĤ¤":127689,"ÑĪий":127690,"меÑīен":127691,"ãģĹãģ¾ãģĨ":127692,"ãĥīãĥ©":127693,"ÙĪضØŃ":127694,"à¸Ńà¸Ļุ":127695,"ĠاÙĦاجتÙħاع":127696,"laÅŁma":127697,"à¸Ħà¸Ńà¸Ļ":127698,"×ŀר×Ļ×Ŀ":127699,"ÙĨاÙħج":127700,"שר×ķת":127701,"اÙĦØ£":127702,"ĠksiÄħż":127703,"Ġан":127704,"ÑĢай":127705,"اÙĩرة":127706,"×ŀ×ĵ×Ķ":127707,"ä¸Ģç·":127708,"ä¸Ģç·Ĵ":127709,"ä¸Ģç·Ĵãģ«":127710,"ÑĢиÑĤоÑĢ":127711,"dıkl":127712,"à¹ģà¸ĸ":127713,"à¹ģà¸Ĥà¹Īà¸ĩ":127714,"екÑĤоÑĢ":127715,"×ŀסע":127716,"ÑĢакÑĤи":127717,"uÄŁu":127718,"×ķ×ijת":127719,"สูà¸ķร":127720,"ĠçalÄ±ÅŁm":127721,"ĠçalÄ±ÅŁmalar":127722,"Ġана":127723,"ãĥĽãĥ¼ãĥł":127724,"Ġbölüm":127725,"Ġبص":127726,"олоÑģ":127727,"ĠìķĬëĬĶ":127728,"à¹Īะ":127729,"ÙĪتر":127730,"ä¹Ĺ":127731,"ستخداÙħ":127732,"פ×Ļ×Ļס":127733,"פ×Ļ×Ļס×ij":127734,"פ×Ļ×Ļס×ij×ķק":127735,"ĠкÑĢаÑģ":127736,"лик":127737,"رÙĬØŃ":127738,"×ŀש׾×Ķ":127739,"à¹Ģยีà¹Īย":127740,"à¹Ģยีà¹Īยม":127741,"виÑģ":127742,"омн":127743,"ÄŁun":127744,"ãĥŃãĥ¼ãĥ³":127745,"أتÙĬ":127746,"à¸ķรี":127747,"çĶ³ãģĹ":127748,"تÙħر":127749,"ìĹĪìĬµëĭĪëĭ¤":127750,"ĠÙĪغÙĬر":127751,"redni":127752,"ĠاÙĦصÙģ":127753,"ĠнаÑģÑĤоÑı":127754,"ĠнаÑģÑĤоÑıÑī":127755,"à¸ķรา":127756,"ĠÑĥÑģлов":127757,"ĠÑĥÑģловиÑı":127758,"ÑĨеп":127759,"×Ķ×Ĺ׾×ĺ":127760,"Ø·ÙĬع":127761,"ĠBakan":127762,"ĠاÙĦرÙĪ":127763,"илÑĮно":127764,"ĠмеÑĤ":127765,"à¸Ķà¸Ńà¸ģ":127766,"ãģĭãĤīãģªãģĦ":127767,"ĠпоÑģÑĤоÑı":127768,"ĠпоÑģÑĤоÑıн":127769,"ĠÑĩаÑģ":127770,"üc":127771,"wró":127772,"бÑĥÑĢ":127773,"ãĥIJãĥĥãĤ¯":127774,"ãĥ©ãĥ³ãĥī":127775,"ĠогÑĢ":127776,"สัà¸į":127777,"สัà¸įà¸įา":127778,"มัà¹Īà¸Ļ":127779,"à¸Ħà¸Ńม":127780,"alık":127781,"Ġнед":127782,"ümüz":127783,"ĠÅĽwie":127784,"ério":127785,"×Ļ×IJ×Ķ":127786,"دÙħات":127787,"ırl":127788,"ĠоÑĤз":127789,"ĠоÑĤзÑĭв":127790,"ä»ĺãģį":127791,"Ġkażde":127792,"миниÑģÑĤ":127793,"ãĤ°ãĥ«":127794,"ë°ĸ":127795,"езн":127796,"اÙĦÙģ":127797,"Ġשק׾":127798,"Ùħض":127799,"ãĥĿãĥ¼ãĥĪ":127800,"ÙħÙĨت":127801,"ÙĤÙĬاÙħ":127802,"Ø´ÙĨ":127803,"×Ļר×ķ×¢":127804,"ãĤŃãĥ£ãĥ³":127805,"доÑĢов":127806,"×ŀ×Ļת×Ļ":127807,"ÙĪÙĦÙĪج":127808,"ÙĥاÙģ":127809,"ĠÑĢазлиÑĩ":127810,"иÑĤеÑĤ":127811,"нолог":127812,"ลà¸ĩà¸Ĺุà¸Ļ":127813,"ĠyaklaÅŁ":127814,"ãĥ¬ãĤ¤":127815,"ê²łëĭ¤":127816,"æ±ĤãĤģ":127817,"رÙĪÙģ":127818,"ĠíĬ":127819,"ĠíĬ¹":127820,"ãģ£ãģıãĤĬ":127821,"à¸Ħวามà¸Ħิà¸Ķ":127822,"×Ķ×Ļס×ĺ":127823,"Ø¥ÙĤ":127824,"ãģ¦ãģĦ":127825,"à¹Ĥà¸Ĭ":127826,"ĠBüyük":127827,"ĠФедеÑĢ":127828,"ÑĨин":127829,"ÑĢова":127830,"ĠاÙĦاÙĤتصاد":127831,"Ġchá":127832,"à¸ĺาà¸Ļ":127833,"ë¥ł":127834,"à¹Ħà¸ķ":127835,"ÃŃpio":127836,"Ùĭا":127837,"ĠобÑıз":127838,"Ùĩج":127839,"Ġì¤ijìļĶ":127840,"ãģ®ãģ§ãģ¯ãģªãģĦ":127841,"باراة":127842,"ãĤ¤ãĥ«":127843,"ĠноÑĢм":127844,"á»īnh":127845,"mö":127846,"möglich":127847,"ÑĨип":127848,"ãĤ¢ãĤ¯":127849,"×Ķ×Ļ":127850,"ÑĨиалÑĮно":127851,"ĠÅĽwi":127852,"تÙĤ":127853,"ĠÑģÑĤоим":127854,"بÙĬعÙĬ":127855,"Ġ׾ש×ŀ":127856,"глÑı":127857,"глÑıд":127858,"ãģ¦ãģıãĤĮ":127859,"ÄĻdzi":127860,"à¸Ĥั":127861,"à¸Ĥัà¹īà¸Ļ":127862,"Ø·ÙĤ":127863,"ĠìĹŃ":127864,"ãģ£ãģ¦ãģĹãģ¾ãģĨ":127865,"ĠdeÄŁerl":127866,"ĠdeÄŁerlendir":127867,"Ġülk":127868,"Ġмног":127869,"à¹ĭ":127870,"ë¿IJ":127871,"ĠУкÑĢа":127872,"ÄŁini":127873,"Ġбезоп":127874,"ĠбезопаÑģ":127875,"à¸Ńà¸Ńà¸ģà¹ģà¸ļà¸ļ":127876,"اظ":127877,"ØŃداث":127878,"леÑĢ":127879,"×Ļ×¥":127880,"×Ļ׳×ĺר׳×ĺ":127881,"larınız":127882,"ØŃÙĬØŃ":127883,"żeli":127884,"à¸Ńัà¸ĩ":127885,"à¸Ńัà¸ĩà¸ģ":127886,"à¸Ńัà¸ĩà¸ģฤษ":127887,"ĠоÑĤлиÑĩ":127888,"ัส":127889,"ëŀį":127890,"ожно":127891,"ãĤ¹ãĥĿ":127892,"ĠÑħоÑĩ":127893,"Ġкап":127894,"еÑĩен":127895,"ØŃÙĦØ©":127896,"ÙĬاÙĩ":127897,"нал":127898,"×ķצר×Ļ×Ŀ":127899,"Ġkald":127900,"åĥį":127901,"ĠاÙĦشخص":127902,"Ġзна":127903,"Ġwzgl":127904,"życz":127905,"ê°Ŀ":127906,"à¸ŀลัà¸ĩ":127907,"íģ¼":127908,"Ġöl":127909,"Ġbụ":127910,"Ø´Ùĩر":127911,"Ġзам":127912,"Ġдев":127913,"×Ļ×ĺת":127914,"تعÙĦÙĤ":127915,"ÙĪÙħØ©":127916,"ãĤĴä½ľ":127917,"ãģįãģ¦":127918,"íĥĿ":127919,"rasında":127920,"ãĤĴæİ¢":127921,"ĠÙħباشر":127922,"راجع":127923,"Ġвозд":127924,"ÙħØŃا":127925,"×ķשר":127926,"ĠиÑģÑĤоÑĢ":127927,"มัà¸ģ":127928,"tıģ":127929,"ثار":127930,"ترÙĨت":127931,"à¹ģà¸Ĥà¹ĩ":127932,"à¹ģà¸Ĥà¹ĩà¸ĩ":127933,"поÑĩ":127934,"Ġ×ij×IJ×ķת":127935,"ë¯Ģ":127936,"ëĿ¼ëıĦ":127937,"à¸Ĭัà¸Ķ":127938,"สà¸ķà¹Į":127939,"ãĥĭãĥĥãĤ¯":127940,"иденÑĤ":127941,"ĠгÑĢÑĥпп":127942,"تخ":127943,"áºł":127944,"ยืà¸Ļ":127945,"ยัà¸Ļ":127946,"óry":127947,"TÃľ":127948,"ãģĹãĤĥ":127949,"ĠпÑĢовед":127950,"лÑıеÑĤ":127951,"ÙħØ®":127952,"ยà¸Ńม":127953,"×Ľ×ł×¡×ª":127954,"ĠاÙĦÙħÙĨت":127955,"Ġolmad":127956,"ר׼×ĸ×Ļ":127957,"ĠвÑģÑĤÑĢ":127958,"ĠиÑģÑģлед":127959,"ÑĤвеÑĢж":127960,"بدÙĪ":127961,"еÑĢÑĤ":127962,"ï»·":127963,"±ħ":127964,"สัมà¸ŀัà¸Ļà¸ĺà¹Į":127965,"ิà¹Īà¸Ļ":127966,"צ×Ļ×ij":127967,"wiÄĻt":127968,"Ġì°¸":127969,"ĠzwiÄħz":127970,"سبÙĪع":127971,"ãĥĥãĤ°":127972,"à¸Ľà¸¥à¸Ńà¸Ķ":127973,"à¸Ľà¸¥à¸Ńà¸Ķà¸łà¸±à¸¢":127974,"ãĤĤãĤĬ":127975,"ÙĤدس":127976,"Ġsprz":127977,"Ġsprzeda":127978,"Ġistedi":127979,"Ġkhu":127980,"Ġден":127981,"ĠkoÅĦ":127982,"Ġ×ij×Ĺ×Ļ":127983,"à¹Ģà¸Ĺà¹īา":127984,"×ķס×Ļ×£":127985,"ãĥĭãĥ¥ãĥ¼":127986,"ĠпÑĢедоÑģÑĤ":127987,"ĠпÑĢедоÑģÑĤав":127988,"à¹Ĥà¸Ł":127989,"év":127990,"ĠاÙĦصØŃ":127991,"صØŃاب":127992,"à¹Ģà¸Īà¹ĩà¸ļ":127993,"влек":127994,"วัà¸ķ":127995,"à¸ĸุ":127996,"ãģĵãģ¨ãģĮãģ§ãģįãģ¾ãģĻ":127997,"ÙĤÙĬÙĤÙĬ":127998,"×ķ×Ĺר":127999,"ÑĭÑĪ":128000,"ĠоÑĤно":128001,"ĠоÑĤноÑĪ":128002,"обилÑĮ":128003,"ÙģØŃ":128004,"ınt":128005,"ıntı":128006,"Ġ׾×ij×ĵ":128007,"íİĺìĿ´ì§Ģ":128008,"ãĥĬãĥ«":128009,"ĠÙħساء":128010,"×Ļ×ĺ×ij":128011,"ÑĮеÑĢ":128012,"ëĦ·":128013,"ÑĭÑĤа":128014,"ĠоÑĩеÑĢ":128015,"à¸Ķืà¹Ī":128016,"à¸Ķืà¹Īม":128017,"ĠNgh":128018,"تعب":128019,"ÙĦاÙĤات":128020,"×ķ׾×ķ×Ĵ×Ļ×Ķ":128021,"ĠìĿ´ê²ĥ":128022,"Ġ×Ķ×ijר":128023,"ìľµ":128024,"à¹Ģà¸Ħลืà¹Īà¸Ńà¸Ļ":128025,"ÙĩØ©":128026,"à¸Īำà¹Ģà¸Ľà¹ĩà¸Ļ":128027,"å¤īãģĪ":128028,"wiÅĽcie":128029,"chod":128030,"chodzÄħ":128031,"вÑĢо":128032,"×ŀ×Ĺ×Ļר":128033,"Ġyı":128034,"Ġyıll":128035,"ì¡Į":128036,"à¹Ħหว":128037,"ãģªãģıãģª":128038,"ĠзавиÑģ":128039,"ĠìĺĪìĪĺ":128040,"ÙģØ°":128041,"ủng":128042,"à¸ŀุà¸Ĺà¸ĺ":128043,"зн":128044,"layan":128045,"ãĤ¡":128046,"à¸ģà¹ĩà¸ķาม":128047,"ĠsaÄŁlam":128048,"รà¸ĵ":128049,"ĠÑģиÑĤ":128050,"ĠÑģиÑĤÑĥ":128051,"ĠاÙĦتÙĨ":128052,"×Ķ×ĸ":128053,"ĠØ·ÙĪÙĬÙĦ":128054,"taÅĤ":128055,"Ġgörd":128056,"å¤īãĤı":128057,"ëĥ¥":128058,"à¸Ħà¹Īà¸Ńย":128059,"×IJ×ķ×ĺ":128060,"ëħIJ":128061,"ãĥ©ãĥ³ãĤ¹":128062,"วัà¸Ĵ":128063,"วัà¸Ĵà¸Ļ":128064,"ĠoluÅŁ":128065,"פע×ķ׾":128066,"ĠszczegóÅĤ":128067,"à¸Ħาสิ":128068,"à¸Ħาสิà¹Ĥà¸Ļ":128069,"powied":128070,"ĠÑĤеб":128071,"หà¸Ļà¹Īวย":128072,"Ġмил":128073,"ØŃÙĥ":128074,"à¸Ĺà¸Ķ":128075,"ĠмаÑĤеÑĢиал":128076,"ÅĤow":128077,"à¹Ģà¸ģีย":128078,"ĠÑģовеÑĢ":128079,"ãĤ©":128080,"à¸Ľà¸£à¸´":128081,"ĠиÑİ":128082,"наÑĩен":128083,"ÑĢенд":128084,"muÅŁtur":128085,"ĠпÑĢодÑĥк":128086,"зд":128087,"ÑıÑĤи":128088,"ÑıÑĤиÑı":128089,"à¹Ģมีย":128090,"راتÙĬج":128091,"Ġamacı":128092,"ש×ķ׾":128093,"ש×ķ׾×Ĺ":128094,"สะà¸Ńา":128095,"สะà¸Ńาà¸Ķ":128096,"פ×Ĵ×¢":128097,"عبة":128098,"dın":128099,"íħĶ":128100,"Ġ×ŀש×Ĺק":128101,"Ġfiyat":128102,"ĠзаÑı":128103,"ĠзаÑıв":128104,"à¹Ĥหล":128105,"à¹Ĥหลà¸Ķ":128106,"à¸ģรุà¸ĩà¹Ģà¸Ĺà¸ŀ":128107,"צ×Ļ×Ļף":128108,"ìļ±":128109,"Ùħب":128110,"Ùħباد":128111,"landır":128112,"ĠвеÑģÑĮ":128113,"Ġhük":128114,"ĠÐĴоз":128115,"ÑĩиÑĤÑĭва":128116,"วล":128117,"×ķצע":128118,"à¸Ĥà¸ĵะà¸Ĺีà¹Ī":128119,"ĠaÅŁaģı":128120,"׾×IJ×ķ×ŀ×Ļ":128121,"trzym":128122,"Ã¤ÃŁig":128123,"owoÅĽci":128124,"ãģĿãĤĤ":128125,"ĠrozwiÄħz":128126,"ĠgÅĤówn":128127,"монÑĤ":128128,"×ŀ×ķ×ŀ":128129,"ĠÑģÑĤан":128130,"ÙĦاÙĤØ©":128131,"prowad":128132,"prowadzi":128133,"ĠÑģоÑģÑĤоÑı":128134,"×Ļ×IJ×ķת":128135,"rı":128136,"gı":128137,"ãĥijãĥij":128138,"ĠналиÑĩ":128139,"×Ķצע":128140,"Ġ׳×Ķ":128141,"à¸Ħัà¸ļ":128142,"عراض":128143,"иж":128144,"ÙĩائÙĬ":128145,"ãĤīãģı":128146,"ожеÑĤ":128147,"ĠобоÑĢ":128148,"ĠобоÑĢÑĥд":128149,"أسÙĦ":128150,"à¹ĩà¸Ķ":128151,"ÑĢÑĥÑĤ":128152,"دÙĬÙħÙĤ":128153,"دÙĬÙħÙĤرا":128154,"Ġjeste":128155,"×ķ×ķ×Ļר":128156,"×ij×ĵ×Ļק":128157,"деÑĢжива":128158,"ãģĬãģı":128159,"ewnÄĻtr":128160,"ewnÄĻtrzn":128161,"à¸ŀฤ":128162,"Ġ×IJ×ķ×Ķ":128163,"ת×Ĺ×ķש":128164,"Ġzob":128165,"дÑĥм":128166,"ĠÑģÑĭ":128167,"ÙĬرا":128168,"ĠwiÄĻks":128169,"à¹ģà¸ķà¸ģà¸ķà¹Īาà¸ĩ":128170,"lararas":128171,"lararası":128172,"íĺĢ":128173,"ëī´":128174,"×ķ×Ĵ׾":128175,"ĠоÑĤмеÑĤ":128176,"ĠÑĢан":128177,"تÙĥÙĦ":128178,"иÑĤелÑĮн":128179,"à¸Ľà¸£à¸°à¸§à¸±":128180,"à¸Ľà¸£à¸°à¸§à¸±à¸ķิ":128181,"ìŀĸ":128182,"можно":128183,"pieczeÅĦ":128184,"pieczeÅĦst":128185,"못":128186,"ìĬ¨":128187,"×ŀס×ŀ":128188,"Ủ":128189,"ศิ":128190,"ศิล":128191,"à¸¨à¸´à¸¥à¸Ľ":128192,"ĠÅļw":128193,"ãĥĥãĤ·ãĥ§ãĥ³":128194,"unitÃł":128195,"Ġmieszka":128196,"ĠmieszkaÅĦ":128197,"przed":128198,"przedsi":128199,"przedsiÄĻb":128200,"przedsiÄĻbior":128201,"à¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺิ":128202,"à¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺà¸´à¸łà¸²à¸ŀ":128203,"ยà¹Ī":128204,"ìķĻ":128205,"รวà¸Ķ":128206,"รวà¸Ķà¹Ģรà¹ĩว":128207,"å½ĵãģŁãĤĬ":128208,"älle":128209,"ÑĥеÑĤÑģÑı":128210,"ãn":128211,"ëłµ":128212,"thè":128213,"ãĤĴåĪ©çĶ¨":128214,"ìµľ":128215,"íĵ¨":128216,"à¸Ĺัà¸ļ":128217,"าà¸Ħม":128218,"ãģĩ":128219,"ëĤĮ":128220,"à¹Ģà¸Ľà¸¥à¹Īา":128221,"â¦":128222,"ë¾":128223,"êĢ":128224,"êĩ":128225,"â¡":128226,"ðŁŁ":128227,"ãIJ":128228,"âº":128229,"áŃ":128230,"áĻ":128231,"áĵ":128232,"á²":128233,"ðĵı":128234,"á¬":128235,"â¯":128236,"ä¨":128237,"êĿ":128238,"ê«":128239,"ðij":128240,"ðĵĥ":128241,"ðĿħ":128242,"":128244,"":128245,"":128247,"ĠعÙĦÙī":128248,"Ġmá»Ļt":128249,"ĠvỼi":128250,"ĠngÆ°á»Ŀi":128251,"ĠØ¥ÙĦÙī":128252,"Ġnhững":128253,"Ġthá»ĥ":128254,"Ġ×IJ×ķ":128255,"Ġ×¢×Ŀ":128256,"اÙĭ":128257,"Ġà¹ģละ":128258,"ĠÙĦا":128259,"ĠnhÆ°":128260,"ĠاÙĦتÙĬ":128261,"Ġ×Ķ×ķ×IJ":128262,"ĠÄijến":128263,"ĠØ£ÙĪ":128264,"Ġvá»ģ":128265,"ĠlÃłm":128266,"Ġsẽ":128267,"ĠcÅ©ng":128268,"Ġợ":128269,"ĠÄijó":128270,"Ġnhiá»ģu":128271,"Ġtại":128272,"Ġtrên":128273,"Ġ×Ĵ×Ŀ":128274,"ĠnhÃł":128275,"Ġ׼×Ļ":128276,"Ġsá»±":128277,"ĠÄijầu":128278,"Ġbá»ĭ":128279,"ĠÙĩذا":128280,"Ġnhất":128281,"Ġphải":128282,"Ġhiá»ĩn":128283,"Ġdụng":128284,"ĠÄijá»Ļng":128285,"ĠاÙĦÙĦÙĩ":128286,"ĠØĮ":128287,"ĠÙĥÙĦ":128288,"Ġviá»ĩc":128289,"ĠnÄĥm":128290,"Ġthì":128291,"Ġhá»įc":128292,"ĠÙĪت":128293,"té":128294,"ĠاÙĨ":128295,"Ġtôi":128296,"Ġ×IJ׳×Ļ":128297,"Ġ׾×Ļ":128298,"Ġ×ŀ×ķ":128299,"ĠngÃły":128300,"ĠnÆ°á»Ľc":128301,"Ġ×Ķ×Ļ×IJ":128302,"Ġ×IJ×Ļ":128303,"ĠhÆ¡n":128304,"ĠÙĩØ°Ùĩ":128305,"ĠÙĪÙĬ":128306,"ĠاÙĦØ°ÙĬ":128307,"Ġ×ķ×ŀ":128308,"Ġgiá":128309,"Ġnhân":128310,"ĠchÃŃnh":128311,"Ġmình":128312,"ĠÐĿа":128313,"Ġthế":128314,"Ġ×Ļ×ķתר":128315,"Ġ×IJ×Ŀ":128316,"Ġnên":128317,"Ġhợ":128318,"Ġhợp":128319,"Ġcòn":128320,"ĠÙĩÙĪ":128321,"ĠcÆ¡":128322,"Ġrất":128323,"ĠViá»ĩt":128324,"Ġبعد":128325,"Ġש×Ļ":128326,"Ġthá»Ŀi":128327,"Ġcách":128328,"ĠÄijá»ĵng":128329,"Ġно":128330,"ĠtrÆ°á»Ŀng":128331,"ØŁ":128332,"ĠÄijá»ĭnh":128333,"ĠÄijiá»ģu":128334,"×Ļ×Ļ×Ŀ":128335,"Ġthá»±c":128336,"nın":128337,"Ġhình":128338,"Ġnói":128339,"Ġcùng":128340,"Ġ×Ķ×Ķ":128341,"ĠØ¥ÙĨ":128342,"Ġ×IJ×ij׾":128343,"ĠnhÆ°ng":128344,"Ġbiết":128345,"Ġже":128346,"Ġchúng":128347,"ĠÄijang":128348,"ĠØ°ÙĦÙĥ":128349,"Ġlên":128350,"Ġkhách":128351,"ĠnÃło":128352,"Ġsá»Ń":128353,"Ġkhác":128354,"Ġë°ı":128355,"Ġlý":128356,"×Ļ×Ļ":128357,"ĠÄijây":128358,"Ġ׾×ŀ":128359,"Ġcần":128360,"Ġtrình":128361,"Ġphát":128362,"ãģ«ãĤĤ":128363,"по":128364,"ĠnÄĥng":128365,"Ġbá»Ļ":128366,"Ġvụ":128367,"ĠÄijá»Ļ":128368,"Ñĩе":128369,"ĠnháºŃn":128370,"ĠtrÆ°á»Ľc":128371,"Ġ×¢×ĵ":128372,"ĠhÃłnh":128373,"ĠØ®ÙĦاÙĦ":128374,"Ġlượng":128375,"Ġcấp":128376,"Ġtá»±":128377,"Ġvì":128378,"ĠtÆ°":128379,"Ġchất":128380,"Ġ׼×ŀ×ķ":128381,"Ġgì":128382,"Ġש׳":128383,"Ġtế":128384,"ת×ķ":128385,"Ġnghiá»ĩp":128386,"Ġmặt":128387,"ĠÙĥÙħا":128388,"Ġ×ij×Ļף":128389,"Ġרק":128390,"Ġthấy":128391,"Ġmáy":128392,"ĠÙģÙī":128393,"Ġdân":128394,"Ġ×IJ×Ĺ×ĵ":128395,"Ġtâm":128396,"Ġ׼×ļ":128397,"Ġ׾×ķ":128398,"во":128399,"Ġtác":128400,"ĠtoÃłn":128401,"ĠÙĪÙħ":128402,"Ġkết":128403,"Ġหรืà¸Ń":128404,"ĠÙĪاÙĦÙħ":128405,"ĠÄijiá»ĥm":128406,"Ġ×ĸ×ķ":128407,"Ġ×ij×ķ":128408,"׼×ķת":128409,"Ġhá»Ļi":128410,"Ġbằng":128411,"تÙĩا":128412,"Ġ׼×ĵ×Ļ":128413,"Ġ×Ķ×Ŀ":128414,"Ġxuất":128415,"ĠÙĤد":128416,"Ġbảo":128417,"Ġtá»ijt":128418,"Ġtình":128419,"ĠÙĩÙĬ":128420,"ĠÄijá»iji":128421,"Ġthiết":128422,"Ġhiá»ĩu":128423,"Ġtiếp":128424,"Ġtạo":128425,"ת×Ķ":128426,"Ġchủ":128427,"oÅĽÄĩ":128428,"Ġgiú":128429,"Ġgiúp":128430,"Ġý":128431,"Ġquả":128432,"Ġloại":128433,"Ġcô":128434,"Ġô":128435,"Ġông":128436,"Ġ×Ķ×ķ":128437,"ĠاÙĦÙĬÙĪÙħ":128438,"ĠtÃŃnh":128439,"га":128440,"Ġphòng":128441,"ĠÄĥn":128442,"ĠعاÙħ":128443,"Ġvá»ĭ":128444,"larını":128445,"rÃŃa":128446,"ĠtỼi":128447,"ĠÄijÆ°á»Ŀng":128448,"ĠgiỼi":128449,"Ġbản":128450,"Ġcầu":128451,"Ġnhiên":128452,"Ġbá»ĩnh":128453,"ĠthÆ°á»Ŀng":128454,"Ġ×IJ×Ļף":128455,"ĠÄijá»ģ":128456,"Ġhá»ĩ":128457,"Ġ×Ļשר×IJ׾":128458,"Ġquá":128459,"ĠÐĹа":128460,"ãģ®ãģ§ãģĻãģĮ":128461,"ĠÐŁÑĢи":128462,"Ġphần":128463,"ĠÙĪÙĦا":128464,"ĠlỼn":128465,"Ġtrá»ĭ":128466,"Ġcảm":128467,"Ġмо":128468,"Ġdùng":128469,"ĠاÙĦÙī":128470,"ĠعÙĦÙĬÙĩ":128471,"ĠìŀĪìĬµëĭĪëĭ¤":128472,"ÙĬÙĤ":128473,"ĠÙĤبÙĦ":128474,"Ġhoặc":128475,"ĠØŃÙĬØ«":128476,"Ġà¸Ĺีà¹Ī":128477,"ĠغÙĬر":128478,"ĠÄijại":128479,"Ġsá»ijng":128480,"нÑĭми":128481,"Ġthức":128482,"Ġפ×Ļ":128483,"ĠÄijiá»ĩn":128484,"ãģªãģĭãģ£ãģŁ":128485,"Ġgiải":128486,"Ġvẫn":128487,"ĠиÑħ":128488,"Ġönce":128489,"ĠváºŃy":128490,"Ġmuá»ijn":128491,"Ġảnh":128492,"à¹ĥà¸Ļà¸ģาร":128493,"ĠQuá»ijc":128494,"Ġkế":128495,"׳×IJ":128496,"Ġס×Ļ":128497,"Ġyêu":128498,"ãģ®ãģĭ":128499,"ĠÄijẹ":128500,"ĠÄijẹp":128501,"Ġchức":128502,"Ġyıl":128503,"ĠTürkiye":128504,"dé":128505,"ĠÙĤاÙĦ":128506,"Ġdá»ĭch":128507,"ĠolduÄŁu":128508,"Ġchá»įn":128509,"ĠتÙħ":128510,"หà¸Ļึà¹Īà¸ĩ":128511,"ãģķãĤĮãģŁ":128512,"Ġpháp":128513,"ìĽĶ":128514,"Ġtiá»ģn":128515,"ãģĹãģ¾ãģĹãģŁ":128516,"Ġש׾×IJ":128517,"ÙĦØ©":128518,"Ġ׾פ׳×Ļ":128519,"Ġ×ij×Ļת":128520,"ĠHÃł":128521,"ĠØŃت":128522,"ĠØŃتÙī":128523,"Ġ×¢×ķ×ĵ":128524,"Ġnó":128525,"Ġtháng":128526,"à¹Ģลืà¸Ńà¸ģ":128527,"ר×Ķ":128528,"ĠtÄĥng":128529,"Ġcái":128530,"Ġtriá»ĥn":128531,"Ġ×IJ×ķת×ķ":128532,"ìłģìĿ¸":128533,"ĠCông":128534,"Ġ׾×Ķ×Ļ×ķת":128535,"Ġгода":128536,"иÑİ":128537,"Ġبعض":128538,"Ġà¸ģาร":128539,"èī¯ãģĦ":128540,"ÙĪت":128541,"Ġliên":128542,"ĠÐĿо":128543,"ĠÐĿе":128544,"çļĦãģª":128545,"ĠÙħت":128546,"ĠÑĤакже":128547,"ĠкоÑĤоÑĢÑĭе":128548,"Ġ×Ļ×ĵ×Ļ":128549,"Ġtrá»įng":128550,"ãĤµãĤ¤ãĥĪ":128551,"ìłģìľ¼ë¡ľ":128552,"ĠtáºŃp":128553,"Ġש׾×Ļ":128554,"íķĺê²Į":128555,"ĠtÃłi":128556,"ĠЯ":128557,"Ġrá»ĵi":128558,"اÙĥ":128559,"ĠthÆ°Æ¡ng":128560,"Ġ×Ķ×ĸ×Ķ":128561,"ĠÙĪÙħÙĨ":128562,"à¸Ĺีà¹Īมี":128563,"Ġcuá»Ļc":128564,"Ġbüyük":128565,"ãģ¨ãģĭ":128566,"Ġ×ij×Ļ×ķתר":128567,"Ġlần":128568,"Ġgöre":128569,"Ġtrợ":128570,"Ġ×ĺ×ķ×ij":128571,"ÑĤÑĮÑģÑı":128572,"Ġthá»ijng":128573,"Ġ׼ש":128574,"Ġtiêu":128575,"Ġ×ŀ×IJ×ķ×ĵ":128576,"ØĽ":128577,"kÄħ":128578,"Ġà¹ĥà¸Ļ":128579,"Ġvấn":128580,"Ġש׾×ķ":128581,"ĠÄijá»ģu":128582,"Ùģت":128583,"Ġê²ĥìĿ´":128584,"Ġhóa":128585,"ĠاÙĦعاÙħ":128586,"ĠÙĬÙĪÙħ":128587,"кой":128588,"Ġbiá»ĩt":128589,"ÑģÑĤо":128590,"Ġ×Ķ×Ļ×ķ":128591,"à¸Ĺีà¹Īà¸Īะ":128592,"Ġ×ĵ×Ļ":128593,"Ġ×IJ×ļ":128594,"Ġán":128595,"صÙĪر":128596,"ĠtrÃŃ":128597,"ĠÐŁÑĢо":128598,"Ġlá»±c":128599,"ãģĹãģ¦ãģĦãģ¾ãģĻ":128600,"ĠbÃłi":128601,"Ġ×ĸ×IJת":128602,"Ġbáo":128603,"à¸ļà¸Ļ":128604,"ĠëĮĢíķľ":128605,"Ġtiế":128606,"Ġtiếng":128607,"Ġbên":128608,"ãģķãĤĮãĤĭ":128609,"sión":128610,"Ġtìm":128611,"×¢×ķ":128612,"mé":128613,"ниÑı":128614,"ãģ»ãģ©":128615,"Ġà¹Ģà¸ŀราะ":128616,"بة":128617,"Ġë¶Ħ":128618,"Ġ×IJ×ĸ":128619,"à¸Ĺà¹Īาà¸Ļ":128620,"ת×Ŀ":128621,"Ġthêm":128622,"Ġhoạt":128623,"yı":128624,"×ĸ×ķ":128625,"Ġgiá»Ŀ":128626,"Ġbán":128627,"à¸Ĥาย":128628,"Ñĩа":128629,"Ġà¹Ĩ":128630,"ĠاÙĦÙħت":128631,"ĠоÑĩенÑĮ":128632,"Ġbất":128633,"Ġtrẻ":128634,"ÑĤÑĢ":128635,"ĠØ£ÙĨÙĩ":128636,"ĠØ«Ùħ":128637,"Ġ׼×ŀ×Ķ":128638,"Ġkhó":128639,"Ġrằng":128640,"ĠÙĪÙģÙĬ":128641,"ний":128642,"ĠhoÃłn":128643,"tó":128644,"Ġ×IJשר":128645,"ĠìĥĿê°ģ":128646,"Ñģа":128647,"Ġ׼×ijר":128648,"ĠÑįÑĤом":128649,"larının":128650,"ĠchÆ°a":128651,"зи":128652,"Ġdẫn":128653,"ĠÐļак":128654,"جÙĪ":128655,"ĠбÑĭло":128656,"ĠÙĬت":128657,"nı":128658,"ÅĤam":128659,"ĠÙĪÙĩÙĪ":128660,"×ij×ķ":128661,"пи":128662,"רת":128663,"Ġquá»ijc":128664,"жд":128665,"ĠÄijÆ¡n":128666,"Ùĥتب":128667,"Ġmắt":128668,"ระà¸ļ":128669,"ระà¸ļà¸ļ":128670,"ĠÙĥاÙĨت":128671,"Ġthân":128672,"สิà¸Ļà¸Ħà¹īา":128673,"×Ĵ×Ļ":128674,"ĠphÆ°Æ¡ng":128675,"à¹Ħมà¹Īà¹Ħà¸Ķà¹ī":128676,"ĠìĦ±":128677,"ĠCác":128678,"Ġ×Ķ×ŀ×ķ":128679,"ĠÑĤем":128680,"Ġ×ĵ×ķ":128681,"à¸Ńะà¹Ħร":128682,"ĠvÄĥn":128683,"ãģªãģ®ãģ§":128684,"ĠNá»Ļi":128685,"Ġ×¢×ķ":128686,"ãĤīãĤĮãĤĭ":128687,"Ġsáng":128688,"Ġgöster":128689,"ãģĵãģ¨ãĤĴ":128690,"Ġtarafından":128691,"Ġма":128692,"ĠпоÑģле":128693,"Ġ׳×Ļת":128694,"Ġ׳×Ļ×ª×Ł":128695,"ĠлеÑĤ":128696,"Ġ׾׳×ķ":128697,"ÑģÑģ":128698,"Ġ×Ļ×ķ":128699,"пе":128700,"ĠÙĪÙĦÙĥ":128701,"ĠÙĪÙĦÙĥÙĨ":128702,"ĠngoÃłi":128703,"ĠÄijá»ĭa":128704,"rzÄħd":128705,"dziaÅĤ":128706,"ĠÙħر":128707,"иÑĤÑĮÑģÑı":128708,"Ġ×IJ×Ĺר×Ļ":128709,"Ġ׾׼׾":128710,"à¸Ĥà¹īà¸Ńม":128711,"à¸Ĥà¹īà¸Ńมูล":128712,"Ġбол":128713,"Ġболее":128714,"جÙħع":128715,"леÑĤ":128716,"Ġlá»ĭch":128717,"ĠÙħØ«ÙĦ":128718,"Ġê·¸ë¦¬ê³ł":128719,"Ġthứ":128720,"ĠdeÄŁil":128721,"ÙĪØŃ":128722,"Ġש׾×ļ":128723,"ĠÙħØŃÙħد":128724,"Ġnếu":128725,"ĠÄijá»ķi":128726,"Ġvừa":128727,"Ġmá»įi":128728,"Ġони":128729,"Ġlúc":128730,"ĠÙĬÙĥÙĪÙĨ":128731,"ì§Ī":128732,"Ġש׾׳×ķ":128733,"ĠÐĶо":128734,"Ġש׳×Ļ":128735,"ลิ":128736,"×IJפשר":128737,"Ġsức":128738,"ê¶Į":128739,"Ġứng":128740,"à¹Ħมà¹Īมี":128741,"Ø·ÙĦب":128742,"ĠÑĩем":128743,"Ġchuyên":128744,"ĠthÃŃch":128745,"Ġ×ķ×Ļ":128746,"íķ©":128747,"ĠÙħصر":128748,"до":128749,"ĠÄijất":128750,"Ġchế":128751,"à¸Ĭืà¹Īà¸Ń":128752,"Ġìĭł":128753,"Ġإذا":128754,"ĠرئÙĬس":128755,"Ġש×Ļש":128756,"Ġgiảm":128757,"Ñģка":128758,"larında":128759,"Ġsợ":128760,"ĠtÃŃch":128761,"ĠÙĦÙĥÙĨ":128762,"ĠبÙħ":128763,"×¢×ķ×ij":128764,"×¢×ķ×ij×ĵ":128765,"ÅĤÄħcz":128766,"larına":128767,"Ġש×Ŀ":128768,"ĠÙĦت":128769,"Ġש×Ķ×ķ×IJ":128770,"tów":128771,"Ġëĭ¤ë¥¸":128772,"ĠØ£Ùĥثر":128773,"ãģ®ãģ§ãģĻ":128774,"׼×Ļ×Ŀ":128775,"ĠolduÄŁunu":128776,"ãģĭãģª":128777,"ãĤĤãģĨ":128778,"ÙĬØŃ":128779,"Ġnhìn":128780,"Ġnghá»ĩ":128781,"ãģ«ãģªãģ£ãģ¦":128782,"па":128783,"Ġquyết":128784,"ÙĦÙĤ":128785,"tá":128786,"Ġluôn":128787,"ĠÄijặc":128788,"Ġ×IJר":128789,"Ġtuá»ķi":128790,"são":128791,"ìĻ¸":128792,"رد":128793,"ĠبÙĩا":128794,"Ġ×Ķ×Ļ×ķ×Ŀ":128795,"×ķ×ķ×Ļ":128796,"ãģ§ãģĻãģŃ":128797,"ĠÑĤого":128798,"Ġthủ":128799,"ãģĹãģŁãģĦ":128800,"رÙĤ":128801,"Ġbắt":128802,"гÑĥ":128803,"Ġtá»Ń":128804,"ÑĪа":128805,"Ġà¸Ľà¸µ":128806,"Ġ×Ķ×IJ×Ŀ":128807,"íı¬":128808,"ża":128809,"Ġ×IJת×Ķ":128810,"Ġná»Ļi":128811,"ĠphÃŃ":128812,"ĠÅŁekilde":128813,"Ġlá»Ŀi":128814,"dıģı":128815,"Ġ׼×IJף":128816,"Ġtüm":128817,"Ġmạnh":128818,"ĠMỹ":128819,"ãģĿãĤĵãģª":128820,"Ġnhá»ı":128821,"ãģªãģĮãĤī":128822,"Ġbình":128823,"ıp":128824,"à¸ŀา":128825,"ĠÄijánh":128826,"ĠÙĪÙĦ":128827,"ר×ķת":128828,"Ġ×IJ×Ļ×ļ":128829,"Ġchuyá»ĥn":128830,"Ùĥا":128831,"ãĤĮãĤĭ":128832,"à¹ģมà¹Ī":128833,"ãĤĪãģı":128834,"ĠÙĪÙĤد":128835,"íĸĪëĭ¤":128836,"ĠnÆ¡i":128837,"ãģ«ãĤĪãģ£ãģ¦":128838,"Ġviết":128839,"Ġà¹Ģà¸ŀืà¹Īà¸Ń":128840,"ëIJĺëĬĶ":128841,"ادÙĬ":128842,"ĠÙģØ¥ÙĨ":128843,"ì¦Ŀ":128844,"ĠÄijặt":128845,"ĠhÆ°á»Ľng":128846,"Ġxã":128847,"Ġönemli":128848,"ãģłãģ¨":128849,"Ġmẹ":128850,"Ġ×ij×Ļ":128851,"Ġ×ĵ×ijר":128852,"ĠváºŃt":128853,"ĠÄijạo":128854,"Ġdá»±ng":128855,"ĠÑĤом":128856,"ĠÙģÙĬÙĩا":128857,"ĠجÙħÙĬع":128858,"ĠthuáºŃt":128859,"stÄĻp":128860,"Ġtiết":128861,"Ø´ÙĬ":128862,"ĠеÑīе":128863,"ãģĻãĤĭãģ¨":128864,"ĠmÃłu":128865,"ĠÑįÑĤого":128866,"Ġvô":128867,"ĠÐŃÑĤо":128868,"ĠtháºŃt":128869,"Ġnữa":128870,"Ġbiến":128871,"Ġnữ":128872,"Ġ׾׼×Ŀ":128873,"×Ļ×Ļף":128874,"Ġست":128875,"ĠÐŀÑĤ":128876,"Ġphụ":128877,"ê¹Įì§Ģ":128878,"Ġ׾×ļ":128879,"Ġkỳ":128880,"à¹ĥà¸Ħร":128881,"Ġgây":128882,"ĠÙĦÙĦÙħ":128883,"Ġtục":128884,"تÙĬÙĨ":128885,"Ġtrợ":128886,"Ġ׾פ×Ļ":128887,"Ġbá»ij":128888,"ĠÐļа":128889,"ĠÄijình":128890,"owÄħ":128891,"sında":128892,"Ġkhiến":128893,"sız":128894,"Ġкогда":128895,"×¡×ľ":128896,"ĠбÑĭл":128897,"à¸Ļà¹īà¸Ńย":128898,"обÑĢаз":128899,"Ġê²ĥìĿ´ëĭ¤":128900,"ëĵ¤ìĿĢ":128901,"ãģ¸ãģ®":128902,"Ġà¹Ģมืà¹Īà¸Ń":128903,"Ġphục":128904,"Ġ×Ĺ׾ק":128905,"Ġhết":128906,"ĠÄija":128907,"à¹Ģà¸Ķà¹ĩà¸ģ":128908,"íĺķ":128909,"lÃŃ":128910,"ê¸ī":128911,"Ġعدد":128912,"ĠÄijá»ĵ":128913,"Ġgần":128914,"Ġ×Ļ×ķ×Ŀ":128915,"ĠsÄ©":128916,"ÑĢÑıд":128917,"Ġquyá»ģn":128918,"Ġ×IJ׾×IJ":128919,"ÙĩÙħا":128920,"׳×Ļ×Ķ":128921,"׾×ķת":128922,"Ġ×Ķר×ij×Ķ":128923,"Ġtiên":128924,"Ġalın":128925,"Ġdá»ħ":128926,"人ãģĮ":128927,"ноÑģ":128928,"лÑģÑı":128929,"ĠÄijÆ°a":128930,"สาว":128931,"иÑĢован":128932,"Ġ×ŀספר":128933,"×Ĵף":128934,"Ġkiến":128935,"ĠШ":128936,"pé":128937,"бÑĥ":128938,"овой":128939,"ба":128940,"ĠØ¥ÙĦا":128941,"×IJ׾×Ļ":128942,"Ġxây":128943,"Ġbợi":128944,"Ġש×ķ":128945,"人ãģ®":128946,"ק×Ļ×Ŀ":128947,"à¹Ģà¸Ķืà¸Ńà¸Ļ":128948,"Ġkhá":128949,"Ġ×ķ׾×Ķ":128950,"×ĵ×ķת":128951,"Ġ×¢×ij×ķר":128952,"ĠبشÙĥÙĦ":128953,"ĠÙĩÙĨاÙĥ":128954,"ÑĤÑĢа":128955,"ĠíķĺëĬĶ":128956,"รà¸Ńà¸ļ":128957,"owaÅĤ":128958,"hé":128959,"Ġdiá»ħn":128960,"Ġ×Ķ׼׾":128961,"Ġأس":128962,"Ġchuyá»ĩn":128963,"ระà¸Ķัà¸ļ":128964,"ĠNhững":128965,"Ġ×IJ×Ĺת":128966,"ĠØŃÙĪÙĦ":128967,"лов":128968,"׳ר":128969,"Ġ×ķ׳":128970,"ĠchÆ¡i":128971,"Ġiçinde":128972,"ÑģÑĤвÑĥ":128973,"Ġphá»ij":128974,"ĠÑģÑĥ":128975,"ç§ģãģ¯":128976,"Ġchứng":128977,"Ġvá»±c":128978,"à¹ģà¸Ń":128979,"ĠláºŃp":128980,"Ġtừng":128981,"å°ijãģĹ":128982,"ĠNguy":128983,"ĠNguyá»ħn":128984,"ĠÙģÙĬÙĩ":128985,"Ġба":128986,"×Ļ×Ļת":128987,"Ġ×ľ×¢×©×ķת":128988,"Ġ×ŀ׼":128989,"Ġnghiá»ĩm":128990,"Ġмного":128991,"Ġее":128992,"ëIJĺìĸ´":128993,"Ġlợi":128994,"Ġ׾׾×IJ":128995,"Ġ׼ף":128996,"ĠchÃŃ":128997,"ãģ§ãģ®":128998,"×Ĺ×ķ":128999,"ש×ķ×Ŀ":129000,"Ġ×ŀר":129001,"ĠÐĶлÑı":129002,"Åģ":129003,"Ġ׼×IJשר":129004,"ĠMá»Ļt":129005,"ĠÙĪاÙĦت":129006,"ĠìĿ´ëŁ°":129007,"ÅŁa":129008,"Ġchiến":129009,"Ġarasında":129010,"Ġ×ij×IJתר":129011,"ãģķãĤĮãģ¦ãģĦãĤĭ":129012,"Ø´ÙĥÙĦ":129013,"Ġtượng":129014,"Ġتت":129015,"ĠCó":129016,"Ġbá»ı":129017,"Ġtá»īnh":129018,"ĠkhÃŃ":129019,"ĠпÑĢоÑģÑĤ":129020,"ĠпÑĢоÑģÑĤо":129021,"ĠÙĪÙĤاÙĦ":129022,"Ġgiáo":129023,"ĠNếu":129024,"×IJ×ŀר":129025,"×¢×ł×Ļ×Ļף":129026,"íݸ":129027,"ÙĩدÙģ":129028,"ĠBá»Ļ":129029,"ĠbÃłn":129030,"Ġnguyên":129031,"Ġgüzel":129032,"สาย":129033,"ì²ľ":129034,"×ŀ×ķר":129035,"Ġphân":129036,"ספק":129037,"ק×ij׾":129038,"ĠاÙĦÙħتØŃ":129039,"ĠاÙĦÙħتØŃدة":129040,"ائد":129041,"Ġ×IJ×ŀר":129042,"ĠkiÅŁi":129043,"ì¤Ģ":129044,"Ġtruyá»ģn":129045,"ĠÙĦÙĩا":129046,"ĠÐľÐ°":129047,"à¸ļริษ":129048,"à¸ļริษั":129049,"à¸ļริษัà¸Ĺ":129050,"Ġש׳×Ļ×Ŀ":129051,"ĠменÑı":129052,"ÅŁe":129053,"Ġdiá»ĩn":129054,"Ġ×IJ׳×Ĺ׳×ķ":129055,"kü":129056,"Ġcá»ķ":129057,"Ġmá»Ĺi":129058,"wä":129059,"ÙħÙĬ":129060,"Ġhiá»ĥu":129061,"ëĭ¬":129062,"Ġ×Ķ×Ĺ׾":129063,"Ġtên":129064,"Ġkiá»ĩn":129065,"ÙĨÙĤÙĦ":129066,"Ġvá»ĩ":129067,"×ĵת":129068,"ĠÐłÐ¾ÑģÑģии":129069,"лÑĥ":129070,"ĠاÙĦعربÙĬØ©":129071,"ĠطرÙĬÙĤ":129072,"Ġ×Ķ×ij×Ļת":129073,"ÑģеÑĢ":129074,"Ġмне":129075,"äu":129076,"Ġtriá»ĩu":129077,"ĠÄijủ":129078,"Ġר×ij":129079,"تÙĩÙħ":129080,"à¸ĭี":129081,"Ġì§Ģê¸Ī":129082,"liÅĽmy":129083,"دعÙħ":129084,"ãģłãĤįãģĨ":129085,"Ñģкие":129086,"Ġhá»ıi":129087,"Ġק×ķ":129088,"ÑĢÑĥÑģ":129089,"ÙĨظر":129090,"ãģ®ãĤĤ":129091,"Ġ×Ķ׼×Ļ":129092,"ĠìĽIJ":129093,"ÙĪÙĩ":129094,"ĠÙĪÙİ":129095,"ĠBạn":129096,"плаÑĤ":129097,"Ġ×ŀ×ŀש":129098,"лÑİб":129099,"ĠнÑĥжно":129100,"ĠthÆ°":129101,"ãģµ":129102,"ãģıãĤīãģĦ":129103,"رش":129104,"ר×ķ×Ĺ":129105,"ĠÙĬتÙħ":129106,"Ġצר×Ļ×ļ":129107,"Ġphá":129108,"มà¸Ńà¸ĩ":129109,"Ġ×ij×IJ×ķפף":129110,"Ġcảnh":129111,"Ġíķľëĭ¤":129112,"Ġ×Ķ×ŀת":129113,"à¸ķà¹Īาà¸ĩà¹Ĩ":129114,"มีà¸ģาร":129115,"ÑģкиÑħ":129116,"ĠÐĴÑģе":129117,"ĠاÙĪ":129118,"جÙĬ":129119,"ãģĵãģ¨ãģ¯":129120,"ĠdÃłi":129121,"Ġhá»ĵ":129122,"èĩªåĪĨãģ®":129123,"à¹Ħหà¸Ļ":129124,"ëĵ¤ìĿĦ":129125,"ĠVÄĥn":129126,"Ġдаж":129127,"Ġдаже":129128,"Ñĭми":129129,"лаÑģÑĮ":129130,"ÙĬÙĪÙĨ":129131,"ÙĨÙĪ":129132,"có":129133,"ãģĹãģ¦ãģĦãģŁ":129134,"ãģłãģĭãĤī":129135,"طاÙĦب":129136,"Ġcá»Ńa":129137,"пÑĢоÑģ":129138,"ãģªãģ©ãģ®":129139,"รุà¹Īà¸Ļ":129140,"Ġchiếc":129141,"лÑĭ":129142,"ĠÑıвлÑıеÑĤÑģÑı":129143,"Ġná»ķi":129144,"ãģ®ãģĬ":129145,"Ġ×IJת×Ŀ":129146,"ĠëķĮ문ìĹIJ":129147,"à¸ģลาà¸ĩ":129148,"ĠbaÅŁka":129149,"ìĦĿ":129150,"ĠÑĨел":129151,"ÙģÙĤ":129152,"ãģ«ãĤĪãĤĭ":129153,"ÙĤا":129154,"Ġçıkar":129155,"Ġcứu":129156,"طا":129157,"Ġשת":129158,"à¹Ĥà¸Ħ":129159,"Ġ×ŀ׾":129160,"Ġ×Ķפר":129161,"Ġгде":129162,"Ġخط":129163,"åīįãģ«":129164,"cjÄĻ":129165,"Ġ×Ĺש×ķ×ij":129166,"ר×Ĵ×¢":129167,"Ġkhoảng":129168,"ĠÄijá»Ŀi":129169,"ĠÐłÐµ":129170,"Ġона":129171,"Ġ×IJ׳×ķ":129172,"ãģ®ãģ«":129173,"ĠاÙĦØ°ÙĬÙĨ":129174,"кÑĥп":129175,"ãĤµãĥ¼ãĥ":129176,"ãĤµãĥ¼ãĥĵ":129177,"ãĤµãĥ¼ãĥĵãĤ¹":129178,"вал":129179,"ге":129180,"Ġgiữa":129181,"ĠKhông":129182,"ĠâĹĭ":129183,"à¸ģลุà¹Īม":129184,"ĠÙħÙĨØ°":129185,"à¸Ńà¹Īาà¸Ļ":129186,"ĠÑģпоÑģоб":129187,"ĠÄijá»Ļi":129188,"ĠdiÄŁer":129189,"Ġà¸ĸà¹īา":129190,"ÙħØ«ÙĦ":129191,"Ġ×Ķ×IJ×Ļ":129192,"ĠدÙĪÙĨ":129193,"ÙĬراÙĨ":129194,"Ñīи":129195,"بÙĨاء":129196,"Ġآخر":129197,"ظÙĩر":129198,"Ġ×ij׼":129199,"ĠاÙĦÙħع":129200,"ãĥĴ":129201,"Ġtất":129202,"Ġmục":129203,"ĠdoÄŁru":129204,"ãģŁãĤī":129205,"Ġס×ķ":129206,"Ġxác":129207,"รà¸Ń":129208,"ĠcÄĥn":129209,"Ġонл":129210,"Ġонлайн":129211,"Ġký":129212,"Ġchân":129213,"Ġà¹Ħมà¹Ī":129214,"اØŃØ©":129215,"rán":129216,"׳×Ļ×Ļ×Ŀ":129217,"Ġ×ijף":129218,"ĠÐĸ":129219,"à¸ķรà¸ĩ":129220,"дÑĭ":129221,"Ġsắc":129222,"ÙĦت":129223,"ãĥŃãĥ¼":129224,"ĠÙĦÙĨ":129225,"Ġר×ķ":129226,"ĠdÆ°á»Ľi":129227,"à¹Ģà¸ĺ":129228,"à¹Ģà¸ĺà¸Ń":129229,"eÄŁi":129230,"Ġ×ķש":129231,"ĠÙĦØ£":129232,"Ġgặp":129233,"Ġcá»ij":129234,"ãģ¨ãģ¦ãĤĤ":129235,"رÙĪس":129236,"Ġ׾×Ķ×Ļ":129237,"Ġ본":129238,"ä¸ĬãģĴ":129239,"Ġmức":129240,"Ñħа":129241,"Ġìŀ¬":129242,"à¸īัà¸Ļ":129243,"ÑĢÑĥж":129244,"Ġaçık":129245,"ÙĪاÙĦ":129246,"Ġ×ĸ×ŀף":129247,"人ãģ¯":129248,"عÙĬÙĨ":129249,"ÑıÑħ":129250,"Ġ×Ĵ×ĵ×ķ׾":129251,"ר×ķ×ij":129252,"gó":129253,"ëĿ¼ê³ł":129254,"ĠarkadaÅŁ":129255,"ÙĨشر":129256,"ĠгодÑĥ":129257,"ĠболÑĮÑĪе":129258,"ãģ¡ãĤĩãģ£ãģ¨":129259,"Ġcâu":129260,"Ġsát":129261,"íĶ¼":129262,"Ġtiến":129263,"íķ´ìķ¼":129264,"ĠÙĪØ£ÙĨ":129265,"à¸Ļาà¸Ļ":129266,"Ġ×ij×IJ×ŀצע":129267,"Ġ×ij×IJ×ŀצע×ķת":129268,"Ġ׾ר":129269,"Ġquản":129270,"ĠÙĪاÙĦØ£":129271,"Ġ×IJ×ķת×Ķ":129272,"Ġìĸ´ëĸ¤":129273,"Ġê²ĥìĿĢ":129274,"ØŃسÙĨ":129275,"Ġmất":129276,"à¸Ħูà¹Ī":129277,"ãĥ¬ãĥ¼":129278,"ĠÐĶа":129279,"Ġolması":129280,"Ġthuá»Ļc":129281,"׳×Ĺ":129282,"íĨł":129283,"Ġsöyle":129284,"ãģĿãģĨãģ§ãģĻ":129285,"ĠتÙĥÙĪÙĨ":129286,"лÑĥÑĩ":129287,"׾×Ļ×ļ":129288,"ĠØ£ØŃد":129289,"лиÑģÑĮ":129290,"ĠвÑģего":129291,"Ġ×Ķר×ij":129292,"Ġ못":129293,"oÄŁ":129294,"oÄŁlu":129295,"ĠìĦł":129296,"ĠкаÑĢ":129297,"à¸łà¸²à¸Ħ":129298,"eÅĦ":129299,"Ġà¸ģà¹ĩ":129300,"Ġaynı":129301,"ĠbÃł":129302,"ãģªãĤĵãģ¦":129303,"Ġ모ëĵł":129304,"ÙĤرار":129305,"ãģĹãģªãģĦ":129306,"ĠÐĴо":129307,"ĠÙĪÙĩÙĬ":129308,"ники":129309,"ãĤĮãģŁ":129310,"Ġchuẩn":129311,"רע":129312,"ÙģرÙĬÙĤ":129313,"ãĤĴåıĹãģij":129314,"ĠÄijúng":129315,"бе":129316,"׼×ķ×Ĺ":129317,"пÑĥ":129318,"Ġ×ķ×Ĵ×Ŀ":129319,"×ŀ׳×Ļ":129320,"íĸ¥":129321,"צ×Ļ×Ŀ":129322,"à¸ĭิ":129323,"ÙĩÙĨ":129324,"нем":129325,"Ġ×ij×ij×Ļת":129326,"رع":129327,"Ġส":129328,"ĠÄIJÃł":129329,"íķĺëĭ¤":129330,"Ġấy":129331,"×Ĺ×ķ×ĵ":129332,"×Ĺ×ķ×ĵש":129333,"ĠÑĩеÑĢез":129334,"Ñĥл":129335,"ĠBình":129336,"Ġê²ĥìĿĦ":129337,"Ġ×Ĵר":129338,"ä»ĺãģij":129339,"×Ĺ׾ק":129340,"ĠتÙĦÙĥ":129341,"à¹ĥสà¹Ī":129342,"szÄħ":129343,"ÙĤاÙħ":129344,"دÙĪر":129345,"ĠÙģÙĤØ·":129346,"Ġhữu":129347,"ĠмогÑĥÑĤ":129348,"Ġgá»įi":129349,"Ġקר":129350,"à¸Īะมี":129351,"تÙĤدÙħ":129352,"Ġعبر":129353,"Ġ׾×Ķ×Ŀ":129354,"ĠÑģамо":129355,"ס×ĵר":129356,"ĠcÃłng":129357,"rÃŃ":129358,"Ġìŀ¥":129359,"ëĵ¤ìĿĺ":129360,"ĠÙĦÙĥ":129361,"поÑĢÑĤ":129362,"Ġkhả":129363,"ĠÑģебÑı":129364,"׳ף":129365,"ĠدÙĪر":129366,"Ġmợ":129367,"Ġcây":129368,"Ġfark":129369,"Ġfarklı":129370,"аÑİÑĤ":129371,"Ġtrá»±c":129372,"wiÄĻksz":129373,"Ġthuá»ijc":129374,"ĠتØŃت":129375,"تÙĦ":129376,"овÑĭе":129377,"ëĤł":129378,"Ġвам":129379,"بÙĦغ":129380,"Ġê°ĻìĿĢ":129381,"íĮIJ":129382,"ÙĦب":129383,"Ġnasıl":129384,"Ġодин":129385,"ман":129386,"ĠعÙĦÙĬÙĩا":129387,"би":129388,"Ġפש×ķ×ĺ":129389,"×ijר×Ļ":129390,"Ġש׳×Ķ":129391,"ĠëıĦ":129392,"ĠÄIJại":129393,"Ġ×IJ×ķת×Ŀ":129394,"ĠاÙĦØŃر":129395,"Ġбо":129396,"à¸Īุà¸Ķ":129397,"Ġrõ":129398,"ĠdeÄŁiÅŁ":129399,"Ġëĭ¨":129400,"ĠÑģлÑĥÑĩа":129401,"ĠÑģлÑĥÑĩае":129402,"Ġ×IJ׳ש×Ļ×Ŀ":129403,"×ĵ×£":129404,"ש×ijת":129405,"Ġש׾׼×Ŀ":129406,"Ġchú":129407,"ników":129408,"Ġtanı":129409,"Ġcáo":129410,"ĠÄijá":129411,"Ġ×IJ×ĵ×Ŀ":129412,"Ġê°ķ":129413,"Ġnhiá»ĩm":129414,"Ġ×ľ×¡":129415,"Ġ×Ľ×ª×ij":129416,"Ġ×Ķספר":129417,"ĠÄijÄĥng":129418,"ĠëijIJ":129419,"à¸ľà¸´":129420,"à¸ľà¸´à¸§":129421,"جا":129422,"Ġê°IJ":129423,"رأ":129424,"ستخدÙħ":129425,"ãģ«ãģªãĤĬãģ¾ãģĻ":129426,"Ġtá»·":129427,"×ĺ×ķר":129428,"говоÑĢ":129429,"ĠвоÑģ":129430,"ĠÙħÙĨÙĩا":129431,"иÑĢоваÑĤÑĮ":129432,"ĠÄijầy":129433,"׳×Ĵ":129434,"ĠÙħÙĪ":129435,"ĠÙħÙĪÙĤع":129436,"ר׼×Ļ":129437,"تÙı":129438,"모":129439,"Ġת×ķ":129440,"ÙĬاÙĭ":129441,"à¹ĥà¸Ķ":129442,"ãĤĬãģ¾ãģĻ":129443,"à¸Ńยูà¹Īà¹ĥà¸Ļ":129444,"ĠØ£ÙĪÙĦ":129445,"ĠأخرÙī":129446,"ĠcÆ°":129447,"صار":129448,"×ŀ×Ĺש×ij":129449,"бÑĢа":129450,"ÅĦski":129451,"бÑĢ":129452,"ĠÙĬÙı":129453,"à¸ģิà¸Ļ":129454,"Ġchá»ijng":129455,"ÙħÙı":129456,"Ġà¸Ħืà¸Ń":129457,"ĠتÙĨ":129458,"tÃŃ":129459,"yÄĩ":129460,"Ġmạng":129461,"ÙģÙĪ":129462,"Ġdünya":129463,"קר×IJ":129464,"Ġק׾":129465,"ĠØŃاÙĦ":129466,"cÃŃa":129467,"Ġà¹Ģรา":129468,"Ġר×ķצ×Ķ":129469,"Ġáp":129470,"ë°ķ":129471,"اÙĤØ©":129472,"ниÑİ":129473,"Ġ×IJ׾×ķ":129474,"Ġ×ŀס×ķ":129475,"ãģ§ãģ¯ãģªãģı":129476,"Ġtrả":129477,"Ġקשר":129478,"miÅŁtir":129479,"ĠlÆ°u":129480,"Ġhá»Ĺ":129481,"ĠбÑĭли":129482,"Ġlấy":129483,"عÙĦÙħ":129484,"Ġözel":129485,"æ°ĹãģĮ":129486,"Ġ×ĵר×ļ":129487,"Ùħد":129488,"sını":129489,"׳×ķש×IJ":129490,"rów":129491,"ÑĩеÑĢ":129492,"êµIJìľ¡":129493,"ĠÐľÐ¾":129494,"лег":129495,"ĠVỼi":129496,"วัà¸Ļà¸Ļีà¹ī":129497,"ÑİÑīие":129498,"ãģĬãģĻ":129499,"ãģĬãģĻãģĻ":129500,"ãģĬãģĻãģĻãĤģ":129501,"ëıħ":129502,"Ġ×Ļ×Ķ×Ļ×Ķ":129503,"×ŀ×ĺר":129504,"Ñıми":129505,"Ġlá»±a":129506,"ĠÄijấu":129507,"à¹Ģสียà¸ĩ":129508,"ĠtÆ°Æ¡ng":129509,"ëĵ±":129510,"ĠÑģÑĤаÑĢ":129511,"à¹ĥà¸ļ":129512,"วัà¸Ķ":129513,"ĠÄ°stanbul":129514,"Ġà¸Īะ":129515,"à¸ķลาà¸Ķ":129516,"ĠبÙĬ":129517,"à¹ģà¸Ļะ":129518,"à¹ģà¸Ļะà¸Ļำ":129519,"ساعد":129520,"Ġبأ":129521,"Ġkiá»ĥm":129522,"ØŃسب":129523,"à¸Ĭัà¹īà¸Ļ":129524,"Ġ×ķ×¢×ķ×ĵ":129525,"овÑĭÑħ":129526,"оÑģнов":129527,"ĠtrÆ°á»Łng":129528,"צ×ij×¢":129529,"ĠÃŃt":129530,"Ġkỹ":129531,"cré":129532,"Ñıм":129533,"êµ°":129534,"ãģĮãģªãģĦ":129535,"ÙĬÙĦØ©":129536,"ãĥķãĤ£":129537,"رÙī":129538,"ĠÙĬجب":129539,"Ġ×IJ×£":129540,"Ġcá»±c":129541,"ãĤīãĤĮãģŁ":129542,"Ġà¸ľà¸¹à¹ī":129543,"Ġà¸Ń":129544,"larımız":129545,"Ġkadın":129546,"Ġê·¸ëŀĺ":129547,"Ġê·¸ëŀĺìĦľ":129548,"ĠëĺIJëĬĶ":129549,"ĠÄijả":129550,"ĠÄijảm":129551,"Ġ×IJ×ķ×ŀר":129552,"Ġyếu":129553,"ciÄħ":129554,"ciÄħg":129555,"Ġtá»ij":129556,"Ġש×IJ׳×Ļ":129557,"ĠdziaÅĤa":129558,"Ñīа":129559,"ĠÄijÃłn":129560,"sına":129561,"ãģĵãĤĮãģ¯":129562,"Ġ×ij׾×Ļ":129563,"Ġ×ij×Ļשר×IJ׾":129564,"лоÑģÑĮ":129565,"Ġgiữ":129566,"ê°IJ":129567,"ÑĢон":129568,"تجار":129569,"глав":129570,"вин":129571,"Ġhạn":129572,"Ġyapılan":129573,"بس":129574,"Ġà¸ŀรà¹īà¸Ńม":129575,"ê´Ģ리":129576,"mÄ±ÅŁtır":129577,"bü":129578,"rück":129579,"ĠBaÅŁkanı":129580,"ĠÙĦÙĬس":129581,"ĠsÆ¡":129582,"à¸Īัà¸ĩหว":129583,"à¸Īัà¸ĩหวัà¸Ķ":129584,"داء":129585,"Ġ×Ķ׼":129586,"vÃŃ":129587,"ש×IJר":129588,"ĠhÆ°á»Łng":129589,"Ġbóng":129590,"ĠChÃŃnh":129591,"Äħc":129592,"à¹Ģà¸ģีà¹Īยวà¸ģัà¸ļ":129593,"Ġtứ":129594,"Ġtức":129595,"ĠÑĨвеÑĤ":129596,"Ġtá»iji":129597,"ĠnghÄ©a":129598,"ÙĦاعب":129599,"دÙĦ":129600,"Ġפע×Ŀ":129601,"hör":129602,"à¸Ĭุà¸Ķ":129603,"à¸ŀู":129604,"à¸ŀูà¸Ķ":129605,"паÑģ":129606,"ĠÅŁu":129607,"ĠtÆ°á»Łng":129608,"خارج":129609,"Ġâm":129610,"ĠинÑĤеÑĢеÑģ":129611,"еннÑĭÑħ":129612,"×IJ׳×Ļ":129613,"بدأ":129614,"ëĿ¼ëĬĶ":129615,"ì¹´":129616,"æĸ¹ãģĮ":129617,"лив":129618,"Ġà¸Ħà¸Ļ":129619,"ער×ļ":129620,"à¸Ĥà¸Ńà¸ĩà¸Ħุà¸ĵ":129621,"пад":129622,"Ġcạnh":129623,"ĠëĤ¨":129624,"ĠÄijâu":129625,"Ġbiá»ĥu":129626,"ãĤĤãģĤãĤĭ":129627,"׾×Ĵ":129628,"Ġสำหรัà¸ļ":129629,"Ġxuá»ijng":129630,"ס×ķ":129631,"Ġذات":129632,"ĠÐľÐµ":129633,"عاÙĦÙħ":129634,"×IJס":129635,"بÙĬØ©":129636,"شا":129637,"ием":129638,"ĠNgÆ°á»Ŀi":129639,"íĺij":129640,"Ñģлов":129641,"Ġпа":129642,"Ġmẫu":129643,"ĠпÑĢоÑĨеÑģÑģ":129644,"ĠNhÃł":129645,"пÑĢоиз":129646,"пÑĢоизвод":129647,"à¸łà¸²à¸¢à¹ĥà¸Ļ":129648,"Ġà¸ļาà¸Ĺ":129649,"×ŀ׳×ķ":129650,"ĠоÑĢган":129651,"רצ×ķ":129652,"×ķ×ŀ×Ļ×Ŀ":129653,"Ġyazı":129654,"Ġdù":129655,"ãĥ¬ãĥ³":129656,"ÙĪÙĦÙĬ":129657,"ยู":129658,"Ġtrò":129659,"à¹Ģà¸ŀลà¸ĩ":129660,"Ġ×ŀ׾×IJ":129661,"à¸ķล":129662,"à¸ķลà¸Ńà¸Ķ":129663,"ĠÄijạt":129664,"Ġ×Ĺ×ĵש":129665,"póÅĤ":129666,"Ġ×ŀ×ĵ×Ļ":129667,"ujÄħc":129668,"×ŀ׳×Ķ׾":129669,"Ġש×ij×ķ":129670,"Ġ×Ķ×ŀשפ×ĺ":129671,"Ġ×IJ׾×Ķ":129672,"ĠÙĪØ°ÙĦÙĥ":129673,"à¹Ģà¸ŀราะ":129674,"ĠÄijoÃłn":129675,"Ġíķ¨ê»ĺ":129676,"Ġdục":129677,"شت":129678,"Ġula":129679,"ĠulaÅŁ":129680,"Ġquý":129681,"Ġ×Ķ×Ĵ×ĵ×ķ׾":129682,"à¸ķัà¹īà¸ĩà¹ģà¸ķà¹Ī":129683,"Ġשר":129684,"Ø´Ùĩد":129685,"׳ש×Ļ×Ŀ":129686,"à¸ŀล":129687,"رÙĪا":129688,"ãĤĮãģ¦":129689,"ĠниÑħ":129690,"Ġдела":129691,"ãģ§ãģįãģªãģĦ":129692,"ÅĤoż":129693,"×IJ×Ĺר":129694,"ì½Ķ":129695,"ãĤ¢ãĥĥãĥĹ":129696,"دÙģع":129697,"Ġtiá»ĩn":129698,"Ġkhá»ı":129699,"Ġkhá»ıe":129700,"ĠاÙĦعاÙħØ©":129701,"ãģ«ãģĤãĤĭ":129702,"ĠÄijá»Ļc":129703,"족":129704,"Ġcụ":129705,"йÑĤе":129706,"Ġзакон":129707,"ĠпÑĢоекÑĤ":129708,"ìĸ¸":129709,"ÙĦØŃ":129710,"ĠçalÄ±ÅŁma":129711,"ãĤĴãģĻãĤĭ":129712,"Ñħи":129713,"عاد":129714,"Ġ׳×ŀצ×IJ":129715,"Ġר×Ļ":129716,"à¸Ńà¸Ńà¸ģมา":129717,"ĠTôi":129718,"Ġthần":129719,"ĠÙĬا":129720,"ลาย":129721,"ĠавÑĤо":129722,"Ġsıra":129723,"ĠÙĥØ«ÙĬر":129724,"ÙħÙĬز":129725,"ĠاÙĦعÙĦÙħ":129726,"æĸ¹ãģ¯":129727,"×ķ×¢×ĵ":129728,"ĠоблаÑģÑĤи":129729,"×Ļ׾×Ļ×Ŀ":129730,"ãģĮåĩº":129731,"à¸ĺุ":129732,"à¸ĺุร":129733,"à¸ĺุรà¸ģิà¸Ī":129734,"ÙĤتÙĦ":129735,"ר×IJ×ķ":129736,"Ġngu":129737,"Ġnguá»ĵn":129738,"Ġมา":129739,"Ġплан":129740,"tório":129741,"Ġcuá»iji":129742,"Ñģком":129743,"ĠاÙĦÙħاض":129744,"ĠاÙĦÙħاضÙĬ":129745,"Ġ×ij×¢×ľ":129746,"Ġר×ij×Ļ×Ŀ":129747,"ĠluáºŃn":129748,"ÙĥÙĪ":129749,"à¸Ĺัà¹īà¸ĩหมà¸Ķ":129750,"ван":129751,"Ġthoại":129752,"à¹Ħà¸Ń":129753,"биÑĢ":129754,"ĠاÙĦض":129755,"تا":129756,"ĠÑĢод":129757,"ĠVÃł":129758,"×ŀ×Ļף":129759,"ĠбÑĭла":129760,"ками":129761,"ĠÐĶе":129762,"tık":129763,"קר×Ļ":129764,"ĠeÄŁitim":129765,"ĠÙĥبÙĬر":129766,"بÙĥ":129767,"ĠÙĦÙĪ":129768,"вой":129769,"Ġãģĵãģ®":129770,"ĠÑĤÑĢÑĥд":129771,"myÅĽl":129772,"ĠsÆ°":129773,"à¸ŀีà¹Ī":129774,"Ġà¹ģลà¹īว":129775,"עק":129776,"Ġ×Ĺ×ijרת":129777,"ระหว":129778,"ระหวà¹Īาà¸ĩ":129779,"×Ļ×Ļ×Ķ":129780,"ĠاÙĦÙĨاس":129781,"ünü":129782,"Ġ׾×ŀ×Ķ":129783,"ĠchÆ°Æ¡ng":129784,"ĠHá»ĵ":129785,"ارت":129786,"ãĤĪãģĨãģ§ãģĻ":129787,"lá":129788,"ק×Ļ×Ļ×Ŀ":129789,"æľ¬å½ĵ":129790,"æľ¬å½ĵãģ«":129791,"ãģĵãĤĵãģª":129792,"Ñģов":129793,"Ġ×ķ×Ĺ":129794,"à¹Ģà¸ģà¹ĩà¸ļ":129795,"ĠкÑĤо":129796,"à¹Ĥรà¸Ħ":129797,"ĠشرÙĥØ©":129798,"عزÙĬ":129799,"عزÙĬز":129800,"Ø·ÙĦÙĤ":129801,"пÑĥÑģÑĤ":129802,"ÙģتØŃ":129803,"ëŀĢ":129804,"Ġhãy":129805,"ضÙħ":129806,"린":129807,"åł´åIJĪãģ¯":129808,"ãĤªãĥ¼":129809,"Ġhắn":129810,"Ġ×IJ×ij×Ļ×ij":129811,"Ġש׾×Ķ×Ŀ":129812,"Ġ×Ķ×Ļ×Ļת×Ķ":129813,"ĠاÙĦدÙĪÙĦØ©":129814,"ĠاÙĦÙĪÙĤ":129815,"ĠاÙĦÙĪÙĤت":129816,"ãģĤãģ¾ãĤĬ":129817,"ĠtaÅŁÄ±":129818,"Ä°N":129819,"עסק":129820,"ãģ¦ãģĦãģŁ":129821,"Ġtá»ķng":129822,"ĠاÙĦØ¥ÙĨس":129823,"ĠاÙĦØ¥ÙĨساÙĨ":129824,"ÑĢеÑĪ":129825,"Ġgái":129826,"ĠÑĨен":129827,"ĠÙģÙĤد":129828,"Ùħات":129829,"ãģķãĤĵãģ®":129830,"Ġphù":129831,"×ĺ×Ķ":129832,"ĠÙĪاÙĦتÙĬ":129833,"ĠبÙĥ":129834,"ìĿ´ëĤĺ":129835,"кÑģ":129836,"ÙħÙĬر":129837,"Ġvùng":129838,"ĠاÙĦشعب":129839,"ĠNhÆ°ng":129840,"ãĥĢãĥ¼":129841,"Ġ×Ĺ×Ļ×Ļ×Ŀ":129842,"Ġشخص":129843,"ק×ķ×ĵ":129844,"ê²Ģ":129845,"עש":129846,"×¢×ķ׾×Ŀ":129847,"צ×ķר":129848,"عÙĤد":129849,"ĠiÅŁlem":129850,"Ġ×Ķ×ij×IJ":129851,"Ġdưỡng":129852,"à¸Łà¸£à¸µ":129853,"ĠphÃŃa":129854,"ãģ®ä¸Ńãģ§":129855,"Ġпи":129856,"ĠngÃłnh":129857,"нима":129858,"ĠÙĩÙĦ":129859,"Ġ×ķ×IJת":129860,"ĠÄijáng":129861,"équipe":129862,"ĠÑįÑĤоÑĤ":129863,"Ġgörev":129864,"매":129865,"Ġquân":129866,"å¼ķãģį":129867,"æĻĤãģ«":129868,"ĠبÙħا":129869,"×ŀ×Ļת":129870,"Ġülke":129871,"Ġ×ŀק×ķ×Ŀ":129872,"×ijף":129873,"æ°ĹæĮģãģ¡":129874,"Ġë§İìĿĢ":129875,"Ġyüksek":129876,"ÑĨенÑĤÑĢ":129877,"ĠÙħجÙĦس":129878,"ç§ģãģ®":129879,"ÙĤدر":129880,"Ġë¶Ģë¶Ħ":129881,"Ġì°¨":129882,"خرج":129883,"ãģĭãģªãĤĬ":129884,"ë³´ëĭ¤":129885,"Ġ×ŀ×Ļ×ĵ×¢":129886,"peÅĤni":129887,"Ġxá»Ń":129888,"ìĹIJìĦľëĬĶ":129889,"ĠباÙĦÙħ":129890,"ĠÙĪÙħا":129891,"ĠÑįÑĤой":129892,"بÙĬÙĨ":129893,"nü":129894,"ØŃز":129895,"ØŃزب":129896,"ĠÑĢабоÑĤа":129897,"ĠNháºŃt":129898,"ÙĦاء":129899,"Ġëĵ¤":129900,"Ġëĵ¤ìĸ´":129901,"ãĤĦãģĻãģĦ":129902,"×Ĺ×ĸק":129903,"Ġ×Ķ×Ĺ×ijר×Ķ":129904,"пиÑĤ":129905,"ãģĭãĤīãģ®":129906,"Ġë§IJìĶĢ":129907,"Ġפ×ķ":129908,"ÙĦÙİ":129909,"à¹Ģà¸ķà¹ĩม":129910,"ĠÐļо":129911,"Ġmówi":129912,"ĠtÃŃn":129913,"ר×Ĵש":129914,"פרק":129915,"Ġtrạng":129916,"ĠÐŀн":129917,"×Ĺ×ķ×¥":129918,"ĠعÙĨدÙħا":129919,"Ġبر":129920,"使ãģĦ":129921,"Ġrá»Ļng":129922,"ëĮĢë¡ľ":129923,"íĪ¬":129924,"Ġktórych":129925,"вид":129926,"ลูà¸ģà¸Ħà¹īา":129927,"ĠmogÄħ":129928,"Ġש×Ĺ":129929,"×ij×Ĺר":129930,"ãĥĸãĥŃãĤ°":129931,"ĠThÃłnh":129932,"Ġ×Ķר×Ļ":129933,"ĠÑģÑĤаÑĤÑĮ":129934,"ĠHá»Ļi":129935,"à¸ļà¹īาà¸ĩ":129936,"çī¹ãģ«":129937,"ĠÄIJức":129938,"èĢħãģ®":129939,"×¢×ŀ×ķ×ĵ":129940,"×ĺר×Ķ":129941,"Ð¥":129942,"ĠÙħÙħا":129943,"ĠeÅŁ":129944,"ĠнеобÑħодимо":129945,"ников":129946,"Ġüzerinde":129947,"aÅĤa":129948,"Ġchá»ĭu":129949,"ĠاÙĦدÙĬÙĨ":129950,"أخبار":129951,"ĠÄijau":129952,"ãģĮå¤ļãģĦ":129953,"jÄħcych":129954,"دخÙĦ":129955,"larınd":129956,"larından":129957,"Ġsẻ":129958,"à¸ŀิà¹Ģศ":129959,"à¸ŀิà¹Ģศษ":129960,"×ª×Ł":129961,"tıģı":129962,"ĠluáºŃt":129963,"ĠÅŀe":129964,"ãĤ«ãĥ¼":129965,"ãģ®ãģĤãĤĭ":129966,"Ġ×Ķ×IJתר":129967,"ĠاÙĦØ¢ÙĨ":129968,"ıldı":129969,"Ġáo":129970,"ĠнаÑĩал":129971,"Ġviá»ĩn":129972,"Ġ×ij×¢×ķ׾×Ŀ":129973,"знаÑĩ":129974,"×Ļ×ĺ×Ķ":129975,"кам":129976,"ĠÐĺз":129977,"à¹Ģà¸Ĥียà¸Ļ":129978,"à¸Ļà¹īà¸Ńà¸ĩ":129979,"ÑĤÑĢо":129980,"à¹Ģà¸Ł":129981,"Ġжизни":129982,"Ġสà¹Īวà¸Ļ":129983,"ĠváºŃn":129984,"Ġê´Ģ볨":129985,"Ġlâu":129986,"ס×ĺר":129987,"קש":129988,"سÙĬر":129989,"Ġ×IJ×ķת×Ļ":129990,"Ġmôi":129991,"ائب":129992,"ĠоÑģÑĤа":129993,"Ġmón":129994,"Ġ×ij×ŀק×ķ×Ŀ":129995,"ĠداخÙĦ":129996,"Ġ×IJ×ķר":129997,"ĠваÑģ":129998,"ÙĥØ´Ùģ":129999,"ìĺ¨":130000,"à¸ĸà¹Īาย":130001,"Ġkullanıl":130002,"Ġtô":130003,"ãģ«ãĤĪãĤĬ":130004,"ĠëĺIJíķľ":130005,"Ġ×¢×ij×ķ×ĵ×Ķ":130006,"Ġriê":130007,"Ġriêng":130008,"Ġyakın":130009,"زا":130010,"Å»":130011,"×IJ×ķ׼׾":130012,"شارÙĥ":130013,"ĠбеÑģ":130014,"×´":130015,"ĠابÙĨ":130016,"ĠTá»ķng":130017,"ÙĨظ":130018,"ÅĽwiad":130019,"ãĤµãĥ¼":130020,"หาย":130021,"ĠGün":130022,"Ġhakkında":130023,"à¹Ģà¸Ĥà¹īามา":130024,"زÙĨ":130025,"ĠÐłÐ¾":130026,"Ġbiá»ĥn":130027,"ãģ©ãģĵ":130028,"ÙģعÙĦ":130029,"زع":130030,"פר×ĺ":130031,"Ġ×Ķף":130032,"Ø£ÙĩÙĦ":130033,"Ġthất":130034,"ØŃÙħÙĦ":130035,"ÑĩÑĥ":130036,"ĠìĤ¬ìĭ¤":130037,"ì°¸":130038,"ĠìľĦíķ´":130039,"ÙĪظ":130040,"ĠÐŁÐ¾Ð´":130041,"Ġkhoản":130042,"ÑĤен":130043,"ĠÙģاÙĦ":130044,"Ñģад":130045,"à¸Ļà¸Ńà¸Ļ":130046,"ĠاÙĦسعÙĪدÙĬØ©":130047,"\"ØĮ":130048,"ĠاÙĦÙĴ":130049,"ãĤīãģļ":130050,"Ġtoán":130051,"Ġchắc":130052,"׼×Ļר":130053,"méd":130054,"média":130055,"زÙĪ":130056,"Ġyanı":130057,"פ׳×Ļ×Ŀ":130058,"ØŃظ":130059,"ĠбеÑģп":130060,"ĠбеÑģплаÑĤ":130061,"ĠбеÑģплаÑĤно":130062,"ĠØ£ÙħاÙħ":130063,"à¸Ńาย":130064,"à¸Ńายุ":130065,"רשת":130066,"Ġgá»ĵ":130067,"Ġgá»ĵm":130068,"Ġuá»ijng":130069,"صب":130070,"kır":130071,"ãĥijãĥ¼":130072,"Ġ׾×ĵעת":130073,"ĠкÑĥпиÑĤÑĮ":130074,"׾×ķ×Ĺ":130075,"ÙĪضع":130076,"ÙĤÙĬÙħ":130077,"à¸Ľà¸²":130078,"жив":130079,"à¸Ķิà¸Ļ":130080,"×IJ×ķפ":130081,"à¹Ģลà¹ĩà¸ģ":130082,"ãĥĥãĥī":130083,"иÑĩеÑģкиÑħ":130084,"ĠChủ":130085,"кÑĢаÑģ":130086,"ÙĪصÙĦ":130087,"pÅĤat":130088,"моÑĢ":130089,"Ġ×Ķ×IJ×ķ":130090,"à¸Ńิà¸Ļ":130091,"ĠíķľêµŃ":130092,"гÑĢе":130093,"Ġìłľê³µ":130094,"ì°½":130095,"Ġê°ľìĿ¸ìłķë³´":130096,"Ġnghá»ĭ":130097,"à¸ĭา":130098,"ØŃساب":130099,"ĠbyÅĤa":130100,"ÙħÙĦÙĥ":130101,"иÑĩеÑģкие":130102,"Ġbác":130103,"ضØŃ":130104,"길":130105,"ש×ŀ×¢":130106,"Ġìĸ´ëĸ»":130107,"Ġìĸ´ëĸ»ê²Į":130108,"ìĽĮ":130109,"اتÙĩ":130110,"à¹Ĥรà¸ĩà¹ģ":130111,"à¹Ĥรà¸ĩà¹ģรม":130112,"خدÙħØ©":130113,"ĠÐłÐ°":130114,"׼×ķ׾×Ŀ":130115,"×ŀש×Ĺק":130116,"ĠÙĪÙĥاÙĨ":130117,"ס×ķ×£":130118,"ĠاÙĦØŃÙĥÙĪÙħØ©":130119,"Ġ×ij×ĺ":130120,"ĠtráºŃn":130121,"Ġ×Ķ×¢×ķ׾×Ŀ":130122,"ĠÃŃch":130123,"tÄħ":130124,"ש×ŀ×ķ":130125,"Ġ×Ķר×IJש×ķף":130126,"Ġíķĺê³ł":130127,"ãģķãĤī":130128,"ãģķãĤīãģ«":130129,"ãģ«ãģĹãģ¦":130130,"Ġà¸ľà¸¡":130131,"ãģ®ãĤĪãģĨãģª":130132,"ĠÙĪÙĤت":130133,"ãĥįãĥĥãĥĪ":130134,"ÙĦعب":130135,"ÙĪØ´":130136,"ìĺ¬":130137,"Ġหาà¸ģ":130138,"ĠmiaÅĤ":130139,"à¸Ĺà¸Ńà¸ĩ":130140,"иÑĤа":130141,"اصر":130142,"илÑģÑı":130143,"зе":130144,"à¸Ľà¸£à¸°à¸¡à¸²à¸ĵ":130145,"ãģĿãĤĮãģ¯":130146,"Ġbır":130147,"Ġbırak":130148,"صÙĨاع":130149,"Ю":130150,"شعر":130151,"Ġ׳×Ĵ×ĵ":130152,"Ġبسبب":130153,"ãĥĿãĤ¤":130154,"ãĥĿãĤ¤ãĥ³ãĥĪ":130155,"ĠاÙĦجÙĪ":130156,"ĠнеÑģколÑĮко":130157,"Ġkiếm":130158,"ÙģÙİ":130159,"Ġضد":130160,"×ij×Ļ×ĺ×ķ×Ĺ":130161,"تابع":130162,"ÙĨز":130163,"ĠBản":130164,"Ġaçıkl":130165,"Ġaçıklama":130166,"Ġà¸Ħุà¸ĵ":130167,"à¸Ĺา":130168,"ÅĤów":130169,"طب":130170,"ÙĨØŃÙĨ":130171,"Ġ×ŀק×ķר":130172,"ĠÄ°s":130173,"Ġдома":130174,"Ġวัà¸Ļ":130175,"ĠdÃłnh":130176,"Ñıн":130177,"миÑĢ":130178,"Ġmô":130179,"ĠvÃłng":130180,"صاب":130181,"sının":130182,"à¸Ħืà¸Ļ":130183,"خبر":130184,"×ĸ׼×ķ":130185,"Ġ×ŀש×Ķ×ķ":130186,"mü":130187,"Ġкомпании":130188,"Ġ×Ķ×¢×Ļר":130189,"ĠÙĥÙĪ":130190,"ÙĤÙĦب":130191,"ĠlỼp":130192,"ики":130193,"׳×ij":130194,"à¹Ĥà¸Ħร":130195,"à¹Ĥà¸Ħรà¸ĩ":130196,"à¹Ĥà¸Ħรà¸ĩà¸ģาร":130197,"×ŀ×ķ×¢×ĵ":130198,"ÑıÑĤÑģÑı":130199,"หลัà¸ĩà¸Īาà¸ģ":130200,"ениÑİ":130201,"Ġשע":130202,"ĠbÆ°á»Ľc":130203,"ãĥ¡ãĥ¼ãĥ«":130204,"ãĤĦãĤĬ":130205,"Ġ×Ļ×ķ×ĵ×¢":130206,"Ġê´Ģíķľ":130207,"ĠاÙĦØ£Ùħر":130208,"Ġbölge":130209,"ĠÑģвой":130210,"ÙĦس":130211,"Ġ×ŀ×Ļ×ķ×Ĺ×ĵ":130212,"ĠëĤ´ìļ©":130213,"ĠأجÙĦ":130214,"ĠÄIJông":130215,"Ġ×ŀ×ł×ª":130216,"Ġìĭľê°Ħ":130217,"ÙĥÙİ":130218,"ãģ¨ãģĦãģĨãģ®ãģ¯":130219,"Ġnależy":130220,"تÙĨظÙĬÙħ":130221,"ĠÑģозда":130222,"Ġphé":130223,"Ġphép":130224,"ãģ§ãģįãģ¾ãģĻ":130225,"ĠعÙĦÙħ":130226,"大ãģįãģª":130227,"ãĤ²ãĥ¼ãĥł":130228,"íħĮ":130229,"Ġ׼×ķ׾׾":130230,"ĠинÑĤеÑĢнеÑĤ":130231,"ĠTừ":130232,"ãģ¨ãģªãĤĭ":130233,"زاÙĦ":130234,"Ġktórym":130235,"Ġnhé":130236,"ìĪľ":130237,"нев":130238,"деÑĢ":130239,"ãĤ¢ãĥĹãĥª":130240,"iá»ĩu":130241,"×ij×Ļ׾":130242,"Ġتس":130243,"ĠÄIJây":130244,"ĠاÙĦخاصة":130245,"Ġà¹Ģà¸Ĭ":130246,"Ġà¹Ģà¸Ĭà¹Īà¸Ļ":130247,"صاد":130248,"Ġdạng":130249,"سعر":130250,"Ġש×Ļ×ŀ×ķש":130251,"×Ĵ×Ļ×Ŀ":130252,"ãģĮãģĤãģ£ãģŁ":130253,"пÑĢов":130254,"пÑĢовод":130255,"Ġ×IJ×Ļ׳×ķ":130256,"Ġ׾ר×IJ":130257,"Ġ׾ר×IJ×ķת":130258,"ĠØ£ÙģضÙĦ":130259,"ĠØŃÙĦ":130260,"ĠأبÙĪ":130261,"ê°ķ":130262,"Ġì§ij":130263,"ãģ®ãĤĪãģĨãģ«":130264,"Ġפ׳×Ļ":130265,"ס×Ļ×Ŀ":130266,"ĠÙĪÙĩذا":130267,"Ġkaç":130268,"Ġéén":130269,"Ġê±´":130270,"ë°Ķ":130271,"Ñĥз":130272,"à¸Ĥà¸Ńà¸ĩà¹Ģรา":130273,"iÅĤ":130274,"ĠÐľÑĭ":130275,"Ġchết":130276,"ĠاÙĦثاÙĨÙĬ":130277,"×IJק":130278,"Ġ×ķ×¢×ľ":130279,"ĠاÙĦطب":130280,"×ij×ĺ×Ĺ":130281,"ĠجدÙĬدة":130282,"ĠعدÙħ":130283,"عز":130284,"สิà¹Īà¸ĩà¸Ĺีà¹Ī":130285,"ãģĻãĤĮãģ°":130286,"ĠÄijô":130287,"ì£ł":130288,"دÙĤ":130289,"номÑĥ":130290,"Ġká»ĥ":130291,"ãĤ¢ãĥ³":130292,"å¤ļãģıãģ®":130293,"à¸Ľà¸£à¸°à¸ģ":130294,"à¸Ľà¸£à¸°à¸ģà¸Ńà¸ļ":130295,"פע×Ļ׾×ķת":130296,"ĠÑģÑĤол":130297,"mayı":130298,"ãģ¤ãģĦ":130299,"Ġyılında":130300,"Ġà¸Īึà¸ĩ":130301,"koÅĦcz":130302,"ĠThông":130303,"ĠакÑĤив":130304,"нÑģÑĤ":130305,"нÑģÑĤÑĢÑĥ":130306,"ĠÃĸz":130307,"Ġת×ŀ×Ļ×ĵ":130308,"ĠÙĥÙĨت":130309,"ÑģиÑģÑĤем":130310,"prés":130311,"présent":130312,"Ġnâ":130313,"Ġnâng":130314,"gÅĤos":130315,"ĠÙĪزÙĬر":130316,"ØŃصÙĦ":130317,"ĠимееÑĤ":130318,"ØŃرÙĥØ©":130319,"à¸ŀà¹Īà¸Ń":130320,"ãĤĴãģĬ":130321,"ĠاستخداÙħ":130322,"×IJ×Ļר×ķ×¢":130323,"ä»ĸãģ®":130324,"Ġש×Ķ×Ŀ":130325,"ãģĹãģŁãĤī":130326,"ש×ŀ×Ļ":130327,"Ñģла":130328,"mı":130329,"Ġbazı":130330,"Ġíķĺì§Ģë§Į":130331,"×ĵ׾":130332,"Ġyaptıģı":130333,"ãĥĬãĥ¼":130334,"׾×Ļ׾×Ķ":130335,"ãģ¨ãģĦãģ£ãģŁ":130336,"ändig":130337,"ĠÅŁa":130338,"ĠÙģÙĬÙħا":130339,"иÑĤелÑı":130340,"×ŀ×ķש":130341,"à¸Ĥà¸Ńà¸ļ":130342,"lük":130343,"Ġhá»ĵi":130344,"Ġëªħ":130345,"ĠاÙĦÙĥØ«ÙĬر":130346,"צ×IJ":130347,"Ġhazır":130348,"طرÙģ":130349,"اÙĬا":130350,"ĠÄijôi":130351,"енд":130352,"ÙĦغ":130353,"×Ĺ×ĸ×ķר":130354,"ĠвÑģег":130355,"ĠвÑģегда":130356,"ëIJĺê³ł":130357,"×ĵ×ķ×ĵ":130358,"ана":130359,"دÙĪÙĦØ©":130360,"Ġhoạch":130361,"عÙĦا":130362,"عÙĦاج":130363,"Ġ×ķ×¢×ĵ":130364,"×Ķ×Ŀ":130365,"кий":130366,"ÙĦÙIJ":130367,"Ġ×¢×ľ×Ļ×ķ":130368,"ÑİÑīий":130369,"Ġngủ":130370,"صÙĨع":130371,"ĠاÙĦعراÙĤ":130372,"à¸ķà¹Īà¸Ńà¹Ħà¸Ľ":130373,"ãģŁãģıãģķãĤĵ":130374,"Ġphạm":130375,"ÙĦاÙĨ":130376,"اتÙĩا":130377,"Ġböyle":130378,"تÙĨÙģÙĬ":130379,"تÙĨÙģÙĬØ°":130380,"Ġש×Ķ×Ļ×IJ":130381,"ÑģÑĥ":130382,"ยาว":130383,"Ġש×ķ׳×Ļ×Ŀ":130384,"Ġ×ŀ×ķ׾":130385,"ĠÑģил":130386,"Ġ×IJ×Ĺר×Ļ×Ŀ":130387,"Ġphủ":130388,"ÙĤطع":130389,"ĠThủ":130390,"à¸Ľà¸£à¸°à¹Ģà¸Ĺศà¹Ħà¸Ĺย":130391,"ÙĨÙĤ":130392,"ĠÄijoạn":130393,"Ġبإ":130394,"пÑĢедел":130395,"×ķת×ķ":130396,"Ġyarı":130397,"пÑĢе":130398,"ĠczÄĻÅĽci":130399,"ØŃÙĥÙħ":130400,"×ķ׳×Ļת":130401,"×¤×¢×ľ":130402,"ãĤĴãģĹãģ¦":130403,"Ġktórzy":130404,"׾×Ŀ":130405,"ĠÄIJiá»ģu":130406,"ĠкоÑĤоÑĢаÑı":130407,"ĠìĿ´ìĥģ":130408,"ãģĤãģ£ãģŁ":130409,"Ġ×ŀ×ĵ×ķ×ijר":130410,"פ×ķ×¢×ľ":130411,"dım":130412,"éĢļãĤĬ":130413,"ĠбÑĥдÑĥÑĤ":130414,"à¹Ģวà¹ĩà¸ļà¹Ħà¸ĭ":130415,"à¹Ģวà¹ĩà¸ļà¹Ħà¸ĭà¸ķà¹Į":130416,"اخر":130417,"×Ĺ×Ļ׾":130418,"Ġ×Ļ׾":130419,"Ġ×Ļ׾×ĵ×Ļ×Ŀ":130420,"×Ĺ×Ļפ":130421,"×Ĺ×Ļפ×ķש":130422,"Ġdòng":130423,"Ġש×ĸ×Ķ":130424,"ÑĮе":130425,"ãģĤãģ¨":130426,"ìŀIJê°Ģ":130427,"×IJ×ĵ":130428,"Ġüz":130429,"Ġüzere":130430,"ظÙĦ":130431,"Ġ×IJ×ķ׾×Ļ":130432,"Ġ×ij×Ļ×ķ×Ŀ":130433,"ÙĦات":130434,"Ġmê":130435,"침":130436,"تØŃد":130437,"تØŃدث":130438,"Ġخاصة":130439,"ĠبرÙĨ":130440,"ĠبرÙĨاÙħج":130441,"ĠHÃłn":130442,"×Ĺס":130443,"ĠÙĪÙĦÙħ":130444,"×¢×Ŀ":130445,"Ġmı":130446,"à¸Łà¸±à¸ĩ":130447,"שע×Ķ":130448,"ÙĪÙģÙĤ":130449,"ס×ij×Ļר":130450,"алÑĮнÑĭй":130451,"×Ĺש×ķ×ij":130452,"ĠnÃłng":130453,"ë³¼":130454,"ĠкоÑĤоÑĢÑĭÑħ":130455,"Ġ×Ĺ×ķק":130456,"tör":130457,"ĠлÑĥÑĩÑĪе":130458,"ãĥijãĥ³":130459,"ลà¹Īาสุà¸Ķ":130460,"ĠجدÙĬد":130461,"ÙĬدة":130462,"à¸Ĺรà¸ĩ":130463,"ãĤĪãĤĬãĤĤ":130464,"ÙĦÙĦ":130465,"ãĤĤãģ£ãģ¨":130466,"ש×ĺ×Ĺ":130467,"Ġ×ķ×IJ×Ļ":130468,"Ġgiá»ijng":130469,"إضاÙģ":130470,"קת":130471,"ë§Ŀ":130472,"ĠzostaÅĤ":130473,"ÑĢоз":130474,"×Ļפ×Ļ×Ŀ":130475,"Ġ׼׾׾":130476,"ת×ķ׼ף":130477,"dıģını":130478,"ÙĤسÙħ":130479,"ĠÑģÑĩиÑĤ":130480,"ĠÑģÑĩиÑĤа":130481,"×ĺ×ķת":130482,"ĠÆ°u":130483,"ĠØ¢ÙĦ":130484,"Ġмом":130485,"ĠмоменÑĤ":130486,"ĠاÙĦتعÙĦÙĬÙħ":130487,"×¢×ľ×ķת":130488,"Ġchữa":130489,"Ġyön":130490,"ĠtrÃł":130491,"ĠØŃÙĬÙĨ":130492,"à¸ĭั":130493,"ĠCá":130494,"×¢×ĸ":130495,"ĠاÙĦØ£ÙħÙĨ":130496,"cÃŃ":130497,"Ġvá»ijn":130498,"Ġà¸Ļาย":130499,"обÑĢа":130500,"ק×IJ":130501,"Ġthiếu":130502,"ãĥŀãĥ¼":130503,"สวà¸Ļ":130504,"Ġgá»Ń":130505,"Ġgá»Ńi":130506,"Ġê¹":130507,"Ġê¹Ģ":130508,"Ġthiá»ĩn":130509,"ÙĤع":130510,"wÄĻ":130511,"Ġнам":130512,"ÑĤол":130513,"Ġsân":130514,"ס×ķ×Ĵ":130515,"Ġgeçir":130516,"ÑĤон":130517,"ева":130518,"ĠÙĪضع":130519,"Ġعشر":130520,"Ñģло":130521,"à¸Īัà¸ļ":130522,"ãĤ·ãĥ¼":130523,"ãĤĤãģĤãĤĬãģ¾ãģĻ":130524,"Ġvẻ":130525,"ĠÄIJá»ĥ":130526,"رÙģع":130527,"ĠاÙĦØ£ÙĪÙĦÙī":130528,"ÑĤаÑĢ":130529,"ãģªãģıãģ¦":130530,"ÙħÙİ":130531,"quÃŃ":130532,"×¢×ł×Ļ×Ļ׳":130533,"ген":130534,"Ġhôm":130535,"à¸Īา":130536,"ĠnhỼ":130537,"ĠاÙĦعربÙĬ":130538,"×IJף":130539,"Ġlá»Ļ":130540,"ĠjeÅĽli":130541,"à¹Ģà¸Ĺà¹Īาà¸Ļัà¹īà¸Ļ":130542,"ĠØ£ÙĨÙĩا":130543,"Ġtuy":130544,"Ġtuyá»ĩt":130545,"Ġتص":130546,"ĠتصÙĨÙĬ":130547,"ĠتصÙĨÙĬÙģ":130548,"Ġê·¸ëŁ¬ëĤĺ":130549,"оÑĨен":130550,"à¸ģิà¸Īà¸ģรรม":130551,"ãĤĦãģ£ãģ¦":130552,"Ġkhá»ıi":130553,"Ġlá»ĩ":130554,"ĠاÙĦÙħجتÙħع":130555,"à¸Ńาà¸Īà¸Īะ":130556,"à¸Īะà¹Ģà¸Ľà¹ĩà¸Ļ":130557,"овÑĭй":130558,"ר×Ŀ":130559,"รà¹īà¸Ńà¸Ļ":130560,"ש×ŀש":130561,"人ãģ«":130562,"Ġüzerine":130563,"פר×Ļ":130564,"duÄŁu":130565,"Ñĩик":130566,"Ġmùa":130567,"Ġ×ŀת×ķ×ļ":130568,"ĠcáºŃp":130569,"ĠتارÙĬØ®":130570,"×ij×ľ×ª×Ļ":130571,"Ġì¢Ģ":130572,"ÙĦع":130573,"باÙĨ":130574,"Ġchút":130575,"Ġ×Ķ×ĸ×ŀף":130576,"née":130577,"ĠLiên":130578,"ĠÙĦÙĦØ£":130579,"ØŃدÙĪد":130580,"Ġ×¢×Ľ×©×Ļ×ķ":130581,"воз":130582,"Ġyaptı":130583,"Ġобо":130584,"à¹ĥหà¹īà¸ģัà¸ļ":130585,"Ġ×ij×Ķ×Ŀ":130586,"ãģıãģ¦":130587,"رأس":130588,"ĠÑģÑĢедÑģÑĤв":130589,"ĠBÃłi":130590,"ãģĵãģ¨ãģ«":130591,"ĠìĤ¬íļĮ":130592,"Ġ모ëijIJ":130593,"×ij×IJ":130594,"Ġtrắng":130595,"ĠاÙĦبÙĦد":130596,"ĠHoÃłng":130597,"либо":130598,"ĠдÑĢÑĥгиÑħ":130599,"Ä°R":130600,"Ñĥма":130601,"ĠJeÅĽli":130602,"ãĤĤãģĹ":130603,"Ġvòng":130604,"Ġ×IJתר×Ļ×Ŀ":130605,"ĠÄijá»įc":130606,"ĠвоÑĤ":130607,"ãģłãģĮ":130608,"ë°°":130609,"à¸Ķูà¹ģล":130610,"Ġ×ŀ׼׾":130611,"ìĹIJëıĦ":130612,"газ":130613,"Ġ׳×ķספ×Ļ×Ŀ":130614,"ãģĵãģ¨ãģ§":130615,"ĠتÙĪ":130616,"ãģ§ãģĤãĤĬ":130617,"à¸Ļัà¹Īà¸ĩ":130618,"ĠможеÑĤе":130619,"szÄĻ":130620,"ãģ®ãģł":130621,"ĠÙħÙĨÙĩ":130622,"Ġbá»ķ":130623,"Ġbüt":130624,"Ġbütün":130625,"ë³´ê³ł":130626,"Ġchá»ĵng":130627,"à¹ģà¸Īà¹īà¸ĩ":130628,"ĠVì":130629,"ĠØŃر":130630,"Ġgiản":130631,"ĠÙħدÙĬÙĨØ©":130632,"تطبÙĬÙĤ":130633,"à¸Īิ":130634,"æĹ¥ãģ®":130635,"бил":130636,"à¸ģà¸Ńà¸ĩ":130637,"ê³³":130638,"ĠØ£Ùħا":130639,"ìĨIJ":130640,"Ġtrái":130641,"ĠвÑģем":130642,"ĠسÙĨØ©":130643,"ĠÑģайÑĤ":130644,"ĠгоÑĤов":130645,"пÑĭ":130646,"ĠëIJł":130647,"ĠاÙĦخط":130648,"ĠاÙĦرئÙĬسÙĬØ©":130649,"Ġíķ©ëĭĪëĭ¤":130650,"ĠìķĦëĭĪëĿ¼":130651,"ĠìĿ´ëłĩ":130652,"ĠìĿ´ëłĩê²Į":130653,")ØĮ":130654,"hält":130655,"ĠØ£Ùħر":130656,"ĠعÙħر":130657,"à¸ģà¹ĩà¸Īะ":130658,"Ġà¸Ĺำà¹ĥหà¹ī":130659,"Ġcân":130660,"Ġ×ij׾":130661,"Ġ×ij׾×ij×ĵ":130662,"פסק":130663,"ĠÙĬÙĤÙĪÙĦ":130664,"нÑĥÑĤÑĮ":130665,"à¹ģà¸Ħ":130666,"Ġקצת":130667,"Ġnằm":130668,"Ġhòa":130669,"bilitÃł":130670,"ĠìĹĨëĭ¤":130671,"Ġ׼פ×Ļ":130672,"ÑĢож":130673,"лага":130674,"Ġ×Ķש×Ļ":130675,"ĠNgoÃłi":130676,"ĠÙĪج":130677,"ĠÙĪجÙĪد":130678,"ĠìľĦíķľ":130679,"ĠusÅĤug":130680,"Ġtuần":130681,"dź":130682,"×ŀ×ķף":130683,"ĠاÙĦعدÙĬد":130684,"Ġchẳng":130685,"สุà¸Ĥà¸łà¸²à¸ŀ":130686,"Ġ×ij×ĵר×ļ":130687,"ĠÑģебе":130688,"ĠìŀĪìĿĦ":130689,"ĠاÙĦØŃاÙĦ":130690,"Ġdá":130691,"ĠcÆ°á»Ŀi":130692,"Ġnghiên":130693,"ieÅĦ":130694,"ĠDÆ°Æ¡ng":130695,"ï¼ħ":130696,"شد":130697,"ãģĦãģ¤ãĤĤ":130698,"ĠвÑĭбоÑĢ":130699,"Ġcá»Ļng":130700,"ש×Ļ׳×ķ×Ļ":130701,"Ġchạy":130702,"Ġ×ij×¢×ľ×Ļ":130703,"اخبار":130704,"íķĺë©°":130705,"żÄħ":130706,"جاز":130707,"Ġ׳ר×IJ×Ķ":130708,"ศู":130709,"ศูà¸Ļ":130710,"ศูà¸Ļยà¹Į":130711,"×Ĵ×¢":130712,"Ġ×¢×ĵ×Ļ":130713,"Ġ×¢×ĵ×Ļ×Ļף":130714,"برا":130715,"ÑĨий":130716,"ĠÄIJá»ĵng":130717,"ÙĤاÙĨÙĪÙĨ":130718,"ĠÄijứng":130719,"ãģĹãģŁãĤĬ":130720,"Ġ×Ĺ×Ļ×Ļ":130721,"ĠëIJľ":130722,"ĠëIJľëĭ¤":130723,"ĠмеждÑĥ":130724,"à¸ŀวà¸ģà¹Ģà¸Ĥา":130725,"ĠBắc":130726,"ลำ":130727,"ë°±":130728,"ĠíĻķ":130729,"มาà¸ģม":130730,"มาà¸ģมาย":130731,"банк":130732,"à¸Ńาà¸ģาร":130733,"ĠhÃł":130734,"Ġ׾׳":130735,"à¸Ńà¸Ń":130736,"Ġë°Ķë¡ľ":130737,"лом":130738,"mática":130739,"ĠØŃد":130740,"ابت":130741,"à¸Ĺีà¹Īà¸Ļีà¹Ī":130742,"ĠcoÅĽ":130743,"ÙģÙĬدÙĬ":130744,"ÙģÙĬدÙĬÙĪ":130745,"ĠмеÑģÑĤо":130746,"Ġphút":130747,"มาà¸ģà¸ģวà¹Īา":130748,"×IJפ":130749,"بÙIJ":130750,"ĠPhú":130751,"ì±Ħ":130752,"ĠÙĪسÙĦÙħ":130753,"à¸Īีà¸Ļ":130754,"поÑĤÑĢеб":130755,"Ġ×Ĺ×ĵש×ķת":130756,"Ø´ÙĪ":130757,"Ġעצ×ŀ×ķ":130758,"ĠعÙħÙĦÙĬØ©":130759,"à¸Ħุà¸ĵà¸łà¸²à¸ŀ":130760,"ãģ¾ãģĻãģĮ":130761,"دعÙĪ":130762,"طرÙĤ":130763,"à¹Ħมà¹Īà¸ķà¹īà¸Ńà¸ĩ":130764,"ë²Ķ":130765,"ìĬ¹":130766,"ĠkÃŃch":130767,"ĠìĹĨëĬĶ":130768,"ĠÑĤам":130769,"ĠÙĨØŃÙĪ":130770,"ĠاÙĦÙĤاÙĨÙĪÙĨ":130771,"×Ĺ×ķ×Ŀ":130772,"Ġkız":130773,"Ġ×ĵ×Ļף":130774,"ĠвÑĢемени":130775,"ãģ£ãģŁãĤĬ":130776,"ĠØ´Ùĩر":130777,"ĠìĦľë¹ĦìĬ¤":130778,"עש×Ķ":130779,"Ġgiác":130780,"ĠاÙĦسÙĦاÙħ":130781,"Ġ×IJש":130782,"ĠполÑĥÑĩа":130783,"à¸Īัà¸Ķà¸ģาร":130784,"коÑĢ":130785,"Ġ×Ķ×ĺ×ķ×ij":130786,"รายà¸ģาร":130787,"주ìĿĺ":130788,"à¹ģà¸ķà¹Īละ":130789,"Ġê·¸ëŁ°ëį°":130790,"à¸Ĺีà¹Īà¹Ģà¸Ľà¹ĩà¸Ļ":130791,"Ġת×ķ×ļ":130792,"بÙĬاÙĨ":130793,"ÐĻ":130794,"oÅĽciÄħ":130795,"ÑĤок":130796,"ĠÃĶ":130797,"ĠÃĶng":130798,"à¹Ħมà¹Īà¹ĥà¸Ĭà¹Ī":130799,"ãģ¿ãģ¦":130800,"ÐŁÐ¾":130801,"ĠЧÑĤо":130802,"íĻ©":130803,"×ĺ×ij×¢":130804,"меÑĤÑĢ":130805,"Ġ×ij×ŀ×Ķ":130806,"Ġ×ij×ŀ×Ķ׾":130807,"Ġ×ij×ŀ×Ķ׾×ļ":130808,"ÑĩÑĮ":130809,"קש×Ķ":130810,"знак":130811,"знаком":130812,"ujÄĻ":130813,"×Ļצר":130814,"ĠاÙĦÙħÙĦÙĥ":130815,"ıyla":130816,"×IJ×ŀת":130817,"à¸Ľà¸´à¸Ķ":130818,"×IJ×Ĺ×ĵ":130819,"راد":130820,"ĠmáºŃt":130821,"ëĭ¤ëĬĶ":130822,"Ġlạnh":130823,"ש׾×ķש":130824,"ØŃدÙĬØ«":130825,"تز":130826,"å¹´ãģ®":130827,"ĠкваÑĢ":130828,"ĠкваÑĢÑĤиÑĢ":130829,"ä½ľãĤĬ":130830,"رÙĪب":130831,"ован":130832,"ĠТе":130833,"à¸Īำà¸ģ":130834,"à¸Īำà¸ģัà¸Ķ":130835,"باط":130836,"×Ĵת":130837,"ĠмаÑĪ":130838,"ĠмаÑĪин":130839,"×Ļצ×Ķ":130840,"ãģ»ãģ¨":130841,"ãģ»ãģ¨ãĤĵãģ©":130842,"ÃŃdo":130843,"ĠÑıзÑĭк":130844,"à¸ļิà¸Ļ":130845,"สà¸ĸาà¸Ļà¸Ĺีà¹Ī":130846,"ĠìĹ´":130847,"ãĤ¦ãĤ§":130848,"ĠcÃł":130849,"пан":130850,"åı£ãĤ³ãĥŁ":130851,"Ġرد":130852,"اÙĤت":130853,"ĠÙĥب":130854,"ĠÙĥبÙĬرة":130855,"ÑģÑĤал":130856,"ש×ŀ×Ĺ":130857,"posición":130858,"ĠÙħÙĦÙĬÙĪÙĨ":130859,"ĠìĿ´ìķ¼":130860,"ĠìĿ´ìķ¼ê¸°":130861,"Ġhút":130862,"ĠÅĽwiat":130863,"Ġë°©ë²ķ":130864,"ĠÑģвеÑĤ":130865,"Ġвидео":130866,"ĠاÙĦÙĨظاÙħ":130867,"Ġtrá»Ŀi":130868,"ĠëĮĢíķ´ìĦľ":130869,"ר×ŀת":130870,"تداÙĪÙĦ":130871,"×ķר×ĵ":130872,"ת×ŀ":130873,"ת×ŀ×ķ׳×ķת":130874,"Ġ×ŀף":130875,"Ġдва":130876,"Ġ×Ķק×ķ":130877,"æĹ¥ãģ«":130878,"Ġ×Ķ×Ĵ×Ļ×¢":130879,"à¹Ģà¸ŀิà¹Īมà¹Ģà¸ķิม":130880,"Ùħارس":130881,"Ġê²ĥìŀħëĭĪëĭ¤":130882,"ãģªãģĦãģ¨":130883,"Ġnhiá»ĩt":130884,"ëIJ©ëĭĪëĭ¤":130885,"Ġ×ij׳×ķש×IJ":130886,"Ġê°Ģìŀ¥":130887,"Ġvợ":130888,"ĠÄijóng":130889,"צ×Ļ׾×ķ×Ŀ":130890,"ê´Ģê³Ħ":130891,"ваÑı":130892,"×IJ×Ļ×ĸ":130893,"×IJ×Ļ×ĸ×Ķ":130894,"ĠÙĨظاÙħ":130895,"ÙħØŃاÙģظ":130896,"Ġtải":130897,"기ëıĦ":130898,"à¸Ľà¸±à¸Īà¸Īุ":130899,"à¸Ľà¸±à¸Īà¸Īุà¸ļัà¸Ļ":130900,"׼×ĵ×ķר":130901,"ĠìķĦìĿ´":130902,"׼׳×Ļס":130903,"à¹Ģà¸ķร":130904,"à¹Ģà¸ķรียม":130905,"Ġngoại":130906,"ĠدÙĪÙĦار":130907,"Ġrẻ":130908,"ĠkhÄĥn":130909,"عدد":130910,"شعب":130911,"czyÄĩ":130912,"ĠاÙĦÙĥر":130913,"ĠÑĩеловека":130914,"ĠÙĪØ¥ÙĨ":130915,"×IJ×ĺ":130916,"ĠthÆ¡":130917,"ĠاÙĦرÙĬاض":130918,"опÑĢедел":130919,"опÑĢеделен":130920,"×Ķ×ŀש×ļ":130921,"ĠÐĿово":130922,"зÑĭва":130923,"ĠاÙĦدÙĪÙĦÙĬ":130924,"ĠÄijáp":130925,"ĠкÑĢед":130926,"ĠкÑĢедиÑĤ":130927,"ового":130928,"Ġmôn":130929,"à¸Ľà¸£à¸°à¹Ĥย":130930,"à¸Ľà¸£à¸°à¹Ĥยà¸Ĭà¸Ļ":130931,"à¸Ľà¸£à¸°à¹Ĥยà¸Ĭà¸Ļà¹Į":130932,"ÑģÑĤе":130933,"ĠThá»ĭ":130934,"دÙĬØ©":130935,"×ŀצ×ķ":130936,"Ùģات":130937,"ק×ĵ×Ŀ":130938,"ìĿ´ëĿ¼ê³ł":130939,"ÙĪØ®":130940,"Ġ×Ĺ×ĸ":130941,"ĠÑĦоÑĤо":130942,"׾×Ļת":130943,"تÙİ":130944,"ÙĪبر":130945,"йÑĤи":130946,"ĠÃ¶ÄŁren":130947,"Ġ×Ķ×ĸ×ķ":130948,"Ġvá»įng":130949,"ÙĤÙĪØ©":130950,"ĠTây":130951,"ĠÐĿи":130952,"Ġש×ķ×ij":130953,"ãģ¨è¨ĢãĤıãĤĮ":130954,"ãģ©ãĤĵãģª":130955,"×Ĺצ×Ļ":130956,"ï½ľ":130957,"Ġ×ķ×Ķ×ķ×IJ":130958,"ä¸Ģãģ¤":130959,"ĠÑģÑĤоиÑĤ":130960,"niÄħ":130961,"×ĺר×Ļ":130962,"ĠдеÑĤей":130963,"нÑıÑĤÑĮ":130964,"ĠÑģделаÑĤÑĮ":130965,"Ġë§İìĿ´":130966,"ä½ķãģĭ":130967,"ãģĽãĤĭ":130968,"à¹Ħหม":130969,"à¸ķิà¸Ķà¸ķà¹Īà¸Ń":130970,"Ġ×ijת×Ĺ":130971,"Ġ×ijת×Ĺ×ķ×Ŀ":130972,"ìĻĦ":130973,"ì§ĢëĬĶ":130974,"ÑģÑĤаÑĤ":130975,"ÑıÑģн":130976,"üb":130977,"Ġthả":130978,"Ġ×ij×IJ×ŀת":130979,"Ġtuyến":130980,"×ĵ×Ļר×Ķ":130981,"Ġ×IJ×Ļש×Ļ":130982,"×ĸ׼ר":130983,"ãģ°ãģĭãĤĬ":130984,"Ġxét":130985,"׼×Ļ×ķ":130986,"׼×Ļ×ķ×ķף":130987,"diÄŁini":130988,"ĠاÙĦÙħÙĪضÙĪع":130989,"ĠháºŃu":130990,"à¸Īาà¸ģà¸ģาร":130991,"×ijס×Ļס":130992,"Ġ×ŀ×Ĵ×Ļ×¢":130993,"×ij×Ļ×¢":130994,"ĠÙĪجÙĩ":130995,"à¹ģà¸Ķà¸ĩ":130996,"à¸Ļาà¸ĩ":130997,"ĠÅŀa":130998,"ì¡´":130999,"ë¡Ģ":131000,"à¸ķะ":131001,"Ġ×Ķ×Ĺ×Ļ×Ļ×Ŀ":131002,"ÙģÙĬد":131003,"ãģ§ãģĻãģĭãĤī":131004,"ê·ľ":131005,"źni":131006,"ĠлÑİдей":131007,"Ġyüzde":131008,"ıyorum":131009,"ĠاÙĦبØŃر":131010,"eño":131011,"паÑĢ":131012,"ÙĬÙĤØ©":131013,"обÑĢ":131014,"ר×ķ×ļ":131015,"تÙĪÙĤع":131016,"ĠاÙĦØ´ÙĬØ®":131017,"åĪĿãĤģãģ¦":131018,"ĠÑĤелеÑĦ":131019,"ĠÑĤелеÑĦон":131020,"Ġthôi":131021,"Ġ×Ļ׼×ķ׾×Ļ×Ŀ":131022,"ĠÅŁirk":131023,"ĠÅŁirket":131024,"Ġìļ°ë¦¬ê°Ģ":131025,"ĠÄijông":131026,"Ġת×ķ×ĵ×Ķ":131027,"ÑģмоÑĤÑĢеÑĤÑĮ":131028,"ĠÙĦÙĩÙħ":131029,"Ġ׾׼":131030,"ĠNó":131031,"ĠØŃاÙĦØ©":131032,"ãģĦãģij":131033,"קר×ķ":131034,"azı":131035,"ãĤ³ãĥ¼":131036,"ĠÙĦÙĦت":131037,"sınız":131038,"ĠHải":131039,"기ìĪł":131040,"ยัà¸ĩà¹Ħมà¹Ī":131041,"ëĭ¤ê³ł":131042,"פ×Ĺ":131043,"Ġ׾×Ĵ×ij×Ļ":131044,"ĠعÙĨÙĩ":131045,"Ġказ":131046,"Ġказино":131047,"بÙĪر":131048,"ÑĦеÑĢ":131049,"Ġê°ĻìĿ´":131050,"تسجÙĬÙĦ":131051,"ĠاÙĦÙħرÙĥز":131052,"ĠThái":131053,"даÑĤÑĮ":131054,"×ŀ×Ļ×Ļ׾":131055,"ĠpaylaÅŁ":131056,"ãģ¤ãģ®":131057,"à¹Ģรืà¸Ń":131058,"nça":131059,"׳×ķ×Ĺ":131060,"Ġ×IJפ×Ļ׾×ķ":131061,"ãģ¨èĢĥãģĪ":131062,"ãģ¨ãģĹãģ¦ãģ¯":131063,"à¹Ģà¸Īà¸Ń":131064,"×ŀפ":131065,"ĠgiriÅŁ":131066,"лиÑĤ":131067,"ÑĤелÑı":131068,"Ñijн":131069,"æ°Ĺãģ«":131070,"Ġgó":131071,"Ġgóp":131072,"åĪĩãĤĬ":131073,"Ġ×Ķ×Ĺ×ĵש":131074,"жал":131075,"Ġ×ĵעת":131076,"éģķãģĨ":131077,"à¹Ģà¸Ĥà¹īาà¹Ħà¸Ľ":131078,"Ġסר×ĺ":131079,"eña":131080,"æĸ°ãģĹãģĦ":131081,"رÙİ":131082,"ĠÐIJÑĢ":131083,"Ġphản":131084,"à¸Īะà¹Ħà¸Ķà¹ī":131085,"Ġ×ijצ×ķר×Ķ":131086,"شاÙĩ":131087,"شاÙĩد":131088,"ÙĪرد":131089,"à¹Ģà¸Ļืà¹Īà¸Ńà¸ĩà¸Īาà¸ģ":131090,"илиÑģÑĮ":131091,"à¹ģละà¸ģาร":131092,"Ġ×Ķ×ĸ׼":131093,"Ġ×Ķ×ĸ׼×ķ×Ļ×ķת":131094,"eiÃŁ":131095,"ãĥ¨":131096,"ìĥĪ":131097,"ĠÃĩa":131098,"Ư":131099,"ש×Ĵ":131100,"ÙĬÙĨØ©":131101,"รà¹īà¸Ńà¸ĩ":131102,"ãĤµãĥ³":131103,"ÑĢоÑģÑģий":131104,"ÑĢоÑģÑģийÑģк":131105,"aÄŁa":131106,"ĠнаÑĩина":131107,"ĠصÙĦÙī":131108,"à¸Ĺุà¸ģà¸Ħà¸Ļ":131109,"íļĮìĤ¬":131110,"ĠлиÑĨ":131111,"Ø´ÙĬر":131112,"ĠØ´ÙĬØ¡":131113,"ÙĬÙĨا":131114,"Ġפ×Ĺ×ķת":131115,"Ġiçeris":131116,"Ġiçerisinde":131117,"ĠØ£ØŃÙħد":131118,"Ġżeby":131119,"ì´Ŀ":131120,"Ġпоказ":131121,"Ġименно":131122,"หà¸Ļัà¸ĩส":131123,"หà¸Ļัà¸ĩสืà¸Ń":131124,"ĠÑĤÑĢе":131125,"สัà¸ĩà¸Ħม":131126,"Ø¥ÙIJ":131127,"ãģĮå¿ħè¦ģ":131128,"ÙĬÙijØ©":131129,"פצ":131130,"íĭ°":131131,"ĠÙħجاÙĦ":131132,"׳פש":131133,"кан":131134,"×Ĺ×ķפ":131135,"×Ĺ×ķפש":131136,"ì²ĺëŁ¼":131137,"оваÑı":131138,"зов":131139,"Ġhạ":131140,"ĠdziÄĻki":131141,"×Ļר×ķ":131142,"Ġ׾×ŀצ":131143,"Ġ׾×ŀצ×ķ×IJ":131144,"×Ļ×ĵ×ķ":131145,"Ġsợ":131146,"Ġ׾×Ķ×Ĵ×Ļ×¢":131147,"ק×ij×¢":131148,"Ġchiá»ģu":131149,"ãĥŀãĤ¤":131150,"ĠdÃłng":131151,"à¹ģà¸Łà¸Ļ":131152,"Ġüye":131153,"×Ļ׳×Ĵ":131154,"à¹Ģรียà¸ģ":131155,"ç§ģãģĮ":131156,"thé":131157,"ĠÑĦилÑĮ":131158,"ĠÑĦилÑĮм":131159,"ĠNgÃły":131160,"Ġжен":131161,"ĠженÑīин":131162,"جÙĬد":131163,"nç":131164,"à¸Ľà¸£à¸²":131165,"×Ļ×ŀ×ķ":131166,"Ġná»ģn":131167,"×IJ×ķ׾×Ŀ":131168,"ĠвозможноÑģÑĤÑĮ":131169,"Ġëĭ¤ìĭľ":131170,"è¦ĭãģŁ":131171,"à¸ĸà¸Ļ":131172,"à¸ĸà¸Ļà¸Ļ":131173,"mızı":131174,"ĠÙħجÙħÙĪعة":131175,"cjÄħ":131176,"ĠÐłÐ¤":131177,"à¸ģำหà¸Ļ":131178,"à¸ģำหà¸Ļà¸Ķ":131179,"ĠìĹ¬ê¸°":131180,"landı":131181,"ниÑĨ":131182,"ÑģÑĤве":131183,"Ġ×ĵ×ijר×Ļ×Ŀ":131184,"ĠskÅĤad":131185,"ãĤĬãģ¾ãģĹãģŁ":131186,"ĠоÑĤкÑĢÑĭÑĤ":131187,"нÑıÑĤ":131188,"ĠÑģвоей":131189,"à¸Īิà¸ķ":131190,"ĠкаÑĩеÑģÑĤве":131191,"ĠettiÄŁi":131192,"ìĤ¬íķŃ":131193,"ĠاÙĦÙĬÙħÙĨ":131194,"иÑĩеÑģкий":131195,"ë¸Į":131196,"Ġ×ij×IJרץ":131197,"ĠاسÙħ":131198,"ĠизвеÑģÑĤ":131199,"rão":131200,"ĠattivitÃł":131201,"à¹Ģà¸Ľà¹ĩà¸Ļà¸ģาร":131202,"ĠاÙĦدÙĥت":131203,"ĠاÙĦدÙĥتÙĪر":131204,"ĠÙĪاØŃدة":131205,"ĠÑģÑĩеÑĤ":131206,"ĠпÑĢиÑĩ":131207,"ĠпÑĢиÑĩин":131208,"ĠÙĪزارة":131209,"Ġhuyá»ĩn":131210,"ĠÙĥتاب":131211,"à¹ģà¸Ļà¹Īà¸Ļ":131212,"à¹ģà¸Ļà¹Īà¸Ļà¸Ńà¸Ļ":131213,"Ġgünü":131214,"гÑĢÑĥз":131215,"ĠاÙĦخاص":131216,"Ġgörül":131217,"׾×ŀ×ĵ":131218,"ĠìłķëıĦ":131219,"×ķ×ij×Ļ׾":131220,"Ġ×ŀקצ×ķ×¢×Ļ":131221,"ĠоÑģобенно":131222,"à¸Ľà¸£à¸°à¸ģา":131223,"à¸Ľà¸£à¸°à¸ģาศ":131224,"acaģını":131225,"ë¶ģ":131226,"à¸łà¸¹à¸¡à¸´":131227,"ĠÑįлекÑĤ":131228,"ĠÑįлекÑĤÑĢо":131229,"Ġקש×Ķ":131230,"سÙĦØ·":131231,"à¸Ĭà¸Ļะ":131232,"×¢×Ļ׾":131233,"ĠЧе":131234,"à¹ģà¸Ļà¹Ī":131235,"lıģ":131236,"lıģın":131237,"Ġ×ŀ×¢×¨×Ľ×ª":131238,"好ãģįãģª":131239,"มาà¸ģà¸Ĥึà¹īà¸Ļ":131240,"×ŀ×¢×ijר":131241,"ĠاÙĦÙħغرب":131242,"ĠпеÑĢи":131243,"ĠпеÑĢиод":131244,"Ġnhạc":131245,"اÙĪÙĬ":131246,"ĠÙĪعÙĦÙī":131247,"أخذ":131248,"ĠCô":131249,"תר×ij×ķת":131250,"×Ĵ×Ķ":131251,"Ġktórej":131252,"×IJ×Ļת":131253,"×ij×ķ×IJ":131254,"делÑĮ":131255,"รีวิ":131256,"รีวิว":131257,"жÑĥ":131258,"Ġ×ij×Ĺ×ķ":131259,"еÑĪÑĮ":131260,"ĠØ£ÙĦÙģ":131261,"ĠاÙĦÙĪØ·ÙĨÙĬ":131262,"ĠاÙĦÙħÙĨØ·ÙĤØ©":131263,"nÄħÄĩ":131264,"Ġthiên":131265,"иÑĩеÑģкой":131266,"ĠاÙĦÙħÙĦ":131267,"ĠعÙħ":131268,"ספר":131269,"Ġnhóm":131270,"ÙĪصÙģ":131271,"ĠChúng":131272,"ĠرÙĤÙħ":131273,"ãģ¾ãģĹãģŁãģĮ":131274,"alité":131275,"ลม":131276,"ĠëĤ´ê°Ģ":131277,"׾ק×ķ×Ĺ":131278,"ĠSÆ¡n":131279,"posição":131280,"miÄĻ":131281,"Ġtránh":131282,"ĠÄIJá»Ļ":131283,"׼×Ĺ":131284,"ãģĤãģ£ãģ¦":131285,"à¸Ńยà¹Īา":131286,"Ġ×ŀ×Ĺ×Ļר":131287,"Ġ×Ķ×Ļת×Ķ":131288,"à¸Ľà¹Īา":131289,"à¸Ńืà¹Īà¸Ļà¹Ĩ":131290,"Ø´ÙĤ":131291,"×ł×¡×Ļ":131292,"림":131293,"ãģ¦ãģĹãģ¾ãģĨ":131294,"Ġ×ŀצ×ij":131295,"ãģ«åĩº":131296,"ÙħÙĪاطÙĨ":131297,"ยัà¸ĩมี":131298,"алÑĮнÑĭе":131299,"sanız":131300,"إسرائÙĬÙĦ":131301,"ĠvÃłi":131302,"ì¤Ħ":131303,"ãģ¨æĢĿãģ£ãģ¦":131304,"×Ļ×ķ׳×Ļ":131305,"çĶŁãģį":131306,"Ġsâu":131307,"ÑĩиÑģÑĤ":131308,"Ġlá»ħ":131309,"ĠGiá":131310,"à¸Ńà¸¸à¸Ľ":131311,"à¸Ńà¸¸à¸Ľà¸ģร":131312,"à¸Ńà¸¸à¸Ľà¸ģรà¸ĵà¹Į":131313,"Ġnhẹ":131314,"rö":131315,"ס×ĺ×Ļ":131316,"ãģķãĤĵãģĮ":131317,"Ġdầu":131318,"عÙİ":131319,"ترا":131320,"×Ĵ×ĵ׾":131321,"Ġtécnica":131322,"׼׳×Ļ×Ŀ":131323,"תקש":131324,"תקש×ķרת":131325,"Ġнего":131326,"était":131327,"Ġmá»ģm":131328,"ÑģеÑĤ":131329,"ĠnháºŃt":131330,"Ġ×ŀ×¢×ľ":131331,"Ġ×Ķ×¢×ij×ķ×ĵ":131332,"Ġ×Ķ×¢×ij×ķ×ĵ×Ķ":131333,"Ġ×Ĵ×Ļ׾":131334,"ãģ¯ãģªãģĦ":131335,"ائØŃ":131336,"ĠздеÑģÑĮ":131337,"×IJ×Ļ׳×ĺר":131338,"ÙħÙIJ":131339,"Ġ×Ļ×Ĺ×ĵ":131340,"راÙģ":131341,"ì²ĺ리":131342,"×ĵ×¢×ķת":131343,"ì¹ľ":131344,"ĠТо":131345,"ĠThế":131346,"충":131347,"Ġ׳׼×ķף":131348,"عÙĬØ´":131349,"низ":131350,"ĠجاÙĨب":131351,"×ŀקצ×ķ×¢":131352,"à¹Ĥà¸ĭ":131353,"ÑģÑĥÑĤ":131354,"ìĸ´ìļĶ":131355,"ãĤĴè¦ĭãģ¦":131356,"ارد":131357,"Ġaçıl":131358,"ĠاÙĦØŃÙĬاة":131359,"à¸ģà¹ĩà¹Ħà¸Ķà¹ī":131360,"ãģĿãĤĮãĤĴ":131361,"عضÙĪ":131362,"ĠгÑĢаж":131363,"ĠгÑĢаждан":131364,"à¸Īะà¸ķà¹īà¸Ńà¸ĩ":131365,"ĠìĿ´ëŁ¬":131366,"ĠìĿ´ëŁ¬íķľ":131367,"Ġtrách":131368,"ÙĨÙİ":131369,"Ġkısa":131370,"ÃĶ":131371,"ÑĪка":131372,"ãģ®äºº":131373,"ĠÐŁÐ¾Ñģ":131374,"ĠÐŁÐ¾Ñģле":131375,"ÑĥлÑĮ":131376,"ÙĪاجÙĩ":131377,"ÙĤرب":131378,"à¸Ľà¸ıิà¸ļัà¸ķิ":131379,"ê°Ļ":131380,"Ġ×ŀ׳":131381,"ĠÑģвои":131382,"براÙħج":131383,"ĠرÙĪ":131384,"пÑĢод":131385,"пÑĢодаж":131386,"ĠbyÅĤy":131387,"วัย":131388,"Ġgörün":131389,"ĠÃĪ":131390,"ÑİÑīим":131391,"ĠÑĤакой":131392,"ÙģÙĪر":131393,"ĠÙģعÙĦ":131394,"Ġбел":131395,"ëIJł":131396,"erÃŃa":131397,"ĠÑģвоÑİ":131398,"Ġlã":131399,"Ġlãnh":131400,"à¹Ģà¸ŀืà¹Īà¸Ńà¹ĥหà¹ī":131401,"ÙĤÙĨ":131402,"تطÙĪÙĬر":131403,"Ġsayı":131404,"ĠÑģейÑĩаÑģ":131405,"Ġ×IJ×Ĺרת":131406,"ק×ķפ×Ķ":131407,"ק×ķרס":131408,"ĠسÙħ":131409,"Ġ×ĺ×Ļפ×ķ׾":131410,"ìĿ´ëĿ¼ëĬĶ":131411,"دراسة":131412,"èµ·ãģĵ":131413,"×Ĺ×Ļ׳":131414,"×Ĺ×Ļ׳×ķ×ļ":131415,"×ĵק":131416,"Ġë§ŀ":131417,"Ġкоманд":131418,"ĠÐijо":131419,"ĠигÑĢÑĭ":131420,"à¸ļี":131421,"ĠØ£Ùİ":131422,"вен":131423,"ĠاÙĦجدÙĬد":131424,"ĠÙĦØ¥":131425,"Ġ×ķ×IJ׳×Ļ":131426,"Ġ×Ķס×Ļ":131427,"иÑĩеÑģкого":131428,"رÙĪØŃ":131429,"à¸ģารศึà¸ģษา":131430,"ĠTrÆ°á»Ŀng":131431,"игÑĢа":131432,"ılması":131433,"ĠмаÑģÑģ":131434,"ãģ¨ãģįãģ«":131435,"à¸Ĺีà¹Īà¸ľà¹Īาà¸Ļ":131436,"à¸Ĺีà¹Īà¸ľà¹Īาà¸Ļมา":131437,"ĠاÙĦسابÙĤ":131438,"Ġ×ŀ×¢×ĺ":131439,"ваÑĤÑĮ":131440,"mÃ¼ÅŁ":131441,"Ġ׾׼×ļ":131442,"Ġtá»ĭch":131443,"ÙģÙĩÙħ":131444,"تدرÙĬب":131445,"Ø´Ùĥ":131446,"Ġ×ij×ŀ×Ļ":131447,"Ġ×ij×ŀ×Ļ×ķ×Ĺ×ĵ":131448,"ÙĤطاع":131449,"ãģªãģĹ":131450,"×ķצ×Ļ×IJ":131451,"ĠÙĪسÙĬ":131452,"зÑĥ":131453,"Ġyat":131454,"Ġyatırım":131455,"ë§İ":131456,"Ġthắng":131457,"ãģĬ客":131458,"ãģĬ客æ§ĺ":131459,"ĠThiên":131460,"ãģ«å¯¾ãģĹãģ¦":131461,"ÑĢиÑģ":131462,"ÙĨتائ":131463,"ÙĨتائج":131464,"Ġ×ŀשר":131465,"Ġ×ŀשר×ĵ":131466,"ĠتعاÙĦ":131467,"ĠتعاÙĦÙī":131468,"ש׳×Ļ":131469,"ÙĩاÙħ":131470,"×IJ׳ש×Ļ×Ŀ":131471,"Ġżycia":131472,"ĠÑĢÑĥблей":131473,"ÙĬض":131474,"Ġkatıl":131475,"ĠÙħÙĪضÙĪع":131476,"Ġvardır":131477,"ĠÙħÙĨØ·ÙĤØ©":131478,"ĠTrần":131479,"ĠвеÑģ":131480,"üp":131481,"ÙħÙĪÙĨ":131482,"ÑĪли":131483,"Ġnóng":131484,"Ø®ÙĦÙģ":131485,"ĠСÑĤа":131486,"ĠдоÑĢ":131487,"ĠдоÑĢог":131488,"ĠwÅĤaÅĽnie":131489,"eÄŁin":131490,"Ġhiá»ĥm":131491,"ĠСам":131492,"ê»ĺìĦľ":131493,"ĠÑĦа":131494,"ãģ»ãģĨ":131495,"ãģ»ãģĨãģĮ":131496,"×ķפ×Ļ×¢":131497,"ê°Ī":131498,"دÙĪÙĦ":131499,"Ġthuê":131500,"Ġchá»Ĺ":131501,"Ġëĭ¹ìĭł":131502,"ãģijãĤĮ":131503,"ãģijãĤĮãģ©":131504,"ë³´íĺ¸":131505,"ãģķãĤĮãģ¦ãģĦãģ¾ãģĻ":131506,"Ġнадо":131507,"ĠìĤ¬ëŀĮëĵ¤":131508,"à¹Ģà¸Ĥà¸ķ":131509,"สมัย":131510,"zÅĤ":131511,"تÙĪر":131512,"Ġשת×Ļ":131513,"vê":131514,"Ġ×ijת×ķ×ļ":131515,"à¸Ĭัย":131516,"ãģĦãģ£ãģŁ":131517,"ìĿij":131518,"Ġtầ":131519,"Ġtầng":131520,"ש׼ר":131521,"Ġê¸Ģ":131522,"Ġ×Ķש׳×Ķ":131523,"ĠاÙĨÙĩ":131524,"ç«ĭãģ¡":131525,"rés":131526,"führen":131527,"رØŃÙħ":131528,"ê·¹":131529,"ĠâĢ«":131530,"Ġsuất":131531,"à¸Łà¸´":131532,"ÙĬÙĩا":131533,"ĠاÙĦاتØŃاد":131534,"Ġtuyá»ĥn":131535,"ãģ¾ãĤĭ":131536,"Ġmại":131537,"Ġngân":131538,"ãĤ°ãĥ©":131539,"欲ãģĹãģĦ":131540,"سار":131541,"ãĤĤãģ®ãģ§ãģĻ":131542,"кие":131543,"Ġseçim":131544,"åħ¥ãĤĬ":131545,"ãģªãģ©ãĤĴ":131546,"ÑĤÑĢи":131547,"ĠÑģпеÑĨ":131548,"Ġأد":131549,"Ġодно":131550,"ÑĪел":131551,"ãĥĩãĥ¼ãĤ¿":131552,"ãĤ·ãĤ¹ãĥĨ":131553,"ãĤ·ãĤ¹ãĥĨãĥł":131554,"è¡Įãģį":131555,"ãģ¨æĢĿãģ£ãģŁ":131556,"à¹Ģà¸ģิà¸Ķà¸Ĥึà¹īà¸Ļ":131557,"ĠÑĤож":131558,"ĠÑĤоже":131559,"Ġsạch":131560,"ĠÑģÑĢок":131561,"ĠклиенÑĤ":131562,"ĠÙħشرÙĪع":131563,"Ġaltında":131564,"Ġì·¨":131565,"ä¸Ńãģ®":131566,"ãģķãģĽãĤĭ":131567,"ãģĻãģ¹":131568,"ãģĻãģ¹ãģ¦":131569,"ê°ľë°ľ":131570,"ĠÄijêm":131571,"ãģªãģĦãģ®ãģ§":131572,"ì²ł":131573,"×¢×ij×ĵ":131574,"Ġdấu":131575,"à¸Ħà¸Ļà¸Ĺีà¹Ī":131576,"ĠCách":131577,"تعÙĦÙĬÙħ":131578,"Ġhại":131579,"ãĤ»ãĥķãĥ¬":131580,"ĠÙĨÙģسÙĩ":131581,"ĠíĨµíķ´":131582,"ÑĪло":131583,"ĠнапÑĢав":131584,"ĠнапÑĢавлен":131585,"ÑĢÑĥÑĩ":131586,"íĶĮ":131587,"Ġ×ijר×Ļ×IJ":131588,"ãģ®ãģ¿":131589,"ãģ«ãģĬãģĦãģ¦":131590,"×ij׳ק":131591,"ãĤ¨ãĥ³":131592,"Ø«ÙĦاث":131593,"Ġmỹ":131594,"ĠÑģайÑĤе":131595,"ĠемÑĥ":131596,"تغÙĬ":131597,"تغÙĬÙĬر":131598,"خصÙĪص":131599,"ÑĤели":131600,"Ġ×ķ׾׼ף":131601,"פע×Ŀ":131602,"ĠпоÑįÑĤомÑĥ":131603,"راÙĨ":131604,"иÑĤелей":131605,"пиÑģан":131606,"×¢×¥":131607,"ĠìĤ¬ìĹħ":131608,"Ùħز":131609,"جÙħÙĬع":131610,"ë©´ìĦľ":131611,"à¸ľà¸¥à¸´à¸ķà¸łà¸±":131612,"à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵ":131613,"à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵà¸ij":131614,"à¸ľà¸¥à¸´à¸ķà¸łà¸±à¸ĵà¸ijà¹Į":131615,"ĠпÑĢимеÑĢ":131616,"ãĤŃãĥ¼":131617,"lâ":131618,"ĠchÄĥm":131619,"缮ãģ®":131620,"ãģĦãģĭ":131621,"ãģ¨è¨ĢãģĨ":131622,"×ĸ×ķ×Ĵ":131623,"Ġ×ij×ĵ×Ļ":131624,"Ġ×ij×ĵ×Ļ×ķק":131625,"ãģĬåºĹ":131626,"à¸ķà¸Ńà¸Ļà¸Ļีà¹ī":131627,"Ġphá»iji":131628,"пÑĤ":131629,"สà¸Ļาม":131630,"Ø·ÙĪ":131631,"صاØŃ":131632,"صاØŃب":131633,"ĠDü":131634,"ĠDünya":131635,"Ġпока":131636,"пал":131637,"ĠÄijảo":131638,"ĠاÙĦÙģÙĪر":131639,"ĠاÙĦÙģÙĪرÙĥس":131640,"Ġmáu":131641,"кÑĢеп":131642,"ĠاÙĦساعة":131643,"ĠгоÑĢода":131644,"ÙģصÙĦ":131645,"айÑĤе":131646,"Ġдог":131647,"ĠдоговоÑĢ":131648,"Ġإذ":131649,"Ġ×ij׼׾׾":131650,"ÙĬتÙĩ":131651,"×Ĵ×ijר":131652,"Ġbirç":131653,"Ġbirçok":131654,"문íĻĶ":131655,"ãģĿãģĨãģª":131656,"راØŃ":131657,"ĠÙħرة":131658,"ĠденÑĮги":131659,"fä":131660,"à¸Ĥà¹īาว":131661,"ĠÑģовÑĢем":131662,"ĠÑģовÑĢеменн":131663,"׾×Ĺ×¥":131664,"èī¯ãģı":131665,"ĠÙģØ£":131666,"Ġ×ķ×ĸ×Ķ":131667,"Ġзани":131668,"Ġзанима":131669,"Ġê°Ģì§Ģê³ł":131670,"ĠhÆ¡i":131671,"ãģªãģ®ãģĭ":131672,"ãĥĨãĥ¬ãĥĵ":131673,"Ġר×ij×ķת":131674,"à¸ķี":131675,"Ġ×ij×©×ł×ª":131676,"ĠTại":131677,"ĠthuáºŃn":131678,"Ñģел":131679,"Ñijм":131680,"dziÄĩ":131681,"ĠÑģка":131682,"ĠÑģкаÑĩ":131683,"ĠÑģкаÑĩаÑĤÑĮ":131684,"×ķ×ŀ×ķ":131685,"гла":131686,"ĠминÑĥÑĤ":131687,"åĩºãģĻ":131688,"Ġ×Ĺ×Ļ×Ļ×ij":131689,"Ġת×Ĵ×ķ×ij×Ķ":131690,"à¸£à¸¹à¸Ľà¹ģà¸ļà¸ļ":131691,"ниÑĨа":131692,"ĠÄ°n":131693,"Ġأع":131694,"ĠضÙħÙĨ":131695,"ÙħثاÙĦ":131696,"ĠyaÅŁan":131697,"ĠìĹ°êµ¬":131698,"ĠLê":131699,"ש׾×Ĺ":131700,"ãģıãģªãĤĭ":131701,"ìĹĨìĿ´":131702,"ĠÑĤÑĢи":131703,"ĠÑĩаÑģÑĤо":131704,"ĠобÑĢаÑĤ":131705,"пло":131706,"دخ":131707,"دخÙĪÙĦ":131708,"سÙĩ":131709,"à¸Ńาà¸ģ":131710,"à¸Ńาà¸ģาศ":131711,"Ġ׼×ĸ×Ķ":131712,"Ġ×Ķעסק":131713,"ĠاÙĦØ£ÙĨ":131714,"å¹´ãģ«":131715,"עש×ķ":131716,"Ġשע×ķת":131717,"ĠmÃłn":131718,"×IJר×Ļ":131719,"sıyla":131720,"ÙģرÙĤ":131721,"ниÑħ":131722,"Ġتست":131723,"è¦ĭãģ¦":131724,"ØŃاÙĪÙĦ":131725,"×IJ×Ļ׼×ķת":131726,"ĠbaÅŁladı":131727,"stÄħ":131728,"stÄħpi":131729,"à¸Ĺีà¹Īà¹Ģรา":131730,"ÙĤرر":131731,"جاب":131732,"Ġ×ijר×ķר":131733,"à¹Ģà¸Ĥà¹īาà¹ĥà¸Ī":131734,"×ŀ×Ĺקר":131735,"alım":131736,"Ġס×Ļפ×ķר":131737,"ãģ§ãģĤãĤĮãģ°":131738,"Ġש×ŀ×ķר×ķת":131739,"Ġ×ķ×ŀ×Ķ":131740,"ãģĵãģĿ":131741,"idée":131742,"ä¸ĭãģķãģĦ":131743,"تÙĨاÙĪÙĦ":131744,"Ġลà¹īาà¸Ļ":131745,"Ġìļ°ë¦¬ëĬĶ":131746,"اÙĨا":131747,"ÑģÑĤой":131748,"боÑĤ":131749,"ĠyaÅŁam":131750,"köy":131751,"Ø¥ÙĦ":131752,"ÑĢÑĭв":131753,"기ìĹħ":131754,"Ġ×Ķ×ŀ×ĵ":131755,"Ġ×Ķ×ŀ×ĵ×Ļ׳×Ķ":131756,"دب":131757,"×¢×Ļ׳×Ļ":131758,"×ŀת×Ĺ":131759,"Ġפר×Ļ":131760,"ãĥĭãĥ¼":131761,"اÙħÙĬ":131762,"Ġnhằm":131763,"ãĤĮãģªãģĦ":131764,"تعرÙģ":131765,"Ġë§ĪìĿĮ":131766,"ìĵ°":131767,"Ġhấp":131768,"ר×Ĵ×Ļ׾":131769,"بÙİ":131770,"ĠrÄĥng":131771,"glÄħd":131772,"ĠÑģиÑģÑĤемÑĭ":131773,"Ġkhóa":131774,"ãģ§ãģĻãĤĪãģŃ":131775,"大ãģįãģı":131776,"기를":131777,"Ġkéo":131778,"ÙĪØ¡":131779,"جاÙħ":131780,"جاÙħع":131781,"Ġ×¢×Ļצ×ķ×ij":131782,"téri":131783,"Ġתש":131784,"Ġ×IJ×ij×Ļ":131785,"ĠChÆ°Æ¡ng":131786,"à¸ļริà¹Ģว":131787,"à¸ļริà¹Ģวà¸ĵ":131788,"ãģ¤ãģı":131789,"Ġ×Ĺ×ķ׾":131790,"עת×Ļ×ĵ":131791,"ש×Ļ×ŀ×Ķ":131792,"ëĤ¨":131793,"Ġש×IJ×Ļף":131794,"ĠÙĪاÙĦØ¥":131795,"ÑĦа":131796,"Ġkhám":131797,"Ġ×ĺ×ķ×ij×Ķ":131798,"ĠвÑĭÑģ":131799,"ĠвÑĭÑģоко":131800,"ĠاÙĦØŃدÙĬØ«":131801,"人ãĤĤ":131802,"dÃ¼ÄŁÃ¼":131803,"×Ļ×Ĺ×ķ×ĵ":131804,"تعÙĦÙĬ":131805,"تعÙĦÙĬÙĤ":131806,"lö":131807,"تØŃدÙĬد":131808,"него":131809,"ĠÑĥдоб":131810,"Ġ׾×ŀ×Ļ":131811,"Ġר×ķצ×Ļ×Ŀ":131812,"Ġجاء":131813,"Ġ×ij×ĸ×ŀף":131814,"à¸Ľà¸ģà¸ķิ":131815,"é«ĺãģı":131816,"à¸Ľà¸¥à¸²":131817,"Ġartık":131818,"Ġbugün":131819,"ק׳×Ļ":131820,"Ġkhoá":131821,"ĠÙħرÙĥز":131822,"ĠìŀIJ기":131823,"درجة":131824,"×ŀשר×ĵ":131825,"Ġgiấy":131826,"Ġchóng":131827,"קפ":131828,"ÙĬبة":131829,"ĠczÄĻsto":131830,"вали":131831,"Ùĥب":131832,"ìŁģ":131833,"สà¸ļาย":131834,"à¸Ľà¸£à¸°à¸Ĭาà¸Ĭà¸Ļ":131835,"×Ĵ×ķ×£":131836,"ëŁī":131837,"ãģ®ãģĵãģ¨":131838,"ลà¸Ń":131839,"Ġnghá»ī":131840,"åŃIJãģ©":131841,"åŃIJãģ©ãĤĤ":131842,"à¹Ħà¸Ķà¹īà¸Ńย":131843,"à¹Ħà¸Ķà¹īà¸Ńยà¹Īาà¸ĩ":131844,"×ĵ×¢":131845,"ĠاÙĦتÙī":131846,"ĠÑģовеÑĤ":131847,"ĠqualitÃł":131848,"åĩºãģĹ":131849,"ĠÑĢÑĥков":131850,"ĠÑĢÑĥковод":131851,"รายละà¹Ģà¸Ńียà¸Ķ":131852,"ãģªãģĭãģªãģĭ":131853,"기ê´Ģ":131854,"Ġ×Ĺ×ķש":131855,"Ġ×Ĺ×ķש×ij":131856,"лоÑĤ":131857,"à¸Ļะà¸Ħรัà¸ļ":131858,"ק×ij×ķצ×Ķ":131859,"Ġthái":131860,"Ġש×ij×Ķ":131861,"ĠÑĪкол":131862,"ĠÙĦÙĥÙĦ":131863,"à¹ĥà¸Ļà¸Ĭà¹Īวà¸ĩ":131864,"ĠÙħÙĥاÙĨ":131865,"ëķĮ":131866,"Ġcải":131867,"ĠChÃŃ":131868,"ÑĥÑĩа":131869,"ìĿµ":131870,"Ġxảy":131871,"à¸Ĭà¸Ļิà¸Ķ":131872,"ĠcáºŃu":131873,"кÑĢов":131874,"ssé":131875,"ĠÙĨÙĪع":131876,"ĠТа":131877,"Ø®Ùħس":131878,"פ×ķס×ĺ":131879,"Ġmắc":131880,"ĠÄijem":131881,"à¸ģารà¹ĥà¸Ĭà¹ī":131882,"ר×ķס":131883,"ĠÐĽÐµ":131884,"Ġthá»Ń":131885,"รà¹Īาà¸ĩà¸ģาย":131886,"üzü":131887,"æĹ¥æľ¬ãģ®":131888,"ê³¼ìłķ":131889,"ש×Ļ×IJ":131890,"ĠìŀĪê³ł":131891,"×ij×ķ׾":131892,"ìķħ":131893,"ĠÙĪاÙĦا":131894,"ĠÐĽÐ¸":131895,"ĠвÑģÑij":131896,"Ġużytkow":131897,"×Ĺ×ķ׾":131898,"رÙģض":131899,"Ġsonuç":131900,"ãģĦãģ¾ãģĽãĤĵ":131901,"ìĤ¬ìĹħ":131902,"ëĪĦ":131903,"ÑĤек":131904,"ĠudziaÅĤ":131905,"лез":131906,"Ġ×Ķ×Ļ×Ļת×Ļ":131907,"ãĤīãĤĮãģ¦":131908,"ÙħسؤÙĪÙĦ":131909,"رار":131910,"ÑĤан":131911,"ĠÄijÃło":131912,"Ġר×ķ×ij":131913,"Ġ×ijש×ij×Ļ׾":131914,"ä»ĬåĽŀãģ¯":131915,"ãĤ¸ãĥ¥":131916,"Ġ×¢×ijר":131917,"ãģĽãģ¦":131918,"полÑĮ":131919,"aklı":131920,"ĠkÃŃnh":131921,"دت":131922,"ложение":131923,"ĠاÙĦÙħص":131924,"ĠاÙĦÙħصرÙĬ":131925,"à¸Īริà¸ĩà¹Ĩ":131926,"ĠاÙĦشرÙĥØ©":131927,"ĠÄijá»ı":131928,"ãĥĽãĥĨ":131929,"ãĥĽãĥĨãĥ«":131930,"Ñįкон":131931,"Ñįконом":131932,"ĠÙĪعÙĨ":131933,"Ġ×ª×ł":131934,"Ġ×ª×ł×IJ×Ļ":131935,"ĠاÙĦدÙĪÙĦÙĬØ©":131936,"Ġì§ĢìĹŃ":131937,"ãģ§ãģĻãģĭ":131938,"ĠваÑĢи":131939,"ĠваÑĢианÑĤ":131940,"ĠاÙĦعرب":131941,"ела":131942,"ĠtÆ°á»Ľng":131943,"skÄħ":131944,"Ġmặc":131945,"สัà¸ģ":131946,"ãĥĵãĥ¼":131947,"Ġ×ij×Ĵ׾":131948,"Ġ×ij×Ĵ׾׾":131949,"ãĥķãĤ¡ãĥ³":131950,"×ij×Ļצ":131951,"×ij×Ļצ×ķ×¢":131952,"лиÑģÑĤ":131953,"à¸Łà¸¸":131954,"à¸Łà¸¸à¸ķ":131955,"à¸Łà¸¸à¸ķà¸ļà¸Ńล":131956,"à¸Ŀà¹Īาย":131957,"ìŀIJìĿĺ":131958,"ĠسÙĪÙģ":131959,"Ġש×Ķת":131960,"Ġ걸":131961,"×¢×ij×ķ×ĵ":131962,"ãģĻãĤĭãģĵãģ¨ãģĮ":131963,"ĠÑĩаÑģÑĤÑĮ":131964,"ãĤ¢ãĥ¡ãĥª":131965,"ãĤ¢ãĥ¡ãĥªãĤ«":131966,"Ġtakım":131967,"ĠsỼ":131968,"ĠsỼm":131969,"שר×Ķ":131970,"è¨ĢãģĨ":131971,"лан":131972,"커":131973,"׼׳×Ķ":131974,"ÙĪÙģÙĬ":131975,"íĹĪ":131976,"luÄŁu":131977,"ĠëĮĢíķ´":131978,"Ġ׾×ij×Ļת":131979,"Ġ×Ķר×IJש×ķ׳×Ķ":131980,"صÙħ":131981,"Ġsöyled":131982,"Ġsöyledi":131983,"à¸Ľà¸²à¸ģ":131984,"Ġardından":131985,"ãģĪãģŁ":131986,"à¸Ĺัà¹Īวà¹Ħà¸Ľ":131987,"Ġ׳×ķסף":131988,"болÑĮ":131989,"ãĤĵãģ§ãģĻãģijãģ©":131990,"ĠлиÑĪÑĮ":131991,"Ġ×ij×IJ×Ļ":131992,"ĠбÑĭÑģÑĤÑĢо":131993,"สัà¸Ļ":131994,"Ġ×ijפ׳×Ļ":131995,"леÑĩ":131996,"ĠاÙĦخبر":131997,"Ġsóc":131998,"Ġthú":131999,"ĠпÑıÑĤ":132000,"ãģĬé¡ĺ":132001,"ãģĬé¡ĺãģĦ":132002,"ÑĤин":132003,"ãģ«ãģ¤ãģĦãģ¦ãģ¯":132004,"פף":132005,"ĠдвÑĥÑħ":132006,"à¸įีà¹Ī":132007,"à¸įีà¹Īà¸Ľ":132008,"à¸įีà¹Īà¸Ľà¸¸":132009,"à¸įีà¹Īà¸Ľà¸¸à¹Īà¸Ļ":132010,"опеÑĢ":132011,"ĠاÙĦبشر":132012,"ĠاÙĦÙħاÙĦ":132013,"ıyoruz":132014,"تØŃÙħÙĬÙĦ":132015,"à¸ģะ":132016,"éĸĵãģ«":132017,"×Ĺ×ķש":132018,"ĠNguyên":132019,"ãģĦãģ¦ãģĦãĤĭ":132020,"дÑĥÑĪ":132021,"שפע":132022,"ÑĪÑĥ":132023,"å®ŁéļĽãģ«":132024,"ĠÑĢайон":132025,"ĠChá»ī":132026,"ÙĨصر":132027,"Ġìļ´":132028,"Ġìļ´ìĺģ":132029,"Ġ×Ķ×ĵ×Ļף":132030,"ØŃدد":132031,"رز":132032,"ĠاÙĦدÙħ":132033,"ĠPháp":132034,"ÑĤÑģÑı":132035,"è¦ĭãģĪ":132036,"Ġtiá»ĥu":132037,"Ġsá»Ńa":132038,"аÑİÑĤÑģÑı":132039,"ĠBá":132040,"Ġ×ķ׼׾":132041,"Ðĸ":132042,"ÑĪим":132043,"ìĿ´ëĬĶ":132044,"лев":132045,"dık":132046,"Ġprésente":132047,"Ġaraç":132048,"صدÙĤ":132049,"Ġпомог":132050,"ĠاÙĦشرÙĤ":132051,"ĠÙĪاÙĦØ°ÙĬ":132052,"رÙĬا":132053,"×ij׳×ķת":132054,"Ġngá»ĵi":132055,"ר×ķפ":132056,"ר×ķפ×IJ":132057,"Ġthấp":132058,"ãĤĦãģ¯":132059,"ãĤĦãģ¯ãĤĬ":132060,"ĠاÙĦجدÙĬدة":132061,"éĿŀ常ãģ«":132062,"ÙĬÙĦÙĬ":132063,"쪽":132064,"تعاÙħÙĦ":132065,"ãģłãģ¨æĢĿãģĦãģ¾ãģĻ":132066,"ÙħÙħ":132067,"иÑĤели":132068,"ãĤµãĤ¤ãĤº":132069,"ادات":132070,"ĠاÙĦÙħاÙĦÙĬØ©":132071,"Ùĥاتب":132072,"кли":132073,"веÑĢÑħ":132074,"ниÑĩ":132075,"Ġ×ľ×¢×ij×ķ×ĵ":132076,"׾×Ļ×Ķ":132077,"ØŃÙİ":132078,"ãĤ¤ãĥĻ":132079,"ãĤ¤ãĥĻãĥ³ãĥĪ":132080,"Ġת×Ĵ×ķ×ij×ķת":132081,"ÑĦон":132082,"ĠдÑĢÑĥгие":132083,"×IJ×ĸ×ķר":132084,"Ġperò":132085,"ìķŀ":132086,"åĢŁãĤĬ":132087,"רצ×Ļ":132088,"×IJ×ĸ":132089,"алÑĮнÑĭÑħ":132090,"Ġê²ĥìľ¼ë¡ľ":132091,"ĠпÑĢаво":132092,"ĠاÙĦأرض":132093,"à¹Ģà¸Ĺà¸Ħ":132094,"à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļ":132095,"à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥล":132096,"à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥลย":132097,"à¹Ģà¸Ĺà¸Ħà¹Ĥà¸Ļà¹Ĥลยี":132098,"צר×Ļ":132099,"ĠÐļÑĥ":132100,"ılma":132101,"決ãĤģ":132102,"اÙĪ":132103,"Ġ×ĵק×ķת":132104,"à¸Ħรู":132105,"ĠÙħستÙĪÙī":132106,"à¸Ľà¹īà¸Ńà¸ĩ":132107,"à¸Ľà¹īà¸Ńà¸ĩà¸ģัà¸Ļ":132108,"×ĵ×ķ×ŀ×Ķ":132109,"ĠÑģегоднÑı":132110,"سÙĪÙĤ":132111,"ר×Ĺ×ķ×ij":132112,"Ġإدارة":132113,"Ñħож":132114,"éģİãģİ":132115,"à¸Ħà¸Ń":132116,"нÑĥл":132117,"×ķ׼×Ķ":132118,"ÙĪاÙģÙĤ":132119,"׼׾׾":132120,"Ġ×Ķ×ĵ×ķ":132121,"ĠlÄ©nh":132122,"Ġkhảo":132123,"×IJ×ŀצע":132124,"머":132125,"Ġ׼×Ļצ":132126,"Ġ׼×Ļצ×ĵ":132127,"ĠдолжнÑĭ":132128,"หวัà¸ĩ":132129,"ãĥĩãĤ¶":132130,"ãĥĩãĤ¶ãĤ¤ãĥ³":132131,"Ġngá»Ŀ":132132,"ä¸Ńãģ«":132133,"à¸ģลัà¸ļมา":132134,"جÙħاÙĦ":132135,"à¸Ķัà¸ĩà¸ģลà¹Īาว":132136,"سÙĥÙĨ":132137,"سÙĨ":132138,"Ġözellikle":132139,"зеÑĢ":132140,"rzÄĻ":132141,"×ŀ×ķר×Ķ":132142,"Ġlạ":132143,"×ŀ×Ļ׳×Ļ":132144,"ר×Ļת":132145,"ãģĿãĤĮãģĮ":132146,"ãģĭãĤĮ":132147,"ĠÙĬÙħÙĥÙĨÙĥ":132148,"öffentlich":132149,"ган":132150,"ĠاÙĦØŃÙĦ":132151,"ĠmiÄĻdzy":132152,"ĠÑĩаÑģÑĤи":132153,"ujÄħcy":132154,"ĠbaÄŁlı":132155,"ĠiliÅŁki":132156,"Ùģاء":132157,"ãĥªãĥ³ãĤ°":132158,"Ġhãng":132159,"ĠконÑĤÑĢ":132160,"ĠконÑĤÑĢол":132161,"коп":132162,"ש×Ļ×¢":132163,"ש×Ļ×¢×ķר":132164,"ĠÐĴаÑĪ":132165,"Ġ×Ķתק":132166,"ÙħÙĨع":132167,"ĠpolÃŃtico":132168,"Ġголов":132169,"ĠØ¥ÙĬ":132170,"Ø¥ÙĨتاج":132171,"à¸ļิ":132172,"ĠговоÑĢ":132173,"ĠговоÑĢиÑĤ":132174,"Ġphá»ķ":132175,"ĠÑģемÑĮ":132176,"ãģ¯ãģĤãĤĬãģ¾ãģĽãĤĵ":132177,"ĠÙĪاست":132178,"×ŀשפ×ĺ":132179,"зем":132180,"×ŀ×ĵ×ijר":132181,"Ġíģ°":132182,"ĠìĿ´ë²Ī":132183,"ê°ĢëĬĶ":132184,"Ġì§ĢìĽIJ":132185,"ĠcaÅĤy":132186,"ĠgeliÅŁtir":132187,"Ñģкое":132188,"posé":132189,"Ġkhô":132190,"à¸ķิà¸Ķà¸ķาม":132191,"missão":132192,"Ġ׾×ŀר":132193,"Ġ׾×ŀר×ķת":132194,"Ġbó":132195,"à¸ķรวà¸Īสà¸Ńà¸ļ":132196,"Ġnghá»ģ":132197,"Ġбиз":132198,"ĠбизнеÑģ":132199,"ÑģÑĤеÑĢ":132200,"ÙĪÙİ":132201,"楽ãģĹãģ":132202,"楽ãģĹãģ¿":132203,"ãģĵãĤĮãģĭãĤī":132204,"wiÄħzan":132205,"สà¸Ńà¸Ļ":132206,"ÙħÙĪر":132207,"׳×ĵ׾":132208,"Ġ×Ķ×IJ×ĵ×Ŀ":132209,"Ġмолод":132210,"ØŃÙħا":132211,"ØŃÙħاÙĬØ©":132212,"ÑģÑĤÑĢан":132213,"Ġbuá»ķi":132214,"ת×Ļ×Ļ×Ŀ":132215,"abileceÄŁi":132216,"LÄ°":132217,"à¹Ģยà¸Ńะ":132218,"à¸Īร":132219,"سÙĥاÙĨ":132220,"à¸Ļัà¸Ķ":132221,"Ġmấy":132222,"ĠÐijа":132223,"sÅĤaw":132224,"ĠÙģÙĦا":132225,"ĠкоÑĤоÑĢой":132226,"ĠплоÑī":132227,"ĠплоÑīад":132228,"ãĤĤãģĤãĤĬ":132229,"szczÄĻ":132230,"×Ļפ×ķ":132231,"ש×ŀת":132232,"owaÅĤa":132233,"Ġnông":132234,"צ×ij×IJ":132235,"ĠìŀĪìĹĪ":132236,"ãģ¾ãģ¨":132237,"ãģ¾ãģ¨ãĤģ":132238,"ÙĤÙĪات":132239,"ãģ¿ãĤĵãģª":132240,"Ġ׼×ŀ×¢×ĺ":132241,"Ġxúc":132242,"ï¼Ĩ":132243,"rÄĻ":132244,"rÄĻcz":132245,"×ĵ×ŀ×Ļ":132246,"ĠtáºŃn":132247,"à¸Ķวà¸ĩ":132248,"ê²½ìłľ":132249,"пÑĥÑĤ":132250,"أربع":132251,"Ġ×ŀשת×ŀש":132252,"ãĤ¿ãĤ¤ãĥĹ":132253,"Ġìłľê°Ģ":132254,"Ġ׾׼ף":132255,"ĠобÑĢазом":132256,"ÙĬÙĥا":132257,"wÅĤ":132258,"wÅĤasn":132259,"ĠاÙĦÙĪØ·ÙĨÙĬØ©":132260,"بÙĬب":132261,"×ŀ׾×Ļ":132262,"кÑĢаÑĤ":132263,"기ìĹIJ":132264,"ÙĤاد":132265,"ĠÙĦدÙī":132266,"à¸Ħวามรูà¹ī":132267,"×ŀ×ĵ×Ļ׳×Ļ×ķת":132268,"겨":132269,"ĠíĺĦìŀ¬":132270,"שת×Ļ":132271,"мол":132272,"Ġmái":132273,"à¸ŀิม":132274,"à¸ŀิมà¸ŀ":132275,"à¸ŀิมà¸ŀà¹Į":132276,"หลวà¸ĩ":132277,"Ġxuyên":132278,"×Ĺסר":132279,"رÙĪÙĨ":132280,"ãģĿãģĨãģĦãģĨ":132281,"ãģĿãĤĮãģŀ":132282,"ãģĿãĤĮãģŀãĤĮ":132283,"Ġ׼ש×Ķ":132284,"ÐŁÑĢав":132285,"×ŀ×ijצע":132286,"عرب":132287,"Ġbüyü":132288,"פ×Ļת×ķ×Ĺ":132289,"à¸Īà¸ļ":132290,"ĠØ£Ùĥبر":132291,"שרת":132292,"×ŀ׼ש×Ļר":132293,"ĠÙĪÙħع":132294,"ãģ®ãģŁãĤģãģ«":132295,"à¸Ļัà¸ļ":132296,"ì°°":132297,"ãĥªãĥķãĤ©":132298,"ãĥªãĥķãĤ©ãĥ¼ãĥł":132299,"ĠcÆ°á»Ŀng":132300,"ĠìłĢíĿ¬":132301,"ÙħÙĨظÙħØ©":132302,"Ġhiçbir":132303,"ãģ§ãģ¯ãģĤãĤĬãģ¾ãģĽãĤĵ":132304,"รà¸Ńย":132305,"ëIJľëĭ¤":132306,"ãģĻãģIJãģ«":132307,"кла":132308,"Ġürünler":132309,"Ġkiá»ĥu":132310,"ĠëĤĺëĬĶ":132311,"ÑĤки":132312,"Ñģим":132313,"Ġchá»īnh":132314,"ãĤĤãģªãģĦ":132315,"ศรี":132316,"æĽ¿ãģĪ":132317,"taÅŁ":132318,"ĠبÙĥÙĦ":132319,"Ġ×ķ×Ļש":132320,"visão":132321,"ä¼Ŀ":132322,"ä¼ĿãģĪ":132323,"ÙĦد":132324,"׾×Ļ×ŀ":132325,"׾×Ļ×ŀ×ķ×ĵ":132326,"tória":132327,"دÙij":132328,"اÙħر":132329,"Ġê·¸ëłĩê²Į":132330,"ĠmateriaÅĤ":132331,"à¸Ĺรา":132332,"à¸Ĺราà¸ļ":132333,"ãģ®æĸ¹ãģĮ":132334,"ãģ¦ãģįãģŁ":132335,"ضغ":132336,"ضغط":132337,"ĠÙĬعÙĨÙĬ":132338,"ело":132339,"×IJ×Ķ×ij×Ķ":132340,"×¢×ŀ":132341,"ÅŁÄ±k":132342,"ìŀIJëĬĶ":132343,"ãĤ¿ãĥ³":132344,"ĠbáºŃt":132345,"×ŀשפ×Ĺ×Ķ":132346,"кÑĢи":132347,"бли":132348,"สัà¸ķ":132349,"สัà¸ķวà¹Į":132350,"ĠسÙĨÙĪات":132351,"ĠPhÆ°Æ¡ng":132352,"ãģ¦ãģĹãģ¾ãģ£ãģŁ":132353,"ãģªãģľ":132354,"Ġ×ij×IJ×ķ":132355,"Ġcán":132356,"سجÙĦ":132357,"Ġlẽ":132358,"ãĤ±ãĥ¼ãĤ¹":132359,"Ġק×Ļ×ij׾":132360,"à¸ļà¸Ĺà¸Ħวาม":132361,"Ġ×ķ׼ף":132362,"ĠпÑĢедÑģÑĤавлен":132363,"Ġná»iji":132364,"Ġcomentário":132365,"ением":132366,"Ġtá»ı":132367,"lÃł":132368,"Ġש×Ķ×Ļ×Ķ":132369,"Ñģлав":132370,"ĠاÙĦÙĪÙĦا":132371,"ĠاÙĦÙĪÙĦاÙĬات":132372,"ÙĦجÙĨØ©":132373,"ק×ķר×IJ":132374,"бÑĭÑĤ":132375,"Ġì¦":132376,"Ġì¦ī":132377,"ãģ§ãģĻãģĹ":132378,"หรืà¸Ńà¹Ħมà¹Ī":132379,"заÑīиÑĤ":132380,"ÙģÙĦسطÙĬÙĨ":132381,"Ġmiá»ħn":132382,"à¹Ģยà¹ĩà¸Ļ":132383,"ĠçalÄ±ÅŁan":132384,"×Ļ×Ĵ×Ķ":132385,"ĠEÄŁ":132386,"ĠEÄŁitim":132387,"ãĥĥãĤ·ãĥ¥":132388,"ĠопÑĭ":132389,"ĠопÑĭÑĤ":132390,"رغ":132391,"رغب":132392,"ĠÑģвоиÑħ":132393,"à¸Ľà¸£à¸°à¸ķ":132394,"à¸Ľà¸£à¸°à¸ķู":132395,"Ġ×ŀ×IJ×ĵ":132396,"׼×ķ׳×Ļ×Ŀ":132397,"à¸Ļี":132398,"ĠвÑĭÑħод":132399,"ãģ®ä¸Ńãģ«":132400,"פ׾×IJ":132401,"ĠÙĪÙĦÙĬس":132402,"פ×ķרס":132403,"פ×ķרס×Ŀ":132404,"ÙħسÙĦÙħ":132405,"Ġngôi":132406,"×ĵ×ŀ×ķת":132407,"ãĤĴ使ãģ£ãģ¦":132408,"ĠпомоÑīÑĮÑİ":132409,"أسر":132410,"блок":132411,"ÙĤÙĩ":132412,"ãģĹãģ¾ãģĦ":132413,"ãģ¨ãģĹãģŁ":132414,"ĠпеÑģ":132415,"ãĥīãĥ«":132416,"×Ĺ×Ŀ":132417,"ãģĹãģªãģĮãĤī":132418,"ĠÐŁÑĢед":132419,"ãĥģãĤ§ãĥĥãĤ¯":132420,"å¼·ãģĦ":132421,"ש×Ļר×ķת":132422,"даеÑĤ":132423,"×Ļ×ij×ķ":132424,"Ġgenç":132425,"илаÑģ":132426,"илаÑģÑĮ":132427,"ĠبÙĦد":132428,"æĤª":132429,"æĤªãģĦ":132430,"Ġ×ŀשת":132431,"æ§ĺãĢħ":132432,"æ§ĺãĢħãģª":132433,"à¸ĺรรมà¸Ĭาà¸ķิ":132434,"ĠÙĥاÙħÙĦ":132435,"ĠاÙĦسÙħ":132436,"×ij×ĺ×Ļ×Ĺ":132437,"cá":132438,"gência":132439,"ãĤ¹ãĤ¿ãĥ¼":132440,"à¸Ĺำà¸ģาร":132441,"×Ļ×ľ×ª":132442,"Ġ×Ļ×ķצ×IJ":132443,"wój":132444,"à¸ļุà¸Ħ":132445,"à¸ļุà¸Ħà¸Ħล":132446,"عتÙħ":132447,"عتÙħد":132448,"ãģĿãĤĮãģ«":132449,"ĠاÙĦتارÙĬØ®":132450,"ÙĤراء":132451,"Ġyönetim":132452,"קשר":132453,"ĠÑģпоÑĢÑĤ":132454,"Ġר×IJש×ķף":132455,"Ġseñal":132456,"Ġchắn":132457,"çĦ¡ãģĦ":132458,"ĠдоÑģÑĤаÑĤ":132459,"ĠдоÑģÑĤаÑĤоÑĩно":132460,"Ġágua":132461,"à¸ģรà¸ĵ":132462,"à¸ģรà¸ĵี":132463,"Ġ×ŀש×ķ":132464,"Ġtrải":132465,"ë²Į":132466,"ujÄħcych":132467,"Ùģرد":132468,"à¹ĥà¸ģล":132469,"à¹ĥà¸ģลà¹ī":132470,"ãĤĭãģ®ãģ¯":132471,"ר×ķ×ķ×Ĺ":132472,"ÙĨÙĥ":132473,"ĠاÙĦÙĨÙĤ":132474,"ãģ®ãģ§ãģĹãĤĩãģĨ":132475,"ãģ®ãģ§ãģĹãĤĩãģĨãģĭ":132476,"ÙħعرÙģ":132477,"ÙħعرÙģØ©":132478,"ÑĥÑīе":132479,"Ġ×ij×¢×Ļקר":132480,"تصÙĦ":132481,"Ġ×Ķ×IJר":132482,"Ġ×Ķ×IJרץ":132483,"ĠÅŀi":132484,"à¸Ĥาà¸Ķ":132485,"íŀĺ":132486,"ãģªãĤĵãģ¨":132487,"ĠìĤ¬ëŀij":132488,"lÃ¼ÄŁÃ¼":132489,"باء":132490,"ĠاÙĦآخر":132491,"ĠfamÃŃlia":132492,"ĠTháng":132493,"ÑīениÑı":132494,"ãĤ¯ãĥŃ":132495,"ĠThứ":132496,"æĽ¸ãģį":132497,"енной":132498,"ìŀ¡":132499,"благ":132500,"благо":132501,"пов":132502,"à¹ģว":132503,"à¸ĩà¸Ħà¹Į":132504,"à¸Ńัà¸Ļà¸Ķัà¸ļ":132505,"ãģĤãģĴ":132506,"รà¹īาย":132507,"ünün":132508,"Ġ×Ļ׼×ķ׾×Ķ":132509,"зон":132510,"ĠÐľÐ¸":132511,"маÑĤеÑĢиал":132512,"Ġë³´ë©´":132513,"ØŃÙģظ":132514,"êÌģ":132515,"ãģ«ãģĻãĤĭ":132516,"Ġת×IJ":132517,"Ġ×Ķס×ķ":132518,"ĠÑģÑĤоÑĢ":132519,"ĠÑģÑĤоÑĢон":132520,"ãĥĪãĥĥãĥĹ":132521,"ÅĤoÅĽÄĩ":132522,"ëħ¼":132523,"ëĵĿ":132524,"ĠÙĪاÙĦع":132525,"ì¶Ķ":132526,"Ġ×Ļצ×IJ":132527,"ĠÑĢаздел":132528,"алÑĮнаÑı":132529,"×IJ׳ש×Ļ":132530,"spoÅĤ":132531,"spoÅĤec":132532,"spoÅĤeczn":132533,"إعÙĦ":132534,"إعÙĦاÙĨ":132535,"ÙĤÙĪÙī":132536,"íķĺë©´ìĦľ":132537,"تطÙĪر":132538,"Ġsiêu":132539,"Ỽt":132540,"дви":132541,"движ":132542,"Ġquần":132543,"kıl":132544,"ĠпÑĢизна":132545,"ĠHã":132546,"ĠHãy":132547,"ĠباÙĦت":132548,"manın":132549,"ãĤ«ãĥ«":132550,"Ġká»·":132551,"ק׾×Ļ":132552,"ëIJĺì§Ģ":132553,"تعÙĦÙħ":132554,"ìĭľìĦ¤":132555,"ìĭ¶":132556,"íĺ¼":132557,"ÙĥÙĬÙģ":132558,"売ãĤĬ":132559,"วิà¸Ĭา":132560,"бал":132561,"ĠØ£ØŃ":132562,"Ġдолжен":132563,"ราà¸ĩ":132564,"ราà¸ĩวั":132565,"ราà¸ĩวัล":132566,"Ùħاء":132567,"جار":132568,"Åļ":132569,"Ġ×ŀ×IJ×ĸ":132570,"ר×ŀ×Ķ":132571,"ãģĭãĤĤãģĹãĤĮãģªãģĦ":132572,"étude":132573,"czÄħc":132574,"Ġgór":132575,"×ł×¡×Ķ":132576,"ÙħÙĬد":132577,"ĠÐŁÐµÑĢе":132578,"أخر":132579,"ãģĿãģ®å¾Į":132580,"à¹Ģà¸Ķียวà¸ģัà¸Ļ":132581,"×ŀ×Ĵ×ķ":132582,"×ŀ×Ĵ×ķ×ķף":132583,"дов":132584,"masına":132585,"×¢×ł×Ķ":132586,"ãĤ±ãĥĥãĥĪ":132587,"סע":132588,"סע×Ļ×£":132589,"ĠTÆ°":132590,"Ġtóc":132591,"íĻľëıĻ":132592,"ĠÐŀд":132593,"ĠÐŀднако":132594,"Ġdolayı":132595,"ؤÙĥد":132596,"ê³Ħíļį":132597,"׾ר":132598,"веÑĩ":132599,"Ġkhợi":132600,"Ġthủy":132601,"×ĵף":132602,"รà¸ģ":132603,"à¸ļัà¸ķร":132604,"à¹Ģà¸ģà¹Īา":132605,"ĠاÙĦثاÙĦ":132606,"ĠاÙĦثاÙĦØ«":132607,"Ġpodrá":132608,"ער×Ļ":132609,"ÙĨجاØŃ":132610,"Ġkhắc":132611,"측":132612,"Ä°M":132613,"ãĤ»ãĥĥãĥĪ":132614,"żenia":132615,"Ġ׾×Ĺ×ijר":132616,"erÃł":132617,"ì´Ī":132618,"Ġküç":132619,"Ġküçük":132620,"اتÙĩÙħ":132621,"à¸ĭà¹Į":132622,"ÙħشارÙĥØ©":132623,"ĠاÙĦبط":132624,"Ġdây":132625,"еннÑĭм":132626,"à¸Ĺีà¹Īà¹Ħมà¹Ī":132627,"ÙĤÙİ":132628,"Ġvượt":132629,"Ġtrì":132630,"ĠwpÅĤyw":132631,"AÅŀ":132632,"зо":132633,"ĠاÙĦسÙĬد":132634,"à¸Ĺะà¹Ģล":132635,"ĠÑģодеÑĢжа":132636,"عطÙĬ":132637,"ĠاÙĦعÙĨ":132638,"èĢħãģĮ":132639,"à¹Ģหà¸Ļ":132640,"à¹Ģหà¸Ļืà¸Ń":132641,"ĠbÃŃ":132642,"Ġüzerinden":132643,"ĠVÅ©":132644,"Ġnuôi":132645,"ÙĨÙħ":132646,"алÑĮного":132647,"×¢×Ļף":132648,"ØŃضر":132649,"ĠоÑĤдел":132650,"ëªĩ":132651,"ìķ¡":132652,"ĠÙĦدÙĬÙĩ":132653,"ìĻľ":132654,"Ġsektör":132655,"Ġвозможно":132656,"ĠÐĶж":132657,"Ġhô":132658,"äºĭãģĮ":132659,"иÑĢование":132660,"алÑĮной":132661,"Ġ미êµŃ":132662,"رØŃÙĦ":132663,"ĠÑįкÑģ":132664,"пÑĢавлÑı":132665,"Ġnhá»Ŀ":132666,"ĠÄijẩ":132667,"ĠÄijẩy":132668,"ÙģÙĥر":132669,"ĠÙĪأضاÙģ":132670,"ãĥIJãĤ¹":132671,"ת×ķ׼׳×Ļת":132672,"ÑĤелей":132673,"ĠØ¥ÙĦÙĬÙĩ":132674,"ãģ¨è¨Ģãģ£ãģ¦":132675,"Ġдве":132676,"Ġchấp":132677,"ĠLö":132678,"à¸Ħลิ":132679,"à¸Ħà¸¥à¸´à¸Ľ":132680,"ĠسÙĪر":132681,"ĠسÙĪرÙĬا":132682,"×ŀ×Ĺ×ķ":132683,"stä":132684,"доб":132685,"Ġniá»ĩm":132686,"ãģ®å¤§":132687,"פר×ķ×Ļק":132688,"פר×ķ×Ļק×ĺ":132689,"ĠChâu":132690,"Ġ×ŀ×Ķ×Ŀ":132691,"Ñģким":132692,"ĠполÑĥÑĩиÑĤÑĮ":132693,"ÙĬÙĪÙħ":132694,"Ø«ÙĪر":132695,"פ×ķ׾×Ļ×ĺ":132696,"פ×ķ׾×Ļ×ĺ×Ļ":132697,"ĠмеÑģÑıÑĨ":132698,"åħ¨ãģ¦":132699,"ĠاÙĦÙħجÙĦس":132700,"ĠاÙĦتاÙĦÙĬ":132701,"Ġ×Ĺר":132702,"åIJijãģij":132703,"׼×ŀ×Ķ":132704,"бед":132705,"أعض":132706,"أعضاء":132707,"ÙĪÙĦد":132708,"วà¹Īาà¸Īะ":132709,"Ġbánh":132710,"à¸Ļิย":132711,"à¸Ļิยม":132712,"à¸Ľà¸£à¸°à¸ģัà¸Ļ":132713,"ÑģÑĤавиÑĤÑĮ":132714,"à¸ŀà¸Ļัà¸Ļ":132715,"ĠÑįÑĦÑĦ":132716,"ĠÑįÑĦÑĦекÑĤив":132717,"ĠавÑĤоÑĢ":132718,"ĠÄIJÄĥng":132719,"ĠthÆ°á»Łng":132720,"ãĤĴæĦŁãģĺ":132721,"à¸ģัà¸ļà¸ģาร":132722,"å¾Įãģ«":132723,"ĠyaÄŁ":132724,"ستاÙĨ":132725,"Ġliá»ģn":132726,"ãģĦãģ¾":132727,"iêu":132728,"à¹Ĥà¸Ķà¸Ļ":132729,"ĠÙĦØ°ÙĦÙĥ":132730,"à¹Ĥรà¸ĩà¹Ģรียà¸Ļ":132731,"צ×Ļ×Ĵ":132732,"ĠاÙĦÙħعÙĦÙĪÙħات":132733,"ç§ģãģŁãģ¡":132734,"à¸Ĺีà¹Īà¸Ħุà¸ĵ":132735,"ãģ«ãģªãģ£ãģ¦ãģĦãĤĭ":132736,"×ŀ×ĵ×Ļ׳×Ķ":132737,"×¡×Ľ×Ŀ":132738,"Ġвне":132739,"à¸ŀà¸Ļัà¸ģà¸ĩาà¸Ļ":132740,"ÑĢей":132741,"à¹Ģà¸Īà¹īาหà¸Ļà¹īาà¸Ĺีà¹Ī":132742,"ĠHiá»ĩn":132743,"Ġmédico":132744,"ĠتØŃÙĤÙĬÙĤ":132745,"ÑĮÑĤе":132746,"miÅŁti":132747,"ÙĤÙĬادة":132748,"ãĤıãģĭãĤĬ":132749,"มาà¸Īาà¸ģ":132750,"ëħĢ":132751,"ãģ«éĸ¢ãģĻãĤĭ":132752,"×IJר×Ĵ×ķף":132753,"mètre":132754,"Ġעצ×ŀ×Ļ":132755,"ĠChúa":132756,"รูà¹īà¸Ī":132757,"รูà¹īà¸Īัà¸ģ":132758,"ì£Ħ":132759,"ëĭµ":132760,"à¹ģà¸Ĺà¹ī":132761,"Ġgeçen":132762,"Ġlança":132763,"ĠاÙĦبØŃØ«":132764,"×ĵ×ŀ×ķ":132765,"ãģ¯ãģĺ":132766,"ãģ¯ãģĺãĤģ":132767,"ĠdönÃ¼ÅŁ":132768,"è¿ijãģı":132769,"à¹Ģสม":132770,"à¹Ģสมà¸Ń":132771,"ëĿ½":132772,"Ġüç":132773,"á»ŀ":132774,"ÑĪаÑı":132775,"à¸Ĺร":132776,"ØŃÙĤÙĬÙĤØ©":132777,"à¸Ĥà¸Ńà¸ĩà¸ģาร":132778,"Ġ무ìĹĩ":132779,"Ġ×Ķ׼ר":132780,"ĠاÙĦصÙĬÙĨ":132781,"ĠлÑİди":132782,"à¸ķาย":132783,"بÙĪÙĦ":132784,"Ġviêm":132785,"Ġthiá»ĩu":132786,"à¸ģà¸Ķ":132787,"Ġ׾×ĵ×ijר":132788,"פ׳×Ķ":132789,"×IJר×ij×¢":132790,"سÙī":132791,"ĠاÙĦسÙĬاس":132792,"ĠاÙĦسÙĬاسÙĬØ©":132793,"ydı":132794,"ÙĪØŃدة":132795,"ĠдеÑıÑĤелÑĮноÑģÑĤи":132796,"Ġ×ķ×Ķ×ŀ":132797,"пеÑĩ":132798,"пеÑĩаÑĤ":132799,"иÑĢованиÑı":132800,"ĠÑģог":132801,"ĠÑģоглаÑģ":132802,"Ġ׼×ĵ":132803,"Ġ׼×ĵ×IJ×Ļ":132804,"ĠиÑģполÑĮзоваÑĤÑĮ":132805,"ספ×ķר×ĺ":132806,"Ġilçe":132807,"expérience":132808,"ĠThá»Ŀi":132809,"Ä°K":132810,"à¹Ħà¸Łà¸Łà¹īา":132811,"ëĵ¤ìĹIJê²Į":132812,"à¸Ľà¸£à¸°à¹Ģà¸ł":132813,"à¸Ľà¸£à¸°à¹Ģà¸łà¸Ĺ":132814,"Ġmümk":132815,"Ġmümkün":132816,"Ġ×IJ×ķ×ª×ł×ķ":132817,"ìĦ±ìĿĦ":132818,"ĠìĿ´ìľł":132819,"زÙĬارة":132820,"Ġoldukça":132821,"rób":132822,"ĠØ£ÙĨا":132823,"Ġ×Ķ×ij×Ļ":132824,"Ñģен":132825,"×¢×Ļקר":132826,"×Ļ×ĵ×ķ×¢":132827,"dzÄħ":132828,"ÙħعÙĦÙĪÙħات":132829,"شاب":132830,"Ġparça":132831,"à¸Ļะà¸Ħะ":132832,"باس":132833,"ĠÑĤоÑĢг":132834,"ĠÑĤоÑĢгов":132835,"Ġ×Ĺ×ĵר":132836,"׼ר×ĺ":132837,"׼ר×ĺ×Ļס":132838,"ĠAyrıca":132839,"ệ":132840,"ìľ¨":132841,"ĠÑĤакие":132842,"Ġ×ŀצ×ķ×Ļ":132843,"ãĥ©ãĥ³ãĤŃãĥ³ãĤ°":132844,"ש×Ļ×ķ×ķק":132845,"åīįãģ®":132846,"ĠBảo":132847,"ÑīÑĥ":132848,"æĹ©ãģı":132849,"ĠPhòng":132850,"à¸ŀระราà¸Ĭ":132851,"פ×Ĺ×ķת":132852,"Ġгл":132853,"Ġглаз":132854,"à¸Ĺà¹Īา":132855,"Ġdạy":132856,"ÑĢоÑģÑĤ":132857,"à¹Ĥà¸Ķยà¹Ģà¸īà¸ŀาะ":132858,"ĠquáºŃn":132859,"Ġ×Ĺ×ijר×ķת":132860,"même":132861,"mÄ±ÅŁtı":132862,"ĠاÙĦتداÙĪÙĦ":132863,"Ġnạn":132864,"Ġ×Ķ×ĵ×Ļ":132865,"ĠاÙĦطرÙĬÙĤ":132866,"×Ĵ×ķת":132867,"Ġ×Ķ×ĵר×ļ":132868,"ujÄħce":132869,"Ġchữ":132870,"ãĤĤãģ®ãģ®":132871,"ë°Ľ":132872,"ãģķãĤĵãģ¯":132873,"Ġyardım":132874,"ĠاÙĦعÙħ":132875,"Ġì§Ħíĸī":132876,"Ġ×Ļ×Ĺ":132877,"Ġ×Ļ×Ĺס×Ļ":132878,"ĠاÙĦÙħدÙĬÙĨØ©":132879,"Ġcú":132880,"à¸ģีฬ":132881,"à¸ģีฬา":132882,"Ġniên":132883,"misión":132884,"׳×Ļס×Ļ":132885,"׳×Ļס×Ļ×ķף":132886,"ĠвозÑĢаÑģÑĤ":132887,"Ġ×¢×ķש×Ķ":132888,"ĠÙħدÙĬر":132889,"ÑıÑģÑĮ":132890,"ØŃجÙħ":132891,"íĻĺê²½":132892,"ĠاÙĦأخرÙī":132893,"uÃŁer":132894,"ĠاÙĦعاÙĦÙħÙĬØ©":132895,"ĠNgá»įc":132896,"êµIJíļĮ":132897,"ä¸Ĭãģ§":132898,"×Ļ×Ķ×ķ×ĵ":132899,"×Ļ×Ķ×ķ×ĵ×Ļ×Ŀ":132900,"Ùħساعدة":132901,"ĠжизнÑĮ":132902,"ĠпоÑĤомÑĥ":132903,"ĠاÙĦÙħÙħÙĦ":132904,"ĠاÙĦÙħÙħÙĦÙĥØ©":132905,"ĠGör":132906,"رÙIJ":132907,"×ŀק×ķ×ŀ×ķת":132908,"åĩºæĿ¥ãĤĭ":132909,"ÑĦÑĤ":132910,"ĠìĿ´ìłľ":132911,"ĠÑĢем":132912,"ĠÑĢемонÑĤ":132913,"ת×ķ×ļ":132914,"æĻĤãģ¯":132915,"ãĤīãĤĮãģªãģĦ":132916,"altı":132917,"家ãģ®":132918,"ĠاÙĦإعÙĦاÙħ":132919,"리ëĬĶ":132920,"ãģĭãĤīãģ¯":132921,"ĠHạ":132922,"ãģĤãģ®":132923,"×ĵ×Ļ×ķף":132924,"رÙĬس":132925,"ĠsocietÃł":132926,"ĠاÙĦÙĥبÙĬر":132927,"Ġ×ij×ŀס":132928,"Ġ×ij×ŀס×Ĵר":132929,"Ġ×ij×ŀס×Ĵרת":132930,"ĠìŀĪìľ¼ë©°":132931,"Ġnặng":132932,"ÙĩÙī":132933,"ĠBÃł":132934,"×ŀר×ķ":132935,"ĠjÄĻ":132936,"ĠjÄĻzy":132937,"ĠjÄĻzyk":132938,"Ġ׼×ŀ×ķ×ijף":132939,"×¢×ľ×Ķ":132940,"à¸Ĺีà¹Īà¹Ħà¸Ķà¹ī":132941,"ãģ¾ãģĹãĤĩãģĨ":132942,"×ŀספר":132943,"ТÐŀ":132944,"سÙĬاسة":132945,"ĠкаждÑĭй":132946,"ë²ł":132947,"tım":132948,"yá»ĩn":132949,"รีà¹Ī":132950,"ĠдеÑĤÑģк":132951,"วิà¸ĺีà¸ģาร":132952,"mówi":132953,"×ĺ×¢×Ŀ":132954,"×Ķצ׾×Ĺ×Ķ":132955,"ضÙĬÙģ":132956,"ĠÑħоÑĤÑı":132957,"ãĤĵãģ§ãģĦãĤĭ":132958,"à¸Ħาà¸Ķ":132959,"à¸Ħรà¸ļ":132960,"ĠкÑĥÑĢÑģ":132961,"ĠbaÅŁarı":132962,"×ijר×ķ":132963,"ÙĬعة":132964,"ĠÐĿÑĥ":132965,"à¸Ħวามà¹Ģà¸Ľà¹ĩà¸Ļ":132966,"Ġ׾×ŀש׾":132967,"Ġì¢ĭìĿĢ":132968,"Ùħؤسس":132969,"Ùħؤسسات":132970,"Ġprécis":132971,"Ġthảo":132972,"à¸ģà¹ĩà¸Ħืà¸Ń":132973,"Ġש׼׾":132974,"führung":132975,"ãģĦãģ§":132976,"à¹ģละมี":132977,"à¸ģà¹ĩมี":132978,"Ġשש":132979,"мел":132980,"Ġкниг":132981,"ĠباÙĦÙĨ":132982,"ĠباÙĦÙĨسبة":132983,"Ġaldı":132984,"ÑĤай":132985,"Ġ×Ĺ×ĵש×Ļ×Ŀ":132986,"å®Łãģ¯":132987,"عÙĪا":132988,"ĠìĿĺ미":132989,"изм":132990,"ÑĢабоÑĤаÑĤÑĮ":132991,"Ùģص":132992,"Ġ×ij׳×ķסף":132993,"ãģ¨ãģĹãģ¦ãĤĤ":132994,"à¹Ģà¸Ľà¹ĩà¸Ļà¸Ĺีà¹Ī":132995,"ĠÑģледÑĥеÑĤ":132996,"èĢĥãģĪãģ¦":132997,"Ġ׼×Ļ×ķ×Ŀ":132998,"ÑģÑĤÑĭ":132999,"׼׾׼׾×Ļ":133000,"æµģãĤĮ":133001,"ãĤĴãģ¤ãģij":133002,"ÑĩаÑĤ":133003,"×Ļ׼×ķף":133004,"×Ļר×Ļ":133005,"larıyla":133006,"ãĤ¤ãĥ¡":133007,"ãĤ¤ãĥ¡ãĥ¼ãĤ¸":133008,"׳×ĸק":133009,"Ġciò":133010,"Ġsın":133011,"Ġsınır":133012,"à¸Ļà¸Ħร":133013,"каÑĤ":133014,"Ġlá»Ĺi":133015,"ëŀĮ":133016,"تÙģاص":133017,"تÙģاصÙĬÙĦ":133018,"ëĨĵ":133019,"ĠÙħض":133020,"ilmiÅŁ":133021,"بارÙĥ":133022,"ÐĿÐĺ":133023,"Ġthẩm":133024,"Ġ×IJ×ķת×ļ":133025,"ĠпÑĢиним":133026,"ĠпÑĢинима":133027,"Ġyönt":133028,"Ġyöntem":133029,"Ġ×ŀק×ij׾":133030,"Ġktórego":133031,"ê·Ģ":133032,"شرÙģ":133033,"داÙħ":133034,"ãģĦãĤįãģĦãĤį":133035,"ĠAlém":133036,"Ġgörü":133037,"Ġgörünt":133038,"Ġgörüntü":133039,"دس":133040,"ÑĪки":133041,"гÑĢад":133042,"Ġlạc":133043,"Ġsữa":133044,"ãĤīãĤĮãģ¾ãģĻ":133045,"oÃłi":133046,"Ñīен":133047,"ãģĭãģªãģĦ":133048,"Ġпоп":133049,"ĠпопÑĥ":133050,"ĠпопÑĥлÑıÑĢ":133051,"ĠاÙĦÙħÙĪÙĤع":133052,"räg":133053,"Ａ":133054,"íķĦ":133055,"ãĤĴè¦ĭãĤĭ":133056,"اÙħا":133057,"ĠاÙĦØŃرب":133058,"ĠÐŁÐ°":133059,"Ġ׾×IJתר":133060,"Ġtá»ijc":133061,"×ij׾×Ķ":133062,"رئÙĬس":133063,"вÑĥ":133064,"ÙĬدÙĬ":133065,"казан":133066,"Ġ×Ĺש×ij×ķף":133067,"hôtel":133068,"×¢×ķ׳×Ķ":133069,"بÙĨÙĬ":133070,"×ŀ×ķ׾":133071,"ĠднÑı":133072,"éĽ£ãģĹãģĦ":133073,"ведениÑı":133074,"Ġ×ķ×ŀת":133075,"напÑĢимеÑĢ":133076,"ÙĤابÙĦ":133077,"Ġrésultat":133078,"ĠÑĢазвиÑĤиÑı":133079,"رÙij":133080,"ìłĦ문":133081,"ĠاÙĦÙħزÙĬد":133082,"ĠìľĦíķ´ìĦľ":133083,"ëĨį":133084,"íĻķ":133085,"ĠThiết":133086,"íĮ¨":133087,"malıdır":133088,"ĠczÅĤ":133089,"ĠczÅĤowie":133090,"ĠczÅĤowiek":133091,"ĠÙĦبÙĨ":133092,"ĠÙĦبÙĨاÙĨ":133093,"üsü":133094,"ãģªãĤĵãģł":133095,"Ġżycie":133096,"ĠÑħоÑĢоÑĪо":133097,"æĸ¹ãģ«":133098,"ëĭ¤ë©´":133099,"иÑĩеÑģкаÑı":133100,"ער×Ļ׼":133101,"ער×Ļ×Ľ×ª":133102,"ãģ¾ãģĽãĤĵãģ§ãģĹãģŁ":133103,"ĠÑģобой":133104,"Ġgá»Ĺ":133105,"ĠделаÑĤÑĮ":133106,"daÄĩ":133107,"аÑĢа":133108,"różni":133109,"à¹Ģลีà¹ī":133110,"à¹Ģลีà¹īย":133111,"à¹Ģลีà¹īยà¸ĩ":133112,"à¸Ŀาà¸ģ":133113,"ĠتÙĤ":133114,"ĠتÙĤدÙĬ":133115,"ĠتÙĤدÙĬÙħ":133116,"หà¸Ļุà¹Īม":133117,"Ġmücade":133118,"Ġmücadele":133119,"ì§Ģ를":133120,"ãĤ¤ãĤ¹":133121,"Ġأساس":133122,"jÄħcego":133123,"ĠÅŁeh":133124,"нÑĤеÑĢ":133125,"ÑĨиÑİ":133126,"ï»»":133127,"ÑİÑīего":133128,"à¹Ĥà¸Ľà¸£à¹ģ":133129,"à¹Ĥà¸Ľà¸£à¹ģà¸ģรม":133130,"ĠmieÄĩ":133131,"ØŃÙĥÙĪÙħØ©":133132,"ãģ§ãģĹãģŁãģĮ":133133,"×Ļס×Ķ":133134,"ãĤĤãģ®ãĤĴ":133135,"Ġ×ŀ×IJת":133136,"สุà¸Ķà¸Ĺà¹īาย":133137,"ĠcÅ©":133138,"ÙĨسب":133139,"ĠпÑĢоÑĩ":133140,"Ġдней":133141,"ĠÑįÑĤиÑħ":133142,"׾×ŀת":133143,"нÑıÑı":133144,"Ñįк":133145,"Ġì§ĢëĤľ":133146,"มหาวิà¸Ĺยา":133147,"มหาวิà¸Ĺยาล":133148,"มหาวิà¸Ĺยาลัย":133149,"dão":133150,"ĠMáy":133151,"ĠêµŃê°Ģ":133152,"à¸ļุรี":133153,"×Ĵ×Ļ׾":133154,"ĠÑĤÑĭÑģÑı":133155,"ĠÑĤÑĭÑģÑıÑĩ":133156,"ÙģÙĥ":133157,"ĠÐĺÑģ":133158,"è¡ĮãĤıãĤĮ":133159,"פר×ĵ":133160,"ãģ¤ãģį":133161,"à¸Ħรà¸Ńà¸ļ":133162,"à¸Ħรà¸Ńà¸ļà¸Ħรัว":133163,"à¸Ĥึà¹īà¸Ļมา":133164,"ä»ĬæĹ¥ãģ¯":133165,"ĠìĤ¬ëŀĮìĿ´":133166,"עצ×ŀ×Ķ":133167,"поÑĢ":133168,"ĠKỳ":133169,"ĠÆ¡n":133170,"ĠthÄĥm":133171,"ÙģاÙĤ":133172,"ãģļãģ«":133173,"Ġ׾קר":133174,"Ġ׾קר×ķ×IJ":133175,"اÙģÙĬØ©":133176,"ÙħÙİا":133177,"гаÑĢ":133178,"صÙĦا":133179,"صÙĦاة":133180,"Ġ×ŀ×ĸ×Ķ":133181,"lıģını":133182,"Ġ×IJ×Ļ׳×Ķ":133183,"кÑĢо":133184,"ĠngÆ°Æ¡i":133185,"Ġвним":133186,"Ġвнимание":133187,"jÄħcy":133188,"ÙĢÙĢÙĢÙĢÙĢ":133189,"ÑģÑħод":133190,"ãģªãĤĵãģĭ":133191,"×ŀ×Ļ׾":133192,"Ġ×Ķ×IJ×Ĺ":133193,"ãĤıãģªãģĦ":133194,"عسÙĥر":133195,"ĠìĦ¸ê³Ħ":133196,"ĠÑĩего":133197,"ĠÑģÑĢедÑģÑĤва":133198,"ĠÐłÐ°Ñģ":133199,"ãģªãģģ":133200,"ÙĨÙģس":133201,"ר×Ļ×ķף":133202,"ÑģÑĥд":133203,"ĠìĿ¸ê°Ħ":133204,"ĠاÙĦÙħÙĤبÙĦ":133205,"ÙĨعÙħ":133206,"تÙĪÙģر":133207,"ש×ij×¢":133208,"ılm":133209,"ılmÄ±ÅŁ":133210,"Ġ×ľ×ª×ª":133211,"تصÙģ":133212,"×Ķפ×ķ×ļ":133213,"à¹ĥà¸Ļà¸Ľà¸µ":133214,"ìĿ´ê³ł":133215,"ÙģÙĪز":133216,"à¸ľà¸¥à¸ĩาà¸Ļ":133217,"ĠGiáo":133218,"à¸ļà¸Ńà¸ģวà¹Īา":133219,"ĠdÄ±ÅŁ":133220,"ĠdÄ±ÅŁÄ±nda":133221,"죽":133222,"ĠdzieÅĦ":133223,"кÑĨии":133224,"иÑĨе":133225,"ãģ®ä¸Ģ":133226,"عش":133227,"пÑĢеÑģÑģ":133228,"หà¸Ļà¹Īà¸Ńย":133229,"ลัà¸ģษà¸ĵะ":133230,"ĠpossibilitÃł":133231,"à¹Ħà¸Ķà¹īรัà¸ļà¸ģาร":133232,"หยุà¸Ķ":133233,"Ġphiên":133234,"çĶŁãģ¾ãĤĮ":133235,"Ø·ÙĪÙĦ":133236,"ÑĦин":133237,"für":133238,"ØŃÙĬاة":133239,"íĸĪìĬµëĭĪëĭ¤":133240,"׼׳×ķת":133241,"à¸Ľà¸£à¸°à¸ª":133242,"à¸Ľà¸£à¸°à¸ªà¸ļ":133243,"à¸Ľà¸£à¸°à¸ªà¸ļà¸ģารà¸ĵà¹Į":133244,"ëIJĺìĹĪ":133245,"Ġkażdy":133246,"Ġluyá»ĩn":133247,"ĠоÑĢганизаÑĨии":133248,"å°ijãģªãģı":133249,"ÑģÑĤÑĢоен":133250,"Ġtécnico":133251,"ק×Ķ׾":133252,"Ġ×ķ×IJ×Ĺ":133253,"ĠعÙĦÙĬÙĥ":133254,"Ñīение":133255,"Ġ×Ķ×Ļ׾×ĵ×Ļ×Ŀ":133256,"ÙĪسائÙĦ":133257,"Ġ×ķ×Ķת":133258,"تÙħÙĬز":133259,"ĠÑģказал":133260,"Ġполи":133261,"Ġ×Ķ×ŀס":133262,"ÙĦÙijÙİ":133263,"Ùħؤسسة":133264,"Ġ×ŀ×Ļ×ĵ":133265,"ãģ£ãģ¡":133266,"ĠëĦĪ무":133267,"à¸ŀี":133268,"Ġtặng":133269,"Ġtấn":133270,"רש×Ŀ":133271,"Ġmédica":133272,"Ġ×¢×ķ×ŀ":133273,"Ġ×¢×ķ×ŀ×ĵ":133274,"ÑĦоÑĢ":133275,"Ùħرة":133276,"Ġvatanda":133277,"ĠvatandaÅŁ":133278,"Ġдело":133279,"à¸Ļม":133280,"ãģ¨åIJĮãģĺ":133281,"ÙģÙī":133282,"ÑģоÑĢ":133283,"Ġ×Ķסר×ĺ":133284,"Ġépoca":133285,"ìłķì±ħ":133286,"ĠÑģвÑıзан":133287,"ضرب":133288,"ĠÙĦÙĨا":133289,"Ġużywa":133290,"ĠاÙĦجÙĬØ´":133291,"ÑİÑĢ":133292,"×ijס×ķ×£":133293,"ĠмÑĥ":133294,"ĠмÑĥзÑĭк":133295,"bilité":133296,"Ġmaç":133297,"سÙİ":133298,"تÙĦÙĥ":133299,"ãģ¬":133300,"ÙĬÙĦا":133301,"ÑĪла":133302,"ÙĢÙĢÙĢ":133303,"Ġодной":133304,"зван":133305,"ĠÑģÑĢаз":133306,"ĠÑģÑĢазÑĥ":133307,"ÙĨظÙħ":133308,"راÙĩ":133309,"ĠÙĦÙĩذا":133310,"׼×ķר":133311,"Ġ×Ķש×ij×ķ×¢":133312,"Ġ×Ķשת":133313,"ĠQuảng":133314,"ãĥ«ãĥ¼":133315,"ãģĪãģªãģĦ":133316,"×ĺ×IJ":133317,"Ġmiá»ģn":133318,"ĠPháºŃt":133319,"ĠاÙĦسÙĪÙĤ":133320,"ÄĤ":133321,"ĠاÙĦجÙħع":133322,"ĠاÙĦجÙħعة":133323,"ÑİÑīей":133324,"aÅĤem":133325,"عتÙĤد":133326,"Ø£ÙĦÙħ":133327,"Ñģке":133328,"ĠìĿ´íķ´":133329,"ÙĨسخ":133330,"è¨ĢãģĦ":133331,"добав":133332,"سبÙĤ":133333,"×¢×ķרר":133334,"ÑĤип":133335,"ãģĿãģĵãģ§":133336,"visión":133337,"عÙĪدة":133338,"먹":133339,"×ŀ×ĸר×Ĺ":133340,"ĠØ¥ØŃ":133341,"Ġ׾×ij×Ļף":133342,"Ġ׾צ×IJת":133343,"Ġyardı":133344,"Ġyardımc":133345,"Ġyardımcı":133346,"Ä°Z":133347,"קפ×Ķ":133348,"tré":133349,"liÄŁini":133350,"клÑİÑĩа":133351,"Ġüretim":133352,"Ġayrı":133353,"ĠkiÅŁiler":133354,"à¸Ħà¹īà¸Ļ":133355,"à¸Ħà¹īà¸Ļหา":133356,"ĠSá»±":133357,"Ġ×Ľ×¡":133358,"Ġ×Ľ×¡×£":133359,"ĠÑĤакиÑħ":133360,"ĠXuân":133361,"Ġлег":133362,"Ġлегко":133363,"Ø«ÙĤاÙģØ©":133364,"ÐĿÐŀ":133365,"ãĤ¹ãĤ¿ãĥĥ":133366,"ãĤ¹ãĤ¿ãĥĥãĥķ":133367,"åIJĪãģĦ":133368,"Ġ×Ķש×Ļ×ŀ×ķש":133369,"manız":133370,"ĠÐĴаÑģ":133371,"gün":133372,"ìľĦìĽIJíļĮ":133373,"Ġwspóln":133374,"ĠÑģвое":133375,"íĥģ":133376,"à¹Ģà¸Ļีย":133377,"ÙĪبة":133378,"вÑıз":133379,"ıdır":133380,"ëIJĺìĹĪëĭ¤":133381,"ĠdeÄŁiÅŁtir":133382,"ãĤĭãģĵãģ¨ãģĮ":133383,"Ġ×Ĺ×ĵש×Ķ":133384,"ãĤīãĤĮãģ¦ãģĦãĤĭ":133385,"×Ĺ×Ļ×Ļ×ij":133386,"ĠÐļаÑĢ":133387,"׳×Ļת×ķ×Ĺ":133388,"Ġק×ĺף":133389,"ר×ĸ":133390,"ÙĪغ":133391,"èªŃãģ¿":133392,"ĠتÙĤÙĪÙħ":133393,"ĠÙĥاÙĦ":133394,"à¸Ŀึà¸ģ":133395,"Ġë°ľìĥĿ":133396,"ológico":133397,"راع":133398,"à¹ģà¸ģà¹īà¹Ħà¸Ĥ":133399,"ĠÑĢабоÑĤÑĥ":133400,"ÙĨÙijÙİ":133401,"à¸Ńยูà¹Īà¸Ĺีà¹Ī":133402,"ĠاÙĦثاÙĨÙĬØ©":133403,"ĠNhân":133404,"ÑħваÑĤ":133405,"öne":133406,"Ġعدة":133407,"à¹ģสà¸ĩ":133408,"ÑĤоп":133409,"пÑĥÑģка":133410,"شراء":133411,"ĠÐļом":133412,"Ġפע×ķ׾×Ķ":133413,"ìĤ¬ìĿ´":133414,"ìĤ¬ìĿ´íĬ¸":133415,"è¡Įãģ£ãģ¦":133416,"Ġ×Ķ×Ķת":133417,"ĠÑģÑĤоÑĢо":133418,"ĠÑģÑĤоÑĢонÑĭ":133419,"درس":133420,"à¸ĭู":133421,"à¸ķà¹Īำ":133422,"ĠأبÙĬ":133423,"подоб":133424,"ãģ«ãģ¦":133425,"ارتÙģاع":133426,"ĠÙħؤ":133427,"иков":133428,"geführt":133429,"มืà¸Ńà¸ĸืà¸Ń":133430,"ĠÙĦÙĤد":133431,"ĠØ£ÙĨÙij":133432,"سÙĬطر":133433,"ãģ¾ãģļãģ¯":133434,"ס×ĵ":133435,"ÑģколÑĮко":133436,"ãģ¿ãģŁãģĦãģª":133437,"×ĵר×Ĵ":133438,"×¢×Ļ×ĵ":133439,"à¹ĥหà¹īà¸ļริà¸ģาร":133440,"ĠÐĶи":133441,"×ij×¢×Ļ×ķת":133442,"Ġ×Ķ×Ĺ×ķ":133443,"пиÑģÑĮ":133444,"ĠاÙĦØ®ÙĦ":133445,"бав":133446,"ĠÄ°lk":133447,"ĠاÙĦØ®Ùħ":133448,"ĠاÙĦØ®ÙħÙĬس":133449,"ĠÙĬÙĤÙĪÙħ":133450,"æĻĤãģ®":133451,"ĠsÅĤow":133452,"ĠØ£ÙĩÙħ":133453,"Ø®ÙĦÙĤ":133454,"ĠأصبØŃ":133455,"Ġchứa":133456,"Ġthác":133457,"ÙģاÙĦ":133458,"Ġchá»Ŀ":133459,"ĠاÙĦخار":133460,"ĠاÙĦخارج":133461,"ĠاÙĦخارجÙĬØ©":133462,"طائر":133463,"ĠtÃł":133464,"ĠtÃłu":133465,"à¸ģลà¹īà¸Ńà¸ĩ":133466,"ĠاÙĦÙħرأ":133467,"ĠاÙĦÙħرأة":133468,"åħ¨ãģı":133469,"ĠÃĸn":133470,"çļĦãģ«ãģ¯":133471,"Ġpièce":133472,"×Ĵ×Ļ×ij":133473,"ĠاÙĦÙĪاÙĤع":133474,"ä»Ĭãģ®":133475,"ĠاÙĦÙħÙĤ":133476,"cznÄħ":133477,"ÙģعاÙĦ":133478,"енного":133479,"ĠÑĦакÑĤ":133480,"ìĭłì²Ń":133481,"ĠÐŀни":133482,"ĠاÙĦبÙĦاد":133483,"овиÑĩ":133484,"ëıĮ":133485,"ÑĦÑĥнкÑĨи":133486,"Ġìĸ´ëĬIJ":133487,"ãĥķãĤ©ãĥ¼":133488,"dÃŃ":133489,"илоÑģÑĮ":133490,"ÙħÙī":133491,"ĠاÙĦØ£ÙħرÙĬÙĥ":133492,"ĠاÙĦØ£ÙħرÙĬÙĥÙĬØ©":133493,"×ĺ×Ļפ×ķ׾":133494,"íĶĦë¡ľê·¸":133495,"íĶĦë¡ľê·¸ëŀ¨":133496,"Ġש×ķ׳×ķת":133497,"Ø´ÙħÙĦ":133498,"ĠпаÑĢа":133499,"Ġ×Ķ×Ĺ×ķק":133500,"ÙĪزارة":133501,"ãģ¨ãģĻãĤĭ":133502,"Ġquảng":133503,"Ġaģır":133504,"ĠاÙĦÙĦج":133505,"ĠاÙĦÙĦجÙĨØ©":133506,"긴":133507,"ĠTân":133508,"جÙħÙĦ":133509,"дол":133510,"à¹ģà¸ŀà¸Ĺย":133511,"à¹ģà¸ŀà¸Ĺยà¹Į":133512,"Ġר×IJש×Ļ":133513,"Ñīей":133514,"Ġçevre":133515,"ĠкомплекÑģ":133516,"Ġ×ij×ŀש×ļ":133517,"Ġaltın":133518,"ĠأعÙħاÙĦ":133519,"ĠÑģвоего":133520,"ãĤĪãģĦ":133521,"×Ĺ׾×Ļ×ĺ":133522,"×ŀ×ł×¢":133523,"Ġר×ij×Ķ":133524,"ĠØ£ÙĬضاÙĭ":133525,"×ĸ׾":133526,"ĠاÙĦسÙĬاسÙĬ":133527,"æĢĿãģĨ":133528,"קרק":133529,"קרקע":133530,"ĠاÙĦÙģرÙĬÙĤ":133531,"биÑĤ":133532,"ק׳×Ķ":133533,"ĠØ¥ÙĨÙĩ":133534,"ĠÐĴам":133535,"ÐłÐŀ":133536,"ãĥĪãĥª":133537,"å¿ħè¦ģãģª":133538,"Ġchâu":133539,"ç¶ļãģij":133540,"Ġçözüm":133541,"gÅĤow":133542,"عÙĤÙĦ":133543,"売ãĤĭ":133544,"iết":133545,"à¸Ĭิà¹īà¸Ļ":133546,"ĠØŃÙĤÙĪÙĤ":133547,"Ø·ÙĦع":133548,"ĠÄijen":133549,"ĠÙĥاÙģØ©":133550,"ãģ®ãģĶ":133551,"Ġë¬":133552,"Ġ물":133553,"Ġë¬¼ë¡ł":133554,"ĠرسÙĪÙĦ":133555,"зам":133556,"замен":133557,"Ġkullanıcı":133558,"×¢×ķ׾":133559,"èī²ãĢħ":133560,"ÑĪиÑĢ":133561,"Ġ×Ĺש":133562,"Ġwygl":133563,"ĠwyglÄħda":133564,"ש×Ļ×ŀ×ķש":133565,"å¿ĺãĤĮ":133566,"×¢×Ļצ×ķ×ij":133567,"ĠاÙĦسÙĪرÙĬ":133568,"å°ijãģªãģĦ":133569,"ĠпоиÑģк":133570,"สำà¸Ļัà¸ģà¸ĩาà¸Ļ":133571,"Ġ×ŀצ×ĵ":133572,"ĠmÃ¼ÅŁ":133573,"ĠmÃ¼ÅŁter":133574,"ĠmÃ¼ÅŁteri":133575,"ĠÙħÙĨÙĩÙħ":133576,"à¸ķำà¹ģ":133577,"à¸ķำà¹ģหà¸Ļ":133578,"à¸ķำà¹ģหà¸Ļà¹Īà¸ĩ":133579,"ÅĽmie":133580,"Ġ×©×ł×ª":133581,"Ġ×Ķפ×Ļ":133582,"פרש":133583,"×¢×ijר×Ļת":133584,"สà¸Ļัà¸ļ":133585,"สà¸Ļัà¸ļสà¸Ļุ":133586,"สà¸Ļัà¸ļสà¸Ļุà¸Ļ":133587,"è¨Ģãģ£ãģ¦":133588,"à¸ģารà¸Īัà¸Ķ":133589,"ĠMoże":133590,"изаÑĨии":133591,"ứt":133592,"ĠÙĪبعد":133593,"ĠdeÄŁild":133594,"ĠdeÄŁildir":133595,"Ġת×ŀ":133596,"Ġ×ŀ×ŀ׳×ķ":133597,"話ãĤĴ":133598,"ĠÑĨена":133599,"Ġthúc":133600,"×Ļ×ŀ×ķף":133601,"ĠBáo":133602,"ãĤĴåıĸãĤĬ":133603,"å®īãģĦ":133604,"Ġ×¢×ķש×Ļ×Ŀ":133605,"èĩªåĪĨãģĮ":133606,"lée":133607,"ãĤĭãģ®ãģ§":133608,"иÑĢÑĥеÑĤ":133609,"ãģ¦ãĤĭ":133610,"ستر":133611,"ĠاÙĦØŃÙĬ":133612,"×Ļ׾×ķת":133613,"Ġ×Ĺ×ij":133614,"ÙĤرأ":133615,"تÙħÙĥÙĨ":133616,"سائÙĦ":133617,"prüf":133618,"ãģĭãģijãģ¦":133619,"ĠÑģобÑģÑĤвенно":133620,"ĠìľĦíķĺìŬ":133621,"׾×Ļ×ĺ":133622,"ãģĮå¤ļãģı":133623,"ÙĬتÙĩا":133624,"ç«ĭãģ¦":133625,"มà¸Ńà¸ļ":133626,"ìĭľìŀ¥":133627,"оÑĢа":133628,"ĠsavaÅŁ":133629,"×ĺ×Ļ×ij×Ļ":133630,"×ij׳×ķ":133631,"Ùħاذا":133632,"기ê°Ħ":133633,"ãģªãģ©ãģ§":133634,"Ġ×ŀת×Ĺ×Ļ׾":133635,"Ġnhiá»ħ":133636,"Ġnhiá»ħm":133637,"каÑĢ":133638,"каÑĢÑĤ":133639,"Ġ׾×Ķשת×ŀש":133640,"׳×Ļ×Ĺ":133641,"ادÙĬØ©":133642,"รายà¸ĩาà¸Ļ":133643,"ĠprzykÅĤad":133644,"Ñīий":133645,"ØŃضÙĪر":133646,"Ġhôn":133647,"ÃĿ":133648,"ת×ķצ×IJ×ķת":133649,"رابط":133650,"Ġbếp":133651,"ĠполÑĥÑĩи":133652,"åĩºä¼ļãģĦç³»":133653,"à¸Ľà¸¥à¹Īà¸Ńย":133654,"ĠاÙĦشباب":133655,"اÙĩÙĦ":133656,"ä»Ĭãģ¾ãģ§":133657,"رجع":133658,"ãĤ¶ãĥ¼":133659,"ÙĤÙģ":133660,"ĠGroÃŁ":133661,"ĠíļĮìĽIJ":133662,"اجر":133663,"Ġ×ij×ŀקר×Ķ":133664,"Ġsegurança":133665,"fühl":133666,"ãģ¦ãģĦãģı":133667,"หมà¸Ń":133668,"ĠкоÑĤоÑĢом":133669,"ĠNÄĥm":133670,"ĠdÅĤugo":133671,"ÙħÙĨØŃ":133672,"ש×ķ×ķ×Ļ":133673,"ĠØ£ÙĬاÙħ":133674,"à¸ªà¸łà¸²à¸ŀ":133675,"rzÄħ":133676,"شرÙĥات":133677,"ãĤĴèĢĥãģĪ":133678,"даÑĢ":133679,"à¸Ľà¸£à¸°à¸Ĭุม":133680,"Ġ×ķ×IJ×ĸ":133681,"iá»ĩn":133682,"ĠtÆ°Æ¡i":133683,"ש×Ļ×Ĺ":133684,"à¸Ńà¹Īà¸Ńà¸Ļ":133685,"æĽ¸ãģĦãģ¦":133686,"Ġngữ":133687,"×ij×Ļ×ĺ×Ĺ":133688,"×ij×Ļ×ĺ×Ĺ×ķף":133689,"Ġsẵ":133690,"Ġsẵn":133691,"ì§ĢëıĦ":133692,"ĠпÑĢеп":133693,"ĠпÑĢепаÑĢаÑĤ":133694,"ĠнаÑĥÑĩ":133695,"ĠÃľnivers":133696,"ĠÃľniversites":133697,"ĠÃľniversitesi":133698,"Ġ×Ĵ×ĵ×ķ׾×Ķ":133699,"Ġ×Ķ×ł×ª":133700,"Ġ×Ķ×ł×ª×ij×¢":133701,"ãģ§ãģĤãģ£ãģŁ":133702,"ĠmiesiÄħ":133703,"ĠmiesiÄħc":133704,"гÑĢам":133705,"гÑĢамм":133706,"ĠبشأÙĨ":133707,"ĠÑħÑĢ":133708,"ק×Ļ×ĵ":133709,"ק×Ļ×ĵ×ķ×Ŀ":133710,"Ø´Ùĥر":133711,"Ġá»ķ":133712,"Ġá»ķn":133713,"ãģĮãģĤãģ£ãģ¦":133714,"ãģķãĤĮãģ¾ãģĻ":133715,"Ġ×Ĺ×ķ×ĵ":133716,"Ġ×Ĺ×ķ×ĵש×Ļ×Ŀ":133717,"ÙħÙĪاجÙĩ":133718,"ÙħÙĪاجÙĩØ©":133719,"أشخاص":133720,"بغ":133721,"à¹Ģรียà¸Ļรูà¹ī":133722,"ãģĹãģ¦ãģĦãģı":133723,"Ġsạn":133724,"å¿ħãģļ":133725,"׳×Ļ×Ĵ":133726,"׳×Ļ×Ĵ×ķ×ĵ":133727,"باÙĦغ":133728,"×Ĺש×ŀ":133729,"×Ĺש×ŀ׾":133730,"Ġnapraw":133731,"ĠnaprawdÄĻ":133732,"Ø´Ùĩاد":133733,"×IJ×ķ×Ķ":133734,"×IJ×ķ×Ķ×ij":133735,"иÑĨÑĭ":133736,"Ġ×Ķר׼×ij":133737,"ëŀij":133738,"Ġתע":133739,"Ġ×Ķ×Ļש":133740,"Ġ×Ķ×Ļשר×IJ":133741,"Ġ×Ķ×Ļשר×IJ׾×Ļ":133742,"Ø£ÙħÙĨ":133743,"ÑİÑīаÑı":133744,"skór":133745,"LERÄ°":133746,"Ġ×Ķ×IJ×Ĺר×ķף":133747,"×¢×ł×§":133748,"ĠÙĪÙĥÙĦ":133749,"ãģĵãģĵãģ§":133750,"Ġquán":133751,"liÄŁin":133752,"à¸ģà¸İหมาย":133753,"Ø·Ùħ":133754,"أجÙĩ":133755,"أجÙĩزة":133756,"ĠErdoÄŁan":133757,"ãģ§ãģĬ":133758,"ĠвÑĢа":133759,"ĠвÑĢаÑĩ":133760,"ĠPhó":133761,"à¸Ĭัà¹Īว":133762,"à¸Ĭัà¹Īวà¹Ĥม":133763,"à¸Ĭัà¹Īวà¹Ĥมà¸ĩ":133764,"Ġphúc":133765,"×Ļפ×ķת":133766,"×¢×Ļ×ķף":133767,"Ġdużo":133768,"ãĥģãĥ¼ãĥł":133769,"ĠÙĬÙİ":133770,"ĠзадаÑĩ":133771,"Ġ×Ĵ×ij×ķ×Ķ×Ķ":133772,"Ġ׼׼׾":133773,"ложен":133774,"état":133775,"ĠngÄĥn":133776,"èµ·ãģį":133777,"ĠTiến":133778,"صعب":133779,"Ġexperiência":133780,"Ø®Ùħ":133781,"à¸ģารà¸Ĺำà¸ĩาà¸Ļ":133782,"سÙĬد":133783,"ĠDá»±":133784,"ĠкоÑĤоÑĢого":133785,"ladıģı":133786,"Ġkhá»ķ":133787,"Ġê³ĦìĨį":133788,"Ñīик":133789,"สà¹Īวà¸Ļà¸ķัว":133790,"зоÑĢ":133791,"ÙĨÙı":133792,"Ġà¸Ķัà¸ĩ":133793,"Ġà¸Ķัà¸ĩà¸Ļัà¹īà¸Ļ":133794,"Ġcấu":133795,"ĠÄijá»ijc":133796,"оÑĦ":133797,"ĠاÙĦأعÙħاÙĦ":133798,"ãģªãģıãģ¦ãĤĤ":133799,"×ķ׼×Ļ×Ŀ":133800,"à¹ģà¸Ľ":133801,"ĠBên":133802,"ãĥ¯ãĥ³":133803,"Ġgiám":133804,"ĠÅŀu":133805,"Ġdáng":133806,"عÙĦÙĬ":133807,"à¹Ģà¸ģษ":133808,"à¹Ģà¸ģษà¸ķร":133809,"ÙĪجب":133810,"ннÑĭе":133811,"ÙĤضاء":133812,"à¸Ħวà¸ļ":133813,"à¸Ħวà¸ļà¸Ħุ":133814,"à¸Ħวà¸ļà¸Ħุม":133815,"ãģ¤ãģ¤":133816,"ĠViá»ĩc":133817,"×ŀ×ij×ĺ":133818,"ש×Ļת×ķ×£":133819,"ĠведÑĮ":133820,"kaza":133821,"kazaÅĤ":133822,"à¸ķำรวà¸Ī":133823,"ãĤ¿ãĥ«":133824,"ĠповÑĭ":133825,"ĠповÑĭÑĪен":133826,"ĠSợ":133827,"ĠìĦ¤ëªħ":133828,"ĠÃĩünkü":133829,"ìĥĿíĻľ":133830,"Ö¾":133831,"ãĤĮãģ¦ãģĦãĤĭ":133832,"Ġ×ijר×IJש":133833,"ר×ķ×Ĵ":133834,"ĠоÑĦи":133835,"ĠоÑĦиÑĨиалÑĮн":133836,"ĠÑĥÑģÑĤанов":133837,"ĠÑĥÑģÑĤановлен":133838,"ĠاÙĦÙħصر":133839,"ĠاÙĦÙħصرÙĬØ©":133840,"ĠÐŁÐ¾ÑįÑĤомÑĥ":133841,"ÙĨصÙģ":133842,"ĠÙĪاÙĦÙĨ":133843,"ĠhÃłi":133844,"à¸Ħิ":133845,"ĠAprès":133846,"ì³IJ":133847,"à¹Ģà¸ĭีย":133848,"×ĵ×ŀ×Ķ":133849,"activité":133850,"à¸Ħิà¸Ķวà¹Īา":133851,"ÑĤÑĢен":133852,"à¹Ģฮ":133853,"ãĥıãĤ¤":133854,"ãģĮå¢ĹãģĪ":133855,"еннаÑı":133856,"Ġìĺ¤ëĬĺ":133857,"ãĥ¢ãĥ³":133858,"ĠконеÑĩно":133859,"ĠÙħÙĤابÙĦ":133860,"clé":133861,"Ġhü":133862,"Ġthẳng":133863,"ìłģìĿ´":133864,"ĠÐIJлекÑģ":133865,"ĠÐIJлекÑģан":133866,"ĠÐIJлекÑģандÑĢ":133867,"ãĥŀãĥ³ãĤ·ãĥ§ãĥ³":133868,"ãģ²ãģ¨ãģ¤":133869,"ãģªãģĬ":133870,"à¹Ģà¸Īà¹īาà¸Ĥà¸Ńà¸ĩ":133871,"ëĵľë¦¬":133872,"شاء":133873,"ĠsaÄŁlık":133874,"ĠÅŁimdi":133875,"×Ļ×IJ׾":133876,"تأثÙĬر":133877,"أسب":133878,"أسباب":133879,"ĠвÑĭполнен":133880,"лок":133881,"ש×Ļ×ij×Ķ":133882,"Ġlắm":133883,"ĠTrÆ°á»Ľc":133884,"Ġ×Ķ×¢×ľ":133885,"리를":133886,"ĠÑĢеж":133887,"ĠÑĢежим":133888,"inté":133889,"intégr":133890,"×Ĵ׳×Ļ":133891,"ĠاÙĦشعر":133892,"Ġmilhões":133893,"Ġpequeño":133894,"ãĤ³ãĥ¼ãĤ¹":133895,"×ķ׼×Ĺ":133896,"à¹Ģà¸Ĭà¹īา":133897,"شرÙĤ":133898,"ĠhÆ°Æ¡ng":133899,"รัà¸IJà¸ļาล":133900,"à¸ģลาย":133901,"à¸ģลายà¹Ģà¸Ľà¹ĩà¸Ļ":133902,"ĠподÑħод":133903,"תש×ķ×ij×Ķ":133904,"ãģıãģªãģ£ãģ¦":133905,"ĠاÙĦØ£ÙħÙħ":133906,"ĠHá»įc":133907,"ĠwspóÅĤpr":133908,"ĠwspóÅĤprac":133909,"ÑĩÑĥв":133910,"ÑĩÑĥвÑģÑĤв":133911,"ÃŃstico":133912,"à¹Ģà¸ģาะ":133913,"ìĽĢ":133914,"Ġназад":133915,"ãĤĭãĤĪãģĨãģ«":133916,"ĠСШ":133917,"ĠСШÐIJ":133918,"мон":133919,"ĠAsÃŃ":133920,"×ķר×Ĵ":133921,"полнен":133922,"×ŀ×¡×ľ":133923,"×ŀ×¡×ľ×ķ׾":133924,"à¹Ģลืà¸Ńà¸Ķ":133925,"à¹Ģริà¹Īมà¸ķà¹īà¸Ļ":133926,"ĠاÙĦØ¥Ùħ":133927,"ĠاÙĦØ¥Ùħارات":133928,"צ×Ķר":133929,"ãĥ¡ãĥªãĥĥãĥĪ":133930,"ĠпоÑĤом":133931,"виз":133932,"ĠÙģترة":133933,"å¾Įãģ®":133934,"ÐĿÐIJ":133935,"×ŀסר":133936,"ÙĬرÙĬ":133937,"pré":133938,"ĠteÅŁek":133939,"ĠteÅŁekkür":133940,"Ġödeme":133941,"داÙĨ":133942,"ãģ¾ãģĹãģ¦":133943,"缮ãģ«":133944,"ĠÑĤеÑĩение":133945,"lard":133946,"lardır":133947,"à¹Ģราà¸Īะ":133948,"ספ×Ļ":133949,"ĠÙĪÙĥØ°ÙĦÙĥ":133950,"Ġhát":133951,"Ġtá»Ļc":133952,"à¸Ħุย":133953,"Ġbức":133954,"ØŃÙĬÙĨ":133955,"èģŀãģĦãģ¦":133956,"Ùħؤشر":133957,"ĠNhÆ°":133958,"Ġменее":133959,"ละà¸Ħร":133960,"Ñģин":133961,"ĠÑĢек":133962,"ĠÑĢекл":133963,"ĠÑĢеклам":133964,"ĠÙģÙĩÙĪ":133965,"Ġ׾×ĸ":133966,"×Ļ׳×ķת":133967,"ĠÅŁart":133968,"ÑģÑĤавка":133969,"Ġíı¬íķ¨":133970,"ãģ«è¡Įãģı":133971,"ï¼Ŀ":133972,"ĠпозволÑıеÑĤ":133973,"Ġת×ķ׼׾×ķ":133974,"овал":133975,"صÙĦØ©":133976,"Ġ׾ש׳×ķת":133977,"ĠÐĺгÑĢ":133978,"ÙħÙĨتجات":133979,"ĠsatÄ±ÅŁ":133980,"Ñģко":133981,"ĠاÙĦØ«ÙĦاثاء":133982,"Ġ×Ķ×ĵ×ijר×Ļ×Ŀ":133983,"ãģĹãģ¾ãģĹãĤĩãģĨ":133984,"بÙĤÙī":133985,"åĬĽãĤĴ":133986,"ĠÃĩok":133987,"ãĥģãĥ¥":133988,"à¹Ģà¸Ĭืà¹īà¸Ń":133989,"ยุà¸Ħ":133990,"ศาล":133991,"Ġק×ķ×ĵ×Ŀ":133992,"×ĸר×Ļ×Ŀ":133993,"ãģ®åł´åIJĪ":133994,"ĠìķĬìķĺ":133995,"ãģĤãĤĬãģ¾ãģĻãģĮ":133996,"×IJשר":133997,"è¡Įãģı":133998,"ãģ»ãģĭ":133999,"æ°Ĺãģ«ãģªãĤĭ":134000,"йдеÑĤ":134001,"íķĺìĺĢëĭ¤":134002,"ستÙħرار":134003,"ĠÐŁÑĢе":134004,"ĠÑģбоÑĢ":134005,"ĠìķĦ무":134006,"ç§ģãĤĤ":134007,"عص":134008,"ĠниÑĩ":134009,"ĠниÑĩего":134010,"ĠпÑĢием":134011,"ק×ķ×ŀ":134012,"ĠìĪĺëıĦ":134013,"Ġì¡´":134014,"Ġì¡´ìŀ¬":134015,"ĠأثÙĨ":134016,"ĠأثÙĨاء":134017,"ĠÙĪاÙĦØŃ":134018,"ãģĮãģ§ãģįãĤĭ":134019,"Ġת×Ķ":134020,"Ġת×Ķ×Ļ×Ķ":134021,"רף":134022,"ĠÑģвÑıзи":134023,"×Ĵשת":134024,"ÑģпекÑĤ":134025,"ס×ij×Ļ×ij":134026,"ס×ij×Ļ×ij×Ķ":134027,"ĠíķĦìļĶíķľ":134028,"تخصص":134029,"Ġжив":134030,"ĠживоÑĤ":134031,"ĠMayıs":134032,"تعا":134033,"تعاÙĪÙĨ":134034,"ĠعÙĨÙĩا":134035,"ówki":134036,"ĠاÙĦÙģÙĦسطÙĬÙĨÙĬ":134037,"ãģłãģijãģ§ãģªãģı":134038,"ìĿ¸ì§Ģ":134039,"ĠاÙĦسÙĪد":134040,"ĠاÙĦسÙĪداÙĨ":134041,"إجراءات":134042,"Ġkötü":134043,"Ġ×Ļתר":134044,"×Ĵ×Ļש×Ķ":134045,"Ġצ×ķר×ļ":134046,"รà¸ĸย":134047,"รà¸ĸยà¸Ļà¸ķà¹Į":134048,"ÑħоÑĤ":134049,"ÐłÐIJ":134050,"ÙĪØ·ÙĨ":134051,"Ġsayısı":134052,"ס×Ĺר":134053,"ÙħÙĪÙĦ":134054,"ãĤĴæĮģãģ£ãģ¦":134055,"عاÙĨ":134056,"Ġtá»Ļi":134057,"ĠвÑĭÑĪе":134058,"Ġtầm":134059,"ãĥĪãĥ¬":134060,"×Ļצ×ķ":134061,"มุม":134062,"سÙĪد":134063,"ìłĦìŀIJ":134064,"ãĤµãĥŃãĥ³":134065,"ìĤ°ìĹħ":134066,"ĠоÑģнован":134067,"Ø®Ùģض":134068,"רצ×Ķ":134069,"بÙĬض":134070,"×ķÖ¹":134071,"ס×Ļ×Ļ×¢":134072,"Ġש×IJ×Ļ":134073,"ĠاÙĦÙĤرآÙĨ":134074,"ĠТакже":134075,"×ŀש×ŀ×¢×ķת":134076,"سÙĩÙĦ":134077,"Ġ×Ķ׳×Ķ":134078,"ãĤĴãģĹãģ¦ãģĦãĤĭ":134079,"×Ļ×Ļס":134080,"×Ķ×ķ×IJ":134081,"ĠBÃŃ":134082,"Ġмало":134083,"ĠëĶ°ëĿ¼ìĦľ":134084,"Ġר×Ĺ×ij":134085,"ãģĮé«ĺãģĦ":134086,"ÙĪاس":134087,"ìĤ¼":134088,"×ł×¢":134089,"ãģ£ãģ¡ãĤĥ":134090,"ĠTüm":134091,"à¸Ńีà¸ģà¸Ķà¹īวย":134092,"ãģĹãģ¦ãģıãģłãģķãģĦ":134093,"ÙĨشاط":134094,"ãĥĹãĥ©ãĥ³":134095,"алиÑģÑĮ":134096,"×ĵ×ľ×ª":134097,"ĠwczeÅĽ":134098,"ĠwczeÅĽniej":134099,"ĠÑįÑĤим":134100,"Ġthá»ĭt":134101,"à¸ļัà¸į":134102,"à¸ļัà¸įà¸Ĭี":134103,"ãģļãģ£ãģ¨":134104,"ÑĢин":134105,"ĠswojÄħ":134106,"íķĺëĬĶëį°":134107,"Ġë§Įëĵ¤ìĸ´":134108,"تشÙĥ":134109,"تشÙĥÙĬÙĦ":134110,"ائÙĩ":134111,"Ġ׾פ×Ĺ×ķת":134112,"ãĥĭãĥ¥":134113,"ãĥĭãĥ¥ãĥ¼ãĤ¹":134114,"׼×IJף":134115,"ãģ§ãģįãģŁ":134116,"звон":134117,"ĠstaÅĤ":134118,"×Ĺ×ijרת×Ļ":134119,"ĠأعÙĦÙĨ":134120,"à¹ģà¸ļà¸ļà¸Ļีà¹ī":134121,"بدء":134122,"ãĤģãģŁ":134123,"Ġ×ŀש×ŀ×¢×ķת":134124,"Ġ×ŀש×ŀ×¢×ķת×Ļ":134125,"örü":134126,"Ġhạnh":134127,"zähl":134128,"ĠLý":134129,"Ġ×ij×Ķת":134130,"Ġ×ij×Ķת×IJ×Ŀ":134131,"баÑĢ":134132,"ì¦Ī":134133,"ä»ĬåĽŀãģ®":134134,"Ġyü":134135,"Ġyüks":134136,"Ġyüksel":134137,"ãĤ½ãĥ¼":134138,"ãģĤãĤĮ":134139,"×ª×ľ×ŀ×Ļ×ĵ":134140,"ãģ¤ãģª":134141,"×ij׳×Ļ×Ŀ":134142,"Ġxếp":134143,"ĠмÑĥжÑĩин":134144,"ĠاÙĦÙĥتاب":134145,"׼×ŀ×ķת":134146,"Ġçe":134147,"ĠçeÅŁ":134148,"ĠçeÅŁit":134149,"ĠçeÅŁitli":134150,"×ĵ×Ļר×ķת":134151,"à¸ļุà¸į":134152,"ĠاÙĦØ¥ÙĦÙĥ":134153,"ĠاÙĦØ¥ÙĦÙĥترÙĪ":134154,"ĠاÙĦØ¥ÙĦÙĥترÙĪÙĨÙĬ":134155,"ĠباÙĦإض":134156,"ĠباÙĦإضاÙģØ©":134157,"Ġyönel":134158,"Ġyönelik":134159,"mysÅĤ":134160,"à¸Ķà¹īวยà¸ģาร":134161,"à¸ģารà¸Ĺำ":134162,"овÑĭм":134163,"أزÙħØ©":134164,"æİ¢ãģĹ":134165,"íļ¨":134166,"Ġ×ķ×IJ×Ŀ":134167,"Ġnghiêm":134168,"ÑĪин":134169,"кал":134170,"Ġcrianças":134171,"èĩªåĪĨãģ§":134172,"Ġнай":134173,"ĠнайÑĤи":134174,"ĠSá»ij":134175,"ĠÃ¶ÄŁrenciler":134176,"ãĥ¶æľĪ":134177,"Ñģан":134178,"ĠJá":134179,"ĠkonuÅŁma":134180,"شرط":134181,"ëĪĪ":134182,"arrière":134183,"ضرÙĪرة":134184,"ãĥĶãĥ³":134185,"עשר":134186,"аÑĢÑĮ":134187,"جÙħاع":134188,"Ġdéco":134189,"Ġ×Ļ×Ķ×ķ×ĵ×Ļ":134190,"à¸ŀลาà¸Ķ":134191,"ĠÙĬÙĥÙĨ":134192,"ĠجاÙħعة":134193,"طبÙĤ":134194,"ĠboÅŁ":134195,"×ķ×ķ×IJ":134196,"×ŀ×ĵ×¢":134197,"ק×ij×ķצת":134198,"פ×Ļר":134199,"jÄħcym":134200,"Ùħشا":134201,"ÙħشاÙĥÙĦ":134202,"צפ×ķף":134203,"إست":134204,"×ŀ׼ר":134205,"سÙħع":134206,"Ġкакой":134207,"ÑĤвоÑĢ":134208,"ØŃج":134209,"Ùģرض":134210,"пÑĢавлен":134211,"Ġникак":134212,"Ġmiá»ĩ":134213,"Ġmiá»ĩng":134214,"Ã¼ÃŁ":134215,"иÑĢовал":134216,"׾×ŀ×ķת":134217,"次ãģ®":134218,"ÙĦØ·":134219,"à¸ķัà¸Ļ":134220,"×Ķת×Ĺ×Ļ׾":134221,"ĠfotoÄŁ":134222,"ĠfotoÄŁraf":134223,"طرØŃ":134224,"à¸Ńà¸Ńà¸ģà¹Ħà¸Ľ":134225,"Ġyên":134226,"Ġпок":134227,"ĠпокÑĥп":134228,"ĠпокÑĥпа":134229,"ÑĨÑĥ":134230,"ĠкомпÑĮÑİ":134231,"ĠкомпÑĮÑİÑĤеÑĢ":134232,"ĠاÙĦÙĥرÙĬÙħ":134233,"تصÙħ":134234,"تصÙħÙĬÙħ":134235,"Ġоказа":134236,"Ġzarówn":134237,"Ġzarówno":134238,"ëĮĢì¶ľ":134239,"ãĤ»ãĥ³ãĤ¿ãĥ¼":134240,"ĠjakoÅĽci":134241,"æĤ©":134242,"æĤ©ãģ¿":134243,"Ø£ÙĨÙĪ":134244,"Ø£ÙĨÙĪاع":134245,"ë¹ł":134246,"Ġìłķë§IJ":134247,"Ġkẻ":134248,"ĠÑģайÑĤа":134249,"Ġ×Ķער×ij":134250,"Ùĩز":134251,"presión":134252,"ĠÑģÑĤен":134253,"ãģ£ãģ¦ãĤĭ":134254,"Ġhızlı":134255,"ÐļÐIJ":134256,"×ŀשפ×Ĺת":134257,"ĠÙĨÙĩا":134258,"ĠÙĨÙĩاÙĬØ©":134259,"ãģ¾ãģĦ":134260,"оÑħÑĢан":134261,"รà¹īà¸Ńย":134262,"ลึà¸ģ":134263,"ĠÙĪباÙĦ":134264,"ãĤĤãģ®ãģĮ":134265,"ר׼×Ļ×ij":134266,"ãĤ¤ãĥ¤":134267,"سؤ":134268,"سؤاÙĦ":134269,"ĠÙĦØ£ÙĨÙĩ":134270,"ĠkonuÅŁtu":134271,"ÐļÑĥпиÑĤÑĮ":134272,"Ġש×IJת×Ķ":134273,"ĠÙĪاÙĦس":134274,"ĠmożliwoÅĽci":134275,"Ġprób":134276,"ëĶ°":134277,"ãģ©ãĤĮ":134278,"ĠÐľÐ¸Ð½":134279,"ĠоÑĢганизм":134280,"ãģ«å¯¾ãģĻãĤĭ":134281,"ĠPré":134282,"Ġprivé":134283,"chè":134284,"ãģĦãģŁãģłãģį":134285,"สà¸Ļุà¸ģ":134286,"ajÄħce":134287,"ĠDzi":134288,"ĠDziÄĻki":134289,"ÅĤatw":134290,"rän":134291,"ränk":134292,"æĿ¥ãģŁ":134293,"Ġ×Ķ×Ļ×Ķ×ķ×ĵ×Ļ":134294,"ãĤ¬ãĥ¼":134295,"ĠÑĢад":134296,"ĠÑĢади":134297,"кÑĤив":134298,"Ø£Ùĩد":134299,"Ø£ÙĩداÙģ":134300,"ש×IJ×Ļר":134301,"ãģ¦ãģĦãģªãģĦ":134302,"Ġfrüh":134303,"Ġокол":134304,"Ġоколо":134305,"Ġregião":134306,"ĠÑĩиÑģле":134307,"Ġponiew":134308,"Ġponieważ":134309,"ìĦ¼íĦ°":134310,"Ġbầu":134311,"Ġê·":134312,"Ġê·ľ":134313,"Ġê·ľìłķ":134314,"ĠHòa":134315,"ĠÑĤоÑĤ":134316,"ãĤĤå¤ļãģĦ":134317,"ĠاÙĦإسÙĦاÙħÙĬØ©":134318,"ãģĭãģĦ":134319,"Ñįн":134320,"ĠÑĥказан":134321,"ĠÑĤакое":134322,"ï¼³":134323,"ëĮĢíķĻ":134324,"ĠgeniÅŁ":134325,"ĠاÙĦØ®ÙĬ":134326,"ĠاÙĦØ®ÙĬارات":134327,"ãĤĴè¡ĮãģĨ":134328,"ש×ŀ×Ķ":134329,"ĠLÃłm":134330,"ÙĪÙĨÙĬ":134331,"Ġ×IJ׾×Ļ×ķ":134332,"Äĺ":134333,"à¹Ħมà¹Īสามารà¸ĸ":134334,"人ãģ¨":134335,"برز":134336,"×Ļס×ķ×ĵ":134337,"×Ĵ׾×Ļ":134338,"ĠÙĬÙĨا":134339,"ĠÙĬÙĨاÙĬر":134340,"ĠкаÑĢÑĤин":134341,"Ġtôn":134342,"à¹Ģà¸ģร":134343,"à¸Ħà¸Ķี":134344,"Ġ׾×IJ×ķר×ļ":134345,"ãĤĤãĤīãģĨ":134346,"ãģĭãģĭãĤĭ":134347,"ании":134348,"ĠaraÅŁtırma":134349,"ÙĦاØŃظ":134350,"ãģĦãĤĦ":134351,"ĠTÃłi":134352,"Ġà¸Ļà¸Ńà¸ģà¸Īาà¸ģ":134353,"Ġà¸Ļà¸Ńà¸ģà¸Īาà¸ģà¸Ļีà¹ī":134354,"ĠÄIJảng":134355,"ãģ£ãģ¦ãģįãģŁ":134356,"Ġà¸ĭึà¹Īà¸ĩà¹Ģà¸Ľà¹ĩà¸Ļ":134357,"Ġtả":134358,"ĠmożliwoÅĽÄĩ":134359,"ĠSản":134360,"ĠÄ°ki":134361,"Ġcắt":134362,"سأÙĦ":134363,"Ġbakım":134364,"شب":134365,"à¸ķีà¹ī":134366,"à¸ŀยาย":134367,"à¸ŀยายาม":134368,"à¸ªà¸±à¸Ľ":134369,"à¸ªà¸±à¸Ľà¸Ķา":134370,"à¸ªà¸±à¸Ľà¸Ķาหà¹Į":134371,"ë°Ģ":134372,"еÑĢÑĭ":134373,"Ġcánh":134374,"Ġthuế":134375,"تبع":134376,"ãģ«åħ¥ãĤĮ":134377,"ÑİÑģÑĮ":134378,"íļĮìĿĺ":134379,"ç°¡åį":134380,"ç°¡åįĺ":134381,"ç°¡åįĺãģ«":134382,"Ġtrúc":134383,"ĠاÙĦÙĥÙĪÙĬ":134384,"ĠاÙĦÙĥÙĪÙĬت":134385,"ãĤıãģijãģ§ãģĻ":134386,"ĠÑģвоб":134387,"ĠÑģвобод":134388,"ĠÑĥÑĩаÑģÑĤник":134389,"สิà¹īà¸Ļ":134390,"ĠпÑĢоÑĦеÑģÑģиона":134391,"ĠпÑĢоÑĦеÑģÑģионалÑĮн":134392,"ÑģпоÑĢ":134393,"×Ĺ×ķ×ij×Ķ":134394,"ÙħعÙĨÙī":134395,"ĠاÙĦÙģترة":134396,"สูà¸ĩสุà¸Ķ":134397,"ãĤıãģļ":134398,"ĠÄijè":134399,"ĠÄijèn":134400,"æ¯Ķãģ¹":134401,"าà¸ĺิ":134402,"Ġmożemy":134403,"à¹ģà¸ĭ":134404,"à¸Īะà¹Ħมà¹Ī":134405,"Ġsắp":134406,"ÐļÐŀ":134407,"Ġpráctica":134408,"ÙĪÙĥاÙĦØ©":134409,"è¾¼ãĤĵãģ§":134410,"ológica":134411,"ĠеÑī":134412,"ĠеÑīÑij":134413,"تعدÙĬÙĦ":134414,"ĠØ£Ùĥد":134415,"Ġצר×Ļ׼":134416,"Ġצר×Ļ׼×Ļ×Ŀ":134417,"Ø«Ùħ":134418,"ĠкÑĢÑĥ":134419,"ĠкÑĢÑĥп":134420,"×ij×Ļק×ķרת":134421,"Ġì¡°ê¸Ī":134422,"ãģ¨ãģįãģ¯":134423,"Ġbạc":134424,"ĠÑĢаÑģпол":134425,"ĠÑĢаÑģполож":134426,"ĠÑĢаÑģположен":134427,"زÙĬÙĨ":134428,"ĠÐļÑĢоме":134429,"ĠاÙĦÙĨظر":134430,"×Ķ×ķ×ĵ":134431,"ĠاÙĦسبت":134432,"ãģ¨æĢĿãģĦ":134433,"ĠpaÅĦst":134434,"ĠpaÅĦstw":134435,"ĠÙĦÙĬست":134436,"ĠбÑĥдÑĥ":134437,"à¸Ĺัà¸Ļà¸Ĺี":134438,"ราม":134439,"ØŃصÙĪÙĦ":134440,"ãģĹãģ¦ãģıãĤĮãĤĭ":134441,"ĠاÙĦإسرائÙĬÙĦ":134442,"ĠاÙĦإسرائÙĬÙĦÙĬ":134443,"ãģĵãĤĮãģ¾ãģ§":134444,"ìĤ¬ë¥¼":134445,"Ġsürü":134446,"à¹Ģวà¸Ńรà¹Į":134447,"à¹Ģà¸ĭà¸Ńรà¹Į":134448,"Ġutilisé":134449,"ĠÑģиÑģÑĤема":134450,"Ġdwó":134451,"Ġdwóch":134452,"Ġpróprio":134453,"Ġëĵ±ìĿĦ":134454,"arrêt":134455,"ĠЧа":134456,"×IJ×ŀ׳×ķת":134457,"عارض":134458,"à¹Ģà¸ģมสà¹Į":134459,"Ġ׾×Ķ×ij×Ļף":134460,"Ġ׾×ij×Ĺ":134461,"Ġ׾×ij×Ĺ×ķר":134462,"สาà¸Ĥา":134463,"ĠÐľÐ¾Ñģкве":134464,"بعد":134465,"ĠاÙĦÙĤرار":134466,"ĠÄIJá»ĭa":134467,"Ġ×Ĺ×Ĵ":134468,"Ùģتر":134469,"ÙĪÙĨØ©":134470,"Ġ×Ķ×ĸ×IJת":134471,"å¸Ĥãģ®":134472,"ãģ»ãģĹãģĦ":134473,"Ġ×ij×¢×Ļר":134474,"ĠÑĤепеÑĢÑĮ":134475,"ìĬµëĭĪê¹Į":134476,"à¹Ħมà¹Īว":134477,"à¹Ħมà¹Īวà¹Īา":134478,"à¹Ħมà¹Īวà¹Īาà¸Īะ":134479,"×ŀ×IJ×Ķ":134480,"æĥħåł±":134481,"æĥħåł±ãĤĴ":134482,"غÙĨ":134483,"ĠпоÑı":134484,"ĠпоÑıви":134485,"éģİãģĶ":134486,"تشغ":134487,"تشغÙĬÙĦ":134488,"вел":134489,"Ġ×Ĺ×ŀ":134490,"ãģ¨ãģªãĤĬãģ¾ãģĻ":134491,"ĠraÄŁ":134492,"ĠraÄŁmen":134493,"ãģĭãģ©ãģĨ":134494,"ãģĭãģ©ãģĨãģĭ":134495,"енко":134496,"ì§Ģê³ł":134497,"Ġ×IJ׾×Ļ×Ķ":134498,"ĠØ£ÙĦ":134499,"à¸Īำหà¸Ļ":134500,"à¸Īำหà¸Ļà¹Īาย":134501,"nızı":134502,"Ġ׾ק×Ĺת":134503,"Ø£ÙĩÙħ":134504,"Ø£ÙĩÙħÙĬØ©":134505,"تغÙĬر":134506,"ש×Ĺר":134507,"ס×ķפר":134508,"×ĵ×Ļר":134509,"èī¯ãģĭãģ£ãģŁ":134510,"×ŀ׾×Ĺ×ŀ×Ķ":134511,"ÑģÑĤвие":134512,"ÑĤÑĢаÑĤ":134513,"ĠاÙĦأخ":134514,"ĠاÙĦأخÙĬرة":134515,"ĠاÙĦØŃصÙĪÙĦ":134516,"Ġcrédito":134517,"צ×Ļ×¢":134518,"ãĥ¬ãĥĻãĥ«":134519,"برÙĬ":134520,"ëIJIJ":134521,"ãģłãģ£ãģ¦":134522,"ĠrealtÃł":134523,"سÙģر":134524,"×ķ׳×ķ":134525,"×Ĵ×ķ×ĵ":134526,"×Ĵ×ķ×ĵ׾":134527,"ฮา":134528,"ãģĹãģ¦ãģĬãĤĬãģ¾ãģĻ":134529,"ĠgÃł":134530,"Ġ׾×ijצע":134531,"å¼ķè¶ĬãģĹ":134532,"Ġ×ŀ×Ļ׾×Ļ":134533,"Ġ×ŀ×Ļ׾×Ļ×ķף":134534,"Ùħدر":134535,"Ùħدرسة":134536,"פ×ķ×ĺ":134537,"à¸Ļà¹īำมัà¸Ļ":134538,"ëģĿ":134539,"عÙĥس":134540,"ĠÙĤض":134541,"ĠÑĢÑĭб":134542,"خطط":134543,"×ŀ×ķס×ĵ":134544,"Ġ׼׾׾×Ļ":134545,"ĠкоÑĤоÑĢое":134546,"צ×Ļ×ķף":134547,"ĠмеÑģÑĤа":134548,"ãģĭãģ¤":134549,"гÑĢÑĥпп":134550,"׾×Ļ׾":134551,"ת×ķ×IJר":134552,"ë³µì§Ģ":134553,"à¹ģà¸ľà¹Īà¸Ļ":134554,"Ġ×ijעת":134555,"æĻĤéĸĵãĤĴ":134556,"ï¼£":134557,"ãģ¨ãģĦãģĨãģĵãģ¨ãģ§":134558,"Ġ׾×Ķק":134559,"Ġ׾×ĸ×Ķ":134560,"ĠìłĢëĬĶ":134561,"ĠاÙĦإرÙĩاب":134562,"ĠìŀĪëĬĶëį°":134563,"ĠÑĤогда":134564,"Ġ×Ķצ×Ļ":134565,"×ķ׾×ĺ":134566,"Ġרפ×ķ×IJ×Ļ":134567,"ãģĵãģ¨ãģ§ãģĻ":134568,"ĠÄijÃŃch":134569,"ØŃÙĬا":134570,"Ġ×Ķ×ŀש×Ĺק":134571,"ãģľãģ²":134572,"Ġ×ŀ×IJפשר":134573,"ãģ¿ãģ¾ãģĹãģŁ":134574,"ĠاÙĦØ£ÙħÙĬرÙĥÙĬ":134575,"ÙħجتÙħع":134576,"Ġساب":134577,"ĠسابÙĤ":134578,"׼×Ļ׾":134579,"Ế":134580,"ãĥªãĤ¹ãĥĪ":134581,"Ġìĥ":134582,"ĠìĥĪ":134583,"ĠìĥĪë¡ľ":134584,"ĠìĥĪë¡ľìļ´":134585,"ĠDá»ĭch":134586,"à¹Ģหมาะสม":134587,"ĠاÙĦÙĨبÙĬ":134588,"׾׾":134589,"ÙĨع":134590,"Ðĵлав":134591,"ÐĵлавнаÑı":134592,"Ùħرض":134593,"Ġ×ķ×ĵ":134594,"تÙĤÙĬ":134595,"تÙĤÙĬÙĬÙħ":134596,"Ġbảng":134597,"ĠÙģÙĤاÙĦ":134598,"×¢×ŀ×Ļ":134599,"дÑĢа":134600,"Ġsuá»ijt":134601,"سرعة":134602,"Ġcá»Ń":134603,"Ġ×Ķ×Ļ×Ĺ×Ļ×ĵ":134604,"سعÙĬد":134605,"à¸Ńาà¸Ĭีà¸ŀ":134606,"ĠسÙĪاء":134607,"ãĤ½ãĥķãĥĪ":134608,"ĠлиÑĩно":134609,"ĠÐļоÑĢ":134610,"اÙĩتÙħ":134611,"اÙĩتÙħاÙħ":134612,"à¸Ńà¸Ķี":134613,"à¸Ńà¸Ķีà¸ķ":134614,"ãģIJãĤīãģĦ":134615,"Ġihtiya":134616,"Ġihtiyaç":134617,"ãģ¾ãģ§ãģ®":134618,"ìĭľìĬ¤":134619,"ìĭľìĬ¤íħľ":134620,"ÑĢÑĥÑĪ":134621,"ãĤĦãģ£ãģ±":134622,"ãĤĦãģ£ãģ±ãĤĬ":134623,"кеÑĢ":134624,"Ġży":134625,"Ġżyw":134626,"клон":134627,"Ġlượt":134628,"þ":134629,"даÑĩи":134630,"türk":134631,"غÙĪ":134632,"ĠигÑĢок":134633,"Ġphê":134634,"Ġ×©×¢×ľ":134635,"ĠاÙĦÙħدÙĨÙĬ":134636,"ĠìŬ룬ë¶Ħ":134637,"ער×Ļ×Ŀ":134638,"ÑħодÑıÑĤ":134639,"Ġxứ":134640,"ÐĹа":134641,"ĠÙģرص":134642,"à¸Īะà¸Ĺำà¹ĥหà¹ī":134643,"íģ´":134644,"×¢×ij×ķר":134645,"à¹Ģหลà¹Īาà¸Ļีà¹ī":134646,"èĢĥãģĪãĤĭ":134647,"ÑĢеÑģÑĤ":134648,"ннÑĭй":134649,"Ġcầm":134650,"داخÙĦ":134651,"ĠÙħÙĦÙĬار":134652,"ĠÐIJл":134653,"ĠвÑĢемен":134654,"à¸Ĭà¹Īวยà¹ĥหà¹ī":134655,"ר×Ļ×ķת":134656,"ëĵ¯":134657,"飲ãģ¿":134658,"׳׾":134659,"שתף":134660,"ĠاÙĦسعÙĪدÙĬ":134661,"uÃŁ":134662,"ìĿ¸ëį°":134663,"ĠìĿ¼ë°ĺ":134664,"ÅĤÄĻ":134665,"Ġmá»iji":134666,"×ŀ×Ļ׳":134667,"ĠاÙĦأطÙģاÙĦ":134668,"Ġçıkan":134669,"école":134670,"ק×Ļש":134671,"ק×Ļש×ķר":134672,"ĠоÑģÑĥÑīеÑģÑĤв":134673,"ĠоÑģÑĥÑīеÑģÑĤвлÑı":134674,"×ij×IJר":134675,"à¹Ħà¸Ľà¸Ķà¹īวย":134676,"Ġ×¢×ķ׾×Ķ":134677,"à¸ģà¹ĩà¹Ħมà¹Ī":134678,"ãĥ¢ãĥĩ":134679,"ãĥ¢ãĥĩãĥ«":134680,"تØŃÙĪÙĦ":134681,"Ġодного":134682,"ת×Ĺ×Ļ×ľ×ª":134683,"Ġتخ":134684,"Ġchcia":134685,"ĠchciaÅĤ":134686,"ãĥIJãĥ³":134687,"èĢħãģ¯":134688,"ĠÙħØŃÙĦ":134689,"Ñģлож":134690,"Ñģложн":134691,"ĠtÄĻ":134692,"Ġçıkt":134693,"Ġçıktı":134694,"ĠCÆ¡":134695,"à¹Ħà¸Ķà¹īà¹Ģลย":134696,"ırken":134697,"à¹Ģà¸Ĥà¹īาสูà¹Ī":134698,"ÙħØŃÙĥ":134699,"ÙħØŃÙĥÙħØ©":134700,"à¸Ħุà¹īม":134701,"à¸Ļà¹Īาà¸Īะ":134702,"лÑİд":134703,"деÑģÑı":134704,"деÑģÑıÑĤ":134705,"ĠлÑİбой":134706,"تØŃرÙĬر":134707,"צע×ĵ":134708,"ĠеÑij":134709,"ĠاÙĦØŃÙĥÙħ":134710,"ĠصباØŃ":134711,"à¹Ģà¸ļà¸Ńรà¹Į":134712,"Ġróżnych":134713,"гиб":134714,"ĠÑģоÑĤ":134715,"ĠÑģоÑĤÑĢÑĥд":134716,"ĠÑģоÑĤÑĢÑĥдник":134717,"ĠобÑĬем":134718,"פ×ĺר":134719,"ãģĻãģĶãģı":134720,"ãģ«éĸ¢ãģĹãģ¦":134721,"вол":134722,"Ø«ÙħاÙĨ":134723,"Ġdần":134724,"æĬľ":134725,"æĬľãģij":134726,"Ġעש":134727,"Ġעש×ķ×Ļ":134728,"ס×ķף":134729,"ãģªãģ®ãģ§ãģĻ":134730,"ãģ¯ãģ©ãģĨ":134731,"×ŀער×ij":134732,"ï¼°":134733,"Ùħصر":134734,"ÙħÙĨاسب":134735,"ÙħÙĨاسبة":134736,"ä¸Ĭãģ®":134737,"×IJ×Ļש×ķר":134738,"ĠìĦ¤ì¹ĺ":134739,"×ŀ×ĵ×Ļ׳×ķת":134740,"×ŀרת":134741,"ãĤĭãģ®ãģĮ":134742,"دÙİ":134743,"ĠاÙĦشرÙĥات":134744,"ìĭľê°Ħ":134745,"ĠÑĢеÑĪение":134746,"ãģĻãĤĭãģ®ãģ¯":134747,"ĠìŀIJìĭłìĿĺ":134748,"׾×ŀ×ķ":134749,"ãģ¨ãģĵãĤįãģ§":134750,"Ġקצר":134751,"Ġmãi":134752,"Ġkültür":134753,"ãĥ©ãĤ¤ãĥĸ":134754,"à¸ľà¸¹à¹īหà¸įิà¸ĩ":134755,"æĻĤéĸĵãģĮ":134756,"клÑİÑĩи":134757,"diÄŁiniz":134758,"มาà¸ģà¹Ĩ":134759,"تØŃÙħÙĦ":134760,"Ġhạt":134761,"ãĤ¦ãĤ£":134762,"пле":134763,"×ŀ׾×IJ":134764,"ÅĤó":134765,"Ġgá»ijc":134766,"Ġ×IJ×ķ×ĵ×ķת":134767,"หวาà¸Ļ":134768,"ĠاÙĦÙĪز":134769,"ĠاÙĦÙĪزراء":134770,"ëĵ¤ê³¼":134771,"ĠصØŃ":134772,"ĠصØŃÙĬÙģØ©":134773,"Ġмм":134774,"تدخÙĦ":134775,"Ġpersönlich":134776,"ĠزÙĬ":134777,"ĠزÙĬادة":134778,"ãĤ·ãĤ¢":134779,"Ġngắn":134780,"à¸Ħลิà¸ģ":134781,"Ġsông":134782,"Ġtüket":134783,"ÑįÑĦÑĦ":134784,"ÑįÑĦÑĦекÑĤ":134785,"ש×Ļ×ij":134786,"Ġاعت":134787,"تض":134788,"تضÙħÙĨ":134789,"ĠاÙĦÙħشرÙĪع":134790,"Ġprodução":134791,"ĠпÑĢименÑı":134792,"ниÑĨÑĭ":134793,"주ëĬĶ":134794,"رÙı":134795,"ĠmÆ¡":134796,"Ġhayatı":134797,"ëŁ½":134798,"Ġücret":134799,"Ġyanında":134800,"Ġprática":134801,"×ij×Ļק×ķר":134802,"ÃľN":134803,"ÑģоÑĤ":134804,"ãĤıãģijãģ§":134805,"Ġдолго":134806,"×ª×Ľ×ķ":134807,"ĠìķĦëĭĮ":134808,"ëį°ìĿ´":134809,"Ġçiz":134810,"ĠchoÄĩ":134811,"Ġ×Ķ×Ļת":134812,"Ġ×Ķ×Ļתר":134813,"Ġsoát":134814,"׼×ij×ĵ":134815,"à¹Ģลà¹Īา":134816,"ĠдеÑĢ":134817,"ĠдеÑĢев":134818,"ãĤĴåħ¥ãĤĮ":134819,"×Ĺ×ķס":134820,"×Ĺ×ķסר":134821,"جÙĬÙĨ":134822,"tón":134823,"onné":134824,"ĠполноÑģÑĤÑĮÑİ":134825,"人ãģŁãģ¡":134826,"Ġprêt":134827,"본":134828,"Ġdécembre":134829,"cılar":134830,"Ġתת":134831,"Ġê²½ìļ°ìĹIJëĬĶ":134832,"ÙĪعد":134833,"è¦ĭãĤĭ":134834,"วิà¸Īัย":134835,"ë¶Ī":134836,"زÙĪا":134837,"زÙĪاج":134838,"dì":134839,"ãģ§ãģĻãĤĪ":134840,"Ġводо":134841,"ĠÙĬÙĪجد":134842,"ÑģоÑģÑĤоÑı":134843,"ÐŀС":134844,"ĠÄIJó":134845,"×Ĺפש":134846,"Ġצ×Ļ×ij×ķר":134847,"ĠاÙĦÙĤØ·":134848,"ĠاÙĦÙĤطاع":134849,"ĠимеÑİÑĤ":134850,"ĠpháºŃn":134851,"×Ľ×¡×¤×Ļ":134852,"полниÑĤелÑĮ":134853,"éĻIJãĤĬ":134854,"ĠÑģÑĢав":134855,"ĠÑģÑĢавн":134856,"ÙħاÙĦÙĥ":134857,"×ĵר×ķ×Ŀ":134858,"çļĨãģķãĤĵ":134859,"ØŃÙĤÙĤ":134860,"à¹ģหลà¹Īà¸ĩ":134861,"ĠاÙĦرسÙħÙĬ":134862,"оÑĩки":134863,"×ĺ×ij×Ĺ":134864,"Ġcanlı":134865,"Ġ׾׾":134866,"Ġ׾׾×ŀ×ķ×ĵ":134867,"×ŀ×ij×ķ":134868,"×ª×Ľ":134869,"×ª×Ľ×ł×Ļת":134870,"ĠاÙĦÙħشار":134871,"ĠاÙĦÙħشارÙĥØ©":134872,"Ä°Åŀ":134873,"ĠسÙĬاسÙĬ":134874,"волÑĮ":134875,"ĠÑģпÑĢав":134876,"æĿ¥ãģ¦":134877,"פ×ķר×ķ×Ŀ":134878,"สำà¹Ģรà¹ĩ":134879,"สำà¹Ģรà¹ĩà¸Ī":134880,"ĠÅŁÃ¶yle":134881,"ĠzostaÅĤa":134882,"ĠHü":134883,"ר×ķש":134884,"دÙĦÙĬÙĦ":134885,"ÑĢид":134886,"שף":134887,"×ŀק×ķר":134888,"ĠÑĥÑĩ":134889,"ĠÑĥÑĩеб":134890,"ĠÑįÑĤа":134891,"кова":134892,"à¸ķà¸Ļà¹Ģà¸Ńà¸ĩ":134893,"ÙĨÙIJ":134894,"à¸Ńีà¸ģà¸Ħรัà¹īà¸ĩ":134895,"ระà¸ļุ":134896,"Ġdữ":134897,"ĠاÙĦØŃاÙĦÙĬ":134898,"׼×ķ׼":134899,"׼×ķ׼×ij":134900,"Ġ×ŀ×IJשר":134901,"Ġtrụ":134902,"ÑĤелем":134903,"Ġвли":134904,"ĠвлиÑı":134905,"Ġש×IJת×Ŀ":134906,"Ġuwag":134907,"ĠuwagÄĻ":134908,"×ĺ×Ļת":134909,"×IJ×ĵ×Ŀ":134910,"à¸Ķุ":134911,"Ġ×Ķ×IJ׾×Ķ":134912,"ĠkarÄ±ÅŁ":134913,"ĠÄIJá»iji":134914,"даÑİÑĤ":134915,"ãģªãģ®ãģ«":134916,"Äħcych":134917,"à¹Ģà¸Ļà¹īà¸Ļ":134918,"ãģĹãģ¦ãģĹãģ¾ãģĨ":134919,"intérieur":134920,"ĠfÃŃsica":134921,"ĠÐŁÐ¾Ð»":134922,"ãģĹãģķ":134923,"à¸Ĺำà¹Ħม":134924,"ĠLâm":134925,"ĠاÙĦÙħسÙĦÙħ":134926,"ĠاÙĦÙħسÙĦÙħÙĬÙĨ":134927,"صØŃØ©":134928,"ìĹĦ":134929,"à¹Ģà¸Ķà¹ĩà¸Ķ":134930,"ĠÑĥÑĩеÑĤ":134931,"âÌģ":134932,"ĠبÙĦا":134933,"ĠاÙĦاجتÙħاعÙĬ":134934,"פרס×Ŀ":134935,"ãĥķãĥ©":134936,"ĠÐļогда":134937,"mieÅĽci":134938,"ĠبÙĬÙĨÙħا":134939,"Ġ×ŀ×IJ×ŀר×Ļ×Ŀ":134940,"Ġ×ij×IJ×ĸ×ķר":134941,"×ķש×Ļ×Ŀ":134942,"ĠÑģдела":134943,"entrée":134944,"à¹Ģà¸Ħà¹īา":134945,"Ñĥгл":134946,"ĠاÙĦÙģÙĨÙĬ":134947,"ĠÐĴоÑĤ":134948,"à¸Ĺีà¹Īมา":134949,"×ķצ×Ĵ":134950,"ÙĤدرة":134951,"Ġ목":134952,"Ġ목ìłģ":134953,"íıīê°Ģ":134954,"ĠاÙĦأربع":134955,"ĠاÙĦأربعاء":134956,"פס×Ļק":134957,"ĠÑıвлÑıÑİÑĤÑģÑı":134958,"بÙĪÙĨ":134959,"ì°¾":134960,"×ŀ×¢×¨×Ľ":134961,"×ŀ×¢×¨×Ľ×ķת":134962,"ãĤ·ãĤ§":134963,"ĠباÙĦØ£":134964,"íĸĪëįĺ":134965,"ĠاÙĦبرÙĨاÙħج":134966,"ĠاÙĦØ£ØŃد":134967,"ĠmÅ©":134968,"ĠmÅ©i":134969,"паÑĤ":134970,"بث":134971,"ĠÑĨенÑĭ":134972,"Ġ×ij×ª×ľ":134973,"è¨ĢãĤıãĤĮ":134974,"ĠاÙĦÙħجاÙĦ":134975,"ĠìĦ¸ìĥģ":134976,"Ġ×Ĵ×ķפ":134977,"ĠнаÑĪей":134978,"ĠкомпаниÑı":134979,"бин":134980,"ölü":134981,"×Ļ×Ļ×ĺ":134982,"Ġ×ŀספ×Ļק":134983,"ยัà¸ĩà¸Ħà¸ĩ":134984,"ĠЧи":134985,"ĠанÑĤи":134986,"ĠÑģÑĢеди":134987,"สà¹Īวà¸Ļà¹ĥหà¸įà¹Ī":134988,"оÑĩка":134989,"íĬ¹ë³Ħ":134990,"วà¹Īาà¸ĩ":134991,"гоÑĢод":134992,"باÙĥ":134993,"à¹Ģสีà¹Īย":134994,"à¹Ģสีà¹Īยà¸ĩ":134995,"ãĤĤãĤīãģĦ":134996,"ק×ķ×Ŀ":134997,"ãģĽãģļ":134998,"ĠاÙĦÙĤاÙĩرة":134999,"Ġ×ij׼×ļ":135000,"ÙħشارÙĬع":135001,"باØŃØ«":135002,"ĠпоÑĩ":135003,"ĠпоÑĩÑĤи":135004,"ĠÑĦоÑĢма":135005,"SÄ°":135006,"Ġ×ŀצ×Ļ×¢":135007,"ลื":135008,"ลืม":135009,"ĠÑĤеÑĢ":135010,"ĠÑĤеÑĢÑĢиÑĤоÑĢ":135011,"ĠÑĤеÑĢÑĢиÑĤоÑĢии":135012,"ĠвмеÑģÑĤ":135013,"ĠвмеÑģÑĤе":135014,"dıkları":135015,"opération":135016,"à¹Ĥห":135017,"صدÙĬ":135018,"صدÙĬÙĤ":135019,"íĸīìłķ":135020,"تجا":135021,"تجاÙĪز":135022,"Ġsuç":135023,"Ġarty":135024,"Ġartyku":135025,"ĠartykuÅĤ":135026,"ãĤ·ãĥ§ãĥĥãĥĹ":135027,"שפ":135028,"שפ×Ļ×¢":135029,"Ġ×Ķש×Ļר×ķת":135030,"à¹ģà¸ĸม":135031,"ë¸Ķ":135032,"ĠukÅĤad":135033,"Ġ×ķ׼×Ļ":135034,"หลาà¸ģ":135035,"หลาà¸ģหลาย":135036,"æĸ¹ãĤĤ":135037,"Ġpodróż":135038,"ĠEÄŁer":135039,"ĠкомнаÑĤ":135040,"ĠÑģамÑĭÑħ":135041,"ĠвкÑĥÑģ":135042,"беж":135043,"Ġ×ijק×ķ":135044,"æİĽãģij":135045,"ãģ¿ãĤĭãģ¨":135046,"ĠiliÅŁkin":135047,"ĠÙĬعÙħÙĦ":135048,"ĠподаÑĢ":135049,"Ġyazılı":135050,"ãĤĴå¾Ĺ":135051,"ĠwystÄĻp":135052,"à¸Ĺีà¹Īà¹ĥà¸Ĭà¹ī":135053,"ØŃادث":135054,"ÙĪÙĬد":135055,"кÑĥлÑĮÑĤ":135056,"кÑĥлÑĮÑĤÑĥÑĢ":135057,"à¸ģารà¹ģà¸Ĥà¹Īà¸ĩ":135058,"à¸ģารà¹ģà¸Ĥà¹Īà¸ĩà¸Ĥ":135059,"à¸ģารà¹ģà¸Ĥà¹Īà¸ĩà¸Ĥัà¸Ļ":135060,"ÙħÙĪظ":135061,"ÙħÙĪظÙģ":135062,"ÙĬÙħÙĬ":135063,"ãĤĵãģ§ãģĻãģĮ":135064,"diÄŁim":135065,"diÄŁimiz":135066,"ĠÐŁÐµÑĢ":135067,"ĠÐŁÐµÑĢв":135068,"Ġmão":135069,"ĠÑģез":135070,"ĠÑģезон":135071,"Ġ×Ķ×ŀ×¢":135072,"ÙħجÙħÙĪعة":135073,"ĠинÑĦоÑĢмаÑĨии":135074,"iếc":135075,"ãng":135076,"ĠÄijấy":135077,"ãģĶç´":135078,"ãģĶç´¹":135079,"ãģĶç´¹ä»ĭ":135080,"Ġadım":135081,"à¹Ħหล":135082,"ĠпÑĢакÑĤи":135083,"ĠпÑĢакÑĤиÑĩ":135084,"ĠпÑĢакÑĤиÑĩеÑģ":135085,"ĠпÑĢакÑĤиÑĩеÑģки":135086,"ĠاÙĦÙĨÙģس":135087,"ĠÑĢабоÑĤе":135088,"ÙĦÙĬÙģ":135089,"ĠاÙĦجÙĨÙĪب":135090,"ĠводÑĭ":135091,"ì¹Ļ":135092,"ĠмиÑĢа":135093,"ĠÄijừng":135094,"ĠпÑĢоÑĤиво":135095,"ĠÑģÑĤÑĢанÑĭ":135096,"ลู":135097,"ìĤ¶":135098,"kreÅĽl":135099,"Ġbulund":135100,"ĠbulunduÄŁu":135101,"à¹ģสà¸Ļ":135102,"ãĤ±ãĤ¢":135103,"ת×Ĺ×ķ×ŀ×Ļ":135104,"ר׼×Ķ":135105,"Ġ׾ק×ķ×Ĺ":135106,"Ġ׾ק×ķ×Ĺ×ķת":135107,"Ġ×Ľ×ª×ķ×ijת":135108,"ĠÙĦÙĥÙħ":135109,"بشر":135110,"ĠrÃłng":135111,"Ġ×ŀ×Ķ×ŀ":135112,"Ġ×IJ×Ĺר×ķת":135113,"Ġбон":135114,"ĠбонÑĥÑģ":135115,"ï½Ĺ":135116,"à¹ģยà¸ģ":135117,"ãģĤãģªãģŁãģ®":135118,"ĠÑĥÑĩаÑģÑĤие":135119,"ĠEyl":135120,"ĠEylül":135121,"ĠçalÄ±ÅŁmaları":135122,"خطر":135123,"ìĿ½":135124,"à¸ģารà¹ĥà¸Ĭà¹īà¸ĩาà¸Ļ":135125,"Ġанализ":135126,"תק×ij׾":135127,"нием":135128,"ĠÄ°ns":135129,"ĠÄ°nsan":135130,"ĠبÙĪاس":135131,"ĠبÙĪاسطة":135132,"Ġ×ł×Ľ×ł×¡":135133,"Ġ×Ķ×ŀ×Ļ×ĵ×¢":135134,"Ġço":135135,"ĠçoÄŁu":135136,"á»ĺ":135137,"ĠêµŃ민":135138,"ãĤĤãģĦãģĦ":135139,"Ġ׼׾×Ļ":135140,"ĠÑģÑĢедне":135141,"gÅĤo":135142,"gÅĤoÅĽ":135143,"Ġnegó":135144,"Ġnegócio":135145,"ĠÑĢегиÑģÑĤ":135146,"ĠÑĢегиÑģÑĤÑĢа":135147,"ĠÑĢегиÑģÑĤÑĢаÑĨии":135148,"Ġtrá»ĵng":135149,"ĠпÑĢÑı":135150,"ĠпÑĢÑıмо":135151,"ëłĪìĿ´":135152,"Ġkém":135153,"кле":135154,"à¸Ļำมา":135155,"ĠÑĦин":135156,"ĠÑĦинанÑģ":135157,"ĠÑĦинанÑģов":135158,"Ġkiá»ĩm":135159,"ยัà¸ĩà¹Ħ":135160,"ยัà¸ĩà¹Ħà¸ĩ":135161,"ยิà¸ĩ":135162,"à¹Ĥà¸Ľ":135163,"ĠполÑĥÑĩил":135164,"×Ļ×ĸ×Ŀ":135165,"à¹ģละà¸Ħวาม":135166,"ĠвообÑīе":135167,"صÙĬر":135168,"ãĥıãĥ³":135169,"ĠاÙĦÙĤاد":135170,"ĠاÙĦÙĤادÙħ":135171,"ĠبدÙĪÙĨ":135172,"عظÙħ":135173,"×ª×ł×ķ×¢":135174,"×ª×ł×ķ×¢×Ķ":135175,"Ø£ÙħÙĦ":135176,"ãģķãģĪ":135177,"ÑĤем":135178,"ÑĤемпеÑĢ":135179,"ÑĤемпеÑĢаÑĤÑĥÑĢ":135180,"Ġ׾×Ļצ×ķר":135181,"ĠrÄĻk":135182,"رسÙĦ":135183,"ìŀIJ를":135184,"Ġ×Ļצ×Ļרת":135185,"ÙĨبÙĬ":135186,"ÑĩнаÑı":135187,"تØŃÙĦÙĬÙĦ":135188,"Ġмик":135189,"ĠмикÑĢо":135190,"ĠSöz":135191,"Ġforça":135192,"Ñģон":135193,"ĠاÙĦعرا":135194,"ĠاÙĦعراÙĤÙĬ":135195,"ĠHá»ĵng":135196,"ãģĻãĤĭãģŁãĤģãģ«":135197,"à¸Ĺีà¹Īà¸Ńยูà¹Ī":135198,"Ġ×ķ×IJ×£":135199,"صÙĬد":135200,"ĠìķĬê³ł":135201,"รัà¸ĩ":135202,"ĠاÙĦتÙĪاصÙĦ":135203,"à¹Ģมà¸ķร":135204,"ÑĥÑģÑĤÑĢой":135205,"ÑĥÑģÑĤÑĢойÑģÑĤв":135206,"mıyor":135207,"ĠباسÙħ":135208,"Ġ×ķ׼×ķ":135209,"ĠGül":135210,"á»IJ":135211,"Ãītat":135212,"غاÙĦ":135213,"Ø¥ÙĨØ´":135214,"Ø¥ÙĨشاء":135215,"TÄ°":135216,"à¸Ĥà¹īาม":135217,"Ġtroch":135218,"ĠtrochÄĻ":135219,"إص":135220,"إصابة":135221,"ĠثاÙĨÙĬ":135222,"ĠاÙĦصØŃØ©":135223,"Ġ×ĸ×Ķ×ķ":135224,"jÄħcej":135225,"ãĥĢãĥ³":135226,"ìĿ¸ìĿ´":135227,"ĠволоÑģ":135228,"ëIJĺë©´":135229,"ĠzakÅĤad":135230,"ãģĻãģĵãģ¨":135231,"以ä¸Ĭãģ®":135232,"Ġ×Ķ×ŀק×ķ×Ŀ":135233,"ÙħشاÙĩ":135234,"ÙħشاÙĩدة":135235,"Ñĩив":135236,"بش":135237,"ยà¹īาย":135238,"Ġsürdür":135239,"ĠNẵ":135240,"ĠNẵng":135241,"ĠигÑĢаÑĤÑĮ":135242,"Ġê·¸ëŁ¬ë©´":135243,"ãĥķãĥ«":135244,"ลà¹Īะ":135245,"Ġtendrá":135246,"ĠbÃły":135247,"à¹Ģà¸Ľà¹ĩà¸Ļà¸ľà¸¹à¹ī":135248,"Ġoko":135249,"ĠokoÅĤo":135250,"wÅĤa":135251,"wÅĤaÅĽci":135252,"wÅĤaÅĽciw":135253,"æĢĿãĤı":135254,"ĠYaÅŁ":135255,"ĠBá»ĩnh":135256,"íıŃ":135257,"بÙĬد":135258,"קרף":135259,"à¹Ģศร":135260,"à¹Ģศรษ":135261,"à¹Ģศรษà¸IJ":135262,"à¹Ģศรษà¸IJà¸ģิà¸Ī":135263,"ĠاÙĦØ£ÙĪرÙĪ":135264,"ĠاÙĦØ£ÙĪرÙĪبÙĬ":135265,"fläche":135266,"ä¹ĹãĤĬ":135267,"Ġbá»ģn":135268,"Ùĩب":135269,"æľĢãĤĤ":135270,"Ġsaç":135271,"à¸Ńำà¹Ģà¸ł":135272,"à¸Ńำà¹Ģà¸łà¸Ń":135273,"Ġأج":135274,"ĠاÙĦداخÙĦ":135275,"ĠاÙĦداخÙĦÙĬØ©":135276,"×ĺ×ķ×ij":135277,"ãĤĤãģªãģı":135278,"ĠлиÑĨа":135279,"à¹ģลà¹īวà¸ģà¹ĩ":135280,"×ĸ׼×Ļר":135281,"ĠquÃł":135282,"ĠÙĥØ°ÙĦÙĥ":135283,"صØŃÙģ":135284,"ĠÃĤu":135285,"ÙĪبا":135286,"à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļà¹ģà¸Ľà¸¥":135287,"à¹Ģà¸Ľà¸¥à¸µà¹Īยà¸Ļà¹ģà¸Ľà¸¥à¸ĩ":135288,"à¸ķัวà¸Ńยà¹Īาà¸ĩ":135289,"Ġrápida":135290,"Ġtasar":135291,"Ġtasarım":135292,"ĠعÙĦÙĬÙĩÙħ":135293,"ס×ķ׾":135294,"cılı":135295,"cılık":135296,"ĠرغÙħ":135297,"ìĭľíĤ¤":135298,"Ġ×IJ׾ק":135299,"Ġ×IJ׾ק×ĺר":135300,"Ġ×IJ׾ק×ĺר×ķ׳×Ļ":135301,"à¹ģà¸ļà¹Īà¸ĩ":135302,"Ġhạng":135303,"ãģ£ãģ¦ãģıãĤĮ":135304,"ĠÙĨتÙĬ":135305,"ĠÙĨتÙĬجة":135306,"ıklı":135307,"غاÙĨ":135308,"à¸Ĥà¹īà¸Ńà¸Ħวาม":135309,"à¸Ľà¸¥à¸²à¸¢":135310,"ĠØ£Ùħس":135311,"à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยว":135312,"à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยวà¸Ĥ":135313,"à¸Ĺีà¹Īà¹Ģà¸ģีà¹Īยวà¸Ĥà¹īà¸Ńà¸ĩ":135314,"Ġdéfin":135315,"Ġdéfini":135316,"ÙģÙĨاد":135317,"ÙģÙĨادÙĤ":135318,"à¹Ħà¸Ķà¹īวà¹Īา":135319,"ãģªãģĦãĤĪãģĨãģ«":135320,"Ġprópria":135321,"ĠPhát":135322,"ãĤĦãģĻãģı":135323,"สวยà¸ĩาม":135324,"ê³łìļĶ":135325,"ÑıеÑĤ":135326,"ãģĭãĤĤãģĹãĤĮãģ¾ãģĽãĤĵãģĮ":135327,"ترجÙħ":135328,"ĠкÑĢаÑģив":135329,"Ġ×ŀר×IJש":135330,"деж":135331,"ĠÙĬÙĪÙĨ":135332,"ĠÙĬÙĪÙĨÙĬÙĪ":135333,"ÑģкоÑĢ":135334,"ĠKasım":135335,"ê³Ħìķ½":135336,"коÑģ":135337,"ĠнаÑĢÑĥ":135338,"ĠнаÑĢÑĥÑĪен":135339,"Ġduże":135340,"accès":135341,"Ġhá»ĵng":135342,"ĠvÅ©":135343,"ãģĦãģŁãģĹãģ¾ãģĻ":135344,"Ġ×ĺ×Ļ":135345,"Ġ×ĺ×Ļ×ķ׾":135346,"lıkları":135347,"Ġquê":135348,"ëħ¸ëıĻ":135349,"ìķĶ":135350,"CIÃĵN":135351,"Ġtắc":135352,"pressão":135353,"ĠìŀĪìľ¼":135354,"สิà¸Ĺà¸ĺิà¹Į":135355,"íĥĦ":135356,"Ġ×Ķ×ŀ×ŀש׾×Ķ":135357,"å¬īãģĹãģĦ":135358,"ĠÄIJặc":135359,"ÙĨزÙĦ":135360,"ĠдÑĢÑĥгой":135361,"дÑĥÑĤ":135362,"ìĪĻ":135363,"Ġthụ":135364,"à¹Ģสร":135365,"à¹Ģสรà¹ĩ":135366,"à¹Ģสรà¹ĩà¸Ī":135367,"Ġtoplant":135368,"Ġtoplantı":135369,"×IJ×ŀף":135370,"×ķ×ľ×ª":135371,"помн":135372,"ĠyoÄŁun":135373,"ÅĦskiego":135374,"ì°©":135375,"ĠØ«ÙĦاث":135376,"ĠØ«ÙĦاثة":135377,"Ġlắng":135378,"릴":135379,"ราà¸Ĭà¸ģาร":135380,"ĠÑģлова":135381,"á»Ĩ":135382,"à¸Ķีà¸ģวà¹Īา":135383,"ãģĶãģĸãģĦãģ¾ãģĻ":135384,"Ġдиз":135385,"Ġдизайн":135386,"férence":135387,"lıklar":135388,"ãģªãĤĵãģ§ãģĻ":135389,"ajÄħcy":135390,"Ġëĭ¤ìĸij":135391,"Ġëĭ¤ìĸijíķľ":135392,"ק×Ļר":135393,"ØŃار":135394,"สูà¹ī":135395,"Ġzro":135396,"Ġzrobi":135397,"ĠzrobiÄĩ":135398,"×ŀ×Ļ׼×Ķ":135399,"à¸Ĭà¹Īวยà¹Ģหลืà¸Ń":135400,"ĠÑįÑĤÑĥ":135401,"ë´ī":135402,"楽ãģĹãģĦ":135403,"سÙĪر":135404,"íķĺê±°ëĤĺ":135405,"ÙħؤتÙħر":135406,"ĠpoczÄħ":135407,"ĠpoczÄħtk":135408,"ĠpoczÄħtku":135409,"ĠعربÙĬ":135410,"اÙĦأر":135411,"اÙĦأردÙĨ":135412,"à¸Ķร":135413,"Åĵuvre":135414,"ĠÙĪÙĥاÙĨت":135415,"ĠÅĽredni":135416,"خضر":135417,"Ġchuyến":135418,"нÑĤ":135419,"ĠìķĮê³ł":135420,"Ġvá»Ŀi":135421,"Ġ×ij×Ļ×ĵ×Ļ":135422,"×ŀ×ĵ×ķ×ijר":135423,"ÙĪÙģر":135424,"ÙĬØ¡":135425,"×ł×Ľ×¡":135426,"ĠÐĽÐ°":135427,"лон":135428,"Ġxấu":135429,"ÙģÙĬÙĨ":135430,"Ġfévrier":135431,"ĠÐŀна":135432,"ĠVá»ģ":135433,"ĠÅŁeyler":135434,"ĠполÑĥÑĩен":135435,"зад":135436,"Ġnét":135437,"à¹Ħà¸Ľà¸¢à¸±à¸ĩ":135438,"×Ĺש×ij×ķ":135439,"à¸ļัà¸Ļà¸Ĺ":135440,"à¸ļัà¸Ļà¸Ĺึà¸ģ":135441,"ĠgerçekleÅŁ":135442,"иÑĩеÑģкое":135443,"ìĪĺê°Ģ":135444,"ثبت":135445,"ãģ¤ãģ¾ãĤĬ":135446,"ĠÑĥÑģловиÑıÑħ":135447,"ëĭ¤ê°Ģ":135448,"รายà¹Ħà¸Ķà¹ī":135449,"׼×IJ×ij":135450,"à¹Ĥà¸Ľà¸£à¹Ĥม":135451,"à¹Ĥà¸Ľà¸£à¹Ĥมà¸Ĭัà¹Īà¸Ļ":135452,"jähr":135453,"jährige":135454,"ק׳×Ļ×Ŀ":135455,"×ŀ×ķק":135456,"×ŀ×ķק×ĵ":135457,"ãģ«è¡Įãģ£ãģ¦":135458,"Ø¢ÙĦ":135459,"ведение":135460,"Ġ×ľ×Ľ×ª×ķ×ij":135461,"جÙħÙĩ":135462,"جÙħÙĩÙĪرÙĬØ©":135463,"à¸īà¸ļ":135464,"à¸īà¸ļัà¸ļ":135465,"ĠCòn":135466,"à¸ľà¸ªà¸¡":135467,"ãģªãģ©ãģĮ":135468,"×IJ×Ķ×ij":135469,"ĠдейÑģÑĤвиÑı":135470,"yız":135471,"à¹Ħมà¹Īà¹Ģà¸Ħย":135472,"جÙĪز":135473,"×Ķ×Ĺ׾×ĺ×Ķ":135474,"fällt":135475,"ãĥĵãĤ¸":135476,"ãĥĵãĤ¸ãĥį":135477,"ãĥĵãĤ¸ãĥįãĤ¹":135478,"Ġ×IJ×Ļ׳×Ŀ":135479,"ĠнаÑħодиÑĤÑģÑı":135480,"ĠdziÅĽ":135481,"ستطÙĬع":135482,"׾×Ļף":135483,"Ø®ÙĦاÙģ":135484,"ÙĩÙIJ":135485,"Ġatrás":135486,"íĺģ":135487,"ãĤĴãģĶ":135488,"Ġ×Ķ×ŀ×ķצר":135489,"ĠBakanlıģı":135490,"ÑİÑīее":135491,"ÙħÙĨاط":135492,"ÙħÙĨاطÙĤ":135493,"Ùģد":135494,"à¸Ļำà¹Ħà¸Ľ":135495,"Ġваж":135496,"Ġважно":135497,"Ġmạch":135498,"׼׳×ķ":135499,"بعث":135500,"lanması":135501,"Ġayr":135502,"Ġayrıl":135503,"ìĤ¬íļĮ":135504,"dÃŃa":135505,"pÅĤyw":135506,"اÙħÙĬØ©":135507,"íĺľ":135508,"×IJ׳×Ĵ׾":135509,"×IJ׳×Ĵ׾×Ļת":135510,"ĠìŀĪëĭ¤ëĬĶ":135511,"Ġساعة":135512,"ĠëĤĺíĥĢ":135513,"bö":135514,"à¸Ħัà¸Ļ":135515,"ĠdziaÅĤania":135516,"Ø©Ùĭ":135517,"ĠngÅ©":135518,"׳צ×Ĺ":135519,"ãģ¯ãģĤãĤĭ":135520,"ĠyaÅŁÄ±nda":135521,"stück":135522,"caracter":135523,"caracterÃŃsticas":135524,"Ġrá»Ńa":135525,"ĠÙħختÙĦÙģØ©":135526,"ãģ«ãģĬãģijãĤĭ":135527,"à¹ģà¸ŀà¸ĩ":135528,"วิà¹Īà¸ĩ":135529,"תפ×ķ":135530,"ساÙĩÙħ":135531,"使ãģĨ":135532,"ÙĥرÙĬ":135533,"×IJפ×Ļ":135534,"...............":135535,"ĠÑĤаким":135536,"×Ļ׼×ķ×Ļ":135537,"شبÙĩ":135538,"جÙĬر":135539,"ãģĿãģ®ãģ¾ãģ¾":135540,"acjÄĻ":135541,"ĠاÙĦترÙĥ":135542,"ĠاÙĦترÙĥÙĬ":135543,"ĠпÑĢавилÑĮно":135544,"ĠتعÙħÙĦ":135545,"à¸ģลà¹īา":135546,"Ġbiên":135547,"Ġ×ij׳×Ļ×Ļת":135548,"ĠклÑĥб":135549,"Ġ×ŀש×Ķ":135550,"вÑĪий":135551,"ãģĵãģ¨ãģĮãģ§ãģįãĤĭ":135552,"à¸ŀัà¸Ļà¸ĺุ":135553,"à¸ŀัà¸Ļà¸ĺุà¹Į":135554,"ר×ķ×Ŀ":135555,"ĠاÙĦÙģرÙĨ":135556,"ĠاÙĦÙģرÙĨسÙĬ":135557,"à¹Ģà¸Ľà¹ĩà¸Ļà¸Ħà¸Ļ":135558,"ãģĹãģ¦ãģĬãĤĬ":135559,"Ġthầy":135560,"ãĤĵãģłãģijãģ©":135561,"ìĶ¨":135562,"ÙħدÙĨ":135563,"تÙĪÙĨ":135564,"ĠмеÑĤал":135565,"ĠмеÑĤалл":135566,"ĠinÃŃcio":135567,"à¸Ńà¸Ńà¸ģà¸Īาà¸ģ":135568,"ëĴ¤":135569,"Ġcuá»ijn":135570,"Ġbuá»Ļc":135571,"ÙĨسÙĬ":135572,"ächt":135573,"×ŀ×Ļ׳×Ļ×Ŀ":135574,"ãģķãģ¦":135575,"ãģĮãģ§ãģį":135576,"ÑĬем":135577,"Ġtái":135578,"ĠЧÑĤ":135579,"ĠЧÑĤобÑĭ":135580,"à¸Ľà¸¥à¸¹à¸ģ":135581,"à¸Ĭุมà¸Ĭà¸Ļ":135582,"нÑģкий":135583,"Ġvững":135584,"Ġ×Ķ׾×ij":135585,"ële":135586,"Ġשע×ijר":135587,"ваÑĤÑĮÑģÑı":135588,"бой":135589,"عÙĪÙĨ":135590,"à¹ģà¸Ķà¸Ļ":135591,"Ġספר×Ļ×Ŀ":135592,"Ġtuyên":135593,"Ġnhiêu":135594,"ĠQuý":135595,"Ġhuyết":135596,"ãĤıãģĭãĤīãģªãģĦ":135597,"Ġ×ŀ׼ף":135598,"Ġ×Ķק׾":135599,"Ġ׾×IJ×ķר":135600,"ĠÄIJiá»ĩn":135601,"شؤ":135602,"شؤÙĪÙĨ":135603,"Ġ×ŀ×Ĺפש":135604,"ĠпоÑģÑĤоÑıнно":135605,"×ŀ×Ļר":135606,"ìħĶ":135607,"ÐŀÑģ":135608,"ÐŀÑģнов":135609,"×ĸ×Ļת":135610,"ĠHá":135611,"ĠÑĩаÑģов":135612,"×IJ×ķ׾×Ļ":135613,"Ġmát":135614,"خرÙĪ":135615,"خرÙĪج":135616,"ÙĤضا":135617,"ÙĤضاÙĬا":135618,"à¹Ģà¸Ľà¸Ńรà¹Į":135619,"ĠÙĬÙĪÙĦ":135620,"ĠÙĬÙĪÙĦÙĬÙĪ":135621,"à¹Ĥà¸Ĺษ":135622,"׳פ׾":135623,"ת×ķש":135624,"ת×ķש×ij×Ļ":135625,"Ġvários":135626,"×ŀר×IJ×Ķ":135627,"ëĿ¼ìĿ´":135628,"ÙĨغ":135629,"×ijצע":135630,"гон":135631,"ĠÄIJược":135632,"عÙı":135633,"пÑĥÑģк":135634,"ĠÙĪاÙĦÙģ":135635,"ücü":135636,"×Ļק×Ļ×Ŀ":135637,"ĠسبÙĬÙĦ":135638,"׾×ijף":135639,"ĠاÙĦÙĤرÙĨ":135640,"ס×ķת":135641,"ĠQuáºŃn":135642,"ãģĵãĤĮãģĮ":135643,"ãĥĸãĥ©ãĥ³ãĥī":135644,"×Ĵ×ŀר":135645,"ĠwartoÅĽci":135646,"ĠÙĪبÙĬÙĨ":135647,"Ġdạ":135648,"ÐIJв":135649,"ÐIJвÑĤо":135650,"Ġolacaktır":135651,"à¸Ļà¸Ĺà¹Į":135652,"Ùħطار":135653,"Ġעק×ij":135654,"Ġתפ":135655,"ãģĹãģ¦ãģĦãģ¦":135656,"צ×ŀ×Ĺ":135657,"à¸Īà¸Ńà¸ĩ":135658,"Ġöde":135659,"ìį¨":135660,"ÙĨاس":135661,"調ãģ¹":135662,"ĠогÑĢомн":135663,"ë³´íĹĺ":135664,"×ĺק":135665,"×ĺקס×ĺ":135666,"ĠbaÅŁv":135667,"ĠbaÅŁvuru":135668,"Ġpomys":135669,"ĠpomysÅĤ":135670,"ãģ«ä¹Ĺ":135671,"Ġש׼ף":135672,"ĠاÙĦÙħسؤÙĪÙĦ":135673,"Ġзан":135674,"ĠзанÑıÑĤ":135675,"ĠdÆ°Æ¡ng":135676,"ãĥĹãĥ¬ãĤ¤":135677,"ลà¸ļ":135678,"ÑĤика":135679,"ĠAralık":135680,"Ġнедо":135681,"Ġmá»Ļ":135682,"Ġoran":135683,"Ġoranı":135684,"Ġktór":135685,"ĠktórÄħ":135686,"Ġ×Ķ×IJ×Ĺר×ķ׳×ķת":135687,"ائÙĨ":135688,"ÅĦs":135689,"ÅĦska":135690,"åĽ½ãģ®":135691,"×ŀ×ĺ×Ļ":135692,"ĠвопÑĢоÑģÑĭ":135693,"à¸Ńà¸ĩà¸Ħà¹Įà¸ģร":135694,"×ŀ×ķצ×IJ":135695,"Ġpóź":135696,"Ġpóźniej":135697,"ש×ŀ×IJ׾":135698,"Ġkaps":135699,"Ġkapsam":135700,"Ġkapsamında":135701,"Ġmáquina":135702,"ĠÅĽwiecie":135703,"ĠhoÃłng":135704,"Ġözgü":135705,"×Ĵ×ķר×Ŀ":135706,"ãģĤãģŁãĤĬ":135707,"à¸ķัà¸Ķสิà¸Ļ":135708,"à¸ķัà¸Ķสิà¸Ļà¹ĥà¸Ī":135709,"бÑĢи":135710,"ãģ«ãģªãĤĭãģ¨":135711,"تÙĥÙĪÙĨ":135712,"Ġ×ķ×Ķ×Ļ×IJ":135713,"Ġchiếu":135714,"ÑģÑĤанав":135715,"ÑģÑĤанавли":135716,"ÑģÑĤанавлива":135717,"×ŀ×ķ×Ĵ":135718,"cité":135719,"ĠKörper":135720,"Ġש×Ĵ×Ŀ":135721,"عظ":135722,"عظÙĬÙħ":135723,"Ġ×Ķ×IJ×Ļש×Ļ":135724,"Ġmatière":135725,"ĠÙģÙĪÙĤ":135726,"Ġkto":135727,"ĠktoÅĽ":135728,"à¸Ļà¹Ĥย":135729,"à¸Ļà¹Ĥยà¸ļาย":135730,"å¾ħãģ¡":135731,"à¹Ģมà¸Ļ":135732,"à¹Ģมà¸Ļู":135733,"AÃĩÃĥO":135734,"Ġtù":135735,"Ġtùy":135736,"ãĥĪãĥ³":135737,"ĠоÑĤказ":135738,"Ġ×ŀ×ķצר":135739,"ülü":135740,"ãģķãĤĵãģ«":135741,"Ġ×Ĺ×ķ×ij":135742,"קר×Ļ×IJ×Ķ":135743,"ĠاÙĦخدÙħات":135744,"ĠÙĦÙħدة":135745,"رؤ":135746,"رؤÙĬØ©":135747,"ãĤĴè¦ĭãģ¤ãģij":135748,"à¸Łà¸²":135749,"Ġréussi":135750,"à¸Ļัà¸ģà¹Ģรียà¸Ļ":135751,"ĠÑĩиÑģл":135752,"à¸ģารà¹Ģลà¹Īà¸Ļ":135753,"Ġhazırl":135754,"Ġhazırlan":135755,"ĠпеÑĢвÑĭй":135756,"лим":135757,"ĠоÑĤзÑĭвÑĭ":135758,"ĠwyjÄħ":135759,"ĠwyjÄħtk":135760,"ĠØ£ÙĤÙĦ":135761,"ס×ļ":135762,"Ġê²°ìłķ":135763,"Ġ׾×ŀעש×Ķ":135764,"Ġlắp":135765,"à¹ģà¸ļร":135766,"à¹ģà¸ļรà¸Ļà¸Ķà¹Į":135767,"วà¹Īาà¹Ģà¸Ľà¹ĩà¸Ļ":135768,"Ġبدا":135769,"ĠبداÙĬØ©":135770,"ãģ¨ãģĦãģĨãģ®ãģĮ":135771,"иÑĩеÑģким":135772,"à¸ģารà¸ŀัà¸Ĵà¸Ļา":135773,"ĠbÃło":135774,"ĠmiaÅĤa":135775,"ywaÄĩ":135776,"ĠMärz":135777,"ĠÙĨسبة":135778,"Ġéconomique":135779,"×ĸ×ŀ":135780,"×ĸ×ŀ׳×Ļ×Ŀ":135781,"æŃ¢ãĤģ":135782,"Ġtủ":135783,"íķĺìĭł":135784,"Ġkażdego":135785,"straÃŁe":135786,"à¸Ĭีà¹ī":135787,"à¹Ģà¸ļา":135788,"ÑĢеÑģÑĥÑĢÑģ":135789,"евой":135790,"شباب":135791,"à¸ķà¹Īาà¸ĩà¸Ľà¸£à¸°à¹Ģà¸Ĺศ":135792,"Ġ×IJ×Ļש":135793,"Ġ×IJ×Ļש×Ļת":135794,"×Ļ×ķפ":135795,"×Ļ×ķפ×Ļ":135796,"ĠìļĶ구":135797,"ì¡°ìĤ¬":135798,"ãģ£ãģŁãĤī":135799,"׾×Ļק":135800,"миниÑģÑĤÑĢ":135801,"ãĤĤãģ®ãģ¯":135802,"ĠlÆ°Æ¡ng":135803,"Ġнаи":135804,"Ġнаибол":135805,"Ġнаиболее":135806,"íİĺ":135807,"à¹ģà¸ŀà¹ī":135808,"ãĤŃãĥ¥":135809,"ĠкоÑĤоÑĢÑĭм":135810,"à¹ģà¸Ĺà¸ĩ":135811,"à¹ģà¸Ĺà¸ĩà¸ļà¸Ńล":135812,"Ġ׳×Ļ×Ķ":135813,"Ġ׳×Ļ×Ķ×ķ׾":135814,"âĤª":135815,"ĠGiải":135816,"ĠиÑģполÑĮзова":135817,"ëł¥ìĿĦ":135818,"ãģĹãģĭãĤĤ":135819,"à¸ģà¹ĩà¸ķà¹īà¸Ńà¸ĩ":135820,"ĠÑĢеб":135821,"ĠÑĢебен":135822,"ĠÑĢебенка":135823,"تÙĪاصÙĦ":135824,"ãĤ°ãĥ«ãĥ¼ãĥĹ":135825,"ãĤĦãĤī":135826,"à¹Ģà¸Ľà¸´à¸Ķà¸ķัว":135827,"бÑĢо":135828,"ë°ĸìĹIJ":135829,"ÙĨÙİا":135830,"×Ķ×Ĵ":135831,"×Ķ×Ĵ׳×Ķ":135832,"à¸Ĺรั":135833,"à¸Ĺรัà¸ŀ":135834,"à¸Ĺรัà¸ŀยà¹Į":135835,"Ġkhá»iji":135836,"עצ×ŀ×ķ":135837,"болезн":135838,"Ġë°ĽìķĦ":135839,"มà¸Ļ":135840,"มà¸Ļุ":135841,"มà¸Ļุษ":135842,"มà¸Ļุษยà¹Į":135843,"âĹĨ":135844,"×ŀצ׾×Ļ×Ĺ":135845,"Ñıвление":135846,"ÙħØ·ÙĦ":135847,"ÙħØ·ÙĦÙĪب":135848,"خاÙĦÙģ":135849,"تÙĪÙĤÙģ":135850,"ãģ§ãģįãģ¾ãģĽãĤĵ":135851,"оÑģÑĤей":135852,"меÑĩа":135853,"기ëĬĶ":135854,"תשע":135855,"صÙĬب":135856,"Ġ×ij×¢×ķ×ĵ":135857,"à¸Ĥà¸Ńà¸ĩà¹Ģà¸Ĥา":135858,"ÑĤÑıж":135859,"ĠÑĥпÑĢав":135860,"ĠÑĥпÑĢавлениÑı":135861,"Ġgénér":135862,"ĠthÃŃ":135863,"פ×ļ":135864,"ĠرÙħض":135865,"ĠرÙħضاÙĨ":135866,"Ġtruyá»ĩn":135867,"إعداد":135868,"ãĤµãĥĿãĥ¼ãĥĪ":135869,"Ġполно":135870,"خاÙħ":135871,"ÐŁÐµÑĤ":135872,"ÐŁÐµÑĤеÑĢ":135873,"ÐŁÐµÑĤеÑĢбÑĥÑĢ":135874,"ÐŁÐµÑĤеÑĢбÑĥÑĢг":135875,"ÙħÙĨتدÙī":135876,"ãģķãĤĮãģ¾ãģĹãģŁ":135877,"ĠëĮĢíķĺìŬ":135878,"à¸ľà¸¹à¹īà¸Ĺีà¹Ī":135879,"Ġ×ŀ×IJ×ķ":135880,"׾׳×ĵ":135881,"оÑĩнÑĭе":135882,"ĠнаÑĩала":135883,"Ġ׾×Ļ׾×ĵ×Ļ×Ŀ":135884,"овое":135885,"ãģĻãĤĭãģĵãģ¨ãģ§":135886,"ĠاÙĦÙĨÙģ":135887,"ĠاÙĦÙĨÙģØ·":135888,"ìŀĪëĬĶ":135889,"غÙĨÙĬ":135890,"פ×ĵ":135891,"ãĤ¾":135892,"ĠCré":135893,"ãģ©ãģ¡ãĤī":135894,"ثاÙĨ":135895,"ÑĢабаÑĤ":135896,"ÑĢабаÑĤÑĭва":135897,"Ġê°Ļëĭ¤":135898,"à¸Īั":135899,"à¸Īัà¸ģร":135900,"Ġchụ":135901,"Ġchụp":135902,"ĠмаÑģÑĤ":135903,"ĠмаÑģÑĤеÑĢ":135904,"Ġnắm":135905,"ĠÑģÑĤали":135906,"Ġ×Ķ×IJ×Ļר×ķ×¢":135907,"ãĤ½ãĥ³":135908,"åĪĨãģĭãĤĬ":135909,"طبع":135910,"بدا":135911,"gráfico":135912,"геÑĢ":135913,"à¸Ķำà¹Ģà¸Ļิà¸Ļà¸ģาร":135914,"Ġsaldır":135915,"Ġsaldırı":135916,"вÑĪиÑħ":135917,"ãģĭãģ£ãģŁãģ§ãģĻ":135918,"Ġyapıyor":135919,"ĠاÙĦÙģت":135920,"צרפת":135921,"здоÑĢов":135922,"×ij×¢×ľ":135923,"Ġ×IJ×ŀ×Ļת×Ļ":135924,"ĠобÑĭ":135925,"ĠобÑĭÑĩ":135926,"ĠобÑĭÑĩно":135927,"Ġ׾×ķ×ŀר":135928,"تÙĥÙĨ":135929,"تÙĥÙĨÙĪÙĦÙĪج":135930,"تÙĥÙĨÙĪÙĦÙĪجÙĬا":135931,"Ġhakkı":135932,"ĠÑĢав":135933,"ĠÑĢавно":135934,"رÙĬÙĥ":135935,"Ġ×ij×ŀ×Ļ×ĵ":135936,"Ġ×ij×ŀ×Ļ×ĵ×Ķ":135937,"à¹ģà¸ģà¹īว":135938,"Ġìĸĺ":135939,"Ġìĸĺ기":135940,"ãģĹãģ¦ãģĦãģ¾ãģĹãģŁ":135941,"Ġkısm":135942,"Ġkısmı":135943,"걸":135944,"åĨħãģ®":135945,"ì§ķ":135946,"à¹Ģหมืà¸Ńà¸Ļà¸ģัà¸Ļ":135947,"ĠÙģÙIJ":135948,"ĠÙģÙIJÙĬ":135949,"ÙĤاعدة":135950,"Ġmożesz":135951,"ÙħصاÙĦ":135952,"ÙħصاÙĦØŃ":135953,"ãģ¾ãģŁãģ¯":135954,"бег":135955,"Ġsıc":135956,"Ġsıcak":135957,"ÑĩиÑģ":135958,"ÑĩиÑģлен":135959,"Ġног":135960,"ãĥģãĥ£ãĥ³":135961,"ãĥ«ãĥī":135962,"Ġgió":135963,"Ġsını":135964,"Ġsınıf":135965,"иваÑĤÑĮ":135966,"Ġquên":135967,"Ġìłģ":135968,"Ġìłģìļ©":135969,"ĠJoão":135970,"Ùģاد":135971,"ĠGlück":135972,"à¸Ĺà¸Ńà¸Ķ":135973,"Ġgói":135974,"ï¼Ĭ":135975,"Ġdétail":135976,"ĠدÙĬسÙħ":135977,"ĠدÙĬسÙħبر":135978,"ë¡ľìĦľ":135979,"×ŀ×ķ×Ĺ":135980,"à¹Ħฮ":135981,"ĠоÑĤд":135982,"ĠоÑĤдÑĭÑħ":135983,"Ġkhuyến":135984,"à¸Ħà¸Ńย":135985,"ĠجÙĨÙĬ":135986,"ĠجÙĨÙĬÙĩ":135987,"ĠاÙĦدÙģاع":135988,"à¸Ļà¹īำหà¸Ļัà¸ģ":135989,"ĠìĤ¬ëŀĮëĵ¤ìĿ´":135990,"Ġthừa":135991,"ĠÃ¶ÄŁrenci":135992,"ĠпомоÑīи":135993,"ĠczÄĻÅĽÄĩ":135994,"ש×ĺר":135995,"ĠNhi":135996,"ĠNhiá»ģu":135997,"׳צ×Ļ":135998,"ĠнаÑĪем":135999,"ĠkarÅŁÄ±laÅŁ":136000,"Ġ×Ķש׳×Ļ×Ŀ":136001,"ĠÄIJÆ°á»Ŀng":136002,"Ġtrú":136003,"ĠÑĢазлиÑĩнÑĭÑħ":136004,"ĠاÙĦØ´Ùĩر":136005,"Ġ×ľ×¢×ķ׾×Ŀ":136006,"ØŃجر":136007,"ĠÄijá»ķ":136008,"ĠìĿĺíķ´":136009,"à¸ļà¹Īà¸Ńย":136010,"Ġ×Ķ×Ļ׾×ĵ":136011,"ãģ¨ãģªãģ£ãģŁ":136012,"Ġ×Ĺ×ķ×ķת":136013,"Ġש×Ļר×ķת×Ļ":136014,"Äħcy":136015,"سرÙĬ":136016,"KÄ°":136017,"פ׳×ķ":136018,"ÑģÑĤÑĢÑĥкÑĤÑĥÑĢ":136019,"ÑĤÑĢÑĥд":136020,"Ġ×Ķקר":136021,"Ġ×Ķקר×ķ×ij":136022,"ĠtháºŃm":136023,"èģŀãģį":136024,"ÙĤÙĪÙĬ":136025,"клÑİÑĩен":136026,"ÑĤеÑħ":136027,"ÑĤеÑħнолог":136028,"è¡Įãģ£ãģŁ":136029,"Ġ×ķ×IJ×Ļף":136030,"ĠÅŁeklin":136031,"ĠÅŁeklinde":136032,"rô":136033,"ÑĢог":136034,"ĠновÑĭе":136035,"Ġס×ij×Ļ×ij":136036,"ĠtecnologÃŃa":136037,"×¡×Ľ":136038,"×¡×Ľ×ķ×Ŀ":136039,"ĠÅŀub":136040,"ĠÅŀubat":136041,"Ġ×Ķ×ŀ׾×IJ":136042,"Ġwypos":136043,"Ġwyposaż":136044,"ãģ¯ä½ķ":136045,"ãĤ¬ãĥ³":136046,"ê°ĸ":136047,"Ġкакие":136048,"Ġçocuklar":136049,"Ġ׾צ×ĵ":136050,"Ġkayıt":136051,"ĠмеÑģÑĤе":136052,"ÙħدÙĬÙĨØ©":136053,"Ġ׼×Ĵ":136054,"Ġ׼×Ĵ×ķף":136055,"ãģĹãģ¦ãĤĭ":136056,"ĠÙħاÙĬÙĪ":136057,"ãģ£ãģ¦ãģĹãģ¾ãģ£ãģŁ":136058,"ĠпÑĢогÑĢаммÑĭ":136059,"à¹ģลà¸Ļà¸Ķà¹Į":136060,"ãĥ¯ãĤ¤":136061,"ער×ķ×¥":136062,"Ñģид":136063,"ĠBöyle":136064,"Ġì²ĺìĿĮ":136065,"Ġתפק×Ļ×ĵ":136066,"ĠTrên":136067,"íĥĪ":136068,"ĠÐłÐ¾ÑģÑģий":136069,"ĠÐłÐ¾ÑģÑģийÑģкой":136070,"ĠsÃłn":136071,"Ġrègle":136072,"ĠyaklaÅŁÄ±k":136073,"à¹Ģลิà¸ģ":136074,"ĠدائÙħ":136075,"Ġ×ķ×Ĵ":136076,"ابر":136077,"Ġbè":136078,"ĠاÙĦÙĤدÙħ":136079,"ĠÑĢеÑĪениÑı":136080,"hiên":136081,"ÑĤик":136082,"ÄĦ":136083,"à¸ļรรยาà¸ģ":136084,"à¸ļรรยาà¸ģาศ":136085,"רצ×ķף":136086,"åĭķãģį":136087,"ĠGäste":136088,"Ġ기본":136089,"ĠÙĬعرÙģ":136090,"ĠSá»Ń":136091,"gÅĤÄĻb":136092,"à¹Ģà¸Ńส":136093,"×IJ×ŀ×Ļף":136094,"ĠпÑĥнк":136095,"ĠпÑĥнкÑĤ":136096,"Ġ×Ļ×ķ×ĵ×¢×Ļ×Ŀ":136097,"ãĤ«ãĥ©ãĥ¼":136098,"Ġ×ijס×ĵר":136099,"Ġbuá»ĵn":136100,"йÑĤ":136101,"йÑĤеÑģÑĮ":136102,"ãĤĴæ±ĤãĤģ":136103,"Ġ×IJ×ª×Ľ×Ŀ":136104,"Ġ모르":136105,"ظرÙĪÙģ":136106,"ÑĩеÑģÑĤво":136107,"ìĸ´ìĦľ":136108,"Ġодна":136109,"Ġkapı":136110,"Ġëħ¸ëł¥":136111,"ĠKüche":136112,"ĠاÙĦتش":136113,"Ø·ÙĬب":136114,"ĠíĬ¹íŀĪ":136115,"ĠвÑĭпÑĥÑģ":136116,"ĠвÑĭпÑĥÑģк":136117,"×ĵת×Ļ":136118,"ĠuÄŁ":136119,"ĠuÄŁra":136120,"ائÙĩا":136121,"Ġthoát":136122,"ãģªãĤĤãģ®":136123,"ÑijÑĢ":136124,"기ê°Ģ":136125,"ĠgeliÅŁme":136126,"تØŃÙĤ":136127,"تØŃÙĤÙĤ":136128,"ĠопаÑģ":136129,"бÑĢоÑģ":136130,"หุ":136131,"หุà¹īà¸Ļ":136132,"ì¼Ģ":136133,"ãĤ¹ãĥŀ":136134,"ãĤ¹ãĥŀãĥĽ":136135,"Ø£Ùģر":136136,"Ø£Ùģراد":136137,"ĠThá»±c":136138,"Ġthắ":136139,"ãĥªãĥ³ãĤ¯":136140,"Ġniá»ģm":136141,"ĠHöhe":136142,"عÙħار":136143,"ÙĥÙĪرÙĪÙĨ":136144,"ÙĥÙĪرÙĪÙĨا":136145,"ĠÄIJến":136146,"ĠÑģамом":136147,"ĠÑĤеле":136148,"ĠÄijoán":136149,"à¸Ħวามà¸Ħิà¸Ķà¹Ģหà¹ĩà¸Ļ":136150,"ĠдиÑģк":136151,"أطÙģاÙĦ":136152,"มารà¹Į":136153,"à¸Ĺหาร":136154,"à¸Ĺà¸Ļ":136155,"ĠبعÙĬد":136156,"ĠاÙĦÙĩÙĨد":136157,"åĩºãģĹãģ¦":136158,"Ġkarde":136159,"ĠkardeÅŁ":136160,"×Ķ×Ļס×ĺ×ķר":136161,"×Ķ×Ļס×ĺ×ķר×Ļ×Ķ":136162,"éģ¸ãģ³":136163,"عاÙħÙĦ":136164,"à¸Ĥยาย":136165,"Ġtürl":136166,"Ġtürlü":136167,"ĠìĿ¼ìĿ´":136168,"Ġmatéria":136169,"Ġ׼׾×ķ×ŀר":136170,"ãĥģãĥ£ãĥ¼":136171,"جÙħاعة":136172,"ĠÑģвоим":136173,"Ø¥ÙĤاÙħØ©":136174,"ä¾ĭãģĪãģ°":136175,"ساب":136176,"آخر":136177,"ÙĤدÙĬر":136178,"×IJ×ŀ×Ļ":136179,"ìĸ»":136180,"Ġ׳×ķספת":136181,"ĠÐĴлад":136182,"ĠÐĴладим":136183,"ĠÐĴладимиÑĢ":136184,"Ġestará":136185,"ãģĵãģĨãģĦãģĨ":136186,"ãĤĴ使çĶ¨":136187,"มาà¸ķร":136188,"มาà¸ķรà¸IJาà¸Ļ":136189,"ãģ£ãģ½":136190,"Ġnú":136191,"Ġnúi":136192,"ยาà¸ĩ":136193,"ĠاÙĦجÙĨس":136194,"Ġüstün":136195,"ëľ»":136196,"ãĤ»ãĥ«":136197,"ãģ¦ãģĦãģįãģ¾ãģĻ":136198,"Ġ×Ĺ×ķ×ĸ":136199,"Ġ×Ĺ×ķ×ĸר":136200,"ĠÐĵлав":136201,"à¹Ĥà¸Ĭà¸Ħ":136202,"íıIJ":136203,"ÙĨتظر":136204,"Ġ×Ĵ×ij×Ļ":136205,"عÙĤب":136206,"intér":136207,"intérêt":136208,"×ŀפ×Ĵ":136209,"×ŀפ×Ĵש":136210,"Ġthù":136211,"اÙģت":136212,"Ġ×ŀשפ":136213,"Ġ×ŀשפ×ĺ×Ļ":136214,"ĠÙħÙĪاÙĤع":136215,"è¦ļ":136216,"è¦ļãģĪ":136217,"×ĵ×Ļף":136218,"à¹Ģรืà¹Īà¸Ńà¸ĩราว":136219,"ãģ¾ãģĤ":136220,"Ġghế":136221,"иÑĢÑĥÑİÑĤ":136222,"à¸ģว":136223,"à¸ģวà¹īาà¸ĩ":136224,"ĠповеÑĢ":136225,"ĠповеÑĢÑħ":136226,"ĠповеÑĢÑħноÑģÑĤ":136227,"׳×ĵר":136228,"ĠконÑĨе":136229,"Ġдолжна":136230,"Ġ×Ļש×Ļר":136231,"acaģız":136232,"ìĹĶ":136233,"ĠnÃŃvel":136234,"Ġör":136235,"Ġörnek":136236,"ÙĥÙģ":136237,"ĠФедеÑĢаÑĨии":136238,"Ġ구ìĦ±":136239,"หัวà¹ĥà¸Ī":136240,"ĠVáºŃy":136241,"мед":136242,"меди":136243,"медиÑĨин":136244,"медиÑĨинÑģк":136245,"ازÙĬ":136246,"×Ĵ×ij×ķ׾":136247,"ÑĦÑĢ":136248,"Ġzusätzlich":136249,"à¸ģà¸ģ":136250,"ĠاÙĦاÙĤتصادÙĬØ©":136251,"Ġhè":136252,"luÄŁun":136253,"جÙİ":136254,"à¹Ħà¸Łà¸¥à¹Į":136255,"ÄIJT":136256,"ãģĿãģ®ä»ĸ":136257,"à¸Ĺิà¹īà¸ĩ":136258,"ĠاÙĦØ£ÙĪ":136259,"رسÙħ":136260,"æ°Ĺãģ¥":136261,"ìĿ´ë©°":136262,"ÑĮев":136263,"صط":136264,"ĠاÙĦاستث":136265,"ĠاÙĦاستثÙħار":136266,"à¸Ńาà¸Ħาร":136267,"ĠÑĤоÑĩно":136268,"ĠVân":136269,"à¸Ńร":136270,"à¸Ńรà¹Īà¸Ńย":136271,"ĠاÙĦسÙĨØ©":136272,"ĠcÆ°á»Ľi":136273,"×Ļ×Ķף":136274,"íį¼":136275,"話ãģĹ":136276,"âĹĭ":136277,"ĠìķĬìĿĢ":136278,"ãĥ¡ãĥ¼ãĤ":136279,"ãĥ¡ãĥ¼ãĤ«":136280,"ãĥ¡ãĥ¼ãĤ«ãĥ¼":136281,"ĠÑĤепло":136282,"å½¼ãĤī":136283,"ĠÄ°z":136284,"ĠÄ°zmir":136285,"íĻį":136286,"Ġrượ":136287,"Ġrượu":136288,"æĢĿãģĦåĩº":136289,"ĠPhạm":136290,"Ġcháu":136291,"צ×Ļ×ķת":136292,"ĠìĿ¼ë³¸":136293,"ìĤ¬ëĬĶ":136294,"ĠÑģоздан":136295,"Ġaracı":136296,"Ġער":136297,"Ġער×Ļ׼×Ķ":136298,"ĠíķĺëĤĺëĭĺìĿĺ":136299,"dziÅĤ":136300,"à¸Ľà¸£à¸°à¸ĺาà¸Ļ":136301,"ĠserÃŃa":136302,"ĠìŀĪëıĦë¡Ŀ":136303,"درج":136304,"íķľëĭ¤ëĬĶ":136305,"à¸Ńาà¸Ĺ":136306,"à¸Ńาà¸Ĺิà¸ķ":136307,"à¸Ńาà¸Ĺิà¸ķยà¹Į":136308,"ÑĤелÑĮнÑĭй":136309,"ĠخدÙħات":136310,"×ŀ׳×ĺ":136311,"Ġlược":136312,"ĠSÃłi":136313,"ĠÙĪاض":136314,"ĠÙĪاضØŃ":136315,"غاز":136316,"ĠdoÄŁal":136317,"Ġ×ijש×Ŀ":136318,"Ġдлин":136319,"Ġإطار":136320,"Ġ×ijספר":136321,"ãĤĴä¸İ":136322,"ãĤĴä¸İãģĪ":136323,"Ġë²ķë¥ł":136324,"ĠÑĥвели":136325,"ĠÑĥвелиÑĩи":136326,"สà¹Ħà¸ķ":136327,"สà¹Ħà¸ķลà¹Į":136328,"à¹Ħà¸ģล":136329,"×ij×Ĺף":136330,"ĠìĿ´íĽĦ":136331,"Ġmunic":136332,"ĠmunicÃŃpio":136333,"تÙħØ«ÙĦ":136334,"ĠÄijáo":136335,"Hôtel":136336,"Ġlá»Ńa":136337,"ĠÄijẳng":136338,"Ñĩки":136339,"شرÙĪ":136340,"شرÙĪØ·":136341,"ĠìĿ´ë¥¼":136342,"ÙĬÙĭا":136343,"×ŀ׾×ļ":136344,"×ŀ×Ķ×Ļר×ķת":136345,"ĠобÑıзаÑĤелÑĮ":136346,"ĠобÑıзаÑĤелÑĮно":136347,"énergie":136348,"Ġmudança":136349,"Ġmụ":136350,"Ġmụn":136351,"Ġnº":136352,"ĠاÙĦتعا":136353,"ĠاÙĦتعاÙĪÙĨ":136354,"ĠاÙĦاجتÙħاعÙĬØ©":136355,"ĠплаÑģÑĤ":136356,"Ġëĵ±ìĿĺ":136357,"ãĥIJãĤ¤ãĤ¯":136358,"ÙĩجÙĪÙħ":136359,"ĠSaúde":136360,"Ġì¤ijìļĶíķľ":136361,"Ġ×Ķצ×Ļ×ij×ķר":136362,"×ª×§×Ł":136363,"ĠاÙĦعاÙĦÙħÙĬ":136364,"ĠболÑĮÑĪой":136365,"ĠÙĥÙĦÙħ":136366,"ĠÙĥÙĦÙħØ©":136367,"ãģ®ãģ§ãģ¯ãģªãģĦãģ§ãģĹãĤĩãģĨãģĭ":136368,"ĠÙħباراة":136369,"Ġש×IJ׳":136370,"Ġש×IJ׳×Ĺ׳×ķ":136371,"ãĤ¹ãĤ¿ãĤ¤ãĥ«":136372,"ĠSaÄŁ":136373,"ĠSaÄŁlık":136374,"ĠhÆ°":136375,"׳×Ĺ×Ķ":136376,"Ġ×ijקר×ij":136377,"طعÙħ":136378,"หิà¸Ļ":136379,"à¸Ĺุà¸ģวัà¸Ļ":136380,"à¸Ħรัà¹īà¸ĩà¸Ĺีà¹Ī":136381,"ĠlÃłnh":136382,"Ġdonné":136383,"ãģĽãģĦ":136384,"جزÙĬرة":136385,"доÑĢож":136386,"ì¼ľ":136387,"تÙĨظÙĬÙģ":136388,"ãĥģãĥ§":136389,"Ġaldıģı":136390,"جاج":136391,"ĠÑĤомÑĥ":136392,"à¸Ľà¸´":136393,"Ġ×ijרשת":136394,"ãģıãģªãĤĬãģ¾ãģĻ":136395,"ĠпÑĢинÑĨип":136396,"Ġ×Ĺ׾×ķ":136397,"ëı¼":136398,"×ķ×Ĵש":136399,"سس":136400,"à¸Ľà¸¹":136401,"Ġhầu":136402,"æĦŁãģĺãĤĭ":136403,"ï¼´":136404,"دÙĪا":136405,"ĠÑģмог":136406,"scrição":136407,"ĠtháºŃn":136408,"Ġר×ķ×IJ×Ķ":136409,"обÑĢажен":136410,"ĠاÙĦتجارÙĬØ©":136411,"طبÙĬع":136412,"jÄħcÄħ":136413,"íĸīìľĦ":136414,"ĠновÑĭй":136415,"Ġ×ŀ×Ĺ×ĵש":136416,"æĮ¯ãĤĬ":136417,"gué":136418,"Ġ×IJ×Ļר×ķ×¢":136419,"Ġ×IJ×Ļר×ķ×¢×Ļ×Ŀ":136420,"ĠاÙĦØ°Ùĩب":136421,"×ĵ×IJ":136422,"تاÙĨ":136423,"ãģłãģĹ":136424,"à¸Ńัà¸ķรา":136425,"à¹Ĥà¸Ī":136426,"بÙĦاد":136427,"×Ķ×Ļ×Ļ׳×ķ":136428,"ĠÑģпе":136429,"ĠÑģпеÑĨиалÑĮно":136430,"ĠÅĽwiata":136431,"ãĤĵãģ§ãģĻãĤĪ":136432,"شرÙĥØ©":136433,"ĠpÅĤyt":136434,"Ġsitué":136435,"Ġ׼×IJ׾×Ķ":136436,"ס×ijר":136437,"Ġkażd":136438,"Ġkażdym":136439,"ãĤĴæĮģãģ¤":136440,"׾×Ķ׾":136441,"׾×Ķ׾ף":136442,"ĠwÅĤas":136443,"ĠwÅĤasne":136444,"ĠsaÄŁlan":136445,"×ŀ×¢×ľ×Ķ":136446,"ĠاÙĦاÙĪÙĦ":136447,"ìĹIJìĦľëıĦ":136448,"×IJ×Ļר×ķפ×Ķ":136449,"تÙĤÙĨÙĬØ©":136450,"Ùħائ":136451,"Ùħائة":136452,"ĠcompañÃŃa":136453,"Ġsürek":136454,"Ġsürekli":136455,"ĠиÑģкÑĥÑģ":136456,"ĠиÑģкÑĥÑģÑģÑĤв":136457,"ĠBürger":136458,"ת×Ĺר":136459,"ת×Ĺר×ķת":136460,"à¸ŀรà¹īà¸Ńมà¸ģัà¸ļ":136461,"Ø´Ùħ":136462,"à¸ĸืà¸Ńวà¹Īา":136463,"è¾¼ãĤĢ":136464,"ä¼ijãģ¿":136465,"ĠاÙĦأب":136466,"ĠÑģÑĤоимоÑģÑĤÑĮ":136467,"ĠпÑĢава":136468,"mayın":136469,"หวย":136470,"ĠاÙĦطبÙĬعÙĬ":136471,"à¸Ĺีà¹Īà¸ŀัà¸ģ":136472,"ĠEstá":136473,"ÑĭваÑİÑĤ":136474,"بسÙĬ":136475,"بسÙĬØ·":136476,"Ġ×ij×¢×ijר":136477,"åı¯èĥ½ãģ§ãģĻ":136478,"Ġ×ĵ×ķ׾":136479,"Ġ×ĵ×ķ׾ר":136480,"ÙĩÙİا":136481,"воÑĢоÑĤ":136482,"ãģ¦ãģĦãģ¾ãģĹãģŁ":136483,"à¹Ĥà¸Ĺรศ":136484,"à¹Ĥà¸Ĺรศั":136485,"à¹Ĥà¸Ĺรศัà¸ŀ":136486,"à¹Ĥà¸Ĺรศัà¸ŀà¸Ĺà¹Į":136487,"Ġק׳":136488,"ĠاÙĦØ«ÙĨ":136489,"ĠاÙĦØ«ÙĨائÙĬØ©":136490,"Ġcoût":136491,"à¸ķิà¸Ķà¸ķัà¹īà¸ĩ":136492,"Ġörg":136493,"Ġörgüt":136494,"ĠاÙĦØ®ÙĦÙĬ":136495,"ĠاÙĦØ®ÙĦÙĬج":136496,"Ġbá»įn":136497,"×ķ׾×ķ×Ĵ×Ļ":136498,"ëŀľ":136499,"ĠÐijолÑĮ":136500,"ĠÐijолÑĮÑĪ":136501,"×Ĵ×ijר×Ļ×Ŀ":136502,"ÙĤÙĬد":136503,"×ij×Ļ×ĺ×ķ×Ļ":136504,"æīĵãģ¡":136505,"ĠolmuÅŁ":136506,"fäh":136507,"fähig":136508,"ลาà¸Ļ":136509,"ĠÙĤطر":136510,"שפ×Ķ":136511,"èªŃãĤĵãģ§":136512,"à¸Ĥวา":136513,"Ġchiếm":136514,"ãĤ¤ãĥ³ãĤ¿":136515,"ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥ":136516,"ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥį":136517,"ãĤ¤ãĥ³ãĤ¿ãĥ¼ãĥįãĥĥãĥĪ":136518,"Ġ׾ש×ŀ×ķר":136519,"ĠترÙĥ":136520,"ĠترÙĥÙĬا":136521,"ר×ķ×ĺ":136522,"ãģ¨æĢĿãģĦãģ¾ãģĹãģŁ":136523,"ĠاÙĦتÙĤ":136524,"ĠdÆ°":136525,"ãģ¦ãģıãĤĮãĤĭ":136526,"ãģĹãģŁãģĵãģ¨":136527,"Ġróżne":136528,"ĠاÙĦØ·ÙģÙĦ":136529,"ĠPosté":136530,"Ġ×ŀש×ķ×Ŀ":136531,"ÑįÑĢ":136532,"ĠÑĢабоÑĤаеÑĤ":136533,"ãĤ·ãĥª":136534,"ãĤ·ãĥªãĥ¼ãĤº":136535,"Ġ×ij×Ķ×Ĺ׾×ĺ":136536,"ק×Ķ×Ļ׾×Ķ":136537,"ãĤ«ãĥ¡":136538,"ãĤ«ãĥ¡ãĥ©":136539,"Ｏ":136540,"ĠìĤ¬ìĿ´":136541,"Ġkì":136542,"ĠthÆ°á»Ľc":136543,"ضبط":136544,"ÙĤبÙĪÙĦ":136545,"åĪ¥ãģ®":136546,"Ġparticulière":136547,"ĠÑģвоем":136548,"Ġעסק":136549,"Ġעסק×Ļ×Ŀ":136550,"×ij×Ĺ×Ļר×ķת":136551,"×ij×Ļ׳×ķ":136552,"à¸ĭà¸Ń":136553,"Ġ×¢×ķ×ijר":136554,"ãģłãģ£ãģŁãģ®ãģ§":136555,"ıldıģı":136556,"Ùħدار":136557,"Ùħدارس":136558,"주ìĭľ":136559,"à¸Ńาศ":136560,"à¸Ńาศัย":136561,"Ġtấm":136562,"à¸ŀิà¸Ī":136563,"à¸ŀิà¸Īาร":136564,"à¸ŀิà¸Īารà¸ĵา":136565,"ÑĤелÑĮнÑĭе":136566,"ÑģкÑĥÑİ":136567,"ÐľÐĺ":136568,"à¹Ģà¸ģา":136569,"à¹Ģà¸ģาหล":136570,"à¹Ģà¸ģาหลี":136571,"×ĵ×Ĺ":136572,"à¹Ģà¸Ĭิà¸ĩ":136573,"ĠدÙĤÙĬÙĤØ©":136574,"íķĻìĥĿ":136575,"Ġש×IJ׾×Ķ":136576,"Ġcontrôle":136577,"Ġsituação":136578,"à¸Ĥà¸Ńà¸ĩà¸ľà¸¹à¹ī":136579,"ÙĨØ·ÙĤ":136580,"ê³¼íķĻ":136581,"หลายà¸Ħà¸Ļ":136582,"Ġnắng":136583,"ÙĤÙı":136584,"ì¡°ê±´":136585,"Ñķ":136586,"ãĥĥãģ¨":136587,"×ŀ×Ļ׾×Ķ":136588,"Grün":136589,"×Ļ×Ļ×¢":136590,"×Ļ×Ļ×¢×ķ×¥":136591,"×ŀ׳׼":136592,"ëŃIJ":136593,"×ŀ×¢×ŀ×ĵ":136594,"สำà¸Ļัà¸ģ":136595,"جدد":136596,"à¸Ħัà¸Ķ":136597,"Ġ×Ķ×ŀשפ":136598,"Ġ×Ķ×ŀשפ×Ĺ×Ķ":136599,"×ŀשק׾":136600,"ÙĦÙı":136601,"Ġtytu":136602,"ĠtytuÅĤ":136603,"ÑĪей":136604,"ĠìĿ¼ë¶Ģ":136605,"ÑĪение":136606,"Ġphóng":136607,"ĠìĹŃìĤ¬":136608,"ãĤ«ãĥ³":136609,"Ġtúi":136610,"ĠÙĨÙĪÙģ":136611,"ĠÙĨÙĪÙģÙħبر":136612,"grün":136613,"ĠاÙĦØ´ÙħاÙĦ":136614,"ÅĽwiadc":136615,"ÅĽwiadczenie":136616,"ער×Ķ":136617,"Ġ×¢×ķ×ij":136618,"Ġ×¢×ķ×ij×ĵ×Ļ×Ŀ":136619,"×ĵ×ķ×Ĵ×ŀ×IJ":136620,"ä»Ĭãģ¯":136621,"Ġvão":136622,"ĠТем":136623,"ÑģилÑĮ":136624,"Ġchợ":136625,"Ùħرا":136626,"ÙħراÙĤب":136627,"à¹Ħมà¹Īรูà¹ī":136628,"Ġرائع":136629,"×IJ׳×Ĺ׳×ķ":136630,"สà¹Īà¸ĩà¹Ģสริม":136631,"צ×Ĺ":136632,"ĠìŀĪìĸ´ìĦľ":136633,"Ġkurulu":136634,"ĠkuruluÅŁ":136635,"ĠÃĸzellik":136636,"ĠÃĸzellikle":136637,"Ġת×Ļק":136638,"Ġghé":136639,"ĠsprzÄĻ":136640,"ĠsprzÄĻt":136641,"ער×ķת":136642,"راØŃØ©":136643,"ãģ£ãģį":136644,"ãģ£ãģįãĤĬ":136645,"ĠìķĦëŀĺ":136646,"stituição":136647,"Ġдолжно":136648,"×Ķרש":136649,"×Ķרש×ŀ×Ķ":136650,"×Ķ׾×ļ":136651,"ãģ¡ãģª":136652,"ãģ¡ãģªãģ¿":136653,"ãģ¡ãģªãģ¿ãģ«":136654,"פ×Ĺ×ĵ":136655,"ĠاÙĦجÙħÙĬع":136656,"×ij×¢×ľ×Ļ":136657,"Ġtrùng":136658,"Ġפת×Ĺ":136659,"×ŀ׾×Ĺ×ŀת":136660,"ãĥĨãĥ¼ãĥ":136661,"ãĥĨãĥ¼ãĥŀ":136662,"Ùħتاب":136663,"Ùħتابعة":136664,"Ġ모ìĬµ":136665,"ÙĬص":136666,"åIJĪãģĨ":136667,"ĠYap":136668,"ĠYapı":136669,"ĠÑģказаÑĤÑĮ":136670,"몰":136671,"à¸Ĺีà¹Īสำà¸Ħัà¸į":136672,"ĠìĹĨìĬµëĭĪëĭ¤":136673,"Ġnhắc":136674,"Ġülkeler":136675,"Ġмногие":136676,"íķĺìħ¨":136677,"มาà¸ģà¸Ĺีà¹Īสุà¸Ķ":136678,"à¸ģà¹īา":136679,"à¸ģà¹īาว":136680,"ĠÄ°yi":136681,"леж":136682,"лежа":136683,"ãĤ¸ãĥ§":136684,"à¸Ĺัà¸ŀ":136685,"اÙĪر":136686,"Ġ×Ĺ×ijר×Ļ":136687,"Ġ׾ש×Ŀ":136688,"첫":136689,"ĠTá»Ń":136690,"×ŀ×ķ׳×Ļ":136691,"ÙĤÙĪد":136692,"à¸ģระà¹Ģà¸Ľ":136693,"à¸ģระà¹Ģà¸Ľà¹ĭ":136694,"à¸ģระà¹Ģà¸Ľà¹ĭา":136695,"ĠпÑĢоблемÑĭ":136696,"Ġaçıs":136697,"Ġaçısından":136698,"Ġ×Ķ×ŀ׼":136699,"ĠÙħعظÙħ":136700,"ÙĤÙĬاس":136701,"ĠпÑĢодолж":136702,"ĠпÑĢодолжа":136703,"ĠverdiÄŁi":136704,"ĠпÑĢедмеÑĤ":136705,"ãģĦãģ¾ãģĻãģĮ":136706,"ĠëĶ°ë¥¸":136707,"ĠاÙĦÙĤÙĬاÙħ":136708,"ĠØ¥ÙĦÙĬÙĩا":136709,"ТÐIJ":136710,"поз":136711,"ãĤ·ãĥ¥":136712,"ä¸ĬãģĮãĤĬ":136713,"à¹Ģà¸Ķิมà¸ŀัà¸Ļ":136714,"à¸ģุล":136715,"ØŃرÙĬØ©":136716,"ק×ij×ķצ×ķת":136717,"믿":136718,"ĠاÙĦÙħÙĨا":136719,"ĠاÙĦÙħÙĨاطÙĤ":136720,"ĠвÑĭпол":136721,"ĠвÑĭполнÑı":136722,"ãĥĭãĤ¢":136723,"Ġê²°êµŃ":136724,"×Ĺ×ķ×ŀ":136725,"×Ĺ×ķ×ŀר×Ļ×Ŀ":136726,"ĠУкÑĢаинÑĭ":136727,"หà¸Ńม":136728,"ר×Ļס":136729,"ĠÑħоÑĤел":136730,"ĠобÑĢазованиÑı":136731,"Ġkhẳng":136732,"ĠmÆ°a":136733,"Ġgörme":136734,"Ġgüçlü":136735,"سعÙī":136736,"มัà¹Īà¸Ļà¹ĥà¸Ī":136737,"íķĺê²łìĬµëĭĪëĭ¤":136738,"ĠполÑĥ":136739,"Ġfünf":136740,"ãģ¨æĢĿãģ£ãģ¦ãģĦãģ¾ãģĻ":136741,"Ġê·¸ê²ĥìĿĢ":136742,"ĠdÃ¼ÅŁÃ¼nce":136743,"ìŀł":136744,"ĠHÆ°á»Ľng":136745,"ĠTiá»ĥu":136746,"Ġçift":136747,"ãģijãģ°":136748,"à¸Īà¸Ļà¸ĸึà¸ĩ":136749,"à¸Ĺำà¹Ħà¸Ķà¹ī":136750,"ĠìŀIJì²´":136751,"Ġdõ":136752,"Ġdõi":136753,"à¸Īัà¸Ļ":136754,"à¸Īัà¸Ļà¸Ĺ":136755,"à¸Īัà¸Ļà¸Ĺรà¹Į":136756,"eceÄŁini":136757,"׳×ķער":136758,"غار":136759,"ĠاÙĦØ£ÙħرÙĬÙĥÙĬ":136760,"داعش":136761,"ĠбезопаÑģноÑģÑĤи":136762,"ĠбÑİ":136763,"ĠбÑİдж":136764,"ĠбÑİджеÑĤ":136765,"ãĥĬãĤ¤":136766,"à¸ŀà¸ļวà¹Īา":136767,"daÄŁ":136768,"×IJ×ķפף":136769,"íĹĮ":136770,"ãĥĢãĤ¤ãĤ¨":136771,"ãĥĢãĤ¤ãĤ¨ãĥĥãĥĪ":136772,"ĠëĮĢíĨµ":136773,"ĠëĮĢíĨµëł¹":136774,"DÄ°":136775,"Ø£ØŃداث":136776,"ĠAÄŁ":136777,"ĠAÄŁust":136778,"ĠAÄŁustos":136779,"ØŃÙĦÙĪÙĦ":136780,"ĠwÅĽ":136781,"ĠwÅĽród":136782,"ĠÑģооÑĤвеÑĤ":136783,"ĠÑģооÑĤвеÑĤÑģÑĤв":136784,"ĠÑģооÑĤвеÑĤÑģÑĤвии":136785,"ĠLuáºŃt":136786,"Ġ׼׾פ×Ļ":136787,"ĠвеÑī":136788,"ĠвеÑīеÑģÑĤв":136789,"ק×Ļ×¥":136790,"ĠبÙĩذا":136791,"عاش":136792,"à¹Ģà¸Ľà¹ĩà¸Ļà¹Ģรืà¹Īà¸Ńà¸ĩ":136793,"ТÐķ":136794,"Ġ×ij×IJ×Ļ׳×ĺר׳×ĺ":136795,"سعد":136796,"Ġ×Ķ×ĺ×Ļפ×ķ׾":136797,"פ×Ļס":136798,"à¸ĩà¹Īายà¹Ĩ":136799,"ĠGerät":136800,"׾×Ļ×ĵ×Ķ":136801,"ĠÑĢиÑģк":136802,"׾ק×Ĺ":136803,"ннаÑı":136804,"ר×Ļ×ĵ":136805,"пÑĢакÑĤи":136806,"пÑĢакÑĤик":136807,"à¸Ĥัà¹īà¸Ļà¸ķà¸Ńà¸Ļ":136808,"à¸Ļà¹Īารัà¸ģ":136809,"larınızı":136810,"à¸Ńà¸Ļุà¸įา":136811,"à¸Ńà¸Ļุà¸įาà¸ķ":136812,"ĠzdjÄĻcia":136813,"Ġbây":136814,"ÑģÑĢ":136815,"ÑģÑĢоÑĩ":136816,"ãĥĭãĥ³ãĤ°":136817,"Ġöner":136818,"Ġöneri":136819,"ĠновÑĭÑħ":136820,"دعÙĪØ©":136821,"Ġgắn":136822,"ĠاÙĦÙĦبÙĨ":136823,"ĠاÙĦÙĦبÙĨاÙĨÙĬ":136824,"ãĥĨãĤ£ãĥ¼":136825,"ĠصØŃÙĬØŃ":136826,"емÑĭÑħ":136827,"çĸ²ãĤĮ":136828,"ĠпÑĢоиÑģ":136829,"ĠпÑĢоиÑģÑħодиÑĤ":136830,"สà¸ķิ":136831,"ĠTết":136832,"Ġ×Ķ׾׾×ķ":136833,"à¹Ģรืà¹Īà¸Ńà¸ĩà¸Ļีà¹ī":136834,"×ŀ×ij׳×Ķ":136835,"Ġconteúdo":136836,"Ġاخت":136837,"ĠاختÙĬار":136838,"ÙħسÙĦ":136839,"ÙħسÙĦسÙĦ":136840,"ëıĪ":136841,"Ġ׾×Ļ×ĵ":136842,"à¸ŀิà¸ĺี":136843,"ĠÑģовÑģ":136844,"ĠÑģовÑģем":136845,"ãģĮãģĤãĤĬãģ¾ãģĹãģŁ":136846,"Ġsóng":136847,"إصÙĦاØŃ":136848,"ë§ģ":136849,"ÙģÙĬر":136850,"ĠJeżeli":136851,"ìłľëıĦ":136852,"dÅĤug":136853,"ìĥģìĿĦ":136854,"ĠcáºŃn":136855,"Ġhá»įp":136856,"أست":136857,"أستاذ":136858,"Ġ×ŀ×Ļש×Ķ":136859,"Ġ×ŀ×Ļש×Ķ×ķ":136860,"ĠdÃły":136861,"ĠchÃłng":136862,"ãģ¡ãĤĥãĤĵãģ¨":136863,"ĠÄijám":136864,"Ġswój":136865,"Ġpoderá":136866,"ĠоÑĤлиÑĩа":136867,"Ġpériode":136868,"ündig":136869,"×ĺ×¢×Ł":136870,"ÑģÑĤÑĢоиÑĤелÑĮ":136871,"רת×Ļ":136872,"Ġ×Ļ×Ķ×Ļ×ķ":136873,"×ľ×¡":136874,"ĠاÙĦÙħÙĨزÙĦ":136875,"à¸Ļิà¹īว":136876,"иÑĦика":136877,"иÑĦикаÑĨи":136878,"ðŁĺī":136879,"Ġadına":136880,"ãĢĤãĢĤãĢĤ":136881,"×IJ×Ļף":136882,"ס×Ļר":136883,"ĠÙĬعد":136884,"çŃĶãģĪ":136885,"اÙĦجز":136886,"اÙĦجزائر":136887,"енÑĮк":136888,"รห":136889,"รหัส":136890,"ĠTürkçe":136891,"꾸":136892,"Ġ×Ļ×ķ׼׾":136893,"Ġש×ķ׳×Ķ":136894,"Ġ×ij×ŀצ×ij":136895,"ĠдейÑģÑĤвиÑĤелÑĮно":136896,"ĠبأÙĨÙĩ":136897,"×ŀק×ĵ":136898,"Ġ×Ķשק":136899,"Ø®ÙĬارات":136900,"Ġfı":136901,"Ġfırs":136902,"Ġfırsat":136903,"ëijĺ":136904,"ĠìĦľìļ¸":136905,"Ġ×Ķ×Ĵ×ķ×£":136906,"رعا":136907,"رعاÙĬØ©":136908,"ĠKết":136909,"кÑģи":136910,"ĠÑĥÑģлÑĥги":136911,"ноÑģÑĤей":136912,"ìļ´ëıĻ":136913,"ĠобÑĬÑı":136914,"ĠобÑĬÑıвл":136915,"неж":136916,"×Ķפ×ļ":136917,"Ġ×ij×¢×Ļ׳×Ļ":136918,"ëĨĴ":136919,"ĠпÑĢоÑĨед":136920,"ĠпÑĢоÑĨедÑĥÑĢ":136921,"Ġihtiy":136922,"Ġihtiyacı":136923,"Ġë°Ķëŀį":136924,"Ġë°ĶëŀįëĭĪëĭ¤":136925,"à¸ģลัว":136926,"ĠÑģложно":136927,"ק×Ļ×Ļ×ŀת":136928,"ĠÄIJình":136929,"ĠÙħÙĦÙģ":136930,"Ġà¹Ĥà¸Ķยมี":136931,"Ġkatkı":136932,"تØŃÙĪÙĬÙĦ":136933,"à¹Ħà¸ŀ":136934,"ĠHá»į":136935,"ñe":136936,"ĠдоÑħод":136937,"Ġthoải":136938,"íķĺìŬìķ¼":136939,"ãĤ¹ãĥĿãĥ¼ãĥ":136940,"ãĤ¹ãĥĿãĥ¼ãĥĦ":136941,"ĠGòn":136942,"Ġkè":136943,"Ġkèm":136944,"éĢ²ãĤģ":136945,"ãĤ¹ãĥ¼ãĥ":136946,"ãĤ¹ãĥ¼ãĥij":136947,"ãĤ¹ãĥ¼ãĥijãĥ¼":136948,"ĠgiÃłu":136949,"Ġإعادة":136950,"Ġ׾×ķק":136951,"Ġ׾×ķק×Ĺ":136952,"ĠÑħоÑĩеÑĤ":136953,"×ĺ׾×ķ×ķ":136954,"×ĺ׾×ķ×ķ×Ļ×ĸ":136955,"×ĺ׾×ķ×ķ×Ļ×ĸ×Ļ×Ķ":136956,"Ġthuyết":136957,"ãģĿãĤĮãģ§":136958,"Ġvardı":136959,"à¹Ħรà¹ī":136960,"عبد":136961,"ĠRepública":136962,"ãĥ¼ãĤ¿ãĥ¼":136963,"Ġ×ŀ×IJ×ķת":136964,"à¹Ħà¸Ľà¹ģลà¹īว":136965,"Ġyapılacak":136966,"ãĤ¹ãĤ¿ãĥ¼ãĥĪ":136967,"ãģ»ãģ¼":136968,"ĠkoÅŁ":136969,"ĠмаÑĤеÑĢи":136970,"Ġsiècle":136971,"ĠاÙĦÙħختÙĦÙģ":136972,"ĠاÙĦÙħختÙĦÙģØ©":136973,"Ġ׾קר×IJ":136974,"Ġ׾קר×IJת":136975,"Ġ×Ķפ×ķ×¢×ľ":136976,"Ġtòa":136977,"ĠrÆ¡i":136978,"åij¨ãĤĬ":136979,"à¸Ŀà¸Ļ":136980,"jÅĽÄĩ":136981,"ĠìķĬìĿĦ":136982,"اÙĨتÙĤاÙĦ":136983,"ëĸł":136984,"иваеÑĤ":136985,"ãĥĪãĥ«":136986,"ĠاÙĦÙģÙĦسطÙĬÙĨÙĬØ©":136987,"à¸ģลà¹Īาววà¹Īา":136988,"اÙĥت":136989,"ĠÃĸl":136990,"ĠÑĢеÑĪи":136991,"ĠÑĢеÑĪил":136992,"Ġ׳×ķספ×ķת":136993,"Ġìłķì¹ĺ":136994,"влеÑĩен":136995,"ÙħرØŃÙĦØ©":136996,"Ġcomeça":136997,"Ġyık":136998,"ìĤ´":136999,"à¸ĺà¸Ļา":137000,"à¸ĺà¸Ļาà¸Ħาร":137001,"à¸Ńà¸Ļา":137002,"à¸Ńà¸Ļาà¸Ħ":137003,"à¸Ńà¸Ļาà¸Ħà¸ķ":137004,"Ġpequeña":137005,"ä»ķäºĭãĤĴ":137006,"ĠبذÙĦÙĥ":137007,"Ġнового":137008,"ãģĹãģ¦ãģĦãģªãģĦ":137009,"ĠاÙĦÙħÙĬاÙĩ":137010,"à¸ģà¹ĩà¹Ģà¸Ľà¹ĩà¸Ļ":137011,"ĠжÑĥÑĢ":137012,"ĠжÑĥÑĢнал":137013,"веÑģ":137014,"ختار":137015,"Ġ매ìļ°":137016,"ĠMã":137017,"ĠавÑĤомаÑĤÑĭ":137018,"ضعÙģ":137019,"ĠاÙĦÙģÙĥر":137020,"ãģ§ãģĻãģ®ãģ§":137021,"ãĥ¡ãĥ³ãĥIJãĥ¼":137022,"ĠкÑĢÑĥг":137023,"ĠاÙĦسÙĦطة":137024,"à¸Ħรัà¹īà¸ĩà¹ģรà¸ģ":137025,"à¸ģระà¸Ĺรว":137026,"à¸ģระà¸Ĺรวà¸ĩ":137027,"ÑĨов":137028,"éķ·ãģĦ":137029,"大ãģįãģĦ":137030,"ĠgeçmiÅŁ":137031,"ìĦ±ìĿ´":137032,"Ġצר×Ļ׼×Ķ":137033,"ĠмоÑī":137034,"ĠмоÑīн":137035,"Ġק×Ļש":137036,"Ġק×Ļש×ķר×Ļ×Ŀ":137037,"ĠNasıl":137038,"гÑĢан":137039,"Ġ×ŀ×ķצר×Ļ×Ŀ":137040,"Ġ×ŀס×ķ×Ĵ":137041,"Ġyür":137042,"Ġyürüt":137043,"Ġ׾×Ĺצ×ķ":137044,"×ķÖ¼":137045,"ĠìŀĪìĹĪëĭ¤":137046,"Ġterör":137047,"ĠThÆ°Æ¡ng":137048,"ĠÙĪÙĬÙħ":137049,"ĠÙĪÙĬÙħÙĥÙĨ":137050,"جÙĪÙĨ":137051,"ĠÙĪغÙĬرÙĩا":137052,"×ŀפ×ķ":137053,"×Ĵ×ķר×ŀ×Ļ×Ŀ":137054,"׼×ij×Ļש":137055,"ĠاÙĦÙĦغ":137056,"ĠاÙĦÙĦغة":137057,"شرÙĥ":137058,"ĠاÙĦراب":137059,"ĠاÙĦرابع":137060,"ĠпÑĢек":137061,"ĠпÑĢекÑĢаÑģ":137062,"ĠпÑĢекÑĢаÑģн":137063,"ĠenergÃŃa":137064,"ק×ĵ×ŀ×Ļ":137065,"ãģıãģªãģ£ãģŁ":137066,"ĠÄijứ":137067,"ĠÄijứa":137068,"Servi":137069,"Serviço":137070,"Ġkaldır":137071,"åĥįãģį":137072,"Ġодеж":137073,"Ġодежд":137074,"물ìĿĦ":137075,"ãģĿãģĨãģ§":137076,"ãģĮãģĤãĤĮãģ°":137077,"ìĻķ":137078,"צ×ĵק":137079,"Ġartır":137080,"Ġileti":137081,"ĠiletiÅŁim":137082,"ãĤĪãģĨãģ§":137083,"ãĥĪãĥ¼":137084,"ãĤ¢ãĥĭ":137085,"ãĤ¢ãĥĭãĥ¡":137086,"×ĺ×Ļ×Ļ׾":137087,"ãĥķãĥªãĥ¼":137088,"ãĥĿãĥ³":137089,"ÐŁÑĢо":137090,"ĠعاÙĦÙĬØ©":137091,"ĠÃ¶ÄŁret":137092,"ĠÃ¶ÄŁretmen":137093,"ĠкаÑĩеÑģÑĤва":137094,"Ġ×Ķ×ĺ×ij×¢":137095,"ĠзнаÑİ":137096,"ãģ¦ãģıãĤĭ":137097,"Ġmừng":137098,"ÙħÙĪت":137099,"ש×ķ×ŀר":137100,"×Ĺ׾×ij":137101,"ĠwzglÄĻ":137102,"ĠwzglÄĻdu":137103,"ë²Ī째":137104,"Ġtá»ĵ":137105,"Ġtá»ĵn":137106,"ãĥ¯ãĥ¼ãĤ¯":137107,"Ġpożycz":137108,"Ġpożyczk":137109,"×Ļ×ķצר×Ļ×Ŀ":137110,"ÙĥرÙħ":137111,"ĠгаÑĢ":137112,"ĠгаÑĢан":137113,"ĠгаÑĢанÑĤи":137114,"ลà¹īาà¸ĩ":137115,"ĠìĺģíĻĶ":137116,"×ĺ×Ļס":137117,"Ġthẻ":137118,"ĠìŀĪëĭ¤ê³ł":137119,"اÙĦتز":137120,"اÙĦتزاÙħ":137121,"ĠнаÑĪи":137122,"isée":137123,"ãģĵãĤĮãĤĴ":137124,"Ġmẽ":137125,"ضÙĦ":137126,"بÙĪت":137127,"Ġ׼׼×Ķ":137128,"hợ":137129,"ĠاÙĦسÙĪرÙĬØ©":137130,"Ġ×ľ×¢×ķ×ŀ":137131,"Ġ×ľ×¢×ķ×ŀת":137132,"ĠbaÅŁar":137133,"ĠbaÅŁarılı":137134,"еÑģÑĤÑĮ":137135,"à¸Ħรี":137136,"à¸Ħรีม":137137,"ĠìłĦì²´":137138,"ĠسÙĬÙĥÙĪÙĨ":137139,"Ġ×ŀ×ĵ×ķ×¢":137140,"ĠëķĮ문ìĿ´ëĭ¤":137141,"Ġcứng":137142,"gerät":137143,"ĠмиÑĢ":137144,"ĠмиÑĢе":137145,"ĠÙĥÙĬÙģÙĬØ©":137146,"Ġפר×ĺ×Ļ×Ŀ":137147,"ĠgoÅĽci":137148,"иÑĤеÑģÑĮ":137149,"ÑĥÑĪки":137150,"ؤÙħÙĨ":137151,"Ġ×IJ׼ף":137152,"ĠاÙĦرجÙĦ":137153,"Ġlá»įc":137154,"à¹Ģรียà¸ģวà¹Īา":137155,"ãģĵãģ®ãĤĪãģĨãģª":137156,"ë§Įíģ¼":137157,"ĠпеÑĩ":137158,"ÙĪÙĦات":137159,"ĠÃľye":137160,"liÄŁinde":137161,"à¸Ħะà¹ģà¸Ļ":137162,"à¸Ħะà¹ģà¸Ļà¸Ļ":137163,"ãĤĭãģĵãģ¨ãģ¯":137164,"วิà¹Ģà¸Ħร":137165,"วิà¹Ģà¸Ħราะ":137166,"วิà¹Ģà¸Ħราะหà¹Į":137167,"ĠвозможноÑģÑĤи":137168,"ĠاÙĦÙĨساء":137169,"ãĥīãĥ©ãĥŀ":137170,"Ġgüc":137171,"Ġgücü":137172,"ĠtÆ°á»Ŀng":137173,"Ġacompaña":137174,"ãĤ¤ãĥ©":137175,"קצ×ij":137176,"ĠYö":137177,"ĠYönet":137178,"ĠYönetim":137179,"à¸ªà¸±à¸¡à¸ľ":137180,"à¸ªà¸±à¸¡à¸ľà¸±à¸ª":137181,"à¸Ļาม":137182,"ĠÄijợi":137183,"à¹ģหà¹Īà¸ĩà¸Ĭาà¸ķิ":137184,"ãģĿãĤĮãģ§ãĤĤ":137185,"ätig":137186,"ת×ķ×Ŀ":137187,"ĠbaÅŁlat":137188,"ĠвÑģей":137189,"ת×Ļק":137190,"ת×Ļק×ķף":137191,"ĠNgô":137192,"ĠGeschä":137193,"ĠGeschäfts":137194,"Ø£Ùħ":137195,"Ø£Ùħراض":137196,"à¹Ģà¸Ĺà¸Ħà¸Ļ":137197,"à¹Ģà¸Ĺà¸Ħà¸Ļิ":137198,"à¹Ģà¸Ĺà¸Ħà¸Ļิà¸Ħ":137199,"ĠменÑĮ":137200,"ĠменÑĮÑĪе":137201,"Ġölç":137202,"Ġölçü":137203,"ĠÙĬجعÙĦ":137204,"ĠÄijỡ":137205,"ש×Ļ׾":137206,"ש×Ļ׾×ķ×ij":137207,"ĠGrÃ¶ÃŁe":137208,"ĠÙĩاتÙģ":137209,"รà¹īาà¸Ļà¸Ńาหาร":137210,"×Ķ׾×Ļ׼":137211,"×Ķ׾×Ļ׼×Ļ":137212,"иÑĢÑĥÑİÑī":137213,"èĭ¥ãģĦ":137214,"ĠÃĸzel":137215,"ãģĦãģŁãĤī":137216,"à¸Ħำà¸ĸาม":137217,"ĠzostaÅĤy":137218,"Ġ×Ķס×Ļפ×ķר":137219,"×Ķ×ķ׾":137220,"×Ķ×ķ׾×ļ":137221,"à¹Ģà¸Ĭà¹Īà¸Ļà¸ģัà¸Ļ":137222,"à¹Ĥà¸Ĩ":137223,"à¹Ĥà¸Ĩษ":137224,"à¹Ĥà¸Ĩษà¸ĵา":137225,"×IJרצ×ķת":137226,"×Ĵרפ×Ļ":137227,"Ġaoût":137228,"ĠÙĬرÙĬد":137229,"تÙĪج":137230,"تÙĪجÙĬÙĩ":137231,"ĠÑįÑĤап":137232,"ãĤ¹ãĤ¿ãĥ³":137233,"Ġkró":137234,"Ġkrótk":137235,"ãĤĴ使ãģĨ":137236,"ì·¨":137237,"éĸ¢ãĤı":137238,"à¸Ķà¹īวยà¸Ħวาม":137239,"à¸Ļำà¹Ģสà¸Ļà¸Ń":137240,"Ġayrıca":137241,"à¸Īà¹īาà¸ĩ":137242,"ĠÑĦоÑĤогÑĢаÑĦ":137243,"ĠвеÑĩ":137244,"ĠвеÑĩеÑĢ":137245,"åĩºãģĹãģŁ":137246,"ĠХо":137247,"Ġ×ŀר×Ĵ×Ļש":137248,"à¹ĥหà¹īà¹Ģà¸Ľà¹ĩà¸Ļ":137249,"ãĤĴ缮":137250,"ãĤĴ缮æĮĩ":137251,"׾×ŀ×Ļ×Ŀ":137252,"nÄħÅĤ":137253,"ĠÑģÑĤанд":137254,"ĠÑģÑĤандаÑĢÑĤ":137255,"ĠSüd":137256,"ĠTâm":137257,"اختبار":137258,"à¹Ģà¸ģà¸Ńรà¹Į":137259,"ÙħسرØŃ":137260,"Ġbiá»ĩn":137261,"بÙı":137262,"ĠصاÙĦ":137263,"ĠصاÙĦØŃ":137264,"ĠPhụ":137265,"íľ´":137266,"ãĥ¬ãĥĵãĥ¥ãĥ¼":137267,"Ġbụng":137268,"Ġrégime":137269,"ĠأشÙĩر":137270,"ĠÑĢабоÑĤник":137271,"à¸Ŀัà¸Ļ":137272,"اعتÙħ":137273,"اعتÙħاد":137274,"ĠзамеÑĤ":137275,"ãģ¾ãģ£ãģ¦":137276,"Ġchặt":137277,"æĿ¥ãĤĭ":137278,"ĠاÙĦÙĤÙĪات":137279,"ãģ«åħ¥ãģ£ãģ¦":137280,"تØŃاÙĦÙģ":137281,"ÙħزÙĬد":137282,"ĠÙĬصÙĦ":137283,"ìĹ¼":137284,"à¹Ģà¸Ĭà¹ĩ":137285,"à¹Ģà¸Ĭà¹ĩà¸Ħ":137286,"Ġká»ĭ":137287,"Ġká»ĭp":137288,"ĠìķĦì§ģ":137289,"×IJ׳×Ĵ":137290,"ĠоблаÑģÑĤÑĮ":137291,"ĠpomocÄħ":137292,"Ġ×ķש׾":137293,"ëĵłì§Ģ":137294,"ĠGiám":137295,"ĠStück":137296,"Ġcháy":137297,"ĠëĤĺìĺ¤":137298,"ש×Ļ×ĺת":137299,"×ŀ×ĵר":137300,"×ŀ×ĵר×Ļ×ļ":137301,"Ġsüreç":137302,"ква":137303,"×ij׾×Ļ×Ŀ":137304,"×Ķת×Ļ":137305,"×Ķת×Ļ×Ļ×Ĺס":137306,"ÙĤباÙĦ":137307,"Ġס×ķ×Ĵ":137308,"Ġס×ķ×Ĵ×Ļ":137309,"ÑģÑĤолÑĮ":137310,"ä½ķãĤĤ":137311,"×ĸ׼×ķר":137312,"è²·ãģĨ":137313,"å®īãģı":137314,"à¸Ħรัà¹īà¸ĩà¸Ļีà¹ī":137315,"köp":137316,"ĠÑģеÑĢвиÑģ":137317,"оÑĩнÑĭÑħ":137318,"ê±°ëŀĺ":137319,"تأÙĥ":137320,"تأÙĥÙĬد":137321,"×ĵ׾ק":137322,"ĠпоÑĩем":137323,"ĠпоÑĩемÑĥ":137324,"пиÑģаÑĤÑĮ":137325,"×ijשר":137326,"ĠHÃłng":137327,"ĠTìm":137328,"Ġtrừ":137329,"ãĤ»ãĥĥãĤ¯ãĤ¹":137330,"×ķ׳×Ĵ":137331,"mızda":137332,"пÑģи":137333,"ĠìŀĪ기":137334,"Ġrút":137335,"زاÙĨ":137336,"تÙĨÙĪع":137337,"ÙħÙĤا":137338,"ÙħÙĤاÙĪÙħØ©":137339,"Ġ׾צ×ķר×ļ":137340,"Ġ×ij×Ļר×ķש׾×Ļ×Ŀ":137341,"ãĥ´ãĤ£":137342,"ebile":137343,"ebileceÄŁi":137344,"ãĥ¦ãĥ¼ãĤ":137345,"ãĥ¦ãĥ¼ãĤ¶":137346,"ãĥ¦ãĥ¼ãĤ¶ãĥ¼":137347,"ãĤĴä½ľãĤĭ":137348,"ÑģмеÑĢ":137349,"ÑģмеÑĢÑĤ":137350,"Ġì§ģ":137351,"Ġì§ģìłij":137352,"ĠÐŁÐ°ÑĢ":137353,"ØŃاض":137354,"ØŃاضر":137355,"ÙħÙĥاÙģ":137356,"ÙħÙĥاÙģØŃØ©":137357,"ลิà¸Ļ":137358,"ãģ¦ãģįãģ¦":137359,"ÑĢоÑģл":137360,"ĠÄ°ÅŁte":137361,"ÙĤصÙĬر":137362,"Ġ×ij×Ĵ×Ļ׾":137363,"Ġ×ŀת×IJ×Ļ×Ŀ":137364,"Ġ×Ķ×Ĺ×ĵ":137365,"Ġ×Ķ×Ĺ×ĵש×Ķ":137366,"ר×ķ×¢":137367,"Ġproduktów":137368,"ĠÙħصدر":137369,"неÑĨ":137370,"ĠاÙĦعÙħÙĦات":137371,"Ġçıkma":137372,"ĠدبÙĬ":137373,"ק×Ļף":137374,"ת×IJר":137375,"ת×IJר×Ļ×ļ":137376,"׳×Ļ×Ļ×ĵ":137377,"صراع":137378,"lève":137379,"צ×Ļר":137380,"à¸Ķัà¸Ļ":137381,"à¹ĥหà¹īà¹Ħà¸Ķà¹ī":137382,"ãĤ¿ãĤ¤ãĥł":137383,"Ġgiảng":137384,"Ð¡ÐŁ":137385,"ĠاÙĦÙħØŃÙĦ":137386,"ĠاÙĦÙħØŃÙĦÙĬØ©":137387,"ĠTất":137388,"׾×ķ×ĺ":137389,"há»ķ":137390,"Ġaméric":137391,"Ġaméricain":137392,"Ġ×ijש׾×ij":137393,"Ġ׾×IJ×ķ×ŀ×Ļ":137394,"Ġpeça":137395,"ĠÑĢазнÑĭÑħ":137396,"ãģĦãĤĭãģ¨":137397,"ãĥĩãĥ³":137398,"סקר":137399,"Ġ×Ķ×ŀ×Ĺ×Ļר":137400,"ãģ¨ãģĦãģĨãĤĤãģ®":137401,"رتبط":137402,"ĠиÑģÑĤоÑĩ":137403,"ĠиÑģÑĤоÑĩник":137404,"สมัà¸Ħรสมาà¸Ĭิà¸ģ":137405,"Ġà¸Ĺัà¹īà¸ĩ":137406,"Ġà¸Ĺัà¹īà¸ĩà¸Ļีà¹ī":137407,"ĠTáºŃp":137408,"ãģ£ãģ¦ãģĦãģĨ":137409,"ĠاÙĦÙĪصÙĪÙĦ":137410,"Ġdécada":137411,"ĠоÑĦоÑĢм":137412,"ĠоÑĦоÑĢмлен":137413,"สำหรัà¸ļà¸ģาร":137414,"Ġogóln":137415,"ãģĨãģ¡ãģ«":137416,"Ġvárias":137417,"ãģĻãģİãĤĭ":137418,"ÙĪÙĩا":137419,"à¹Ĥà¸Ľà¸£à¸Ķ":137420,"ĠÐłÐ¾ÑģÑģиÑı":137421,"人ãĢħ":137422,"ãģĹãģ¦ãģįãģŁ":137423,"Ġsırasında":137424,"Ġngôn":137425,"سÙĨØ©":137426,"تÙħتع":137427,"×ŀ׼×ij×Ļ":137428,"Ġnhấn":137429,"×¢×ŀ×Ļ×ĵ":137430,"Ứ":137431,"жиÑĤÑĮ":137432,"ãĤīãģĽ":137433,"gráf":137434,"gráfica":137435,"ĠÙĤÙĪÙĦ":137436,"ĠÙĤÙĪÙĦÙĩ":137437,"ëĭ¨ì²´":137438,"หà¹īา":137439,"หà¹īาม":137440,"使ãģ£ãģ¦":137441,"ת×Ļ×ij":137442,"ת×Ļ×ijת":137443,"iá»ĥu":137444,"à¹ģà¸Ĭม":137445,"à¹ģà¸Ĭà¸¡à¸Ľ":137446,"à¹ģà¸Ĭà¸¡à¸Ľà¹Į":137447,"Ậ":137448,"ĠëĤĺëĿ¼":137449,"ĠÙħباشرة":137450,"ĠtrÄĥm":137451,"سÙĥÙĪ":137452,"ĠاÙĦØ°Ùī":137453,"Ġbiç":137454,"Ġbiçim":137455,"تراجع":137456,"ĠобеÑģп":137457,"ĠобеÑģпеÑĩ":137458,"ĠобеÑģпеÑĩива":137459,"ĠвоздÑĥÑħ":137460,"ÑĭваÑĤÑĮ":137461,"ÙĦØŃÙĤ":137462,"ĠMüdü":137463,"ĠMüdürl":137464,"ĠMüdürlÃ¼ÄŁÃ¼":137465,"Ġyaptır":137466,"Ġפרס":137467,"Ġפרס×ķ×Ŀ":137468,"Ø·ÙĪر":137469,"ÑģÑĤвоваÑĤÑĮ":137470,"ìŀ¥ìĿĦ":137471,"à¸Ĺีà¹Īà¸Ķีà¸Ĺีà¹Īสุà¸Ķ":137472,"à¸Ńัล":137473,"ÑĢÑİ":137474,"ÙħستÙĤبÙĦ":137475,"ÑģлÑĥÑĪ":137476,"ÑģлÑĥÑĪа":137477,"èªįãĤģ":137478,"Ġ׾×Ļ×ŀ":137479,"Ġ׾×Ļ×ŀ×ķ×ĵ×Ļ":137480,"תש×ķ×ij":137481,"תש×ķ×ij×ķת":137482,"ĠgerçekleÅŁtiril":137483,"ĠاÙĦاتÙģاÙĤ":137484,"ĠÑĥÑĢовне":137485,"ĠÑĤÑĢав":137486,"Ġ×Ķ×ŀ×ķף":137487,"ØŃÙģاظ":137488,"ĠÙħÙIJ":137489,"ĠÙħÙIJÙĨ":137490,"ĠÙħÙIJÙĨÙĴ":137491,"Ġdemás":137492,"×ŀ×ķ×ĸ×Ļק×Ķ":137493,"ש×Ļ×Ĺ×Ķ":137494,"Ġbú":137495,"алÑĮнÑĭм":137496,"ãĤıãģŁ":137497,"ãĤıãģŁãģĹ":137498,"ĠاÙĦÙħÙĪاد":137499,"×ª×Ľ×ł":137500,"×ª×Ľ×ł×ķף":137501,"ãĥŃãĥĥãĤ¯":137502,"hiếu":137503,"ĠÑĥме":137504,"ÙħØŃاÙĪÙĦØ©":137505,"×IJ×ķשר":137506,"ĠконкÑĥÑĢ":137507,"ĠконкÑĥÑĢÑģ":137508,"Ġ×ŀ×ij×Ĺ":137509,"Ġ×ŀ×ij×Ĺ×Ļ×ł×ª":137510,"Ġanlam":137511,"Ġanlamı":137512,"Ġliá»ĩt":137513,"ĠвÑħод":137514,"ĠHình":137515,"ĠÙĨÙĬ":137516,"ĠÙĨÙĬÙĪز":137517,"ãĤ¸ãĥ£ãĥ¼":137518,"×ij×Ļ×¥":137519,"ÑĤелÑĮнÑĭÑħ":137520,"à¸Ĺุà¸ģà¸Ńยà¹Īาà¸ĩ":137521,"ĠkiÅŁinin":137522,"Ø£Ùĥثر":137523,"ĠиÑģÑĤоÑĢии":137524,"Ġë³ĢíĻĶ":137525,"×¤×ľ×¡×ĺ":137526,"×¤×ľ×¡×ĺ×Ļ׳×Ļ":137527,"ĠÑģеÑĤ":137528,"ĠÑģеÑĤи":137529,"dıģımız":137530,"íķĺëıĦë¡Ŀ":137531,"×Ķר":137532,"×Ķר×ij×Ķ":137533,"ãģĻãĤĭãģĵãģ¨ãģ¯":137534,"Ġphiếu":137535,"تØŃسÙĬÙĨ":137536,"ĠÅĽrod":137537,"ĠÅĽrodow":137538,"ĠÅĽrodowisk":137539,"ĠÑĢаÑģÑħод":137540,"برÙĬد":137541,"ĠرÙĬ":137542,"ĠرÙĬاÙĦ":137543,"Ġ×ķ׼×ļ":137544,"ì§ĢìļĶ":137545,"׼×ŀ×ķ":137546,"Ġ×¢×ľ×Ļ×Ķ×Ŀ":137547,"fÃŃcio":137548,"Ġkararı":137549,"tıģını":137550,"ĠСов":137551,"ĠСовеÑĤ":137552,"ãģĬéĩijãĤĴ":137553,"междÑĥ":137554,"междÑĥна":137555,"междÑĥнаÑĢод":137556,"междÑĥнаÑĢодн":137557,"Ġmá»Ŀi":137558,"ĠاÙĦØ¥ÙĬر":137559,"ĠاÙĦØ¥ÙĬراÙĨÙĬ":137560,"ĠاÙĦرÙĪسÙĬ":137561,"صÙĨد":137562,"صÙĨدÙĪÙĤ":137563,"ĠاÙĦØ¥ÙĨترÙĨت":137564,"Ġtắm":137565,"ĠÑĤакого":137566,"Ġ×ij׾×ķ×Ĵ":137567,"Ġücrets":137568,"Ġücretsiz":137569,"×Ĺ×ĸ×Ļר":137570,"ìĸ´ìķ¼":137571,"ĠPhần":137572,"ï¼ľ":137573,"Ġ×ĺ×ij×¢":137574,"Ġ×ĺ×ij×¢×Ļ":137575,"×IJ×ŀ×IJ":137576,"اÙĤÙĦ":137577,"Ġcondições":137578,"ÙĤاتÙĦ":137579,"ĠÑĢезÑĥлÑĮÑĤаÑĤе":137580,"ĠÑģвоими":137581,"צ×ij×Ļ×¢":137582,"géni":137583,"Ġzes":137584,"Ġzespo":137585,"ĠzespoÅĤ":137586,"ÑĪив":137587,"Ġפר×ĺ×Ļ×ķת":137588,"ÙħستشÙģ":137589,"ÙħستشÙģÙī":137590,"شرع":137591,"ĠkoÅĽci":137592,"Ġ×Ķ×IJ×Ļ׳×ĺר׳×ĺ":137593,"ĠЧеÑĢ":137594,"поÑĩÑĤ":137595,"Ġactivités":137596,"çŁ¥ãģ£ãģ¦":137597,"Ġ×ij×ĸ×Ķ":137598,"Ġyüzden":137599,"ãģªãĤĬãģ¾ãģĽãĤĵ":137600,"Ġíĺ¹":137601,"Ġíĺ¹ìĿĢ":137602,"Ġ×ŀש׳×Ķ":137603,"ĠÐĴеÑĢ":137604,"Ġ×ij×IJ×ķת×ķ":137605,"éĿ¢çĻ½":137606,"éĿ¢çĻ½ãģĦ":137607,"شرØŃ":137608,"gründe":137609,"ÙģØ´":137610,"ÙģØ´ÙĦ":137611,"Ġséjour":137612,"ë´IJ":137613,"Ġrôle":137614,"شعار":137615,"емÑĭе":137616,"ĠاÙĦجسÙħ":137617,"алÑĮное":137618,"Ġìĥģíĥľ":137619,"Ｄ":137620,"ë¯Ģë¡ľ":137621,"ĠÙĨÙĤØ·":137622,"ĠÙĨÙĤطة":137623,"ãģĿãģĨãģł":137624,"ãģĻãĤĭãģ®ãģĮ":137625,"หู":137626,"Ġnhá»ĭ":137627,"Ġeconómica":137628,"ס×ĺ×ķ×ĵ":137629,"ס×ĺ×ķ×ĵ׳×ĺ":137630,"มีà¹Ĥà¸Ńà¸ģาส":137631,"Ġgestão":137632,"รูà¹īวà¹Īา":137633,"Ġloạt":137634,"ĠاÙĦÙħÙı":137635,"ĠاÙĦØŃÙħÙĦ":137636,"ĠاÙĦعÙħÙĦÙĬØ©":137637,"Ġê²ĥëıĦ":137638,"ĠÐľÐ¾Ñģква":137639,"ק×ĺ×ķר":137640,"ĠподÑĢоб":137641,"ĠподÑĢобн":137642,"ĠlÆ°ng":137643,"تÙģس":137644,"تÙģسÙĬر":137645,"ĠاÙĦبع":137646,"ĠاÙĦبعض":137647,"ئت":137648,"ÐķÐĿ":137649,"ìĹ°êµ¬":137650,"à¹ĥหà¹īà¸Ħุà¸ĵ":137651,"ãģĤãĤĬãģ¾ãģĹãģŁ":137652,"Ġbirka":137653,"Ġbirkaç":137654,"ĠÄ°sl":137655,"ĠÄ°slam":137656,"çĹĽãģ¿":137657,"Ġhảo":137658,"ĠмаÑı":137659,"ĠiÅŁÃ§i":137660,"ש×":137661,"ש×ģ":137662,"à¸ģารà¹Ģมืà¸Ńà¸ĩ":137663,"×ķ×Ķר":137664,"Ġchó":137665,"ëĨĢ":137666,"Ġyanlı":137667,"ĠyanlÄ±ÅŁ":137668,"幸ãģĽ":137669,"×IJר×Ĵ×ķ׳×Ļ":137670,"à¸Ńาà¸Īาร":137671,"à¸Ńาà¸Īารยà¹Į":137672,"ĠинÑĦоÑĢмаÑĨиÑİ":137673,"ÐĵÐŀ":137674,"׳×Ĺש":137675,"ĠìķĮìķĦ":137676,"ĠÑħаÑĢакÑĤеÑĢиÑģÑĤ":137677,"ĠÑħаÑĢакÑĤеÑĢиÑģÑĤик":137678,"à¸Ħุà¸ĵสามารà¸ĸ":137679,"è¦ĭãģĪãĤĭ":137680,"à¸Ĭัà¸Ķà¹Ģà¸Ī":137681,"à¸Ĭัà¸Ķà¹Ģà¸Īà¸Ļ":137682,"ĠdziaÅĤal":137683,"ĠdziaÅĤalnoÅĽci":137684,"à¹Ĥà¸ŀสà¸ķà¹Į":137685,"ĠÐļол":137686,"ĠÙģÙĩÙĬ":137687,"Ġ×ŀפ׳×Ļ":137688,"Ġ×Ķקשר":137689,"ÙħرÙĥ":137690,"ÙħرÙĥز":137691,"Ġhoá":137692,"Ġапп":137693,"ĠаппаÑĢаÑĤ":137694,"Ġpami":137695,"ĠpamiÄĻ":137696,"ĠpamiÄĻta":137697,"Ġçünkü":137698,"×ĵ×ķף":137699,"ãģ¯ãģĵãģ¡ãĤī":137700,"ĠMÃł":137701,"ĠÙĬÙĤدÙħ":137702,"ĠпÑĢез":137703,"ĠпÑĢезиденÑĤ":137704,"à¸Ńุà¸ķ":137705,"à¸Ńุà¸ķสา":137706,"à¸Ńุà¸ķสาห":137707,"à¸Ńุà¸ķสาหà¸ģรรม":137708,"ì§ĢìĽIJ":137709,"Ġ×IJפשר×ķת":137710,"schüt":137711,"schütz":137712,"ĠTiên":137713,"Ġsayılı":137714,"ĠгÑĢÑĥппÑĭ":137715,"оÑĩнÑĭй":137716,"Ġ×ľ×¢×ŀ×ķ×ĵ":137717,"ĠwrzeÅĽ":137718,"ĠwrzeÅĽnia":137719,"ĠÄIJầu":137720,"à¹Ģà¸Ĥà¹īารà¹Īวม":137721,"nızda":137722,"Ø®ÙĬص":137723,"Ġgünc":137724,"Ġgüncel":137725,"ĠÙĦÙĩØ°Ùĩ":137726,"ĠÙĬعتبر":137727,"légi":137728,"ãĤıãģĭãĤĭ":137729,"Ġrừng":137730,"ظÙĩ":137731,"ظÙĩÙĪر":137732,"Ġ×ŀ×ij×Ļף":137733,"Ġ기íĥĢ":137734,"åĪĩãĤĮ":137735,"lanmÄ±ÅŁ":137736,"à¸Ĺีà¹Īมีà¸Ħวาม":137737,"Ġhá»ģ":137738,"تÙĪجÙĩ":137739,"ĠاÙĦإدارة":137740,"Ġútil":137741,"ספ×ķ":137742,"à¸Ħวามรัà¸ģ":137743,"à¹Ĥฮ":137744,"ĠполиÑĤ":137745,"ĠполиÑĤик":137746,"Ġsatın":137747,"ĠÅŀimdi":137748,"×ŀ×ķר×Ļ×Ŀ":137749,"ìķĺëĭ¤":137750,"×Ĺ×ķ×ķ":137751,"×Ĺ×ķ×ķ×Ļ×Ķ":137752,"à¸Ħà¸Ńมà¸ŀิ":137753,"à¸Ħà¸Ńมà¸ŀิว":137754,"à¸Ħà¸Ńมà¸ŀิวà¹Ģà¸ķà¸Ńรà¹Į":137755,"Ġاذا":137756,"تخاذ":137757,"ãĤ¨ãĥ«":137758,"Ġpossibilité":137759,"ยืà¸Ļยัà¸Ļ":137760,"Ġünivers":137761,"Ġüniversite":137762,"ĠاÙĦدÙĪرÙĬ":137763,"ĠìķĬëĬĶëĭ¤":137764,"ĠìĦľë¡ľ":137765,"ØŃاÙĦ":137766,"Ġë¨":137767,"Ġ먼":137768,"Ġ먼ìłĢ":137769,"à¸Ĺีà¹Īà¸ĸูà¸ģ":137770,"ì§ľ":137771,"Ġskóry":137772,"лÑĮÑĨ":137773,"à¹ĥà¸Ĭà¹īà¹Ģวลา":137774,"×ijקשת":137775,"ĠØ°ÙĪ":137776,"æĹ¥ãĢħ":137777,"ĠкоÑĤоÑĢÑĥÑİ":137778,"ĠÑĥÑĢовенÑĮ":137779,"깨":137780,"à¹Ħà¸Ĺ":137781,"ãĤµãĥĹãĥª":137782,"ãĤ¸ãĥ§ãĥ³":137783,"ãģĻãģ¹ãģį":137784,"ĠGór":137785,"ãĥĪãĤ¤":137786,"ãĥĪãĤ¤ãĥ¬":137787,"ĠyaÅŁama":137788,"Ġdá»ĭp":137789,"Ġbữa":137790,"à¸ĭุ":137791,"Ġölüm":137792,"ãģ£ãģ¦ãģıãĤĭ":137793,"à¸ģารà¸Ħà¹īา":137794,"שער":137795,"ĠÑĤипа":137796,"ĠгеÑĢ":137797,"ĠгеÑĢо":137798,"רקע":137799,"Ġuważ":137800,"Ġuważa":137801,"ש×ŀף":137802,"Ġhastalık":137803,"ãĤıãĤĮãĤĭ":137804,"baÅŁÄ±":137805,"ÑĩÑĤо":137806,"Ġ×ij×ŀר׼×ĸ":137807,"Ġìļ°ë¦¬ìĿĺ":137808,"ĠÙĥاÙĨÙĪا":137809,"Ġأبر":137810,"ĠأبرÙĬÙĦ":137811,"층":137812,"à¹Ħà¸Ĥà¹Ī":137813,"ĠÙĪÙĦÙĪ":137814,"à¸Ĺัว":137815,"à¸Ĺัวรà¹Į":137816,"ĠÙĪØ£Ùĥد":137817,"à¸Ĭวà¸Ļ":137818,"׾×ķק":137819,"æį¨":137820,"æį¨ãģ¦":137821,"Ġİçin":137822,"péri":137823,"Ġyal":137824,"Ġyalnız":137825,"ÑĮÑıн":137826,"Ġgắng":137827,"à¸ģà¹ĩยัà¸ĩ":137828,"ĠУкÑĢаин":137829,"ĠÑģами":137830,"ĠпÑĢоведен":137831,"à¸ķà¸ģà¹ģà¸ķà¹Īà¸ĩ":137832,"ĠQuân":137833,"éparation":137834,"ĠbaÅŁÄ±nda":137835,"Ġznale":137836,"Ġznaleź":137837,"ĠznaleźÄĩ":137838,"ãĤ±ãĥ¼":137839,"ãĥİãĥ¼":137840,"à¸ĸูà¸ģà¸ķà¹īà¸Ńà¸ĩ":137841,"몸":137842,"ĠëıĮ":137843,"ĠëıĮìķĦ":137844,"ĠSchüler":137845,"ĠподгоÑĤов":137846,"ĠподгоÑĤовк":137847,"عرÙĪ":137848,"عرÙĪض":137849,"laÅŁtır":137850,"ĠÑģоÑģÑĤавлÑıеÑĤ":137851,"ĠпÑĢоизвод":137852,"ĠпÑĢоизводÑģÑĤва":137853,"ĠоÑģнове":137854,"ĠØ´ÙħاÙĦ":137855,"à¸ģรี":137856,"ĠgörÃ¼ÅŁme":137857,"оÑĩек":137858,"Ġ×Ĺ×ijר×Ļ×Ŀ":137859,"Ùħخاط":137860,"Ùħخاطر":137861,"ï¼Ń":137862,"רפ×IJ":137863,"ĠMẹ":137864,"ยà¸Ńมรัà¸ļ":137865,"Ġvết":137866,"خذ":137867,"ĠاÙĦتط":137868,"ĠاÙĦتطبÙĬÙĤ":137869,"à¸Ļึà¸ģ":137870,"Ġ×Ķ×Ľ×ł×¡×ª":137871,"ĠогÑĢани":137872,"ĠогÑĢаниÑĩен":137873,"ĠÃĩalÄ±ÅŁ":137874,"ĠاÙĦÙħÙĨتدÙī":137875,"à¸Īำà¸Ļวà¸Ļมาà¸ģ":137876,"ĠÑĤоÑĢÑĢ":137877,"ĠÑĤоÑĢÑĢенÑĤ":137878,"ĠìĤ´ìķĦ":137879,"à¸ŀลัà¸ĩà¸ĩาà¸Ļ":137880,"à¸Ĭัà¸Ļ":137881,"ĠÐIJндÑĢ":137882,"Ġréalisé":137883,"×ŀש×IJ":137884,"à¹ģà¸Ĭ":137885,"à¹ģà¸Ĭรà¹Į":137886,"Ġбог":137887,"มาà¹ģลà¹īว":137888,"ĠاÙĦÙĨار":137889,"Ġolmadıģı":137890,"×ĵ×¢×Ķ":137891,"ĠÑĥвеÑĢ":137892,"ĠÑĥвеÑĢен":137893,"ãĤĭãĤĤãģ®":137894,"أد":137895,"أدÙĪات":137896,"Ġ×Ķ×ĸ×ķ×Ĵ":137897,"إعÙĦاÙħ":137898,"há»ı":137899,"ĠNähe":137900,"ĠÑĤеÑģÑĤ":137901,"Ġ×ŀ×ķ׼ר":137902,"Ġë¬¸ìłľê°Ģ":137903,"ת×ķצ×IJ×Ķ":137904,"mó":137905,"móvel":137906,"ĠاÙĦتجارة":137907,"ĠмногиÑħ":137908,"обÑīа":137909,"Ġעסק×Ļ":137910,"ĠEducação":137911,"קש×Ļ×Ŀ":137912,"établ":137913,"établissement":137914,"Ġделе":137915,"иÑĢÑĥеÑĤÑģÑı":137916,"آثار":137917,"Ġ×Ķ×ŀר׼×ĸ×Ļ":137918,"ãĥIJãĥ«":137919,"ĠвÑģÑĤÑĢеÑĩ":137920,"ãģĴãĤĭ":137921,"ĠciÄħ":137922,"ĠciÄħgu":137923,"ÙĬست":137924,"à¸łà¸²à¸§":137925,"à¸łà¸²à¸§à¸°":137926,"Ø£Ùħر":137927,"Ġожи":137928,"Ġожида":137929,"Ġủy":137930,"ãĥŀãĥ«":137931,"راس":137932,"оÑĩной":137933,"ת×Ĵ×ķ×ij×ķת":137934,"تعرÙĬÙģ":137935,"ĠÑģоÑĨиалÑĮно":137936,"ãĤĴéĸĭ":137937,"ĠиÑģÑģледова":137938,"Ġdú":137939,"Ġdúvida":137940,"ĠskÅĤ":137941,"ĠskÅĤada":137942,"Ġhäufig":137943,"ĠвÑĭбÑĢ":137944,"ĠвÑĭбÑĢаÑĤÑĮ":137945,"ãģ®ãģ§ãģ¯ãģªãģĦãģĭ":137946,"ĠÑģилÑĮно":137947,"ÑĤвеÑĢжден":137948,"רפ":137949,"רפ×ķ×IJ×Ķ":137950,"æĢĿãģĦãģ¾ãģĻ":137951,"ØŃرص":137952,"ש×ķתף":137953,"Ùħسجد":137954,"à¹Ĥà¸Ĭวà¹Į":137955,"емÑģÑı":137956,"вÑĪие":137957,"Ġмл":137958,"Ġмлн":137959,"Ġ׾×Ķ×ij×Ļ×IJ":137960,"ĠÙĬتعÙĦÙĤ":137961,"à¸ķูà¹ī":137962,"ĠпÑĢаз":137963,"ĠпÑĢазд":137964,"ĠпÑĢаздник":137965,"Ġнем":137966,"Ġнемного":137967,"ĠsÃłng":137968,"تÙĨسÙĬ":137969,"تÙĨسÙĬÙĤ":137970,"Ġtá»Ŀ":137971,"Ġмеди":137972,"ãģ«æĪ":137973,"ãģ«æĪ»":137974,"à¸Ħวà¹īา":137975,"ãģĭãģijãĤĭ":137976,"×ij׾×ķת":137977,"ĠÑįкÑģп":137978,"ĠÑįкÑģпеÑĢÑĤ":137979,"ĠдевÑĥÑĪ":137980,"ĠдевÑĥÑĪк":137981,"ĠØŃص":137982,"ÙĨشأ":137983,"ãģĮãģĤãĤĭãģ®ãģ§":137984,"ĠتراÙħ":137985,"ĠتراÙħب":137986,"أسÙĪاÙĤ":137987,"Ġ׾פ׳×ķת":137988,"Ġاﻷ":137989,"ãģ«ãģı":137990,"ãģ«ãģıãģĦ":137991,"ĠأعÙĦÙī":137992,"Ġ׾×Ķ×ŀש×Ļ×ļ":137993,"räu":137994,"ש×ŀ×Ļ×Ŀ":137995,"åĪĨãģij":137996,"ãģĻãģ§":137997,"ãģĻãģ§ãģ«":137998,"×Ķ׾׼×Ķ":137999,"×Ĺ׾×Ļ×£":138000,"Ġì±ħ":138001,"Ġì±ħìŀĦ":138002,"à¹Ģà¸Īริ":138003,"à¹Ģà¸Īริà¸į":138004,"éģĬãģ³":138005,"جسد":138006,"สาà¸ĺ":138007,"สาà¸ĺาร":138008,"สาà¸ĺารà¸ĵ":138009,"Ġbasın":138010,"ÑĢаг":138011,"гад":138012,"ĠhoÅŁ":138013,"íķµ":138014,"×ij×Ĺ×Ļר×Ķ":138015,"×ŀס×ļ":138016,"ĠìłľíĴĪ":138017,"تÙħÙĪÙĬÙĦ":138018,"ĠLÆ°u":138019,"ë¡ľë¶ĢíĦ°":138020,"Ġпоб":138021,"Ġпобед":138022,"ÙħÙĨØ°":138023,"常ãģ«":138024,"ÙĤس":138025,"ĠاÙĦÙħصدر":138026,"ĠÙĪاÙĦاست":138027,"Ġkhắp":138028,"ĠاÙĦجاÙĨب":138029,"Ġnguyá»ĩn":138030,"éĸĵéģķãģĦ":138031,"ĠÑģÑĤÑĢа":138032,"ĠÑģÑĤÑĢаÑħ":138033,"ĠÑģÑĤÑĢаÑħов":138034,"รีà¸ļ":138035,"ĠxÆ°Æ¡ng":138036,"Ġì°¾":138037,"Ġì°¾ìķĦ":138038,"Ġngại":138039,"гал":138040,"à¸ĭีà¹Ī":138041,"Ġ×ijפ×Ļ×Ļס×ij×ķק":138042,"ЦенÑĤÑĢ":138043,"Ġavaliação":138044,"Ġeconómico":138045,"×ĸף":138046,"ĠÐľÐ°Ðº":138047,"Ġinterés":138048,"à¸ģลิà¹Īà¸Ļ":138049,"ÑģÑĤÑĮÑİ":138050,"ĠÄijÆ°Æ¡ng":138051,"å¼·ãģı":138052,"ĠKhách":138053,"à¹Ģà¸Ļืà¹īà¸Ńหา":138054,"ĠYazı":138055,"è²·ãģ£ãģ¦":138056,"ÐłÐķ":138057,"à¹Ģà¸ŀิà¹Īมà¸Ĥึà¹īà¸Ļ":138058,"สมà¸ļู":138059,"สมà¸ļูรà¸ĵà¹Į":138060,"ĠмиÑĢов":138061,"×Ĵ׳×Ļ×Ŀ":138062,"ĠÄijức":138063,"à¸Ńารà¹Į":138064,"صاص":138065,"ãģĬãĤĪ":138066,"ãģĬãĤĪãģ³":138067,"êÌī":138068,"ĠاÙĦÙħؤتÙħر":138069,"ĠاÙĦÙħرØŃÙĦØ©":138070,"สà¸Ńà¸ļà¸ĸาม":138071,"Ġà¸Īาà¸ģà¸Ļัà¹īà¸Ļ":138072,"Ġتعد":138073,"ãģĿãģ®ãģŁãĤģ":138074,"Ġkháng":138075,"à¸Ļิà¸Ķ":138076,"ãĥĬãĥ³":138077,"ëĦ¤ìļĶ":138078,"ĠاÙĦاØŃت":138079,"ĠاÙĦاØŃتÙĦاÙĦ":138080,"ìļķ":138081,"Ġмодели":138082,"ĠпÑĢоÑĨенÑĤ":138083,"à¸ŀวà¸ģà¹Ģรา":138084,"Ġ×Ķצ×ĵ":138085,"Ġ×Ķצ×ĵ×ĵ×Ļ×Ŀ":138086,"stände":138087,"׳×Ĵר":138088,"Ġdotyc":138089,"ĠdotyczÄħ":138090,"ĠdotyczÄħce":138091,"ĠÅĽwiÄĻt":138092,"×ŀר×Ķ":138093,"ãģĻãģĶãģĦ":138094,"ãĥĩãĤ£ãĥ³ãĤ°":138095,"à¸ģารสรà¹īาà¸ĩ":138096,"ëĤ¬":138097,"Ġì°¸ìŬ":138098,"ÑģÑħ":138099,"ÑģÑħем":138100,"ÙħÙĪس":138101,"Ġnấu":138102,"Ġ׾×ŀ×¢×ľ×Ķ":138103,"à¹Ģà¸Ľà¹īา":138104,"à¹Ģà¸Ľà¹īาหมาย":138105,"Ġmùi":138106,"ائز":138107,"íĽĪ":138108,"×Ĺ×ij×ķר×Ķ":138109,"à¸ľà¸¹à¹īà¹ĥà¸Ĭà¹ī":138110,"Ġpaź":138111,"Ġpaździ":138112,"Ġpaździern":138113,"Ġpaździernika":138114,"ลà¸ĩà¹Ħà¸Ľ":138115,"ÙĤاع":138116,"ĠcháºŃm":138117,"Ġözellikleri":138118,"ĠÄIJo":138119,"ĠÄIJoÃłn":138120,"жение":138121,"Ġhẳ":138122,"Ġhẳn":138123,"ĠaÅŁk":138124,"ï½į":138125,"ãĥijãĤ¹":138126,"×Ķ×ķר×IJ×ķת":138127,"ĠÅ»":138128,"ĠÅ»y":138129,"×ŀ×ĸ׾":138130,"ĠÑĥкÑĢа":138131,"ĠÑĥкÑĢаин":138132,"à¹Ģà¸Ĭิ":138133,"à¹Ģà¸Ĭิà¸į":138134,"ÐłÐĺ":138135,"ĠzwiÄħzku":138136,"×Ķ×Ĺ׾×ĺת":138137,"ãĤĵãģ§ãģĻãĤĪãģŃ":138138,"ãģ¦ãģĬãĤĬ":138139,"ложиÑĤÑĮ":138140,"×ŀ×ķ׳×Ļ×Ŀ":138141,"ฮิ":138142,"ì°¬":138143,"ĠاÙĦÙħشترÙĥ":138144,"ĠdÃ¼ÅŁÃ¼k":138145,"агенÑĤ":138146,"ĠاÙĦأسبÙĪع":138147,"ĠÙĤرÙĬب":138148,"инд":138149,"индив":138150,"индивид":138151,"индивидÑĥ":138152,"индивидÑĥалÑĮн":138153,"förder":138154,"Ġseçen":138155,"Ġseçenek":138156,"Ġétant":138157,"ĠлÑİбим":138158,"казÑĭваеÑĤ":138159,"วิà¸Ļ":138160,"Ġ×Ķ×ij×IJ×Ļ×Ŀ":138161,"Ġдов":138162,"ĠдоволÑĮ":138163,"ĠдоволÑĮно":138164,"×¢×ĵ×Ļ×£":138165,"Ġokre":138166,"ĠokreÅĽ":138167,"ĠokreÅĽlon":138168,"ĠترÙĬد":138169,"à¹Ģมืà¹Īà¸Ńวัà¸Ļà¸Ĺีà¹Ī":138170,"ãĤĪãģĭãģ£ãģŁ":138171,"Cumh":138172,"Cumhur":138173,"Cumhurba":138174,"CumhurbaÅŁ":138175,"CumhurbaÅŁkan":138176,"CumhurbaÅŁkanı":138177,"Ġnợ":138178,"à¸ľà¸¹à¹īà¹Ģลà¹Īà¸Ļ":138179,"Ġcomplète":138180,"à¹Ģà¸ŀศ":138181,"دÙIJ":138182,"Ġdüz":138183,"Ġdüzey":138184,"ãģ§ãģĤãĤĭãģĵãģ¨":138185,"extérieur":138186,"׳":138187,"Ġinformação":138188,"ãĤ¯ãĥªãĥĭãĥĥãĤ¯":138189,"ĠPubli":138190,"ĠPublié":138191,"ר×ķ×ĵ":138192,"à¸Ħà¸§à¸²à¸¡à¸Ľà¸¥à¸Ńà¸Ķà¸łà¸±à¸¢":138193,"ĠØ£ÙĬض":138194,"ĠØ£ÙĬضÙĭا":138195,"تسبب":138196,"ãģ¤ãĤĤãĤĬ":138197,"изма":138198,"à¸Ĥึà¹īà¸Ļà¹Ħà¸Ľ":138199,"ÙĥÙIJ":138200,"ÙĦÙĪÙħ":138201,"Ġשצר":138202,"Ġשצר×Ļ×ļ":138203,"ãģ¯ãĤĤãģ¡ãĤįãĤĵ":138204,"Ġкан":138205,"Ġканал":138206,"ãģ«ãģªãģ£ãģ¦ãģĦãģ¾ãģĻ":138207,"ĠاÙĦØ£Ùĥثر":138208,"تاØŃ":138209,"ÙĨتÙĩ":138210,"ÙĨتÙĩاء":138211,"اÙĪÙĬØ©":138212,"ĠBugün":138213,"нÑģкого":138214,"à¸Ķà¹Īวà¸Ļ":138215,"évolution":138216,"ãģ£ãģ¦ãģĦãģ¾ãģĹãģŁ":138217,"ãĤħ":138218,"ĠVÆ°Æ¡ng":138219,"à¸łà¸²à¸ŀย":138220,"à¸łà¸²à¸ŀยà¸Ļ":138221,"à¸łà¸²à¸ŀยà¸Ļà¸ķรà¹Į":138222,"Ġ×Ķצ׾×Ļ×Ĺ":138223,"ĠاÙĦإسÙĦاÙħÙĬ":138224,"ÙĦÙĬب":138225,"Ġedição":138226,"ÑģÑĤÑĢел":138227,"Ġkhúc":138228,"ÙĨÙħÙĪØ°":138229,"ÙĨÙħÙĪذج":138230,"׾צ×Ķ":138231,"ÑģÑĤавил":138232,"à¸ĸา":138233,"สรà¹īาà¸ĩà¸Ħวาม":138234,"ãģĦãģ£ãģ±":138235,"ãģĦãģ£ãģ±ãģĦ":138236,"ÑģÑĤавлен":138237,"ĠاÙĦÙĤدس":138238,"Ġngược":138239,"بخ":138240,"สหร":138241,"สหรั":138242,"สหรัà¸IJ":138243,"Ġأغ":138244,"Ġأغسط":138245,"Ġأغسطس":138246,"ãģĨãģ¾":138247,"ãģĨãģ¾ãģı":138248,"ĠêµŃìłľ":138249,"ØŃضار":138250,"Ġdừng":138251,"æĬ¼ãģĹ":138252,"تÙĪا":138253,"تÙĪاجد":138254,"ש×ŀ×Ĺ×Ķ":138255,"ãģıãĤĵ":138256,"Ġ×ijעצ":138257,"Ġ×ijעצ×Ŀ":138258,"×ŀ׳×Ļ×ķת":138259,"×ķ×Ļ×ĵ":138260,"×ķ×Ļ×ĵ×IJ×ķ":138261,"à¸Ĭิà¸ĩ":138262,"ĠpracÄĻ":138263,"ĠзаÑĤ":138264,"ĠзаÑĤем":138265,"ĠìŀIJìľł":138266,"Ġì¤Ģ":138267,"Ġì¤Ģë¹Ħ":138268,"ĠbáºŃ":138269,"ĠbáºŃc":138270,"Ġ×Ķ×ŀצ×ij":138271,"ĠÙĤÙĬÙħØ©":138272,"à¹Ģà¸Ńà¹Ģà¸Ĭ":138273,"à¹Ģà¸Ńà¹Ģà¸Ĭีย":138274,"Ġperchè":138275,"ĠاÙĦعسÙĥر":138276,"ĠاÙĦعسÙĥرÙĬØ©":138277,"جÙĬب":138278,"ëŀµ":138279,"ÙħÙĩر":138280,"ÙħÙĩرجاÙĨ":138281,"ÙħراÙĥ":138282,"ÙħراÙĥز":138283,"Ġоднако":138284,"à¸Ķีà¹Ĩ":138285,"Ġצפ×ķ":138286,"Ġkullanılan":138287,"Ġкино":138288,"ãĥĨãĤ£ãĥ³ãĤ°":138289,"ĠGiỼi":138290,"تÙĪز":138291,"تÙĪزÙĬع":138292,"ยิà¸Ļ":138293,"ยิà¸Ļà¸Ķี":138294,"ĠcÅĵur":138295,"ĠiÅŁaret":138296,"Ġ×ij×¢×ĸר":138297,"Ġ×ij×¢×ĸרת":138298,"ĠпаÑĨи":138299,"ĠпаÑĨиенÑĤ":138300,"ãģ¿ãģŁãģĦãģ§ãģĻ":138301,"вез":138302,"лина":138303,"оде":138304,"Ġ×IJ×ķ×ª×Ł":138305,"dıģınız":138306,"ĠÐIJв":138307,"ĠÐIJвÑĤоÑĢ":138308,"ï¼®":138309,"ĠCần":138310,"ĠاÙĦاخ":138311,"ĠاÙĦاخبار":138312,"Ġê±°ìĿĺ":138313,"Ġatenção":138314,"ĠgeldiÄŁi":138315,"ãĤªãĤ¹":138316,"ãĤªãĤ¹ãĤ¹":138317,"ãĤªãĤ¹ãĤ¹ãĥ¡":138318,"евÑĭе":138319,"кÑĢÑĭл":138320,"à¹Ģà¸Ĭียà¸ĩ":138321,"à¹Ģà¸Ĭียà¸ĩà¹ĥหมà¹Ī":138322,"Ġmarço":138323,"ĠاÙĦÙħادة":138324,"Ġгол":138325,"Ġsprzedaży":138326,"Ġíķ´ê²°":138327,"ĠÐķго":138328,"ê¹Ģ":138329,"Ġ׾ק×ij×ľ×ª":138330,"ĠاÙĦÙģÙĨاÙĨ":138331,"Ġcomunicación":138332,"à¹Ģสà¹īà¸Ļà¸Ĺาà¸ĩ":138333,"íĺ¹":138334,"à¸Ĭำ":138335,"à¸Ĭำระ":138336,"Ġ׼×IJ×ŀ":138337,"Ġ׼×IJ×ŀ×ķר":138338,"à¸Ĭà¹Īาà¸ĩ":138339,"زÙĩر":138340,"Ġklientów":138341,"иваÑİÑĤ":138342,"анг":138343,"׳×ļ":138344,"Ġgá»įn":138345,"ÃľR":138346,"ìĺģìĥģ":138347,"Ġغزة":138348,"ìĿĮìĿĦ":138349,"Ġbezpo":138350,"ĠbezpoÅĽ":138351,"ĠbezpoÅĽredni":138352,"ĠاÙĦÙħÙĪا":138353,"ĠاÙĦÙħÙĪاطÙĨ":138354,"ĠاÙĦÙħÙĪاطÙĨÙĬÙĨ":138355,"ãĤĮãģ¾ãģĻ":138356,"ĠмаÑĤÑĩ":138357,"×IJ×ķף":138358,"ĠرسÙħÙĬ":138359,"ĠÑįкон":138360,"ĠÑįконом":138361,"ĠÑįкономиÑĩеÑģк":138362,"ãĥľãĥ¼":138363,"ĠдиÑĢ":138364,"ĠдиÑĢекÑĤоÑĢ":138365,"ĠÑģкоÑĢо":138366,"à¸ļำ":138367,"à¸ļำร":138368,"à¸ļำรุà¸ĩ":138369,"ĠÑĦÑĥÑĤ":138370,"ĠÑĦÑĥÑĤбол":138371,"Ġ×IJ×Ļ׾":138372,"Ġì¤ijêµŃ":138373,"ìľ¤":138374,"eÄŁe":138375,"à¹Ħà¸ģà¹Ī":138376,"traî":138377,"traîn":138378,"ĠÑĤÑĢÑĥб":138379,"à¹Ģà¸ļื":138380,"à¹Ģà¸ļืà¹īà¸Ńà¸ĩ":138381,"à¹ģมà¸Ļ":138382,"ĠتØŃدÙĬØ«":138383,"Ġ×Ľ×¢×ª":138384,"ØŃاسب":138385,"lıģa":138386,"ק×Ļ×Ļ×ŀ×Ļ×Ŀ":138387,"оÑģÑĤÑĮÑİ":138388,"à¸Ŀั":138389,"à¸Ŀัà¹Īà¸ĩ":138390,"شغÙĦ":138391,"ìĽ¹":138392,"Ġкаждого":138393,"Ġbölümü":138394,"หà¸Ļี":138395,"ĠistediÄŁi":138396,"ĠtrÆ°ng":138397,"ãĥĮ":138398,"ฮà¸Ń":138399,"Ø£ÙĨØ´":138400,"Ø£ÙĨشطة":138401,"ĠاÙĦÙħسÙĬ":138402,"ĠاÙĦÙħسÙĬØŃ":138403,"ลัà¸ģษà¸ĵà¹Į":138404,"Ġná»Ńa":138405,"à¸Ĺีà¹Īà¸ķà¹īà¸Ńà¸ĩà¸ģาร":138406,"ÑĪек":138407,"лÑij":138408,"Ġש×Ļ×Ķ":138409,"Ġש×Ļ×Ķ×Ļ×Ķ":138410,"Ġkhuôn":138411,"ĠÑĤÑĢебованиÑı":138412,"Ġ×ľ×¢×ĸ×ķר":138413,"ĠاÙĦعÙħر":138414,"ราà¸Ħาà¸ĸูà¸ģ":138415,"ÙĩÙıÙħÙĴ":138416,"üst":138417,"üstü":138418,"Ġденег":138419,"Ġnạ":138420,"à¸Ĥà¸Ļม":138421,"Ġблаг":138422,"Ġблагод":138423,"ĠблагодаÑĢ":138424,"ĠблагодаÑĢÑı":138425,"إسÙĦاÙħ":138426,"à¸Ļิว":138427,"çŁ¥ãĤīãģªãģĦ":138428,"Ø«ÙĤØ©":138429,"ĠголоÑģ":138430,"×IJ×ķר×Ĺ":138431,"Ġtrứng":138432,"Ġодном":138433,"ĠkoÅĦcu":138434,"Ġ×ķרק":138435,"WiÄĻ":138436,"WiÄĻcej":138437,"Ġ×IJ×Ļ׼×ķת":138438,"Ġ×IJ×Ļ׼×ķת×Ļ":138439,"ÑģоÑģ":138440,"Ġjeżeli":138441,"以ä¸ĭãģ®":138442,"å°ıãģķ":138443,"å°ıãģķãģª":138444,"ологии":138445,"ĠобÑģлÑĥж":138446,"ĠобÑģлÑĥжива":138447,"Ùĥتابة":138448,"Ġê´Ģìĭ¬":138449,"עש×Ļר":138450,"Ġarasındaki":138451,"ĠÑĢайона":138452,"ÙĪاجب":138453,"Ġ×ij×Ĺ×Ļ×Ļ":138454,"íķ´ì£¼":138455,"Ġgóc":138456,"айл":138457,"ĠTình":138458,"æļ®ãĤī":138459,"æļ®ãĤīãģĹ":138460,"æĻĤãģ«ãģ¯":138461,"ĠгоÑĢоде":138462,"Ġ׼×IJ×Ļ׾":138463,"Ġ׼×IJ×Ļ׾×ķ":138464,"ĠCá»Ļng":138465,"ãģ©ãģĨãģĹãģ¦ãĤĤ":138466,"×Ĺ×ķ×£":138467,"تØŃرÙĥ":138468,"ĠÑģловам":138469,"à¸Īะà¸Ĭà¹Īวย":138470,"ĠاÙĦÙħستÙĤبÙĦ":138471,"ÙĤض":138472,"ÙĤضÙĬ":138473,"×ijס×ķפ":138474,"×ijס×ķפ×ķ":138475,"iÄĻÄĩ":138476,"ĠYıl":138477,"Ø´ÙĬØ®":138478,"à¸Ħุà¸ĵà¸Īะ":138479,"ש×ŀ×ķת":138480,"Ġتعرض":138481,"Ġanálise":138482,"ĠÑģобиÑĢа":138483,"à¹Ģà¸ŀà¸Ĭ":138484,"à¹Ģà¸ŀà¸Ĭร":138485,"Ġвели":138486,"Ġвелик":138487,"สัà¹īà¸Ļ":138488,"Ġpopulação":138489,"รà¹Īวมà¸ģัà¸Ļ":138490,"×Ĺ×ŀ":138491,"×Ĺ×ŀ×Ļש×Ļ":138492,"ס×Ļס":138493,"åĨħãģ§":138494,"ĠsobÄħ":138495,"ĠYay":138496,"ĠYayın":138497,"ãĥ¡ãĥĭãĥ¥ãĥ¼":138498,"ĠпÑĢедоÑģÑĤавлÑı":138499,"ãģłãģ¨æĢĿãģĨ":138500,"Ġê³łê°Ŀ":138501,"Ġодним":138502,"à¹ĥà¸Ļà¹Ģรืà¹Īà¸Ńà¸ĩ":138503,"Ġsá»ķ":138504,"ĠÐĹдеÑģÑĮ":138505,"ĠизменениÑı":138506,"ĠìĿ¼ìĿĦ":138507,"ãģªãģ®ãģł":138508,"кладÑĭва":138509,"ÑĢма":138510,"Ġ×ķ×ij׼׾":138511,"تأÙħÙĬÙĨ":138512,"ĠпÑĢиÑıÑĤ":138513,"ĠпÑĢиÑıÑĤн":138514,"ÙħÙħار":138515,"ÙħÙħارسة":138516,"ãģ¨ãģªãģ£ãģ¦":138517,"ĠجÙħÙĬÙĦ":138518,"Ġì§Ī":138519,"Ġì§Ī문":138520,"Ġquestão":138521,"ié":138522,"iéndo":138523,"หà¹īà¸Ńà¸ĩà¸ŀัà¸ģ":138524,"ãĥijãĥ¼ãĥĪ":138525,"ÑĤвеÑĢжда":138526,"нÑģкой":138527,"зал":138528,"มุà¹Īà¸ĩ":138529,"á»Ĭ":138530,"Ġ×Ķ×IJ×Ĺר×ķ׳×Ķ":138531,"ĠThÆ°":138532,"주민":138533,"ĠاÙĦعب":138534,"évén":138535,"événement":138536,"ÙĤÙĪاعد":138537,"دÙı":138538,"ĠìķĬìĬµëĭĪëĭ¤":138539,"Ġ보기":138540,"Ġyapılması":138541,"à¹Ģราà¸ģ":138542,"à¹Ģราà¸ģà¹ĩ":138543,"ØŃذر":138544,"ÙĤصر":138545,"ãģ¦ãģĹãģ¾ãģĦãģ¾ãģĹãģŁ":138546,"Ġà¹Ģà¸Ľà¹ĩà¸Ļà¸ķà¹īà¸Ļ":138547,"ãģ¨ãģ«":138548,"ãģ¨ãģ«ãģĭ":138549,"ãģ¨ãģ«ãģĭãģı":138550,"нÑĨе":138551,"звÑĥк":138552,"ãģĹãĤĪãģĨãģ¨":138553,"ĠاÙĦصØŃÙĬØ©":138554,"Ġש×Ķ×Ļ×ķ":138555,"ĠDiÄŁer":138556,"ÙĤÙĦÙĤ":138557,"ãĤ¸ãĥ£ãĥ³":138558,"Ġrá»Ŀi":138559,"ĠлеÑĩ":138560,"ĠлеÑĩениÑı":138561,"تباد":138562,"تبادÙĦ":138563,"צפ×Ķ":138564,"à¸Ħวามà¹Ģหà¹ĩà¸Ļ":138565,"Ġشب":138566,"ĠشبÙĥØ©":138567,"ר×Ļק":138568,"Ùħعد":138569,"Ùħعدات":138570,"dıģında":138571,"Ġ×ijש׳×Ļ×Ŀ":138572,"Ġ×Ķ×Ļשר×IJ׾":138573,"Ġ×Ķ×Ļשר×IJ׾×Ļת":138574,"Ġsınav":138575,"׳צ×Ļ×Ĵ":138576,"วัà¸ķà¸ĸุ":138577,"ĠاÙĦبرÙĦÙħ":138578,"ĠاÙĦبرÙĦÙħاÙĨ":138579,"tivitÃł":138580,"ãĤĵãģłãĤįãģĨ":138581,"ק×Ļ×Ļ×ŀ":138582,"ÙĦÙĬÙĥ":138583,"ĠÄijò":138584,"ĠÄijòi":138585,"ĠÐĺнÑĤеÑĢ":138586,"ĠÐĺнÑĤеÑĢнеÑĤ":138587,"ãģ«ãģ¨ãģ£ãģ¦ãģ¯":138588,"ãģ£ãģĵ":138589,"ק×ķס":138590,"ستØŃÙĤ":138591,"æķĻãģĪãģ¦":138592,"ãĥĢãĥ¡":138593,"ĠÙħÙĨزÙĦ":138594,"à¹Ģà¸ĭà¹ĩà¸Ļ":138595,"使ãģĪãĤĭ":138596,"è¦ĭç©į":138597,"è¦ĭç©įãĤĤãĤĬ":138598,"Ø£Ùģ":138599,"Ø£ÙģÙĥار":138600,"ĠигÑĢов":138601,"ĠигÑĢовÑĭе":138602,"ĠmÄĻż":138603,"ĠmÄĻżczy":138604,"ĠmÄĻżczyzn":138605,"ĠاÙĦØŃÙĤÙĬÙĤÙĬ":138606,"عبر":138607,"׼×ķ׾׳×ķ":138608,"íĿ¥":138609,"×ŀ×IJ×ķ×Ĺר":138610,"ختص":138611,"ãĥŀãĥŀ":138612,"Ġ×IJ×Ĺ×ķ×ĸ":138613,"íĮĢ":138614,"Ġrá»iji":138615,"ĠвÑĤоÑĢ":138616,"ĠвÑĤоÑĢой":138617,"Ġlẫn":138618,"пÑĢом":138619,"пÑĢомÑĭÑĪ":138620,"пÑĢомÑĭÑĪлен":138621,"пÑĢомÑĭÑĪленн":138622,"ĠоÑĤноÑĪениÑı":138623,"Ġsứ":138624,"ĠмобилÑĮ":138625,"ĠмобилÑĮн":138626,"ĠÑįÑĤомÑĥ":138627,"Ġtạp":138628,"ĠìĤ¬ê±´":138629,"ĠìķĮ볤":138630,"ÙĥÙı":138631,"ÙĥÙıÙħÙĴ":138632,"Ġק×ķר×Ķ":138633,"ĠÑĦиÑĢ":138634,"ĠÑĦиÑĢм":138635,"Ġsıkıntı":138636,"׳׼":138637,"׳׼×ķף":138638,"ÙĪÙĦÙĪجÙĬ":138639,"ØŃاÙĨ":138640,"Ġloạn":138641,"Ġ×IJ×ľ×£":138642,"Ġmắn":138643,"abhäng":138644,"abhängig":138645,"ĠÑĥÑĢовнÑı":138646,"Ġ׾×ij×ĵ×ķק":138647,"ÙĬÙħÙĨ":138648,"layın":138649,"Ġhải":138650,"Ġзавод":138651,"ĠìķĦ주":138652,"สà¸ĸา":138653,"สà¸ĸาà¸ļัà¸Ļ":138654,"Ġgüvenlik":138655,"à¹Ģà¸Ķà¹Īà¸Ļ":138656,"×ij×ĵק":138657,"ĠëĪ":138658,"ĠëĪĦ":138659,"ĠëĪĦ구":138660,"éĩįè¦ģãģª":138661,"รà¸Ńà¸ĩรัà¸ļ":138662,"schlie":138663,"schlieÃŁen":138664,"Ġìĸ¼":138665,"Ġìĸ¼ë§Ī":138666,"Ġìĸ¼ë§ĪëĤĺ":138667,"ÑĤики":138668,"íķľëĭ¤ê³ł":138669,"ãģłãģ£ãģŁãĤī":138670,"Ġ×Ķ×Ļ×ĺ×ij":138671,"ãģªãģijãĤĮãģ°ãģªãĤīãģªãģĦ":138672,"âÌ":138673,"ậ":138674,"Ġphạt":138675,"akÄ±ÅŁ":138676,"ãģ¦ãģĹãģ¾ãģĦãģ¾ãģĻ":138677,"à¹Ģà¸ĭà¹ĩ":138678,"ĠСегоднÑı":138679,"Ġinsanların":138680,"Ġdéveloppe":138681,"תפר":138682,"תפר×Ļ×ĺ":138683,"اÙĨتشار":138684,"ê°ij":138685,"François":138686,"Ø£ÙĦع":138687,"Ø£ÙĦعاب":138688,"ãĤĴè¶ħ":138689,"ãĤĴè¶ħãģĪ":138690,"Ġê°ĻìĬµëĭĪëĭ¤":138691,"ãĤ³ãĥ¬":138692,"ĠмеÑģÑıÑĨев":138693,"íĮħ":138694,"ĠاÙĦجاÙħعة":138695,"ìĿ¸íĦ°":138696,"ìĿ¸íĦ°ëĦ·":138697,"×ĵר×ķש":138698,"ĠÙĪأشار":138699,"ĠпÑĢавила":138700,"ãģĿãģĵãģ«":138701,"×Ĺ×ŀ×ĵ":138702,"à¹Ģหà¸ķุà¸ģารà¸ĵà¹Į":138703,"Ġê²½íĹĺ":138704,"ãģ¶ãĤĬ":138705,"׾ש":138706,"׾ש×ķף":138707,"à¹Ģà¸ĸ":138708,"ĠDoÄŁu":138709,"ĠиÑģполÑĮзование":138710,"ĠçocuÄŁu":138711,"магазине":138712,"ĠÄijiá»ĥn":138713,"Ġaslı":138714,"Ġaslında":138715,"Ġdoença":138716,"Ġساع":138717,"Ġساعات":138718,"ĠиÑģполÑĮзованиÑı":138719,"ר×ķצ×Ļ×Ŀ":138720,"ĠзнаÑĩиÑĤ":138721,"ĠÑĢам":138722,"ĠÑĢамкаÑħ":138723,"거리":138724,"ĠпÑĭÑĤа":138725,"ãĥģãĥ³":138726,"ĠпоÑģк":138727,"ĠпоÑģколÑĮ":138728,"ĠпоÑģколÑĮкÑĥ":138729,"إبر":138730,"إبراÙĩ":138731,"إبراÙĩÙĬÙħ":138732,"ĠÑĤÑĢеÑħ":138733,"ĠGenç":138734,"سÙĪÙģ":138735,"ĠveÃŃculo":138736,"ĠNgân":138737,"ĠоÑĩеÑĢедÑĮ":138738,"à¸Ħรึà¹Īà¸ĩ":138739,"×IJ×ij×Ļ":138740,"à¸ķà¹īม":138741,"ãĤĴè¡ĮãģĦ":138742,"ĠاÙĦسابÙĤØ©":138743,"наÑĨи":138744,"наÑĨиона":138745,"наÑĨионалÑĮн":138746,"Ġgestión":138747,"تÙĤد":138748,"ĠاÙĦبÙĬاÙĨ":138749,"ĠاÙĦبÙĬاÙĨات":138750,"ĠاÙĦاÙĨتخاب":138751,"ĠاÙĦاÙĨتخابات":138752,"à¹Ģà¸Ĭà¹Īา":138753,"×ĵ×IJ×Ĵ":138754,"Ġ׾×Ĵ×ŀר×Ļ":138755,"ĠتØŃتاج":138756,"Ġthôn":138757,"à¸ķà¹īà¸Ńà¸Ļ":138758,"à¸ķà¹īà¸Ńà¸Ļรัà¸ļ":138759,"女ãģ®":138760,"女ãģ®åŃIJ":138761,"Ġthợ":138762,"Ø·ØŃÙĨ":138763,"ารà¹Įà¸Ķ":138764,"ת×ŀ×Ļ×ĵ":138765,"ĠÑģамÑĭм":138766,"Ġìĭľíĸī":138767,"إصد":138768,"إصدار":138769,"ĠNghá»ĩ":138770,"ìķķ":138771,"سئ":138772,"سئÙĦ":138773,"à¸Ńาร":138774,"à¸Ńารม":138775,"à¸Ńารมà¸ĵà¹Į":138776,"à¹ģฮ":138777,"׳×ĺ׾":138778,"Ġì¢ĭìķĦ":138779,"×ķ׾׾":138780,"Ġ×ij×Ľ×ª×ij":138781,"ãĤ«ãĥ©":138782,"צע×Ļר×Ļ×Ŀ":138783,"تعبÙĬر":138784,"Ġ×ŀקר×Ķ":138785,"ĠÑĦакÑĤоÑĢ":138786,"ĠتÙħاÙħ":138787,"ĠتÙħاÙħا":138788,"ëįķ":138789,"ĠvÆ°á»Ŀ":138790,"ĠvÆ°á»Ŀn":138791,"ĠdÄ±ÅŁÄ±":138792,"ãģĦãģ¡":138793,"Ġ׾ק׳×ķת":138794,"ĠاÙĦعÙĦاÙĤات":138795,"пÑĥб":138796,"пÑĥбли":138797,"Ø¥ÙĬÙħ":138798,"Ø¥ÙĬÙħاÙĨ":138799,"à¸Ńำà¸Ļา":138800,"à¸Ńำà¸Ļาà¸Ī":138801,"åIJ«ãģ¾ãĤĮ":138802,"ãĤĭãģŁãĤģãģ«":138803,"ס×Ĵ":138804,"ס×Ĵ׳×ķף":138805,"تØŃدÙĬ":138806,"Ġauprès":138807,"ĠاÙĦجÙĩا":138808,"ĠاÙĦجÙĩاز":138809,"Ġ×ŀת×Ĺת":138810,"еннÑĥÑİ":138811,"Ġзим":138812,"à¸ģาà¹ģà¸Ł":138813,"Ġ×ijת×ķר":138814,"Ġnghè":138815,"Ġnghèo":138816,"ĠÐĽÑİ":138817,"ĠÐĽÑİб":138818,"תקצ×Ļ×ij":138819,"×ŀעש×Ķ":138820,"ĠاÙĦبÙĬت":138821,"צ×Ļפ":138822,"ĠобÑıзан":138823,"ĠMá»Ĺi":138824,"ĠТÑĥÑĢ":138825,"ĠÙĪباÙĦت":138826,"ĠÙĪباÙĦتاÙĦÙĬ":138827,"Ġdécision":138828,"Ġبد":138829,"Ġبدأت":138830,"Ġcục":138831,"Ġbask":138832,"Ġbaskı":138833,"Ġhatırl":138834,"Ġhatırla":138835,"å°ıãģķãģĦ":138836,"Ġgerçekten":138837,"à¸ľà¸±à¸ģ":138838,"åı¯èĥ½ãģª":138839,"×ŀ×IJס":138840,"ĠcrÃŃtica":138841,"ĠìĿĺìĽIJ":138842,"عÙĤÙĪد":138843,"×ĺ׼׳":138844,"×ĺ׼׳×ķ׾×ķ×Ĵ×Ļ×Ķ":138845,"è¨ĢãģĪãģ°":138846,"ĠÙĤÙĨا":138847,"ĠÙĤÙĨاة":138848,"ĠìĿ´ê²ĥìĿĢ":138849,"تصر":138850,"à¸Łà¸±à¸Ļ":138851,"ĠÑĢеÑĨеп":138852,"ĠÑĢеÑĨепÑĤ":138853,"ĠبÙĨÙģس":138854,"ÑĢоÑĪ":138855,"ĠмаÑĢÑĤа":138856,"Ġsonras":138857,"Ġsonrası":138858,"×ķ×ijש":138859,"ãĥªãĤ¹ãĤ¯":138860,"ĠFrançais":138861,"á»ļ":138862,"ê°Ķ":138863,"Ġ×Ķ×ijר×Ļת":138864,"פ×Ļצ":138865,"פ×Ļצ×ķ×Ļ":138866,"ĠÙĦÙħاذا":138867,"ĠÐļиев":138868,"ĠÑģмÑĭÑģл":138869,"ê¸Īìľµ":138870,"ãĤ·ãĥ£ãĥ«":138871,"ãĥ©ãĤ¤ãĥĪ":138872,"ìĽĥ":138873,"×ŀ×Ĺר":138874,"ãĨį":138875,"Ġkullanım":138876,"Ġ×IJצ׾׳×ķ":138877,"ĠtÃłn":138878,"ãĥıãĥ¼":138879,"ãģ¨ãģ¨ãĤĤ":138880,"ãģ¨ãģ¨ãĤĤãģ«":138881,"ÑĢег":138882,"ÑĢеги":138883,"ÑĢегион":138884,"ãģªãģıãģªãĤĭ":138885,"Ġchảy":138886,"ĠجÙĩØ©":138887,"ÅĦskiej":138888,"à¸Ńีà¹Ģม":138889,"à¸Ńีà¹Ģมล":138890,"ãģįãģ£ãģ¨":138891,"ĠìĺĪìĤ°":138892,"Ġkitabı":138893,"Ġeducação":138894,"ĠbuluÅŁ":138895,"ологиÑı":138896,"ĠконкÑĢ":138897,"ĠконкÑĢеÑĤ":138898,"×Ĵ×Ļר":138899,"ĠпÑĢедлаг":138900,"ĠпÑĢедлагаеÑĤ":138901,"ĠYên":138902,"Ġíķľë²Ī":138903,"Ġ×ŀר׼×ĸ×Ļ":138904,"à¹Ģà¸Ľà¸´à¸Ķà¹Ģà¸ľà¸¢":138905,"ÑĤвеÑĢд":138906,"ĠHá»ĩ":138907,"ĠÐĵÑĢ":138908,"à¸Ŀà¹īา":138909,"×Ķשק":138910,"×Ķשקע×Ķ":138911,"ĠнаÑĥк":138912,"ìłIJìĿĦ":138913,"ĠнелÑĮ":138914,"ĠнелÑĮз":138915,"ĠнелÑĮзÑı":138916,"гин":138917,"ĠBöl":138918,"ĠBölge":138919,"Ġвла":138920,"ĠвлаÑģÑĤи":138921,"à¹Ģà¸Ļà¹ĩ":138922,"à¹Ģà¸Ļà¹ĩà¸ķ":138923,"골":138924,"Ġöld":138925,"Ġöldür":138926,"×Ľ×ł×¢":138927,"ĠاÙĦÙĩÙĬئة":138928,"تارÙĬØ®":138929,"ĠÐijÑĢ":138930,"ĠÑģмож":138931,"ĠÑģможеÑĤе":138932,"ĠLúc":138933,"à¹Ħà¸Ľà¸ĸึà¸ĩ":138934,"ĠBakanı":138935,"Ġerklärt":138936,"ĠÐIJна":138937,"Ġscène":138938,"åķıãģĦ":138939,"åķıãģĦåIJĪãĤıãģĽ":138940,"ÙħÙĩÙĨد":138941,"ÙħÙĩÙĨدس":138942,"Ġназвание":138943,"иваниÑı":138944,"ãĤĴå¤īãģĪ":138945,"ä»ĺãģįåIJĪ":138946,"ãĥijãĤ½":138947,"ãĥijãĤ½ãĤ³ãĥ³":138948,"æĺİãĤī":138949,"æĺİãĤīãģĭ":138950,"à¹Ģà¸Ńà¸ģสาร":138951,"à¹Ģà¸ģิà¸Ļà¹Ħà¸Ľ":138952,"леп":138953,"ãģĹãģŁãĤĤãģ®":138954,"ĠCâm":138955,"ĠCâmara":138956,"ק×ķ׾׳×ķ×¢":138957,"Ġ×ij×Ĵ×Ļף":138958,"Ġoczy":138959,"ĠoczywiÅĽcie":138960,"attivitÃł":138961,"ãĥĵãĥ¥ãĥ¼":138962,"Ġeducación":138963,"Ä°YE":138964,"ê¹ĮìļĶ":138965,"ãĤ¨ãĥªãĤ¢":138966,"неÑģÑĤи":138967,"Ġmóg":138968,"ĠmógÅĤ":138969,"Ġק×ĺ׳×Ļ×Ŀ":138970,"ĠPrä":138971,"Ġ×ľ×¢×ij×ķר":138972,"بÙĨÙī":138973,"зол":138974,"золоÑĤ":138975,"ĠwnÄĻtr":138976,"ĠwnÄĻtrz":138977,"Ġconstrução":138978,"รัà¸ļรà¸Ńà¸ĩ":138979,"سجÙĨ":138980,"Ġק×ķ׳":138981,"ס×Ļפ×ķר":138982,"ĠÙħدÙī":138983,"رضÙī":138984,"плав":138985,"ï¼¥":138986,"Ġila":138987,"Ġilaç":138988,"ãĤĭãģ¹ãģį":138989,"ĠÙħÙĪÙĤÙģ":138990,"à¸ģรุ":138991,"à¸ģรุà¸ĵา":138992,"chodzÄħc":138993,"ĠÑĤÑĭÑģ":138994,"ÐķвÑĢо":138995,"ĠÙĬØŃدث":138996,"ãĥ¡ãĤ¤ãĥ³":138997,"ĠاÙĦصØŃÙĬ":138998,"ĠÐĶан":138999,"دعاء":139000,"ãĤ´ãĥ¼ãĥ«":139001,"×©×ł×ª×Ļ":139002,"×©×ł×ª×Ļ×Ļ×Ŀ":139003,"à¸Ķà¹īวยà¸ģัà¸Ļ":139004,"Ġolacaģı":139005,"Ġ×ij×ŀ×Ĺ×Ļר":139006,"×Ķק":139007,"×Ķק×ŀת":139008,"ãĥ¢ãĥİ":139009,"ĠçalÄ±ÅŁtı":139010,"Ġjóvenes":139011,"ãģĦãģıãĤī":139012,"ĠÙħعدÙĦ":139013,"ĠCÅ©ng":139014,"ĠSegún":139015,"Ġdönemde":139016,"Ġ׾×Ļ×ĵ×Ļ":139017,"ãģįãģ¡":139018,"ãģįãģ¡ãĤĵ":139019,"ãģįãģ¡ãĤĵãģ¨":139020,"ÙģرÙĨس":139021,"ÙģرÙĨسا":139022,"åIJijãģį":139023,"Ġcampaña":139024,"ĠÑģамоÑģÑĤоÑı":139025,"ĠÑģамоÑģÑĤоÑıÑĤелÑĮно":139026,"á»Ģ":139027,"ÙĤÙĪا":139028,"سÙĦاØŃ":139029,"à¸ģระà¹ģ":139030,"à¸ģระà¹ģส":139031,"ĠполÑĮзÑĥ":139032,"nqu":139033,"nquête":139034,"รà¹Īวมà¸ģัà¸ļ":139035,"ëĬIJëĥIJ":139036,"à¸Ĺีมà¸Ĭาà¸ķิ":139037,"Ġyıllık":139038,"ìĬ¬":139039,"ĠأصØŃاب":139040,"illé":139041,"Ġdóla":139042,"Ġdólares":139043,"Ġкож":139044,"Ġкожи":139045,"ลà¹īà¸Ń":139046,"à¹Ģรียà¸ļร":139047,"à¹Ģรียà¸ļรà¹īà¸Ńย":139048,"à¹Ģà¸ŀิ":139049,"à¹Ģà¸ŀิà¹Īà¸ĩ":139050,"ÑĢиÑĤоÑĢи":139051,"Ġíijľ":139052,"ĠíijľíĺĦ":139053,"ĠпеÑĢев":139054,"ĠпеÑĢевод":139055,"פ×Ĵ×Ļ×¢×Ķ":139056,"ĠdeÄŁerlendirme":139057,"Ùģائ":139058,"ĠвÑĭгод":139059,"ınızı":139060,"×ķ׼×Ļ×Ĺ":139061,"ĠдоÑģÑĤиг":139062,"ĠngÃłn":139063,"æĢĿãģ£ãģŁ":139064,"ĠÐķÑģÑĤÑĮ":139065,"ĠاÙĦرغÙħ":139066,"ĠzwiÄħzane":139067,"ربط":139068,"à¸Ļึà¸ĩ":139069,"Ġ׾×Ĺ×ķק":139070,"Ġszczególn":139071,"Ġszczególnie":139072,"ĠباستخداÙħ":139073,"ĠfÃŃsico":139074,"עס":139075,"עס×ķק":139076,"سÙĦÙĪÙĥ":139077,"ĠاØŃد":139078,"ÑĩÑijÑĤ":139079,"×ĸ׼×Ķ":139080,"Ġlá»ĩnh":139081,"ĠÙĪØŃت":139082,"ĠÙĪØŃتÙī":139083,"à¸Ħวามสามารà¸ĸ":139084,"à¸Ńยูà¹Īà¹ģลà¹īว":139085,"à¸ģารà¹Ģà¸Ķิà¸Ļà¸Ĺาà¸ĩ":139086,"تخذ":139087,"צ×Ļ×ķ×ĵ":139088,"ĠاÙĦأس":139089,"ĠاÙĦأسÙĩÙħ":139090,"Ġtá»ĩ":139091,"ãģ£ãģ¦ãģĦãģ¦":139092,"สรุ":139093,"à¸ªà¸£à¸¸à¸Ľ":139094,"ĠкомÑĦ":139095,"ĠкомÑĦоÑĢÑĤ":139096,"ìĺ¤ëĬĶ":139097,"ĠÑĢазв":139098,"ĠÑĢазвива":139099,"ланд":139100,"hänge":139101,"ĠبÙĨسبة":139102,"à¹Ģà¸Ĥียว":139103,"עצ×Ŀ":139104,"Ġ×ľ×ľ×Ľ×ª":139105,"ÑģоÑĨиалÑĮн":139106,"Ġëĭ¤ìĿĮê³¼":139107,"Ġרש×ķ×ŀ":139108,"×ŀר×Ĺ×ij":139109,"سÙĤØ·":139110,"Ġalanı":139111,"ĠÄijá»ĩ":139112,"é£Łãģ¹ãĤĭ":139113,"à¸Ķึà¸ĩ":139114,"Ġgegenüber":139115,"ĠبÙĩØ°Ùĩ":139116,"à¸ĸืà¸Ńà¹Ģà¸Ľà¹ĩà¸Ļ":139117,"ëķħ":139118,"à¸Ħà¸Ļà¹Ħà¸Ĺย":139119,"ãĤ¢ãĤ¦":139120,"ãĤ¢ãĤ¦ãĥĪ":139121,"ศัà¸ģ":139122,"ศัà¸ģà¸Ķิ":139123,"ศัà¸ģà¸Ķิà¹Į":139124,"ÙĤÙĪاÙĨ":139125,"ÙĤÙĪاÙĨÙĬÙĨ":139126,"Ġhá»Ļp":139127,"ãģªãģıãģªãģ£ãģ¦":139128,"Ġ×IJ×ŀ׳":139129,"Ġ×IJ×ŀ׳×Ŀ":139130,"à¹Ģà¸ķืà¸Ńà¸Ļ":139131,"ĠзавиÑģим":139132,"ĠзавиÑģимоÑģÑĤи":139133,"ת×Ļ×IJ":139134,"ת×Ļ×IJ×ķר":139135,"å§ĭãĤģãģŁ":139136,"Ġngá»į":139137,"Ġngá»įt":139138,"íĴį":139139,"ê³¼ìŀ¥":139140,"Ġbại":139141,"ãģ§ãģįãģ¦":139142,"Ġcomeçar":139143,"à¸Ľà¸£à¸²à¸ģ":139144,"à¸Ľà¸£à¸²à¸ģà¸ı":139145,"ĠгодÑĭ":139146,"меÑģ":139147,"ĠاÙĦÙħستÙĪÙī":139148,"ĠÑģамÑĭе":139149,"ллеÑĢ":139150,"ãģ£ãģ¦ãģĹãģ¾ãģĦãģ¾ãģĻ":139151,"ãģ¨ãģ®ãģĵãģ¨":139152,"bió":139153,"à¸ģลà¹Īà¸Ńà¸ĩ":139154,"ĠاÙĦزÙĪج":139155,"ãģ«è¡Įãģ£ãģŁ":139156,"à¸Ħà¹Īà¸Ńà¸Ļ":139157,"à¸Ħà¹Īà¸Ńà¸Ļà¸Ĥà¹īาà¸ĩ":139158,"ĠbaÄŁl":139159,"ĠbaÄŁlant":139160,"ĠbaÄŁlantı":139161,"確ãģĭ":139162,"確ãģĭãģ«":139163,"ãĥľãĥ¼ãĥ«":139164,"çµĤãĤıãĤĬ":139165,"ש×ŀר":139166,"à¸Ĺีà¹Īสามารà¸ĸ":139167,"ÙĦزÙħ":139168,"даеÑĤÑģÑı":139169,"รัà¸ļà¸Ľà¸£à¸°":139170,"รัà¸ļà¸Ľà¸£à¸°à¸Ĺาà¸Ļ":139171,"å¤īãĤıãĤĬ":139172,"ï¼¢":139173,"ĠìĺĪìĪĺëĭĺ":139174,"ãĤĪãģĨãģ¨":139175,"มัà¸ģà¸Īะ":139176,"ĠHÆ°Æ¡ng":139177,"ÙĨÙģØ°":139178,"×ŀ×ĵ×ĵ":139179,"ĠìĿ¸ìłķ":139180,"ÑħодиÑĤÑĮ":139181,"ĠзавиÑģиÑĤ":139182,"×ķ×ĵ×Ļ×¢":139183,"ãģĵãģ¨ãģĮãģĤãĤĬãģ¾ãģĻ":139184,"عراÙĤ":139185,"سطØŃ":139186,"à¸ģำà¹Ħร":139187,"ëĵ¤ëıĦ":139188,"×Ļצ×Ļר×Ķ":139189,"ãģĨãģĵãģ¨":139190,"ÙĦاØŃÙĤ":139191,"ãģĦãĤĮãģ°":139192,"ĠиÑģполÑĮзÑĥÑİÑĤ":139193,"ĠBợi":139194,"Ġשק׾×Ļ×Ŀ":139195,"ÑĨикл":139196,"ÐIJÐŀ":139197,"Ġ×ijש׳×Ķ":139198,"ÙĨشط":139199,"Ġש×Ļ׳×ķ×Ļ":139200,"Ġש×Ļ׳×ķ×Ļ×Ļ×Ŀ":139201,"Ġpoblación":139202,"ĠHÆ°ng":139203,"ระว":139204,"ระวัà¸ĩ":139205,"رÙĬاضة":139206,"رصد":139207,"تÙĤÙĦÙĬ":139208,"تÙĤÙĦÙĬد":139209,"Ġülkem":139210,"Ġülkemiz":139211,"à¸Ĭะ":139212,"ãĤ¯ãĥªãĥ¼ãĥł":139213,"èģŀãģĦãģŁ":139214,"Ġważ":139215,"Ġważne":139216,"ê±°ëĵł":139217,"ê±°ëĵłìļĶ":139218,"×ŀ×IJ×ijק":139219,"×Ĺ×ĵש×ķת":139220,"ĠWroc":139221,"ĠWrocÅĤaw":139222,"ĠKültür":139223,"sist":139224,"sistência":139225,"×¢×ĸר×Ķ":139226,"ĠgÆ°Æ¡ng":139227,"รà¹īาà¸Ļà¸Ħà¹īา":139228,"ĠÙĪØ£ÙĪضØŃ":139229,"ándose":139230,"ãĤ·ãĥ¼ãĥ³":139231,"×IJ׳ר×Ĵ":139232,"×IJ׳ר×Ĵ×Ļ×Ķ":139233,"ãģªãģĦãģ§ãģĻ":139234,"Ġkhủng":139235,"Ġ문ìĦľ":139236,"Ġ×ij×ĵ×ijר":139237,"×ĵ×Ļ×ķ":139238,"×ĵ×Ļ×ķ×ķ×Ĺ":139239,"Ġrégl":139240,"ÙħÙĪاد":139241,"обоÑĢ":139242,"обоÑĢоÑĤ":139243,"Ġ×Ķ×ij׾":139244,"Ġ×Ķ×ij׾×ķ×Ĵ":139245,"ØŃاÙħ":139246,"ĠاÙĦعاص":139247,"ĠاÙĦعاصÙħØ©":139248,"пеÑĢаÑĤоÑĢ":139249,"تخÙĦ":139250,"تخÙĦص":139251,"ãģŁãģłãģĹ":139252,"تسÙħ":139253,"à¹Ĥรà¸ĩà¸ŀ":139254,"à¹Ĥรà¸ĩà¸ŀยา":139255,"à¹Ĥรà¸ĩà¸ŀยาà¸ļาล":139256,"ĠYük":139257,"ĠYüksek":139258,"Ġש׳×Ļת":139259,"Ġש׳×Ļ×ª×Ł":139260,"liÄŁe":139261,"Ġפת":139262,"Ġפת×ķ×Ĺ":139263,"ĠbeÄŁ":139264,"ĠbeÄŁen":139265,"Ġ×ŀ×ķר":139266,"Ġ×ŀ×ķר׼×ij":139267,"ĠرساÙĦØ©":139268,"íĨµìĭł":139269,"Ġavalia":139270,"Ġavaliações":139271,"Ġmanh":139272,"Ġmanhã":139273,"Ġìķŀ":139274,"Ġìķŀìľ¼ë¡ľ":139275,"ÙĤتر":139276,"ÙĤترØŃ":139277,"à¹Ģà¸ģืà¸Ń":139278,"à¹Ģà¸ģืà¸Ńà¸ļ":139279,"Ġproposé":139280,"Ø£Ùħا":139281,"Ø£ÙħاÙĥÙĨ":139282,"ĠÐŀÐŀ":139283,"ĠÐŀÐŀÐŀ":139284,"ÙħÙĤار":139285,"ÙħÙĤارÙĨØ©":139286,"ëĦIJ":139287,"ãģĦãģŁãģłãģı":139288,"ÙĤÙĬÙĦ":139289,"ĠнаÑĪиÑħ":139290,"ãĤ«ãĥĥãĥĹ":139291,"×Ĺ×ľ×ª":139292,"Ġëĭ¤ë§Į":139293,"à¸Ĺัà¹Īวà¹Ĥลà¸ģ":139294,"ãĥįãĤ¿":139295,"ØŃساس":139296,"ãģ«ãģªãĤĮ":139297,"جائ":139298,"جائزة":139299,"échange":139300,"économ":139301,"économie":139302,"ТÐĺ":139303,"×¡×ª×Ľ×ľ":139304,"à¸Ĺัà¹īà¸ĩสà¸Ńà¸ĩ":139305,"ĠاÙĦخاÙħ":139306,"ĠاÙĦخاÙħس":139307,"ק×ĺ×¢":139308,"auważ":139309,"à¸ľà¸¹à¹īà¸Ĭาย":139310,"à¹ģà¸Ľà¸¥à¸ģ":139311,"åIJĮæĻĤãģ«":139312,"знаниÑı":139313,"ãģĦãģŁãģłãģįãģ¾ãģĹãģŁ":139314,"Ġ×ŀ×ij׾×Ļ":139315,"à¸Ĥà¸Ńà¹ĥหà¹ī":139316,"ĠاÙĦتربÙĬØ©":139317,"Ġdécouvert":139318,"Ġżyciu":139319,"après":139320,"Ġyab":139321,"Ġyabanc":139322,"Ġyabancı":139323,"ĠbaÅŁlayan":139324,"ìĹĪëįĺ":139325,"Ġhesabı":139326,"Ġë§Įìķ½":139327,"ë§Īëĭ¤":139328,"ĠThánh":139329,"ãĥ´ãĤ¡":139330,"à¸Ľà¸£à¸±à¸ļà¸Ľà¸£":139331,"à¸Ľà¸£à¸±à¸ļà¸Ľà¸£à¸¸à¸ĩ":139332,"ĠMặc":139333,"à¹Ģหà¸ķà¸¸à¸ľà¸¥":139334,"ĠÐijез":139335,"ĠcapacitÃł":139336,"ÅĤeÅĽ":139337,"ĠпÑĢеим":139338,"ĠпÑĢеимÑĥÑīеÑģÑĤв":139339,"ĠÅļwiÄĻt":139340,"Ġpublié":139341,"×ŀעצ×ij":139342,"ÙħشارÙĥات":139343,"à¸łà¸²à¸©":139344,"à¸łà¸²à¸©à¸µ":139345,"Ġdeuxième":139346,"ĠÙħØŃاÙģظ":139347,"ĠÙħØŃاÙģظة":139348,"ĠSchön":139349,"､":139350,"Ġ×Ķ×ij×¢":139351,"Ġ×Ķ×ij×¢×Ļ×Ķ":139352,"ĠÙĪاÙĦÙĦÙĩ":139353,"è¨Ģãģ£ãģŁ":139354,"à¸ķà¹īาà¸Ļ":139355,"วรรà¸ĵ":139356,"à¸Ĺิศ":139357,"ĠbaÅŁÄ±na":139358,"ĠmogÄĻ":139359,"ש×Ļפ×ķר":139360,"ĠÙĪعد":139361,"ĠÙĪعدÙħ":139362,"Ġhistórico":139363,"Ġkısı":139364,"ĠìĿ´ê²Į":139365,"ĠPolÃŃtica":139366,"ĠÑģиÑĤÑĥаÑĨии":139367,"ĠkoÅĦca":139368,"×ij×ĵ×Ļק×Ķ":139369,"ĠاÙĦسÙĬارات":139370,"ãģªãĤīãģ°":139371,"ãĤµãĥ©":139372,"ãĤĭãģĵãģ¨ãģĮãģ§ãģįãĤĭ":139373,"Ġdecisão":139374,"×ķ×ķ×ĵ":139375,"läss":139376,"lässig":139377,"Ġ׾×Ļשר×IJ׾":139378,"ĠÙĬأتÙĬ":139379,"ר×ķ×ĸ":139380,"Ã¶ÄŁ":139381,"Ã¶ÄŁret":139382,"Ã¶ÄŁretim":139383,"Ġдек":139384,"Ġдекаб":139385,"ĠдекабÑĢÑı":139386,"Ġש×Ĺ×ķר":139387,"ãģ¦ãģıãĤĮãģŁ":139388,"عبارة":139389,"Ġélectrique":139390,"ĠاÙĦتÙĨÙħÙĬØ©":139391,"جرÙī":139392,"ĠìĪĺíĸī":139393,"à¸Ĺู":139394,"ĠÑĢеалÑĮно":139395,"ÑģпоÑģоб":139396,"à¸Ħลà¹īาย":139397,"ĠسعÙĪد":139398,"önü":139399,"ĠÙģÙħÙĨ":139400,"تÙĥÙĪ":139401,"تÙĥÙĪÙĬÙĨ":139402,"ĠкаÑĩеÑģÑĤво":139403,"ĠконÑĤак":139404,"ĠконÑĤакÑĤ":139405,"ĠsözleÅŁme":139406,"à¸Ńà¹īาà¸ĩ":139407,"ĠتÙĪÙģ":139408,"ĠتÙĪÙģÙĬر":139409,"×Ķ×ĸ×ĵ":139410,"×Ķ×ĸ×ĵ×ŀ׳×ķת":139411,"ĠØ·ÙĪÙĬÙĦØ©":139412,"Ġtérmino":139413,"Ġ×IJ×Ļפ×Ķ":139414,"ãĥĵãĥ«":139415,"สà¹Ĥม":139416,"สà¹Ĥมสร":139417,"ĠاÙĦاث":139418,"ĠاÙĦاثÙĨÙĬÙĨ":139419,"евиÑĩ":139420,"Ġopinión":139421,"à¸Ľà¸§à¸Ķ":139422,"åı¤ãģĦ":139423,"รà¹Īา":139424,"ĠBiaÅĤ":139425,"ĠÑģÑĤал":139426,"ĠÑģÑĤало":139427,"ólogo":139428,"ĠìķĦëĭĪëĭ¤":139429,"Ġ×IJ×Ļת":139430,"Ġ×IJ×Ļת×ķ":139431,"à¹Ģหà¹ĩà¸Ļวà¹Īา":139432,"à¸ļารà¹Į":139433,"çĦ¼":139434,"çĦ¼ãģį":139435,"ĠìĿ´ìļ©ìŀIJ":139436,"ĠнекоÑĤоÑĢÑĭе":139437,"ksz":139438,"ksztaÅĤ":139439,"ksztaÅĤc":139440,"ãĤŃãĥ£ãĥĥãĤ·":139441,"ãĤŃãĥ£ãĥĥãĤ·ãĥ³ãĤ°":139442,"ĠroÅĽ":139443,"ĠroÅĽlin":139444,"ÑĢажа":139445,"×ij׳×Ļ×Ļ×Ķ":139446,"à¸Ľà¸£à¸ªà¸´":139447,"à¸Ľà¸£à¸ªà¸´à¸ķ":139448,"Ġgördü":139449,"×ŀ׳×Ķ×Ļ×Ĵ":139450,"å¤īãĤıãģ£ãģ¦":139451,"Ġ×IJ×Ķ":139452,"Ġ×IJ×Ķ×ijת×Ļ":139453,"à¹Ģรà¹Īà¸ĩ":139454,"Ġönünde":139455,"Ġê·¸ëĥ¥":139456,"полиÑĤ":139457,"полиÑĤиÑĩеÑģк":139458,"ãĥ¡ãĥĩãĤ£":139459,"ãĥ¡ãĥĩãĤ£ãĤ¢":139460,"ĠDetay":139461,"ĠDetaylı":139462,"ĠاÙĦصÙģØŃØ©":139463,"à¸ģารà¹Ģà¸ĩิà¸Ļ":139464,"Ġìµľê·¼":139465,"׼ש׾":139466,"Ｉ":139467,"вÑĪего":139468,"íķĺìĭ¤":139469,"ĠÐŃÑĤ":139470,"ĠÐŃÑĤоÑĤ":139471,"สื":139472,"สืà¸ļ":139473,"Ġngừng":139474,"ĠдокÑĥменÑĤов":139475,"даваÑĤÑĮ":139476,"ĠاÙĦشخصÙĬØ©":139477,"Ġצע×Ļר":139478,"درÙĥ":139479,"سØŃب":139480,"à¹Ħมà¹Īà¸Ħà¹Īà¸Ńย":139481,"Ġ×Ķ×ŀק×ķ×ŀ×Ļ":139482,"สัà¹Īà¸ĩà¸ĭืà¹īà¸Ń":139483,"Ġê·¸ê²ĥìĿĦ":139484,"ãģĤãĤĭãģĦ":139485,"ãģĤãĤĭãģĦãģ¯":139486,"×IJ×ķ×ĺ×ķ×ij":139487,"×IJ×ķ×ĺ×ķ×ij×ķס":139488,"кÑĨион":139489,"ĠÐľÐ¾Ð¶Ð½Ð¾":139490,"ãģıãģł":139491,"ãģıãģłãģķ":139492,"ĠинÑĦоÑĢмаÑĨиÑı":139493,"ï»Ł":139494,"ĠìŀijìĹħ":139495,"Ġ×Ļ×ķסף":139496,"إدارة":139497,"ĠاÙĦØŃاج":139498,"×ł×¡×Ļ×¢×Ķ":139499,"изаÑĨиÑı":139500,"×IJ׾×ij":139501,"×IJ׾×ij×ķ×Ŀ":139502,"пед":139503,"Ġק×ĺ׳×Ķ":139504,"ĠÙĨÙģسÙĩا":139505,"ĠMinistério":139506,"Ġпен":139507,"ĠпенÑģи":139508,"ãĥIJãĥ©ãĥ³ãĤ¹":139509,"Ġ×Ķת×ķר×Ķ":139510,"Ġtạm":139511,"ĠìĹŃìĭľ":139512,"｡":139513,"Ġthá»±":139514,"Ġısı":139515,"컨":139516,"ãģĹãģ£ãģĭãĤĬãģ¨":139517,"ĠxÆ°a":139518,"Ġcặp":139519,"×Ĺ×Ļ×ij×ķר":139520,"วัà¸Ĵà¸Ļà¸ĺรรม":139521,"stär":139522,"stärke":139523,"ĠÑģамÑĭй":139524,"pisa":139525,"pisaÄĩ":139526,"ĠoluÅŁan":139527,"ĠاÙĦØ¥ÙħاÙħ":139528,"ĠcÄĥng":139529,"Ġgünl":139530,"Ġgünlük":139531,"Ġ׳ש×IJר":139532,"Ġkhiá»ĥn":139533,"ç¶ļãģijãĤĭ":139534,"stitución":139535,"Ġcapacité":139536,"Ġjaki":139537,"ĠjakiÅĽ":139538,"вÑĪиÑģ":139539,"вÑĪиÑģÑĮ":139540,"פע×ķ׾×ķת":139541,"ĠØŃÙĬات":139542,"ĠØŃÙĬاتÙĩ":139543,"Ġникогда":139544,"ÐĽÐ¬":139545,"Ġ×Ķ×¢×ķ×ij":139546,"Ġ×Ķ×¢×ķ×ij×ĵ×Ķ":139547,"ĠchÃło":139548,"หลายà¹Ĩ":139549,"ĠÑıн":139550,"ĠÑıнваÑĢ":139551,"ĠÑıнваÑĢÑı":139552,"à¸Īำà¹Ģà¸Ľà¹ĩà¸Ļà¸ķà¹īà¸Ńà¸ĩ":139553,"Ġhöher":139554,"ãģķãĤĮãģ¦ãģĦãģŁ":139555,"สà¸ĩสั":139556,"สà¸ĩสัย":139557,"ĠاÙĦاس":139558,"ĠاÙĦاسÙĦاÙħ":139559,"ĠاÙĦØ´Ùħس":139560,"สà¸ĸาà¸Ļี":139561,"ãĤ¯ãĥ©ãĤ¹":139562,"à¸ŀรร":139563,"à¸ŀรรà¸Ħ":139564,"põ":139565,"põe":139566,"Ġporém":139567,"à¸Ľà¸£à¸°à¸ªà¸ĩ":139568,"à¸Ľà¸£à¸°à¸ªà¸ĩà¸Ħà¹Į":139569,"powiedzie":139570,"powiedzieÄĩ":139571,"ĠмогÑĥ":139572,"Ġжел":139573,"Ġжелез":139574,"ĠاÙĦØ«ÙĤ":139575,"ĠاÙĦØ«ÙĤاÙģÙĬ":139576,"ĠпÑĢавило":139577,"Ġgdyż":139578,"פש×ķ×ĺ":139579,"ÑĢабоÑĤка":139580,"ĠÙĥرة":139581,"شدد":139582,"ÙħارÙĥ":139583,"ÙħÙĥØ©":139584,"ĠподпиÑģ":139585,"×ĺ×ķ×ķ×Ĺ":139586,"ĠÅĽc":139587,"ĠÅĽcian":139588,"ĠرجاÙĦ":139589,"Ġ×ª×ľ×ķ×Ļ":139590,"иÑĪ":139591,"иÑĪÑĮ":139592,"Ġmédec":139593,"Ġmédecin":139594,"ëįĶëĿ¼ëıĦ":139595,"ĠÑĤебÑı":139596,"Ġ׾×Ķ×ķס×Ļ×£":139597,"ãģĬ話":139598,"Ġà¹ģà¸ķà¹Īà¸ģà¹ĩ":139599,"داÙģ":139600,"داÙģع":139601,"ĠCùng":139602,"ãĥ»ãĥ»ãĥ»ãĥ»":139603,"ê¶ģ":139604,"ĠdeberÃŃa":139605,"หà¸Ļà¹Īวยà¸ĩาà¸Ļ":139606,"ĠvaÌĢ":139607,"Ġעצ×ŀ":139608,"Ġעצ×ŀ×Ŀ":139609,"à¹Ģà¸Ĭืà¹Īà¸Ńวà¹Īา":139610,"שקע":139611,"Ġ×Ķ׼×ķ׾":139612,"Ġ×Ķ׼×ķ׾׾":139613,"нибÑĥд":139614,"нибÑĥдÑĮ":139615,"ĠëĦĪíĿ¬":139616,"ĠобÑĢаÑī":139617,"ĠобÑĢаÑīа":139618,"Ġ×¢×ij×ķ×ĵת":139619,"ĠاÙĦÙħÙĨتخب":139620,"ıyord":139621,"ıyordu":139622,"ÙĪØ°":139623,"×Ĺש×Ļ×ij×ķת":139624,"Ġ×Ķ×¢×Ļק":139625,"Ġ×Ķ×¢×Ļקר×Ļ":139626,"ì¢Į":139627,"ยุà¹Ĥร":139628,"ยุà¹Ĥà¸£à¸Ľ":139629,"ĠапÑĢ":139630,"ĠапÑĢелÑı":139631,"szed":139632,"szedÅĤ":139633,"дон":139634,"à¹Ģà¸ķิà¸ļ":139635,"à¹Ģà¸ķิà¸ļà¹Ĥà¸ķ":139636,"коло":139637,"Ġkażdej":139638,"帰":139639,"帰ãĤĬ":139640,"Ġмилли":139641,"Ġмиллион":139642,"ç¾İåij³ãģĹãģĦ":139643,"تÙĤار":139644,"تÙĤارÙĬر":139645,"ĠìĿ´ë£¨":139646,"ĠìĿ´ë£¨ìĸ´":139647,"Ġsprzedaż":139648,"×Ķ×ķצ×IJ×ķת":139649,"ãĤ¢ãĤ¯ãĤ»":139650,"ãĤ¢ãĤ¯ãĤ»ãĤ¹":139651,"ר×ķ×¥":139652,"ĠгоÑģÑĥдаÑĢÑģÑĤвенн":139653,"Ø£ØŃÙĥ":139654,"Ø£ØŃÙĥاÙħ":139655,"ĠoluÅŁu":139656,"ĠAç":139657,"ĠAçık":139658,"ãĤ¸ãĥ¼":139659,"ç´łæĻ´":139660,"ç´łæĻ´ãĤīãģĹãģĦ":139661,"Ġ×ijש×ij×ķ×¢":139662,"بذ":139663,"بذÙĦ":139664,"สาà¹Ģหà¸ķุ":139665,"Ġpozosta":139666,"ĠpozostaÅĤ":139667,"ØŃرÙħ":139668,"Ġimportância":139669,"leÅŁtirme":139670,"ĠдÑĢев":139671,"Ġmóvil":139672,"ĠAynı":139673,"Ġналог":139674,"Ġналогов":139675,"Ġ×Ĺ×Ļפ×Ķ":139676,"ĠÑĦоÑĢмÑĥ":139677,"à¸Ĺà¸Ķสà¸Ńà¸ļ":139678,"ĠksiÄħżki":139679,"ĠmaÅĤe":139680,"ÙħسأÙĦ":139681,"ÙħسأÙĦØ©":139682,"＾＾":139683,"çãeste":139684,"éviter":139685,"ĠконÑģÑĤÑĢÑĥк":139686,"ĠконÑģÑĤÑĢÑĥкÑĨи":139687,"ï¾ŀ":139688,"Ġת×ķ׼׳":139689,"ãĤ¹ãĥĪãĥ¬ãĤ¹":139690,"ĠاÙĦاÙĤتصادÙĬ":139691,"×ŀ×ĵ×Ļ":139692,"ĠwÅĤad":139693,"ĠwÅĤadz":139694,"Ø®ÙĪÙģ":139695,"ĠмаÑĤеÑĢиалов":139696,"ãģ¨ãģ£ãģ¦ãĤĤ":139697,"Ġznajdu":139698,"ĠznajdujÄħ":139699,"Ùģئة":139700,"ãģ©ãģ®ãĤĪãģĨãģª":139701,"æĬijãģĪ":139702,"׳×Ĺ׾":139703,"Ġdüny":139704,"Ġdünyan":139705,"Ġdünyanın":139706,"гÑĢани":139707,"гÑĢаниÑĩ":139708,"Ġ×Ķש׾×Ļש×Ļ":139709,"Ġ×Ķ×IJש":139710,"åıĬãģ³":139711,"ìĭŃìĭľ":139712,"ìĭŃìĭľìĺ¤":139713,"Ġдолл":139714,"ĠдоллаÑĢ":139715,"ĠповÑĤоÑĢ":139716,"Ġ×Ĺ×Ļ׳×Ŀ":139717,"תפת×Ĺ":139718,"Ñĥвели":139719,"ÑĥвелиÑĩен":139720,"ãĤ«ãĥª":139721,"rawid":139722,"rawidÅĤow":139723,"×ķ×ķ׾":139724,"ãĥŁãĥ¥":139725,"ì½ĺ":139726,"ĠByÅĤ":139727,"ÐľÐIJ":139728,"عÙIJ":139729,"ĠÑģовеÑĢÑĪ":139730,"ĠÑģовеÑĢÑĪенно":139731,"Ġмой":139732,"Ġ×ķ׾×IJ×Ĺר":139733,"æħ£":139734,"æħ£ãĤĮ":139735,"ØŃاÙģظ":139736,"Ġ무ë£Į":139737,"à¸Ħà¸ĵะà¸ģรรม":139738,"à¸Ħà¸ĵะà¸ģรรมà¸ģาร":139739,"Ġìĸ´ëĶĶ":139740,"Ġdiferen":139741,"Ġdiferença":139742,"ĠاÙĦأساس":139743,"ĠاÙĦأساسÙĬØ©":139744,"Ġ׾×IJ×Ĺר×ķ׳×Ķ":139745,"ê·ł":139746,"Ġ×Ķש׳×Ļ×Ļ×Ķ":139747,"ìľĦìĽIJìŀ¥":139748,"ลุà¸ģ":139749,"çiler":139750,"Ġ×Ķ×IJ׾×ķ":139751,"èģŀãģı":139752,"Ġ×ķ×IJפ×Ļ׾×ķ":139753,"ĠÑĢеализ":139754,"ĠÑĢеализаÑĨи":139755,"ระยะà¹Ģวลา":139756,"ĠجداÙĭ":139757,"تباع":139758,"ĠvehÃŃculo":139759,"Ġдолг":139760,"à¸Ľà¸£à¸´à¸¡à¸²à¸ĵ":139761,"ì¦IJ":139762,"Ġ׾×ŀק×ķ×Ŀ":139763,"ĠìĤ¬ì§Ħ":139764,"à¸Ĭà¹īา":139765,"Ġ×ŀ×¢×ķ׾×Ķ":139766,"Ġgörm":139767,"Ġgörmek":139768,"ĠÙĪÙĩØ°Ùĩ":139769,"пеÑĢв":139770,"пеÑĢвÑĭÑħ":139771,"ê·¸ëŀĺ":139772,"ĠاÙĦبرÙĬØ·":139773,"ĠاÙĦبرÙĬطاÙĨÙĬ":139774,"ĠиÑİнÑı":139775,"ĠÐĵоÑĢ":139776,"Ġ׾ש׾×Ŀ":139777,"ÐIJÐĿ":139778,"ĠназнаÑĩен":139779,"ооÑĢ":139780,"ооÑĢÑĥж":139781,"Ġözelli":139782,"ĠözelliÄŁi":139783,"Ġниже":139784,"ç¶ļãģijãģ¦":139785,"ĠаÑĢенд":139786,"Ġkatılı":139787,"Ġkatılım":139788,"ĠإطÙĦاÙĤ":139789,"ĠÙĪإذا":139790,"ĠокÑĤÑı":139791,"ĠокÑĤÑıбÑĢÑı":139792,"à¹Ĥà¸ķà¹":139793,"à¹Ĥà¸ķà¹Ĭ":139794,"à¹Ĥà¸ķà¹Ĭะ":139795,"Ġoldukları":139796,"ÙħÙĪÙĤع":139797,"ëĤ©":139798,"ãģ¨æĢĿãģ£ãģ¦ãģĦãĤĭ":139799,"Ġש×Ļ׼×ķ׾":139800,"วาà¸Ķ":139801,"سÙĬÙĦ":139802,"à¸Ĥวั":139803,"à¸Ĥวัà¸į":139804,"تØŃÙĥÙħ":139805,"ìĤŃ":139806,"Ġconnaît":139807,"×ł×¤×ª×Ĺ":139808,"Ġchặ":139809,"Ġchặn":139810,"ĠÙħØŃÙħ":139811,"ĠÙħØŃÙħÙĪد":139812,"ãģ´":139813,"ĠпÑĢодÑĥкÑĨии":139814,"здÑĢав":139815,"ãģĶè¦":139816,"ãģĶ覧":139817,"×IJ×ij×IJ":139818,"Ġvéritable":139819,"ĠØ·ÙģÙĦ":139820,"ãĥĪãĥ©ãĥĸãĥ«":139821,"곡":139822,"Ġת×ŀ×ķ׳×Ķ":139823,"Ġkiên":139824,"ĠÙĤادر":139825,"Ø¥ÙĤÙĦÙĬÙħ":139826,"ĠпÑĢедпÑĢи":139827,"ĠпÑĢедпÑĢиÑıÑĤиÑı":139828,"ĠbÄĥng":139829,"Ġayında":139830,"Ġgấp":139831,"еÑħал":139832,"ĠgiÃłnh":139833,"Ġдав":139834,"Ġдавно":139835,"ìĺĢëĭ¤":139836,"à¸Ļัà¸ģà¹Ģà¸ķ":139837,"à¸Ļัà¸ģà¹Ģà¸ķะ":139838,"Ùħستشار":139839,"ستراتÙĬج":139840,"ستراتÙĬجÙĬ":139841,"رÙħز":139842,"ĠtÄ©nh":139843,"ë¡Ń":139844,"ĠÑĩеÑĤ":139845,"ĠÑĩеÑĤÑĭ":139846,"ĠÑĩеÑĤÑĭÑĢе":139847,"ĠEntão":139848,"Ġصغ":139849,"ĠصغÙĬرة":139850,"×ij×Ļ×ĺ×ķ׾":139851,"خطÙĪØ·":139852,"ĠÑĢазвиÑĤие":139853,"Ġamacıyla":139854,"à¸Ĺีวี":139855,"ĠоÑģÑĤ":139856,"ĠоÑģÑĤалÑĮн":139857,"ש×ķ׾×Ĺף":139858,"Ġ׼׳×Ļס":139859,"Ġ׼׳×Ļס×Ķ":139860,"ĠdáºŃy":139861,"ĠyaÅŁayan":139862,"Ġ×ŀ×Ķ×ķ×ķ×Ķ":139863,"ĠÑĥÑģи":139864,"ĠÑĥÑģили":139865,"×ŀפ×Ļ":139866,"ĠпÑĢоведениÑı":139867,"Ġرب":139868,"ĠربÙħا":139869,"ĠاÙĦØ£ÙĪسط":139870,"Ġìľłì§Ģ":139871,"Ġpracownik":139872,"Ġpracowników":139873,"×ŀס×ķרת":139874,"ÙĤارب":139875,"à¸Ħวามรูà¹īสึà¸ģ":139876,"à¹ģหละ":139877,"ĠاÙĦÙĨÙĤد":139878,"Ġ×IJ׾פ×Ļ":139879,"Ùħسئ":139880,"ÙħسئÙĪÙĦ":139881,"евÑĭÑħ":139882,"клÑİÑĩениÑı":139883,"×ij×Ļ׳":139884,"×ij×Ļ׳×Ļ×Ķ×Ŀ":139885,"ש×ķ×IJ×Ķ":139886,"ĠÅŁark":139887,"ĠÅŁarkı":139888,"Ġsürec":139889,"Ġsürecin":139890,"à¹Ģà¸Ħรà¸Ķ":139891,"à¹Ģà¸Ħรà¸Ķิà¸ķ":139892,"ãĥIJãĥ¬":139893,"ĠشأÙĨ":139894,"à¹Ģà¸Ńาà¹Ħวà¹ī":139895,"niÄĻcie":139896,"רצ×Ĺ":139897,"ĠaÅŁama":139898,"׳פ×Ĵ×¢":139899,"Ġthá»Ŀ":139900,"Ġkhuẩn":139901,"diÄŁinde":139902,"ÑıÑīиÑħ":139903,"ãĥĺãĥ«":139904,"Ġüberh":139905,"Ġüberhaupt":139906,"ĠÑĤÑĢебова":139907,"ĠdÅĤugi":139908,"×ĺ×Ļף":139909,"à¸Ĥà¸Ļาà¸Ķà¹ĥหà¸įà¹Ī":139910,"ĠاÙĦØ£Ùĩ":139911,"ĠاÙĦØ£ÙĩÙĦÙĬ":139912,"ĠMüd":139913,"ĠMüdürü":139914,"Ġ×Ļ×Ķ×ķ×ĵ×Ķ":139915,"ÑĭваеÑĤÑģÑı":139916,"ساط":139917,"×Ķ×ª×ł×Ķ×Ĵ":139918,"×Ķ×ª×ł×Ķ×Ĵ×ķת":139919,"à¸ģà¸²à¸£à¸ľà¸¥à¸´à¸ķ":139920,"íĴĢ":139921,"สà¸ĸาà¸Ļà¸ģารà¸ĵà¹Į":139922,"ĠоÑĦ":139923,"ĠоÑĦиÑģ":139924,"ĠÙĦعبة":139925,"ĠstronÄĻ":139926,"Ġר×IJ×ķ×Ļ":139927,"×Ĺ×ij׾":139928,"ĠÑĢÑĭн":139929,"ĠÑĢÑĭнке":139930,"Ġ׾×ŀ×¢×Ł":139931,"اسÙĦ":139932,"หัà¸Ļ":139933,"Ġ×IJ×Ĺ×Ļ":139934,"ĠпÑĢодол":139935,"ê°Ģìŀħ":139936,"Ġ×ijר×Ĺ":139937,"Ġ×ijר×Ĺ×ij×Ļ":139938,"джеÑĢ":139939,"Ġ׾×Ĺ׾":139940,"Ġ׾×Ĺ׾×ķ×ĺ":139941,"Ġ׾×Ĺ׾×ķ×ĺ×Ļף":139942,"ศาสà¸Ļา":139943,"ãĤ¢ãĤ¤ãĥĨ":139944,"ãĤ¢ãĤ¤ãĥĨãĥł":139945,"Ġפר×ķפ":139946,"جزاء":139947,"ลà¸Ńย":139948,"ĠciaÅĤa":139949,"Ġgiết":139950,"ĠзнаÑĩиÑĤелÑĮно":139951,"Ġolmadıģ":139952,"Ġolmadıģını":139953,"нд":139954,"ндекÑģ":139955,"تأÙĥد":139956,"Ġìĸ¸":139957,"Ġìĸ¸ìłľ":139958,"aydın":139959,"ãĥīãĥ¬ãĤ¹":139960,"Ġsắt":139961,"Ġíĺ¸íħĶ":139962,"Ġë¶ģ":139963,"Ġë¶ģíķľ":139964,"ãĥijãĤ¤":139965,"Ġ×ŀש×Ĺק×Ļ":139966,"à¸Ħà¸Ļà¸Ńืà¹Īà¸Ļ":139967,"ĠизгоÑĤов":139968,"ĠизгоÑĤовлен":139969,"à¹Ģà¸ģียร":139970,"à¹Ģà¸ģียรà¸ķิ":139971,"תקשר":139972,"ĠÑĢаÑģÑĩеÑĤ":139973,"สà¹Ģà¸ķ":139974,"Ġlänger":139975,"ĠiÅŁlet":139976,"ĠiÅŁletme":139977,"ĠعÙĦÙĬÙĨ":139978,"ĠعÙĦÙĬÙĨا":139979,"élection":139980,"ĠاÙĦغربÙĬØ©":139981,"íĭĢ":139982,"ãĤĤãĤīãģĪ":139983,"Ġкниги":139984,"أسÙħ":139985,"أسÙħاء":139986,"Ġthá»ı":139987,"Ġthá»ıa":139988,"หà¸Ļู":139989,"Ġ×ł×¢×©×Ķ":139990,"à¸łà¸²à¸¢à¹ĥà¸ķà¹ī":139991,"à¸ŀืà¸Ĭ":139992,"رÙĬØ·":139993,"ÙģÙĪض":139994,"ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸãģĦãģ¾ãģĹãģŁ":139995,"ש×ĵ×Ķ":139996,"Ġngá»±c":139997,"ĠÑģеÑĢÑĮ":139998,"ĠÑģеÑĢÑĮезн":139999,"Tôi":140000,"Ġfiyatları":140001,"ĠвÑģÑİ":140002,"ĠCódigo":140003,"Ġ×Ķש×IJ":140004,"Ġ×Ķש×IJ׾×Ķ":140005,"ĠPública":140006,"إخ":140007,"إخÙĪاÙĨ":140008,"ĠзаÑıвил":140009,"ãĥ¦ãĥ¼":140010,"ר×IJ×Ļת":140011,"volución":140012,"Ġszko":140013,"ĠszkoÅĤy":140014,"جرÙĬدة":140015,"Ġpensé":140016,"ìī¬":140017,"ĠBüyükÅŁehir":140018,"ĠØ£ÙħرÙĬ":140019,"ĠØ£ÙħرÙĬÙĥÙĬ":140020,"à¸Ļัà¸ģศึà¸ģษา":140021,"Ġtodav":140022,"ĠtodavÃŃa":140023,"ĠСан":140024,"ĠСанкÑĤ":140025,"íķĺìŀIJ":140026,"ØŃÙĪاÙĦ":140027,"׼×ķשר":140028,"à¹Ģลยà¸Ħรัà¸ļ":140029,"Ġalgu":140030,"Ġalguém":140031,"Ùģز":140032,"Ġçekil":140033,"Ġ×ĵר׼×Ļ×Ŀ":140034,"ãĥIJãĥ©":140035,"à¸ģà¹ĩสามารà¸ĸ":140036,"สà¹Īวà¸Ļลà¸Ķ":140037,"íı°":140038,"ĠPúb":140039,"ĠPúblico":140040,"à¹ģà¸Ļวà¸Ĺาà¸ĩ":140041,"×IJת×Ĵר":140042,"شاش":140043,"شاشة":140044,"ciÅĽni":140045,"ĠÃľrün":140046,"ÙĦÙĪØŃ":140047,"ĠاÙĦبÙĨ":140048,"ĠاÙĦبÙĨÙĥ":140049,"ì¡°ì¹ĺ":140050,"Ġorganización":140051,"ãģĤãĤĬãģĮãģ¨ãģĨãģĶãģĸãģĦãģ¾ãģĻ":140052,"sätze":140053,"ĠÑģемей":140054,"ÙĤصد":140055,"ÑģÑĤвеннÑĭе":140056,"Ġprécéd":140057,"Ġprécédent":140058,"à¸ģรุà¸ĩà¹Ģà¸Ĺà¸ŀฯ":140059,"ãģ¨è¨ĢãģĦ":140060,"×ij׳×Ļ×Ļף":140061,"ĠØŃÙĪ":140062,"ĠØŃÙĪاÙĦÙĬ":140063,"סקס":140064,"ĠsaÄŁlamak":140065,"Ġ׾צ×Ļ×Ļף":140066,"ק×ĵש":140067,"Ġ×Ķ×ŀ×¢×¨×Ľ×ª":140068,"Ġ׾×Ķ×¢×ij×Ļר":140069,"Ġgünd":140070,"Ġgündem":140071,"ĠнаÑĪего":140072,"à¹ĥà¸Ļà¸ŀืà¹īà¸Ļà¸Ĺีà¹Ī":140073,"à¹Ģà¸Ħรืà¸Ń":140074,"à¹Ģà¸Ħรืà¸Ńà¸Ĥ":140075,"à¹Ģà¸Ħรืà¸Ńà¸Ĥà¹Īาย":140076,"ظاÙĩرة":140077,"ÙħÙĨظÙħ":140078,"ÙħÙĨظÙħات":140079,"Ùħتاز":140080,"追ãģĦ":140081,"dıkt":140082,"dıktan":140083,"ĠëįĶìļ±":140084,"ĠÐĿапÑĢимеÑĢ":140085,"twór":140086,"×ŀ×ķעצ×Ķ":140087,"ÙĥÙĪÙĥ":140088,"Щ":140089,"×ŀ×ĺפ׾":140090,"ólica":140091,"訪ãĤĮ":140092,"ĠëĮĢë¶Ģ":140093,"ĠëĮĢë¶Ģë¶Ħ":140094,"ãĤ¯ãĥªãĥĥãĤ¯":140095,"ãĤĴéģ¸":140096,"ãĤĴéģ¸ãģ¶":140097,"Ġpowsta":140098,"ĠpowstaÅĤ":140099,"Ġrazón":140100,"×ij×ķ×Ĺר":140101,"ĠÑģообÑīил":140102,"Ġק×ij×ķ×¢":140103,"rêt":140104,"à¸Ķีà¸Ĥึà¹īà¸Ļ":140105,"×ŀסע×ĵ":140106,"×ŀסע×ĵ×ķת":140107,"ĠÃĸsterreich":140108,"Ġ׳×Ĺש×ij":140109,"Ùħبادرة":140110,"ì´ī":140111,"×Ĵ׳×ĺ×Ļ":140112,"ä¿¡ãģĺ":140113,"duÄŁ":140114,"duÄŁunu":140115,"Ġphú":140116,"ĠاÙĦأخÙĬر":140117,"Ġتعتبر":140118,"landırıl":140119,"ãģ¨ãģ¯ãģĦ":140120,"ãģ¨ãģ¯ãģĦãģĪ":140121,"ĠاÙĦØ·ÙĦ":140122,"ĠاÙĦØ·ÙĦاب":140123,"ĠNº":140124,"éģ¿ãģij":140125,"اÙĦÙħع":140126,"اÙĦÙħعرÙĪÙģ":140127,"à¸ªà¸łà¸²":140128,"éĽ¢ãĤĮ":140129,"ĠпомоÑīÑĮ":140130,"ĠзнаеÑĤ":140131,"ãĥĹãĥ¬ãĤ¼":140132,"ãĥĹãĥ¬ãĤ¼ãĥ³ãĥĪ":140133,"Ġsupérieur":140134,"Ġש׾×Ļש×Ļ":140135,"ĠاÙĦÙĨÙĪع":140136,"ãĤĵãģ§ãģĻãģŃ":140137,"à¸Ńà¸ļรม":140138,"Ġgiá»įng":140139,"ĠwzglÄĻd":140140,"ĠاÙĦÙģÙĤر":140141,"èrent":140142,"Ġ×ŀ×IJ×Ĺ":140143,"Ġ×ŀ×IJ×Ĺ×ķר×Ļ":140144,"×Ĵ×Ĵ":140145,"×Ļ×Ļ×ij":140146,"ÙħÙĦاب":140147,"ÙħÙĦابس":140148,"Ġhükü":140149,"Ġhükümet":140150,"Ġ×ŀ×Ĵ×Ļ×ij":140151,"ĠÐŀÑĩ":140152,"ĠÐŀÑĩенÑĮ":140153,"æĹ©ãģĦ":140154,"Ġconstrucción":140155,"Ġthượng":140156,"ï¼ĭ":140157,"Ġcoração":140158,"à¹Ģหลà¹ĩà¸ģ":140159,"ĠBaÅŁb":140160,"ĠBaÅŁbakan":140161,"éĢ£ãĤĮ":140162,"ãģĻãĤĭãģĵãģ¨ãģĮãģ§ãģįãģ¾ãģĻ":140163,"ĠÙĤاÙħت":140164,"ĠاÙĥثر":140165,"ÙģاعÙĦ":140166,"ĠÑĦоÑĢ":140167,"ĠÑĦоÑĢÑĥм":140168,"غذÙĬ":140169,"ĠiÅŁle":140170,"ĠiÅŁleml":140171,"ĠiÅŁlemleri":140172,"ĠìĤ¬ëŀĮìĿĢ":140173,"ĠìŀijìĦ±":140174,"Ġë§Ī볨":140175,"ÙħجÙĦس":140176,"หมู":140177,"дв":140178,"двиг":140179,"двига":140180,"à¹Ģสียà¸Ĭีวิà¸ķ":140181,"×Ķתפת×Ĺ":140182,"×Ķתפת×Ĺ×ķת":140183,"ĠмеÑĤÑĢо":140184,"ĠÑģенÑĤ":140185,"ĠÑģенÑĤÑı":140186,"ĠÑģенÑĤÑıбÑĢÑı":140187,"곧":140188,"Ġ×ľ×¤×¢":140189,"Ġ×ľ×¤×¢×ŀ×Ļ×Ŀ":140190,"à¹Ģà¸ļีย":140191,"詳ãģĹãģı":140192,"çķ°ãģªãĤĭ":140193,"ĠÄ°lçe":140194,"ĠAtat":140195,"ĠAtatür":140196,"ĠAtatürk":140197,"รุà¹Īà¸ĩ":140198,"Ġkaldı":140199,"Ġ주ìŀ¥":140200,"Ġprésence":140201,"Ġнаб":140202,"ĠнаблÑİ":140203,"ĠнаблÑİда":140204,"ĠÑģамого":140205,"×Ĵ×ķש":140206,"×ŀ×ĺ×ķפ":140207,"×ŀ×ĺ×ķפ׾":140208,"ĠвÑĭбиÑĢа":140209,"ĠìŀIJ리":140210,"åĪĨãģĭãĤīãģªãģĦ":140211,"ĠзÑĥб":140212,"Ġש׼×ijר":140213,"Ġدائ":140214,"ĠدائÙħا":140215,"ĠпаÑĢÑĤи":140216,"ï¼²":140217,"ĠاÙĬضا":140218,"ĠÑħоз":140219,"ĠÑħозÑı":140220,"ĠÑħозÑıй":140221,"ĠÑħозÑıйÑģÑĤв":140222,"ĠاÙĦأج":140223,"ĠاÙĦأجÙĨب":140224,"ĠاÙĦأجÙĨبÙĬØ©":140225,"ĠÐĹна":140226,"ĠApós":140227,"ĠÑįнеÑĢ":140228,"ĠÑįнеÑĢги":140229,"Ġyans":140230,"Ġyansı":140231,"ĠJusti":140232,"ĠJustiça":140233,"Ġprévu":140234,"มวล":140235,"ìŀ¥ëĭĺ":140236,"à¸ģระà¸ļ":140237,"à¸ģระà¸ļวà¸Ļ":140238,"à¸ģระà¸ļวà¸Ļà¸ģาร":140239,"×ŀ×ŀ":140240,"×ŀ×ŀ×ķצע":140241,"Ġhẹ":140242,"Ġhẹn":140243,"здание":140244,"ĠakÅŁ":140245,"ĠakÅŁam":140246,"×ĺ×ķפ":140247,"Ġgerekt":140248,"Ġgerekti":140249,"ĠgerektiÄŁini":140250,"Ġnarz":140251,"ĠnarzÄĻdzi":140252,"épo":140253,"époque":140254,"ĠThần":140255,"Ġwysoko":140256,"ĠwysokoÅĽci":140257,"à¸ľà¸¹à¹īà¸Ľ":140258,"à¸ľà¸¹à¹īà¸Ľà¹Īวย":140259,"ĠÙĬبدÙĪ":140260,"ÑĤелÑĮного":140261,"ĠвзглÑıд":140262,"ĠjednÄħ":140263,"ĠìĿĺ견":140264,"Ġà¸Ĥà¸ĵะà¸Ĺีà¹Ī":140265,"פ×Ļ×ĵ":140266,"ìĥģëĭ´":140267,"Ġmỡ":140268,"×Ķ×ŀ׾":140269,"×Ķ×ŀ׾צ×ķת":140270,"ĠÑģоÑģÑĤо":140271,"ĠÑģоÑģÑĤоиÑĤ":140272,"Ġави":140273,"Ġавиа":140274,"ĠLänder":140275,"تصÙĪÙĬر":140276,"×ŀ×ĵ×Ļ×Ķ":140277,"ìłĪì°¨":140278,"ãģ¨ãĤĬ":140279,"ãģ¨ãĤĬãģĤ":140280,"ãģ¨ãĤĬãģĤãģĪ":140281,"ãģ¨ãĤĬãģĤãģĪãģļ":140282,"ĠÑĢÑıд":140283,"ĠÑĢÑıдом":140284,"ĠNhất":140285,"ĠاÙĦÙĥاÙħÙĦ":140286,"×Ĺ׾׾":140287,"ĠGiấy":140288,"צ×ĺר":140289,"צ×ĺרף":140290,"Ġ׾×ij×ĺ׾":140291,"ĠимеÑĤÑĮ":140292,"ס×ŀ×ķ×ļ":140293,"Ġparticipação":140294,"íķľëĭ¤ë©´":140295,"ÙħÙĨتدÙĬ":140296,"ÙħÙĨتدÙĬات":140297,"ĠeÄŁlen":140298,"gänge":140299,"ربØŃ":140300,"ãĤ®ãĥ£":140301,"ĠاÙĦرÙĤÙħ":140302,"à¸ĭà¹īำ":140303,"ĠHóa":140304,"×ŀר×Ĺק":140305,"ØŃÙħاÙħ":140306,"بÙĪÙĥ":140307,"ĠArtÃŃculo":140308,"ãĥĦãĤ¢ãĥ¼":140309,"×Ķפ׼×Ķ":140310,"×Ĺ׾×ķף":140311,"ĠпеÑĢеÑħод":140312,"lenmiÅŁ":140313,"زراعة":140314,"Ġseñor":140315,"ãģ£ãģ¦ãģįãģ¦":140316,"إش":140317,"إشارة":140318,"ĠpodÃŃa":140319,"ĠÃľlke":140320,"нÑģкаÑı":140321,"Ġadapté":140322,"Ġdüzenlen":140323,"Ġdüzenlenen":140324,"ĠÑģÑĤала":140325,"ĠÙĬØŃتاج":140326,"Ġnier":140327,"Ġnieruch":140328,"Ġnieruchomo":140329,"ĠnieruchomoÅĽci":140330,"ãģĵãģ¨ãģĮãģĤãĤĭ":140331,"ยà¸Ńà¸Ķà¹Ģยีà¹Īยม":140332,"ĠÙħج":140333,"ĠÙħجاÙĨÙĬ":140334,"Ġзаб":140335,"Ġзабол":140336,"Ġзаболев":140337,"ĠзаболеваниÑı":140338,"ĠÅĽro":140339,"ĠÅĽrodk":140340,"ĠÅĽrodków":140341,"Ġ×Ķ׾×IJ×ķ×ŀ×Ļ":140342,"ĠdokÅĤad":140343,"ĠdokÅĤadnie":140344,"ãģŁãģıãģªãģĦ":140345,"ãģ¯ãģļãģ§ãģĻ":140346,"ãģ¨æĢĿãģ£ãģ¦ãģĦãģŁ":140347,"écran":140348,"ìĹħì²´":140349,"trzymaÅĤ":140350,"ÑģÑĤвеннÑĭй":140351,"ĠNotÃŃc":140352,"ĠNotÃŃcias":140353,"ÙħرÙĬ":140354,"ÙħرÙĬض":140355,"æ°Ĺè»":140356,"æ°Ĺ軽":140357,"æ°Ĺ軽ãģ«":140358,"ëĵ£":140359,"Ġ×ĵ×ķ×IJר":140360,"Ġ׾×ŀ׳":140361,"Ġ׾×ŀ׳×ķ×¢":140362,"ĠçalÄ±ÅŁÄ±yor":140363,"ĠÅŁidd":140364,"ĠÅŁiddet":140365,"ĠMặt":140366,"ĠateÅŁ":140367,"ĠполÑĥÑĩениÑı":140368,"à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩมืà¸Ń":140369,"ĠgrÃ¶ÃŁer":140370,"دائ":140371,"دائرة":140372,"Ġbulun":140373,"Ġbulunmaktadır":140374,"à¹Ģหร":140375,"à¹Ģหรีย":140376,"à¹Ģหรียà¸į":140377,"à¸Ļัà¸ģà¸Ĺà¹Īà¸Ńà¸ĩà¹Ģà¸Ĺีà¹Īยว":140378,"Ġalanında":140379,"ĠÑĥзна":140380,"ĠлеÑĩение":140381,"売ãĤĮ":140382,"Ġçevir":140383,"ĠdesteÄŁi":140384,"ĠheiÃŁt":140385,"âĸ²":140386,"ØŃØ·":140387,"à¸Ħำà¸ķà¸Ńà¸ļ":140388,"ãĤªãĥ³ãĥ©ãĤ¤ãĥ³":140389,"Ġ×ij×Ĺ×Ļ×Ļ×Ŀ":140390,"ãĥ¦ãĥĭ":140391,"Ġdüzenleme":140392,"ĠmodalitÃł":140393,"سرط":140394,"سرطاÙĨ":140395,"×ŀ׼×ķף":140396,"ĠданнÑĭй":140397,"ترت":140398,"ترتÙĬب":140399,"à¸ļาà¸ĩà¸Ħà¸Ļ":140400,"ĠÄIJá»ĭnh":140401,"มูล":140402,"มูลà¸Ħà¹Īา":140403,"ÙĨÙĤص":140404,"à¸ģารรัà¸ģษา":140405,"ĠÑĦон":140406,"ĠÑĦонд":140407,"ãĤĪãģĨãģ«ãģªãģ£ãģŁ":140408,"ÙħعاÙĦ":140409,"ÙħعاÙĦجة":140410,"ĠOsman":140411,"ĠOsmanlı":140412,"иÑĩеÑģком":140413,"à¸Ńยาà¸ģà¸Īะ":140414,"ãģķãģ¾ãģĸ":140415,"ãģķãģ¾ãģĸãģ¾":140416,"ãģķãģ¾ãģĸãģ¾ãģª":140417,"Ġת×ķ׼׾":140418,"עצ×ij":140419,"ĠاÙĦعسÙĥ":140420,"ĠاÙĦعسÙĥرÙĬ":140421,"Ġvéhic":140422,"Ġvéhicule":140423,"Ġ×Ļצ×Ĺק":140424,"ĠاÙĦÙĪØŃ":140425,"ĠاÙĦÙĪØŃÙĬد":140426,"ĠاÙĦعدÙĪ":140427,"ĠQuản":140428,"Ġê³µëıĻ":140429,"بدÙĦ":140430,"ĠÄijảng":140431,"Ġmá»ĩnh":140432,"Ġniezb":140433,"ĠniezbÄĻ":140434,"ĠniezbÄĻdn":140435,"Ġyayınlan":140436,"обÑīи":140437,"Ġgötür":140438,"צפ":140439,"צפ×ķ×Ļ":140440,"ĠÙĦÙĬبÙĬ":140441,"ĠÙĦÙĬبÙĬا":140442,"ØŃÙĪا":140443,"Ġдоб":140444,"ĠдобÑĢо":140445,"иÑĢÑĥем":140446,"ĠاÙĦØŃÙĥÙĪÙħÙĬØ©":140447,"mÃ¤ÃŁig":140448,"Ġedición":140449,"влекаÑĤелÑĮ":140450,"влекаÑĤелÑĮн":140451,"Ġ×ª×©×ľ×ķ×Ŀ":140452,"Ġ×Ķש×ķ׳×Ļ×Ŀ":140453,"มิà¸ĸุ":140454,"มิà¸ĸุà¸Ļ":140455,"มิà¸ĸุà¸Ļายà¸Ļ":140456,"é£Łãģ¹ãģ¦":140457,"ĠìĪĺì§ij":140458,"ס×ij×Ļ":140459,"ĠиÑİлÑı":140460,"Ġà¹Ħà¸Ķà¹īà¹ģà¸ģà¹Ī":140461,"׾×Ĺ×Ŀ":140462,"trä":140463,"trägt":140464,"ãģĿãĤĤãģĿãĤĤ":140465,"ÐĿÐķ":140466,"ĠвнÑĥÑĤ":140467,"ĠвнÑĥÑĤÑĢи":140468,"ãģ¨ä¸Ģç·Ĵãģ«":140469,"ãĤ«ãĥķãĤ§":140470,"Ġ×ij×Ĺ×ĵר":140471,"×Ĺ×ŀש":140472,"ãĤ¨ãĥį":140473,"ãĤ¨ãĥįãĥ«":140474,"ãĤ¨ãĥįãĥ«ãĤ®":140475,"ãĤ¨ãĥįãĥ«ãĤ®ãĥ¼":140476,"à¸Ĥà¸Ńà¸ĩà¸ķัวà¹Ģà¸Ńà¸ĩ":140477,"بÙĤاء":140478,"פס×Ļ׼":140479,"פס×Ļ׼×ķ׾×ķ×Ĵ":140480,"ãĥ¡ãĥĥ":140481,"ãĥ¡ãĥĥãĤ»":140482,"ãĥ¡ãĥĥãĤ»ãĥ¼ãĤ¸":140483,"ÙĦÙĤب":140484,"AÄŀ":140485,"שק×Ļ×¢":140486,"ÙĤساÙħ":140487,"×ĵ×ķ×Ĵ×ŀ×Ķ":140488,"æ·±ãģĦ":140489,"íĸĪëĬĶëį°":140490,"ĠrozwiÄħzanie":140491,"à¸Ļัà¹Īà¸Ļà¹Ģà¸Ńà¸ĩ":140492,"×Ļצ×ij":140493,"Ġtrông":140494,"à¹ĥà¸Ĭà¹īà¸ļริà¸ģาร":140495,"ĠاÙĦÙħÙĪسÙħ":140496,"ĠдеÑĤи":140497,"ãģĹãģĭãģªãģĦ":140498,"ס×Ļף":140499,"Ġréférence":140500,"à¹ģหà¹īà¸ĩ":140501,"ãĤĤãĤīãģ£ãģŁ":140502,"Ġ׾ר׼":140503,"Ġ׾ר׼×ķש":140504,"شعÙĪر":140505,"ĠÐijог":140506,"Ġlazım":140507,"Ġ×Ļש׳×Ŀ":140508,"ĠпаÑĢÑĤ":140509,"ĠпаÑĢÑĤнеÑĢ":140510,"ĠÑĥника":140511,"ĠÑĥникалÑĮн":140512,"Ġmatériel":140513,"×ŀרק":140514,"ĠphÆ°á»Ŀng":140515,"Ġзай":140516,"Ġзайм":140517,"ÙģÙĤد":140518,"UniversitÃł":140519,"×¢×¨×Ľ×Ļ×Ŀ":140520,"Ġbaño":140521,"ĠноÑı":140522,"ĠноÑıбÑĢÑı":140523,"à¸Ľà¹īาย":140524,"Ġtats":140525,"Ġtatsäch":140526,"Ġtatsächlich":140527,"ĠÑĤÑĢеÑĤÑĮ":140528,"Ñįм":140529,"ãĥĻãĥ¼ãĤ¹":140530,"Ġnhá»±a":140531,"ìĬ¤íģ¬":140532,"ĠعبداÙĦÙĦÙĩ":140533,"Ġת×ķר×Ķ":140534,"أشÙĬ":140535,"أشÙĬاء":140536,"ĠÙĦÙĦغا":140537,"ĠÙĦÙĦغاÙĬØ©":140538,"ÙħÙĪاÙĤ":140539,"ÙħÙĪاÙĤÙģ":140540,"ĠgÅĤówna":140541,"ĠartÄ±ÅŁ":140542,"Ġ×ŀק×ķ×ŀ×Ļ":140543,"ãĤ¯ãĥ©ãĥĸ":140544,"ĠسÙĪÙī":140545,"ĠìŬìĦ±":140546,"اسر":140547,"اسرائÙĬÙĦ":140548,"Ġ×ł×Ľ×ª×ij":140549,"ยà¹īà¸Ńà¸Ļ":140550,"Ġdeberá":140551,"Ġphẫu":140552,"ÑİÑīем":140553,"ĠÙĦدÙĬÙĨا":140554,"×ŀ×ĺ×Ķ":140555,"Ġ׳×ķ׾×ĵ":140556,"ĠвÑģÑĤÑĢеÑĩа":140557,"ãĤīãĤĮãģ¦ãģĦãģ¾ãģĻ":140558,"ĠcaÅĤej":140559,"ยึ":140560,"ยึà¸Ķ":140561,"поÑĤен":140562,"поÑĤенÑĨи":140563,"ĠлиÑĤ":140564,"ĠлиÑĤеÑĢ":140565,"ĠлиÑĤеÑĢаÑĤÑĥÑĢ":140566,"Ġкаждом":140567,"ĠíĮIJ":140568,"ĠíĮIJëĭ¨":140569,"à¸Īู":140570,"Ġpresença":140571,"ãģªãĤĵãģ§":140572,"ÙħÙĬاÙĩ":140573,"инÑĦоÑĢм":140574,"инÑĦоÑĢмаÑĨион":140575,"инÑĦоÑĢмаÑĨионн":140576,"ĠìŀIJìĹ°":140577,"ר׼ש":140578,"Ġödül":140579,"ç¶ļãģı":140580,"ĠпÑģ":140581,"ĠпÑģиÑħ":140582,"ĠпÑģиÑħолог":140583,"تذÙĥر":140584,"Ġìŀħìŀ¥":140585,"ลà¸Ķà¹Į":140586,"ìĦłê±°":140587,"ãģ£ãģ¦ãģĬãĤĬãģ¾ãģĻ":140588,"Ġ×Ļ×¢":140589,"Ġ×Ļעק×ij":140590,"ĠاÙĦطعاÙħ":140591,"ãĥĨãĤ¹ãĥĪ":140592,"ĠTuấn":140593,"Ġparticipación":140594,"×ŀ×ķ×ŀ×Ĺ×Ķ":140595,"×Ĵרס×Ķ":140596,"ĠاÙĦتÙĨÙģÙĬ":140597,"ĠاÙĦتÙĨÙģÙĬØ°ÙĬ":140598,"ĠбезопаÑģн":140599,"gef":140600,"gefähr":140601,"Ø´ÙĪر":140602,"ĠmyÅĽli":140603,"ÙĪاشÙĨ":140604,"ÙĪاشÙĨØ·ÙĨ":140605,"׳×ķסע":140606,"ÙĥÙĩ":140607,"ÙĥÙĩرب":140608,"ÙĥÙĩرباء":140609,"ĠmusiaÅĤ":140610,"ìĭ¸":140611,"ãĥĸãĥ©ãĥĥãĤ¯":140612,"Ġcréé":140613,"ÙĨÙĩار":140614,"owoÅĽÄĩ":140615,"ÙħØŃاÙĥÙħ":140616,"ĠwÅĤaÅĽ":140617,"ĠwÅĤaÅĽc":140618,"ĠwÅĤaÅĽciciel":140619,"ĠÙĬؤ":140620,"ĠÙĬؤدÙĬ":140621,"×ŀ×¢×ķ׳":140622,"×IJ×ij׾":140623,"خطأ":140624,"ĠÑħолод":140625,"×ĸ×ķ׾":140626,"ãģĵãĤĮãĤī":140627,"ãģĵãĤĮãĤīãģ®":140628,"Ġbásica":140629,"ฤà¸Ķ":140630,"ฤà¸Ķูà¸ģ":140631,"ฤà¸Ķูà¸ģา":140632,"ฤà¸Ķูà¸ģาล":140633,"èIJ½ãģ¡çĿĢ":140634,"ãģªãģĦãģĵãģ¨":140635,"صÙĪÙħ":140636,"ÙĨجØŃ":140637,"׳ק×ķ×ĵ":140638,"׳ק×ķ×ĵת":140639,"клаÑģÑģ":140640,"íķĺìĭľëĬĶ":140641,"ëĦĺ":140642,"Ġש×IJ×Ļ׳×ķ":140643,"ĠСейÑĩаÑģ":140644,"mayacaģı":140645,"Ġyapılır":140646,"ĠcategorÃŃa":140647,"عباد":140648,"ĠТеп":140649,"ĠТепеÑĢÑĮ":140650,"×Ķ×Ļס×ĺ×ķר×Ļ":140651,"hế":140652,"ãĤ³ãĥ¼ãĥī":140653,"Ġcabeça":140654,"جÙħا":140655,"جÙħاÙĩ":140656,"جÙħاÙĩÙĬر":140657,"ä½İãģĦ":140658,"ĠÑĤоваÑĢов":140659,"à¸Ĭาวà¸ļà¹īาà¸Ļ":140660,"ĠÑģÑĤанов":140661,"ĠÑģÑĤановиÑĤÑģÑı":140662,"ĠавÑĤомобилÑĮ":140663,"ĠÑģлÑĥÑĩай":140664,"à¸Ńัà¸ŀ":140665,"ĠGiriÅŁ":140666,"ĠìĿ¼ëĭ¨":140667,"ĠпÑĢоÑģ":140668,"ĠпÑĢоÑģмоÑĤÑĢ":140669,"ãģªãģıãģªãģ£ãģŁ":140670,"à¸¡à¸µà¸Ľà¸±à¸įหา":140671,"ïºİ":140672,"écoute":140673,"ĠÙħÙĪجÙĪد":140674,"ĠسرÙĬع":140675,"ĠÙĪÙĩÙĨا":140676,"ĠÙĪÙĩÙĨاÙĥ":140677,"à¸Ħุà¸ĵสม":140678,"à¸Ħุà¸ĵสมà¸ļัà¸ķิ":140679,"Ġìļ°ìĦł":140680,"à¸ŀระà¸ŀุà¸Ĺà¸ĺ":140681,"好ãģ¿":140682,"ظÙĦÙħ":140683,"ĠмакÑģ":140684,"ĠмакÑģималÑĮ":140685,"ĠмакÑģималÑĮно":140686,"ãĥªãĤ¢ãĥ«":140687,"à¹ģมà¹īวà¹Īา":140688,"ĠاÙĦØŃÙĪار":140689,"ãĥĹãĥ©ãĤ¹":140690,"ĠعÙĦاÙĤØ©":140691,"ĠíĸīëıĻ":140692,"Ġgönderil":140693,"Ġlãi":140694,"ĠsaÄŁlıkl":140695,"ĠsaÄŁlıklı":140696,"ĠÑĪаг":140697,"Ġ×ij×IJר×Ķ":140698,"prowadziÄĩ":140699,"ãģĦãģıãģ¤ãģĭ":140700,"ĠبتارÙĬØ®":140701,"Ġ×ij×IJ×ķת×Ķ":140702,"Ġmóc":140703,"ĠÐľÐ½Ðµ":140704,"ãĥĹãĥ¬ãĥ¼":140705,"×IJ×ĸר×Ĺ":140706,"åł´åIJĪãģ«ãģ¯":140707,"使ãģĪ":140708,"à¹Ģรืà¸Ńà¸Ļ":140709,"ĠÐŁÐµÑĤ":140710,"ĠÐŁÐµÑĤÑĢ":140711,"ãģ«åħ¥ãĤĭ":140712,"Ùħادة":140713,"à¹Ģà¸ĩืà¹Īà¸Ńà¸Ļ":140714,"à¹Ģà¸ĩืà¹Īà¸Ńà¸Ļà¹Ħà¸Ĥ":140715,"ĠÑģоÑģÑĤоÑıние":140716,"ônica":140717,"ĠÑĦев":140718,"ĠÑĦевÑĢа":140719,"ĠÑĦевÑĢалÑı":140720,"Ġ×ķ×ĸ":140721,"Ġ×ķ×ĸ×IJת":140722,"à¸Ħริ":140723,"à¸Ħริส":140724,"ĠÐķÑīе":140725,"ãģ£ãģ¦ãģĹãģ¾ãģĦãģ¾ãģĹãģŁ":140726,"ĠпÑĢавиÑĤелÑĮ":140727,"ĠпÑĢавиÑĤелÑĮÑģÑĤв":140728,"Ġtäglich":140729,"Ġëĭ¹ìĭľ":140730,"×ŀ×ķ×¢×ŀ×ĵ":140731,"ĠдвоÑĢ":140732,"æīķ":140733,"æīķãģĦ":140734,"ĠÑģÑĤанеÑĤ":140735,"ĠвоздейÑģÑĤв":140736,"ĠвоздейÑģÑĤви":140737,"Ġfête":140738,"à¹Ģสา":140739,"תק×ķ×ķ×Ķ":140740,"Ġuyar":140741,"Ġuyarı":140742,"à¸ģลัà¸ļà¹Ħà¸Ľ":140743,"ĠgiÆ°á»Ŀng":140744,"Ġва":140745,"ĠваÑĪи":140746,"ĠÄijáºŃu":140747,"ĠSpaÃŁ":140748,"ĠìķĦë§Ī":140749,"à¹Ħà¸Ķà¹īà¸ĩà¹Īาย":140750,"Ġ×Ķ×ŀ×ijקש":140751,"æĸ°ãģŁ":140752,"æĸ°ãģŁãģª":140753,"ılıyor":140754,"план":140755,"Ġ×Ķ×ijר×Ļ×IJ×ķת":140756,"ĠaÄŁrı":140757,"Ġsaygı":140758,"建ãģ¦":140759,"Ġnajwyż":140760,"Ġnajwyższ":140761,"سÙĬاسات":140762,"ãģĬå¾Ĺ":140763,"ĠاÙĦعÙĦÙĬ":140764,"ĠاÙĦعÙĦÙĬا":140765,"Ġcorazón":140766,"ì¹ĺë£Į":140767,"หัวà¸Ĥà¹īà¸Ń":140768,"ĠبØŃÙĬ":140769,"ĠبØŃÙĬØ«":140770,"звезд":140771,"بÙĪابة":140772,"ÐĽÐĺ":140773,"ÙĦازÙħ":140774,"Ġrozp":140775,"Ġrozpoc":140776,"ĠrozpoczÄĻ":140777,"触ãĤĮ":140778,"ĠاÙĦجÙħÙĩ":140779,"ĠاÙĦجÙħÙĩÙĪر":140780,"ĠspÄĻd":140781,"ĠspÄĻdz":140782,"วิà¸Ĺยาศาสà¸ķรà¹Į":140783,"иваеÑĤÑģÑı":140784,"Ġданной":140785,"Ġreprésente":140786,"ĠÄijá»ĭch":140787,"Ġ×¢×ŀ×ķק":140788,"à¸Ńัà¸Ļà¸ķร":140789,"à¸Ńัà¸Ļà¸ķราย":140790,"Ġestratég":140791,"Ġestratégia":140792,"padÅĤ":140793,"Ġвполн":140794,"Ġвполне":140795,"ĠпÑĢедоÑģÑĤавлен":140796,"×Ĺ׾×ķק":140797,"×Ĺ׾×ķקת":140798,"ãĤ¢ãĥĬ":140799,"ĠاÙĦغذ":140800,"ĠاÙĦغذائÙĬ":140801,"ĠÑĥзн":140802,"ĠÑĥзнаÑĤÑĮ":140803,"à¸ĭà¹īาย":140804,"å½ĵãģ¦":140805,"ØŃÙĬاء":140806,"Ġbásico":140807,"ק×ķ×ij×¢":140808,"ĠاÙĦÙħباراة":140809,"ĠاÙĦÙĩاتÙģ":140810,"Ġ׼׳×Ĵ×ĵ":140811,"à¸Ľà¸£à¸°à¸«à¸¢":140812,"à¸Ľà¸£à¸°à¸«à¸¢à¸±à¸Ķ":140813,"Ðļак":140814,"à¸Ĺีà¹Īà¸Ļà¹Īา":140815,"à¸Ĺีà¹Īà¸Ļà¹Īาสà¸Ļà¹ĥà¸Ī":140816,"ãģ¾ãģģ":140817,"ï½¢":140818,"Ñģкоп":140819,"Ġsonrasında":140820,"ĠurzÄħd":140821,"ĠurzÄħdzenia":140822,"׼×ķ×ķ׳":140823,"׼×ķ×ķ×ł×ª":140824,"Ġ׾×Ķת×ŀ×ķ×ĵ":140825,"Ġ׾×Ķת×ŀ×ķ×ĵ×ĵ":140826,"ĠÑģли":140827,"ĠÑģлиÑĪ":140828,"ĠÑģлиÑĪком":140829,"ĠÑģÑĤÑĥд":140830,"ĠÑģÑĤÑĥденÑĤ":140831,"Ġ×Ķ×ķ×ĵ":140832,"Ġ×Ķ×ķ×ĵ×¢×Ķ":140833,"ë¹Ħìļ©":140834,"à¸Ńยาà¸ģà¹ĥหà¹ī":140835,"Ġbá»ģ":140836,"ยุà¸Ĺà¸ĺ":140837,"ÐĺÐĿ":140838,"سائر":140839,"أصÙĪÙĦ":140840,"ĠاÙĦغرÙģ":140841,"ãģĵãģ¨ãĤĤãģĤãĤĬãģ¾ãģĻ":140842,"è¾¼ãģ¾ãĤĮ":140843,"ĠاÙĦسابع":140844,"Ġcủ":140845,"ãģĦãģŁãģłãģĦãģŁ":140846,"ì§ĵ":140847,"ìĤ¬ë¬´":140848,"powiedź":140849,"تÙģÙĥ":140850,"تÙģÙĥÙĬر":140851,"иÑĢовки":140852,"ĠíĨµíķ´ìĦľ":140853,"ãĤ¨ãĤ¹ãĥĨ":140854,"ĠдеÑıÑĤелÑĮноÑģÑĤÑĮ":140855,"ĠданнÑĭм":140856,"Ġ×¢×ķר":140857,"Ġ×¢×ķר׼×Ļ":140858,"×ķ×ĵעת":140859,"Ġhayatını":140860,"ĠbÄħd":140861,"ĠbÄħdź":140862,"obsÅĤug":140863,"à¹Ģà¸ŀียà¸ĩà¹ģà¸Ħà¹Ī":140864,"à¸ĭà¹Īา":140865,"è²łãģij":140866,"ĠÑģÑĤÑĢем":140867,"ĠÄijá»īnh":140868,"ĠÐłÑĥÑģ":140869,"ĠNữ":140870,"Ġ׾×Ķש×Ļ×Ĵ":140871,"Ġjednoc":140872,"Ġjednocze":140873,"ĠjednoczeÅĽnie":140874,"Ġ×Ķ×Ĵ×ij×ķ×Ķ":140875,"أخÙĦاÙĤ":140876,"ĠнаÑģел":140877,"ĠнаÑģелениÑı":140878,"ĠÙĬÙĨب":140879,"ĠÙĬÙĨبغÙĬ":140880,"ãģĮãģĭ":140881,"ãģĮãģĭãģĭ":140882,"×Ĵעת":140883,"ÐŀÐł":140884,"ĠналиÑĩии":140885,"Ġë§Īì§Ģ":140886,"Ġë§Īì§Ģë§ī":140887,"ĠíĸīìĤ¬":140888,"ĠtreÅĽci":140889,"Ġê°Ģì¹ĺ":140890,"ì¦ĺ":140891,"Ġаналог":140892,"×Ķצעת":140893,"влад":140894,"владе":140895,"ĠÑģделал":140896,"Ġ׳×Ĵ×Ļש":140897,"Ġ׳×Ĵ×Ļש×ķת":140898,"полнение":140899,"à¸Ĩà¹Īา":140900,"ĠDön":140901,"׼׾׼׾×Ķ":140902,"×ŀ×ĸ×Ĵ":140903,"ÙħÙģ":140904,"ÙħÙģÙĩ":140905,"ÙħÙģÙĩÙĪÙħ":140906,"×Ķ×ĵ":140907,"×Ķ×ĵפס":140908,"×Ķ×ĵפס×Ķ":140909,"ãģĻãģİãģ¦":140910,"ĠгÑĢ":140911,"ĠгÑĢн":140912,"×ŀ×ĺ×ķס":140913,"Ġ기ìĸµ":140914,"ï¾Ł":140915,"ĠpÅĤyn":140916,"ĠGründe":140917,"ĠBücher":140918,"ĠwedÅĤug":140919,"ãģ¾ãģłãģ¾ãģł":140920,"Ġ׳×Ķ×ĵר":140921,"ĠÙĬستطÙĬع":140922,"ĠHiá»ĩp":140923,"ãĤŃãĥ£ãĥ³ãĥļ":140924,"ãĤŃãĥ£ãĥ³ãĥļãĥ¼ãĥ³":140925,"Ġthá»ķ":140926,"Ġeuropéenne":140927,"à¸ļัà¸ĩ":140928,"à¸ļัà¸ĩà¸Ħัà¸ļ":140929,"ĠszczegóÅĤowo":140930,"׳שק":140931,"ãĥķãĥ©ãĥ³ãĤ¹":140932,"×ŀ×ķ×ŀ×Ĺ×Ļ":140933,"Ġcomún":140934,"Ġçarp":140935,"ØŃتÙĬا":140936,"ØŃتÙĬاج":140937,"ØŃتÙĬاجات":140938,"ëĭ´ëĭ¹":140939,"ä½ķ度":140940,"ä½ķ度ãĤĤ":140941,"×ĵ×ijק":140942,"ãģįãĤĮ":140943,"ãģįãĤĮãģĦ":140944,"Ġкам":140945,"ĠкамеÑĢ":140946,"ĠespecÃŃfico":140947,"Ġteléfono":140948,"à¸ķัà¹īà¸ĩà¸Ńยูà¹Ī":140949,"IÅŀ":140950,"ãģ©ãĤĵãģ©":140951,"ãģ©ãĤĵãģ©ãĤĵ":140952,"עצ×ŀ×IJ×Ļ":140953,"à¸Ķัà¸ĩà¸Ļีà¹ī":140954,"ĠÑĦоÑĢмиÑĢов":140955,"ĠÑĦоÑĢмиÑĢова":140956,"×ķ×ŀ×ij":140957,"Ġkullanımı":140958,"ÐľÐŀ":140959,"עש×Ļ":140960,"עש×Ļ×Ļ×Ķ":140961,"Ġönlem":140962,"à¹Ģà¸Ńà¹ĩ":140963,"à¹Ģà¸Ńà¹ĩม":140964,"×ŀשק×Ļ×¢":140965,"ר×Ļ×Ĺ":140966,"à¸Ĥัà¸Ķ":140967,"ĠíĻľ":140968,"ĠíĻľìļ©":140969,"à¸ĭะ":140970,"ãĤĪãģĨãģ«ãģªãĤĬãģ¾ãģĹãģŁ":140971,"ĠÑĢаÑģпÑĢ":140972,"ĠÑĢаÑģпÑĢоÑģÑĤ":140973,"ĠÑĢаÑģпÑĢоÑģÑĤÑĢан":140974,"ĠÑĢаÑģпÑĢоÑģÑĤÑĢанен":140975,"׼×Ļ×ķף":140976,"ÙĤبض":140977,"تصرÙĬØŃ":140978,"تصرÙĬØŃات":140979,"ĠоÑĢи":140980,"ĠоÑĢиг":140981,"ĠоÑĢигина":140982,"ĠоÑĢигинал":140983,"ĠاÙĦعاÙĦÙĬ":140984,"à¹ģหà¹Īà¸ĩà¸Ļีà¹ī":140985,"ãĥķãĤ¡ãĥ¼":140986,"ãģ¦ãģĦãģį":140987,"ãģ¦ãģĦãģįãģŁãģĦ":140988,"פתר":140989,"פתר×ķ׳×ķת":140990,"Ġ×ij×Ļ×Ĺ":140991,"Ġ×ij×Ļ×Ĺ×ĵ":140992,"Ġodby":140993,"ĠodbyÅĤ":140994,"ĠоÑĩеÑĢед":140995,"ĠtrÆ°Æ¡ng":140996,"ãĤŃãĥ³":140997,"×ŀ×ķפ":140998,"×ŀ×ķפע":140999,"ëĵľë¦½":141000,"ëĵľë¦½ëĭĪëĭ¤":141001,"à¸ŀืà¹īà¸Ļà¸IJาà¸Ļ":141002,"ìŀIJ격":141003,"ĠViá»ĩn":141004,"ĠDespués":141005,"Ġ×IJ׾×Ļ׳×ķ":141006,"Ġdurée":141007,"íĩ´":141008,"Ġmüzik":141009,"iếu":141010,"ĠÑĢазмеÑīен":141011,"ĠкÑĥд":141012,"ĠкÑĥда":141013,"غض":141014,"غضب":141015,"ĠTambém":141016,"à¸Īัà¸Ķสà¹Īà¸ĩ":141017,"à¸ģารà¹ģสà¸Ķà¸ĩ":141018,"onomÃŃa":141019,"Ġанг":141020,"Ġангли":141021,"Ġанглий":141022,"ĠанглийÑģк":141023,"Ġznal":141024,"Ġznalaz":141025,"ĠznalazÅĤ":141026,"תר×Ĵ":141027,"תר×Ĵ×ķ×Ŀ":141028,"ĠÑģнов":141029,"ĠÑģнова":141030,"ĠÑĩаÑģа":141031,"Ġcommunauté":141032,"ĠespecÃŃfica":141033,"ĠLá»ĭch":141034,"Ġlié":141035,"Ùģجر":141036,"à¹Ģà¸ģà¹Īà¸ĩ":141037,"عاÙĦ":141038,"عاÙĦج":141039,"Ø£ÙĨظ":141040,"Ø£ÙĨظÙħØ©":141041,"ESÄ°":141042,"ĠاÙĦØŃدÙĬد":141043,"à¸ŀระà¸Ńà¸ĩà¸Ħà¹Į":141044,"Ġפרשת":141045,"Ġдвиж":141046,"ĠдвижениÑı":141047,"ĠاÙĦجارÙĬ":141048,"à¸ĺาà¸Ļี":141049,"неÑģен":141050,"ĠاÙĦÙĨÙĩائÙĬ":141051,"ĠбеÑĢ":141052,"ĠбеÑĢем":141053,"ĠбеÑĢеменн":141054,"Ġdépartement":141055,"à¹Ģà¸Ĺีย":141056,"à¹Ģà¸Ĺียà¸ļ":141057,"ĠÐľÐ°ÑĢи":141058,"ĠнекоÑĤоÑĢÑĭÑħ":141059,"обеÑģп":141060,"обеÑģпеÑĩен":141061,"×Ĺ×ķ×ĸ":141062,"×Ĺ×ķ×ĸ×Ķ":141063,"ÙĨتج":141064,"à¸Īะà¹Ħà¸Ķà¹īรัà¸ļ":141065,"á»°":141066,"Ġéléments":141067,"عط":141068,"عطاء":141069,"Ġtắt":141070,"iá»ĩm":141071,"ÑİÑīиÑħÑģÑı":141072,"ãģĹãģ°":141073,"ãģĹãģ°ãĤīãģı":141074,"ĠпоможеÑĤ":141075,"à¸Ĥà¸ĵะà¸Ļีà¹ī":141076,"Ġעשר×ķת":141077,"éģķãģ£ãģ¦":141078,"ĠпÑĢог":141079,"ĠпÑĢогн":141080,"ĠпÑĢогноз":141081,"ĠtÅĤ":141082,"ĠtÅĤum":141083,"ĠtÅĤumacz":141084,"Tür":141085,"Türkiye":141086,"ãģįãģ£":141087,"ãģįãģ£ãģĭãģij":141088,"Ġ×Ķ׳×ķ׼":141089,"Ġ×Ķ׳×ķ׼×Ĺ×Ļ":141090,"ĠìĥĿìĤ°":141091,"ĠÑĦоÑĢмÑĭ":141092,"ç¾İãģĹãģĦ":141093,"à¸Ľà¸£à¸¶à¸ģ":141094,"à¸Ľà¸£à¸¶à¸ģษา":141095,"Ġlumière":141096,"ãĤªãĥ¼ãĥĹ":141097,"ãĤªãĥ¼ãĥĹãĥ³":141098,"à¸Ľà¸·à¸Ļ":141099,"วัสà¸Ķ":141100,"วัสà¸Ķุ":141101,"еÑĢÑĤв":141102,"ÙĥÙĦÙģ":141103,"ï½£":141104,"à¸ĺรรมà¸Ķา":141105,"׳×ĺר":141106,"ĠпÑĢедÑģÑĤавлÑıеÑĤ":141107,"Ġanálisis":141108,"Ġbãi":141109,"باÙĤÙĬ":141110,"à¸Ľà¸£à¸°à¹Ģà¸Ķ":141111,"à¸Ľà¸£à¸°à¹Ģà¸Ķà¹ĩà¸Ļ":141112,"ĠÑģлÑĥÑĩаÑı":141113,"ĠÑģлÑĥÑĩаÑıÑħ":141114,"ÐĽÐIJ":141115,"สัà¸ĩà¹Ģà¸ģ":141116,"สัà¸ĩà¹Ģà¸ģà¸ķ":141117,"Ġprzec":141118,"Ġprzecież":141119,"ÙħصÙĦ":141120,"ÙħصÙĦØŃØ©":141121,"ש×ķק×ķ׾×ĵ":141122,"ĠобоÑĢÑĥдованиÑı":141123,"ĠtrwaÅĤ":141124,"رÙĪÙħ":141125,"ìķĪëĤ´":141126,"ĠNghá»ĭ":141127,"خش":141128,"à¸ļาà¸Ħาร":141129,"à¸ļาà¸Ħารà¹Īา":141130,"ĠопÑĨион":141131,"ĠÑģозданиÑı":141132,"ãĤ³ãĤ¹ãĥĪ":141133,"Ġ×Ķ×¢×ľ×Ļ":141134,"Ġ×Ķ×¢×ľ×Ļ×ķף":141135,"läuft":141136,"ãĥĻãĤ¹ãĥĪ":141137,"Ġrê":141138,"Ġrêve":141139,"×IJ×ij×Ļ×ij":141140,"×Ļ×Ļ×ļ":141141,"ë¶Ļ":141142,"ãĤ¤ãĥ³ãĥī":141143,"ÅĤoży":141144,"ÅĤożyÄĩ":141145,"عائÙĦ":141146,"عائÙĦØ©":141147,"Ø£ÙĪر":141148,"Ø£ÙĪراÙĤ":141149,"à¸Ĺà¹īà¸Ńà¸ĩà¸ĸ":141150,"à¸Ĺà¹īà¸Ńà¸ĩà¸ĸิà¹Īà¸Ļ":141151,"Ġähn":141152,"Ġähnlich":141153,"ãĥŁãĥĭ":141154,"à¸ľà¸¹":141155,"à¸ľà¸¹à¹īà¸Ļ":141156,"à¸ľà¸¹à¹īà¸Ļำ":141157,"ĠмаÑĤеÑĢиалÑĭ":141158,"ĠкапиÑĤ":141159,"ĠкапиÑĤал":141160,"Ｆ":141161,"Ġseçil":141162,"Ġhứng":141163,"Ġintéressant":141164,"ãģ£ãģ¦ãģĦãģı":141165,"ĠeÄŁer":141166,"ëIJĺìĹĪìĬµëĭĪëĭ¤":141167,"ĠanlaÅŁma":141168,"ãģĶåĪ©çĶ¨":141169,"Ġ×ij×ĸ׼":141170,"Ġ×ij×ĸ׼×ķת":141171,"ëĿ¼ë©´":141172,"ĠÙĬÙĪس":141173,"ĠÙĬÙĪسÙģ":141174,"أسÙĦØŃØ©":141175,"ĠGefühl":141176,"ĠноÑĢмалÑĮн":141177,"ãĥĻãĥ³":141178,"ãģķãĤĮãĤĭãģĵãģ¨":141179,"ĠÐijеÑģ":141180,"ãģ¨ãģĦãģĪãģ°":141181,"ĠÙħÙĩÙħ":141182,"ĠÙħÙĩÙħØ©":141183,"ãģ§ãģĹãĤĩãģĨãģŃ":141184,"ĠêµŃëĤ´":141185,"à¹Ģมà¹ĩà¸Ķ":141186,"×ŀ×ijקר":141187,"ĠاÙĦدÙĨÙĬ":141188,"ĠاÙĦدÙĨÙĬا":141189,"à¸Ĭู":141190,"кÑĢÑĥÑĤ":141191,"Ġthoáng":141192,"Ġ׳×ĵר":141193,"Ġ׳×ĵרש":141194,"ĠÑĢаÑģÑģказал":141195,"ĠAuÃŁerdem":141196,"פ×IJר":141197,"פ×IJרק":141198,"Ġ×ŀש×Ĺק×Ļ×Ŀ":141199,"צר׼×Ļ×Ŀ":141200,"×ŀ×ĵ×ķ":141201,"×ŀ×ĵ×ķ×Ļק":141202,"èĭ¦ãģĹ":141203,"ĠÑģиг":141204,"ĠÑģигнал":141205,"ĠMá»įi":141206,"Ġtrữ":141207,"ĠnastÄĻp":141208,"ĠnastÄĻpnie":141209,"Ġì¶Ķì§Ħ":141210,"ĠاÙĦÙģÙĨد":141211,"ĠاÙĦÙģÙĨدÙĤ":141212,"koÅĦczyÅĤ":141213,"สีà¹Ī":141214,"ק×Ļ×ij":141215,"ק×Ļ×ij×ķ×¥":141216,"ĠнÑĥжнÑĭ":141217,"大åĪĩ":141218,"大åĪĩãģª":141219,"æıĽãģĪ":141220,"ת×ķס":141221,"ת×ķספת":141222,"ãģ£ãģ¦ãģĦãģªãģĦ":141223,"ĠмÑı":141224,"ĠмÑıг":141225,"ĠмÑıгк":141226,"Ġjakie":141227,"ĠjakieÅĽ":141228,"à¸ķำà¸ļ":141229,"à¸ķำà¸ļล":141230,"ĠìŀĪì§Ģ":141231,"×ij×ĺ×IJ":141232,"ĠоÑĤлиÑĩно":141233,"ÙĤÙIJ":141234,"ĠавÑĤомоб":141235,"ĠавÑĤомоби":141236,"ĠавÑĤомобилÑı":141237,"دÙĬÙħÙĤراطÙĬ":141238,"ĠاÙĦÙĪا":141239,"ĠاÙĦÙĪاØŃد":141240,"ĠسÙĪرÙĬØ©":141241,"أغÙĦ":141242,"أغÙĦب":141243,"ĠÑįкÑĢан":141244,"ãĥĹãĥ©ãĤ¤":141245,"ĠjesteÅĽ":141246,"ãĥIJãĥª":141247,"Ġ×Ķ×IJ×ķ×ķ×Ļר":141248,"ائÙĥ":141249,"à¸Ńยà¹Īาà¸ĩยิà¹Īà¸ĩ":141250,"ÑĢекÑĤ":141251,"Ġumo":141252,"Ġumoż":141253,"Ġumożli":141254,"Ġumożliw":141255,"Ġumożliwia":141256,"Ġnächste":141257,"ĠìŀĪì§Ģë§Į":141258,"ĠпÑĢедн":141259,"ĠпÑĢедназ":141260,"ĠпÑĢедназнаÑĩен":141261,"Ġmaçı":141262,"Ġpomi":141263,"ĠpomiÄĻd":141264,"ĠpomiÄĻdzy":141265,"ĠاÙĦÙĦÙĤاء":141266,"à¹Ģà¸Ķà¸Ńะ":141267,"ĠновоÑģÑĤи":141268,"×ŀ×Ĺ׾×Ķ":141269,"رÙĬاضÙĬ":141270,"à¸Ķà¸Ļ":141271,"à¸Ķà¸Ļà¸ķรี":141272,"بصر":141273,"ìĬ¤íĥĢ":141274,"scripción":141275,"Ġnapisa":141276,"ĠnapisaÅĤ":141277,"Ġ׳ש×ŀ×¢":141278,"ĠاÙĦÙħØŃÙĦÙĬ":141279,"Ġhiá»ĥn":141280,"×IJ×Ĺ":141281,"×IJ×Ĺר×IJ×Ļ":141282,"ĠгÑĢаниÑĨ":141283,"æīĭç¶ļãģį":141284,"Ùĥسب":141285,"Ġà¹ģà¸ķà¹Īà¸ĸà¹īา":141286,"à¸Ķาวà¸Ļà¹Į":141287,"à¸Ķาวà¸Ļà¹Įà¹Ĥหลà¸Ķ":141288,"ãĤĭãģĵãģ¨ãģĮãģ§ãģįãģ¾ãģĻ":141289,"åŁºæľ¬çļĦãģ«":141290,"ÙĪÙĦاد":141291,"räume":141292,"دÙģاع":141293,"×Ļצע":141294,"ĠOczy":141295,"ĠOczywiÅĽcie":141296,"ĠÅģ":141297,"ĠÅģa":141298,"اÙĦÙĬاب":141299,"اÙĦÙĬاباÙĨ":141300,"áºłI":141301,"ĠBirliÄŁi":141302,"×Ķ×ķצ":141303,"×Ķ×ķצ×IJת":141304,"ĠÄijua":141305,"Ġê·¸ëŁ¬ëĭĪê¹Į":141306,"Ġréalité":141307,"عÙĦاÙĤات":141308,"Jeste":141309,"JesteÅĽ":141310,"Ġмнож":141311,"ĠмножеÑģÑĤво":141312,"Ｋ":141313,"ãĥĹãĥŃãĤ¸ãĤ§":141314,"ãĥĹãĥŃãĤ¸ãĤ§ãĤ¯ãĥĪ":141315,"ĠÑĦл":141316,"ظÙĨ":141317,"×Ĵ׾×Ĵ׾":141318,"ĠmÅĤodzie":141319,"ĠmÅĤodzież":141320,"à¸Ļà¹īำà¸ķา":141321,"à¸Ļà¹īำà¸ķาล":141322,"ÐĽÐķ":141323,"×ij×ķ×ĺ":141324,"Ġ׾×Ķ×Ĵ×Ļ×ĵ":141325,"ãģĵãģ¨ãĤĤãģĤãĤĭ":141326,"زاد":141327,"×ŀ×Ļ×ĵ×¢":141328,"ĠgÅĤównie":141329,"ãĥıãĤ¦":141330,"ãĥıãĤ¦ãĤ¹":141331,"бел":141332,"Ġétape":141333,"ðŁĺĢ":141334,"ĠмоделÑĮ":141335,"aģını":141336,"ש×Ĺק":141337,"ש×Ĺקף":141338,"Ġniño":141339,"à¸Ĭà¹īาà¸ĩ":141340,"à¹Ģลีย":141341,"ĠÑĦоÑĢме":141342,"ĠاÙĦشرÙĬÙģ":141343,"ĠÑĥдаÑĢ":141344,"arriv":141345,"arrivée":141346,"ĠmiesiÄĻ":141347,"ĠmiesiÄĻcy":141348,"ØŃرÙĥ":141349,"ØŃرÙĥات":141350,"ĠDiá»ħn":141351,"ÐĿЫ":141352,"ãģ¾ãģ£ãģŁãģı":141353,"Ġ×Ļר×ķק":141354,"еÑģÑĤеÑģÑĤв":141355,"еÑģÑĤеÑģÑĤвенн":141356,"Ġê·¸ëŁ¼":141357,"ĠاÙĦÙħتÙĪ":141358,"ĠاÙĦÙħتÙĪسط":141359,"Ġbénéfic":141360,"Ġbénéficie":141361,"Ġwybra":141362,"ĠwybraÄĩ":141363,"ĠاÙĦزÙħÙĨ":141364,"ĠпÑĢинÑı":141365,"ĠпÑĢинÑıл":141366,"ÙģرØŃ":141367,"Ġksz":141368,"ĠksztaÅĤ":141369,"ĠksztaÅĤt":141370,"ק׾×ĺ":141371,"×ij×ĵ×Ļקת":141372,"Ġgiấ":141373,"Ġgiấc":141374,"ĠproprietÃł":141375,"деÑĢжан":141376,"ĠKöln":141377,"ĠGüzel":141378,"×Ļפ×ķ×Ļ":141379,"ĠCuá»Ļc":141380,"ÑįÑĤаж":141381,"ترÙĥÙĬ":141382,"ترÙĥÙĬز":141383,"ложений":141384,"ĠпÑĥ":141385,"ĠпÑĥÑĤи":141386,"اختÙĦاÙģ":141387,"åĩºãģ¦ãģıãĤĭ":141388,"à¸ļุà¸ģ":141389,"âĿ¤":141390,"ÑĦан":141391,"פש×ĺ":141392,"à¸ļัà¸Ļà¹Ģà¸Ĺ":141393,"à¸ļัà¸Ļà¹Ģà¸Ĺิà¸ĩ":141394,"ĠاÙĦساد":141395,"ĠاÙĦسادس":141396,"ĠاÙĦÙĤÙĪÙħ":141397,"ĠاÙĦÙĤÙĪÙħÙĬ":141398,"Ġyönetici":141399,"ÙĩÙĪات":141400,"ÙĩÙĪاتÙģ":141401,"Ġresponsável":141402,"ĠподдеÑĢжива":141403,"ĠاÙĦسÙĦØ·":141404,"ĠاÙĦسÙĦطات":141405,"ãģĹãģ¦ãģĬãģı":141406,"ãĥļãĥĥãĥĪ":141407,"à¸Ľà¸¸à¹Īม":141408,"ĠoglÄħda":141409,"ÙĨاÙĤ":141410,"ÙĨاÙĤØ´":141411,"à¸Ħà¸Ńà¸Ļà¹Ĥà¸Ķ":141412,"ĠMüsl":141413,"ĠMüslü":141414,"ĠMüslüman":141415,"ĠMoż":141416,"ĠMożna":141417,"Ġnumérique":141418,"Ġvá»ı":141419,"ĠسÙĬتÙħ":141420,"ĠyerleÅŁ":141421,"монÑĤаж":141422,"Ġgoût":141423,"ãģ¦ãģĬãĤĬãģ¾ãģĻ":141424,"ĠKhánh":141425,"Ġедин":141426,"ĠединÑģÑĤв":141427,"اÙĨØ®Ùģ":141428,"اÙĨØ®Ùģاض":141429,"ìĭľíĹĺ":141430,"Ġlặng":141431,"ĠÑĢолÑĮ":141432,"à¸ķัวà¹ģà¸Ĺà¸Ļ":141433,"à¸Ħà¹Īาà¹ĥà¸Ĭà¹ī":141434,"à¸Ħà¹Īาà¹ĥà¸Ĭà¹īà¸Īà¹Īาย":141435,"Ġverfüg":141436,"Ġverfügbar":141437,"ìĻĶëĭ¤":141438,"ãģĦãģļ":141439,"ãģĦãģļãĤĮ":141440,"ĠиÑģÑģледованиÑı":141441,"меÑīа":141442,"×Ķ×Ĺ":141443,"×Ķ×Ĺ×ĸר":141444,"à¹ģà¸Łà¸Ĭัà¹Īà¸Ļ":141445,"تصرÙģ":141446,"إرÙĩاب":141447,"ĠexercÃŃcio":141448,"Ġélev":141449,"Ġélevé":141450,"สัà¸įà¸įาà¸ĵ":141451,"ÃĸZ":141452,"ãĥĹãĥŃãĤ°":141453,"ãĥĹãĥŃãĤ°ãĥ©":141454,"ãĥĹãĥŃãĤ°ãĥ©ãĥł":141455,"ĠwewnÄĻtrzn":141456,"Ġhenüz":141457,"é£Ľãģ³":141458,"à¹Ģà¸Ķà¸Ńรà¹Į":141459,"ÑģÑĥж":141460,"ÑģÑĥжден":141461,"شعÙĪب":141462,"ãģ²ãģ¨ãĤĬ":141463,"ĠwyÅĤÄħ":141464,"ĠwyÅĤÄħcznie":141465,"ĠплоÑħо":141466,"ÐĶÐķ":141467,"Ầ":141468,"ÙģعاÙĦÙĬ":141469,"ÙģعاÙĦÙĬات":141470,"ĠاÙĦعشر":141471,"ÑģÑĤÑĥпил":141472,"Ġyarg":141473,"Ġyargı":141474,"нÑİÑİ":141475,"×ķ×IJ×ij":141476,"Ġuç":141477,"Ġuçak":141478,"ë²½":141479,"تÙĪÙĤÙĬ":141480,"تÙĪÙĤÙĬع":141481,"Ġì¤ijìĭ¬":141482,"׳×Ļ×ķ×ķ×ĺ":141483,"Ø£ÙĥÙĦ":141484,"ç½®ãģĦãģ¦":141485,"éłĤãģį":141486,"Ġ×Ķת×ij":141487,"Ġ×Ķת×ij×Ļ×¢×Ķ":141488,"Ġdürfen":141489,"ÙħÙĤاÙĦ":141490,"ÙħÙĤاÙĦات":141491,"ĠزÙħÙĨ":141492,"à¸ŀฤศ":141493,"à¸ŀฤศà¸Ī":141494,"à¸ŀฤศà¸Īิà¸ģ":141495,"à¸ŀฤศà¸Īิà¸ģายà¸Ļ":141496,"ĠнеÑģколÑĮ":141497,"ĠнеÑģколÑĮки":141498,"ĠнеÑģколÑĮкиÑħ":141499,"Ġcriança":141500,"มิà¸ķร":141501,"×ŀ׼×Ļר×ķת":141502,"à¸ģารà¸ļริหาร":141503,"Ġtélécharg":141504,"Ġ×IJ×ķ×Ķ×ijת":141505,"ĠBüro":141506,"ä½ľãģ£ãģŁ":141507,"ĠKiÅŁi":141508,"ç¾İåij³ãģĹ":141509,"à¹Ģลยà¸Ħà¹Īะ":141510,"à¸ŀà¸ļà¸ģัà¸ļ":141511,"à¸Īà¹īา":141512,"Ġçer":141513,"Ġçerç":141514,"Ġçerçeve":141515,"ãĤĴä½ľãģ£ãģ¦":141516,"ĠпеÑĢвÑĥÑİ":141517,"×ŀצר×Ļ×Ŀ":141518,"×IJ׾×ķ×Ķ":141519,"×IJ׾×ķ×Ķ×Ļ×Ŀ":141520,"Ġagré":141521,"Ġagréable":141522,"Ġayır":141523,"Ä°LÄ°":141524,"ãĤ¥":141525,"ĠíĺĦ":141526,"ĠíĺĦìĭ¤":141527,"ثاÙĦØ«":141528,"ת×ĸ":141529,"ת×ĸ×ķ׳×Ķ":141530,"ãģ¨ãģĦãģ£ãģ¦":141531,"ãģ¨ãģĦãģ£ãģ¦ãĤĤ":141532,"ĠابÙĪ":141533,"ĠÑģобак":141534,"é£Łãģ¹ãģŁ":141535,"Ġданном":141536,"à¹Ģลิ":141537,"à¹Ģลิศ":141538,"Ġíļ":141539,"Ġíļ¨":141540,"Ġíļ¨ê³¼":141541,"ãĤĤãĤīãģĪãĤĭ":141542,"׳צ׾":141543,"ÑĦик":141544,"ÑĦикÑģ":141545,"ĠjesteÅĽmy":141546,"ת×Ĺ×ķש×Ķ":141547,"à¹Ħมà¹Īà¸Ħวร":141548,"ĠØŃسÙĬÙĨ":141549,"à¸ģารลà¸ĩà¸Ĺุà¸Ļ":141550,"ë´¤":141551,"ĠÐĺменно":141552,"à¸ļà¸Ńรà¹Į":141553,"à¸ļà¸Ńรà¹Įà¸Ķ":141554,"ĠCảnh":141555,"ìĦľë¹ĦìĬ¤":141556,"Ġполов":141557,"Ġполовин":141558,"ĠзамеÑĩа":141559,"ãģĦãĤįãĤĵãģª":141560,"Ġ×ij×Ļק":141561,"Ġ×ij×Ļקש":141562,"лÑĥÑĪ":141563,"ãĤĴè¿İ":141564,"ãĤĴè¿İãģĪ":141565,"جرÙĬÙħØ©":141566,"Ġtây":141567,"ĠاÙĦÙĨÙĪ":141568,"ĠاÙĦÙĨÙĪÙĪÙĬ":141569,"ÃĤN":141570,"ì¿ł":141571,"หà¸Ļาว":141572,"Ġ×ij×Ĺש×ij×ķף":141573,"زار":141574,"à¸Ķาร":141575,"à¸Ķารา":141576,"ĠÅĽl":141577,"ĠÅĽlub":141578,"มีà¸Ħวามสุà¸Ĥ":141579,"Ġnhu":141580,"ĠnhuáºŃn":141581,"ÙħØŃطة":141582,"à¹Ģสืà¹īà¸Ńà¸ľà¹īา":141583,"ĠТолÑĮко":141584,"ĠÙĥس":141585,"ĠÙĥسارة":141586,"ÙħشرÙĪع":141587,"niÄĻcia":141588,"×¢×Ľ×©×Ļ×ķ":141589,"تÙĦÙģ":141590,"تÙĦÙģزÙĬ":141591,"تÙĦÙģزÙĬÙĪÙĨ":141592,"ĠlÆ°á»Ľi":141593,"ĠÐľÐ¾ÑģквÑĭ":141594,"Ġréserve":141595,"ĠanlaÅŁ":141596,"ĠanlaÅŁÄ±l":141597,"ĠedeceÄŁi":141598,"รà¸Ńà¸ĩà¹Ģà¸Ĺà¹īา":141599,"Ġبط":141600,"ĠبطرÙĬ":141601,"ĠبطرÙĬÙĤØ©":141602,"ãģ¦ãģĹãģ¾ãģ£ãģ¦":141603,"ãĤĤãĤīãģ£ãģ¦":141604,"برج":141605,"æ±ļ":141606,"æ±ļãĤĮ":141607,"Ġchoc":141608,"Ġchocia":141609,"Ġchociaż":141610,"Ġzobac":141611,"ĠzobaczyÄĩ":141612,"пÑĢÑı":141613,"пÑĢÑıжен":141614,"ĠÑĨиÑĦ":141615,"ĠÑĨиÑĦÑĢ":141616,"Ġмам":141617,"ĠвзÑıÑĤÑĮ":141618,"Ġchạm":141619,"جسÙħ":141620,"ØŃÙħاس":141621,"à¹Ģลà¹Īม":141622,"à¸ŀิษ":141623,"×Ķפ׼×ķ":141624,"à¸Ĭà¹Īà¸Ńà¸ĩà¸Ĺาà¸ĩ":141625,"Ġвек":141626,"Ġвека":141627,"Æ¡Ìģ":141628,"Æ¡Ìģi":141629,"ĠTiá»ģn":141630,"Ġtrầm":141631,"мÑĭÑĪ":141632,"мÑĭÑĪл":141633,"ĠÑĤÑĥ":141634,"ĠÑĤÑĥÑĢиÑģÑĤ":141635,"Ġchc":141636,"ĠchcÄħ":141637,"Ġавг":141638,"ĠавгÑĥÑģÑĤ":141639,"ĠавгÑĥÑģÑĤа":141640,"ס×IJ×ķת":141641,"Ġר×Ĵ׾":141642,"à¸ľà¸¥à¸ģระà¸Ĺ":141643,"à¸ľà¸¥à¸ģระà¸Ĺà¸ļ":141644,"å¤īãĤıãĤĭ":141645,"Ġ×Ķ×IJ×Ĺר×ķ׳×Ļ×Ŀ":141646,"سÙģÙĬر":141647,"ĠÑĩаÑīе":141648,"ãģĦãĤī":141649,"ãģĦãĤīãģ£":141650,"ãģĦãĤīãģ£ãģĹãĤĥ":141651,"×ķ×ŀ׳×Ļ×Ŀ":141652,"Ġarttır":141653,"ĠChá»ĭ":141654,"Ġì¡°ì§ģ":141655,"ĠÑĥÑģпеÑħ":141656,"Ġ×¢×ķס":141657,"Ġ×¢×ķסק":141658,"ĠìĥĿëªħ":141659,"ÑĨиÑĤ":141660,"Ġregión":141661,"ÐŀÐĿ":141662,"ĠdoÄŁum":141663,"ĠyaÅŁad":141664,"ĠyaÅŁadıģı":141665,"à¸Ĺà¸Ķลà¸Ńà¸ĩ":141666,"Ġgözü":141667,"ש×Ļר×Ķ":141668,"дÑĥмал":141669,"Ġdaģı":141670,"Ġdaģıt":141671,"à¸Ĺีมà¸ĩาà¸Ļ":141672,"Ġtiá»ģm":141673,"ĠاÙĦÙĥبر":141674,"ĠاÙĦÙĥبرÙī":141675,"ì¹Ń":141676,"ĠGünc":141677,"ĠGüncelle":141678,"ĠGüncelleme":141679,"ê¹Ĭ":141680,"ĠобоÑĢÑĥдование":141681,"ĠÑĢеÑĪа":141682,"Ụ":141683,"ĠпиÑĤ":141684,"ĠпиÑĤаниÑı":141685,"à¹Ģรียà¸ļ":141686,"×Ľ×ª×Ļ×ij×Ķ":141687,"Ġпон":141688,"ĠпонÑĢав":141689,"ĠпонÑĢави":141690,"Ġ×Ķ×ķ׾×ĵ":141691,"Ġ×Ķ×ķ׾×ĵת":141692,"Ġê²ģ":141693,"Ġê²ģëĭĪëĭ¤":141694,"ĠпеÑĢвой":141695,"ãĥ©ãĤ¤ãĥķ":141696,"ĠÅŁiir":141697,"krÄĻ":141698,"krÄĻc":141699,"Ġthiá»ĥu":141700,"à¹Ģลยà¸Ĺี":141701,"à¹Ģลยà¸Ĺีà¹Ģà¸Ķียว":141702,"×ĺ×¢×ł×ķת":141703,"ائÙĩÙħ":141704,"Ġ×IJס×ķר":141705,"ĠплаÑĤеж":141706,"تردد":141707,"Ġmożliwe":141708,"ĠkhỼ":141709,"ĠkhỼp":141710,"تÙģاعÙĦ":141711,"ĠÑĪколÑĮ":141712,"ĠÑĪколÑĮн":141713,"ĠÙĤصة":141714,"Ġmétier":141715,"nÄĻÅĤa":141716,"หลà¹Īà¸Ń":141717,"Ġủng":141718,"Ġprzegl":141719,"ĠprzeglÄħd":141720,"ĠاÙĦÙħتعÙĦ":141721,"ĠاÙĦÙħتعÙĦÙĤØ©":141722,"ĠÑģÑĭн":141723,"Ġволн":141724,"ãĥĩãĥ¼ãĥĪ":141725,"ĠÐŃÑĤи":141726,"ĠкÑĢоме":141727,"à¸Ħารà¹Į":141728,"׳ק×ķ×ĵ×Ķ":141729,"Ġ׾ש×ŀ×ķ×¢":141730,"Ġ×ĸ×ķ׼ר":141731,"Ｇ":141732,"ÙĬÙİا":141733,"Ġgiá»ıi":141734,"åĥįãģı":141735,"ĠÑģни":141736,"ĠÑģнижен":141737,"à¹ģà¸Ķà¸Ķ":141738,"รุà¸Ļ":141739,"รุà¸Ļà¹ģรà¸ĩ":141740,"Ġhiá»ĩp":141741,"ografÃŃa":141742,"à¹Ģà¸Īà¸Ńรà¹Į":141743,"Ġдвиг":141744,"ĠдвигаÑĤ":141745,"ĠдвигаÑĤел":141746,"Ġüy":141747,"Ġüyeler":141748,"Ġüyeleri":141749,"ĠбÑĥк":141750,"ĠбÑĥкв":141751,"ãĤĤå¤ļãģı":141752,"Ġthiá»ĩt":141753,"ĠPaÃŃs":141754,"ĠطبÙĬعÙĬ":141755,"à¹ģà¸Īà¸ģ":141756,"ĠاÙĦصØŃÙĬØŃ":141757,"Ġappré":141758,"Ġappréci":141759,"Ġdecisión":141760,"Ġë°ĺëĵľ":141761,"Ġë°ĺëĵľìĭľ":141762,"ĠÑĤебе":141763,"ãĤ·ãĥ¼ãĤº":141764,"ãĤ·ãĥ¼ãĤºãĥ³":141765,"ĠдалÑĮн":141766,"ĠìĬ¤":141767,"ĠìĬ¤ìĬ¤":141768,"ĠìĬ¤ìĬ¤ë¡ľ":141769,"ĠThá»ĥ":141770,"ĠkarÅŁ":141771,"ĠkarÅŁÄ±s":141772,"ĠkarÅŁÄ±sında":141773,"ĠKön":141774,"ĠKönig":141775,"ивание":141776,"×ij×ķצע":141777,"глаÑģ":141778,"Ġtwó":141779,"Ġtwórc":141780,"à¸Ľà¸ģà¸Ħร":141781,"à¸Ľà¸ģà¸Ħรà¸Ńà¸ĩ":141782,"ĠGÅĤ":141783,"ĠGÅĤówn":141784,"ĠUnterstüt":141785,"ĠUnterstützung":141786,"ĠдÑĥÑħ":141787,"ĠдÑĥÑħов":141788,"Ø£ÙħاÙĨ":141789,"×Ĺשש":141790,"تظ":141791,"تظاÙĩر":141792,"ĠлÑİбом":141793,"à¸ķาร":141794,"à¸ķาราà¸ĩ":141795,"Ġkról":141796,"Ø£ØŃدث":141797,"ì¡Įëĭ¤":141798,"ÐļÑĥÑĢÑģ":141799,"ãĥĥãĥĦ":141800,"×ŀק×ķ×ij׾":141801,"ĠÑģимвол":141802,"Ġdésorm":141803,"Ġdésormais":141804,"wüns":141805,"wünsche":141806,"Ñĥни":141807,"ÑĥниÑĨип":141808,"ÑĥниÑĨипалÑĮн":141809,"หลัà¸ģสูà¸ķร":141810,"ÙĨتشر":141811,"Ġал":141812,"Ġалк":141813,"Ġалког":141814,"Ġалкогол":141815,"ĠÑĥÑĩиÑĤÑĭва":141816,"à¸ģำà¸ģัà¸ļ":141817,"Ġ×ľ×¤×¢×ķ׾":141818,"ĠìĹ°ê²°":141819,"sÄħd":141820,"ĠاÙĦØ£ÙĬ":141821,"ĠاÙĦØ£ÙĬاÙħ":141822,"غÙĬاب":141823,"ĠнаÑĢ":141824,"ĠнаÑĢко":141825,"×ŀ×ķ×ĵ×¢":141826,"ĠÑģеÑĢии":141827,"пиÑģÑĭва":141828,"สิว":141829,"ç¶ļãģĦãģ¦":141830,"çĶ³ãģĹè¾¼ãģ¿":141831,"Ġ׾×Ĵר":141832,"Ġ׾×Ĵר×ķ×Ŀ":141833,"Ġдем":141834,"Ġдемо":141835,"Ġë³´ëĤ´":141836,"تÙĩدÙĬد":141837,"ĠÙħØ´ÙĬرا":141838,"Ġduy":141839,"Ġduyá»ĩt":141840,"ĠwiÄĻksze":141841,"ÙħعاÙĬ":141842,"ÙħعاÙĬÙĬر":141843,"ĠGda":141844,"ĠGdaÅĦsk":141845,"Ġrah":141846,"Ġrahats":141847,"Ġrahatsız":141848,"ר×ķצ×Ķ":141849,"lös":141850,"lösung":141851,"ĠТаким":141852,"ÑĪед":141853,"ÑĪедÑĪ":141854,"عزÙĦ":141855,"Ġרש×Ļ×ŀת":141856,"Ġ׾×Ķ×Ļ׼":141857,"Ġ׾×Ķ×Ļ×Ľ×ł×¡":141858,"ĠпÑĥÑĤ":141859,"ĠпÑĥÑĤеÑĪ":141860,"ĠпÑĥÑĤеÑĪеÑģÑĤв":141861,"ĠnotÃŃcia":141862,"ĠalÄ±ÅŁ":141863,"ĠalÄ±ÅŁver":141864,"ĠalÄ±ÅŁveriÅŁ":141865,"ĠwÅĤos":141866,"ĠwÅĤosów":141867,"Ġبغ":141868,"Ġبغداد":141869,"Ġveröffent":141870,"Ġveröffentlicht":141871,"ĠKhá":141872,"Ġtán":141873,"ëIJĺ기":141874,"Ġ방문":141875,"ÙģÙĬÙĦ":141876,"à¹Ģà¸ģิà¸Ķà¸Īาà¸ģ":141877,"åı¯æĦĽ":141878,"åı¯æĦĽãģĦ":141879,"à¸ĸุà¸ĩ":141880,"ĠzewnÄĻtrzn":141881,"à¸łà¸²à¸©à¸²à¸Ńัà¸ĩà¸ģฤษ":141882,"Ġmáxima":141883,"Ġulus":141884,"Ġuluslararası":141885,"Ġ׳×Ķ׳":141886,"à¸Ĥà¹Īาวสาร":141887,"ĠìĿĺìĤ¬":141888,"à¹Ģหลืà¸Ńà¸ĩ":141889,"ĠدÙĤ":141890,"ĠدÙĤائÙĤ":141891,"สืà¹Īà¸Ńสาร":141892,"먼":141893,"ĠÑģоÑģÑĤоÑıнии":141894,"สมาà¸Ħม":141895,"á»Ĥ":141896,"ĠÐľÐ¾Ñģков":141897,"ĠÐľÐ¾ÑģковÑģк":141898,"×ŀס×ķ×Ĵ׾":141899,"ãģĭãģĭãĤĬ":141900,"ĠTruyá»ģn":141901,"à¹ģà¸Ĥà¹ĩà¸ĩà¹ģรà¸ĩ":141902,"×ŀ×Ĺ×ĸ×Ļק":141903,"à¹Ĥà¸ģà¹ī":141904,"ÙĬسر":141905,"ìĶ©":141906,"×IJ×ķק":141907,"×IJ×ķק×ĺ":141908,"×IJ×ķק×ĺ×ķ×ijר":141909,"Ġproximité":141910,"ÙħÙĨÙĩج":141911,"ĠاÙĦجز":141912,"ĠاÙĦجزائ":141913,"ĠاÙĦجزائرÙĬ":141914,"ĠÄIJiá»ĥm":141915,"Ġденеж":141916,"Ġденежн":141917,"ÙģØŃص":141918,"Ùģئ":141919,"ĠÐijÑĥд":141920,"×Ĵ×Ļ×ĵ×ķ׾":141921,"ĠÐĴедÑĮ":141922,"عÙĦاÙħØ©":141923,"Ġ×IJ×Ĺר×ķ׳×ķת":141924,"ãģĦãģŁãģłãģĦãģ¦":141925,"سÙĦØŃ":141926,"ØŃÙĦÙħ":141927,"زÙĪار":141928,"Ùĥسر":141929,"×ĺקס":141930,"Ġбан":141931,"Ġбанков":141932,"ĠпÑĢож":141933,"ĠпÑĢожива":141934,"liwo":141935,"liwoÅĽci":141936,"ĠTiếp":141937,"ĠاÙĦÙħÙĨاسب":141938,"ĠاÙĦØ®ÙĬار":141939,"ãģĬãģĭ":141940,"ãģĬãģĭãģĴ":141941,"à¸Ķà¸Ńà¸ģà¹Ħมà¹ī":141942,"ämp":141943,"ämpfe":141944,"à¸ķัà¹īà¸ĩà¹ĥà¸Ī":141945,"ĠзаÑīиÑĤ":141946,"ĠзаÑīиÑĤÑĭ":141947,"ĠThÆ°á»Ŀng":141948,"ĠصÙģ":141949,"ĠصÙģØŃØ©":141950,"×Ĺ×ķרף":141951,"ãĥIJãĥĥãĤ°":141952,"Ġ×ĵ×Ļ×Ĵ":141953,"Ġ×ĵ×Ļ×Ĵ×Ļ×ĺ":141954,"Ġ×ĵ×Ļ×Ĵ×Ļ×ĺ׾×Ļ":141955,"Ġ×Ķ×Ĺ×ķ׾×Ļ×Ŀ":141956,"веÑī":141957,"веÑīа":141958,"ĠкÑĥлÑĮÑĤ":141959,"ĠкÑĥлÑĮÑĤÑĥ":141960,"ĠкÑĥлÑĮÑĤÑĥÑĢÑĭ":141961,"ĠاÙĦاÙĨترÙĨت":141962,"Ġhöch":141963,"Ġhöchst":141964,"Ġíĺķ":141965,"Ġíĺķíĥľ":141966,"Ġвой":141967,"ĠвойнÑĭ":141968,"ÐĽÐŀ":141969,"ìĭłìļ©":141970,"Ġ×ŀ×ij×ķס":141971,"Ġ×ŀ×ij×ķסס":141972,"×ŀ׳×Ļ×¢":141973,"Ġfiyatı":141974,"ĠÑģлÑĥж":141975,"ĠÑģлÑĥжбÑĭ":141976,"à¸Ĺัศ":141977,"à¸Ĺัศà¸Ļ":141978,"ãģĵãģ¨ãģĮå¤ļãģĦ":141979,"Ġ×Ķ×ŀשת":141980,"Ġ×Ķ×ŀשת×ŀש":141981,"å¯ĦãģĽ":141982,"×ŀש׾×ķ×Ĺ":141983,"æĻĤçĤ¹":141984,"æĻĤçĤ¹ãģ§":141985,"à¸ŀรี":141986,"à¸ŀรีà¹Ģมีย":141987,"à¸ŀรีà¹Ģมียรà¹Į":141988,"à¸ŀรีà¹Ģมียรà¹Įลีà¸ģ":141989,"Ġdifficolt":141990,"ĠdifficoltÃł":141991,"ãĥ¬ãĤ¹ãĥĪ":141992,"ãĥ¬ãĤ¹ãĥĪãĥ©ãĥ³":141993,"สมà¹Ģà¸Ķà¹ĩ":141994,"สมà¹Ģà¸Ķà¹ĩà¸Ī":141995,"Ġжид":141996,"Ġжидк":141997,"ĠzupeÅĤ":141998,"ĠzupeÅĤnie":141999,"ĠÙħجر":142000,"ĠÙħجرد":142001,"ãģĮå§ĭ":142002,"ãģĮå§ĭãģ¾":142003,"ãĤŃãĥ£ãĥ©":142004,"Ġ×IJ×ķ×ķ×Ļר":142005,"ãģĬäºĴ":142006,"ãģĬäºĴãģĦ":142007,"ĠpotrÃł":142008,"ĠPaÅĦst":142009,"ĠPaÅĦstwo":142010,"ĠبÙĬاÙĨ":142011,"ĠبÙĬاÙĨات":142012,"Ġиногда":142013,"ĠÑĢа":142014,"ĠÑĢаÑģÑĤв":142015,"ĠÑĢаÑģÑĤвоÑĢ":142016,"Ġ×ĸ×ŀ׳":142017,"ยิà¹īม":142018,"ÄĨ":142019,"ãģ¾ãģķ":142020,"ãģ¾ãģķãģ«":142021,"ãĥķãĤ¡ãĤ¤ãĥ«":142022,"ĠgördÃ¼ÄŁÃ¼":142023,"สà¸ĩà¸Ħร":142024,"สà¸ĩà¸Ħราม":142025,"ĠArkadaÅŁ":142026,"ĠrozwiÄħzania":142027,"×ŀ×ķ×ĺ":142028,"piÄĻ":142029,"piÄĻt":142030,"صغر":142031,"สย":142032,"สยาม":142033,"ãĤĨãģ£ãģıãĤĬ":142034,"Ġtrần":142035,"ĠeconomÃŃa":142036,"Ġgehören":142037,"ãĤ·ãĥ§ãĥ¼":142038,"ĠsÅĤucha":142039,"à¸ŀà¸Ńà¹ĥà¸Ī":142040,"ĠоÑĤмеÑĤил":142041,"ÙĨتÙĤÙĦ":142042,"Ġpropósito":142043,"ĠваÑĪего":142044,"Ġnhắn":142045,"à¹ģà¸ĸว":142046,"ĠкомиÑģ":142047,"ĠкомиÑģÑģи":142048,"ważnie":142049,"ĠyavaÅŁ":142050,"×ŀ×Ļק":142051,"×ŀ×Ļק×ķ×Ŀ":142052,"ש×IJ×ľ×ª":142053,"Ġyıllarda":142054,"ĠЮ":142055,"ĠЮÑĢ":142056,"×ł×¡×Ļ×ij×ķת":142057,"תצ":142058,"תצ×ķ×Ĵ":142059,"ĠоднÑĥ":142060,"Ġà¸Ńยà¹Īาà¸ĩà¹Ħร":142061,"Ġà¸Ńยà¹Īาà¸ĩà¹Ħรà¸ģà¹ĩà¸ķาม":142062,"ëģ¼":142063,"à¹Ħลà¹Ī":142064,"تسÙĦÙĬÙħ":142065,"بÙĦاغ":142066,"Ġìī":142067,"Ġìī½":142068,"Ġìī½ê²Į":142069,"ãĥļãĥ³":142070,"звÑĥÑĩ":142071,"ĠWäh":142072,"ĠWährend":142073,"Ġ×Ļ×Ļת":142074,"Ġ×Ļ×Ļ×ª×Ľ×Ł":142075,"Ġkhuyên":142076,"Ġvẽ":142077,"ĠамеÑĢ":142078,"ĠамеÑĢик":142079,"ĠамеÑĢикан":142080,"ĠамеÑĢиканÑģк":142081,"عجب":142082,"ãĥĽãĥ¼ãĥłãĥļãĥ¼ãĤ¸":142083,"ĠникÑĤо":142084,"ĠÙĤÙİ":142085,"ĠÙĤÙİاÙĦ":142086,"ĠÙĤÙİاÙĦÙİ":142087,"ÐIJÐĹ":142088,"ÙħجÙħÙĪع":142089,"ÙħجÙħÙĪعات":142090,"ĠnecessitÃł":142091,"Ġpobli":142092,"Ġpobliżu":142093,"Ġphấn":142094,"ĠСообÑī":142095,"ÙħÙĤاط":142096,"ÙħÙĤاطع":142097,"Ġ×Ķצ×ķר×ļ":142098,"laÅŁtırma":142099,"วิà¸Ķ":142100,"วิà¸Ķี":142101,"วิà¸Ķีà¹Ĥà¸Ń":142102,"Ġ그리ìĬ¤":142103,"Ġ그리ìĬ¤ëıĦ":142104,"ãĤ¿ãĤ¤ãĥŁ":142105,"ãĤ¿ãĤ¤ãĥŁãĥ³ãĤ°":142106,"ק×ĺ×Ĵ×ķר":142107,"ק×ĺ×Ĵ×ķר×Ļ×Ķ":142108,"Ġ×Ĺ×ķפ":142109,"Ġ×Ĺ×ķפש×Ļ":142110,"أجر":142111,"Ġимени":142112,"ĠÑĢанее":142113,"à¹Ģà¸ŀืà¹Īà¸Ńà¸Ļà¹Ĩ":142114,"ĠJesús":142115,"Ñģоедин":142116,"Ñģоединен":142117,"Ġר×Ĺ×ķק":142118,"à¹Ĥà¸ļรา":142119,"à¹Ĥà¸ļราà¸ĵ":142120,"ĠHÆ¡n":142121,"ĠtháºŃp":142122,"تعÙĬÙĬÙĨ":142123,"ĠtartÄ±ÅŁ":142124,"ĠtartÄ±ÅŁma":142125,"ĠGespr":142126,"ĠGespräch":142127,"תר×ķפ":142128,"תר×ķפ×ķת":142129,"Ġcatégorie":142130,"ĠоказÑĭва":142131,"ĠналиÑĩие":142132,"Ġprésenté":142133,"Ġkull":142134,"Ġkulland":142135,"Ġkullandı":142136,"Ġünl":142137,"Ġünlü":142138,"ĠÙģÙĥرة":142139,"изаÑĤоÑĢ":142140,"×IJ×ķ׳":142141,"×IJ×ķ׳×Ļ×ij":142142,"×IJ×ķ׳×Ļ×ijרס":142143,"×IJ×ķ׳×Ļ×ijרס×Ļ×ĺת":142144,"ĠÑĢаÑģÑģмаÑĤ":142145,"ĠÑĢаÑģÑģмаÑĤÑĢ":142146,"ĠÑĢаÑģÑģмаÑĤÑĢива":142147,"تÙĥÙĦÙħ":142148,"ÙĥترÙĪ":142149,"ÙĥترÙĪÙĨÙĬ":142150,"ĠÑģоÑĩеÑĤ":142151,"ĠÑģоÑĩеÑĤа":142152,"ãĤĴè¦ĭãģĽ":142153,"Ġngừa":142154,"ĠÐłÐµÑģп":142155,"ĠÐłÐµÑģпÑĥб":142156,"ĠÐłÐµÑģпÑĥблик":142157,"ãĤ¦ãĤ©":142158,"ãĤ¦ãĤ©ãĥ¼":142159,"ĠÐľÐµÐ¶Ð´Ñĥ":142160,"ĠìŀĪê²Į":142161,"Ġmâ":142162,"ĠìļĶì²Ń":142163,"ضار":142164,"ลุà¹īà¸Ļ":142165,"ëĮĢíķĻêµIJ":142166,"×ĸ×Ļ׼":142167,"×ĸ×Ļ׼ר×ķף":142168,"ãĤ¹ãĥļ":142169,"ãĤ¹ãĥļãĥ¼ãĤ¹":142170,"ĠкÑĢаÑģоÑĤ":142171,"Ｈ":142172,"ê¼Ń":142173,"ãĤĴéĽĨ":142174,"ãĤĴéĽĨãĤģ":142175,"ë°Ŀ":142176,"Ġ×Ķ׳×IJ":142177,"Ġ×Ķ׳×IJש×Ŀ":142178,"Ġê°Ģìļ´":142179,"Ġê°Ģìļ´ëį°":142180,"تÙĥÙĦÙģØ©":142181,"ĠØŃÙĤÙĬÙĤÙĬ":142182,"Ġhalk":142183,"Ġhalkın":142184,"ÑİÑīÑĥÑİ":142185,"ĠÑģпин":142186,"סר×ĺף":142187,"ĠпеÑĢвого":142188,"Ġполож":142189,"ĠположиÑĤелÑĮн":142190,"Ġдл":142191,"ĠдлиÑĤелÑĮн":142192,"ĠVÄ©nh":142193,"ê´´":142194,"ĠÑģÑĭÑĢ":142195,"ĠíĨµíķĺìŬ":142196,"ë³ijìĽIJ":142197,"à¹Ĥรà¸ĩà¸ĩาà¸Ļ":142198,"รัà¸ļà¸ľà¸´à¸Ķ":142199,"รัà¸ļà¸ľà¸´à¸Ķà¸Ĭà¸Ńà¸ļ":142200,"تجÙĨب":142201,"sÅĤ":142202,"sÅĤuch":142203,"ãĤ¢ãĥ«ãĥIJ":142204,"ãĤ¢ãĥ«ãĥIJãĥł":142205,"ëī´ìĬ¤":142206,"Ġpatië":142207,"Ġpatiënt":142208,"Ġìĺ¤í":142209,"Ġìĺ¤íŀ":142210,"Ġìĺ¤íŀĪ":142211,"Ġìĺ¤íŀĪ볤":142212,"ĠDerne":142213,"ĠDerneÄŁi":142214,"wróci":142215,"wróciÄĩ":142216,"ĠобÑī":142217,"ĠобÑīеÑģÑĤв":142218,"ĠобÑīеÑģÑĤвенно":142219,"ĠêµIJìĪĺ":142220,"tıģımız":142221,"Ġ×Ķ×ŀש×Ļ×ij":142222,"körper":142223,"Ġпозвол":142224,"ĠпозволиÑĤ":142225,"ĠChiến":142226,"أخÙĪ":142227,"ĠAydın":142228,"à¸Ķà¹īาà¸Ļล":142229,"à¸Ķà¹īาà¸Ļลà¹Īาà¸ĩ":142230,"Ġdru":142231,"Ġdruż":142232,"Ġdrużyn":142233,"Ġë°ľíijľ":142234,"ĠThảo":142235,"جÙĩاد":142236,"à¸ģระà¸Ĺูà¹ī":142237,"ĠкÑĢов":142238,"ĠкÑĢови":142239,"Ġiçerik":142240,"Ġnadzie":142241,"ĠnadziejÄĻ":142242,"ĠСмоÑĤÑĢ":142243,"Ġphức":142244,"جتÙħاع":142245,"جتÙħاعÙĬØ©":142246,"компон":142247,"компоненÑĤ":142248,"Ġбил":142249,"ĠбилеÑĤ":142250,"ãĥIJãĥ³ãĥī":142251,"ĠPolÃŃcia":142252,"اÙĦتÙĩ":142253,"اÙĦتÙĩاب":142254,"ØŃرÙģ":142255,"تخط":142256,"تخطÙĬØ·":142257,"ãĤ³ãĥ¼ãĥ":142258,"ãĤ³ãĥ¼ãĥĴ":142259,"ãĤ³ãĥ¼ãĥĴãĥ¼":142260,"･･･":142261,"à¸ĭà¸Ńย":142262,"Ġcrédit":142263,"è²·ãģ£ãģŁ":142264,"ĠпоÑĢÑıд":142265,"ĠпоÑĢÑıдке":142266,"Ġphó":142267,"Ġwida":142268,"ĠwidaÄĩ":142269,"جرائÙħ":142270,"à¸ľà¸µ":142271,"ĠbÄĻdÄĻ":142272,"Ġ×ŀפת×Ĺ":142273,"ãĥijãĥ¼ãĥ":142274,"ãĥijãĥ¼ãĥĨ":142275,"ãĥijãĥ¼ãĥĨãĤ£":142276,"ãĥijãĥ¼ãĥĨãĤ£ãĥ¼":142277,"ĠKaż":142278,"ĠKażdy":142279,"ĠнеобÑħодимоÑģÑĤи":142280,"à¸Łà¸Ńรà¹Į":142281,"à¸Łà¸Ńรà¹Įม":142282,"ĠмалÑĭÑĪ":142283,"ĠплоÑĤ":142284,"ĠÑĥÑģÑĤÑĢой":142285,"ĠÑĥÑģÑĤÑĢойÑģÑĤва":142286,"à¸ĸà¸Ńà¸Ļ":142287,"ĠoluÅŁturul":142288,"ĠÅĽwiad":142289,"ĠÅĽwiadom":142290,"ÙħعÙĩد":142291,"ĠпÑĢоизведен":142292,"Æł":142293,"ר×Ļש":142294,"Ùħستث":142295,"ÙħستثÙħر":142296,"׳×Ļ×Ļר":142297,"pañ":142298,"Ġ;-)":142299,"Ġë°ľê²¬":142300,"Ġgörüyor":142301,"ÙħؤÙĦÙģ":142302,"ĠÄIJá»ģ":142303,"ĠاÙĦÙĨÙĪاب":142304,"×Ĺק×Ļר×Ķ":142305,"Ġmá»ıi":142306,"è¿°ãģ¹":142307,"ÐĿик":142308,"ìŀĸìķĦ":142309,"ìŀĸìķĦìļĶ":142310,"prowadziÅĤ":142311,"lóg":142312,"lógica":142313,"פס×ĺ":142314,"פס×ĺ×Ļ×ij׾":142315,"Ġ×ŀ×ĵ×Ķ":142316,"Ġ×ŀ×ĵ×Ķ×Ļ×Ŀ":142317,"ãģĵãģĵãģ¾ãģ§":142318,"×Ķת×Ĺ":142319,"×Ķת×Ĺ׾×Ķ":142320,"Ġפ×ķס":142321,"Ġפ×ķס×ĺ×Ļ×Ŀ":142322,"Ġнев":142323,"Ġневоз":142324,"Ġневозможно":142325,"ĠdostÄĻpny":142326,"ĠغاÙĦ":142327,"ĠغاÙĦب":142328,"ĠbezpieczeÅĦst":142329,"ĠbezpieczeÅĦstwa":142330,"åĪĨãģĭãĤĭ":142331,"ĠFührung":142332,"à¸ģีà¹ī":142333,"gemÃ¤ÃŁ":142334,"à¸Ĭà¹Īวà¸ĩà¹Ģวลา":142335,"Ġìļ°ë¦¬ëĤĺ":142336,"Ġìļ°ë¦¬ëĤĺëĿ¼":142337,"ãģ¥ãģıãĤĬ":142338,"ĠاÙĦÙħسÙĦ":142339,"ĠاÙĦÙħسÙĦØŃØ©":142340,"Ġliberté":142341,"клÑİÑĩение":142342,"Ġzamów":142343,"Ġzamówienia":142344,"รà¸ĸà¹Ħà¸Ł":142345,"Ø£ÙģÙĦ":142346,"Ø£ÙģÙĦاÙħ":142347,"Ùħراج":142348,"Ùħراجعة":142349,"Ġë¹ĦêµIJ":142350,"ĠاÙĦتاب":142351,"ĠاÙĦتابعة":142352,"Ġë§ĮëĤĺ":142353,"ĠбÑĥм":142354,"ĠбÑĥмаг":142355,"Ġgénero":142356,"Ġìŀĺ못":142357,"×ŀפ×ķר×ĺ":142358,"è²·ãģĦçī©":142359,"ĠÙĦدÙĬÙĥ":142360,"Ġ×ľ×¢×Ļת":142361,"Ġ×ľ×¢×Ļת×Ļ×Ŀ":142362,"ĠsÅĤab":142363,"ĠпÑĢедÑģÑĤавлÑı":142364,"ãĤ¿ãĤ¤ãĥĪ":142365,"ãĤ¿ãĤ¤ãĥĪãĥ«":142366,"Ùħص":142367,"ÙħصطÙģ":142368,"ÙħصطÙģÙī":142369,"Ġdifficulté":142370,"ãĥĨãĤ£ãĥĸ":142371,"ĠpewnoÅĽci":142372,"ĠpewnoÅĽciÄħ":142373,"Ġ무ìĬ¨":142374,"إرس":142375,"إرساÙĦ":142376,"ĠдалÑĮ":142377,"ĠдалÑĮÑĪе":142378,"Ġ×ľ×ł×¡":142379,"Ġ×ľ×ł×¡×ķת":142380,"หมูà¹Īà¸ļà¹īาà¸Ļ":142381,"×ŀס×ŀ׼×Ļ":142382,"أسÙĦÙĪب":142383,"ĠzwÅĤ":142384,"ĠzwÅĤas":142385,"ĠzwÅĤaszc":142386,"ĠzwÅĤaszcza":142387,"ĠпÑĢеж":142388,"ĠпÑĢежде":142389,"ĠоÑĢганизаÑĨиÑı":142390,"Ġdönemin":142391,"Ġdöneminde":142392,"ĠỦ":142393,"ĠỦy":142394,"ä¸ĭãģĴ":142395,"ĠпоÑģледние":142396,"Ġgüne":142397,"ĠgüneÅŁ":142398,"Ġ×IJ×ĸר":142399,"Ġ×IJ×ĸר×Ĺ×Ļ":142400,"ãģ§ãģĤãĤįãģĨ":142401,"ĠÙĨÙĤ":142402,"ĠÙĨÙĤاط":142403,"æŃ£ãģĹãģĦ":142404,"ĠÑĢег":142405,"ĠÑĢегиона":142406,"ĠFörder":142407,"ê²½ìĺģ":142408,"dıklar":142409,"dıklarını":142410,"trzymaÄĩ":142411,"أشÙĥ":142412,"أشÙĥاÙĦ":142413,"×Ķת×IJ":142414,"×Ķת×IJ×ŀ×Ķ":142415,"à¸Ĺำà¹ĥหà¹īà¹Ģà¸ģิà¸Ķ":142416,"ĠGebä":142417,"ĠGebäude":142418,"ĠСеÑĢг":142419,"ĠСеÑĢгей":142420,"ĠздоÑĢов":142421,"ĠздоÑĢовÑĮÑı":142422,"Ġrãi":142423,"ĠпÑĢедÑĥÑģ":142424,"ĠпÑĢедÑĥÑģмоÑĤÑĢ":142425,"ĠпÑĢедÑĥÑģмоÑĤÑĢен":142426,"Ġ×Ķצ×Ļ×ij":142427,"Ġ×Ķצ×Ļ×ij×ķר×Ļ":142428,"Ġdésir":142429,"ĠноÑĩ":142430,"ĠноÑĩÑĮ":142431,"möglichkeiten":142432,"Ġ×IJ×Ĺר×ķ׳×Ļ×Ŀ":142433,"Ġsoirée":142434,"ĠNháºŃn":142435,"Ùª":142436,"à¸Ľà¸£à¸°à¸§à¸±à¸ķิศาสà¸ķรà¹Į":142437,"êµIJíĨµ":142438,"ĠأخÙĬ":142439,"Ġdécid":142440,"Ġdécidé":142441,"Ġwyja":142442,"ĠwyjaÅĽni":142443,"Ġสิ":142444,"Ġสิà¸ĩ":142445,"Ġสิà¸ĩหา":142446,"Ġสิà¸ĩหาà¸Ħม":142447,"à¹ģà¸Ńรà¹Į":142448,"หà¸Ļà¹īาà¸Īà¸Ń":142449,"סתר":142450,"Ġê¶":142451,"Ġê¶Į":142452,"Ġê¶Į리":142453,"plätze":142454,"بطÙĦ":142455,"ê±´ìĦ¤":142456,"Ġ×IJ×Ļ×ŀ×Ļ":142457,"Ġ×IJ×Ļ×ŀ×Ļ×Ļ׾":142458,"ãģ½":142459,"تراث":142460,"×IJ׾×Ļ×ŀ×ķת":142461,"ĠdisponÃŃveis":142462,"Ġzale":142463,"Ġzależy":142464,"à¸Ľà¸£à¸°à¸Ĭาสัมà¸ŀัà¸Ļà¸ĺà¹Į":142465,"ĠÅļwiat":142466,"Ġporówn":142467,"Ġporówna":142468,"Ġ׾×ĺ×ķ×ijת":142469,"×Ķ×ĸ×ŀ׳×Ķ":142470,"Ġ×Ľ×ª×ķצ×IJ×Ķ":142471,"Ġ×ijק׾":142472,"Ġ×ijק׾×ķת":142473,"ĠоÑĤкÑĢ":142474,"ĠоÑĤкÑĢÑĭва":142475,"ãĥijãĥ¯ãĥ¼":142476,"ë¿IJë§Į":142477,"ĠвÑģÑı":142478,"ĠвÑģÑıк":142479,"ãģ¨ãģªãģ£ãģ¦ãģĦãĤĭ":142480,"ĠgiáºŃn":142481,"ĠокÑĢÑĥ":142482,"ĠокÑĢÑĥжа":142483,"ĠокÑĢÑĥжаÑİÑī":142484,"ĠUniversität":142485,"ĠÑĢож":142486,"ĠÑĢожд":142487,"ĠÑĢождениÑı":142488,"Ø®ÙĬÙĦ":142489,"Ġкомпаний":142490,"ĠÑĢазлиÑĩнÑĭе":142491,"ĠЦена":142492,"׳×Ļ×ķ×ĸ":142493,"׳×Ļ×ķ×ĸ׾":142494,"׳×Ļ×ķ×ĸ׾×ĺר":142495,"Ġê³µê°Ħ":142496,"Ġê°ľëħIJ":142497,"landırma":142498,"ĠÑĥдален":142499,"à¸ŀัà¸ģà¸ľ":142500,"à¸ŀัà¸ģà¸ľà¹Īà¸Ńà¸Ļ":142501,"Ġprotección":142502,"ĠbÅĤ":142503,"ĠbÅĤÄĻd":142504,"ÃĪ":142505,"Ġíĸīë³µ":142506,"ĠÅŁÃ¼":142507,"ĠÅŁÃ¼phe":142508,"ĠíĶ":142509,"ĠíĶ¼":142510,"ĠíĶ¼íķ´":142511,"Ġëĭ¤ë¥´":142512,"à¹Ħมà¹Īà¹Ģà¸ģิà¸Ļ":142513,"ãģ¿ãģª":142514,"ãģ¿ãģªãģķãĤĵ":142515,"ĠпоÑĤÑĢеб":142516,"ĠпоÑĤÑĢебиÑĤел":142517,"ĠاÙĦÙĥÙĦاÙħ":142518,"ìķĦë²Ħ":142519,"ìķĦë²Ħì§Ģ":142520,"ãĤĴ使ãģ£ãģŁ":142521,"Ġbụi":142522,"ĠпоÑĤеÑĢ":142523,"ĠпоÑĤеÑĢÑı":142524,"ĠØ¢ÙĦاÙģ":142525,"ĠнаÑģÑĤоÑıÑīее":142526,"ãģıãģªãĤĬãģ¾ãģĹãģŁ":142527,"clusão":142528,"ãĤ³ãĥĶãĥ¼":142529,"צפ×Ļ":142530,"צפ×Ļ×Ļ×Ķ":142531,"Ø®ÙĦا":142532,"Ø®ÙĦاص":142533,"ลà¹īำ":142534,"ãĥ¯ãĤ¤ãĥ³":142535,"Ġมีà¸Ļา":142536,"Ġมีà¸Ļาà¸Ħม":142537,"شخص":142538,"شخصÙĬات":142539,"Ġ×ĸק":142540,"Ġ×ĸק×ķק":142541,"×Ļ×Ļצ":142542,"×Ļ×Ļצ×Ĵ":142543,"èĢĥãģĪæĸ¹":142544,"Ġürünü":142545,"ĠиÑģпол":142546,"ĠиÑģполни":142547,"Ġcompañero":142548,"קצ×Ķ":142549,"×ŀ×¢×ł×Ļק":142550,"ÙħØŃÙħد":142551,"Ġcámara":142552,"Ġпед":142553,"Ġпедаг":142554,"Ġпедагог":142555,"маÑĢ":142556,"маÑĢк":142557,"×Ķ×ª×ł×Ĵ×ĵ":142558,"ĠìĨĮê°ľ":142559,"ĠcomunitÃł":142560,"곤":142561,"ĠNgÃłi":142562,"สà¸ĩà¸ļ":142563,"ĠmieszkaÅĦców":142564,"ĠÙĨÙĩائÙĬ":142565,"ivité":142566,"Ġиде":142567,"ĠидеалÑĮн":142568,"ĠأسبÙĪع":142569,"Ġ×Ļ×¢×ľ":142570,"Ġ׾ר×IJש":142571,"Ġ׾ר×IJש×ķ׳×Ķ":142572,"ĠзапиÑģи":142573,"ĠкоÑĢпÑĥÑģ":142574,"วà¸ĩศ":142575,"วà¸ĩศà¹Į":142576,"ĠÐĶм":142577,"ĠÐĶмиÑĤ":142578,"ĠÐĶмиÑĤÑĢ":142579,"Ġkönnt":142580,"Ġbölges":142581,"Ġbölgesinde":142582,"׼×Ļ׼":142583,"׼×Ļ׼ר":142584,"ĠاÙĦإثÙĨ":142585,"ĠاÙĦإثÙĨÙĬÙĨ":142586,"Ġngá»Ļ":142587,"ì¹ł":142588,"دراج":142589,"Ġuda":142590,"ĠudaÅĤo":142591,"ìºIJ":142592,"برÙĨاÙħج":142593,"ĠÑģÑĥдеб":142594,"ĠÑģÑĥдебн":142595,"Ġzunächst":142596,"ĠEducación":142597,"ãģ¨ãģªãģ£ãģ¦ãģĦãģ¾ãģĻ":142598,"Ġ×Ķ×IJ×ŀ×Ļת×Ļ":142599,"ĠÄ°nt":142600,"ĠÄ°nternet":142601,"ĠcaÅĤego":142602,"ãĥĹãĥªãĥ³":142603,"إبد":142604,"إبداع":142605,"ĠпоÑĢÑĤал":142606,"à¹Ĥà¸ķà¹ī":142607,"Ġ×Ķקש×ķר":142608,"плод":142609,"ĠÙħد":142610,"ĠÙħدرÙĬد":142611,"×ŀסע×ĵ×Ķ":142612,"ĠØ´ÙĬئ":142613,"ĠØ´ÙĬئا":142614,"à¸ģà¹Īà¸Ńสรà¹īาà¸ĩ":142615,"Ġì°¸ê³ł":142616,"à¹Ģà¸Ĺร":142617,"à¹Ģà¸Ĺรà¸Ķ":142618,"Ġ×ij×ŀקר×Ļ×Ŀ":142619,"Ġbât":142620,"Ġbâtiment":142621,"åij¼ãģ³":142622,"ç´łæķµ":142623,"ç´łæķµãģª":142624,"przedsiÄĻbiorst":142625,"przedsiÄĻbiorstw":142626,"Ġ×ł×ª×ķ׳×Ļ×Ŀ":142627,"×Ĺ׾×ķ×Ŀ":142628,"รวย":142629,"ÙħÙĪضÙĪع":142630,"ĠÑģобÑĢан":142631,"ведÑĥÑī":142632,"ĠÑĤеаÑĤ":142633,"ĠÑĤеаÑĤÑĢ":142634,"meye":142635,"meyeceÄŁi":142636,"ĠpieniÄħ":142637,"ĠpieniÄħd":142638,"ĠpieniÄħdze":142639,"ÑĢезиденÑĤ":142640,"ØŃصر":142641,"ìĺ¥":142642,"à¹Ģยืà¸Ńà¸Ļ":142643,"ĠÑĥни":142644,"ĠÑĥнивеÑĢ":142645,"ĠÑĥнивеÑĢÑģ":142646,"ĠÑĥнивеÑĢÑģиÑĤеÑĤ":142647,"ĠاÙĦرØŃ":142648,"ĠاÙĦرØŃÙħÙĨ":142649,"ĠÑĤеÑħнолог":142650,"ĠÑĤеÑħнологии":142651,"ìĹIJëĦĪ":142652,"ìĹIJëĦĪì§Ģ":142653,"ĠíķŃ":142654,"ĠíķŃìĥģ":142655,"à¸ĺา":142656,"à¸ĺาà¸ķุ":142657,"ĠEspañol":142658,"×ĵ×Ĵש":142659,"Ġêµī":142660,"Ġêµīìŀ¥":142661,"Ġêµīìŀ¥íŀĪ":142662,"ĠÅĤat":142663,"ĠÅĤatwo":142664,"Ġká»ĭch":142665,"إز":142666,"إزاÙĦØ©":142667,"ĠдейÑģÑĤвие":142668,"ĠsaÄŁlayan":142669,"สุà¸Ķยà¸Ńà¸Ķ":142670,"ĠzostaÄĩ":142671,"ĠdisponÃŃvel":142672,"ïºį":142673,"verständ":142674,"verständlich":142675,"twor":142676,"tworzyÄĩ":142677,"عجز":142678,"à¹Ģà¸Ĥà¹īม":142679,"ยà¹Īà¸Ńม":142680,"Ġstratég":142681,"Ġstratégie":142682,"à¸ľà¸¥à¹Ħมà¹ī":142683,"Ġê°ģì¢ħ":142684,"ĠÙħÙĪا":142685,"ĠÙħÙĪاض":142686,"ĠÙħÙĪاضÙĬع":142687,"اØŃتج":142688,"اØŃتجاج":142689,"ĠẤ":142690,"ĠẤn":142691,"×ŀ×ŀש׾×Ķ":142692,"ĠÅŁekil":142693,"×ŀ×Ĺ׾":142694,"×ŀ×Ĺ׾×ķת":142695,"Ġà¸ĺ":142696,"Ġà¸ĺัà¸Ļ":142697,"Ġà¸ĺัà¸Ļวา":142698,"Ġà¸ĺัà¸Ļวาà¸Ħม":142699,"Ġìĭ¤ìłľ":142700,"Ġìĭ¤ìłľë¡ľ":142701,"ì¤ijìķĻ":142702,"ëįĶëĿ¼":142703,"ĠÑĪиÑĢ":142704,"ĠÑĪиÑĢоко":142705,"Ġsolución":142706,"วาà¸ĩà¹ģà¸ľà¸Ļ":142707,"×IJ×ķ×ĺ×ķ×ŀ":142708,"×IJ×ķ×ĺ×ķ×ŀ×ĺ×Ļ":142709,"ĠÑĢеÑģÑĤ":142710,"ĠÑĢеÑģÑĤоÑĢ":142711,"ĠÑĢеÑģÑĤоÑĢан":142712,"ëį¸":142713,"ÑĤÑĢад":142714,"ÑĤÑĢади":142715,"ÑĤÑĢадиÑĨион":142716,"ÑĤÑĢадиÑĨионн":142717,"มะà¹Ģรà¹ĩ":142718,"มะà¹Ģรà¹ĩà¸ĩ":142719,"à¹Ĥส":142720,"Ġolmasını":142721,"×ŀ×ķסר":142722,"ĠоÑĤноÑĪении":142723,"Ġê°ĢëĬ¥ìĦ±":142724,"Ġyuk":142725,"Ġyukarı":142726,"ìĨĶ":142727,"ĠÑģÑĦ":142728,"ĠÑģÑĦеÑĢе":142729,"Ġק×ķפ":142730,"ãĤ±ãĥ¼ãĤ":142731,"ãĤ±ãĥ¼ãĤŃ":142732,"âĢķâĢķ":142733,"ĠاÙĦØ£ÙĦÙħ":142734,"ĠاÙĦØ£ÙĦÙħاÙĨÙĬ":142735,"ẢN":142736,"ת×ķ׼׳×Ļ×ķת":142737,"ĠÑģÑĥÑīеÑģÑĤвÑĥеÑĤ":142738,"æĪijãĢħ":142739,"ĠاÙĦصادر":142740,"ĠTrá»įng":142741,"Ġад":142742,"ĠадминиÑģÑĤ":142743,"ĠадминиÑģÑĤÑĢа":142744,"ĠадминиÑģÑĤÑĢаÑĨи":142745,"ĠдÑĢÑĥгими":142746,"ÑģпеÑĪ":142747,"عÙĦاÙħات":142748,"Ġаб":142749,"ĠабÑģол":142750,"ĠабÑģолÑİÑĤ":142751,"ĠабÑģолÑİÑĤно":142752,"ฤà¸Ķู":142753,"étr":142754,"étranger":142755,"нÑıÑĤи":142756,"нÑıÑĤие":142757,"×¢×ķ׳":142758,"×¢×ķ׳ש":142759,"ĠÙĤائ":142760,"ĠÙĤائÙĦا":142761,"ĠмаÑģ":142762,"ĠмаÑģло":142763,"ãĥīãĤ¤":142764,"ãĥīãĤ¤ãĥĦ":142765,"å¿ħè¦ģãģĮãģĤãĤĬãģ¾ãģĻ":142766,"×ŀ×ķ×ĸ×Ļ×IJ":142767,"×ŀ×ķ×ĸ×Ļ×IJ×ķף":142768,"ĠNgoại":142769,"Ġkênh":142770,"à¸ģารà¸Ńà¸Ńà¸ģà¹ģà¸ļà¸ļ":142771,"×ŀפק":142772,"×ŀפק×ĵ":142773,"ÙħÙĨاز":142774,"ÙħÙĨازÙĦ":142775,"ë·°":142776,"íŤ":142777,"ÙħÙĩارات":142778,"Ġpropriété":142779,"פ×Ĵ×Ļש×Ķ":142780,"ÑĩÑĢ":142781,"ÑĩÑĢеж":142782,"ÑĩÑĢежден":142783,"×Ķ×ķצ×IJ×Ķ":142784,"ØŃÙĥÙĬÙħ":142785,"ĠíĻĪ":142786,"ĠíĻĪíİĺìĿ´ì§Ģ":142787,"åݳ":142788,"åݳãģĹãģĦ":142789,"×¢×ŀ×ĵ×Ķ":142790,"ĠAuÃŁen":142791,"سÙĪØ¡":142792,"ë¹Ī":142793,"ĠÙĪØ®":142794,"ĠÙĪخاصة":142795,"инÑĤеÑĢ":142796,"инÑĤеÑĢеÑģ":142797,"èĩ´ãģĹãģ¾ãģĻ":142798,"Ġhüküm":142799,"à¹Ħà¸Ĥมัà¸Ļ":142800,"Ġdavran":142801,"ĠdavranÄ±ÅŁ":142802,"à¹Ģà¸ķียà¸ĩ":142803,"вÑĢем":142804,"вÑĢеменно":142805,"à¹Ģà¸Ĺศà¸ģา":142806,"à¹Ģà¸Ĺศà¸ģาล":142807,"å¼ķãģ£":142808,"å¼ķãģ£è¶ĬãģĹ":142809,"×IJר×ķ×Ĺ":142810,"×IJר×ķ×Ĺת":142811,"à¹Ģวิ":142812,"à¹Ģวิรà¹Į":142813,"à¸Ńยà¹Īาà¸ĩรวà¸Ķà¹Ģรà¹ĩว":142814,"ĠìŬíĸī":142815,"ĠÑĢанÑĮ":142816,"ĠÑĢанÑĮÑĪе":142817,"Ġzobow":142818,"ĠzobowiÄħ":142819,"ĠzobowiÄħz":142820,"Ġ×ķ׼×ŀ×ķ×ijף":142821,"ĠاÙĦÙħÙĩ":142822,"ĠاÙĦÙħÙĩÙĨÙĬ":142823,"ãĤ¢ãĤ¸":142824,"ãĤ¢ãĤ¸ãĤ¢":142825,"ë°©ìĨ¡":142826,"à¸Ńà¸Ńà¸ģà¸ģำลัà¸ĩ":142827,"à¸Ńà¸Ńà¸ģà¸ģำลัà¸ĩà¸ģาย":142828,"améli":142829,"améliorer":142830,"å½ĵãģŁãĤĬåīį":142831,"Ġregelm":142832,"ĠregelmÃ¤ÃŁig":142833,"ãģĬåĭ":142834,"ãģĬåĭ§":142835,"ãģĬåĭ§ãĤģ":142836,"ĠmÆ°á»Ŀi":142837,"برÙħج":142838,"ĠNatürlich":142839,"ĠDÅ©ng":142840,"ĠاÙĦرجاÙĦ":142841,"Ġthép":142842,"ĠolmuÅŁtur":142843,"×ŀ×ķס×Ļק×Ķ":142844,"fälle":142845,"주íĥĿ":142846,"ĠاÙĦÙģرص":142847,"ĠnajwiÄĻks":142848,"ĠnajwiÄĻkszy":142849,"ĠçaÄŁ":142850,"ĠçaÄŁrı":142851,"ì¸ł":142852,"ĠvÃŃct":142853,"ĠvÃŃctima":142854,"ĠÑģовеÑĢÑĪен":142855,"×Ķ×Ļ×Ļת×Ļ":142856,"à¹Ģà¸Ķี":142857,"à¹Ģà¸Ķีà¹ĭ":142858,"à¹Ģà¸Ķีà¹ĭยว":142859,"üyü":142860,"Ġдоп":142861,"Ġдополн":142862,"ĠдополниÑĤелÑĮно":142863,"à¹ģà¸ķà¸ģà¸ķà¹Īาà¸ĩà¸ģัà¸Ļ":142864,"Ġál":142865,"Ġálbum":142866,"à¸Ľà¸£à¸°à¸Īà¸³à¸Ľà¸µ":142867,"ĠÑĦедеÑĢ":142868,"ĠÑĦедеÑĢалÑĮн":142869,"ĠobsÅĤ":142870,"ĠobsÅĤugi":142871,"à¹Ģรืà¹Ī":142872,"à¹Ģรืà¹Īà¸Ńย":142873,"à¹Ģรืà¹Īà¸Ńยà¹Ĩ":142874,"ëģĮ":142875,"Ġnghìn":142876,"ĠBaÅŁkanlıģı":142877,"تأسÙĬ":142878,"تأسÙĬس":142879,"Ġ×ij×ij×ķקר":142880,"Ġ×¢×ij×ķ×ĵ×ķת":142881,"ĠبصÙĪرة":142882,"ãĤıãģijãģ§ãģ¯ãģªãģĦ":142883,"führer":142884,"ãĤ¹ãĤŃ":142885,"ãĤ¹ãĤŃãĥ«":142886,"ĠاÙĦÙĤض":142887,"ĠاÙĦÙĤضÙĬØ©":142888,"ĠдолжноÑģÑĤ":142889,"ÙģارÙĤ":142890,"Ġcomeçou":142891,"Ġorganisé":142892,"Ġxuân":142893,"ĠÑģообÑīаеÑĤ":142894,"ĠпÑĢид":142895,"ĠпÑĢидеÑĤÑģÑı":142896,"TÃľRK":142897,"ãĥ¬ãĥ¼ãĤ·ãĥ§ãĥ³":142898,"Không":142899,"استÙģ":142900,"استÙģادة":142901,"ä¸ĬãģĮãģ£ãģ¦":142902,"Ġumie":142903,"ĠumiejÄĻ":142904,"ĠumiejÄĻtn":142905,"ĠumiejÄĻtnoÅĽci":142906,"ëĤ¸":142907,"à¹Ģà¸Ļà¸Ńรà¹Į":142908,"×ĵ×ķ×ķ×Ĺ":142909,"ÃŃsimo":142910,"IÃĬ":142911,"IÃĬN":142912,"Ġalcanç":142913,"Ġà¸ķุ":142914,"Ġà¸ķุลา":142915,"Ġà¸ķุลาà¸Ħม":142916,"ש׾×ĺ×ķף":142917,"Ġélè":142918,"Ġélèves":142919,"ĠÄiju":142920,"ĠÄijuá»ķi":142921,"ĠØ£Ùģ":142922,"ĠØ£ÙģرÙĬ":142923,"ĠØ£ÙģرÙĬÙĤÙĬ":142924,"ĠØ£ÙģرÙĬÙĤÙĬا":142925,"ãĤĴæİ¢ãģĻ":142926,"ĠпÑĢедложениÑı":142927,"جاد":142928,"ĠÑħоÑĤÑĮ":142929,"Ñģал":142930,"Ñģалон":142931,"à¸Ľà¸£à¸°à¹Ģม":142932,"à¸Ľà¸£à¸°à¹Ģมิà¸Ļ":142933,"ãĤŃãĥĥãĥģ":142934,"ãĤŃãĥĥãĥģãĥ³":142935,"×ij×ĵ×Ļק×ķת":142936,"Ġchù":142937,"Ġchùa":142938,"ÐĴиде":142939,"ÐĴидео":142940,"иÑĢовка":142941,"ĠÑħоÑĤиÑĤе":142942,"Ġspécifique":142943,"รสà¸Ĭาà¸ķิ":142944,"è¾¼ãĤĵãģł":142945,"伸ãģ³":142946,"×Ķצ׾×Ĺת":142947,"ãģ©ãģ®ãĤĪãģĨãģ«":142948,"سعادة":142949,"Ġлид":142950,"ĠлидеÑĢ":142951,"มà¸ĩ":142952,"มà¸ĩà¸Ħล":142953,"ØŃاÙħÙĦ":142954,"หลุà¸Ķ":142955,"à¸Ńยà¹Īาà¸ĩà¸ķà¹Īà¸Ń":142956,"à¸Ńยà¹Īาà¸ĩà¸ķà¹Īà¸Ńà¹Ģà¸Ļืà¹Īà¸Ńà¸ĩ":142957,"ãģķãģĽãģ¦éłĤ":142958,"تسÙĪÙĬ":142959,"تسÙĪÙĬÙĤ":142960,"ĠaÅŁaģıd":142961,"ĠaÅŁaģıdaki":142962,"ĠÑĨелÑĮ":142963,"ĠÑĨелÑĮÑİ":142964,"ĠAraÅŁtırma":142965,"à¸Ĥัà¸ļรà¸ĸ":142966,"ÙĩØ°Ùĩ":142967,"ลà¸ĩà¸Ĺะ":142968,"ลà¸ĩà¸Ĺะà¹Ģà¸ļ":142969,"ลà¸ĩà¸Ĺะà¹Ģà¸ļียà¸Ļ":142970,"تÙĥاÙħÙĦ":142971,"Ġcio":142972,"Ġcioè":142973,"ãģ¦ãģĬãģı":142974,"ĠاÙĦصØŃÙģÙĬ":142975,"ĠíĬ¹ìłķ":142976,"полниÑĤÑĮ":142977,"ãĤĵãģĺãĤĥãģªãģĦ":142978,"ãĤĵãģĺãĤĥãģªãģĦãģĭ":142979,"ĠاÙĦجÙĩ":142980,"ĠاÙĦجÙĩات":142981,"ĠÑĥÑģпеÑĪно":142982,"Ġвок":142983,"ĠвокÑĢÑĥг":142984,"ĠÑģиÑĤÑĥаÑĨиÑı":142985,"Ġ×Ķ×IJ×ŀר":142986,"Ġ×Ķ×IJ×ŀר×Ļק":142987,"Ġ×Ķ×IJ×ŀר×Ļק×IJ×Ļ":142988,"×ŀ×Ĵ×ĸ":142989,"×ŀ×Ĵ×ĸ×Ļף":142990,"ĠакÑĤÑĥ":142991,"ĠакÑĤÑĥалÑĮн":142992,"éta":142993,"étais":142994,"ĠmogÅĤa":142995,"ĠÑĤоÑĩки":142996,"Ġ×ŀ×Ķ×ŀ×¢":142997,"Ġ×ŀ×Ķ×ŀ×¢×¨×Ľ×ª":142998,"à¸¡à¸µà¸Ľà¸£à¸°à¸ªà¸´à¸Ĺà¸ĺà¸´à¸łà¸²à¸ŀ":142999,"×Ļר×Ļ×ĵ×Ķ":143000,"×Ĵר×ŀ׳":143001,"×Ĵר×ŀ׳×Ļ×Ķ":143002,"Ġглав":143003,"Ġглавное":143004,"Ġ미ëŀĺ":143005,"Ġ׳׼×ķ׳×Ķ":143006,"ĠÙĪØ·ÙĨÙĬ":143007,"opport":143008,"opportunitÃł":143009,"Ġhủy":143010,"ĠÙĦتØŃ":143011,"ĠÙĦتØŃÙĤÙĬÙĤ":143012,"Ġórg":143013,"Ġórgão":143014,"ãĤ¹ãĥĶ":143015,"ãĤ¹ãĥĶãĥ¼ãĥī":143016,"Ġönü":143017,"Ġönüne":143018,"ÙħعاÙħÙĦ":143019,"ש×ŀ×Ļר×Ķ":143020,"ĠвеÑģÑĮма":143021,"ĠwiÄĻkszo":143022,"ĠwiÄĻkszoÅĽÄĩ":143023,"ĠاستراتÙĬج":143024,"ĠاستراتÙĬجÙĬØ©":143025,"ĠÙģØ¥":143026,"ĠÙģإذا":143027,"à¹Ģà¸Ĭืà¹Īà¸Ńม":143028,"à¹Ģà¸Ĭืà¹Īà¸Ńมà¸ķà¹Īà¸Ń":143029,"Ġ׾פר":143030,"Ġ׾פר×ĺ×Ļ×Ŀ":143031,"ÙħضÙĬ":143032,"ĠGerçek":143033,"Ġçocukların":143034,"ÙĪثائÙĤ":143035,"ĠÙħساءÙĭ":143036,"Ġunterstützt":143037,"Ġprést":143038,"Ġpréstamo":143039,"ĠÐłÐ°Ð·Ð¼ÐµÑĢ":143040,"ĠÅŁeker":143041,"Ġséculo":143042,"×ij×Ķ×Ļר":143043,"Ø´ÙĩÙĪر":143044,"Ġà¸Ńีà¸ģ":143045,"Ġà¸Ńีà¸ģà¸Ĺัà¹īà¸ĩ":143046,"Ġllegó":143047,"à¸¨à¸´à¸¥à¸Ľà¸°":143048,"æĪijãģĮ":143049,"æĪijãģĮ家":143050,"عÙĤÙĪ":143051,"عÙĤÙĪبات":143052,"ĠFälle":143053,"ĠsÅĤuż":143054,"ĠsÅĤużb":143055,"ĠاÙĦØŃÙĤÙĪÙĤ":143056,"ĠплиÑĤ":143057,"ĠиноÑģÑĤ":143058,"ĠиноÑģÑĤÑĢан":143059,"ĠиноÑģÑĤÑĢанн":143060,"à¹ĥà¸Ļà¸Ĥà¸ĵะà¸Ĺีà¹Ī":143061,"ãĤ«ãĥĨ":143062,"ãĤ«ãĥĨãĤ´":143063,"ãĤ«ãĥĨãĤ´ãĥª":143064,"à¸Ńิส":143065,"à¸Ńิสระ":143066,"à¹Ģà¸ľà¸¢à¹ģ":143067,"à¹Ģà¸ľà¸¢à¹ģà¸ŀร":143068,"à¹Ģà¸ľà¸¢à¹ģà¸ŀรà¹Ī":143069,"ãģĬãģĦ":143070,"ãģĬãģĦãģĹãģĦ":143071,"استÙĤÙĦ":143072,"استÙĤÙĦاÙĦ":143073,"تØŃض":143074,"تØŃضÙĬر":143075,"åĬ©ãģij":143076,"ÙħراÙģÙĤ":143077,"Ġ×ĵ×ķר":143078,"Ġ×ĵ×ķרש":143079,"×ŀת×Ļ×Ļ×Ĺס":143080,"ס×Ļ׼":143081,"ס×Ļ׼×ķ×Ŀ":143082,"íĮĮíĬ¸":143083,"ĠwyÅĽ":143084,"ĠwyÅĽw":143085,"ĠwyÅĽwiet":143086,"ĠwyÅĽwietl":143087,"ĠاÙĦاÙĨساÙĨ":143088,"ĠStraÃŁen":143089,"Ｌ":143090,"ãģ«åŁº":143091,"ãģ«åŁºãģ¥":143092,"ĠcapÃŃtulo":143093,"ลุย":143094,"Ġ×Ķ×ŀקצ×ķ×¢×Ļ":143095,"ãģĤãĤĭç¨ĭ度":143096,"Ợ":143097,"ĠاÙĦÙĦا":143098,"ĠاÙĦÙĦازÙħØ©":143099,"æķĻãģĪ":143100,"Ġרש×IJ×Ļ":143101,"зав":143102,"завиÑģ":143103,"завиÑģим":143104,"à¸Ľà¸±à¸Īà¸Īัย":143105,"à¹Ģà¸ĭล":143106,"à¹Ģà¸ĭลลà¹Į":143107,"Ġdifférence":143108,"ĠAltın":143109,"ĠкÑĢай":143110,"ĠкÑĢайне":143111,"Ġзло":143112,"Ġgünümüz":143113,"ĠнаÑĤÑĥÑĢ":143114,"ĠнаÑĤÑĥÑĢалÑĮн":143115,"×Ĵ×ķ׾ש×Ļ×Ŀ":143116,"ĠкаÑĤегоÑĢ":143117,"ĠкаÑĤегоÑĢии":143118,"Ġзнак":143119,"à¸ģà¹Īà¸Ńà¸Ļหà¸Ļà¹īา":143120,"à¸ģà¹Īà¸Ńà¸Ļหà¸Ļà¹īาà¸Ļีà¹ī":143121,"ĠÙħÙĨت":143122,"ĠÙħÙĨتخب":143123,"ãĥĽãĥ¼ãĥ«":143124,"ĠевÑĢо":143125,"สว":143126,"สวม":143127,"ĠìľĦìĽIJ":143128,"ĠìľĦìĽIJëĭĺ":143129,"ĠاÙĦØŃÙĪØ«":143130,"ĠاÙĦØŃÙĪØ«ÙĬ":143131,"ĠÑģодеÑĢжиÑĤ":143132,"ãĥķãĤ¡ãĥĥãĤ·ãĥ§ãĥ³":143133,"Ġà¸ģัà¸Ļ":143134,"Ġà¸ģัà¸Ļย":143135,"Ġà¸ģัà¸Ļยายà¸Ļ":143136,"ãĤªãĥª":143137,"ãĤªãĥªãĤ¸":143138,"ãĤªãĥªãĤ¸ãĥĬãĥ«":143139,"ĠбÑĢенд":143140,"ãĤĴæĮģãģ£ãģ¦ãģĦãĤĭ":143141,"Ġinversión":143142,"Ġê°ĸ":143143,"Ġê°ĸê³ł":143144,"ĠnovitÃł":143145,"ê´Ģê´ij":143146,"Ġà¸ŀฤษ":143147,"Ġà¸ŀà¸¤à¸©à¸łà¸²":143148,"Ġà¸ŀà¸¤à¸©à¸łà¸²à¸Ħม":143149,"×ķר×Ĺ×Ļ×Ŀ":143150,"׼׾×ķ׾":143151,"Ġngạc":143152,"×Ļ×Ļש":143153,"×Ļ×Ļש×ķ×ij":143154,"fäll":143155,"fällig":143156,"ĠÑĤÑĢебÑĥеÑĤÑģÑı":143157,"Ġcará":143158,"Ġcarácter":143159,"ĠprincÃŃpio":143160,"ĠÅĤaz":143161,"ĠÅĤazien":143162,"ĠÅĤazienk":143163,"Ġgiãn":143164,"ÑģÑĤÑĢаива":143165,"Ùħساب":143166,"ÙħسابÙĤØ©":143167,"à¹Ģà¸Ħรืà¹Īà¸Ńà¸ĩà¸Ķืà¹Īม":143168,"ترÙĥÙĬب":143169,"volução":143170,"ĠÐŁÐ¾Ñĩ":143171,"ĠÐŁÐ¾Ñĩем":143172,"ĠÐŁÐ¾ÑĩемÑĥ":143173,"казалоÑģÑĮ":143174,"ĠпÑĢименениÑı":143175,"à¹Ģà¸Ĺียม":143176,"íĮĶ":143177,"à¸Ĥà¹īà¸Ńà¹Ģสà¸Ļà¸Ń":143178,"à¸Ľà¸±à¸įà¸įา":143179,"ĠобÑĥÑĩ":143180,"ĠобÑĥÑĩениÑı":143181,"ĠÑģеÑĢи":143182,"ĠÑģеÑĢиал":143183,"Ġinglés":143184,"ĠÙĦÙĥرة":143185,"Ġ×ĺ׾":143186,"Ġ×ĺ׾פ×ķף":143187,"Ġìłij":143188,"Ġìłijê·¼":143189,"×IJ×ķ×Ĵ":143190,"×IJ×ķ×Ĵ×ķס":143191,"×IJ×ķ×Ĵ×ķס×ĺ":143192,"ĠболÑĮÑĪое":143193,"ĠÐļонеÑĩно":143194,"×¢×Ļת×ķ׳":143195,"×¢×Ļת×ķ׳×IJ×Ļ":143196,"Ġкнопк":143197,"Ġзн":143198,"ĠзнаÑĤÑĮ":143199,"ĠÄijá»±":143200,"ĠÄijá»±ng":143201,"влаж":143202,"влажн":143203,"×ŀ×Ļ×ĺ×ij":143204,"ãĤ¬ãĤ¤":143205,"ãĤ¬ãĤ¤ãĥī":143206,"..........":143207,"Ġà¸ģุม":143208,"Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀ":143209,"Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļ":143210,"Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļà¸ĺ":143211,"Ġà¸ģà¸¸à¸¡à¸łà¸²à¸ŀัà¸Ļà¸ĺà¹Į":143212,"bez":143213,"bezpieczeÅĦst":143214,"bezpieczeÅĦstw":143215,"ãĥijãĥijæ´»":143216,"عاط":143217,"عاطÙģ":143218,"ĠÄijáºŃm":143219,"ĠзÑĢ":143220,"ĠзÑĢениÑı":143221,"Ġborç":143222,"Ġнедел":143223,"ĠнеделÑİ":143224,"Ġhá»ı":143225,"Ġhá»ıng":143226,"ìŀ¥ìķł":143227,"ìŀ¥ìķłìĿ¸":143228,"ĠاÙĦعÙĦاÙĤØ©":143229,"Ġíģ¬":143230,"Ġíģ¬ê²Į":143231,"à¹Ħรà¹Ī":143232,"à¸ļาà¸Ķ":143233,"à¸ļาà¸Ķà¹Ģà¸Īà¹ĩà¸ļ":143234,"à¸Ŀรั":143235,"à¸Ŀรัà¹Īà¸ĩ":143236,"à¸Ŀรัà¹Īà¸ĩà¹Ģศ":143237,"à¸Ŀรัà¹Īà¸ĩà¹Ģศส":143238,"רע×Ļ":143239,"רע×Ļ×ķ׳×ķת":143240,"ĠëĮ":143241,"ĠëĮĵ":143242,"ĠëĮĵê¸Ģ":143243,"Ġnajb":143244,"Ġnajbli":143245,"Ġnajbliż":143246,"Ġnajbliższ":143247,"ĠиÑģполÑĮзÑĥеÑĤÑģÑı":143248,"ĠcientÃŃf":143249,"ĠcientÃŃfico":143250,"×¢×ŀק":143251,"Ġgợi":143252,"Ø´ØŃÙĨ":143253,"ĠÅĽm":143254,"ĠÅĽmier":143255,"ĠÅĽmierci":143256,"à¸Ħาสิà¹Ĥà¸Ļà¸Ńà¸Ńà¸Ļà¹Ħลà¸Ļà¹Į":143257,"×Ĺש×ijת×Ļ":143258,"Ġningu":143259,"Ġninguém":143260,"è¾¼ãĤģ":143261,"ãģ·":143262,"ĠÑĥг":143263,"ĠÑĥгол":143264,"ï½°":143265,"פת×Ļ×Ĺ":143266,"פת×Ļ×Ĺת":143267,"Ġ×Ķר×IJש×ķ׳×Ļ×Ŀ":143268,"pósito":143269,"ãĤŃãĥ¬ãĤ¤":143270,"ãģ©ãģĵãĤį":143271,"à¹Ģà¸Ĺà¹Īาà¹Ħ":143272,"à¹Ģà¸Ĺà¹Īาà¹Ħหร":143273,"à¹Ģà¸Ĺà¹Īาà¹Ħหรà¹Ī":143274,"ĠинÑĤеÑĢÑĮеÑĢ":143275,"ĠØŃاج":143276,"ĠØŃاجة":143277,"สีà¸Ĥาว":143278,"ìĸ¼":143279,"Ġná»Ļ":143280,"Ġná»Ļp":143281,"ĠÃŃnd":143282,"ĠÃŃndice":143283,"สำรวà¸Ī":143284,"Ġкаждой":143285,"Ġhotéis":143286,"ĠnastÄĻ":143287,"ĠnastÄĻpn":143288,"Ġ×Ķק×ķ×ĵ":143289,"Ġ×Ķק×ķ×ĵ×Ŀ":143290,"פ×ķפ":143291,"פ×ķפ×ķ׾":143292,"פ×ķפ×ķ׾ר×Ļ":143293,"вÑĪей":143294,"ãĤ·ãĥ³ãĥĹ":143295,"ãĤ·ãĥ³ãĥĹãĥ«":143296,"ĠzdjÄĻÄĩ":143297,"ĠгÑĢÑĥппа":143298,"ĠпомеÑī":143299,"ĠпомеÑīениÑı":143300,"ãģ©ãģĨãģĦãģĨ":143301,"ĠиÑģпÑĭÑĤа":143302,"ĠogÅĤ":143303,"ĠogÅĤos":143304,"ĠogÅĤoszen":143305,"ĠogÅĤoszeni":143306,"สรà¹īาà¸ĩสรร":143307,"สรà¹īาà¸ĩสรรà¸Ħà¹Į":143308,"à¸ŀรรà¸ĵ":143309,"ĠçıkÄ±ÅŁ":143310,"ĠÑĩаÑģÑĤноÑģÑĤи":143311,"Ġ×ķ×Ļ×ķתר":143312,"ç¶ļãģįãĤĴ":143313,"ç¶ļãģįãĤĴèªŃ":143314,"ç¶ļãģįãĤĴèªŃãĤĢ":143315,"à¸ģรั":143316,"à¸ģรัม":143317,"гÑĢаÑĦ":143318,"Ġвлад":143319,"ĠвладелÑĮ":143320,"ĠвладелÑĮÑĨ":143321,"ĠistediÄŁ":143322,"ĠistediÄŁiniz":143323,"×ij×ľ×¢":143324,"×ij×ľ×¢×ĵ×Ļ":143325,"ÙħÙĪاÙģ":143326,"ÙħÙĪاÙģÙĤØ©":143327,"Ġ×Ļ×ķר":143328,"Ġ×Ļ×ķרק":143329,"ãĤ«ãĥ¼ãĥīãĥŃãĥ¼ãĥ³":143330,"ĠاÙĦÙħØ´ÙĥÙĦ":143331,"ĠاÙĦÙħØ´ÙĥÙĦØ©":143332,"ĠêµŃíļĮ":143333,"ספ×ĺ":143334,"ספ×ĺ×ŀ":143335,"ספ×ĺ×ŀ×ijר":143336,"Ġìĸ´ëłµ":143337,"ÙĥاÙħ":143338,"ÙĥاÙħÙĬرا":143339,"schlü":143340,"schlüsse":143341,"ĠØ«ÙĨ":143342,"ĠØ«ÙĨائÙĬ":143343,"ìī½":143344,"ĠÐŀÑģоб":143345,"ĠÐŀÑģобенно":143346,"ĠинвеÑģÑĤи":143347,"ĠинвеÑģÑĤиÑĨи":143348,"اØŃتÙħ":143349,"اØŃتÙħاÙĦ":143350,"EÄŀ":143351,"EÄŀÄ°":143352,"íķĺê²łëĭ¤":143353,"Ġ×IJ×ijר×Ķ":143354,"Ġ×IJ×ijר×Ķ×Ŀ":143355,"Ġ×ij×Ĺ×Ļ׳×Ŀ":143356,"Ø£ÙĪض":143357,"Ø£ÙĪضاع":143358,"Ġdél":143359,"Ġdélai":143360,"Ġ×IJ×ķ×Ķ×ij×Ļ×Ŀ":143361,"ĠÑģоÑħ":143362,"ĠÑģоÑħÑĢ":143363,"ĠÑģоÑħÑĢани":143364,"ĠдоÑģÑĤиж":143365,"ĠдоÑģÑĤижени":143366,"สิà¹Īà¸ĩà¹ģ":143367,"สิà¹Īà¸ĩà¹ģวà¸Ķ":143368,"สิà¹Īà¸ĩà¹ģวà¸Ķล":143369,"สิà¹Īà¸ĩà¹ģวà¸Ķลà¹īà¸Ńม":143370,"ĠاÙĦÙħباشر":143371,"ĠÑĦиг":143372,"ĠÑĦигÑĥÑĢ":143373,"можем":143374,"׾×ŀ×Ļ×ĵ×Ķ":143375,"Ġciné":143376,"Ġcinéma":143377,"Ġbada":143378,"ĠbadaÅĦ":143379,"جبÙĩØ©":143380,"Ġдеп":143381,"ĠдепÑĥÑĤ":143382,"ĠдепÑĥÑĤаÑĤ":143383,"Ġdistância":143384,"ĠاÙĦÙħعار":143385,"ĠاÙĦÙħعارضة":143386,"thèse":143387,"ünc":143388,"üncü":143389,"Ġданного":143390,"ĠBelgi":143391,"ĠBelgië":143392,"Ġ×ij×ijק":143393,"Ġ×ij×ijקש×Ķ":143394,"ยà¹Īาà¸Ļ":143395,"Ġsolução":143396,"Ġ×Ķצ×ĺר":143397,"Ġ×Ķצ×ĺרפ×ķ":143398,"ĠØ£ÙĨØŃ":143399,"ĠØ£ÙĨØŃاء":143400,"ĠدÙħØ´":143401,"ĠدÙħØ´ÙĤ":143402,"มัà¹ī":143403,"มัà¹īย":143404,"Ùħغرب":143405,"استعÙħاÙĦ":143406,"ĠSÅĤow":143407,"ĠëıĻìĭľ":143408,"ĠëıĻìĭľìĹIJ":143409,"ĠÑģоÑģ":143410,"ĠÑģоÑģед":143411,"ì²ŃìĨĮ":143412,"ì²ŃìĨĮëħĦ":143413,"ĠгÑĢаÑĦ":143414,"ĠгÑĢаÑĦик":143415,"ĠìŀijìĿĢ":143416,"Ġyeti":143417,"ĠyetiÅŁtir":143418,"ĠìĿ´ê²ĥìĿ´":143419,"หà¹Īาà¸ĩ":143420,"Ø¥ÙħÙĥاÙĨ":143421,"Ø¥ÙħÙĥاÙĨÙĬØ©":143422,"استعراض":143423,"Ùħخدر":143424,"ĠÑĩÑĥÑĤÑĮ":143425,"ÙħدÙĬر":143426,"ÙħدÙĬرÙĬØ©":143427,"Ġà¹Ģมษ":143428,"Ġà¹Ģมษายà¸Ļ":143429,"ĠмеÑħ":143430,"ĠмеÑħаниз":143431,"ĠмеÑħанизм":143432,"ĠÑģÑĥм":143433,"ĠÑģÑĥммÑĥ":143434,"Ġvö":143435,"Ġvöll":143436,"Ġvöllig":143437,"ĠдÑĢÑĥз":143438,"ĠдÑĢÑĥзÑĮÑı":143439,"ãĤĴåĪ©çĶ¨ãģĹãģ¦":143440,"à¸ļรรà¸Īุ":143441,"pożycz":143442,"×ŀש׼":143443,"×ŀ×©×Ľ×ł×ª":143444,"×ŀ×©×Ľ×ł×ª×IJ":143445,"Ġeuropéen":143446,"Ġproprié":143447,"Ġpropriétaire":143448,"Ġkhấu":143449,"ãģĦãģŁãģłãģijãĤĭ":143450,"Ġtecrü":143451,"Ġtecrübe":143452,"×Ķ×ij":143453,"×Ķ×ij׳×Ķ":143454,"ĠcuÌ":143455,"ĠcuÌī":143456,"ĠcuÌīa":143457,"×IJ×ķ×ķ":143458,"×IJ×ķ×ķ×Ļר×Ķ":143459,"Ġ׼×ķ׾×ķ":143460,"Ulus":143461,"Uluslararası":143462,"Ġ׳×ķת":143463,"Ġ׳×ķ×ª×Ł":143464,"ãģ«åIJij":143465,"ãģ«åIJijãģijãģ¦":143466,"ë¹Ľ":143467,"à¸Ĺัà¸ģษ":143468,"à¸Ĺัà¸ģษะ":143469,"سÙĤÙĪ":143470,"سÙĤÙĪØ·":143471,"Ġвн":143472,"ĠвнеÑĪ":143473,"ĠвнеÑĪне":143474,"Ġurz":143475,"ĠurzÄĻd":143476,"Ġámb":143477,"Ġámbito":143478,"à¸Ńà¸ĺิ":143479,"à¸Ńà¸ĺิà¸ļาย":143480,"ĠÅĤad":143481,"ĠÅĤadn":143482,"ê±´ì¶ķ":143483,"wództ":143484,"wództw":143485,"Ġquestões":143486,"Ġשק":143487,"Ġשק×Ļ×ij׾":143488,"ĠmiejscowoÅĽci":143489,"Ġвал":143490,"ĠвалÑİÑĤ":143491,"häuser":143492,"หà¸Ļà¸Ńà¸ĩ":143493,"ãģ¨åħ±":143494,"ãģ¨åħ±ãģ«":143495,"ãĥıãĥ¼ãĥī":143496,"Ġê°ľìµľ":143497,"ĠоÑģновном":143498,"ĠмÑıÑģ":143499,"اعت":143500,"اعتÙĤاÙĦ":143501,"สà¸ĸิ":143502,"สà¸ĸิà¸ķิ":143503,"Ngu":143504,"Nguá»ĵn":143505,"ĠÙħجÙĦ":143506,"ĠÙħجÙĦØ©":143507,"à¹ģà¸Ĥà¸Ļ":143508,"ĠاÙĦÙĦÙĬبÙĬ":143509,"פע×Ļ׾×ķ×Ļ×ķת":143510,"Ġ×Ķרפ×ķ×IJ×Ļ":143511,"פר×ķפ":143512,"פר×ķפ×Ļ׾":143513,"ק׾×IJ":143514,"ק׾×IJס×Ļ":143515,"ÙĥتشÙģ":143516,"ãģ«ãģªãģ£ãģ¦ãģĹãģ¾ãģĨ":143517,"à¹Ģà¸Ħลà¹ĩà¸Ķ":143518,"à¹Ģà¸Ħลà¹ĩà¸Ķลัà¸ļ":143519,"Ġì»´":143520,"Ġì»´íĵ¨":143521,"Ġì»´íĵ¨íĦ°":143522,"Ġ×Ĺ×Ļ×ķ×ij×Ļ":143523,"Ġnäm":143524,"Ġnämlich":143525,"åij¼ãģ°":143526,"åij¼ãģ°ãĤĮ":143527,"ĠÑĢол":143528,"ĠÑĢоли":143529,"Ġspécialisé":143530,"à¸Ļวัà¸ķ":143531,"à¸Ļวัà¸ķà¸ģรรม":143532,"ÙĨصÙĪص":143533,"пеÑĢед":143534,"пеÑĢедаÑĩ":143535,"thèque":143536,"Ġר×IJ×Ļת×Ļ":143537,"ãĥĢãĤ¦ãĥ³":143538,"ãĤıãģĭ":143539,"ãĤıãģĭãģ£ãģ¦":143540,"беÑĢеж":143541,"ĠÑģек":143542,"ĠÑģекÑĢ":143543,"ĠÑģекÑĢеÑĤ":143544,"ĠпоÑģÑĤоÑıнн":143545,"à¸Ĥà¸Ļสà¹Īà¸ĩ":143546,"Ġmük":143547,"Ġmükem":143548,"Ġmükemmel":143549,"еÑĤеÑģÑĮ":143550,"ĠاÙĦسÙĨÙĪات":143551,"ĠìłĦíĺĢ":143552,"Ġ×Ķ×ŀק×ķר×Ļ":143553,"Ġmüd":143554,"Ġmüdah":143555,"Ġmüdahale":143556,"Ġwyb":143557,"Ġwybór":143558,"Ġtendência":143559,"إدار":143560,"إدارÙĬØ©":143561,"Ġunterstützen":143562,"ת×ijר":143563,"ת×ijרר":143564,"Ġdiá":143565,"Ġdiálogo":143566,"ĠÃĸnce":143567,"ĠÃĸnceki":143568,"ãĤ¹ãĥĿãĥĥãĥĪ":143569,"ëĦ£":143570,"ĠGeli":143571,"ĠGeliÅŁ":143572,"ãĤĴéĢļ":143573,"ãĤĴéĢļãģĹãģ¦":143574,"ĠFuÃŁball":143575,"Ġsalari":143576,"Ġsalarié":143577,"ĠпÑĢодÑĥкÑĤов":143578,"صÙģÙĤØ©":143579,"รวà¸ļ":143580,"รวà¸ļรวม":143581,"à¹ĥà¸Ļà¸IJาà¸Ļ":143582,"à¹ĥà¸Ļà¸IJาà¸Ļะ":143583,"Ġkayna":143584,"Ġkaynaģı":143585,"ĠìŀijíĴĪ":143586,"ĠвÑĭÑĢаж":143587,"ĠвÑĭÑĢажен":143588,"ĠÑģÑĤеп":143589,"ĠÑģÑĤепени":143590,"ĠاÙĦÙħÙĪجÙĪد":143591,"ĠاÙĦÙħÙĪجÙĪدة":143592,"ลà¹īม":143593,"ĠnajczÄĻ":143594,"ĠnajczÄĻÅĽcie":143595,"ĠnajczÄĻÅĽciej":143596,"Ġzwy":143597,"Ġzwyk":143598,"ĠzwykÅĤ":143599,"Ġê·¸ëłĩì§Ģ":143600,"à¸ģระà¸Ī":143601,"à¸ģระà¸Īาย":143602,"Ġëĭµ":143603,"Ġëĭµë³Ģ":143604,"ĠÑĢеак":143605,"ĠÑĢеакÑĨи":143606,"ĠÅĽwież":143607,"ĠÑģÑĤоимоÑģÑĤи":143608,"ÙħÙĨاÙĤ":143609,"ÙħÙĨاÙĤØ´":143610,"ÙħÙĨاÙĤشة":143611,"ĠÑħоÑĩÑĥ":143612,"ãĥľãĥ¼ãĥī":143613,"Ġróżnic":143614,"ĠкÑĢÑĭ":143615,"ĠкÑĢÑĭÑĪ":143616,"âľĵ":143617,"ãĤ³ãĥ³ãĥĨãĥ³":143618,"ãĤ³ãĥ³ãĥĨãĥ³ãĥĦ":143619,"ĠпÑĢедпоÑĩ":143620,"×ŀר×ij×Ļת":143621,"ĠØ´Ùĥ":143622,"ĠØ´Ùĥرا":143623,"Ġдал":143624,"Ġдалек":143625,"Ġдалеко":143626,"برÙĬØ·":143627,"برÙĬطاÙĨÙĬا":143628,"عÙĨا":143629,"عÙĨاÙĬØ©":143630,"ĠÑĢаÑģÑģказ":143631,"ĠÑĢаÑģÑģказÑĭва":143632,"Ø£ÙĦÙĪ":143633,"Ø£ÙĦÙĪاÙĨ":143634,"æĮģãģ£ãģ¦":143635,"æĮģãģ£ãģ¦ãģĦ":143636,"Ùħبادئ":143637,"×Ķ×¢×ijר":143638,"×Ķ×¢×ijרת":143639,"Ġyayı":143640,"Ġyayıml":143641,"Ġyayımla":143642,"mát":143643,"máticos":143644,"à¸ģัà¸ĩ":143645,"à¸ģัà¸ĩวล":143646,"Ġ×ľ×¤×ª":143647,"Ġ×ľ×¤×ª×ķ×Ĺ":143648,"à¸ŀฤà¸ķิ":143649,"à¸ŀฤà¸ķิà¸ģรรม":143650,"íĤ¬":143651,"ĠокÑĢÑĥг":143652,"Ġ×ŀצ×ķ×ķ×Ķ":143653,"ÐĽÐµÐ½Ð¸":143654,"ÐĽÐµÐ½Ð¸Ð½":143655,"ĠTriá»ģu":143656,"ãĤ³ãĥŁãĥ¥":143657,"ãĤ³ãĥŁãĥ¥ãĥĭ":143658,"ãĤ³ãĥŁãĥ¥ãĥĭãĤ±":143659,"ãĤ³ãĥŁãĥ¥ãĥĭãĤ±ãĥ¼ãĤ·ãĥ§ãĥ³":143660,"ÙĥÙĨÙĬ":143661,"ÙĥÙĨÙĬسة":143662,"ãĤĴä¸Ńå¿ĥ":143663,"ãĤĴä¸Ńå¿ĥãģ«":143664,"ĠmiÄĻdz":143665,"ĠmiÄĻdzyn":143666,"ĠmiÄĻdzynar":143667,"ĠmiÄĻdzynarod":143668,"ĠmiÄĻdzynarodow":143669,"ÙĦÙĨ":143670,"ÙĦÙĨدا":143671,"برش":143672,"برشÙĦÙĪÙĨ":143673,"برشÙĦÙĪÙĨØ©":143674,"à¸ģระà¸ķุ":143675,"à¸ģระà¸ķุà¹īà¸Ļ":143676,"Ġgı":143677,"Ġgıda":143678,"à¸Ľà¸£à¸°à¸Ĺัà¸ļ":143679,"à¸Ľà¸£à¸°à¸Ĺัà¸ļà¹ĥà¸Ī":143680,"Ġë¶Ī구":143681,"Ġë¶Ī구íķĺê³ł":143682,"ĠÙĨØ·":143683,"ĠÙĨطاÙĤ":143684,"ĠÐľÐ¾Ð¶ÐµÑĤ":143685,"Präs":143686,"Präsident":143687,"ĠÑģкоÑĢ":143688,"ĠÑģкоÑĢоÑģÑĤÑĮ":143689,"Ġ×Ķ×ij×ķקר":143690,"еÑħаÑĤÑĮ":143691,"Ġgạo":143692,"Ġש×IJ×Ļ׳×Ŀ":143693,"Ġ×ij׳×ķ×Ĵ":143694,"Ġ×ij׳×ķ×Ĵ×¢":143695,"ĠопиÑģание":143696,"Ġuczni":143697,"Ġuczniów":143698,"à¹Ģà¸Ńà¹ĩà¸Ļ":143699,"Ġتش":143700,"ĠتشرÙĬÙĨ":143701,"Ġnhãn":143702,"빨":143703,"Ġcaractère":143704,"×¢×ľ×Ļ":143705,"×¢×ľ×Ļ×Ļ×Ķ":143706,"楽ãģĹãĤģãĤĭ":143707,"ĠÑģаÑħ":143708,"ĠÑģаÑħаÑĢ":143709,"дÑĥмаÑĤÑĮ":143710,"ĠÐĴозможно":143711,"صÙĬاÙĨ":143712,"صÙĬاÙĨØ©":143713,"ömür":143714,"สล":143715,"สลà¹ĩ":143716,"สลà¹ĩà¸Ń":143717,"สลà¹ĩà¸Ńà¸ķ":143718,"롯":143719,"Ġthói":143720,"grÃ¶ÃŁe":143721,"ĠksiÄĻ":143722,"ĠksiÄĻg":143723,"ĠÑĢом":143724,"ĠÑĢоман":143725,"ÙĤاسÙħ":143726,"×ŀ×ij×ķ×Ĵ":143727,"×ŀ×ij×ķ×Ĵר×Ļ×Ŀ":143728,"besch":143729,"beschäft":143730,"beschäftig":143731,"×Ķצע×Ķ":143732,"ĠÃģrea":143733,"ĠзаÑıвк":143734,"Ĺ":143735,"ĠлÑİбого":143736,"Ġม":143737,"Ġมà¸ģร":143738,"Ġมà¸ģราà¸Ħม":143739,"ÑĦиз":143740,"ÑĦизиÑĩеÑģк":143741,"инÑĦ":143742,"инÑĦек":143743,"инÑĦекÑĨи":143744,"اÙĦØ·":143745,"اÙĦطائÙģ":143746,"Ġколл":143747,"ĠколлекÑĤив":143748,"езжа":143749,"ĠسبØŃ":143750,"ĠسبØŃاÙĨ":143751,"ĠسبØŃاÙĨÙĩ":143752,"schlä":143753,"schläge":143754,"Ġди":143755,"Ġдиаг":143756,"ĠдиагноÑģÑĤ":143757,"ĠоÑĤмеÑĤиÑĤÑĮ":143758,"ТЬ":143759,"ĠاÙĦدر":143760,"ĠاÙĦدراسÙĬ":143761,"עצ×ŀ":143762,"עצ×ŀ×IJ×ķת":143763,"Ġdémarch":143764,"Ġdémarche":143765,"Ġ×ĺ×ķ×¢":143766,"Ġ×ĺ×ķ×¢×Ł":143767,"Ġfuncionários":143768,"ỵ":143769,"׾׼×IJ":143770,"׾׼×IJ×ķר×Ķ":143771,"à¸ĭà¹Ī":143772,"à¸ĭà¹Īà¸Ńม":143773,"ĠÑĩÑĥв":143774,"ĠÑĩÑĥвÑģÑĤво":143775,"âĸ¼":143776,"пÑĥÑī":143777,"пÑĥÑīен":143778,"ĠмеÑĢ":143779,"ĠмеÑĢоп":143780,"ĠмеÑĢопÑĢи":143781,"ĠмеÑĢопÑĢиÑıÑĤиÑı":143782,"Ġuçu":143783,"ĠuçuÅŁ":143784,"ãĤĴåĪ©çĶ¨ãģĻãĤĭ":143785,"aÄŁ":143786,"aÄŁlı":143787,"ìĺĪìĪł":143788,"à¹ģยà¹Ī":143789,"ĠاÙĦÙĥÙħ":143790,"ĠاÙĦÙĥÙħبÙĬ":143791,"ĠاÙĦÙĥÙħبÙĬÙĪتر":143792,"تÙĪÙĬ":143793,"تÙĪÙĬتر":143794,"à¹Ģà¸Ĭีà¹Īยว":143795,"à¹Ģà¸Ĭีà¹Īยวà¸Ĭา":143796,"à¹Ģà¸Ĭีà¹Īยวà¸Ĭาà¸į":143797,"á»Ķ":143798,"Ġhiếm":143799,"ذاÙĥرة":143800,"Ġ×Ķ×ŀ×Ļ×ķ×Ĺ×ĵ":143801,"ĠìĪľ":143802,"ĠìĪľê°Ħ":143803,"ĠKı":143804,"ĠKısa":143805,"ĠgeleceÄŁi":143806,"пÑĢоÑĦеÑģÑģиона":143807,"пÑĢоÑĦеÑģÑģионал":143808,"Ġogó":143809,"Ġogóle":143810,"ĠgÅĤów":143811,"ĠgÅĤówne":143812,"ĠÑģÑĤилÑĮ":143813,"×IJפ׾":143814,"×IJפ׾×Ļק":143815,"×IJפ׾×Ļקצ×Ļ×Ķ":143816,"สมารà¹Į":143817,"สมารà¹Įà¸Ĺ":143818,"สมารà¹Įà¸Ĺà¹Ĥà¸Ł":143819,"สมารà¹Įà¸Ĺà¹Ĥà¸Łà¸Ļ":143820,"Ġthánh":143821,"ÐŁÐ¾Ð´":143822,"ÐŁÐ¾Ð´ÑĢоб":143823,"ÐŁÐ¾Ð´ÑĢобнее":143824,"ĠاÙĦتÙĪÙĨ":143825,"ĠاÙĦتÙĪÙĨسÙĬ":143826,"Ġbahçe":143827,"à¹ģà¸ģà¹īà¸Ľà¸±à¸įหา":143828,"éducation":143829,"europ":143830,"europä":143831,"europäische":143832,"ĠKsi":143833,"ĠKsiÄĻ":143834,"ĠëĦĺ":143835,"ĠëĦĺìĸ´":143836,"Ġvüc":143837,"Ġvücud":143838,"Ġyayg":143839,"Ġyaygın":143840,"Ġniekt":143841,"Ġniektóry":143842,"Ġniektórych":143843,"ãģŃãģĩ":143844,"Ġкаж":143845,"ĠкажеÑĤÑģÑı":143846,"каж":143847,"кажеÑĤ":143848,"ĠاÙĦدÙĬÙħÙĤرا":143849,"ĠاÙĦدÙĬÙħÙĤراط":143850,"ĠاÙĦدÙĬÙħÙĤراطÙĬØ©":143851,"æŃ©":143852,"æŃ©ãģĦãģ¦":143853,"Ġvaz":143854,"Ġvazge":143855,"Ġvazgeç":143856,"ĠминималÑĮ":143857,"ĠминималÑĮн":143858,"ãĥijãĤ¿":143859,"ãĥijãĤ¿ãĥ¼ãĥ³":143860,"ĠëĬ":143861,"ĠëĬIJ":143862,"ĠëĬIJëĤĮ":143863,"ãģ¡ãĤĩãģĨ":143864,"ãģ¡ãĤĩãģĨãģ©":143865,"Ġà¸ģร":143866,"Ġà¸ģรà¸ģà¸İ":143867,"Ġà¸ģรà¸ģà¸İาà¸Ħม":143868,"تجدÙĬد":143869,"ĠشاÙħÙĦ":143870,"หลัà¸ģà¸IJาà¸Ļ":143871,"ĠмаÑĢÑĪ":143872,"ĠмаÑĢÑĪÑĢÑĥÑĤ":143873,"ĠvÃŃt":143874,"ĠvÃŃtima":143875,"Ġquizá":143876,"aygı":143877,"×ĵ×ijר×Ļ×ķ":143878,"Ġизд":143879,"Ġиздели":143880,"ĠизделиÑı":143881,"пла":143882,"плаÑĩ":143883,"плаÑĩива":143884,"ä»»ãģĽ":143885,"Ġéquipé":143886,"ä¹ħãģĹãģ":143887,"ä¹ħãģĹãģ¶":143888,"ä¹ħãģĹãģ¶ãĤĬ":143889,"ĠкаÑĤ":143890,"ĠкаÑĤал":143891,"ĠкаÑĤалог":143892,"สà¹īม":143893,"ĠÑĢей":143894,"ĠÑĢейÑĤ":143895,"ĠÑĢейÑĤинг":143896,"Ġthuyá»ģn":143897,"ĠاÙĦÙħÙĤدس":143898,"espère":143899,"ãģ«åħ¥ãģ£ãģŁ":143900,"หมายà¹Ģลà¸Ĥ":143901,"ת×Ĺ×ķשת":143902,"à¸Ļà¹Īะ":143903,"ĠpeÅĤ":143904,"ĠpeÅĤne":143905,"Ġpérd":143906,"Ġpérdida":143907,"หมวà¸Ķ":143908,"หมวà¸Ķหมูà¹Ī":143909,"иÑĩеÑģкÑĥÑİ":143910,"çµĤãĤı":143911,"çµĤãĤıãģ£ãģŁ":143912,"Ġ×Ĵ×ķ×Ĵ׾":143913,"à¸Ĺำà¸Ħวาม":143914,"à¸Ĺำà¸Ħวามสะà¸Ńาà¸Ķ":143915,"Hotéis":143916,"ĠзаÑĢ":143917,"ĠзаÑĢегиÑģÑĤ":143918,"ĠзаÑĢегиÑģÑĤÑĢи":143919,"ĠзаÑĢегиÑģÑĤÑĢиÑĢова":143920,"ĠÑģобÑĭÑĤи":143921,"ĠÑģобÑĭÑĤиÑı":143922,"Ġ×ĸ׼×IJ":143923,"ÙħÙĨظÙĪÙħØ©":143924,"Ġ×Ķ×ŀצ":143925,"Ġ×Ķ×ŀצ×Ļ×IJ×ķת":143926,"ÙħÙĥÙĪÙĨ":143927,"ÙħÙĥÙĪÙĨات":143928,"ä¸ĬãģĮãĤĭ":143929,"ĠmÄĻ":143930,"ĠmÄĻsk":143931,"หรืà¸Ńà¹Ģà¸Ľà¸¥à¹Īา":143932,"ëĤ®":143933,"Ġnoktas":143934,"Ġnoktası":143935,"ĠболÑĮÑĪим":143936,"ĠлÑĥÑĩÑĪиÑħ":143937,"Ø´ÙĩÙĬد":143938,"à¸Ńำà¸Ļ":143939,"à¸Ńำà¸Ļวย":143940,"à¸Ńำà¸Ļวยà¸Ħวาม":143941,"à¸Ńำà¸Ļวยà¸Ħวามสะà¸Ķวà¸ģ":143942,"Ġев":143943,"ĠевÑĢ":143944,"ĠевÑĢоп":143945,"ĠевÑĢопей":143946,"à¸īาย":143947,"ìĦŃ":143948,"ÙħÙģا":143949,"ÙħÙģاÙĪض":143950,"ÙħÙģاÙĪضات":143951,"ë¹Į":143952,"赤ãģ¡ãĤĥãĤĵ":143953,"ĠÑĥдалоÑģÑĮ":143954,"ĠХоÑĤ":143955,"ĠХоÑĤÑı":143956,"przedsiÄĻbiorc":143957,"ĠHôm":143958,"íķĺìĺĢìĬµëĭĪëĭ¤":143959,"Ġнаг":143960,"ĠнагÑĢÑĥз":143961,"ĠнагÑĢÑĥзк":143962,"Ġ×ij×Ļ׳׾×IJ×ķ×ŀ×Ļ":143963,"Ġê°ĢëĬ¥íķľ":143964,"ĠHữu":143965,"à¸Ńุà¸Ķ":143966,"à¸Ńุà¸Ķม":143967,"ת×ķפ":143968,"ת×ķפע×Ķ":143969,"ĠmiÅĤo":143970,"ĠmiÅĤoÅĽci":143971,"ksiÄħż":143972,"ksiÄħżka":143973,"ĠاÙĦÙĦعبة":143974,"à¸īาà¸ģ":143975,"สะสม":143976,"×ŀתר":143977,"×ŀתר×Ĺש":143978,"Ġlégère":143979,"Ġ׾צפ":143980,"Ġ׾צפ×Ļ×Ķ":143981,"ĠиÑģÑĤоÑĢиÑı":143982,"ĠãĥĪãĥ©":143983,"ĠãĥĪãĥ©ãĥĥãĤ¯":143984,"ĠãĥĪãĥ©ãĥĥãĤ¯ãĥIJãĥĥãĤ¯":143985,"Ġка":143986,"ĠкаÑĦе":143987,"×ŀס×ŀ×ļ":143988,"Ġcüm":143989,"Ġcümle":143990,"à¹Ģà¸Ħลืà¹Īà¸Ńà¸Ļà¹Ħหว":143991,"ãģĬãģĿ":143992,"ãģĬãģĿãĤīãģı":143993,"ìŀIJëıĻ":143994,"ìŀIJëıĻì°¨":143995,"à¸Ńัà¸ķ":143996,"à¸Ńัà¸ķà¹Ĥà¸Ļ":143997,"à¸Ńัà¸ķà¹Ĥà¸Ļมั":143998,"à¸Ńัà¸ķà¹Ĥà¸Ļมัà¸ķิ":143999,"ĠÅŁik":144000,"ĠÅŁikay":144001,"ĠÅŁikayet":144002,"extrême":144003,"krä":144004,"kräfte":144005,"ëĤĻ":144006,"íķij":144007,"ì²Ļ":144008,"íĺĪ":144009,"ì°į":144010,"âĻ¡":144011,"ìŀĶ":144012,"뢰":144013,"íĿĶ":144014,"íĿIJ":144015,"âĩĴ":144016,"맼":144017,"ìĬĪ":144018,"á»Ĵ":144019,"ìĺµ":144020,"âĹİ":144021,"íĤ¨":144022,"ê¿Ī":144023,"ìĪ¨":144024,"ìĽ¨":144025,"맥":144026,"ï½Ģ":144027,"Ｊ":144028,"Ẩ":144029,"ãħİ":144030,"ÑĹ":144031,"ìĦ¬":144032,"ì¹¼":144033,"Ｖ":144034,"ìĽł":144035,"룴":144036,"Åĥ":144037,"ëĤ¼":144038,"ëĭIJ":144039,"âĢ¹":144040,"ë¦Ń":144041,"ì§IJ":144042,"âĢ¤":144043,"Ãħ":144044,"뾨":144045,"íĦ¸":144046,"íľĺ":144047,"ê²ģ":144048,"ë´ħ":144049,"Ãĺ":144050,"ëŃĶ":144051,"ëĺij":144052,"âĹĩ":144053,"ìĹĺ":144054,"ï»´":144055,"맹":144056,"ï¾Ŀ":144057,"ìĬ·":144058,"íĥķ":144059,"ï¼ł":144060,"ì»´":144061,"ëłĮ":144062,"ì½ľ":144063,"ﻹ":144064,"ãħł":144065,"졸":144066,"ëħ¹":144067,"âĤº":144068,"âĸ¶":144069,"íĥIJ":144070,"êµ´":144071,"íij¸":144072,"ÑĶ":144073,"íĶ½":144074,"Ðħ":144075,"ë°¤":144076,"Ôģ":144077,"첨":144078,"ì¶ĺ":144079,"ë²Ĺ":144080,"멸":144081,"ï¼»":144082,"ï¼½":144083,"ï¼·":144084,"ì°Į":144085,"ÃĴ":144086,"íı´":144087,"ìĵ¸":144088,"ì´Į":144089,"ëģĶ":144090,"ëĶ©":144091,"ëĩĮ":144092,"ë©Ģ":144093,"벨":144094,"ï¼µ":144095,"맡":144096,"ëĭ«":144097,"฿":144098,"ãģ±":144099,"ìĩ¼":144100,"ìºł":144101,"뮤":144102,"ê±±":144103,"컬":144104,"âĦĥ":144105,"ëĶ±":144106,"ëĥĪ":144107,"ìĭ±":144108,"íĻĪ":144109,"ëŀIJ":144110,"ìħĢ":144111,"ìłł":144112,"ÐĨ":144113,"ëłī":144114,"ï½ħ":144115,"ï½ı":144116,"íĻĢ":144117,"뼰":144118,"á»®":144119,"íĤ¹":144120,"ê½ĥ":144121,"ﻤ":144122,"ïºĶ":144123,"꺼":144124,"ìķī":144125,"âĻ¦":144126,"ï½ģ":144127,"ìĵ´":144128,"ãĢī":144129,"ì°®":144130,"ì¤ĺ":144131,"Ừ":144132,"ëģĦ":144133,"ëIJ¨":144134,"ìķĮ":144135,"íĿĺ":144136,"íħIJ":144137,"ãĢĪ":144138,"겪":144139,"ëĭ¥":144140,"ê²¼":144141,"á»Į":144142,"맨":144143,"ëģĬ":144144,"벤":144145,"ëijĶ":144146,"íĿ¡":144147,"Ử":144148,"ë¬ĺ":144149,"ãģī":144150,"ëŀ«":144151,"íĶĪ":144152,"íħį":144153,"ìŀĥ":144154,"ï½ī":144155,"ìģľ":144156,"âĸ½":144157,"묻":144158,"âĸ³":144159,"Ｘ":144160,"ìģĺ":144161,"춰":144162,"ìĬ´":144163,"ìķ±":144164,"ìĩĦ":144165,"Ắ":144166,"ï´¿":144167,"ï´¾":144168,"âĤ½":144169,"ëĦĵ":144170,"룩":144171,"쳤":144172,"ê´ľ":144173,"ÃĻ":144174,"Ỿ":144175,"ï¿£":144176,"ëĵŃ":144177,"ë©ĺ":144178,"ê»´":144179,"ëł´":144180,"Ðĥ":144181,"묵":144182,"ì§Ŀ":144183,"ãģº":144184,"ðŁĺĤ":144185,"ëŀ¬":144186,"ìłĬ":144187,"ê´Ħ":144188,"ìŀĬ":144189,"íŀĮ":144190,"ìĦ¯":144191,"âĪĢ":144192,"âĸ¡":144193,"ëĢĮ":144194,"ëŀĻ":144195,"ï½ĥ":144196,"Ặ":144197,"ï¾Ħ":144198,"ïºĺ":144199,"ë¹¼":144200,"ÃĮ":144201,"âĸ·":144202,"ê¸į":144203,"ë©ĭ":144204,"ãģĥ":144205,"ìĺĨ":144206,"ìĺ®":144207,"몬":144208,"롤":144209,"볬":144210,"ëĬ¦":144211,"âĸª":144212,"ì¼ĵ":144213,"ìľĪ":144214,"짧":144215,"ï½½":144216,"ëĥī":144217,"ï¾Į":144218,"ëĺIJ":144219,"ï¼ĥ":144220,"á»Ħ":144221,"ì´¬":144222,"춤":144223,"ï¼¹":144224,"ï»Ń":144225,"âĤ«":144226,"ï½ĩ":144227,"ìĺ·":144228,"ëĸ¨":144229,"âī«":144230,"릿":144231,"⾨":144232,"Ù±":144233,"쯤":144234,"ê¹Ķ":144235,"ðŁĺĬ":144236,"ìĪ«":144237,"ê³±":144238,"êµ³":144239,"ï½ĭ":144240,"à¸Į":144241,"Äł":144242,"ëĶ¸":144243,"ë°ij":144244,"ìħĭ":144245,"íİ´":144246,"âľħ":144247,"íĥij":144248,"ëĪĩ":144249,"íı¼":144250,"ðŁĺį":144251,"ìĺĽ":144252,"ﻣ":144253,"Ñĺ":144254,"ì©Į":144255,"ë¦ħ":144256,"ìĿį":144257,"ｸ":144258,"ëįľ":144259,"ãģħ":144260,"íݼ":144261,"ëĭĿ":144262,"ë¿Į":144263,"ì¼°":144264,"ìĭ«":144265,"ë°¥":144266,"íĽĮ":144267,"ì¨Į":144268,"ë¹Ļ":144269,"ï½İ":144270,"ë´Ħ":144271,"ìĦ¹":144272,"ï½²":144273,"ìĮĵ":144274,"Òij":144275,"ë°į":144276,"ëłĢ":144277,"íĨ¤":144278,"ｯ":144279,"ë¤Ħ":144280,"꽤":144281,"ï½Ĵ":144282,"ìķ¨":144283,"ï½¼":144284,"ê¹IJ":144285,"íģIJ":144286,"âĦĸ":144287,"맺":144288,"ﺮ":144289,"ëħģ":144290,"겸":144291,"ï»ł":144292,"íĬľ":144293,"Ź":144294,"ë¥Ń":144295,"ëĪī":144296,"ï½Ķ":144297,"íĮ¬":144298,"ìŀĩ":144299,"ï¬ģ":144300,"ﻨ":144301,"ëij¥":144302,"ëŀĦ":144303,"Ù¬":144304,"íĭ´":144305,"ìŀī":144306,"Ú¾":144307,"ìĽħ":144308,"ï»®":144309,"ëĭī":144310,"âīª":144311,"âĹĦ":144312,"ëĪĮ":144313,"íĽ¼":144314,"ì¤į":144315,"Ÿ":144316,"줬":144317,"ì¾Į":144318,"ï½ĵ":144319,"ï¾Ĭ":144320,"ðŁı»":144321,"ï¾ī":144322,"Ðģ":144323,"íĺIJ":144324,"ï¾Ļ":144325,"꼬":144326,"íŀIJ":144327,"âĢ¥":144328,"ëŁŃ":144329,"ë§ŀ":144330,"ìĥ¤":144331,"ïºĴ":144332,"íĭ±":144333,"ë½ij":144334,"Ãķ":144335,"âĪļ":144336,"ëĤĦ":144337,"ê¹Ŀ":144338,"ëĨĪ":144339,"Ẻ":144340,"ìħĪ":144341,"ìĮį":144342,"âĢ¡":144343,"ï¼±":144344,"ìģ¨":144345,"âĺº":144346,"ëĴ·":144347,"ìĺ³":144348,"ðŁijį":144349,"몽":144350,"ëĤŃ":144351,"ïºŃ":144352,"ë©Ī":144353,"á»Ī":144354,"íķĢ":144355,"ëĭĻ":144356,"ë¦ĩ":144357,"ìķ¤":144358,"ìį¼":144359,"ãĥµ":144360,"Ñ£":144361,"ìľĹ":144362,"âŃIJ":144363,"ï¾ĺ":144364,"íŬ":144365,"ê¾¼":144366,"ìķĹ":144367,"ï»Į":144368,"ê±·":144369,"ëħķ":144370,"롱":144371,"ìķĬ":144372,"ï¾Ģ":144373,"ìĩł":144374,"íĮ©":144375,"ﺪ":144376,"ë§Ļ":144377,"＿":144378,"ê¿Ķ":144379,"íİľ":144380,"룸":144381,"íĶĶ":144382,"ﻳ":144383,"ëıķ":144384,"ìĭ¼":144385,"á»İ":144386,"ë§ĺ":144387,"ì¢ĭ":144388,"íĨ¡":144389,"ï½±":144390,"íĿij":144391,"Ỹ":144392,"ì¦Į":144393,"칸":144394,"ëŃĺ":144395,"ï¾Ĺ":144396,"ï»ĭ":144397,"íĬĢ":144398,"ë¥Ļ":144399,"콩":144400,"ëģĹ":144401,"ëį´":144402,"ìħľ":144403,"¸":144404,"ë»IJ":144405,"ìĥµ":144406,"ê²IJ":144407,"ëĵ¬":144408,"룰":144409,"ãħĭ":144410,"ìĹī":144411,"á»ĸ":144412,"ëĦĮ":144413,"ｶ":144414,"ë´ĩ":144415,"ëĤ³":144416,"ãĤľ":144417,"ëĸ»":144418,"íİĢ":144419,"ëį©":144420,"íķ¸":144421,"÷":144422,"ê¼¼":144423,"ëĶľ":144424,"ë°´":144425,"ë©į":144426,"âĹ¯":144427,"ìĹij":144428,"ìĻ¼":144429,"ïºij":144430,"ë¶ķ":144431,"롬":144432,"ï½Į":144433,"íĨ¨":144434,"ﺴ":144435,"ëłĺ":144436,"ê°¤":144437,"ìĪ²":144438,"Ñĵ":144439,"ìħī":144440,"ï»ĵ":144441,"ëĪĶ":144442,"ëį§":144443,"âĢ¼":144444,"ﻲ":144445,"ê°±":144446,"ê¿Ģ":144447,"ëĭ·":144448,"Ẹ":144449,"Ẫ":144450,"ÆĴ":144451,"ëį¤":144452,"ìĪŃ":144453,"ï½Ĥ":144454,"ï½Ī":144455,"Åł":144456,"룬":144457,"ѵ":144458,"ëĸ¡":144459,"ëĥĦ":144460,"ìĦ°":144461,"ëĵĪ":144462,"ï¾ĥ":144463,"ëĩ¨":144464,"ï½IJ":144465,"êµ½":144466,"ìĹ½":144467,"ëĤĢ":144468,"묶":144469,"ï½·":144470,"ìıŁ":144471,"íĺĶ":144472,"ê¼Ī":144473,"ëģĪ":144474,"ì¥IJ":144475,"ïºĹ":144476,"ÄĮ":144477,"ëĪł":144478,"ëĸ¼":144479,"íĢ´":144480,"âī¥":144481,"ëĭŃ":144482,"ì±Ļ":144483,"ê»ı":144484,"멤":144485,"ìĥĺ":144486,"ëį®":144487,"룡":144488,"ìĤ½":144489,"ãĪľ":144490,"Ĩ":144491,"âĢ§":144492,"ｺ":144493,"Ä£":144494,"ì¦ī":144495,"ï¼¼":144496,"Û©":144497,"âĪĻ":144498,"ë°ı":144499,"ë¹ħ":144500,"ðŁĺĽ":144501,"íĪ´":144502,"ðŁĴķ":144503,"ãĢĴ":144504,"ìŀĺ":144505,"ﺤ":144506,"ï½ĸ":144507,"ë©ľ":144508,"ë²¼":144509,"ëĿĦ":144510,"ëļľ":144511,"ï»ĺ":144512,"ìĥĮ":144513,"ï½Ħ":144514,"ì©Ķ":144515,"ï½Ļ":144516,"ﺩ":144517,"Ûŀ":144518,"âĺİ":144519,"ìł¤":144520,"ëIJ©":144521,"ÅĿ":144522,"âŀ¡":144523,"ﻧ":144524,"Ðı":144525,"ì«ĵ":144526,"ê³½":144527,"Éij":144528,"ãĥ²":144529,"ëĤ«":144530,"ë¦ī":144531,"ì¢ģ":144532,"ë°Ń":144533,"ðŁĺģ":144534,"ë¹µ":144535,"첩":144536,"컵":144537,"ðŁĺĺ":144538,"ë±ħ":144539,"âīĪ":144540,"ë¹ļ":144541,"ï»ľ":144542,"ðŁĻı":144543,"íģ°":144544,"ìĦŀ":144545,"ï¾ļ":144546,"ìĺ¹":144547,"ë¼Ī":144548,"ëĤ¯":144549,"ëŀ©":144550,"íļ¡":144551,"ï½ķ":144552,"íĥĵ":144553,"ëĿł":144554,"ê³ģ":144555,"ëĵĢ":144556,"ìĹł":144557,"Ｚ":144558,"ë§ij":144559,"ëĭ¿":144560,"쿨":144561,"ãİ¡":144562,"ÐĬ":144563,"íĦ±":144564,"Ũ":144565,"ﺳ":144566,"ï¾ı":144567,"âĭħ":144568,"ê¼´":144569,"âī¤":144570,"íĮģ":144571,"Ω":144572,"궤":144573,"ìĪį":144574,"âľ¿":144575,"콤":144576,"ëĪħ":144577,"íĨ±":144578,"ãħľ":144579,"áIJħ":144580,"ÅĴ":144581,"ðŁijī":144582,"ﻦ":144583,"Ъ":144584,"ë¥ľ":144585,"íķ«":144586,"ï¾ĭ":144587,"âĻ«":144588,"ê¹ľ":144589,"ë°¸":144590,"ëĶĺ":144591,"íĿī":144592,"ï¾ģ":144593,"ï¾Ľ":144594,"볼":144595,"ê²¹":144596,"쿼":144597,"ﻬ":144598,"âŀ¤":144599,"ðŁĻģ":144600,"ïºł":144601,"ëĨ¨":144602,"믹":144603,"ê¸ĭ":144604,"ë»Ķ":144605,"ê¹ĥ":144606,"ëijij":144607,"íĭ¸":144608,"íİĻ":144609,"âŀĸ":144610,"ãĥ½":144611,"ì§ļ":144612,"ｬ":144613,"ﻥ":144614,"íĮ½":144615,"âĢĴ":144616,"ìĮĢ":144617,"ìŃī":144618,"ëļ±":144619,"ãĤŀ":144620,"íĭĪ":144621,"ãĤIJ":144622,"ëīĺ":144623,"Σ":144624,"ê³°":144625,"ë¹Ĺ":144626,"ï¾İ":144627,"ðŁĺŃ":144628,"íĿł":144629,"ìĹ¿":144630,"ê°ļ":144631,"ì¤Į":144632,"맵":144633,"ï½³":144634,"ãģ¢":144635,"ï»Ĺ":144636,"âī¦":144637,"Ú¤":144638,"ëłģ":144639,"ê¼½":144640,"ﻫ":144641,"âī§":144642,"ì´Ľ":144643,"ìłĿ":144644,"Ằ":144645,"âĻ£":144646,"ìºĺ":144647,"âĪĩ":144648,"ê²ī":144649,"ë°Ł":144650,"ï»Ķ":144651,"íĸĩ":144652,"âĸĴ":144653,"ðŁijı":144654,"Ãŀ":144655,"ðŁĺĨ":144656,"ﺼ":144657,"âĿĹ":144658,"ìºĶ":144659,"칩":144660,"ëĸ¤":144661,"ëĥħ":144662,"âĶľ":144663,"ï½»":144664,"ÎĶ":144665,"áĥ¦":144666,"ìŀİ":144667,"âĺĢ":144668,"âĪ¼":144669,"ðŁĶ¥":144670,"ë°Į":144671,"ìłĸ":144672,"íĹĽ":144673,"Îķ":144674,"ïºĥ":144675,"ë¶ī":144676,"âĪŀ":144677,"íĥŃ":144678,"Ãĭ":144679,"âģĦ":144680,"ãħĩ":144681,"ëĦ¥":144682,"ëĭ®":144683,"ëł·":144684,"íĮĿ":144685,"캡":144686,"ë·Ķ":144687,"ì©į":144688,"íĤ´":144689,"ëļ«":144690,"âĵĴ":144691,"íķį":144692,"âĻĤ":144693,"ï¾Ĩ":144694,"âĨ©":144695,"ìį©":144696,"ïºķ":144697,"íĿĻ":144698,"Ñľ":144699,"íĤ·":144700,"íĿ°":144701,"íĥ±":144702,"ëķIJ":144703,"ï¾Ĵ":144704,"×ĥ":144705,"ëĮĦ":144706,"ìĺ´":144707,"ìķµ":144708,"ê¹¥":144709,"ëŀŃ":144710,"쪼":144711,"ãİĿ":144712,"ðŁĺħ":144713,"ëıĭ":144714,"몫":144715,"ﺸ":144716,"뮬":144717,"ë²ħ":144718,"ëijł":144719,"ìħ°":144720,"ì»·":144721,"ëĶª":144722,"ëħĶ":144723,"ãħ¡":144724,"ìĶ»":144725,"íķı":144726,"ëį±":144727,"ﺨ":144728,"ï¾į":144729,"ï½µ":144730,"ì¢Ģ":144731,"íİĮ":144732,"ï»°":144733,"ﺣ":144734,"Æ£":144735,"ðŁ¤£":144736,"ï·º":144737,"ëĤļ":144738,"âĭĨ":144739,"ë³į":144740,"ðŁĺĦ":144741,"ìĸĢ":144742,"ìĻł":144743,"ëĨĶ":144744,"íŨ":144745,"ï»Ľ":144746,"ï»Ŀ":144747,"Ỷ":144748,"ìĸĺ":144749,"ìİĦ":144750,"ÚĨ":144751,"ï»ŀ":144752,"ëĢIJ":144753,"ê²Ķ":144754,"ﻵ":144755,"âŦ":144756,"íļŁ":144757,"ê¹ģ":144758,"ê°ĵ":144759,"ëĶ´":144760,"ìıĺ":144761,"ëļĿ":144762,"ỳ":144763,"ëŀ´":144764,"ëĦī":144765,"âĺŀ":144766,"ï½ĺ":144767,"Ž":144768,"ë¦İ":144769,"âĸ¬":144770,"ëŃī":144771,"âĩĽ":144772,"ìį¬":144773,"ïºŁ":144774,"Ëľ":144775,"ë¶ĵ":144776,"ìĽ°":144777,"Åľ":144778,"ëŃĩ":144779,"Ỳ":144780,"Ëļ":144781,"ëķĢ":144782,"âĺij":144783,"ðŁı¼":144784,"ìĸ½":144785,"âĮĴ":144786,"Ðİ":144787,"ɾ":144788,"íĮ¡":144789,"ï¾ħ":144790,"ìŀŃ":144791,"ｨ":144792,"칫":144793,"ìľĮ":144794,"ÒĽ":144795,"굿":144796,"ëĭ¦":144797,"âĶĶ":144798,"ï¾ij":144799,"ì§ĸ":144800,"ìºĦ":144801,"ãĢĥ":144802,"ʼ":144803,"ê²Ł":144804,"ｧ":144805,"Ä¢":144806,"íİł":144807,"맷":144808,"ê°ĩ":144809,"ìĭ¹":144810,"ðŁĴ¦":144811,"ï¾ľ":144812,"ëĬĻ":144813,"벡":144814,"Å¿":144815,"ðŁĺĭ":144816,"ðŁĴª":144817,"ì¿Ħ":144818,"ë©ķ":144819,"ìѤ":144820,"ëĬĦ":144821,"ðŁĮ¸":144822,"ãĤĿ":144823,"Çİ":144824,"ï½ļ":144825,"ÄĹ":144826,"ëģĵ":144827,"ê¶IJ":144828,"áµī":144829,"ãĥĤ":144830,"ê»į":144831,"ðŁĺ¦":144832,"ãĢĿ":144833,"ðŁ¤Ĺ":144834,"ÑŁ":144835,"ìĹİ":144836,"âľĮ":144837,"ìīIJ":144838,"ÃĨ":144839,"íĹIJ":144840,"ðŁİī":144841,"Îij":144842,"ï½Ń":144843,"ðŁĴĻ":144844,"ìĽ¬":144845,"íĢĺ":144846,"ﻢ":144847,"ðŁĺİ":144848,"íij¼":144849,"íĿ©":144850,"ï»Ħ":144851,"íħĢ":144852,"ëłIJ":144853,"쥬":144854,"Ðĭ":144855,"ìĥ·":144856,"뾬":144857,"ðŁĺĥ":144858,"ëĦ¬":144859,"륨":144860,"ìĽį":144861,"ï½Ĩ":144862,"ï½´":144863,"ãĥħ":144864,"Ãı":144865,"ﻪ":144866,"âĻł":144867,"ëĬ¬":144868,"ë±Ģ":144869,"ë°ĭ":144870,"ìĥĢ":144871,"ï½¾":144872,"ëĤ±":144873,"컸":144874,"ðŁĴĸ":144875,"ðŁijĮ":144876,"Ñŀ":144877,"짱":144878,"ËĨ":144879,"ðŁĵļ":144880,"âŃķ":144881,"ï¬Ĥ":144882,"ﻡ":144883,"ëij¬":144884,"íĪ¼":144885,"âĸ¸":144886,"ê°¯":144887,"ê¹ħ":144888,"ï½®":144889,"ëĺ¥":144890,"Ä¡":144891,"íĮŁ":144892,"ÐĮ":144893,"ìĨŁ":144894,"ïºĵ":144895,"ﻼ":144896,"ÃĽ":144897,"ãĥ¾":144898,"ëĮĵ":144899,"íĴĭ":144900,"ìķĵ":144901,"ï½¹":144902,"ëĤ¡":144903,"ðŁijĩ":144904,"Ẽ":144905,"ãĢŁ":144906,"ðŁĮŁ":144907,"íĥł":144908,"ãĢĨ":144909,"âĢŁ":144910,"ë¸IJ":144911,"ðŁĮ¹":144912,"ìł¼":144913,"ðŁĵĮ":144914,"ìĶ¬":144915,"âĹĢ":144916,"ðŁĴĵ":144917,"ê¹İ":144918,"ìĤIJ":144919,"ìĶĮ":144920,"ÑĽ":144921,"âĶĪ":144922,"ë²³":144923,"ãİŀ":144924,"Õ¡":144925,"íĤµ":144926,"ðŁ¤Ķ":144927,"ëĢĶ":144928,"ìĬIJ":144929,"íĻī":144930,"⾦":144931,"ëľ¯":144932,"ìł¯":144933,"ëĶ§":144934,"Φ":144935,"ËĪ":144936,"ìī¼":144937,"âĹĬ":144938,"ëľ©":144939,"ëľ°":144940,"ï¾IJ":144941,"ë¿Ķ":144942,"ìĹ®":144943,"ì·Į":144944,"ﺧ":144945,"ÎĴ":144946,"ëµĻ":144947,"ï»Ĭ":144948,"ì°Ķ":144949,"íİĦ":144950,"ðŁĴĹ":144951,"Ẵ":144952,"ì°¢":144953,"íľ¼":144954,"ê½Ĥ":144955,"ì±Ķ":144956,"ìī´":144957,"âĸ¾":144958,"íĪ°":144959,"ëĭĽ":144960,"âĿ£":144961,"ｪ":144962,"ðŁĴľ":144963,"Ëĺ":144964,"ãħ¤":144965,"âĨĹ":144966,"íĸĦ":144967,"âĻ¬":144968,"ìķ°":144969,"ïºľ":144970,"âī¡":144971,"ãĢĵ":144972,"ìij¥":144973,"íĮį":144974,"íīģ":144975,"ë»Ĺ":144976,"íľł":144977,"íľ©":144978,"âľĪ":144979,"íĢĦ":144980,"ìĸĩ":144981,"ì¢ĩ":144982,"íŀĻ":144983,"몹":144984,"ãĤĽ":144985,"ðŁĺ±":144986,"ëįŁ":144987,"à¹ħ":144988,"굶":144989,"Ù«":144990,"ìĶģ":144991,"âľª":144992,"ï¾Ī":144993,"ðŁĻĮ":144994,"âļ¡":144995,"Îļ":144996,"ì¼Ī":144997,"ï¾Ķ":144998,"ï¾Ĥ":144999,"êµī":145000,"ﺻ":145001,"ðŁĴĭ":145002,"á¹£":145003,"ÓĻ":145004,"ìĨľ":145005,"ìĹ£":145006,"âľ©":145007,"ìľĻ":145008,"ﺰ":145009,"Ẳ":145010,"ìŀ£":145011,"âĿĮ":145012,"âĺģ":145013,"ìķİ":145014,"Ľ":145015,"Ûģ":145016,"ãĦ±":145017,"ëŁ¿":145018,"íĮ¸":145019,"ê½ī":145020,"ìıł":145021,"ðŁįĢ":145022,"âĨĶ":145023,"ëŃ¡":145024,"ï»ģ":145025,"ï¼Ħ":145026,"ðŁĴ¥":145027,"âĺĽ":145028,"íĹ·":145029,"ëij¡":145030,"Îł":145031,"Τ":145032,"âĦĵ":145033,"ﺷ":145034,"ÎĻ":145035,"ëıĶ":145036,"짤":145037,"âĶĥ":145038,"ãĦ·":145039,"ÇĴ":145040,"ðŁ¥°":145041,"ëĶķ":145042,"ìļ¥":145043,"ì¸Ħ":145044,"íĽĶ":145045,"ïºĩ":145046,"ﺬ":145047,"ðŁĺ¢":145048,"빡":145049,"ìĶ¹":145050,"ų":145051,"ËĿ":145052,"íİij":145053,"ï¾ĵ":145054,"ðŁĴļ":145055,"ëĬij":145056,"꺾":145057,"íĨ°":145058,"ÿ":145059,"ÐĦ":145060,"ëĮIJ":145061,"ë½Ģ":145062,"ì·Ħ":145063,"ðŁĵį":145064,"ðŁĻĪ":145065,"âĹĪ":145066,"ê¿ĩ":145067,"ì¼Ħ":145068,"íİ«":145069,"ðŁĩ·":145070,"âĶĭ":145071,"âļł":145072,"ë±ī":145073,"ìį°":145074,"ìĻĪ":145075,"ɪ":145076,"ïºĭ":145077,"ðŁĺľ":145078,"ÎŁ":145079,"ðŁĻĤ":145080,"âļ½":145081,"ÅĪ":145082,"ë¹Ķ":145083,"íĮľ":145084,"à¹ı":145085,"ìĸ¹":145086,"íĪŃ":145087,"ðŁ¥ĩ":145088,"ãĦ´":145089,"ëĶ¥":145090,"ìŃĪ":145091,"âĪĨ":145092,"ëĸ³":145093,"ë±ĥ":145094,"ìŀ¦":145095,"ï»IJ":145096,"Îľ":145097,"⾧":145098,"Ïį":145099,"ìłĵ":145100,"âĹķ":145101,"ëĴĢ":145102,"ï»Ģ":145103,"ðŁĶ´":145104,"ê½ģ":145105,"ëĮĪ":145106,"ëİĮ":145107,"ãĤİ":145108,"â¦ģ":145109,"콧":145110,"ﯾ":145111,"âĿ¯":145112,"à¸ħ":145113,"ðŁĻĦ":145114,"âĿĢ":145115,"ðŁĶ¹":145116,"âĩIJ":145117,"êµµ":145118,"âĩĶ":145119,"ë¶IJ":145120,"ðŁĴĽ":145121,"ξ":145122,"íĥ¬":145123,"âĿĦ":145124,"Ò£":145125,"ãĢ°":145126,"âĪij":145127,"âĺ¼":145128,"âīł":145129,"Ò¯":145130,"ﺯ":145131,"꿨":145132,"âľĸ":145133,"Êĸ":145134,"íĢĢ":145135,"ê¾Ģ":145136,"íĹĿ":145137,"âĶ£":145138,"ãİľ":145139,"ëĶĽ":145140,"뾸":145141,"ﺫ":145142,"ê¿°":145143,"ðŁĩ¹":145144,"ÇIJ":145145,"ÛĴ":145146,"룻":145147,"ïºĸ":145148,"Ñļ":145149,"ëĬł":145150,"Ûķ":145151,"깡":145152,"ë¿ľ":145153,"ì²¼":145154,"ï¨ij":145155,"륵":145156,"ìį¸":145157,"íħħ":145158,"íij¹":145159,"ÖĢ":145160,"ï³Į":145161,"ãħ£":145162,"ìij¤":145163,"ì½ķ":145164,"ëķł":145165,"ðŁĮ¿":145166,"íĥĶ":145167,"ìĽģ":145168,"ζ":145169,"âŀľ":145170,"ìĬĺ":145171,"íĽĹ":145172,"멧":145173,"ìīĺ":145174,"Õ¶":145175,"á¹ĩ":145176,"ðŁİģ":145177,"ｿ":145178,"ï¼Ĥ":145179,"á¼IJ":145180,"âľķ":145181,"âŀ¢":145182,"ëĦ¨":145183,"컫":145184,"ì¯Ķ":145185,"ì°ľ":145186,"ðŁĴ°":145187,"íħĿ":145188,"ãİı":145189,"볶":145190,"Òĵ":145191,"âĨ³":145192,"ìĥ´":145193,"íģĺ":145194,"âĸĢ":145195,"ë²Ļ":145196,"à¸ĥ":145197,"ὶ":145198,"Äķ":145199,"â¬ĩ":145200,"ë¤ĺ":145201,"ðŁİµ":145202,"âľļ":145203,"ïºı":145204,"Ρ":145205,"âĹī":145206,"ðŁĴ«":145207,"ÐĪ":145208,"ìĸĦ":145209,"ì§Ļ":145210,"ï»ĥ":145211,"ðĿijĴ":145212,"ëŃĦ":145213,"âĿ¥":145214,"âĿĸ":145215,"âĺĿ":145216,"ʹ":145217,"ḥ":145218,"âĢ¿":145219,"ãħħ":145220,"ê¸ģ":145221,"ëķ¡":145222,"ëį¥":145223,"âĪ©":145224,"ê»Ħ":145225,"ë®Į":145226,"Ò±":145227,"âĪĹ":145228,"ëłĻ":145229,"ïºĮ":145230,"ËIJ":145231,"ðŁĺ³":145232,"ðŁij©":145233,"ðŁİ¶":145234,"쿵":145235,"ðŁ¤©":145236,"ê·¤":145237,"ëĮĶ":145238,"ïºIJ":145239,"Ïİ":145240,"춥":145241,"ï½Ĭ":145242,"á¹Ń":145243,"뤼":145244,"âĸ«":145245,"ì§ł":145246,"á¼Ģ":145247,"ê»ij":145248,"ëĮģ":145249,"íĢ¸":145250,"âĻĽ":145251,"ðŁĴŀ":145252,"âĸ°":145253,"ðĿijĸ":145254,"ëĿ¤":145255,"द":145256,"ì´ĺ":145257,"ðŁĺĩ":145258,"ëĶ¤":145259,"ÎĹ":145260,"ðŁĻĩ":145261,"ËĽ":145262,"ì©¡":145263,"âĪ§":145264,"Õ¥":145265,"ÑĻ":145266,"ëIJ¬":145267,"ëĸĦ":145268,"ðŁĮ·":145269,"ìĹĮ":145270,"ðŁĺ¥":145271,"ëĪ´":145272,"ï»ļ":145273,"ÉĽ":145274,"ïºĦ":145275,"ï»ı":145276,"ÅĮ":145277,"ë²ļ":145278,"ìĭ£":145279,"ïºĢ":145280,"Îĵ":145281,"ðŁĺĮ":145282,"ËĻ":145283,"ëŀı":145284,"ðŁĶ¸":145285,"ðŁĵ·":145286,"ëģ½":145287,"íģ½":145288,"ðŁĴ¡":145289,"ðŁĮ±":145290,"ëºı":145291,"ìģł":145292,"ìĥIJ":145293,"ëıĹ":145294,"츰":145295,"ëĪķ":145296,"ÎĿ":145297,"âģī":145298,"ðŁĮ¼":145299,"íĮł":145300,"âĭ¯":145301,"áĥĺ":145302,"⾤":145303,"ê±Ķ":145304,"íĮİ":145305,"ðŁĴ¯":145306,"ìıĻ":145307,"íĹī":145308,"ÙŃ":145309,"ì½°":145310,"ﺿ":145311,"ï»±":145312,"ì±Į":145313,"âĺķ":145314,"ðŁİĢ":145315,"ÄĿ":145316,"ë°§":145317,"ìĤ¿":145318,"áijķ":145319,"ðŁįĥ":145320,"âĩ¨":145321,"ÎĽ":145322,"맴":145323,"ë³ķ":145324,"áijIJ":145325,"âĸĵ":145326,"ðĿijľ":145327,"âĻ»":145328,"íĤ¥":145329,"Õ¸":145330,"ãĪ±":145331,"ëºĢ":145332,"첸":145333,"ïºĽ":145334,"ðŁıĨ":145335,"ðŁĩª":145336,"âĿĵ":145337,"ÄĢ":145338,"ì½¥":145339,"ðŁĩ§":145340,"á½·":145341,"âľĤ":145342,"ìŀ¼":145343,"李":145344,"ðŁĵ¸":145345,"âĻ¯":145346,"ÉĶ":145347,"ὸ":145348,"âĮª":145349,"ï»ĸ":145350,"不":145351,"âļ«":145352,"âĶĹ":145353,"ðŁĮĪ":145354,"ﻩ":145355,"ðŁĵ²":145356,"ÏĪ":145357,"ðŁĺ¡":145358,"ðĿijİ":145359,"ìľ½":145360,"짬":145361,"ì§Ĭ":145362,"á½³":145363,"ìĮ¤":145364,"ëĤį":145365,"âīĴ":145366,"ðŁij¨":145367,"âĺĺ":145368,"Ó©":145369,"âĤĵ":145370,"âĪĤ":145371,"ï¹ģ":145372,"ðŁĴIJ":145373,"íħĥ":145374,"ðŁı½":145375,"ê·Ħ":145376,"ðŁĺı":145377,"ðŁĮº":145378,"ðŁĺĶ":145379,"ｫ":145380,"âľİ":145381,"ëµĪ":145382,"ðŁĩ¸":145383,"âĢ£":145384,"âŀĶ":145385,"ëĺĺ":145386,"ìĥ¬":145387,"Êĥ":145388,"â¬ħ":145389,"ì©IJ":145390,"ðŁĻĨ":145391,"ðŁİĦ":145392,"ľ":145393,"⣶":145394,"áĥIJ":145395,"âĺ»":145396,"ì±ķ":145397,"ìģ©":145398,"ë½ķ":145399,"캣":145400,"ðŁijĪ":145401,"ðŁĻĭ":145402,"ï¾ĸ":145403,"Òļ":145404,"Õ«":145405,"ìĮĪ":145406,"벧":145407,"ðŁĩ®":145408,"ï½Ŀ":145409,"ðŁįģ":145410,"ìĹ¥":145411,"ij":145412,"ë½IJ":145413,"íį½":145414,"íĽij":145415,"âĤ¹":145416,"ãħģ":145417,"ìĶ½":145418,"ðŁĶģ":145419,"य":145420,"ê¾¹":145421,"ëīľ":145422,"âĹ¡":145423,"íķĮ":145424,"Îĺ":145425,"룹":145426,"ìĻĵ":145427,"ðŁĩ¦":145428,"ðŁijĢ":145429,"âĶĮ":145430,"ῦ":145431,"ëĦĽ":145432,"ìĦ£":145433,"ìŃĻ":145434,"ï±ł":145435,"Îŀ":145436,"Ê»":145437,"ῶ":145438,"âĿĿ":145439,"ê±Ģ":145440,"ëĸ´":145441,"ãĦ¹":145442,"ðŁĴİ":145443,"Ϲ":145444,"âĽħ":145445,"ï»ķ":145446,"ãĥ±":145447,"ï½Ľ":145448,"ëĮķ":145449,"ë¹½":145450,"ì¥Ķ":145451,"쿤":145452,"ðŁĸ¤":145453,"ÑĴ":145454,"ê¹į":145455,"ëİĢ":145456,"ìĭ¯":145457,"뻤":145458,"ðŁĵŀ":145459,"ðŁĵ£":145460,"ðŁĺĿ":145461,"ìį¹":145462,"ìĹ¡":145463,"ì°IJ":145464,"á½IJ":145465,"ï»Ī":145466,"âľį":145467,"Äı":145468,"ðŁĮŀ":145469,"âĦ¦":145470,"ê½Ŀ":145471,"ë»ĺ":145472,"ìĪ±":145473,"âĶĺ":145474,"ðŁĮ»":145475,"âĤ´":145476,"âŀ¨":145477,"íIJģ":145478,"ê¶Ī":145479,"âĺ¢":145480,"ðŁĺĪ":145481,"ｩ":145482,"âĦĹ":145483,"ê°Ń":145484,"ê°¸":145485,"ë»ij":145486,"쥴":145487,"컥":145488,"ï¤Ĭ":145489,"ï»Ĵ":145490,"ðŁĺķ":145491,"âĺĶ":145492,"ìĺIJ":145493,"ðŁļĹ":145494,"ëĹĦ":145495,"ë§ı":145496,"Õ½":145497,"âĸ»":145498,"⣵":145499,"ìī°":145500,"ï»ij":145501,"âĻ©":145502,"Î¥":145503,"ðŁĺ£":145504,"âĬĤ":145505,"ãħĤ":145506,"ìħ¸":145507,"íıĦ":145508,"âľ½":145509,"ì¦Ļ":145510,"âĸ£":145511,"ê±į":145512,"ê¿ĭ":145513,"ì«Ħ":145514,"ìºĩ":145515,"ðŁĩµ":145516,"ðŁijij":145517,"âľĺ":145518,"ðĿijĽ":145519,"ìį½":145520,"ìºī":145521,"וּ":145522,"ðŁĶº":145523,"âĦ®":145524,"íĥ¤":145525,"ðŁĩº":145526,"ðŁĴµ":145527,"íħ¨":145528,"ï½ij":145529,"Ψ":145530,"ìĥ¹":145531,"ìĸķ":145532,"ì¹µ":145533,"ðŁĵ±":145534,"व":145535,"ðŁijĬ":145536,"ðŁĴĦ":145537,"ðŁĴĿ":145538,"ãĮĶ":145539,"ìĻģ":145540,"Ðĩ":145541,"à®IJ":145542,"âĸ¹":145543,"á´Ľ":145544,"âĹĺ":145545,"뺨":145546,"íĥī":145547,"ìĸĮ":145548,"ðŁIJ¶":145549,"ãĤij":145550,"Ëĩ":145551,"Åı":145552,"á½¹":145553,"ìħ§":145554,"ï¹°":145555,"ðĿij¡":145556,"ðŁĶĿ":145557,"ðŁĺ»":145558,"ðŁĴĥ":145559,"ðŁ¤¦":145560,"ðŁįĴ":145561,"íĢµ":145562,"âľĨ":145563,"ë¹´":145564,"理":145565,"ï»Ļ":145566,"á´Ĺ":145567,"ðŁĮ´":145568,";":145569,"ëĮij":145570,"ì¨ĭ":145571,"쵸":145572,"ðŁİĪ":145573,"ðŁıł":145574,"á½±":145575,"ÛĨ":145576,"á¿ĸ":145577,"âĢĽ":145578,"ì°¼":145579,"íķ¥":145580,"íĹ´":145581,"ðŁĩ¬":145582,"ì°Ŀ":145583,"âĪł":145584,"ï¼ĩ":145585,"âĬĻ":145586,"âĿij":145587,"ëĦĭ":145588,"ëŀĹ":145589,"ë°ī":145590,"ìĹĬ":145591,"ì¢Ĩ":145592,"íĮ¥":145593,"ï°²":145594,"ðŁĵĸ":145595,"ðŁĺ®":145596,"âļª":145597,"ðŁĺļ":145598,"âĿŀ":145599,"ðĿijŁ":145600,"ðŁİĤ":145601,"Åķ":145602,"áIJĪ":145603,"꺽":145604,"ì±ł":145605,"ïºĿ":145606,"ê¿ī":145607,"áĥł":145608,"ðŁıĥ":145609,"ðŁĴ¸":145610,"âĿģ":145611,"âĹ¾":145612,"Úª":145613,"á¹ĥ":145614,"íĬ¬":145615,"ðŁĩ±":145616,"íİŃ":145617,"ðŁĺŀ":145618,"ë¾°":145619,"á¹Ľ":145620,"뼸":145621,"âĿĤ":145622,"êĴ³":145623,"âĶIJ":145624,"íĵ°":145625,"âŀł":145626,"ê´ĺ":145627,"ëħĺ":145628,"뻥":145629,"ì¾ħ":145630,"ðŁĺIJ":145631,"âĪª":145632,"ðŁijģ":145633,"âĪ´":145634,"âĹģ":145635,"ëºIJ":145636,"ìŀ¤":145637,"ì±Ĺ":145638,"ðŁı¾":145639,"Χ":145640,"á½»":145641,"âŀ¥":145642,"ìŁĪ":145643,"ï»ī":145644,"âĸĮ":145645,"ãĥ®":145646,"ðŁ¤¤":145647,"âĩĵ":145648,"ì¼ł":145649,"á´ı":145650,"맬":145651,"뻣":145652,"ðŁĴ¬":145653,"ðŁįĵ":145654,"ĸ":145655,"Ù¹":145656,"Ê¿":145657,"á½°":145658,"ëķľ":145659,"ì°¡":145660,"ì°»":145661,"íİį":145662,"ðŁİ¯":145663,"ðŁįĤ":145664,"ðŁij§":145665,"âĻ¢":145666,"áĨŀ":145667,"âĻ§":145668,"âļľ":145669,"âľī":145670,"ëĵ¦":145671,"ëŃ£":145672,"ìĪı":145673,"ìĵ±":145674,"ÅŃ":145675,"ÊĬ":145676,"âĴ¸":145677,"âĩ©":145678,"ðŁĴĶ":145679,"Õµ":145680,"Ðī":145681,"Ò»":145682,"맣":145683,"ìĽľ":145684,"ì¿¡":145685,"íĽħ":145686,"íĽ¤":145687,"ﺢ":145688,"âľĭ":145689,"âĪĪ":145690,"ðŁĮį":145691,"Êľ":145692,"ëĬª":145693,"ëĴ¹":145694,"ﺲ":145695,"âĸĦ":145696,"ãħĪ":145697,"ëļ¤":145698,"íİ©":145699,"âĪ¨":145700,"ðŁ¤ª":145701,"áĥļ":145702,"곶":145703,"íĬķ":145704,"ðŁĺ¬":145705,"âĪ«":145706,"ðŁijĭ":145707,"ÒIJ":145708,"íĬ¿":145709,"ðŁĶµ":145710,"ðŁĴ¨":145711,"ðŁĮĻ":145712,"ëĩ©":145713,"âľ³":145714,"ë¨ģ":145715,"ëºĦ":145716,"ìĻij":145717,"ìºħ":145718,"íıĪ":145719,"ðĿijĻ":145720,"ðŁĴĺ":145721,"ãİ¥":145722,"âĿı":145723,"âľ°":145724,"ﯿ":145725,"ëµIJ":145726,"ì¼IJ":145727,"ﺱ":145728,"Õ´":145729,"ï¬Ģ":145730,"âľ´":145731,"ðŁ¤Ń":145732,"ðŁijĨ":145733,"âĽĶ":145734,"ê·ĵ":145735,"ìĮĮ":145736,"ðŁ¤·":145737,"ÛĶ":145738,"ðŁ§¡":145739,"ðŁĺĵ":145740,"Îĸ":145741,"âı°":145742,"ê²ľ":145743,"ëĭ³":145744,"ëİħ":145745,"ë°Ī":145746,"ï®IJ":145747,"ðŁı¡":145748,"âĨª":145749,"âĵĶ":145750,"âľĬ":145751,"ϲ":145752,"ÜIJ":145753,"ðŁĩ³":145754,"ÖĤ":145755,"âľı":145756,"ìĸĹ":145757,"ì«Ļ":145758,"ðŁĺ²":145759,"ÄŃ":145760,"âĻŃ":145761,"âĶı":145762,"âĹĮ":145763,"ðŁĺ¯":145764,"áµĴ":145765,"íĬł":145766,"Ä·":145767,"Êģ":145768,"à¤Ł":145769,"á¹ģ":145770,"á¼°":145771,"á¿Ĩ":145772,"â«":145773,"⫸":145774,"ëį«":145775,"ì³ĩ":145776,"켤":145777,"íĽ¨":145778,"ðŁĴŁ":145779,"ÊĢ":145780,"ʳ":145781,"ëĵIJ":145782,"âķ°":145783,"âĿĩ":145784,"ÇĢ":145785,"ÇĶ":145786,"É´":145787,"âĺļ":145788,"âĺľ":145789,"ê¶Ĥ":145790,"ì«Ĵ":145791,"ì±Ī":145792,"ðŁĩ¨":145793,"ðŁİ¥":145794,"ðŁĵĿ":145795,"ħ":145796,"ðĿijIJ":145797,"ÛĪ":145798,"ब":145799,"ì¬IJ":145800,"íĹ¥":145801,"âĻ¨":145802,"ðŁį´":145803,"ï¹ı":145804,"Ëĭ":145805,"ðŁ¥º":145806,"âĸ¨":145807,"íĻĭ":145808,"âĪħ":145809,"ëģĻ":145810,"ëŀł":145811,"ìĨ¥":145812,"âĢĸ":145813,"ðŁ¤ĺ":145814,"ðŁIJ»":145815,"áµķ":145816,"ÇĿ":145817,"âĺı":145818,"ïºļ":145819,"ï»Ĥ":145820,"ðŁļ©":145821,"ìĪŁ":145822,"ËĬ":145823,"⤵":145824,"ðŁĴ§":145825,"ãħį":145826,"ë©©":145827,"Ƭ":145828,"Îĩ":145829,"âĩ§":145830,"âĵļ":145831,"ìĤ¯":145832,"ìĪ¯":145833,"ëĨĭ":145834,"âľ¯":145835,"ðŁļĢ":145836,"Úĺ":145837,"Ú¨":145838,"âľŃ":145839,"ê²ħ":145840,"íĮ°":145841,"íľĻ":145842,"ðŁĮĬ":145843,"ðŁİĵ":145844,"ðŁĺĻ":145845,"Ëĥ":145846,"ðŁĴģ":145847,"ðŁijİ":145848,"âĺ¹":145849,"ðŁĺ«":145850,"ðŁĴ»":145851,"ëĤµ":145852,"ìĿĬ":145853,"íĮ»":145854,"Ò³":145855,"á½²":145856,"âŀŀ":145857,"ëĤij":145858,"ëĿĪ":145859,"죤":145860,"ﻯ":145861,"ðŁĩ©":145862,"ðŁ¥³":145863,"âĴ¼":145864,"ðŁ¦ĭ":145865,"âĺĤ":145866,"ðŁĺ°":145867,"ðŁĻĥ":145868,"ðŁĺĴ":145869,"Ûİ":145870,"Ïķ":145871,"Ḥ":145872,"룽":145873,"ìĬ¥":145874,"ðĿijī":145875,"ÉIJ":145876,"ðŁįİ":145877,"âķ¯":145878,"âķ¹":145879,"າ":145880,"ï¾ł":145881,"ë¹ķ":145882,"ïºĨ":145883,"ʺ":145884,"Ó§":145885,"âĨł":145886,"ëĥĩ":145887,"ìİĪ":145888,"ìŁ¤":145889,"ï±¢":145890,"âķ¬":145891,"âĺł":145892,"ðŁİĬ":145893,"ãįį":145894,"ãİİ":145895,"âĺ°":145896,"âľĥ":145897,"ãħī":145898,"ë¯Ī":145899,"빤":145900,"ìıŃ":145901,"ðĿij¢":145902,"ðŁIJ¾":145903,"Åĭ":145904,"ðŁij¶":145905,"âĶĽ":145906,"ï¿¢":145907,"áĥ¡":145908,"ļ":145909,"ÅĨ":145910,"ÑIJ":145911,"ìĥĽ":145912,"ìĺĮ":145913,"챤":145914,"íħģ":145915,"íļĥ":145916,"ï³Ĭ":145917,"ðĿijĶ":145918,"ðŁĩ«":145919,"âĭ°":145920,"ðŁĺ¨":145921,"âĤ©":145922,"Õ¬":145923,"á¸į":145924,"á»´":145925,"âĨĺ":145926,"âĺ¯":145927,"ãħı":145928,"ìł¬":145929,"âĻĶ":145930,"ðŁĶĶ":145931,"ðŁĺł":145932,"ðŁĻĬ":145933,"à®ľ":145934,"á¹ħ":145935,"âĹIJ":145936,"âĿĪ":145937,"âŀ½":145938,"ìĥħ":145939,"ðĿijł":145940,"Æ¢":145941,"âĭĻ":145942,"ê°Ľ":145943,"ëĿµ":145944,"ë£Ł":145945,"ìıľ":145946,"ïºģ":145947,"ðŁĴŃ":145948,"âĬĥ":145949,"ðŁIJ°":145950,"ãħĮ":145951,"Üĵ":145952,"âŀķ":145953,"á½ģ":145954,"ìķ³":145955,"ðĿijĿ":145956,"ðŁİ¬":145957,"É¡":145958,"à¤Ĺ":145959,"áIJī":145960,"ì©ľ":145961,"춧":145962,"ï³ī":145963,"ï»ħ":145964,"ðĿIJŀ":145965,"श":145966,"ðŁĵ¢":145967,"ðŁįĭ":145968,"ðŁĴħ":145969,"ï¾ķ":145970,"â¬Ĩ":145971,"âĪµ":145972,"ðŁ¤ij":145973,"áĥ£":145974,"ÆĦ":145975,"ѹ":145976,"á¼Ķ":145977,"ê°ł":145978,"ê´Į":145979,"ê·IJ":145980,"뼴":145981,"ì±ĺ":145982,"ï®Ń":145983,"ﺹ":145984,"ﺾ":145985,"âľĹ":145986,"âĿ¦":145987,"ðŁij¦":145988,"áĥĹ":145989,"Ù²":145990,"á½´":145991,"âĪı":145992,"âľ®":145993,"ê¹°":145994,"ë²µ":145995,"ìĦĢ":145996,"ì©Ŀ":145997,"ïºŀ":145998,"ﺽ":145999,"ðŁĩŃ":146000,"ËĤ":146001,"ðŁįij":146002,"ðŁįĮ":146003,"ðŁĶ»":146004,"깬":146005,"ìĬŃ":146006,"ìľ·":146007,"ðŁĽij":146008,"ǧ":146009,"ë¼Ľ":146010,"ﺡ":146011,"ﺺ":146012,"ðĿijļ":146013,"ðŁĵ¦":146014,"ðŁĶİ":146015,"ðŁĹĵ":146016,"áĥĶ":146017,"âľĴ":146018,"âľ¡":146019,"ðŁĮµ":146020,"âĶķ":146021,"ëĢĿ":146022,"ðŁįĬ":146023,"âĺĥ":146024,"ìĺħ":146025,"ব":146026,"ðŁ¦ģ":146027,"âݯ":146028,"ðŁIJķ":146029,"Ñ¿":146030,"।":146031,"à¼ĭ":146032,"ê·Ī":146033,"ì«Į":146034,"ðŁĩ°":146035,"âĿī":146036,"ì«Ģ":146037,"íĿĦ":146038,"ðĿIJ¢":146039,"ðŁļ¨":146040,"âĻ¤":146041,"ðŁĺ©":146042,"ðŁįį":146043,"ðŁĺij":146044,"ðŁļļ":146045,"ÖĦ":146046,"ë«":146047,"뫼":146048,"à¤ı":146049,"á¿·":146050,"âĮ©":146051,"âĺIJ":146052,"âŀ£":146053,"긱":146054,"꼿":146055,"ëĦĿ":146056,"ìı´":146057,"ìļ¤":146058,"쿱":146059,"íİIJ":146060,"ðŁĴ¢":146061,"ì´IJ":146062,"âĩij":146063,"âĶĵ":146064,"âģ¾":146065,"ÜĿ":146066,"ðŁį°":146067,"â´°":146068,"Æı":146069,"ÏŁ":146070,"Úº":146071,"Ûĥ":146072,"áĦĴ":146073,"âĪŁ":146074,"âĿį":146075,"ãĦ²":146076,"ìľħ":146077,"ì¤ı":146078,"ðŁĩ²":146079,"êºĦ":146080,"ðŁİ¤":146081,"âľ£":146082,"â¸Ŀ":146083,"︵":146084,"ວ":146085,"áĢĻ":146086,"âķł":146087,"Õ¯":146088,"âı©":146089,"ðĿij£":146090,"ðŁĴ£":146091,"Åĺ":146092,"à¥IJ":146093,"âģĥ":146094,"âĮĺ":146095,"ê»Į":146096,"ìĮĶ":146097,"ðĿijĺ":146098,"ðŁ¤ĵ":146099,"Õ¿":146100,"à¤Ń":146101,"âĮļ":146102,"âľĿ":146103,"ðŁIJ¼":146104,"ËĮ":146105,"âķļ":146106,"ï¦Ĺ":146107,"âĿķ":146108,"âķ£":146109,"ðŁIJ±":146110,"த":146111,"Ѿ":146112,"à¤ļ":146113,"à¤ľ":146114,"ìĪĦ":146115,"ìļľ":146116,"ðŁİ®":146117,"ÉĴ":146118,"Ú·":146119,"àºį":146120,"âĨµ":146121,"âĪĺ":146122,"âĿĬ":146123,"ë¿į":146124,"ìIJĪ":146125,"ìļĺ":146126,"쯧":146127,"íĥ¯":146128,"ìĸı":146129,"︰":146130,"ðŁĩ¯":146131,"ðŁ§ļ":146132,"ðŁĺµ":146133,"ðŁĺ·":146134,"ðŁĮ³":146135,"ລ":146136,"Äī":146137,"Ä¥":146138,"⾶":146139,"῾":146140,"âĬ±":146141,"âĺ¾":146142,"ê°ī":146143,"ê¼°":146144,"ëºij":146145,"ðŁĶĬ":146146,"ðŁĸIJ":146147,"Ť":146148,"Ò«":146149,"à®®":146150,"âĮĪ":146151,"âĹĹ":146152,"ëĦµ":146153,"ëħľ":146154,"ëľ¹":146155,"ðĿij¥":146156,"ðŁĴ¿":146157,"ðŁĽĴ":146158,"ÊĴ":146159,"áŀĵ":146160,"ðŁIJĿ":146161,"ðŁ¦Ħ":146162,"ðŁį·":146163,"âĺŁ":146164,"︶":146165,"ðŁ¤Ł":146166,"Ô±":146167,"âĨ²":146168,"âĪİ":146169,"âľ«":146170,"ëĩ½":146171,"ëıIJ":146172,"ëķĦ":146173,"靈":146174,"ï§Ŀ":146175,"ïºĻ":146176,"ðŁij»":146177,"ðŁĵº":146178,"êµ¼":146179,"ìĮ©":146180,"ðŁĮ²":146181,"ȱ":146182,"íĶķ":146183,"ðŁĺ¤":146184,"ãĮ¢":146185,"ÊĶ":146186,"ड":146187,"á¼Ī":146188,"ëİĥ":146189,"멱":146190,"ë®Ī":146191,"ðĿIJ«":146192,"âĬķ":146193,"ëĥł":146194,"뻬":146195,"íĭĶ":146196,"Õ¤":146197,"á¼±":146198,"âľ¥":146199,"âĺĦ":146200,"âĪ¥":146201,"âļķ":146202,"ðŁijĦ":146203,"ðŁİħ":146204,"àºĻ":146205,"âĶ¬":146206,"á½µ":146207,"Õ¾":146208,"Öģ":146209,"âĹĶ":146210,"ê¿į":146211,"ëĸµ":146212,"ë©İ":146213,"ë®´":146214,"ìķ´":146215,"áĥľ":146216,"ἡ":146217,"âĶĬ":146218,"âķ®":146219,"âĹ¼":146220,"ðŁį¾":146221,"ðŁĽį":146222,"ðŁijĹ":146223,"ðŁ¤ŀ":146224,"âľĦ":146225,"ÕĢ":146226,"ল":146227,"Ëī":146228,"⣨":146229,"į":146230,"ÏĬ":146231,"á´ľ":146232,"ë¹³":146233,"ï³ĭ":146234,"ï¿ł":146235,"Ī":146236,"âĤ¸":146237,"âľ±":146238,"ê»IJ":146239,"ëĭ»":146240,"맸":146241,"ìŀ¿":146242,"쩨":146243,"ìŃIJ":146244,"ì°¿":146245,"íħŁ":146246,"ðĿIJ§":146247,"ðĿijij":146248,"ðŁĮİ":146249,"ðŁĵ®":146250,"ðŁķĶ":146251,"âĹĻ":146252,"âĹ»":146253,"âŀ§":146254,"ìŁĿ":146255,"⾬":146256,"ãĥ°":146257,"âģĪ":146258,"âĵĺ":146259,"ðŁĴĮ":146260,"ï¬ĥ":146261,"àºĶ":146262,"ìĶ°":146263,"ðŁĺª":146264,"×Ģ":146265,"ìĥ¨":146266,"ïŃĭ":146267,"ðŁįķ":146268,"ðŁĺ´":146269,"ϳ":146270,"á¼Ħ":146271,"á½ħ":146272,"âĩ¢":146273,"âķŃ":146274,"ìĺ»":146275,"íĬ¤":146276,"Üĺ":146277,"⤴":146278,"âĹį":146279,"áŀŁ":146280,"ðŁįº":146281,"áŀļ":146282,"ðŁıĬ":146283,"ðŁIJ·":146284,"ÊĮ":146285,"ὺ":146286,"âģ»":146287,"ê½Į":146288,"ëĪĹ":146289,"ëĹı":146290,"ì¿°":146291,"íĢ¼":146292,"íįħ":146293,"ï·²":146294,"ðŁĮı":146295,"ðŁį«":146296,"ðŁį³":146297,"ðŁİ°":146298,"ðŁij°":146299,"ðŁĴ²":146300,"á¥Ļ":146301,"ðŁIJŁ":146302,"ï¿¡":146303,"ðŁĹ£":146304,"ðŁįľ":146305,"âľ²":146306,"ãİ¢":146307,"ðŁĶ°":146308,"Ἰ":146309,"á½ij":146310,"Äİ":146311,"áĦĢ":146312,"âĻķ":146313,"ëłĿ":146314,"ìĪ´":146315,"ïŃŃ":146316,"Óľ":146317,"ÔĢ":146318,"ëĢľ":146319,"ëĥĶ":146320,"ìĬĽ":146321,"ì«ij":146322,"캥":146323,"캬":146324,"ðĿij¦":146325,"ðŁĶ¶":146326,"쾨":146327,"ðĿIJļ":146328,"ðŁį»":146329,"ðŁĴį":146330,"ðŁ¤¡":146331,"ðŁķĬ":146332,"â½ĩ":146333,"âĵIJ":146334,"ðŁįŃ":146335,"ðŁįª":146336,"ðŁĶĨ":146337,"Ò¡":146338,"á´ĩ":146339,"ÉĹ":146340,"ÜĶ":146341,"âĦİ":146342,"âĿĥ":146343,"ëĹĢ":146344,"ï²Ķ":146345,"ïºĪ":146346,"ðĿIJ»":146347,"ðŁĴĬ":146348,"ðŁļ«":146349,"Ñ°":146350,"ѳ":146351,"ष":146352,"âĹł":146353,"ðŁij¤":146354,"ï¾ĩ":146355,"âĺĵ":146356,"ðŁįµ":146357,"ðŁ¤¨":146358,"âĸŃ":146359,"à®´":146360,"Ü¢":146361,"ܬ":146362,"à´®":146363,"ðŁķº":146364,"Ô¹":146365,"Õ£":146366,"à´¯":146367,"á´Ģ":146368,"âĮī":146369,"âľIJ":146370,"âŀ¦":146371,"ê¹½":146372,"ëĮľ":146373,"ðŁı¥":146374,"ðŁĵ©":146375,"Ò¹":146376,"Óĺ":146377,"à¤ħ":146378,"âĿ§":146379,"ÆĹ":146380,"âĹ½":146381,"ðŁij«":146382,"ðŁİ§":146383,"ðŁij£":146384,"âľ»":146385,"ðŁĻħ":146386,"ðŁĺĸ":146387,"ðŁĴ®":146388,"ະ":146389,"ðŁĶľ":146390,"ðŁįĦ":146391,"ðŁ¤Ŀ":146392,"áĥĿ":146393,"áŀĢ":146394,"âĩ¦":146395,"ʾ":146396,"Ò®":146397,"Õ¼":146398,"à¤Ĩ":146399,"âĹħ":146400,"âļĵ":146401,"âļĸ":146402,"ê¿©":146403,"ë¯Ħ":146404,"ìIJIJ":146405,"ìŀ°":146406,"ì§Ń":146407,"íĭĭ":146408,"íݨ":146409,"íĻ§":146410,"ï²ij":146411,"ðŁİĹ":146412,"Ù³":146413,"ðŁij¸":146414,"ম":146415,"ðŁijķ":146416,"Úµ":146417,"âĢ¾":146418,"âŀ°":146419,"ðŁij¯":146420,"ðŁİ¼":146421,"ðŁıģ":146422,"ĺ":146423,"Êı":146424,"Ú³":146425,"âı±":146426,"ê½Ī":146427,"ëĿĮ":146428,"ìĮī":146429,"ìĹ·":146430,"ìŀ´":146431,"íĹ¹":146432,"íľ¨":146433,"ðĿĹ²":146434,"ðŁĮIJ":146435,"ðŁİĻ":146436,"ðŁıµ":146437,"íĽĻ":146438,"ðĿijħ":146439,"ðŁĺ¶":146440,"âĵħ":146441,"âķ¥":146442,"ðŁįı":146443,"ï¦İ":146444,"Õ©":146445,"ðĿIJĦ":146446,"Ó£":146447,"Ú¿":146448,"âĻļ":146449,"ðŁĶĹ":146450,"ḫ":146451,"âĭ®":146452,"âĸ¦":146453,"âĽ½":146454,"âľµ":146455,"ãħĨ":146456,"ãħĬ":146457,"ëĦĻ":146458,"ëĿ¨":146459,"ë¥Ħ":146460,"ìĦ¦":146461,"짰":146462,"짹":146463,"íīĪ":146464,"ï§ij":146465,"ï»ĩ":146466,"ðŁĮ¾":146467,"ðŁıĸ":146468,"ðŁIJij":146469,"ðŁĴ³":146470,"ðŁĵĨ":146471,"Ûĩ":146472,"Üķ":146473,"á½½":146474,"ëĦľ":146475,"à´²":146476,"à´³":146477,"àºŃ":146478,"áĥĽ":146479,"âĿĶ":146480,"âijħ":146481,"áĥ¥":146482,"ðŁĵħ":146483,"âŀ³":146484,"á´µ":146485,"﹡":146486,"ﹶ":146487,"ÎĨ":146488,"थ":146489,"áīµ":146490,"âĿĻ":146491,"âĿ±":146492,"ëīł":146493,"ëİł":146494,"ëıĽ":146495,"ë¿ħ":146496,"ìĶ¸":146497,"íij¯":146498,"íŀī":146499,"íŀĽ":146500,"ï§Ħ":146501,"ïŃĺ":146502,"ﺦ":146503,"ﻸ":146504,"ðĿijĤ":146505,"ðĿijı":146506,"Ïij":146507,"Úł":146508,"áĢĶ":146509,"áŀĶ":146510,"á¹¢":146511,"ëĦ¸":146512,"ðĿIJ¨":146513,"ðŁĩ´":146514,"Õ°":146515,"ðŁijł":146516,"ðŁįĨ":146517,"ðŁıĢ":146518,"ðŁijIJ":146519,"ðŁįĩ":146520,"ðŁIJ£":146521,"áĪŃ":146522,"ܪ":146523,"ðŁĮĢ":146524,"áŀĺ":146525,"âĩĦ":146526,"ðĿIJĢ":146527,"ÊĻ":146528,"âĶ¼":146529,"ðŁı¿":146530,"Æ·":146531,"Èł":146532,"ѽ":146533,"âĤ¨":146534,"ê´Ń":146535,"ê¹»":146536,"ëĶ¨":146537,"ìĪĢ":146538,"ì¾°":146539,"íĨĪ":146540,"ﮧ":146541,"ﯽ":146542,"ðŁĶħ":146543,"ðŁĶ®":146544,"Å¢":146545,"Ê°":146546,"Ѹ":146547,"ण":146548,"âĬĹ":146549,"ëªĦ":146550,"ï¹·":146551,"ïºħ":146552,"ðĿIJµ":146553,"ðŁĮ¶":146554,"ðŁĵ°":146555,"ðŁĶ·":146556,"ðŁĸĴ":146557,"ðŁ¤²":146558,"ëī©":146559,"ðŁİĨ":146560,"ðŁ§IJ":146561,"ðŁį®":146562,"âĨº":146563,"âĿ¢":146564,"ðŁijª":146565,"ðŁij±":146566,"âĨ¡":146567,"áŀı":146568,"Úķ":146569,"ðŁį¹":146570,"ðŁĴĢ":146571,"Ë®":146572,"Ó¨":146573,"Öħ":146574,"à¤ĩ":146575,"âĤ¡":146576,"âĪķ":146577,"âĺī":146578,"ê¹¼":146579,"ê¼IJ":146580,"콸":146581,"ðĿIJ¬":146582,"ðŁıħ":146583,"ðŁijĻ":146584,"ðŁĴī":146585,"ðŁ¤Ļ":146586,"Èĺ":146587,"ɳ":146588,"ɹ":146589,"Ùº":146590,"áĢĦ":146591,"ῳ":146592,"âļĺ":146593,"âĿĨ":146594,"ëĨī":146595,"ìĸį":146596,"ìĺĩ":146597,"ì¥ĺ":146598,"íĸħ":146599,"íĻij":146600,"ï®Ĭ":146601,"ï¿Ń":146602,"ðĿĴIJ":146603,"ðĿĹ¢":146604,"ðŁĶĸ":146605,"ðŁĶ¨":146606,"ðŁļij":146607,"ðŁļ²":146608,"Ƹ":146609,"âĹ¥":146610,"ðĿIJŃ":146611,"ðŁį½":146612,"âĹij":146613,"âĵĩ":146614,"ðŁĶ±":146615,"âľ¼":146616,"ï¹ĥ":146617,"âķ±":146618,"ãĢĹ":146619,"ðŁıĭ":146620,"ðŁļ´":146621,"ðĿIJ®":146622,"Äļ":146623,"Õı":146624,"Ķ":146625,"áĥij":146626,"Ṭ":146627,"ÄĪ":146628,"ÄĴ":146629,"Ò°":146630,"Óķ":146631,"âIJ":146632,"âIJ£":146633,"âĹ¢":146634,"âļĻ":146635,"ãħĹ":146636,"ê°¬":146637,"곪":146638,"ê»Ģ":146639,"ëĦ´":146640,"ëİģ":146641,"ëĿĶ":146642,"묽":146643,"ëŃį":146644,"ìĩ³":146645,"ì°¹":146646,"íĮ¹":146647,"íŀĿ":146648,"ï®ĭ":146649,"ï¶Ī":146650,"ðĿĴĤ":146651,"ðŁ¥Ģ":146652,"ðŁ¦ħ":146653,"Êĺ":146654,"á¼ij":146655,"âģİ":146656,"ðŁįŀ":146657,"âĨĸ":146658,"âĨĻ":146659,"ðŁİĥ":146660,"âĦ¡":146661,"âĭ±":146662,"ðŁĶį":146663,"ನ":146664,"áµĥ":146665,"âĶ«":146666,"⦿":146667,"ðŁĩ»":146668,"Ƥ":146669,"Òı":146670,"Ò·":146671,"Ûī":146672,"à®ķ":146673,"ḳ":146674,"בּ":146675,"ðŁĨĶ":146676,"ÚŃ":146677,"Û¦":146678,"áħ¡":146679,"âĦ¹":146680,"ê¿İ":146681,"ëķĶ":146682,"ë¼ī":146683,"ìļ§":146684,"ì²µ":146685,"ì´¨":146686,"íĬĪ":146687,"íĸIJ":146688,"ðĿĹĺ":146689,"ðŁĩ¿":146690,"ðŁİĸ":146691,"ðŁijħ":146692,"ðŁĵĺ":146693,"ðŁļĻ":146694,"ðŁĽµ":146695,"ල":146696,"⼵":146697,"ðĿIJ³":146698,"ðĿIJ¸":146699,"âļĶ":146700,"ðŁijŃ":146701,"Óij":146702,"âĶ¯":146703,"ðŁħ¿":146704,"ðŁĺ¹":146705,"ï¿«":146706,"⼤":146707,"ðŁĴĩ":146708,"ðŁĵİ":146709,"ðŁĸĭ":146710,"স":146711,"ðĿIJį":146712,"IJ":146713,"Ïĭ":146714,"Ѭ":146715,"Ú¬":146716,"ÜĴ":146717,"á´¬":146718,"ï¨Ħ":146719,"É£":146720,"Ëij":146721,"ϵ":146722,"ÒĿ":146723,"Û¥":146724,"Üł":146725,"à¹Ľ":146726,"áĥķ":146727,"áĬķ":146728,"ᾶ":146729,"âĤ·":146730,"âĩ¾":146731,"âķ©":146732,"âĸIJ":146733,"âĺª":146734,"âĺ®":146735,"âĿļ":146736,"âĿŃ":146737,"âŀ±":146738,"âµİ":146739,"ãıĬ":146740,"ë©ĵ":146741,"ìĹ¾":146742,"ìªĦ":146743,"íĵĮ":146744,"íķ¼":146745,"ïѬ":146746,"ðĿijĨ":146747,"ðĿijŀ":146748,"ðĿĸĬ":146749,"ðŁİ¸":146750,"ðŁıĦ":146751,"ðŁijµ":146752,"ðŁĴł":146753,"ðŁĶĺ":146754,"ðŁ¥Ĥ":146755,"Ū":146756,"à·ĥ":146757,"á´¼":146758,"âĬ°":146759,"ë³ı":146760,"ë´£":146761,"ï¥ľ":146762,"ðŁĵĪ":146763,"ðŁķ¯":146764,"ðŁ§Ģ":146765,"âĻIJ":146766,"ðŁĨĹ":146767,"ðŁĵķ":146768,"ðŁ§ģ":146769,"Ü«":146770,"âĿIJ":146771,"Õķ":146772,"à½ķ":146773,"âŀĿ":146774,"à¦ķ":146775,"ðĿIJ¶":146776,"É¢":146777,"ÎĦ":146778,"áĨ¢":146779,"âĤ±":146780,"Õį":146781,"à¡ķ":146782,"á´°":146783,"ḩ":146784,"⼷":146785,"âĿ®":146786,"ê¡ĵ":146787,"ëı¤":146788,"ëĹIJ":146789,"ëµĮ":146790,"ìijĪ":146791,"íı¿":146792,"íŵ":146793,"ðĿIJİ":146794,"ðŁĨĺ":146795,"ðŁıŁ":146796,"É¥":146797,"Õ»":146798,"à¡Ķ":146799,"à¤ĸ":146800,"á´¸":146801,"âİĻ":146802,"âİ¥":146803,"âı³":146804,"ëģķ":146805,"ëĬī":146806,"ì¡į":146807,"칡":146808,"禮":146809,"ï¬Ł":146810,"ﮫ":146811,"ﮯ":146812,"ï±ĥ":146813,"ï·»":146814,"ﺵ":146815,"ðĿĹĶ":146816,"ðĿĹ¡":146817,"ðŁİ¨":146818,"ðŁĶĴ":146819,"ÚĽ":146820,"ध":146821,"âŀ¹":146822,"áĢĢ":146823,"ðŁįħ":146824,"âŤ":146825,"à¤ł":146826,"ðŁIJ¥":146827,"áĥĴ":146828,"ðŁıĿ":146829,"ðŁį¼":146830,"ãĮ§":146831,"âĿĽ":146832,"ðŁIJĪ":146833,"য":146834,"áĢŀ":146835,"ãĢĸ":146836,"áŀĻ":146837,"প":146838,"ÕĨ":146839,"âĬĨ":146840,"âľ¾":146841,"ðŁIJĹ":146842,"ﹿ":146843,"Ħ":146844,"ÜŁ":146845,"à²ł":146846,"ಥ":146847,"áŀī":146848,"á´¥":146849,"á´©":146850,"á½Ģ":146851,"ὡ":146852,"âĨķ":146853,"âŀ¯":146854,"ê¡ij":146855,"ëij£":146856,"ë±Į":146857,"ìĪij":146858,"ìľĶ":146859,"ìŀ½":146860,"ì¨į":146861,"ðĿijĢ":146862,"ðŁĮĮ":146863,"ðŁį¦":146864,"ðŁį©":146865,"ðŁIJļ":146866,"ðŁĵĴ":146867,"ðŁĵ¹":146868,"ðŁ¥ij":146869,"Äĭ":146870,"ËĹ":146871,"Ñ«":146872,"Õ¢":146873,"Ú°":146874,"âĮĢ":146875,"âĹĤ":146876,"âĹ£":146877,"⾼":146878,"âĿĴ":146879,"âĿĺ":146880,"âŀĻ":146881,"âŀ²":146882,"ãİį":146883,"ê¡IJ":146884,"ëŀĸ":146885,"ìĬĿ":146886,"ìĽ¤":146887,"ì¡ĭ":146888,"쨰":146889,"íĹĻ":146890,"兩":146891,"ï³į":146892,"ï»İ":146893,"ðĿijĵ":146894,"ðŁĵĬ":146895,"ðŁļ¼":146896,"ï¦ģ":146897,"ðĿķĴ":146898,"ðŁijľ":146899,"ðŁij¿":146900,"ðŁĩ½":146901,"à·Ħ":146902,"âĸ´":146903,"ãįī":146904,"âĬĩ":146905,"ðŁ§¸":146906,"Ú¡":146907,"â¾ĥ":146908,"ðŁĹ»":146909,"âĵij":146910,"ðŁ¤¸":146911,"ðŁ¤¯":146912,"êĴ°":146913,"ðĿIJĵ":146914,"âĶ´":146915,"êĴ±":146916,"áĢĺ":146917,"âĽĦ":146918,"ï¹¹":146919,"ÓĶ":146920,"áĥ±":146921,"Ü¡":146922,"ßŀ":146923,"âĻı":146924,"⾸":146925,"ìij¨":146926,"ðĿIJĿ":146927,"ðĿIJ¥":146928,"ðŁįī":146929,"ðŁij¼":146930,"ðŁ¥Ŀ":146931,"ÆĶ":146932,"ݬ":146933,"फ":146934,"àºļ":146935,"á´´":146936,"á½ĸ":146937,"âĤ¶":146938,"âİ¢":146939,"âĿħ":146940,"⣫":146941,"ãİĽ":146942,"뮨":146943,"ëºĮ":146944,"ë¼ĺ":146945,"ìĨĿ":146946,"ìľ³":146947,"ìŀĮ":146948,"ì£Ĺ":146949,"ìªĺ":146950,"컹":146951,"ï·¼":146952,"ïºĤ":146953,"ðĿIJ´":146954,"ðĿIJ¼":146955,"ðŁĮļ":146956,"ðŁı«":146957,"ðŁĴ¤":146958,"ðŁĴ¶":146959,"ðŁĴ¼":146960,"Êķ":146961,"ʽ":146962,"â²Ł":146963,"ãīł":146964,"ê¡Ĵ":146965,"ëľĢ":146966,"ìĥ¾":146967,"츤":146968,"ï¥ģ":146969,"ðĿļĬ":146970,"ðŁļĥ":146971,"âŀĽ":146972,"ìħ´":146973,"áĦĭ":146974,"âĩĹ":146975,"立":146976,"âĺĸ":146977,"ðŁIJ¦":146978,"⸾":146979,"ðŁĴ´":146980,"ðŁ¤ļ":146981,"ãĬĹ":146982,"âĮĽ":146983,"áĪĽ":146984,"༺":146985,"â½ī":146986,"ðŁı¢":146987,"âĵŀ":146988,"âĺ½":146989,"ãĢĻ":146990,"ðŁ¤®":146991,"ÅIJ":146992,"áĥ¬":146993,"ðĿĹ»":146994,"ðŁįĸ":146995,"ÆĬ":146996,"ÊŁ":146997,"ßĭ":146998,"à¤ĭ":146999,"áµĶ":147000,"á¿ĥ":147001,"âĦī":147002,"âĮĭ":147003,"âı²":147004,"âĵĪ":147005,"âĵ¢":147006,"âķĶ":147007,"âļij":147008,"âĿĭ":147009,"âĿİ":147010,"⵾":147011,"âµ£":147012,"ëĴĪ":147013,"ëľģ":147014,"ë¶ĩ":147015,"ìį»":147016,"ìĺŃ":147017,"짢":147018,"íĹĢ":147019,"ï§Ĭ":147020,"טּ":147021,"ﱡ":147022,"ðĿIJº":147023,"ðĿij§":147024,"ðĿĺ¦":147025,"ðŁĵ¥":147026,"ðŁĺŁ":147027,"ðŁ¥IJ":147028,"Äĸ":147029,"ɨ":147030,"áĢIJ":147031,"áĥĵ":147032,"áºĵ":147033,"ἶ":147034,"á½Ħ":147035,"âĤ¤":147036,"âĮľ":147037,"âĮŁ":147038,"âİł":147039,"⼸":147040,"âµį":147041,"âµı":147042,"âµĵ":147043,"ãĢĺ":147044,"ë·¸":147045,"íħ¼":147046,"ï¦Į":147047,"ïŃĦ":147048,"ïŃİ":147049,"ðĿĻļ":147050,"ðĿļĺ":147051,"à¼ĵ":147052,"ëŃħ":147053,"áIJĽ":147054,"ãݾ":147055,"ï¨Ģ":147056,"ðŁĹ½":147057,"âĻŀ":147058,"Ëĸ":147059,"âĹŀ":147060,"ðŁ¤«":147061,"ðŁĺĹ":147062,"ｦ":147063,"ðŁ¤¢":147064,"âģĩ":147065,"ãĢµ":147066,"ðŁįĶ":147067,"áĬł":147068,"ðŁĺ¼":147069,"ðĿĹ®":147070,"ðŁIJ³":147071,"ðĿIJĭ":147072,"ðŁĨļ":147073,"ðŁĶĽ":147074,"Ñ»":147075,"ܨ":147076,"ல":147077,"âľŀ":147078,"âµĻ":147079,"êµ£":147080,"츨":147081,"ðĿIJľ":147082,"ðĿĺ°":147083,"ðŁĶ½":147084,"Ç»":147085,"Ç¿":147086,"Êĩ":147087,"ÎIJ":147088,"ÐĢ":147089,"Ñ¡":147090,"Ѳ":147091,"ÒĴ":147092,"Ù¶":147093,"ßķ":147094,"න":147095,"áIJģ":147096,"âģŀ":147097,"âĸ§":147098,"âĽĪ":147099,"âľľ":147100,"âľ¹":147101,"âŁ¹":147102,"â¤ĩ":147103,"ê²Ĭ":147104,"ê¾ľ":147105,"ë¯IJ":147106,"ë³IJ":147107,"ìħ©":147108,"ìIJ¬":147109,"ìij¹":147110,"ï¤Ķ":147111,"ï¦ļ":147112,"ï¬ł":147113,"ïŃĶ":147114,"ﺶ":147115,"ðĿĴı":147116,"ðĿĸĨ":147117,"ðĿŶ":147118,"ðŁıĤ":147119,"ðŁIJ½":147120,"ðŁĴ©":147121,"ðŁĵ½":147122,"ðŁĹ¨":147123,"ðŁĹº":147124,"ðŁĺ¸":147125,"ðŁ¥§":147126,"ÅĹ":147127,"Êİ":147128,"ÒĻ":147129,"ײ":147130,"à¤Ī":147131,"á¼´":147132,"á¿ij":147133,"âµī":147134,"ãħĵ":147135,"ì½´":147136,"ðĿĸĵ":147137,"ðŁĵĹ":147138,"ðŁĶª":147139,"ðŁĸį":147140,"ÏĴ":147141,"ðŁij¬":147142,"áĥĻ":147143,"âĨ¬":147144,"âĶ¤":147145,"âĽ¹":147146,"âĻŁ":147147,"ðŁļ¶":147148,"ðŁij¾":147149,"âĪĭ":147150,"ðŁIJ¯":147151,"à¼İ":147152,"âľ·":147153,"ï¨Ļ":147154,"âĶ»":147155,"ðŁij¹":147156,"áĦī":147157,"ສ":147158,"â¾ı":147159,"â½ħ":147160,"ãİĸ":147161,"Ñ´":147162,"Õ®":147163,"Ú¼":147164,"áĢķ":147165,"áĨ¼":147166,"ëŃı":147167,"ðŁIJ¸":147168,"ðŁļ£":147169,"ÆĿ":147170,"Ô»":147171,"áĥ¢":147172,"ðŁį¯":147173,"ɦ":147174,"Õ¦":147175,"âĻĭ":147176,"שׂ":147177,"ðĿŦ":147178,"Çļ":147179,"ɱ":147180,"à¤ī":147181,"á´Ħ":147182,"âĻĵ":147183,"⼰":147184,"âŁª":147185,"ëĥĺ":147186,"뢸":147187,"ìĤij":147188,"ï®Ķ":147189,"ðĿķĸ":147190,"ðĿŧ":147191,"ðŁĩ¼":147192,"ðŁĵĭ":147193,"ðŁļľ":147194,"ðŁ¥¤":147195,"Ä®":147196,"Å·":147197,"ßĬ":147198,"॥":147199,"ப":147200,"áŀĦ":147201,"áµĢ":147202,"á¸ħ":147203,"á¼¢":147204,"âĪĿ":147205,"âĬ¹":147206,"âĴ¶":147207,"âķ´":147208,"⼱":147209,"âĽ³":147210,"âĽº":147211,"âŀŁ":147212,"ãıĦ":147213,"ê¸Ķ":147214,"ê¹Ł":147215,"ëĩ°":147216,"ë¹»":147217,"ìĤ¥":147218,"ìĽ»":147219,"ì°Ł":147220,"íĥ°":147221,"íĨº":147222,"íļ½":147223,"老":147224,"量":147225,"ï³Ŀ":147226,"ðĿIJ¦":147227,"ðĿĴľ":147228,"ðĿĴŁ":147229,"ðĿļĹ":147230,"ðŁİŃ":147231,"ðŁıĵ":147232,"ðŁı³":147233,"ðŁıº":147234,"ðŁIJį":147235,"ðŁijĥ":147236,"ðŁĴı":147237,"ðŁ¤ĸ":147238,"ðŁ¤µ":147239,"Õ²":147240,"âµĶ":147241,"ëĺ¬":147242,"念":147243,"ÊĤ":147244,"áĨ«":147245,"áŀij":147246,"ðĿĸİ":147247,"ðĿĹĸ":147248,"áĦĥ":147249,"âĩł":147250,"áĢ¡":147251,"à½Ħ":147252,"âŀ¸":147253,"ï¦Ļ":147254,"âĩļ":147255,"ðŁIJ¬":147256,"ðŁIJ¢":147257,"â¾Ĵ":147258,"ðŁIJ¤":147259,"ðŁĶ«":147260,"ãĢŀ":147261,"︺":147262,"ðŁĺº":147263,"â½´":147264,"ðŁĨķ":147265,"âģ¿":147266,"ðŁį¨":147267,"à²ķ":147268,"ðŁļĺ":147269,"áŀħ":147270,"à¦ħ":147271,"áŀ¢":147272,"à¨ľ":147273,"âļĮ":147274,"ãĢ½":147275,"à·´":147276,"âĵĽ":147277,"áĢľ":147278,"ìĨ¨":147279,"Ë©":147280,"ÜĹ":147281,"âĭ¼":147282,"ðŁĻī":147283,"ÅĬ":147284,"Éĵ":147285,"ʲ":147286,"ΰ":147287,"Ѽ":147288,"Ô¿":147289,"à¡IJ":147290,"à¼ľ":147291,"ས":147292,"ᶾ":147293,"âĤ²":147294,"âĨ¨":147295,"âĬ¥":147296,"âķ§":147297,"âĻľ":147298,"ãĭ¡":147299,"ë´¬":147300,"ë¶ij":147301,"ìī¿":147302,"ìİħ":147303,"ìł±":147304,"ì°§":147305,"ﲡ":147306,"ðĿĴĽ":147307,"ðĿķ£":147308,"ðĿĹľ":147309,"ðŁį²":147310,"ðŁİ©":147311,"ðŁIJIJ":147312,"ðŁIJł":147313,"ðŁij½":147314,"ðŁĴij":147315,"ðŁĵľ":147316,"ðŁķµ":147317,"ðŁļĮ":147318,"ðŁĽ£":147319,"Êĭ":147320,"Ó¯":147321,"Ù¸":147322,"ßĶ":147323,"ßĻ":147324,"à¡ĵ":147325,"á´į":147326,"ḿ":147327,"âıº":147328,"âĸ¥":147329,"뤽":147330,"íľij":147331,"ðĿIJ¹":147332,"ðĿĸĶ":147333,"ðĿļİ":147334,"ðŁĵĦ":147335,"ðŁ¦·":147336,"Æĥ":147337,"à¦Ł":147338,"âĮĤ":147339,"âĺŃ":147340,"â²ļ":147341,"ëĿķ":147342,"ðŁİ£":147343,"à®ĩ":147344,"à½Ĩ":147345,"áħµ":147346,"áĹľ":147347,"âĢ½":147348,"âĮ£":147349,"âģ½":147350,"ðŁĵ¬":147351,"ðŁ¤§":147352,"âĩª":147353,"â½£":147354,"âĹŁ":147355,"ï¨Ĺ":147356,"êĴª":147357,"ðŁĽĢ":147358,"ÇĤ":147359,"ðŁ¥¶":147360,"ðŁİį":147361,"ï¿©":147362,"ðŁijĴ":147363,"áµĪ":147364,"︿":147365,"áħ©":147366,"⾦":147367,"à°¤":147368,"á´ĸ":147369,"ਬ":147370,"àºĹ":147371,"༻":147372,"Ѻ":147373,"ਪ":147374,"á´³":147375,"ðĿIJĪ":147376,"à»Ģ":147377,"á´¿":147378,"âĤį":147379,"âĩ¡":147380,"âĽª":147381,"ðĿIJĤ":147382,"ðĿĴķ":147383,"ðŁIJľ":147384,"Êį":147385,"ѱ":147386,"à½ĥ":147387,"ë®IJ":147388,"ìĽ¡":147389,"ìľģ":147390,"ðĿIJ¿":147391,"ðĿķł":147392,"ðŁijĽ":147393,"ƪ":147394,"Ϻ":147395,"Ó¬":147396,"Ù¿":147397,"Ý£":147398,"àªī":147399,"ஹ":147400,"à½ij":147401,"áĨ¯":147402,"áµĩ":147403,"âĩ¥":147404,"âıª":147405,"âĻ°":147406,"âļŃ":147407,"âļ¾":147408,"ãħĦ":147409,"êĢ°":147410,"ê°Ĺ":147411,"ê²ĭ":147412,"ê²»":147413,"ê¶ľ":147414,"ê¼ĩ":147415,"ê½¹":147416,"ëĤŁ":147417,"ëħĪ":147418,"ëĭ¢":147419,"맣":147420,"ëªĨ":147421,"ëµĢ":147422,"ì½±":147423,"íĩĺ":147424,"íľľ":147425,"茶":147426,"ï±µ":147427,"ï²¢":147428,"ﲤ":147429,"ðĿĴĬ":147430,"ðĿĺ¯":147431,"ðŁįĹ":147432,"ðŁıį":147433,"ðŁIJĺ":147434,"ðŁĵ¡":147435,"ðŁĶŀ":147436,"ðŁ¤³":147437,"ðŁ¥ģ":147438,"ðŁ¥Ĺ":147439,"ðŁ¦Ĭ":147440,"ĵ":147441,"Ʀ":147442,"ǵ":147443,"ɯ":147444,"Îı":147445,"ÕĦ":147446,"Ü¥":147447,"à½ģ":147448,"ᨳ":147449,"âķ«":147450,"ãİī":147451,"ë·´":147452,"ìĨİ":147453,"ìİĮ":147454,"죵":147455,"íĽł":147456,"離":147457,"ï³ı":147458,"ﻺ":147459,"ðĿijģ":147460,"ðĿijĩ":147461,"ðĿĴĨ":147462,"ðŁİł":147463,"ðŁIJĶ":147464,"ðŁijŁ":147465,"Åĸ":147466,"à¤Į":147467,"á¾½":147468,"ê¦Ĵ":147469,"à®Ł":147470,"á´±":147471,"ðŁı°":147472,"ðŁIJŀ":147473,"à½Ģ":147474,"áĢħ":147475,"âĬ¿":147476,"ðŁIJ§":147477,"áĽģ":147478,"â¼Ī":147479,"âĶ¿":147480,"ðŁ¥´":147481,"⼿":147482,"ðŁ§ľ":147483,"ãħ¿":147484,"âĦ«":147485,"ãĢ³":147486,"ãĬĻ":147487,"â¼Ģ":147488,"怜":147489,"ðŁı¬":147490,"ðŁĵ»":147491,"áĬĽ":147492,"áĦħ":147493,"àºĬ":147494,"àºĽ":147495,"áħ³":147496,"ðŁij®":147497,"à®±":147498,"âĺĩ":147499,"ðĿIJı":147500,"à´µ":147501,"à»ģ":147502,"à½ı":147503,"ར":147504,"ᥱ":147505,"âĤ£":147506,"復":147507,"ïŃĻ":147508,"ï´©":147509,"ï¹Ĥ":147510,"ðŁį£":147511,"ðŁķ¹":147512,"Ïĸ":147513,"ම":147514,"ຢ":147515,"áĭŃ":147516,"âİĿ":147517,"âĹĿ":147518,"âĻĪ":147519,"âĻİ":147520,"ê½¥":147521,"ì³Ķ":147522,"ì¼ij":147523,"ï±°":147524,"ðĿijĥ":147525,"ðŁĮª":147526,"ðŁį¡":147527,"Åİ":147528,"ʦ":147529,"ѧ":147530,"Óİ":147531,"Ô´":147532,"ÚĪ":147533,"ßĵ":147534,"ߧ":147535,"à¤Ķ":147536,"áĪ«":147537,"áĪµ":147538,"áĹ©":147539,"á´ł":147540,"á¼ł":147541,"âĢĹ":147542,"âģij":147543,"âĦı":147544,"âĸĩ":147545,"â²£":147546,"ãĦ³":147547,"ãī®":147548,"ê³Ĺ":147549,"ëĦĴ":147550,"ëĸ«":147551,"ë¡Ħ":147552,"ë¹°":147553,"ë½ģ":147554,"ìĦģ":147555,"ìĮĺ":147556,"ìŁĮ":147557,"ì³ī":147558,"ì¼ķ":147559,"כּ":147560,"ï³İ":147561,"ﹸ":147562,"ï¹¾":147563,"ðĿIJĨ":147564,"ðĿij·":147565,"ðĿĽ¼":147566,"ðŁİı":147567,"ðŁİŀ":147568,"ðŁIJĻ":147569,"ðŁijĤ":147570,"ðŁĵģ":147571,"ðŁĸ±":147572,"ðŁļį":147573,"ðŁļ§":147574,"ðŁĽ¡":147575,"ðŁ¤Ĵ":147576,"ðŁ¥ŀ":147577,"ðŁ¥©":147578,"ðŁ¦Ģ":147579,"ðŁ¦ĸ":147580,"Ë¢":147581,"Üļ":147582,"வ":147583,"áĢģ":147584,"áī°":147585,"âıŃ":147586,"âĻ¿":147587,"ê³ĺ":147588,"ëıĿ":147589,"ëķĥ":147590,"ìħĮ":147591,"ìĴ¸":147592,"ìĽŁ":147593,"íħĦ":147594,"íľ«":147595,"ï§ĺ":147596,"￬":147597,"ðŁı·":147598,"ðŁĶ§":147599,"ðŁ¥Ī":147600,"Æĸ":147601,"áŀĩ":147602,"áŀĸ":147603,"âģº":147604,"âĹľ":147605,"âŀ©":147606,"ê¦Ń":147607,"ëĻ¤":147608,"ïŃ¼":147609,"ðĿĻĸ":147610,"ðĿĻ£":147611,"ðĿĻ¤":147612,"ðŁĮĿ":147613,"ðŁĶij":147614,"ðŁĽł":147615,"àºĩ":147616,"âĺ£":147617,"ãĦ¨":147618,"ðĿĸĹ":147619,"Óĵ":147620,"âĨ£":147621,"ðŁ¥ī":147622,"ðŁĮł":147623,"ðŁĺ½":147624,"ãİł":147625,"ŧ":147626,"ðŁIJĴ":147627,"ï§IJ":147628,"ðŁĺ¿":147629,"âĪ¬":147630,"ðŁIJ®":147631,"⣱":147632,"ಡ":147633,"â¾¼":147634,"à°²":147635,"˶":147636,"âĸ¿":147637,"ÕĪ":147638,"áŀİ":147639,"áħ¥":147640,"áŀĹ":147641,"Õ§":147642,"ðŁ¤IJ":147643,"ðŁįł":147644,"ত":147645,"ය":147646,"âĻį":147647,"ìĺĻ":147648,"íĺĵ":147649,"ﹺ":147650,"ðŁĽ³":147651,"Åī":147652,"á´İ":147653,"âıľ":147654,"âĶ³":147655,"긷":147656,"ì¡Ķ":147657,"ðĿĴĪ":147658,"ðĿĴį":147659,"ðĿĴ¹":147660,"ðĿĵĩ":147661,"ðĿķŁ":147662,"ðĿĹ¹":147663,"ðŁĮħ":147664,"ðŁı´":147665,"ÄĶ":147666,"Ĥ":147667,"ŵ":147668,"Ǿ":147669,"Ïŀ":147670,"϶":147671,"Ô³":147672,"ÜĨ":147673,"ß©":147674,"à¡Ĵ":147675,"à¤ĺ":147676,"à¶ļ":147677,"à½ĸ":147678,"áģĬ":147679,"áĥŀ":147680,"áĦĤ":147681,"áĭ«":147682,"á´º":147683,"ḣ":147684,"Ḫ":147685,"á¹Ĥ":147686,"á¼·":147687,"á¿ĩ":147688,"âĩĮ":147689,"âı¬":147690,"âĻĮ":147691,"⮣":147692,"â´»":147693,"ⵣ":147694,"ê¦ķ":147695,"ꦪ":147696,"ꦮ":147697,"ê²Ħ":147698,"ê¾IJ":147699,"ëĥij":147700,"ëķĭ":147701,"롸":147702,"ë¬Ģ":147703,"ìĩ¤":147704,"ìĪ©":147705,"ìľķ":147706,"ìŃĺ":147707,"ì·°":147708,"ì·¸":147709,"íľĢ":147710,"藍":147711,"ï§į":147712,"ï±Ħ":147713,"ï³ij":147714,"ðĿIJ¤":147715,"ðĿĴĵ":147716,"ðĿĴ¶":147717,"ðĿĹ¼":147718,"ðĿĻĬ":147719,"ðŁĩ¾":147720,"ðŁĮĽ":147721,"ðŁĮ®":147722,"ðŁİĩ":147723,"ðŁİ²":147724,"ðŁıĽ":147725,"ðŁij¥":147726,"ðŁij´":147727,"ðŁĴĨ":147728,"ðŁĵĤ":147729,"ðŁĵ§":147730,"ðŁķIJ":147731,"ðŁĸķ":147732,"ðŁĺ§":147733,"ðŁĻĢ":147734,"ðŁļĴ":147735,"ðŁĽ«":147736,"ðŁ¤ł":147737,"ðŁ¥ļ":147738,"ðŁ¥Ľ":147739,"ðŁ¥£":147740,"ǯ":147741,"ȧ":147742,"ÎĬ":147743,"Ò²":147744,"×°":147745,"Ûij":147746,"áĥ©":147747,"áĦĮ":147748,"áĪį":147749,"áī¥":147750,"áıĤ":147751,"âģ±":147752,"âĬ¢":147753,"âĹĵ":147754,"âĿ°":147755,"ë¿¡":147756,"ìĽ©":147757,"íģŃ":147758,"íĨ³":147759,"íĬĦ":147760,"íĵ¸":147761,"北":147762,"若":147763,"ï±IJ":147764,"ﱯ":147765,"ï³ļ":147766,"ðĿĸĺ":147767,"ðĿĺĢ":147768,"ðŁIJĬ":147769,"ðŁIJĮ":147770,"ðŁijļ":147771,"ðŁĵĥ":147772,"ðŁļĽ":147773,"ðŁļª":147774,"ðŁ¤°":147775,"Ä´":147776,"áĥ®":147777,"áŨ":147778,"âĻ®":147779,"â²ŀ":147780,"ãĪĶ":147781,"ìħį":147782,"ãħĥ":147783,"率":147784,"ມ":147785,"Õİ":147786,"Õº":147787,"⬼":147788,"⽤":147789,"ðĿIJ²":147790,"âŀµ":147791,"áĢĽ":147792,"âĶħ":147793,"âĨŁ":147794,"â¼Ĭ":147795,"ðŁĮ½":147796,"ðŁļ¿":147797,"ï¦Ĭ":147798,"ãĦ£":147799,"⼩":147800,"ï©Ľ":147801,"ðŁį±":147802,"⾨":147803,"à´¤":147804,"áŀģ":147805,"àºŀ":147806,"Êļ":147807,"ðĿIJĴ":147808,"à´±":147809,"áŀľ":147810,"ன":147811,"à°Ĺ":147812,"à´ļ":147813,"âĩ£":147814,"ï¦ķ":147815,"Õħ":147816,"Æĺ":147817,"âĤ¦":147818,"âĶĦ":147819,"ï¦Ł":147820,"嶺":147821,"ðĿIJģ":147822,"ðĿIJĥ":147823,"ðŁį¸":147824,"ðŁIJ²":147825,"Ŷ":147826,"Éĸ":147827,"ßĺ":147828,"ฦ":147829,"à½Ķ":147830,"áĨ·":147831,"âģķ":147832,"âĵĤ":147833,"âĿľ":147834,"便":147835,"אַ":147836,"ðĿĹĿ":147837,"ðĿĹ¿":147838,"ðŁİ¾":147839,"ðŁĹĿ":147840,"ðŁ¦Į":147841,"Æħ":147842,"Ǫ":147843,"ÒĹ":147844,"ÜĽ":147845,"ßł":147846,"à¡ij":147847,"áī£":147848,"áĬŃ":147849,"ṡ":147850,"âŀ¼":147851,"âŀ¾":147852,"â´±":147853,"ãī¡":147854,"곯":147855,"ë½Ī":147856,"ìĤĺ":147857,"ìīij":147858,"ì«ĺ":147859,"íĮĥ":147860,"íĻ°":147861,"ï¤Ĺ":147862,"ðŁĮ¬":147863,"ðŁĮ°":147864,"ðŁį¤":147865,"Ä»":147866,"Åĩ":147867,"ƨ":147868,"Éķ":147869,"Ò¢":147870,"Òº":147871,"Öį":147872,"×±":147873,"Ú±":147874,"Ú½":147875,"ÛIJ":147876,"à¤Ľ":147877,"à·Ģ":147878,"à¹ļ":147879,"ຫ":147880,"á´¹":147881,"á½Ķ":147882,"á¾³":147883,"âĤĴ":147884,"âĨ´":147885,"âĩĿ":147886,"âīħ":147887,"âĮ¨":147888,"âĵĵ":147889,"âĸ¢":147890,"âļ¬":147891,"âŀŃ":147892,"â²Ĵ":147893,"ãİ¿":147894,"ê¿´":147895,"ëĪ±":147896,"ëį¬":147897,"ëİIJ":147898,"ëIJ«":147899,"ëĶ«":147900,"ë±ģ":147901,"ìĥ¥":147902,"íĮ¼":147903,"ïŃĵ":147904,"ﮥ":147905,"ï²°":147906,"ðĿIJĩ":147907,"ðĿIJij":147908,"ðĿijĮ":147909,"ðĿĵª":147910,"ðĿķļ":147911,"ðĿĺª":147912,"ðĿĺ¼":147913,"ðĿļĽ":147914,"ðŁĩ¶":147915,"ðŁĮĦ":147916,"ðŁĮķ":147917,"ðŁĮ¤":147918,"ðŁĮ§":147919,"ðŁį¬":147920,"ðŁİĭ":147921,"ðŁİ»":147922,"ðŁı¨":147923,"ðŁIJĩ":147924,"ðŁijĵ":147925,"ðŁĵIJ":147926,"ðŁĵĻ":147927,"ðŁĶ¼":147928,"ðŁķĴ":147929,"ðŁĸı":147930,"ðŁĸ¥":147931,"ðŁ¤¬":147932,"ðŁ¥Ĭ":147933,"ðŁ¥Ĵ":147934,"ßĮ":147935,"àºĦ":147936,"á¼µ":147937,"âķ¡":147938,"Ⲥ":147939,"â´¼":147940,"âµ¢":147941,"ãĪ¯":147942,"ëĵ¸":147943,"ëŁĩ":147944,"ëºį":147945,"ðĿĻ§":147946,"ðŁįĪ":147947,"ðŁĶ¬":147948,"ðŁĸĬ":147949,"ðŁ¤¾":147950,"Ë¡":147951,"Ü©":147952,"âĮ¡":147953,"âŃij":147954,"Ⲧ":147955,"ë©ī":147956,"ì¼Ń":147957,"￤":147958,"ðĿĴİ":147959,"ðĿĹ¥":147960,"ðŁIJµ":147961,"ðŁķ¶":147962,"ðŁķ¸":147963,"ðŁ¤ľ":147964,"Õª":147965,"áĪĭ":147966,"ðŁ¥µ":147967,"ï°ģ":147968,"áµIJ":147969,"âķĵ":147970,"áĢĸ":147971,"âĭĪ":147972,"Éŀ":147973,"âŀ®":147974,"॰":147975,"ãĨģ":147976,"ðŁĴ±":147977,"ðŁıŃ":147978,"áĨ¨":147979,"ðŁįļ":147980,"ðŁ¦IJ":147981,"á´»":147982,"âĺĮ":147983,"à´ķ":147984,"Õ±":147985,"áħ®":147986,"ðĿIJĮ":147987,"Ŧ":147988,"àºķ":147989,"âľĻ":147990,"˳":147991,"Ôµ":147992,"âķĴ":147993,"ðĿĹĹ":147994,"ðĿĹł":147995,"Úļ":147996,"ধ":147997,"âĨĿ":147998,"âĻī":147999,"ãĮ»":148000,"ì¹Ĭ":148001,"ðĿĹº":148002,"ðŁ§ĺ":148003,"ì³£":148004,"ï¬Ŀ":148005,"ðŁijº":148006,"ÇŁ":148007,"ÎĪ":148008,"Ϋ":148009,"Ñ¥":148010,"Ô²":148011,"Õ¨":148012,"ܦ":148013,"à¦Ĩ":148014,"থ":148015,"áIJ¢":148016,"á¼ģ":148017,"á¼ĺ":148018,"ἦ":148019,"âĵĿ":148020,"ãĪ°":148021,"ãİĹ":148022,"겡":148023,"ë¨Ģ":148024,"ì£Ķ":148025,"ì´¤":148026,"ìµĿ":148027,"林":148028,"ïŃĬ":148029,"ï²Ł":148030,"ðĿIJ·":148031,"ðĿijĭ":148032,"ðĿĵī":148033,"ðĿĺµ":148034,"ðŁĴ·":148035,"ðŁĽ©":148036,"ðŁ§¹":148037,"ÅĶ":148038,"Êŀ":148039,"Ë¥":148040,"ÎĮ":148041,"Ñ©":148042,"ÓIJ":148043,"Ół":148044,"Úij":148045,"ÚĴ":148046,"ߨ":148047,"àªĪ":148048,"áIJĥ":148049,"ṯ":148050,"âĤĭ":148051,"âĤµ":148052,"âĦħ":148053,"âĦł":148054,"âĪ£":148055,"âīº":148056,"âī»":148057,"âĬĽ":148058,"âĮIJ":148059,"âİĵ":148060,"âĺ¸":148061,"âĻĴ":148062,"âļĴ":148063,"âľĩ":148064,"âľł":148065,"â´·":148066,"âµĸ":148067,"ãĦ¸":148068,"ãī¢":148069,"ãī°":148070,"êĩ´":148071,"ê´¸":148072,"êºł":148073,"ëĤı":148074,"ëĤ¢":148075,"ëIJĢ":148076,"뺴":148077,"ìĥľ":148078,"ìįħ":148079,"줫":148080,"챦":148081,"ìºij":148082,"ì¼ģ":148083,"쿳":148084,"íĤģ":148085,"íħ¡":148086,"íĴĤ":148087,"íĴī":148088,"íľĦ":148089,"ïŃª":148090,"ﮬ":148091,"ﯦ":148092,"ﱪ":148093,"ï²ı":148094,"ï´Ģ":148095,"ï»Ĩ":148096,"￦":148097,"ðĿijĹ":148098,"ðĿĸĻ":148099,"ðŁĮ¡":148100,"ðŁįĿ":148101,"ðŁį§":148102,"ðŁİ«":148103,"ðŁıĺ":148104,"ðŁıª":148105,"ðŁIJĭ":148106,"ðŁIJĽ":148107,"ðŁIJº":148108,"ðŁijĸ":148109,"ðŁijŀ":148110,"ðŁij·":148111,"ðŁĵĢ":148112,"ðŁĶĦ":148113,"ðŁĶĮ":148114,"ðŁķĻ":148115,"ðŁĻį":148116,"ðŁĻİ":148117,"ðŁ¦į":148118,"Ç°":148119,"ÉŁ":148120,"ÊĨ":148121,"Ô¼":148122,"Úľ":148123,"ড":148124,"শ":148125,"áĴĥ":148126,"Ἡ":148127,"âĵķ":148128,"â²Ī":148129,"ê°°":148130,"ê¹ł":148131,"êºħ":148132,"ëĦ¹":148133,"ë¯ĵ":148134,"íIJĪ":148135,"臨":148136,"ï®ij":148137,"ﲨ":148138,"ðĿĴī":148139,"ðĿĴĶ":148140,"ðĿŨ":148141,"ðĿĻŀ":148142,"ðĿļĴ":148143,"ðĿļķ":148144,"ðŁIJİ":148145,"ðŁ¤ķ":148146,"ðŁ§Ķ":148147,"Ï°":148148,"ÔĿ":148149,"âĮĬ":148150,"âĴ¾":148151,"ãī£":148152,"ïŃ©":148153,"ðĿļŀ":148154,"Êij":148155,"দ":148156,"áĦĩ":148157,"âīĥ":148158,"â²Ģ":148159,"ìŁİ":148160,"ðĿij¶":148161,"ðĿĵ²":148162,"ðŁİ·":148163,"ðŁļ¹":148164,"àºģ":148165,"áłł":148166,"ãĦļ":148167,"ðŁIJ¿":148168,"áĽļ":148169,"âķ³":148170,"ðŁIJŃ":148171,"âĴ¹":148172,"ðĿĸļ":148173,"âĻĸ":148174,"ãĪ²":148175,"âĨ¾":148176,"áĦĨ":148177,"âķĽ":148178,"ðŁ¤į":148179,"â½¥":148180,"ðŁĮ¨":148181,"âĪ®":148182,"ãĮĺ":148183,"ãįij":148184,"ï¹Ģ":148185,"âĵĹ":148186,"âĬĦ":148187,"ðŁı¹":148188,"ËĴ":148189,"ðŁ¤±":148190,"ãıľ":148191,"ðŁİĮ":148192,"ï¥Ń":148193,"ণ":148194,"ðŁİ¹":148195,"ãĬŁ":148196,"à´°":148197,"ðĿIJĶ":148198,"à´¨":148199,"à½ļ":148200,"âľº":148201,"Õ·":148202,"ðŁij³":148203,"à¦ľ":148204,"âĺĭ":148205,"âĻĬ":148206,"ãĢĽ":148207,"Èĭ":148208,"à®°":148209,"áĥ¨":148210,"âĦķ":148211,"íijĢ":148212,"ðĿĵĥ":148213,"ðŁ¦Ķ":148214,"Ä¿":148215,"ÅĢ":148216,"Ƴ":148217,"Éļ":148218,"Öĥ":148219,"Ü£":148220,"ߣ":148221,"à¦Ń":148222,"ৡ":148223,"ර":148224,"ຣ":148225,"à½ĩ":148226,"Ḩ":148227,"á½Ī":148228,"⽬":148229,"ê¡Ķ":148230,"ì³Ħ":148231,"ï¨ī":148232,"ðĿIJ¡":148233,"ðĿĺ¢":148234,"ðŁį¿":148235,"ðŁİŁ":148236,"ðŁıī":148237,"ðŁĶIJ":148238,"ðŁļħ":148239,"ðŁ¤½":148240,"Æį":148241,"Ç«":148242,"ǽ":148243,"Èļ":148244,"Îī":148245,"Ó¤":148246,"Óª":148247,"ÕĬ":148248,"Ù¼":148249,"Ú´":148250,"ßĿ":148251,"à¶ľ":148252,"á¼ķ":148253,"á¿¥":148254,"âİŀ":148255,"ãĢļ":148256,"ãī¤":148257,"곸":148258,"ê·ģ":148259,"ëĵĦ":148260,"ëĵķ":148261,"ì¨Ķ":148262,"챨":148263,"ðĿIJ¾":148264,"ðĿij»":148265,"ðĿĶ¼":148266,"ðĿķĿ":148267,"ðĿĺŃ":148268,"ðŁĨĻ":148269,"ðŁĵ¤":148270,"ðŁĶŁ":148271,"ðŁĹ¼":148272,"Äľ":148273,"Æģ":148274,"Æ¿":148275,"dz":148276,"Ç·":148277,"Éĥ":148278,"Éł":148279,"Êī":148280,"ʧ":148281,"˲":148282,"Ï´":148283,"Õģ":148284,"Õŀ":148285,"Öĩ":148286,"ÛĤ":148287,"Ûĵ":148288,"ßĹ":148289,"ߦ":148290,"হ":148291,"ள":148292,"à´¸":148293,"à»Ĥ":148294,"áĪĿ":148295,"áĪª":148296,"áĭµ":148297,"áIJĬ":148298,"áĴª":148299,"áļĸ":148300,"áŀĽ":148301,"á´¢":148302,"áµı":148303,"áµŃ":148304,"ᶫ":148305,"á¸ı":148306,"áºĴ":148307,"á¼¥":148308,"á½ķ":148309,"á½¼":148310,"âĤĬ":148311,"âĦĤ":148312,"âĦ©":148313,"âĩī":148314,"âī£":148315,"âĮł":148316,"âİŁ":148317,"âı®":148318,"âķĺ":148319,"âĹĸ":148320,"âĺ©":148321,"âĻij":148322,"âĻ²":148323,"âļĽ":148324,"ãĦŁ":148325,"ãī±":148326,"ãİļ":148327,"ê¡ķ":148328,"êªĸ":148329,"ê°¹":148330,"ê²Ĩ":148331,"êµĦ":148332,"ëĩ¬":148333,"ëĭ¯":148334,"ëıł":148335,"ëĴ¬":148336,"ëĸĪ":148337,"ëĸ½":148338,"ëĺĶ":148339,"ëŀ¸":148340,"ë¸ħ":148341,"뻳":148342,"ë¿Ł":148343,"ìĤµ":148344,"ìĬī":148345,"ìľ°":148346,"ìłĭ":148347,"ìłĶ":148348,"쥡":148349,"ìŃĿ":148350,"켬":148351,"íĪĩ":148352,"íīľ":148353,"íįĦ":148354,"íĽ¾":148355,"íĿ£":148356,"朗":148357,"勞":148358,"ï¦ľ":148359,"獵":148360,"ï§ľ":148361,"ï¨Ī":148362,"שׁ":148363,"הּ":148364,"ïŃ½":148365,"ï®ī":148366,"ï¯ŀ":148367,"ï°Ĵ":148368,"ï±ĩ":148369,"ï¿Ħ":148370,"ðĿIJħ":148371,"ðĿijĦ":148372,"ðĿijº":148373,"ðĿĴĹ":148374,"ðĿĵ®":148375,"ðĿķĽ":148376,"ðĿķŀ":148377,"ðĿĸij":148378,"ðĿĺģ":148379,"ðĿĺĨ":148380,"ðĿĺ¶":148381,"ðĿĻ¢":148382,"ðĿļľ":148383,"ðŁĮĥ":148384,"ðŁĮ¦":148385,"ðŁįŁ":148386,"ðŁİİ":148387,"ðŁıĻ":148388,"ðŁIJ©":148389,"ðŁIJ«":148390,"ðŁIJ´":148391,"ðŁijĶ":148392,"ðŁĵī":148393,"ðŁĵĽ":148394,"ðŁĶī":148395,"ðŁĸ¼":148396,"ðŁĹĥ":148397,"ðŁĹ¯":148398,"ðŁļĩ":148399,"ðŁļIJ":148400,"ðŁļµ":148401,"ðŁ¤¶":148402,"ðŁ¥ĭ":148403,"ðŁ¥ĵ":148404,"ðŁ¥®":148405,"ðŁ¦İ":148406,"ðŁ¦ł":148407,"ðŁ§Ĵ":148408,"ðŁ§¨":148409,"ÆIJ":148410,"Çį":148411,"ÓĢ":148412,"ÔĽ":148413,"ರ":148414,"à´Ļ":148415,"áĢĴ":148416,"ê²Ŀ":148417,"ê¹¹":148418,"ë©¥":148419,"ìĸĶ":148420,"ï¤ģ":148421,"ï¤ı":148422,"ï¦ī":148423,"ï¦ĵ":148424,"ï§ī":148425,"ï²Ŀ":148426,"ðĿĹŀ":148427,"ðĿű":148428,"ðŁĮĭ":148429,"ðŁį¶":148430,"à¦ļ":148431,"ìķľ":148432,"ðĿIJ¯":148433,"ðĿļĿ":148434,"à°¨":148435,"à½ĺ":148436,"à½ł":148437,"á¡¥":148438,"á¾°":148439,"âģį":148440,"âĶ°":148441,"⬾":148442,"ðĿIJł":148443,"ðĿij¯":148444,"ðĿĹĽ":148445,"ðĿĵ»":148446,"ðĿĸĪ":148447,"âŀ»":148448,"áŀł":148449,"⡱":148450,"â»ij":148451,"ðŁ§µ":148452,"廉":148453,"ðŁijĺ":148454,"ãĤĶ":148455,"â¼Ł":148456,"ãĬ¤":148457,"ï¦Ŀ":148458,"ãĮ¦":148459,"âĢ¸":148460,"ðŁĶĻ":148461,"ã¹":148462,"㹦":148463,"ï¹ħ":148464,"ï©Į":148465,"ãī¨":148466,"︽":148467,"âį¥":148468,"ðŁļī":148469,"ðŁ¥ľ":148470,"âĵľ":148471,"â»Ŀ":148472,"ï¨ľ":148473,"ðŁĴĴ":148474,"áĦij":148475,"â¾ŀ":148476,"ï¨ģ":148477,"à´ª":148478,"áĦİ":148479,"âŀ´":148480,"ষ":148481,"áħ¬":148482,"áŀ§":148483,"âĨ¢":148484,"âķ¦":148485,"âľij":148486,"ˬ":148487,"ÕIJ":148488,"à¼Ķ":148489,"ʤ":148490,"˨":148491,"à¤ŀ":148492,"à»ĥ":148493,"à¼ļ":148494,"âĵ¥":148495,"âķľ":148496,"ðŁIJĸ":148497,"á¼Ļ":148498,"ἤ":148499,"ìĨ°":148500,"ÈĤ":148501,"ʱ":148502,"à®ļ":148503,"áĥ§":148504,"á´ĭ":148505,"á´®":148506,"âĿ¡":148507,"âŀ·":148508,"ëĿ¡":148509,"梨":148510,"ﯡ":148511,"ðĿķķ":148512,"ðŁħ°":148513,"ðŁ¦¸":148514,"Ǹ":148515,"Óŀ":148516,"Ô¶":148517,"ÖĨ":148518,"Úģ":148519,"Ûĭ":148520,"áİ¥":148521,"᾿":148522,"âĶŃ":148523,"âĶ®":148524,"êĢĢ":148525,"ê±ĺ":148526,"ëIJŃ":148527,"ë½Ħ":148528,"ìĶIJ":148529,"ì¸Į":148530,"íģł":148531,"íĻ±":148532,"ï¥ī":148533,"ï¨ĸ":148534,"ðĿij´":148535,"ðĿĸĴ":148536,"ðĿĺ¨":148537,"ðĿļĮ":148538,"ðŁIJ¡":148539,"ðŁij¢":148540,"ðŁĵĶ":148541,"Åħ":148542,"Æİ":148543,"È©":148544,"Òª":148545,"Ôĥ":148546,"áĥ«":148547,"á¸ĩ":148548,"⼣":148549,"ê»Ń":148550,"ë¨Ħ":148551,"ìŁĢ":148552,"줴":148553,"íļIJ":148554,"盧":148555,"ðŁŁ¢":148556,"Ƨ":148557,"ȼ":148558,"ÊĿ":148559,"ËĦ":148560,"Ëħ":148561,"Ëį":148562,"˧":148563,"Ò¥":148564,"ÕĶ":148565,"Øı":148566,"ؼ":148567,"ßIJ":148568,"ßľ":148569,"à¤ĵ":148570,"à¦Ļ":148571,"à®ĵ":148572,"ප":148573,"à¼į":148574,"à¼Ĵ":148575,"ལ":148576,"áĢĤ":148577,"áĢĬ":148578,"áĦĦ":148579,"áĪĺ":148580,"áĭĬ":148581,"áĮį":148582,"áijĭ":148583,"áŀĤ":148584,"áł¢":148585,"á¡Ŀ":148586,"á´¦":148587,"áµį":148588,"ᵨ":148589,"ḡ":148590,"ḯ":148591,"á¼£":148592,"âģĤ":148593,"âĦĺ":148594,"âĦľ":148595,"âĦ³":148596,"âĦµ":148597,"âĨ¦":148598,"âĩĨ":148599,"âĪ·":148600,"âĬļ":148601,"âĮ«":148602,"âĮ¯":148603,"âİĽ":148604,"âİľ":148605,"âݤ":148606,"âݦ":148607,"âİ®":148608,"âijī":148609,"âĶī":148610,"âķĻ":148611,"âĸĤ":148612,"âĹŃ":148613,"âĺĬ":148614,"âĺį":148615,"âĺĴ":148616,"âļĨ":148617,"⼧":148618,"âĽ²":148619,"âŀĺ":148620,"â¥Ħ":148621,"â´³":148622,"â´½":148623,"âµĪ":148624,"ãī¯":148625,"ãİij":148626,"㧬":148627,"êĻ¬":148628,"ê§ģ":148629,"곬":148630,"ê´ŀ":148631,"ê»ľ":148632,"ëħĵ":148633,"ëĭ¼":148634,"ëįĸ":148635,"ëĸ±":148636,"ëĿ°":148637,"롹":148638,"뢴":148639,"ë£Ģ":148640,"뤳":148641,"ë¨ķ":148642,"ëŃ¥":148643,"ìĦ¶":148644,"ìħ¤":148645,"ìĮķ":148646,"ìįª":148647,"ìı©":148648,"ìĴĢ":148649,"ìĶ¯":148650,"ìĿĶ":148651,"ìĿľ":148652,"ìłŃ":148653,"짦":148654,"쨩":148655,"첬":148656,"ì³¥":148657,"켯":148658,"íĢ«":148659,"íĢŃ":148660,"íĥ¸":148661,"íĵģ":148662,"íķ¬":148663,"íŸ":148664,"íĽķ":148665,"íľŃ":148666,"íĿĹ":148667,"ï¤Į":148668,"浪":148669,"刺":148670,"ï¬Ħ":148671,"ï¬ħ":148672,"ïŃij":148673,"ïŃ«":148674,"ïŃº":148675,"ï®Ĥ":148676,"ﮢ":148677,"ﮨ":148678,"ï°İ":148679,"ï°ł":148680,"ï²£":148681,"ï³IJ":148682,"ï³Ĵ":148683,"ï³ĺ":148684,"ï³ľ":148685,"ï¹¼":148686,"￨":148687,"ðĿIJ©":148688,"ðĿĴļ":148689,"ðĿķĶ":148690,"ðĿķ¤":148691,"ðĿĸĮ":148692,"ðĿĹ£":148693,"ðĿĹ°":148694,"ðĿĹ´":148695,"ðĿĺĤ":148696,"ðĿĺ¥":148697,"ðĿĺ®":148698,"ðĿĺ¸":148699,"ðĿĻĢ":148700,"ðĿĽ¾":148701,"ðĿľı":148702,"ðŁĮģ":148703,"ðŁĮľ":148704,"ðŁĮ¥":148705,"ðŁĮ¯":148706,"ðŁįIJ":148707,"ðŁİĴ":148708,"ðŁıĶ":148709,"ðŁıķ":148710,"ðŁı®":148711,"ðŁIJĤ":148712,"ðŁIJī":148713,"ðŁIJ¹":148714,"ðŁĶķ":148715,"ðŁĶļ":148716,"ðŁķij":148717,"ðŁķ£":148718,"ðŁĹŀ":148719,"ðŁĹ¡":148720,"ðŁĹ¿":148721,"ðŁļĨ":148722,"ðŁļĬ":148723,"ðŁļĵ":148724,"ðŁļķ":148725,"ðŁļ¾":148726,"ðŁĽģ":148727,"ðŁĽİ":148728,"ðŁĽı":148729,"ðŁ¤´":148730,"ðŁ¥ķ":148731,"ðŁ¥ĸ":148732,"ðŁ¥ł":148733,"ðŁ¥¥":148734,"ðŁ¦Ĩ":148735,"ðŁ¦ī":148736,"ðŁ¦ļ":148737,"ðŁ§ij":148738,"ðŁ§¥":148739,"ðŁ§¿":148740,"Å°":148741,"ƺ":148742,"ɧ":148743,"àªĩ":148744,"ண":148745,"áĪĪ":148746,"áĬ¤":148747,"áĭ®":148748,"áĮĪ":148749,"áĮµ":148750,"ᥲ":148751,"âĵŁ":148752,"êĻ³":148753,"ê°Ĭ":148754,"ëķģ":148755,"ëķ¨":148756,"ìĬģ":148757,"例":148758,"גּ":148759,"ðĿĸį":148760,"ðĿĺĮ":148761,"ðĿĺ³":148762,"ðĿĻ©":148763,"ðŁįĻ":148764,"ðŁĸĸ":148765,"áī³":148766,"áĭ¨":148767,"áĸĩ":148768,"áŀĮ":148769,"ṧ":148770,"âķª":148771,"âŀļ":148772,"â²ĺ":148773,"êķ":148774,"êķ¥":148775,"路":148776,"ﮣ":148777,"ï¯ł":148778,"ðĿĴĸ":148779,"ðĿķĺ":148780,"ðĿĸĩ":148781,"ðĿĹŁ":148782,"ðĿĹª":148783,"ðĿĹ¯":148784,"ðĿĻł":148785,"ðŁĵı":148786,"à¦Ĺ":148787,"âĴ»":148788,"â²ł":148789,"ðĿĵµ":148790,"Ê£":148791,"à°ľ":148792,"áĬ¢":148793,"áŀIJ":148794,"ḷ":148795,"âĦĽ":148796,"âĩĢ":148797,"âĩĬ":148798,"êĴ¦":148799,"ê¦ł":148800,"ﮤ":148801,"ðŁįĽ":148802,"ðŁ¤Ľ":148803,"ᨾ":148804,"âŀº":148805,"áķ¯":148806,"áĽı":148807,"âĩĤ":148808,"âĶ¹":148809,"âĻĹ":148810,"ðŁĸ¨":148811,"ê¦ı":148812,"ર":148813,"áļ¨":148814,"ðŁ¤¥":148815,"ðŁ§¢":148816,"ãIJĤ":148817,"ãĦ¥":148818,"ðŁĸĮ":148819,"â¼Ĵ":148820,"ãĬ§":148821,"âį©":148822,"ðŁ¦ij":148823,"âĶ·":148824,"ï©IJ":148825,"ï©¡":148826,"ðĵĪ":148827,"ðĵĪĴ":148828,"â»Ħ":148829,"ï¨Ĵ":148830,"âĦª":148831,"Ò§":148832,"ÚĮ":148833,"âĢ¶":148834,"âºł":148835,"â»ģ":148836,"âĨ¸":148837,"áĦIJ":148838,"ãħIJ":148839,"à»Ħ":148840,"áĹª":148841,"âĨ¼":148842,"âĩĭ":148843,"âĩĺ":148844,"âĮij":148845,"âĸ©":148846,"ðĿIJĹ":148847,"ÄĬ":148848,"à¦ī":148849,"ìīł":148850,"ɤ":148851,"ßį":148852,"ßı":148853,"áµĹ":148854,"âĤ¥":148855,"âĵī":148856,"âĶł":148857,"âĶ¨":148858,"âķĦ":148859,"ä¤":148860,"ä¤Ģ":148861,"껸":148862,"ï®ģ":148863,"ðĵĤ":148864,"ðĵĤĥ":148865,"ðŁ¦ķ":148866,"ÆĽ":148867,"à¦ĩ":148868,"ãıĺ":148869,"﮼":148870,"Úĵ":148871,"ÚĿ":148872,"à¦ĵ":148873,"ද":148874,"á´ħ":148875,"á½Ļ":148876,"âģ¼":148877,"âĸİ":148878,"⼩":148879,"äĶ":148880,"äĶĢ":148881,"뻡":148882,"ìĽ½":148883,"íģĦ":148884,"良":148885,"ï±ī":148886,"ï¹»":148887,"ðĿĸĭ":148888,"ðĿĻĪ":148889,"ðĿĻª":148890,"ðĿĻ¶":148891,"ðŁIJĦ":148892,"ðŁIJĨ":148893,"áİ¢":148894,"á¸Į":148895,"âĿ´":148896,"ðŁı¸":148897,"ÈĿ":148898,"ɸ":148899,"Îħ":148900,"Ïľ":148901,"Ó¢":148902,"Õ¹":148903,"à´ħ":148904,"àºĪ":148905,"áĭ°":148906,"áijİ":148907,"áłµ":148908,"á¡ł":148909,"á´ī":148910,"ḵ":148911,"á¿´":148912,"âĵ£":148913,"âĶ¶":148914,"⽯":148915,"ê²¥":148916,"ê¿ĺ":148917,"ëģİ":148918,"ëİĪ":148919,"ëĶ¯":148920,"ë²°":148921,"ìĺ¯":148922,"ìĽ¸":148923,"ìŀĹ":148924,"ì§ĺ":148925,"쬬":148926,"ì·¬":148927,"íģħ":148928,"íĵĶ":148929,"íĽĿ":148930,"冷":148931,"魯":148932,"沈":148933,"ï¯ĸ":148934,"ðĿĵħ":148935,"ðĿĻĦ":148936,"ðŁĵ¶":148937,"ðŁĹĴ":148938,"ðŁ¥Ķ":148939,"ðŁ¥Ń":148940,"Å®":148941,"Å´":148942,"Æī":148943,"Æ«":148944,"Çģ":148945,"Ç£":148946,"Ǻ":148947,"Ǽ":148948,"Èį":148949,"ȯ":148950,"Éľ":148951,"ʬ":148952,"Ëģ":148953,"ˤ":148954,"˵":148955,"ÏĽ":148956,"Ò¤":148957,"Ò¬":148958,"Óı":148959,"ÓĽ":148960,"Ó¡":148961,"Ó³":148962,"ÔĮ":148963,"Ô¬":148964,"Õ³":148965,"Ù»":148966,"Úī":148967,"Ú§":148968,"Üľ":148969,"ߪ":148970,"à¤Ŀ":148971,"à¦Ľ":148972,"à¨Ĩ":148973,"àªķ":148974,"ડ":148975,"à®İ":148976,"à°¬":148977,"ൻ":148978,"ർ":148979,"à¶ł":148980,"à¶Ń":148981,"බ":148982,"à·Ĩ":148983,"༽":148984,"áĢļ":148985,"áħ¢":148986,"áĨ¸":148987,"áĪĢ":148988,"áĪķ":148989,"áĪ°":148990,"áī¡":148991,"áī¤":148992,"áĬ¦":148993,"áĬ«":148994,"áĭĭ":148995,"áĭį":148996,"áݯ":148997,"áijŃ":148998,"áķĹ":148999,"᣼":149000,"á¥Ĵ":149001,"á©ī":149002,"áŃº":149003,"á´¡":149004,"áµĺ":149005,"ᵼ":149006,"ᶳ":149007,"á¸ģ":149008,"á¸ĭ":149009,"á¹Ļ":149010,"á¹Ŀ":149011,"Ṧ":149012,"áºħ":149013,"á¼Ĥ":149014,"á½ĥ":149015,"á½į":149016,"ὧ":149017,"á¾·":149018,"âĢµ":149019,"âĤİ":149020,"âĦĿ":149021,"âħĢ":149022,"âĨŀ":149023,"âĨ§":149024,"âĩħ":149025,"âĪĥ":149026,"âīı":149027,"âī½":149028,"âĬŀ":149029,"âĬ¡":149030,"âĬ§":149031,"âĬ¶":149032,"âĭĦ":149033,"âİĴ":149034,"âİ¡":149035,"âİ£":149036,"âݪ":149037,"âıİ":149038,"âĵĥ":149039,"âĵĸ":149040,"âĵ¨":149041,"âķĭ":149042,"âķĸ":149043,"âķ¢":149044,"âķ²":149045,"âĸĨ":149046,"âĸĬ":149047,"âĸį":149048,"âĸ®":149049,"âĺ¡":149050,"âĺ¦":149051,"âĺ±":149052,"âĺ¿":149053,"âĻĺ":149054,"âĻĿ":149055,"âļ°":149056,"âĽij":149057,"âŀª":149058,"â¤Ŀ":149059,"⤢":149060,"⤷":149061,"⧫":149062,"â¨Ń":149063,"⨯":149064,"â±£":149065,"â²İ":149066,"⵼":149067,"ãħĶ":149068,"ãĪı":149069,"ãī²":149070,"ãī³":149071,"ãĬij":149072,"ãĭĽ":149073,"ãİIJ":149074,"겤":149075,"ê·¿":149076,"ê¹ŀ":149077,"껨":149078,"ê¼į":149079,"꿸":149080,"ëĥ¬":149081,"ëĩIJ":149082,"ëĭł":149083,"ëį¯":149084,"ëĹĮ":149085,"ëĹij":149086,"ë¥Ģ":149087,"ëªĥ":149088,"몯":149089,"뱡":149090,"ë³ĵ":149091,"ë³½":149092,"뵾":149093,"ìĤ³":149094,"ìħ¥":149095,"ìĩ½":149096,"ìı¨":149097,"ìı¸":149098,"ìķį":149099,"ìĸĸ":149100,"ìŁ¨":149101,"ì¢ĥ":149102,"ì¢į":149103,"ì¥ij":149104,"짼":149105,"ì©ĥ":149106,"ì®ľ":149107,"쮸":149108,"ì³ij":149109,"ì´¥":149110,"ì¾ĥ":149111,"íħ¦":149112,"íĪ¿":149113,"íĵ½":149114,"íķ³":149115,"íĸı":149116,"íĹł":149117,"íĿ«":149118,"ï¤ĵ":149119,"ï¤ĺ":149120,"ï¥İ":149121,"略":149122,"ï¦ħ":149123,"尿":149124,"ï§ĩ":149125,"ï¬Ĩ":149126,"דּ":149127,"ï®ĩ":149128,"ï®Ī":149129,"ï®Ŀ":149130,"ﮩ":149131,"ï®±":149132,"ï¯ĺ":149133,"ï¯Ļ":149134,"ﯢ":149135,"ﯣ":149136,"ﯤ":149137,"ﯥ":149138,"ï±Ĥ":149139,"ï²Ĩ":149140,"ﲪ":149141,"ï´¼":149142,"ïºī":149143,"ïºĬ":149144,"ﺥ":149145,"ðĿij¨":149146,"ðĿij©":149147,"ðĿij²":149148,"ðĿĴĮ":149149,"ðĿĴª":149150,"ðĿĴ®":149151,"ðĿĵĤ":149152,"ðĿĵĪ":149153,"ðĿĵ¯":149154,"ðĿĶ¨":149155,"ðĿķĢ":149156,"ðĿķĨ":149157,"ðĿķ¦":149158,"ðĿķ§":149159,"ðĿķ«":149160,"ðĿķ·":149161,"ðĿŵ":149162,"ðĿŸ":149163,"ðĿĺĦ":149164,"ðĿĺĻ":149165,"ðĿĺł":149166,"ðĿĺ¬":149167,"ðĿĻį":149168,"ðĿĻij":149169,"ðĿĻ¡":149170,"ðĿĻ¨":149171,"ðĿĻ·":149172,"ðĿļį":149173,"ðĿĽ¿":149174,"ðŁĥ":149175,"ðŁĥı":149176,"ðŁħĺ":149177,"ðŁī":149178,"ðŁīij":149179,"ðŁİ¡":149180,"ðŁİª":149181,"ðŁİ±":149182,"ðŁİ³":149183,"ðŁİº":149184,"ðŁıİ":149185,"ðŁıĹ":149186,"ðŁıļ":149187,"ðŁıŀ":149188,"ðŁı¦":149189,"ðŁı§":149190,"ðŁIJģ":149191,"ðŁIJħ":149192,"ðŁIJĵ":149193,"ðŁĴĤ":149194,"ðŁĵij":149195,"ðŁĵĵ":149196,"ðŁĵ¨":149197,"ðŁĵ«":149198,"ðŁĶĭ":149199,"ðŁĶŃ":149200,"ðŁĶ¯":149201,"ðŁķĹ":149202,"ðŁļĤ":149203,"ðŁļ¢":149204,"ðŁļ¦":149205,"ðŁļ¬":149206,"ðŁĽĭ":149207,"ðŁĽĮ":149208,"ðŁĽ¬":149209,"ðŁĽ¶":149210,"ðŁŁ¡":149211,"ðŁ¥ĺ":149212,"ðŁ¥Ł":149213,"ðŁ¥¦":149214,"ðŁ¦ĩ":149215,"ðŁ¦Ī":149216,"ðŁ§Ĭ":149217,"ðŁ§Ĺ":149218,"ðŁ§¤":149219,"Ê·":149220,"˹":149221,"á¹ļ":149222,"á½¥":149223,"âĦŁ":149224,"겯":149225,"껫":149226,"ë°·":149227,"ìĥĨ":149228,"ìĽĿ":149229,"ì¨ī":149230,"ì«ı":149231,"ï¯ķ":149232,"ðĿľĭ":149233,"ɲ":149234,"ÒŃ":149235,"ÓĪ":149236,"à½Ľ":149237,"áĭĵ":149238,"áĻŃ":149239,"áł©":149240,"á¹®":149241,"âĦĴ":149242,"âĨ»":149243,"âµĥ":149244,"ëĢ¨":149245,"볧":149246,"ìī¥":149247,"ìĮľ":149248,"ìŶ":149249,"ì¨Ī":149250,"쪾":149251,"íı½":149252,"íļĶ":149253,"íĽµ":149254,"露":149255,"ï¦IJ":149256,"ï§Ĺ":149257,"ï§ļ":149258,"אָ":149259,"ðĿIJĬ":149260,"ðĿķĹ":149261,"ðĿĹļ":149262,"ðĿļĸ":149263,"ðŁħ´":149264,"Èĥ":149265,"ÉĿ":149266,"ϱ":149267,"ÓĹ":149268,"ढ":149269,"áħł":149270,"áī¦":149271,"áijĮ":149272,"áĴ¼":149273,"áŀ¡":149274,"᳨":149275,"áłŃ":149276,"á¨ħ":149277,"á¨Ķ":149278,"á´ĺ":149279,"ᶦ":149280,"á¸İ":149281,"á¼ħ":149282,"á¼¹":149283,"âĨ¯":149284,"âĵİ":149285,"ãıĮ":149286,"êī":149287,"êīĤ":149288,"ëĨ§":149289,"ëĿ±":149290,"좡":149291,"íĪ½":149292,"ï¤ĩ":149293,"ï¤Ľ":149294,"ðĿIJķ":149295,"ðĿĵ¸":149296,"ðĿĵ¼":149297,"ðĿĹķ":149298,"ðĿĺĪ":149299,"ðŁı£":149300,"ðŁı¤":149301,"ðŁĹĦ":149302,"Ñ·":149303,"Òł":149304,"áµĸ":149305,"Ἠ":149306,"ë¬Ħ":149307,"ï°´":149308,"âĪ½":149309,"ÕŃ":149310,"Ú¹":149311,"à¥Ł":149312,"áĢĨ":149313,"áŀĴ":149314,"ãĢ¶":149315,"ꦫ":149316,"ï¸ĵ":149317,"ðĿIJĽ":149318,"ðĿĺĹ":149319,"ðŁıľ":149320,"ì«Ń":149321,"ðŁ§ŀ":149322,"à½Ĥ":149323,"âĨ¿":149324,"âĩı":149325,"âĵģ":149326,"âĶ§":149327,"âķģ":149328,"âķ¤":149329,"ê¦Ĺ":149330,"ꦤ":149331,"ðŁıĪ":149332,"áŀķ":149333,"Ô½":149334,"àªĹ":149335,"à¬Ĩ":149336,"âķķ":149337,"ï½ł":149338,"⼦":149339,"⼯":149340,"â¾·":149341,"âĶĸ":149342,"à¬ĵ":149343,"âĺĹ":149344,"âįĭ":149345,"ï¨Ŀ":149346,"â¼¥":149347,"寧":149348,"âĦĬ":149349,"ãĢ´":149350,"âį¢":149351,"ð¡Ī":149352,"ð¡Ī½":149353,"難":149354,"ãĢ»":149355,"ãıĥ":149356,"說":149357,"ï¨ĺ":149358,"ðŁIJĥ":149359,"ðŁĨĸ":149360,"ðŁĹ¾":149361,"ãĦĩ":149362,"Þĭ":149363,"â¼¼":149364,"ï¨Ń":149365,"ÞĢ":149366,"ÞĦ":149367,"ÞĪ":149368,"ÞIJ":149369,"âĮĦ":149370,"â»ĺ":149371,"ãŁ¢":149372,"áħ§":149373,"ðIJĮ¿":149374,"Ë»":149375,"à²Ĺ":149376,"áĢĩ":149377,"áŀĬ":149378,"âķĩ":149379,"ãĩ¼":149380,"ãİ°":149381,"ÕĴ":149382,"ÜĪ":149383,"ߥ":149384,"à¿IJ":149385,"áĢŁ":149386,"âĨ¥":149387,"âķĮ":149388,"â½Ģ":149389,"â½°":149390,"â¾Ĭ":149391,"äĦ":149392,"äĦĢ":149393,"ðĵIJ":149394,"ðĵIJį":149395,"ðŁİ¦":149396,"âĤ¯":149397,"âĬĺ":149398,"âĦį":149399,"ʵ":149400,"Ѷ":149401,"Úĥ":149402,"à¦Ķ":149403,"à´¦":149404,"áݶ":149405,"áĵķ":149406,"Ṩ":149407,"âĤł":149408,"âĩ°":149409,"âĹĴ":149410,"â¿Ĭ":149411,"ê·±":149412,"ì¹ķ":149413,"íĪ©":149414,"ïŃĢ":149415,"ðĿĴ¸":149416,"ðĿĵĬ":149417,"ðĿĺ©":149418,"Ǧ":149419,"É«":149420,"áĬ¨":149421,"ȹ":149422,"ʯ":149423,"Ϊ":149424,"ÚĢ":149425,"áĮ¸":149426,"áİ»":149427,"áıķ":149428,"áı´":149429,"á²Ĥ":149430,"Ὠ":149431,"âıĿ":149432,"âĺĻ":149433,"ëĥ¨":149434,"ëĦ¼":149435,"ëĪĻ":149436,"ë£ħ":149437,"ìĶ¼":149438,"ìķĿ":149439,"ìļ¬":149440,"ìľ±":149441,"ï¥Ĥ":149442,"惡":149443,"יּ":149444,"ïŃģ":149445,"ï³Ī":149446,"ðĿĶħ":149447,"ðĿĺ¤":149448,"ðĿĻı":149449,"ðĿĻĻ":149450,"ðŁķī":149451,"ðŁ§Ļ":149452,"á¸ij":149453,"ê´¼":149454,"ëģį":149455,"ëĹ´":149456,"ëĿ³":149457,"ë°ŀ":149458,"ë°¢":149459,"ëµĺ":149460,"ìĤĶ":149461,"ìĦĦ":149462,"ì¼ļ":149463,"íĢł":149464,"íĬ±":149465,"íĮĸ":149466,"ï¤ij":149467,"領":149468,"隸":149469,"ï´į":149470,"ðĿĺ·":149471,"Ĭ":149472,"Ŭ":149473,"ÆĢ":149474,"Æĭ":149475,"Æľ":149476,"Çij":149477,"Çĺ":149478,"Çŀ":149479,"Ç¥":149480,"Ç®":149481,"É°":149482,"ɶ":149483,"É·":149484,"ɽ":149485,"ÊĪ":149486,"ÊIJ":149487,"Ëİ":149488,"ËŁ":149489,"˦":149490,"˯":149491,"ÏIJ":149492,"Ïĵ":149493,"Ï¢":149494,"Ϥ":149495,"Ϫ":149496,"ÏŃ":149497,"Ï®":149498,"Ï»":149499,"Ñł":149500,"ÑŃ":149501,"Ò¨":149502,"ÓĿ":149503,"Ô¡":149504,"Ô·":149505,"Õī":149506,"Õĵ":149507,"Õĸ":149508,"Õļ":149509,"ÕĿ":149510,"Öİ":149511,"Ø¿":149512,"Úħ":149513,"Úį":149514,"ÚĶ":149515,"ÛĬ":149516,"Û¾":149517,"ÜĻ":149518,"ÝĴ":149519,"Ýĺ":149520,"ßĴ":149521,"ßĸ":149522,"à¤Ĭ":149523,"à¤IJ":149524,"à¦ı":149525,"à¦ĸ":149526,"à§Ł":149527,"મ":149528,"હ":149529,"à®ħ":149530,"à®Ĩ":149531,"à°¡":149532,"à°°":149533,"à²ļ":149534,"ಮ":149535,"ಯ":149536,"à´Ł":149537,"à´·":149538,"ൾ":149539,"à¶ij":149540,"à¶ŀ":149541,"༼":149542,"à½ĵ":149543,"áĢĵ":149544,"áĤ¦":149545,"áĥĸ":149546,"áĥŃ":149547,"áĥ¯":149548,"áħ¨":149549,"áħª":149550,"áĨ°":149551,"áĪģ":149552,"áĪİ":149553,"áĪĵ":149554,"áĪ¥":149555,"áĪ²":149556,"áĪ´":149557,"áĪ»":149558,"áīł":149559,"áī²":149560,"áī¶":149561,"áĬ£":149562,"áĬ¥":149563,"áĬª":149564,"áĭĺ":149565,"áĭ²":149566,"áĭ¶":149567,"áĮ£":149568,"áį¡":149569,"áį£":149570,"áݬ":149571,"áݾ":149572,"áIJ¡":149573,"áķķ":149574,"áĸ±":149575,"áĹIJ":149576,"áĹŃ":149577,"áĺī":149578,"áļ±":149579,"ἣ":149580,"áŀ¥":149581,"áŁĶ":149582,"áł£":149583,"áłª":149584,"áł°":149585,"áł´":149586,"á¤ĸ":149587,"ᥣ":149588,"á®":149589,"᮳":149590,"á¯":149591,"á¯Ļ":149592,"á°":149593,"á°į":149594,"á´Ĭ":149595,"á´¾":149596,"áµģ":149597,"áµİ":149598,"áµŀ":149599,"ᵤ":149600,"á¶ħ":149601,"á¶ĺ":149602,"ᶣ":149603,"ᶢ":149604,"ᶤ":149605,"ᶱ":149606,"ᶻ":149607,"á¸ī":149608,"á¸ŀ":149609,"Ḻ":149610,"á¹ĵ":149611,"á¹Ĺ":149612,"Ṫ":149613,"áºĬ":149614,"áºı":149615,"áºĽ":149616,"á¼ĥ":149617,"á¼Į":149618,"Ἷ":149619,"á½Ĥ":149620,"á½ĵ":149621,"á½Ĺ":149622,"ὦ":149623,"á¾±":149624,"á¾´":149625,"á¿ĺ":149626,"á¿Ł":149627,"Ὸ":149628,"âģĺ":149629,"âĤij":149630,"âĤĽ":149631,"âĤ¿":149632,"âĦĩ":149633,"âĦŀ":149634,"âĦ±":149635,"âĩŁ":149636,"âĩ²":149637,"âĪ¤":149638,"âĪ¶":149639,"âīĤ":149640,"âī¾":149641,"âĬ¨":149642,"âĬ³":149643,"âĬ·":149644,"âĭĮ":149645,"âĭĺ":149646,"âĮķ":149647,"âĮ¥":149648,"âĮµ":149649,"âĮº":149650,"âį£":149651,"âį²":149652,"âįµ":149653,"âİĩ":149654,"âıĥ":149655,"âıIJ":149656,"âıł":149657,"âı¤":149658,"âı¶":149659,"âı¸":149660,"âı¹":149661,"âijĤ":149662,"âĴ·":149663,"âĴº":149664,"âĵ¡":149665,"âĵ¤":149666,"âĶ¾":149667,"âĸĺ":149668,"âĸµ":149669,"âĹª":149670,"âĹ·":149671,"âĺ¨":149672,"âĺ«":149673,"âĺ²":149674,"âĺ³":149675,"âĻĨ":149676,"âļ¤":149677,"âļ¥":149678,"âĽĵ":149679,"⼴":149680,"âĽ¾":149681,"âŀ«":149682,"âŀ¿":149683,"⣷":149684,"â¤ij":149685,"⤫":149686,"⤶":149687,"⤽":149688,"⧪":149689,"â¨Ģ":149690,"⩽":149691,"⬡":149692,"⬢":149693,"⬤":149694,"â²ĸ":149695,"Ⲫ":149696,"âµĢ":149697,"⸮":149698,"⸽":149699,"ãĢł":149700,"ãĢ·":149701,"ãĦĮ":149702,"ãĦĺ":149703,"ãħij":149704,"ãĪİ":149705,"ãĪIJ":149706,"ãĬľ":149707,"ãĮĵ":149708,"ãĮł":149709,"ãİŁ":149710,"ãݤ":149711,"ãݧ":149712,"㬮":149713,"äĪ":149714,"äĪĢ":149715,"ä°":149716,"ä°Ģ":149717,"êħ":149718,"êħī":149719,"êĩĹ":149720,"êĪ":149721,"êĪį":149722,"ê§Ĥ":149723,"ê§Ĭ":149724,"êªĢ":149725,"ê²Ī":149726,"ê²į":149727,"ê³Ģ":149728,"êµł":149729,"ê½IJ":149730,"ê¾Ī":149731,"꿱":149732,"ëĥı":149733,"ëĦij":149734,"ëħ¤":149735,"ëĩ¸":149736,"ëĪ¼":149737,"ëīħ":149738,"ëĬ£":149739,"ëĭº":149740,"ëįŀ":149741,"ëIJĮ":149742,"ëķ¸":149743,"ëĺł":149744,"ëĻĩ":149745,"ëĻĪ":149746,"ëľ½":149747,"ëŀĶ":149748,"ëłľ":149749,"ë£IJ":149750,"ë§Ģ":149751,"ë§Ĭ":149752,"ëªĢ":149753,"ë¬Ń":149754,"믾":149755,"ë³ľ":149756,"ë´Ĭ":149757,"ëµī":149758,"ë·ľ":149759,"ë¸Ģ":149760,"ë¹ĭ":149761,"ìģĦ":149762,"ìĤ£":149763,"ìĤ»":149764,"ìĦµ":149765,"ìħĴ":149766,"ìīĪ":149767,"ìīĶ":149768,"ìĬĮ":149769,"ìĬĻ":149770,"ìIJ´":149771,"ìĵº":149772,"ìķļ":149773,"ìķº":149774,"ìĸľ":149775,"ìĹª":149776,"ìĺľ":149777,"ìĻ¤":149778,"ìļĽ":149779,"ìļº":149780,"ìĿħ":149781,"ìĿı":149782,"ìĿŃ":149783,"ìĿ¶":149784,"ìłĽ":149785,"ì¡Ī":149786,"ì¢ī":149787,"ì¢Ķ":149788,"ì©ł":149789,"ìŃĮ":149790,"쯩":149791,"ì´£":149792,"ì¸ķ":149793,"ì¹Ł":149794,"쾡":149795,"ì¿Ļ":149796,"íģĩ":149797,"íģī":149798,"íĩĢ":149799,"íĪ¶":149800,"íĸij":149801,"íĸ¤":149802,"íĹħ":149803,"íľı":149804,"íĿĿ":149805,"ï¤Ĵ":149806,"ï¤ķ":149807,"郎":149808,"ï¥ħ":149809,"ï¥ĩ":149810,"ï¥ı":149811,"ï¥ļ":149812,"ï¥Ł":149813,"ï¦Ħ":149814,"ï¦Ī":149815,"令":149816,"囹":149817,"零":149818,"ï§ģ":149819,"ï§ĥ":149820,"ï§Ķ":149821,"ï§ł":149822,"泥":149823,"燐":149824,"ïŃIJ":149825,"ïŃĸ":149826,"ïѦ":149827,"ïŃ´":149828,"ïѵ":149829,"ïѶ":149830,"ïѸ":149831,"ï®Į":149832,"ï®İ":149833,"ï®ŀ":149834,"ï®Ł":149835,"ﮡ":149836,"ﮪ":149837,"ï¯Ķ":149838,"ï¯Ĺ":149839,"ï¯ļ":149840,"ï¯Ľ":149841,"ï¯Ŀ":149842,"ï¯Ł":149843,"ﯧ":149844,"ﯨ":149845,"ﯫ":149846,"ﯯ":149847,"ﯰ":149848,"ﯱ":149849,"ﯲ":149850,"ﯳ":149851,"ﯴ":149852,"ﯵ":149853,"ﯶ":149854,"ï°Ģ":149855,"ï±ħ":149856,"ï±Ķ":149857,"ï±´":149858,"ï²ģ":149859,"ï³ķ":149860,"ï·½":149861,"ï¸ķ":149862,"︱":149863,"ï¹£":149864,"ï¹½":149865,"ï»į":149866,"ï¾±":149867,"ðĿIJĻ":149868,"ðĿIJ½":149869,"ðĿij¤":149870,"ðĿij®":149871,"ðĿijµ":149872,"ðĿĴĥ":149873,"ðĿĴĦ":149874,"ðĿĵŃ":149875,"ðĿĵ·":149876,"ðĿĶĸ":149877,"ðĿĶŀ":149878,"ðĿĶ¢":149879,"ðĿĶ¦":149880,"ðĿĶ¬":149881,"ðĿķĦ":149882,"ðĿķĬ":149883,"ðĿķİ":149884,"ðĿķĻ":149885,"ðĿķľ":149886,"ðĿķŃ":149887,"ðĿķ³":149888,"ðĿķ¸":149889,"ðĿķ¾":149890,"ðĿĸī":149891,"ðĿĸı":149892,"ðĿĺĩ":149893,"ðĿĺī":149894,"ðĿĺĸ":149895,"ðĿĺĽ":149896,"ðĿĺŀ":149897,"ðĿĺ«":149898,"ðĿĺ¾":149899,"ðĿĻĩ":149900,"ðĿĻī":149901,"ðĿĻĭ":149902,"ðĿĻİ":149903,"ðĿĻĺ":149904,"ðĿĻ¥":149905,"ðĿļĥ":149906,"ðĿļIJ":149907,"ðĿļĶ":149908,"ðĿľĥ":149909,"ðŁĦ·":149910,"ðŁħĿ":149911,"ðŁħ¾":149912,"ðŁĨĤ":149913,"ðŁĨĵ":149914,"ðŁĮĤ":149915,"ðŁĮĨ":149916,"ðŁĮī":149917,"ðŁĮij":149918,"ðŁĮĺ":149919,"ðŁĮ©":149920,"ðŁĮ«":149921,"ðŁį¢":149922,"ðŁį¥":149923,"ðŁİĽ":149924,"ðŁİ¢":149925,"ðŁİ´":149926,"ðŁij¡":149927,"ðŁĴ¾":149928,"ðŁĵŃ":149929,"ðŁĶĪ":149930,"ðŁĶ¦":149931,"ðŁĶ²":149932,"ðŁĶ³":149933,"ðŁķĵ":149934,"ðŁķķ":149935,"ðŁķĺ":149936,"ðŁķŁ":149937,"ðŁķ·":149938,"ðŁĹ³":149939,"ðŁļĦ":149940,"ðŁļĶ":149941,"ðŁļĸ":149942,"ðŁĽIJ":149943,"ðŁĽ¤":149944,"ðŁĽ¸":149945,"ðŁł":149946,"ðŁł³":149947,"ðŁ¤¹":149948,"ðŁ¥ĥ":149949,"ðŁ¥¨":149950,"ðŁ¥ª":149951,"ðŁ¥¾":149952,"ðŁ¦ĥ":149953,"ðŁ¦Ĵ":149954,"ðŁ¦Ļ":149955,"ðŁ¦¶":149956,"ðŁ§ł":149957,"ðŁ§ª":149958,"ðŁ§Ń":149959,"ðŁ§²":149960,"ð£·":149961,"ð£·Ń":149962,"ð¦ĺ":149963,"ð¦ĺĴ":149964,"Æij":149965,"ÇĻ":149966,"È®":149967,"Øł":149968,"ÚĦ":149969,"ÜĢ":149970,"ߢ":149971,"áīĢ":149972,"áĬIJ":149973,"áİł":149974,"áºŀ":149975,"ëĪŀ":149976,"ëķŁ":149977,"ë£ģ":149978,"ë¤Ĺ":149979,"ìĦ¥":149980,"ìħij":149981,"ìĸIJ":149982,"ìĽĽ":149983,"ì£ķ":149984,"íİı":149985,"íĽĵ":149986,"梁":149987,"ï³Ľ":149988,"ï´«":149989,"ðĸ§":149990,"ðĸ§·":149991,"ðĿķģ":149992,"ðŁIJª":149993,"ðŁĴĪ":149994,"ðŁĵł":149995,"ðŁķĽ":149996,"ðŁķ´":149997,"ÑĿ":149998,"ÓĬ":149999,"ॲ":150000,"પ":150001,"áĥ¤":150002,"áįIJ":150003,"ᶰ":150004,"á¼Ŀ":150005,"Ὡ":150006,"âĭĭ":150007,"âĴ½":150008,"âĻ¾":150009,"â½Ķ":150010,"⾯":150011,"ãĦĴ":150012,"ãħļ":150013,"ëIJį":150014,"ë·ģ":150015,"ìĭĢ":150016,"ìļĿ":150017,"쥰":150018,"캴":150019,"íĭī":150020,"íĿ½":150021,"ï¦Ģ":150022,"樂":150023,"ï§ħ":150024,"ï§ĵ":150025,"ïŃ¯":150026,"ï®Ĩ":150027,"ðIJ¤ķ":150028,"ðĿIJŁ":150029,"ðĿĴħ":150030,"ðĿĵľ":150031,"ðĿĶ°":150032,"ðĿĶ»":150033,"ðĿĺį":150034,"ðĿĻ¯":150035,"ðŁĦ½":150036,"ðŁħĤ":150037,"ðŁħĶ":150038,"ðŁħ½":150039,"ðŁĵ´":150040,"ðŁ§ĸ":150041,"ÓĴ":150042,"Ḳ":150043,"ëī¼":150044,"Çı":150045,"Èĵ":150046,"ʸ":150047,"ÕĤ":150048,"Ûħ":150049,"ß¡":150050,"ߣ":150051,"ய":150052,"à°Ī":150053,"ಸ":150054,"ຮ":150055,"à¼ķ":150056,"áĢİ":150057,"áĨ¡":150058,"áIJĭ":150059,"áIJķ":150060,"áij¯":150061,"áŀĨ":150062,"á¨ķ":150063,"á©Ī":150064,"âģħ":150065,"âĨļ":150066,"âĶİ":150067,"âł©":150068,"â²Ĥ":150069,"â²Ķ":150070,"Ⲩ":150071,"ãĬļ":150072,"íĵ²":150073,"ðĿijĪ":150074,"ðĿij¬":150075,"ðĿij¹":150076,"ðĿĴ¾":150077,"ðĿĵ±":150078,"ðĿĵ½":150079,"ðĿķ¯":150080,"ðĿķ»":150081,"ðĿĺ½":150082,"ðĿļĨ":150083,"ðŁĦ°":150084,"ðŁIJ¨":150085,"Òķ":150086,"à²ħ":150087,"ï¨Ĩ":150088,"ðĿij°":150089,"ðŁĦ¸":150090,"Ôİ":150091,"Øį":150092,"Ùµ":150093,"ಶ":150094,"áĢĪ":150095,"áĺĹ":150096,"᳸":150097,"á¡¡":150098,"ᨲ":150099,"á©ģ":150100,"á´·":150101,"ᵧ":150102,"âķ¨":150103,"âļģ":150104,"â¾Ŀ":150105,"ãĢ¼":150106,"ãĦı":150107,"êĴ«":150108,"ꦥ":150109,"ꦩ":150110,"ꦲ":150111,"ìĺ¼":150112,"íĵIJ":150113,"ðĵĩ":150114,"ðĵĩ¼":150115,"ðĿķ¿":150116,"ðŁĽ´":150117,"먾":150118,"ವ":150119,"à´İ":150120,"à¼Ģ":150121,"âĩĸ":150122,"ãĪ«":150123,"âĵĢ":150124,"áħ´":150125,"áļ¾":150126,"áĽŀ":150127,"Ἣ":150128,"ᥴ":150129,"âĨĽ":150130,"âĨ¶":150131,"âĩ¤":150132,"âķŁ":150133,"âĺ·":150134,"âļIJ":150135,"ðŁ§´":150136,"á¹³":150137,"âĶį":150138,"âĶĴ":150139,"âĶ©":150140,"âĶ¦":150141,"â¾µ":150142,"àªľ":150143,"ત":150144,"âĩĻ":150145,"âĶ±":150146,"âķĢ":150147,"â½Ĭ":150148,"ï½Ł":150149,"ଡ":150150,"ðł®":150151,"ðł®·":150152,"âķĥ":150153,"â°Ķ":150154,"ãĬ¦":150155,"ðŁİIJ":150156,"ãĩ°":150157,"â¼Ŀ":150158,"â¾Ķ":150159,"â½Ĵ":150160,"âłĴ":150161,"都":150162,"ï©Ĵ":150163,"免":150164,"ï©ĸ":150165,"ðĵı¸":150166,"ãĮĥ":150167,"ðĸ¤":150168,"ðĸ¤IJ":150169,"ï¦Ń":150170,"âĬħ":150171,"â¾³":150172,"ä´¥":150173,"ï©ķ":150174,"ðŁĮĶ":150175,"áŀĭ":150176,"âļį":150177,"â¼ĭ":150178,"ãİĺ":150179,"ðIJĮ²":150180,"É©":150181,"áİij":150182,"âĨ®":150183,"âĩĥ":150184,"âļİ":150185,"ãĩ±":150186,"ãĭ©":150187,"ãĮ¶":150188,"êĻª":150189,"ëݬ":150190,"ï¨IJ":150191,"ï¨Ľ":150192,"ï©Ĭ":150193,"ï©į":150194,"ðĵħ":150195,"ðĵħº":150196,"Ï¡":150197,"Èij":150198,"ÉĤ":150199,"Ôĵ":150200,"ßİ":150201,"à´§":150202,"áĢī":150203,"áĢĭ":150204,"áĢij":150205,"áĢł":150206,"áļĻ":150207,"á¨Ħ":150208,"ᨩ":150209,"ᨹ":150210,"á©ĵ":150211,"ᬾ":150212,"á´Ļ":150213,"áµij":150214,"âĤŃ":150215,"âĨ°":150216,"âľģ":150217,"â½IJ":150218,"ãĭ¯":150219,"ãĮ½":150220,"íĨ¢":150221,"錄":150222,"ðŁĤ":150223,"ðŁĤ»":150224,"ÈĴ":150225,"ͺ":150226,"Ô¥":150227,"Õij":150228,"Ú¶":150229,"à§İ":150230,"ථ":150231,"àºĸ":150232,"àºľ":150233,"ຽ":150234,"áĥ»":150235,"áħ¯":150236,"áĭŀ":150237,"áĸķ":150238,"á´Ī":150239,"á¶Ĩ":150240,"Ḿ":150241,"á¹¼":150242,"Ῠ":150243,"âĦĭ":150244,"âĦŃ":150245,"âĪ±":150246,"âĮĵ":150247,"âĶĩ":150248,"âĶ¢":150249,"â±®":150250,"â²Ħ":150251,"ãĩ¾":150252,"ãĪ¬":150253,"븡":150254,"ìIJī":150255,"íĻĽ":150256,"ðĿķª":150257,"ƹ":150258,"Ͳ":150259,"Óģ":150260,"Û¼":150261,"ফ":150262,"áħŁ":150263,"áīĨ":150264,"áįĪ":150265,"áºĸ":150266,"á½ī":150267,"âĶ¸":150268,"⽩":150269,"êľ":150270,"êľ¥":150271,"êµħ":150272,"ëĤĶ":150273,"ëĦł":150274,"ëĩĹ":150275,"ëĻĿ":150276,"ìļ¯":150277,"ìļ·":150278,"ìŁĽ":150279,"ì·IJ":150280,"íŁ¬":150281,"íŁ®":150282,"íŁ°":150283,"ï¦Ĩ":150284,"鈴":150285,"ï²ŀ":150286,"ﳤ":150287,"ï³¥":150288,"ðIJĮ¸":150289,"ðĿĶı":150290,"ðĿķ®":150291,"ðĿĺ£":150292,"à¦Ī":150293,"âıı":150294,"ãĦĸ":150295,"ê²ĩ":150296,"ëĸĺ":150297,"ëľ·":150298,"ëŀĴ":150299,"ë¡ĵ":150300,"ë¢ī":150301,"ë£ĥ":150302,"ë§ĭ":150303,"ë²ĭ":150304,"ìĤ·":150305,"ìĪķ":150306,"ìĮ¨":150307,"ìĵ»":150308,"ìĸĬ":150309,"ìĻ¬":150310,"ìĿ»":150311,"ì¦ģ":150312,"쵤":150313,"ì·ĥ":150314,"íĢľ":150315,"íħī":150316,"íįł":150317,"íıħ":150318,"íij±":150319,"íķķ":150320,"íĸł":150321,"íĿķ":150322,"ÆĻ":150323,"Æļ":150324,"Æŀ":150325,"Çĥ":150326,"ÇĬ":150327,"Çľ":150328,"Ǥ":150329,"ÇŃ":150330,"ǹ":150331,"ÈĢ":150332,"Èģ":150333,"Èħ":150334,"Èī":150335,"ÈĹ":150336,"ÈŁ":150337,"Ȥ":150338,"È¥":150339,"Ȩ":150340,"ȵ":150341,"Ⱥ":150342,"È»":150343,"ÉĮ":150344,"É®":150345,"Êħ":150346,"Ê¥":150347,"ʨ":150348,"Ëĵ":150349,"ËĶ":150350,"Ëł":150351,"Ë£":150352,"˸":150353,"Í´":150354,"ÏĹ":150355,"Ïĺ":150356,"ÏĻ":150357,"Ïļ":150358,"ÏĿ":150359,"Ϩ":150360,"Ϭ":150361,"Ͼ":150362,"Ï¿":150363,"Ѫ":150364,"ÒĢ":150365,"Òľ":150366,"Ò¼":150367,"Ò½":150368,"ÓĤ":150369,"Óħ":150370,"Óĩ":150371,"Óį":150372,"Óĸ":150373,"ÓŁ":150374,"Ó«":150375,"Ó±":150376,"ÔĨ":150377,"Ôĩ":150378,"Ôº":150379,"Õĭ":150380,"Öī":150381,"ØĪ":150382,"ØĬ":150383,"ؽ":150384,"ؾ":150385,"Ù·":150386,"ÚĤ":150387,"ÚĬ":150388,"Úĸ":150389,"ÚĹ":150390,"Ú£":150391,"Ú«":150392,"Ú¸":150393,"ÛĢ":150394,"Ûį":150395,"Û½":150396,"Üī":150397,"ܤ":150398,"ݧ":150399,"Ý´":150400,"Þĥ":150401,"Þ¤":150402,"Þ¥":150403,"ßļ":150404,"߼":150405,"ߤ":150406,"àłį":150407,"àłĵ":150408,"àł³":150409,"à¡¢":150410,"à¥ł":150411,"à§ł":150412,"৺":150413,"à¨Ĭ":150414,"à¨IJ":150415,"ਮ":150416,"ਯ":150417,"ਰ":150418,"ਸ":150419,"àªĨ":150420,"ળ":150421,"વ":150422,"ઽ":150423,"à¬Į":150424,"à¬ĺ":150425,"ଽ":150426,"à®ĥ":150427,"ஸ":150428,"à°Ĩ":150429,"à°ķ":150430,"à°¦":150431,"à²Ĩ":150432,"à²Ĭ":150433,"à²Į":150434,"à²IJ":150435,"à²Ľ":150436,"ತ":150437,"ದ":150438,"ಪ":150439,"ಲ":150440,"ಹ":150441,"à´Ĩ":150442,"à´ı":150443,"à´Ĺ":150444,"à´«":150445,"à´¹":150446,"ൺ":150447,"ൽ":150448,"à¶ħ":150449,"à¶Ĭ":150450,"à¶Ķ":150451,"ට":150452,"ණ":150453,"ධ":150454,"à¼Ħ":150455,"à¼ħ":150456,"à¼Ĭ":150457,"à½Ļ":150458,"ཡ":150459,"ཧ":150460,"à¿Ģ":150461,"à¿Ļ":150462,"áĢĿ":150463,"áĢ§":150464,"áĢ©":150465,"áĢ¿":150466,"áģµ":150467,"áĤģ":150468,"áĤ½":150469,"áĥĤ":150470,"áĥª":150471,"áĦĬ":150472,"áĦ¢":150473,"áħ¦":150474,"áħŃ":150475,"áĨ®":150476,"áĨ±":150477,"áĨ»":150478,"áĩ":150479,"áĩĤ":150480,"áĪħ":150481,"áĪī":150482,"áĪĮ":150483,"áĪIJ":150484,"áĪĴ":150485,"áĪĻ":150486,"áĪļ":150487,"áĪľ":150488,"áĪŀ":150489,"áĪ©":150490,"áĪ³":150491,"áĪº":150492,"áĪ½":150493,"áīħ":150494,"áī¢":150495,"áī±":150496,"áī´":150497,"áĬĥ":150498,"áĬį":150499,"áĬĸ":150500,"áĬ®":150501,"áĬ¸":150502,"áĭĽ":150503,"áĭĿ":150504,"áĭ³":150505,"áĮģ":150506,"áĮħ":150507,"áĮ¥":150508,"áĮ¦":150509,"áĮ¨":150510,"áįĬ":150511,"áįį":150512,"áįķ":150513,"áįĸ":150514,"áį¢":150515,"áį¤":150516,"áİĴ":150517,"áݪ":150518,"áıģ":150519,"áıIJ":150520,"áıŁ":150521,"áIJĤ":150522,"áIJĸ":150523,"áIJĿ":150524,"áIJŀ":150525,"áIJŁ":150526,"áIJł":150527,"áijĸ":150528,"áĴĭ":150529,"áĴį":150530,"áĴ¡":150531,"áĵ«":150532,"áĶķ":150533,"áķĭ":150534,"áķij":150535,"áķĻ":150536,"áķļ":150537,"áķĽ":150538,"áķ¤":150539,"áķ¦":150540,"áķ®":150541,"áķ¼":150542,"áĸĵ":150543,"áĹĹ":150544,"áĹ¢":150545,"áĹ¯":150546,"áĹ·":150547,"áĺĦ":150548,"áĺij":150549,"áĽĤ":150550,"áĽĻ":150551,"áŀį":150552,"áłĨ":150553,"áł¡":150554,"᳦":150555,"áł®":150556,"áł¯":150557,"áł²":150558,"áł·":150559,"á¡į":150560,"á¡ŀ":150561,"ᡤ":150562,"á¡´":150563,"ᡵ":150564,"á¤ĵ":150565,"á¥ĸ":150566,"ᥰ":150567,"ᨦ":150568,"ᨧ":150569,"ᨨ":150570,"ᨪ":150571,"ᨬ":150572,"ᨯ":150573,"ᨳ":150574,"ᨵ":150575,"á©ĥ":150576,"á¬ķ":150577,"áŃ£":150578,"á±":150579,"á±ļ":150580,"á²ł":150581,"á´ĵ":150582,"á´¶":150583,"áµĤ":150584,"áµĮ":150585,"áµ¥":150586,"áµ´":150587,"á¶ĩ":150588,"á¸Ī":150589,"ḳ":150590,"ḧ":150591,"Ḵ":150592,"Ḿ":150593,"á¹Ģ":150594,"á¹ĸ":150595,"á¹Ł":150596,"á¹ł":150597,"ṫ":150598,"á¹±":150599,"á¹·":150600,"ṿ":150601,"áºĦ":150602,"áºį":150603,"áºij":150604,"áºĹ":150605,"á¼ī":150606,"á¼ĵ":150607,"á¼Ń":150608,"á½ĭ":150609,"á½Ĵ":150610,"á½ł":150611,"á½£":150612,"á¾Ħ":150613,"á¾ı":150614,"á¾ij":150615,"á¾Ĺ":150616,"ᾦ":150617,"ᾧ":150618,"á¾¾":150619,"á¿Ħ":150620,"á¿ĵ":150621,"á¿¡":150622,"Ῥ":150623,"âģļ":150624,"âĤĮ":150625,"âĦģ":150626,"âĦĶ":150627,"âĦ£":150628,"âĦ§":150629,"âĦ¯":150630,"âĦ°":150631,"âĦ´":150632,"âħħ":150633,"âĨľ":150634,"âĨ«":150635,"âĨŃ":150636,"âĨ±":150637,"âĨ¹":150638,"âĨ½":150639,"âĩĩ":150640,"âĩľ":150641,"âĩµ":150642,"âĪī":150643,"âĪĬ":150644,"âĪĸ":150645,"âĪľ":150646,"âĪ¾":150647,"âīĢ":150648,"âīĭ":150649,"âīĮ":150650,"âīĵ":150651,"âīľ":150652,"âī´":150653,"âī¿":150654,"âĬĬ":150655,"âĬĭ":150656,"âĬĶ":150657,"âĬĸ":150658,"âĬ£":150659,"âĬ¦":150660,"âĭİ":150661,"âĭª":150662,"âĭ²":150663,"âĮ¦":150664,"âĮ§":150665,"âįº":150666,"âİĪ":150667,"âݨ":150668,"âݬ":150669,"âݳ":150670,"âݼ":150671,"âݾ":150672,"âıĮ":150673,"âıļ":150674,"âı«":150675,"âı¯":150676,"âıµ":150677,"âĴľ":150678,"âĴĿ":150679,"âĴ«":150680,"âĵĦ":150681,"âĵĬ":150682,"âĵĻ":150683,"âĵ©":150684,"âĶij":150685,"âĶĻ":150686,"âĶļ":150687,"âĶ¥":150688,"âķħ":150689,"âķī":150690,"âķį":150691,"âķı":150692,"âķŀ":150693,"âĸļ":150694,"âĸ¯":150695,"âĹĥ":150696,"âĹļ":150697,"âŬ":150698,"âĹ´":150699,"âĺĪ":150700,"âĺ¤":150701,"âĺ¥":150702,"âĺ§":150703,"âĺ¬":150704,"âĻģ":150705,"âĻ±":150706,"âļĥ":150707,"âļĦ":150708,"âļħ":150709,"âļı":150710,"âļļ":150711,"âļŀ":150712,"âļŁ":150713,"âļ±":150714,"âļ²":150715,"âľĢ":150716,"⾣":150717,"âľ¢":150718,"âĿµ":150719,"âŁ¡":150720,"⣦":150721,"⣧":150722,"âŁ³":150723,"âŁ¾":150724,"âŁ¿":150725,"âłĩ":150726,"â¤Ħ":150727,"⤺":150728,"â¥Ĥ":150729,"⥹":150730,"â§ī":150731,"⧼":150732,"⧽":150733,"â¨į":150734,"â¬Ĭ":150735,"⬣":150736,"âŃŀ":150737,"â®ŀ":150738,"⮳":150739,"â¯Ī":150740,"â¯ij":150741,"ⱳ":150742,"â±±":150743,"â²Ń":150744,"â´¹":150745,"âµķ":150746,"⸾":150747,"⺫":150748,"â¼Ĩ":150749,"â¼ł":150750,"â½Ł":150751,"â½¼":150752,"â¾Ľ":150753,"⾧":150754,"â¿ĥ":150755,"â¿»":150756,"ãĤķ":150757,"ãĤŁ":150758,"ãĦĽ":150759,"ãĦ¡":150760,"ãĦ¶":150761,"ãĦº":150762,"ãħĴ":150763,"ãħŁ":150764,"ãĨĢ":150765,"ãĩ»":150766,"ãĪij":150767,"ãĪŃ":150768,"ãĪ®":150769,"ãĪ³":150770,"ãĪ¹":150771,"ãī¥":150772,"ãī¦":150773,"ãī¹":150774,"ãī¿":150775,"ãĬŀ":150776,"ãĬ¨":150777,"ãĭij":150778,"ãĭ¥":150779,"ãĭ´":150780,"ãĭº":150781,"ãİĦ":150782,"ãİķ":150783,"ãݯ":150784,"ãıĤ":150785,"ãıĪ":150786,"ãıĵ":150787,"ãıĸ":150788,"ãı±":150789,"ãIJ±":150790,"ãŁģ":150791,"ã¢":150792,"㢨":150793,"ã¨":150794,"㨳":150795,"㫪":150796,"ã«´":150797,"㶳":150798,"㺾":150799,"äĢ":150800,"äĢĢ":150801,"äĭ":150802,"äĭĮ":150803,"äĮĢ":150804,"äIJĢ":150805,"äłĢ":150806,"äł":150807,"äł¼":150808,"ä§":150809,"ä§ŀ":150810,"䨰":150811,"䨺":150812,"ä´Ģ":150813,"ä·":150814,"ä·ħ":150815,"ä·¸":150816,"êĤ":150817,"êĤ«":150818,"êĮ":150819,"êĮ¼":150820,"êį":150821,"êį²":150822,"êĴµ":150823,"êĵ":150824,"êĵ½":150825,"êĻŃ":150826,"êĿĽ":150827,"êĿ¥":150828,"êŀ":150829,"êŀĬ":150830,"ê¦Ĩ":150831,"ê¦ĩ":150832,"ê¦Ł":150833,"ꦨ":150834,"ê§Ī":150835,"ê©":150836,"ê©Ł":150837,"êªĭ":150838,"êªij":150839,"êªķ":150840,"êªĹ":150841,"êªľ":150842,"ꪮ":150843,"ꪱ":150844,"ꪻ":150845,"ꪼ":150846,"ê«Ģ":150847,"ê«Ŀ":150848,"ê°ĥ":150849,"ê°ĺ":150850,"ê±ľ":150851,"ê²ĵ":150852,"ê²ļ":150853,"ê³Ļ":150854,"ê³¾":150855,"ê´Ĺ":150856,"ê´Ļ":150857,"êµĽ":150858,"ê¶ĥ":150859,"ê¶ķ":150860,"궨":150861,"긩":150862,"긿":150863,"ê¹Ħ":150864,"ê¹Ĩ":150865,"ê¹ī":150866,"ê¹ĵ":150867,"ê¹¢":150868,"ê¹£":150869,"깸":150870,"꺳":150871,"ê¿ı":150872,"ê¿ķ":150873,"꿧":150874,"ëĢ©":150875,"ëģħ":150876,"ëĥµ":150877,"ëĦĸ":150878,"ëĦĹ":150879,"ëĦ¢":150880,"ëħĤ":150881,"ëĨIJ":150882,"ëĩľ":150883,"ëĪĭ":150884,"ëĪļ":150885,"ëīį":150886,"ëī¨":150887,"ëĬļ":150888,"ëĬ¡":150889,"ëĭľ":150890,"ëĭª":150891,"ëĮĺ":150892,"ëĮ¤":150893,"ëĮ¸":150894,"ëİŁ":150895,"ëı¨":150896,"ëIJĦ":150897,"ëIJı":150898,"ëIJ´":150899,"ëIJ¸":150900,"ëijģ":150901,"ëij¿":150902,"ëĴ¨":150903,"ëĵ·":150904,"ëĶ®":150905,"ëĶ²":150906,"ëķ§":150907,"ëĸĶ":150908,"ëĸª":150909,"ëĺŃ":150910,"ëļĢ":150911,"ëļł":150912,"ëĽĶ":150913,"뼩":150914,"ëľħ":150915,"ëŀķ":150916,"ëŀ°":150917,"ëŁIJ":150918,"ëł¡":150919,"ë¡ŀ":150920,"ë¡£":150921,"롵":150922,"ë£Ħ":150923,"ë£į":150924,"뤳":150925,"ë¦į":150926,"ë¦ı":150927,"릳":150928,"ë§Ħ":150929,"ë§Ĩ":150930,"ë§į":150931,"맾":150932,"맫":150933,"맻":150934,"먮":150935,"ë©Ĥ":150936,"ë©Ń":150937,"몴":150938,"묾":150939,"묳":150940,"묫":150941,"묾":150942,"ëѬ":150943,"ë®ĺ":150944,"뮹":150945,"ë¯ķ":150946,"ë¯ľ":150947,"ë°¨":150948,"ë°ª":150949,"ë±Ķ":150950,"ë²ĺ":150951,"ë²Ľ":150952,"ë²±":150953,"ë²´":150954,"ë´½":150955,"뵤":150956,"뵨":150957,"ë·Ĺ":150958,"ë·ĺ":150959,"ë¸ĵ":150960,"븾":150961,"빪":150962,"ëºĥ":150963,"ëºĺ":150964,"뺵":150965,"ë»´":150966,"ë¼IJ":150967,"ë¾Ķ":150968,"ìģŃ":150969,"ìĤł":150970,"ìĤ®":150971,"ìĥı":150972,"ìĥĻ":150973,"ìĦº":150974,"ìħ¢":150975,"ìĨĢ":150976,"ìĨħ":150977,"ìĨ¤":150978,"ìĨ¦":150979,"ìĨ¬":150980,"ìĩ±":150981,"ìĪµ":150982,"ìĭ¨":150983,"ìĭ´":150984,"ìĮ°":150985,"ìįľ":150986,"ìİĹ":150987,"ìİĺ":150988,"ìݼ":150989,"ìijī":150990,"ìijĿ":150991,"ìij»":150992,"ìĴĶ":150993,"ìĴ¯":150994,"ìĵ©":150995,"ìķIJ":150996,"ìķĸ":150997,"ìĸł":150998,"ìĸ¾":150999,"ìĹĥ":151000,"ìĹĹ":151001,"ìĹľ":151002,"ìŨ":151003,"ìĺĤ":151004,"ìĺĦ":151005,"ìĺı":151006,"ìĺ¾":151007,"ìĺ¿":151008,"ìľ§":151009,"ìĿIJ":151010,"ìĿĸ":151011,"ìĿ·":151012,"ìŀį":151013,"ìŀı":151014,"ìŀ¨":151015,"ìŀª":151016,"ìŀ³":151017,"ìł¡":151018,"ìł´":151019,"ìł¹":151020,"ì¡Ģ":151021,"졪":151022,"졵":151023,"ì¢IJ":151024,"좨":151025,"ì£Į":151026,"ì£Ļ":151027,"죳":151028,"ì¦ij":151029,"짥":151030,"짴":151031,"짾":151032,"ì¨ĵ":151033,"ì¨ķ":151034,"ì©°":151035,"ì©»":151036,"쩼":151037,"ìªĹ":151038,"ì¬Ķ":151039,"ì¬ĺ":151040,"ì®®":151041,"ì¯ķ":151042,"ì¯ĺ":151043,"ì°İ":151044,"ì°¯":151045,"ì±ĥ":151046,"ì±µ":151047,"첧":151048,"ì²®":151049,"첯":151050,"쳬":151051,"ì´ĭ":151052,"ì´¢":151053,"ìµ¥":151054,"춣":151055,"ì¸Ī":151056,"ì¸Ļ":151057,"캤":151058,"ìºŃ":151059,"컽":151060,"ì¼Ļ":151061,"콬":151062,"ì¾Ģ":151063,"ì¿ħ":151064,"쿽":151065,"íĢħ":151066,"íģ¦":151067,"íĤħ":151068,"íĥ¶":151069,"íĥ¹":151070,"íĦĶ":151071,"íħ£":151072,"íĨĦ":151073,"íĨ§":151074,"íĨ¹":151075,"íĩ¼":151076,"íī¤":151077,"íĬ½":151078,"íĭĤ":151079,"íĭij":151080,"íįĪ":151081,"íįĻ":151082,"íį¿":151083,"íݶ":151084,"íIJĿ":151085,"íĴľ":151086,"íĵĿ":151087,"íĵª":151088,"íĵ±":151089,"íĵ·":151090,"íĵ¼":151091,"íĶĻ":151092,"íĶł":151093,"íķļ":151094,"íķĽ":151095,"íķŀ":151096,"íķŁ":151097,"íķ§":151098,"íķ¶":151099,"íĸĬ":151100,"íĸĭ":151101,"íĸį":151102,"íĸĶ":151103,"íĸĺ":151104,"íĸ¡":151105,"íĸ¬":151106,"íĹ£":151107,"íĹ¿":151108,"íĺĸ":151109,"íĺŃ":151110,"íļ°":151111,"íĽį":151112,"íĽ½":151113,"íĿŁ":151114,"íĿŃ":151115,"íĿ´":151116,"íŀľ":151117,"ï¤ī":151118,"ï¤Ń":151119,"爐":151120,"蘆":151121,"祿":151122,"ï¥Ģ":151123,"ï¥ij":151124,"ï¥Ĵ":151125,"ï¥ķ":151126,"ï¥ĺ":151127,"ï¥Ļ":151128,"參":151129,"塞":151130,"殺":151131,"勵":151132,"ï¦ĭ":151133,"ï¦ı":151134,"ï¦Ķ":151135,"ï¦ĸ":151136,"ï¦ĺ":151137,"ï¦Ľ":151138,"ï¦ł":151139,"瑩":151140,"羚":151141,"了":151142,"僚":151143,"料":151144,"ï§Ĩ":151145,"ï§ĸ":151146,"ï§Ľ":151147,"ï§ŀ":151148,"ï§Ł":151149,"裏":151150,"麟":151151,"狀":151152,"什":151153,"ï¨ĥ":151154,"ï¨ļ":151155,"諸":151156,"ï©Ł":151157,"ﬤ":151158,"שּׁ":151159,"לּ":151160,"ïŃĴ":151161,"ïŃķ":151162,"ïŃĽ":151163,"ïŃĿ":151164,"ïŃŀ":151165,"ïŃŁ":151166,"ïѤ":151167,"ïѧ":151168,"ïѨ":151169,"ïŃ®":151170,"ïŃ°":151171,"ïѱ":151172,"ïŃ·":151173,"ïŃ¹":151174,"ïŃ»":151175,"ï®Ģ":151176,"ï®ĥ":151177,"ï®Ħ":151178,"ï®ħ":151179,"ï®į":151180,"ï®Ĵ":151181,"ï®ĵ":151182,"ï®ķ":151183,"ﮦ":151184,"ï®®":151185,"ï®°":151186,"ï¯ĵ":151187,"ï¯ľ":151188,"ﯩ":151189,"ﯪ":151190,"ﯬ":151191,"ï¯Ń":151192,"ﯮ":151193,"ﯷ":151194,"ﯹ":151195,"ﯻ":151196,"ﯼ":151197,"ï°ĥ":151198,"ï°Į":151199,"ï°IJ":151200,"ï°ĺ":151201,"ï°Ļ":151202,"ï°ľ":151203,"ï°ŀ":151204,"ï°¢":151205,"ï°®":151206,"ï°°":151207,"ï°¼":151208,"ï°¿":151209,"ï±Ģ":151210,"ï±ģ":151211,"ï±Ī":151212,"ï±ĭ":151213,"ï±ı":151214,"ï±Ń":151215,"ï²Ģ":151216,"ï²ĩ":151217,"ï²Ī":151218,"ï²ĭ":151219,"ï²İ":151220,"ï²Ĵ":151221,"ï²ľ":151222,"ï²ł":151223,"ﲬ":151224,"ï²»":151225,"ï³ĩ":151226,"ï³Ķ":151227,"ï³£":151228,"ﳫ":151229,"ï´ĺ":151230,"ï´°":151231,"ï´½":151232,"ï¶":151233,"ﶰ":151234,"ï¸ĸ":151235,"︴":151236,"︹":151237,"ï¹į":151238,"ï¹Ĺ":151239,"ï¹¢":151240,"﹤":151241,"﹩":151242,"ï¹±":151243,"ï¾°":151244,"ï¿Ĥ":151245,"ï¿®":151246,"ðIJĮ°":151247,"ðIJĮ¹":151248,"ðIJĮº":151249,"ðIJĮ½":151250,"ðIJįĤ":151251,"ðIJįĥ":151252,"ðIJįĦ":151253,"ðIJİ":151254,"ðIJݹ":151255,"ðIJ¤Ĥ":151256,"ðIJ¤į":151257,"ðIJ¤ı":151258,"ðIJ¤ĵ":151259,"ðIJŃī":151260,"ðIJŃį":151261,"ðIJ°ĩ":151262,"ðIJ°°":151263,"ðijĤ":151264,"ðijĤĦ":151265,"ðijĺ":151266,"ðijĺģ":151267,"ðĴĢ":151268,"ðĴĢ¸":151269,"ðĴģ":151270,"ðĴģº":151271,"ðĴĦ":151272,"ðĴĦ·":151273,"ðĴĬ":151274,"ðĴĬij":151275,"ðĴĭ":151276,"ðĴĭĹ":151277,"ðĴĮ":151278,"ðĴĮ¨":151279,"ðĵĥ¢":151280,"ðĵĥ°":151281,"ðĸł":151282,"ðĸłļ":151283,"ðĿĦĥ":151284,"ðĿĦħ":151285,"ðĿĦķ":151286,"ðĿĦĻ":151287,"ðĿĦ±":151288,"ðĿĦ´":151289,"ðĿĦ¹":151290,"ðĿħİ":151291,"ðĿħª":151292,"ðĿĨ£":151293,"ðĿĨ³":151294,"ðĿĨ¹":151295,"ðĿĩĬ":151296,"ðĿĩĹ":151297,"ðĿĩļ":151298,"ðĿĩľ":151299,"ðĿĩł":151300,"ðĿIJī":151301,"ðĿIJĸ":151302,"ðĿIJĺ":151303,"ðĿIJ£":151304,"ðĿIJ±":151305,"ðĿijĬ":151306,"ðĿijŃ":151307,"ðĿij¼":151308,"ðĿij½":151309,"ðĿĴ°":151310,"ðĿĴ·":151311,"ðĿĴ¿":151312,"ðĿĵģ":151313,"ðĿĵĭ":151314,"ðĿĵİ":151315,"ðĿĵĴ":151316,"ðĿĵĺ":151317,"ðĿĵ¢":151318,"ðĿĵ¦":151319,"ðĿĵ«":151320,"ðĿĵ¿":151321,"ðĿĶİ":151322,"ðĿĶ±":151323,"ðĿĶ´":151324,"ðĿĶ·":151325,"ðĿĶ¸":151326,"ðĿĶ½":151327,"ðĿķĤ":151328,"ðĿķĥ":151329,"ðĿķĭ":151330,"ðĿķı":151331,"ðĿķIJ":151332,"ðĿķ¥":151333,"ðĿķ´":151334,"ðĿķº":151335,"ðĿĸIJ":151336,"ðĿĸĽ":151337,"ðĿĸĿ":151338,"ðĿĸŀ":151339,"ðĿĹ©":151340,"ðĿĹ³":151341,"ðĿĹ½":151342,"ðĿĺĬ":151343,"ðĿĺĭ":151344,"ðĿĺĶ":151345,"ðĿĺ±":151346,"ðĿĺ´":151347,"ðĿĺ¿":151348,"ðĿĻĴ":151349,"ðĿĻĿ":151350,"ðĿĻŁ":151351,"ðĿĻ¬":151352,"ðĿĻŃ":151353,"ðĿĻ»":151354,"ðĿĻ¾":151355,"ðĿļĪ":151356,"ðĿļĭ":151357,"ðĿļij":151358,"ðĿļŁ":151359,"ðĿļł":151360,"ðĿļ£":151361,"ðĿĽ½":151362,"ðĿľĤ":151363,"ðĿľĶ":151364,"ðĿľĻ":151365,"ðŁĢ":151366,"ðŁĢĦ":151367,"ðŁĦ²":151368,"ðŁĦ¶":151369,"ðŁħIJ":151370,"ðŁħĸ":151371,"ðŁħļ":151372,"ðŁħĽ":151373,"ðŁħ¦":151374,"ðŁħ¶":151375,"ðŁħ»":151376,"ðŁħ¼":151377,"ðŁĨĥ":151378,"ðŁĨĨ":151379,"ðŁĨİ":151380,"ðŁĪ¯":151381,"ðŁĪ²":151382,"ðŁĪ¹":151383,"ðŁĮĩ":151384,"ðŁĮĵ":151385,"ðŁįĺ":151386,"ðŁİij":151387,"ðŁİ¿":151388,"ðŁıı":151389,"ðŁıĴ":151390,"ðŁı©":151391,"ðŁı¯":151392,"ðŁIJĢ":151393,"ðŁijĿ":151394,"ðŁĴ¹":151395,"ðŁĴº":151396,"ðŁĵŁ":151397,"ðŁĵª":151398,"ðŁĵ¼":151399,"ðŁĶĢ":151400,"ðŁĶĤ":151401,"ðŁĶĥ":151402,"ðŁĶĩ":151403,"ðŁĶĵ":151404,"ðŁĶ¢":151405,"ðŁĶ¤":151406,"ðŁĶ©":151407,"ðŁķĸ":151408,"ðŁķļ":151409,"ðŁķľ":151410,"ðŁķĿ":151411,"ðŁķŀ":151412,"ðŁķł":151413,"ðŁķ¢":151414,"ðŁķ³":151415,"ðŁĸĩ":151416,"ðŁĸij":151417,"ðŁĸ¶":151418,"ðŁĹģ":151419,"Ѩ":151420,"Úİ":151421,"á¡Į":151422,"Ḱ":151423,"áºĢ":151424,"á¼®":151425,"á½Ŀ":151426,"âĦ¬":151427,"âļ§":151428,"⼤":151429,"㳬":151430,"êĻĭ":151431,"ê¸ij":151432,"ëĶī":151433,"ëĹį":151434,"ë¡ij":151435,"ë¯ij":151436,"ë»ħ":151437,"ë¼Ŀ":151438,"ìĦIJ":151439,"ìī¡":151440,"ìĭ²":151441,"ìı±":151442,"ìŤ":151443,"ìĿ©":151444,"ìĿ¿":151445,"ìŁĻ":151446,"ìł°":151447,"ì¥ī":151448,"íĬŃ":151449,"íķ®":151450,"ï®ı":151451,"ðŁħ±":151452,"ðŁĨĴ":151453,"ðŁķĭ":151454,"Éĺ":151455,"Êĵ":151456,"Õĥ":151457,"à´´":151458,"à½ħ":151459,"áĨº":151460,"áĪĬ":151461,"áĪ¨":151462,"áĪ¾":151463,"áīIJ":151464,"áĮĥ":151465,"áĮ½":151466,"áĶŃ":151467,"áłĤ":151468,"ᳬ":151469,"ᨸ":151470,"á©ĭ":151471,"á¶ı":151472,"á¾Ķ":151473,"á¿IJ":151474,"á¿ļ":151475,"âĻĻ":151476,"âļĤ":151477,"âļĹ":151478,"â¡¢":151479,"⤦":151480,"ëĸ°":151481,"ë¤Ĥ":151482,"맳":151483,"ë±ĭ":151484,"ë±IJ":151485,"ìĽ¢":151486,"ìľ¾":151487,"ì³ħ":151488,"ì»ģ":151489,"íģ»":151490,"íĥĻ":151491,"íĵĸ":151492,"íĵŃ":151493,"íķ±":151494,"íĽľ":151495,"ï¤ħ":151496,"ï¤Ĩ":151497,"ï¦ĥ":151498,"里":151499,"ï¨Ĥ":151500,"ðIJ¤Ķ":151501,"ðIJŃĵ":151502,"ðIJ°¼":151503,"ðĿĵŀ":151504,"ðĿĵ°":151505,"ðĿĻľ":151506,"ðĿļģ":151507,"ðŁħ¢":151508,"ðŁıĩ":151509,"Ȳ":151510,"ʶ":151511,"ÔĪ":151512,"Ôij":151513,"Ýĵ":151514,"Ý¥":151515,"à¤ij":151516,"ॱ":151517,"à¬ī":151518,"à°³":151519,"à°µ":151520,"à²Ł":151521,"áĢı":151522,"áģ¼":151523,"áī¨":151524,"áĬĴ":151525,"áĭ©":151526,"áĮĦ":151527,"áĮĶ":151528,"áIJ§":151529,"áĴĮ":151530,"áĶħ":151531,"áĶĬ":151532,"áłĦ":151533,"á¨ģ":151534,"á¸ĥ":151535,"ḻ":151536,"âĶŀ":151537,"âĺµ":151538,"âļ£":151539,"â²¢":151540,"ãĪª":151541,"䶵":151542,"ê²Ļ":151543,"ê²´":151544,"ê³Ĥ":151545,"롼":151546,"ìĨĬ":151547,"ì¼ĩ":151548,"íĭį":151549,"íĵ¬":151550,"íĵ®":151551,"íĵ¶":151552,"íĵ»":151553,"臘":151554,"ï¥ł":151555,"辰":151556,"ïŃ²":151557,"ðIJŃĬ":151558,"ðIJ±ħ":151559,"ðĸ¥":151560,"ðĸ¥¨":151561,"ðĿij³":151562,"ðĿĵķ":151563,"ðĿĵ¬":151564,"ðĿĵ¹":151565,"ðĿĵ¾":151566,"ðĿĶĵ":151567,"ðĿķį":151568,"ðĿķ¡":151569,"ðĿķ±":151570,"ðĿĸĸ":151571,"ðĿĺı":151572,"ðĿĺIJ":151573,"ðĿĺļ":151574,"ðĿĻ®":151575,"ðĿĻ°":151576,"ðĿĻ¸":151577,"ðĿĻº":151578,"ðĿĻ¼":151579,"ðĿĻ½":151580,"ðĿĻ¿":151581,"ðĿļĦ":151582,"ðĿļı":151583,"ðŁħħ":151584,"ðŁħĵ":151585,"ÆĪ":151586,"àłĮ":151587,"áĻ³":151588,"áļĮ":151589,"áĽħ":151590,"áĽIJ":151591,"á¤Ĭ":151592,"á¸Ĭ":151593,"âĶ½":151594,"âķĬ":151595,"âĽĩ":151596,"âĽı":151597,"âĿª":151598,"âĿ«":151599,"⣰":151600,"ãĦį":151601,"ãĦĵ":151602,"ãĦ§":151603,"ãħĸ":151604,"ãī«":151605,"ê¦Ķ":151606,"ï±Ĭ":151607,"àºĤ":151608,"áħ£":151609,"á¥Ķ":151610,"ᥤ":151611,"âĨ¤":151612,"âĨ·":151613,"âĩŀ":151614,"âĸ¤":151615,"âŀ¶":151616,"ãĪ¼":151617,"嘆":151618,"ðĵı§":151619,"âĶ²":151620,"âĢ´":151621,"âĴŁ":151622,"âĴ¡":151623,"â°Ĥ":151624,"â°į":151625,"â°İ":151626,"â°IJ":151627,"â°ij":151628,"â°Ł":151629,"â°ł":151630,"â°¡":151631,"â¼Ń":151632,"ãĬ¥":151633,"âĴł":151634,"⽺":151635,"ãĩº":151636,"ãĩ½":151637,"ï¨Ĭ":151638,"áķ·":151639,"âį¨":151640,"âºŁ":151641,"â½Ĺ":151642} \ No newline at end of file diff --git a/VideoX-Fun/models/put models here.txt b/VideoX-Fun/models/put models here.txt new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/VideoX-Fun/pulsar2_configs/config.json b/VideoX-Fun/pulsar2_configs/config.json new file mode 100644 index 0000000000000000000000000000000000000000..55317546ef2632c06bb29be52d6f9b5b659fcdae --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/config.json @@ -0,0 +1,45 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "./onnx-calibration/transformer_body_inputs.tar", + "calibration_size": -1, + // "calibration_mean": [123.675, 116.28, 103.53], + // "calibration_std": [58.395, 57.12, 57.375], + "calibration_format":"NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "EndToEnd", + "conv_bias_data_type": "FP32", + "enable_smooth_quant": true, + "disable_auto_refine_scale": true, + "layer_configs": [ + { + "start_tensor_names": ["DEFAULT"], + "end_tensor_names": ["DEFAULT"], + "data_type": "U8" + } + + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT", + "tensor_format": "RGB", + "tensor_layout": "NCHW", + "src_dtype": "FP32", + "src_layout": "NCHW", + "mean": [0, 0, 0], + "std": [1, 1, 1] + } + ], + "compiler": { + "npu_perf": false + } +} + diff --git a/VideoX-Fun/pulsar2_configs/config_controlnet.json b/VideoX-Fun/pulsar2_configs/config_controlnet.json new file mode 100644 index 0000000000000000000000000000000000000000..0f54677ccfdee8f2fa90d2b8bf62334ef27ba67e --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/config_controlnet.json @@ -0,0 +1,46 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "./onnx-calibration/controlnet_inputs.tar", + "calibration_size": -1, + // "calibration_mean": [123.675, 116.28, 103.53], + // "calibration_std": [58.395, 57.12, 57.375], + "calibration_format":"NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "EndToEnd", + "conv_bias_data_type": "FP32", + "enable_smooth_quant": true, + "disable_auto_refine_scale": true, + "layer_configs": [ + { + "start_tensor_names": ["DEFAULT"], + "end_tensor_names": ["DEFAULT"], + "data_type": "U16" + } + + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT", + // "tensor_format": "RGB", + // "tensor_layout": "NCHW", + "src_dtype": "FP32", + // "src_layout": "NCHW", + "mean": [0, 0, 0], + "std": [1, 1, 1] + } + ], + "compiler": { + "npu_perf": false, + "check": 0 + } +} + diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/auto_00.json b/VideoX-Fun/pulsar2_configs/subgraphs/auto_00.json new file mode 100644 index 0000000000000000000000000000000000000000..2eb41c38b26e73d0152b4648e2e97e44bc4aea3d --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/auto_00.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/auto_00.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_00.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_00.json new file mode 100644 index 0000000000000000000000000000000000000000..d8a1427df7fa10af727bfe19fa6ecb5f65a35876 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_00.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_00.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": false, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_01.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_01.json new file mode 100644 index 0000000000000000000000000000000000000000..f66d16643eaf8e1ae99de329a8e6d2ed7369b47c --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_01.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_01.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_02.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_02.json new file mode 100644 index 0000000000000000000000000000000000000000..7ff5a82ed4a5f4b12af2caa077b033f2fde4f440 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_02.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_02.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_03.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_03.json new file mode 100644 index 0000000000000000000000000000000000000000..ee29b3d61bfaa55e43624d8ebf7fd40ac3582117 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_03.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_03.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_04.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_04.json new file mode 100644 index 0000000000000000000000000000000000000000..a52b326832e107a8c661cda1310b87d177eb8ac2 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_04.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_04.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_05.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_05.json new file mode 100644 index 0000000000000000000000000000000000000000..f67bf1d1b8548f1d7a04d26807226800a87e605b --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_05.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_05.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_06.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_06.json new file mode 100644 index 0000000000000000000000000000000000000000..9aa3d39444a147880d1430119bbabdfc07bb1fa0 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_06.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_06.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_07.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_07.json new file mode 100644 index 0000000000000000000000000000000000000000..4924745aec83a3971b2dbd4e6ba41eeb390586ce --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_07.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_07.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_08.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_08.json new file mode 100644 index 0000000000000000000000000000000000000000..b83bd72f4413b8d085fac49f7368fef35b58351c --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_08.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_08.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_09.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_09.json new file mode 100644 index 0000000000000000000000000000000000000000..0807ebbc2fb8f771623fb4b72c3a18dbf75cda75 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_09.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_09.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_10.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_10.json new file mode 100644 index 0000000000000000000000000000000000000000..332f1316f1fc4d6149e78eddef4cd81b20aae69a --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_10.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_10.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_11.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_11.json new file mode 100644 index 0000000000000000000000000000000000000000..70c4f871e44ecea552f1a9ff05540ba6595f5f0a --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_11.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_11.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_12.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_12.json new file mode 100644 index 0000000000000000000000000000000000000000..b16bf996ad242e297b243538a652e695b0b3366f --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_12.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_12.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_13.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_13.json new file mode 100644 index 0000000000000000000000000000000000000000..dc56bd2d8cb7f563b19f62d76f70c3e8c9f21363 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_13.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_13.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_14.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_14.json new file mode 100644 index 0000000000000000000000000000000000000000..075b5a97edb4f48a5b1090bbebaae6f360551323 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_14.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_14.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_15.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_15.json new file mode 100644 index 0000000000000000000000000000000000000000..b0d1c4543c2eca0766c125fda46475198e64dc42 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_15.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_15.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_16.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_16.json new file mode 100644 index 0000000000000000000000000000000000000000..79ad556d743418d009c7c9b67ebf8a8a78ceb533 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_16.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_16.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_17.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_17.json new file mode 100644 index 0000000000000000000000000000000000000000..ea414d4671a97b7e5eba11a35c17e6908082a304 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_17.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_17.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_18.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_18.json new file mode 100644 index 0000000000000000000000000000000000000000..7360c35ee4cd705789642146a1979d78dbc266c8 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_18.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_18.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_19.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_19.json new file mode 100644 index 0000000000000000000000000000000000000000..2f2a9a0ca8dab113c0039fd2d5a09181e151ae00 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_19.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_19.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_20.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_20.json new file mode 100644 index 0000000000000000000000000000000000000000..113276b5ab53cadc1ce4ecd0a0a4e28918cb81f7 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_20.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_20.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_21.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_21.json new file mode 100644 index 0000000000000000000000000000000000000000..b8b08ea954ac9ea6903ea882500ef42c31dd4efe --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_21.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_21.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_22.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_22.json new file mode 100644 index 0000000000000000000000000000000000000000..ad0a881df9782da711877cd50b1c1ec55d1a2157 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_22.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_22.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_23.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_23.json new file mode 100644 index 0000000000000000000000000000000000000000..affa1aec141d63ec80969d0f7cbe72375b4bccec --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_23.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_23.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_24.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_24.json new file mode 100644 index 0000000000000000000000000000000000000000..7a4b9825c26eec02d69f5909d2a492236634f744 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_24.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_24.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_25.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_25.json new file mode 100644 index 0000000000000000000000000000000000000000..950e77b174b4ad11a2465747be224b247e26ee90 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_25.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_25.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_26.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_26.json new file mode 100644 index 0000000000000000000000000000000000000000..8e642ddc7b7d9d8371091543fdb5b20afa37a806 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_26.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_26.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_27.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_27.json new file mode 100644 index 0000000000000000000000000000000000000000..9739c8384f7330ca868dd489fcd9cf174d7a5eb1 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_27.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_27.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_28.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_28.json new file mode 100644 index 0000000000000000000000000000000000000000..f5187693050c548b13b15c094c11cc9336af2970 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_28.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_28.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_29.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_29.json new file mode 100644 index 0000000000000000000000000000000000000000..b5aedd38a13beda6aae31e5eeeff6d956b1d812b --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_29.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_29.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_30.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_30.json new file mode 100644 index 0000000000000000000000000000000000000000..8b5ca8118279cca43c5a8e8a180d553873059364 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_30.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_30.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_31.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_31.json new file mode 100644 index 0000000000000000000000000000000000000000..43c62cc0fb3e8574ae3464e68d83c8feaa4c21e7 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_31.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_31.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/cfg_32.json b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_32.json new file mode 100644 index 0000000000000000000000000000000000000000..063093909e99537b945514d9b474fb38f7dabeee --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/cfg_32.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-calibration-subgraphs/cfg_32.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs/subgraph_configs_list.txt b/VideoX-Fun/pulsar2_configs/subgraphs/subgraph_configs_list.txt new file mode 100644 index 0000000000000000000000000000000000000000..3db364206ed33a075ad0c376469cbd011b5f8034 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs/subgraph_configs_list.txt @@ -0,0 +1,34 @@ +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/auto_00.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_00.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_01.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_02.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_03.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_04.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_05.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_06.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_07.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_08.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_09.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_10.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_11.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_12.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_13.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_14.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_15.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_16.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_17.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_18.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_19.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_20.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_21.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_22.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_23.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_24.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_25.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_26.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_27.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_28.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_29.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_30.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_31.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs/cfg_32.json diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/auto_00.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/auto_00.json new file mode 100644 index 0000000000000000000000000000000000000000..a3d551802a234860876dd7e469a8e97317f4076e --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/auto_00.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/auto_00.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_00.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_00.json new file mode 100644 index 0000000000000000000000000000000000000000..74f98a9445908880422e2d133d1f72415b41faae --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_00.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_00.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": false, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_01.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_01.json new file mode 100644 index 0000000000000000000000000000000000000000..7b3f8f9fbfaf2bdeff2e520d6badf3dd5d2ea583 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_01.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_01.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_02.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_02.json new file mode 100644 index 0000000000000000000000000000000000000000..900ffe2be5c93d32482b45829565da685727c606 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_02.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_02.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_03.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_03.json new file mode 100644 index 0000000000000000000000000000000000000000..cb077879e0e4d583c16903b373f255c78f1d1a0e --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_03.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_03.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_04.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_04.json new file mode 100644 index 0000000000000000000000000000000000000000..aa0fe3f817fdd620115d621226dce487167b0558 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_04.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_04.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_05.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_05.json new file mode 100644 index 0000000000000000000000000000000000000000..c38071941a52d176cf138fb121c9dd2152509bb4 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_05.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_05.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_06.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_06.json new file mode 100644 index 0000000000000000000000000000000000000000..33c9944c3c4ca47709c6cdeaea72cb1971659e58 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_06.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_06.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_07.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_07.json new file mode 100644 index 0000000000000000000000000000000000000000..63d2499cc52a31c00343916826a296348f1e79a0 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_07.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_07.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_08.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_08.json new file mode 100644 index 0000000000000000000000000000000000000000..5d186133abc2a192ff86ac4f676bcc30efd23eec --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_08.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_08.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_09.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_09.json new file mode 100644 index 0000000000000000000000000000000000000000..83fe12a8cc6fedd2fc3a2285d2007248168534f8 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_09.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_09.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_10.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_10.json new file mode 100644 index 0000000000000000000000000000000000000000..2eec056b9e9c15d60991db81d2b95309017cf6c1 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_10.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_10.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_11.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_11.json new file mode 100644 index 0000000000000000000000000000000000000000..11a02cc8ad7f05db2444e5c9515cfe3b129d309c --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_11.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_11.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_12.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_12.json new file mode 100644 index 0000000000000000000000000000000000000000..b33f8a4dbfa46ba9bc18bc57e4c4380b2c98afd2 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_12.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_12.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_13.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_13.json new file mode 100644 index 0000000000000000000000000000000000000000..75b74fd7b302f295d8bb51f1b9ad15c2e8d60ba6 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_13.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_13.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_14.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_14.json new file mode 100644 index 0000000000000000000000000000000000000000..29d7f350da65a34399df2ef8c7c00374fe956107 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_14.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_14.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_15.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_15.json new file mode 100644 index 0000000000000000000000000000000000000000..b6e7d17f72c2a4fd52c37a2be4356296ca3a7efd --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_15.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_15.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_16.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_16.json new file mode 100644 index 0000000000000000000000000000000000000000..83e2e79c58bd2098f5fe949bf1ef928191a6eb36 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_16.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_16.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_17.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_17.json new file mode 100644 index 0000000000000000000000000000000000000000..d2eb32806117f325269bda82c8dfe756c6b33d22 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_17.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_17.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_18.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_18.json new file mode 100644 index 0000000000000000000000000000000000000000..7930905ab00bd7bfb37d81caf18dfb464d02a58b --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_18.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_18.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_19.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_19.json new file mode 100644 index 0000000000000000000000000000000000000000..69df686f1cf0a2551dc15e2fd8779290d04ecf88 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_19.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_19.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_20.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_20.json new file mode 100644 index 0000000000000000000000000000000000000000..694f18c7d2590eb870ea60d477075e7c59f0af88 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_20.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_20.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_21.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_21.json new file mode 100644 index 0000000000000000000000000000000000000000..97b61742779a3d26a08c442fd2b2c2e398469794 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_21.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_21.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_22.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_22.json new file mode 100644 index 0000000000000000000000000000000000000000..6cc42d223b4d5c2c47edd50bafaa8a187e58ecda --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_22.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_22.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_23.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_23.json new file mode 100644 index 0000000000000000000000000000000000000000..68ef55ac1f9997dda133f3eca94c06dec35a581b --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_23.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_23.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_24.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_24.json new file mode 100644 index 0000000000000000000000000000000000000000..f58b5908e1edd9eb6ff0c02f202a97ffce6ecbc1 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_24.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_24.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_25.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_25.json new file mode 100644 index 0000000000000000000000000000000000000000..771f1855bfeead38612deb0a108e1bf47c0d95ff --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_25.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_25.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_26.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_26.json new file mode 100644 index 0000000000000000000000000000000000000000..8c3a70d22fe5b8d091a098e8c453686976d586d1 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_26.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_26.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_27.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_27.json new file mode 100644 index 0000000000000000000000000000000000000000..9907d12912a639790bd79b558cdf6a204c0092fa --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_27.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_27.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_28.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_28.json new file mode 100644 index 0000000000000000000000000000000000000000..30f1660c32711bfddce1a24776d700306f6c4cd6 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_28.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_28.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_29.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_29.json new file mode 100644 index 0000000000000000000000000000000000000000..814ba29e9f00b7c89d1a5dff73c9aeb76eb204ce --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_29.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_29.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_30.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_30.json new file mode 100644 index 0000000000000000000000000000000000000000..68cb8d945af2895c6f0421f3eb9976eda0729563 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_30.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_30.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_31.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_31.json new file mode 100644 index 0000000000000000000000000000000000000000..b281c5dbe6a6ce71bc122323781d6136e7ddca70 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_31.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_31.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_32.json b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_32.json new file mode 100644 index 0000000000000000000000000000000000000000..7e0dd9b4408effab3a799c23f5851114b474267a --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_32.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/transformer_body_only_1728x992_simp_slim/subgraph-calib/cfg_32.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} \ No newline at end of file diff --git a/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/subgraph_configs_list.txt b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/subgraph_configs_list.txt new file mode 100644 index 0000000000000000000000000000000000000000..4d137cd57b6bbe90c26d36db02a52b77b84e148a --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/subgraph_configs_list.txt @@ -0,0 +1,34 @@ +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/auto_00.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_00.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_01.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_02.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_03.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_04.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_05.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_06.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_07.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_08.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_09.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_10.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_11.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_12.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_13.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_14.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_15.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_16.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_17.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_18.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_19.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_20.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_21.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_22.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_23.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_24.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_25.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_26.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_27.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_28.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_29.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_30.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_31.json +/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/pulsar2_configs/subgraphs_1728x992/cfg_32.json diff --git a/VideoX-Fun/pulsar2_configs/transformers.json b/VideoX-Fun/pulsar2_configs/transformers.json new file mode 100644 index 0000000000000000000000000000000000000000..628547c7a6b417b68865f0400f89ff401ad95f45 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/transformers.json @@ -0,0 +1,38 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "./onnx-calibration-no-controlnet/transformer.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": true, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} diff --git a/VideoX-Fun/pulsar2_configs/transformers_subgraph.json b/VideoX-Fun/pulsar2_configs/transformers_subgraph.json new file mode 100644 index 0000000000000000000000000000000000000000..6f55984875c79fd4806d43913fb4d88af50cd311 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/transformers_subgraph.json @@ -0,0 +1,207 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "./onnx-calibration-no-controlnet/transformer.tar", + "calibration_size": 4, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "EndToEnd", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT", + "tensor_format": "AutoColorSpace", + "tensor_layout": "NCHW" + } + ], + "compiler": { + "check": 0, + "sub_configs": [ + { + "start_tensor_names": ["timestep"], + "end_tensor_names": ["/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["prompt_embeds"], + "end_tensor_names": ["/model/Slice_1_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["latent_model_input", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/Slice_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/Slice_1_output_0", "/model/Slice_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.0/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.0/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.1/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.1/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.2/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.2/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.3/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.3/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.4/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.4/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.5/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.5/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.6/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.6/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.7/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.7/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.8/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.8/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.9/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.9/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.10/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.10/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.11/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.11/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.12/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.12/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.13/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.13/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.14/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.14/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.15/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.15/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.16/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.16/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.17/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.17/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.18/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.18/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.19/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.19/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.20/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.20/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.21/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.21/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.22/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.22/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.23/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.23/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.24/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.24/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.25/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.25/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.26/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.26/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.27/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.27/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.28/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.28/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.29/Add_4_output_0"], + "check_mode": "CheckPerLayer" + } + ] + } +} + + diff --git a/VideoX-Fun/pulsar2_configs/transformers_subgraph_1728x992.json b/VideoX-Fun/pulsar2_configs/transformers_subgraph_1728x992.json new file mode 100644 index 0000000000000000000000000000000000000000..c0949dbf147a2e02ef9ce239667254fa0c68feec --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/transformers_subgraph_1728x992.json @@ -0,0 +1,207 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "./onnx-calibration-no-controlnet/transformer.tar", + "calibration_size": 4, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": true, + "precision_analysis_method": "EndToEnd", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT", + "tensor_format": "AutoColorSpace", + "tensor_layout": "NCHW" + } + ], + "compiler": { + "check": 0, + "sub_configs": [ + { + "start_tensor_names": ["timestep"], + "end_tensor_names": ["/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["prompt_embeds"], + "end_tensor_names": ["/model/Slice_2_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["latent_model_input", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/Slice_1_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/Slice_2_output_0", "/model/Slice_1_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.0/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.0/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.1/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.1/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.2/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.2/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.3/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.3/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.4/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.4/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.5/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.5/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.6/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.6/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.7/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.7/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.8/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.8/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.9/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.9/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.10/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.10/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.11/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.11/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.12/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.12/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.13/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.13/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.14/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.14/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.15/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.15/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.16/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.16/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.17/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.17/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.18/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.18/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.19/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.19/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.20/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.20/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.21/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.21/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.22/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.22/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.23/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.23/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.24/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.24/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.25/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.25/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.26/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.26/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.27/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.27/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.28/Add_4_output_0"], + "check_mode": "CheckPerLayer" + }, + { + "start_tensor_names": ["/model/layers.28/Add_4_output_0", "/model/t_embedder/mlp/mlp.2/Gemm_output_0"], + "end_tensor_names": ["/model/layers.29/Add_4_output_0"], + "check_mode": "CheckPerLayer" + } + ] + } +} + + diff --git a/VideoX-Fun/pulsar2_configs/vae_decoder.json b/VideoX-Fun/pulsar2_configs/vae_decoder.json new file mode 100644 index 0000000000000000000000000000000000000000..f4aaf707863bd4e8e7c092aa3f6c3cbf61e67d79 --- /dev/null +++ b/VideoX-Fun/pulsar2_configs/vae_decoder.json @@ -0,0 +1,46 @@ +{ + "model_type": "ONNX", + "npu_mode": "NPU3", + "onnx_opt": { + "disable_onnx_optimization": false, + "enable_onnxsim": false, + "model_check": false, + "disable_transformation_check": true, + "save_tensors_data": false + }, + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "./onnx-calibration-no-controlnet/vae_decoder.tar", + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": false, + "precision_analysis_method": "EndToEnd", + "layer_configs": [ + { + "start_tensor_names": [ + "DEFAULT" + ], + "end_tensor_names": [ + "DEFAULT" + ], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT", + "tensor_format": "AutoColorSpace", + "tensor_layout": "NCHW" + } + ], + "compiler": { + "check": 0, + "max_dynamic_batch_size": 1 + } +} diff --git a/VideoX-Fun/pyproject.toml b/VideoX-Fun/pyproject.toml new file mode 100644 index 0000000000000000000000000000000000000000..c0adf3c1e13dc9531426c16c48a5ff2ece0afafa --- /dev/null +++ b/VideoX-Fun/pyproject.toml @@ -0,0 +1,15 @@ +[project] +name = "videox-fun" +description = "VideoX-Fun is a video generation pipeline that can be used to generate AI images and videos, as well as to train baseline and Lora models for Diffusion Transformer. We support direct prediction from pre-trained baseline models to generate videos with different resolutions, durations, and FPS. Additionally, we also support users in training their own baseline and Lora models to perform specific style transformations." +version = "1.0.0" +license = {file = "LICENSE"} +dependencies = ["Pillow", "einops", "safetensors", "timm", "tomesd", "torch>=2.1.2", "torchdiffeq", "torchsde", "decord", "datasets", "numpy", "scikit-image", "opencv-python", "omegaconf", "SentencePiece", "albumentations", "imageio[ffmpeg]", "imageio[pyav]", "tensorboard", "beautifulsoup4", "ftfy", "func_timeout", "accelerate>=0.25.0", "gradio>=3.41.2,<=3.48.0", "diffusers>=0.30.1,<=0.31.0", "transformers>=4.46.2"] + +[project.urls] +Repository = "https://github.com/aigc-apps/VideoX-Fun" +# Used by Comfy Registry https://comfyregistry.org + +[tool.comfy] +PublisherId = "bubbliiiing" +DisplayName = "VideoX-Fun" +Icon = "" diff --git a/VideoX-Fun/reports/cogvideox_fun/report_v1.md b/VideoX-Fun/reports/cogvideox_fun/report_v1.md new file mode 100644 index 0000000000000000000000000000000000000000..802186ea7f854de912ebf43d497cdac5eb72134e --- /dev/null +++ b/VideoX-Fun/reports/cogvideox_fun/report_v1.md @@ -0,0 +1,36 @@ +# CogVideoX FUN v1 Report +In CogVideoX-FUN, we trained on approximately 1.2 million data points based on CogVideoX, supporting image and video predictions. It accommodates pixel values for video generation across different resolutions of 512x512x49, 768x768x49, and 1024x1024x49, as well as videos with different aspect ratios. Moreover, we support the generation of videos from images and the reconstruction of videos from other videos. + +Compared to CogVideoX, CogVideoX FUN also highlights the following features: +- Introduction of the InPaint model, enabling the generation of videos from images with specified starting and ending images. +- Training the model based on token lengths. This allows for the implementation of various sizes and resolutions within the same model. + +## InPaint Model +We used [CogVideoX](https://github.com/THUDM/CogVideo/) as the foundational structure, referencing [EasyAnimate](https://github.com/aigc-apps/EasyAnimate) for the model training to generate videos from images. + +During video generation, the **reference video** is encoded using VAE, with the **black area in the above image representing the part to be reconstructed, and the white area representing the start image**. This is stacked with noise latents and input into the Transformer for video generation. We perform 3D resizing on the **masked area**, directly resizing it to fit the canvas size of the video that needs reconstruction. + +Then, we concatenate the latent, the encoded reference video, and the masked area, inputting them into DiT for noise prediction to obtain the final video. +The pipeline structure of CogVideoX FUN is as follows: +ui + +## Token Length-Based Model Training +We collected approximately 1.2 million high-quality data for the training of CogVideoX-Fun. During the training, we resized the videos based on different token lengths. The entire training process is divided into three phases, with each phase corresponding to 13312 (for 512x512x49 videos), 29952 (for 768x768x49 videos), and 53248 (for 1024x1024x49 videos). + +Taking CogVideoX-Fun-2B as an example: +- In the 13312 phase, the batch size is 128 with 7k training steps. +- In the 29952 phase, the batch size is 256 with 6.5k training steps. +- In the 53248 phase, the batch size is 128 with 5k training steps. + +During training, we combined high and low resolutions, enabling the model to support video generation from any resolution between 512 and 1280. For example, with a token length of 13312: +- At a resolution of 512x512, the number of video frames is 49. +- At a resolution of 768x768, the number of video frames is 21. +- At a resolution of 1024x1024, the number of video frames is 9. + +These resolutions and corresponding lengths were mixed for training, allowing the model to generate videos at different resolutions. + +## Resize 3D Embedding +In adapting CogVideoX-2B to the CogVideoX-Fun framework, it was found that the source code obtains 3D embeddings in a truncated manner. This approach only accommodates a single resolution; when the resolution changes, the embedding should also change. +ui + +Referencing Pixart-Sigma, the above image is from the Pixart-Sigma paper. We used Positional Embeddings Interpolation (PE Interpolation) to resize 3D embeddings. PE Interpolation is more conducive to convergence than directly generating cosine and sine embeddings for different resolutions. diff --git a/VideoX-Fun/reports/cogvideox_fun/report_v1_1.md b/VideoX-Fun/reports/cogvideox_fun/report_v1_1.md new file mode 100644 index 0000000000000000000000000000000000000000..b716d8c72db3cc2f64bbe66b3095c6d6b8d16a34 --- /dev/null +++ b/VideoX-Fun/reports/cogvideox_fun/report_v1_1.md @@ -0,0 +1,32 @@ +# CogVideoX FUN v1.1 Report + +In CogVideoX-FUN v1.1, we performed additional filtering on the previous dataset, selecting videos with larger motion amplitudes rather than still images in motion, resulting in approximately 0.48 million videos. The model continues to support both image and video prediction, accommodating pixel values from 512x512x49, 768x768x49, 1024x1024x49, and videos with different aspect ratios. We support both image-to-video generation and video-to-video reconstruction. + +Additionally, we have released training and prediction code for adding control signals, along with the initial version of the Control model. + +Compared to version 1.0, CogVideoX-FUN V1.1 highlights the following features: +- In the 5b model, Noise has been added to the reference images, increasing the motion amplitude of the videos. +- Released training and prediction code for adding control signals, along with the initial version of the Control model. + +## Adding Noise to Reference Images + +Building on the original CogVideoX-FUN V1.0, we drew upon [CogVideoX](https://github.com/THUDM/CogVideo/) and [SVD](https://github.com/Stability-AI/generative-models) to add Noise upwards to the non-zero reference images to disrupt the original images, aiming for greater motion amplitude. + +In our 5b model, Noise has been added, while the 2b model only performed fine-tuning with new data. This is because, after attempting to add Noise in the 2b model, the generated videos exhibited excessive motion amplitude, leading to deformation and damaging the output. The 5b model, due to its stronger generative capabilities, maintains relatively stable outputs during motion. + +Furthermore, the prompt words significantly influence the generation results, so please describe the actions in detail to increase dynamism. If unsure how to write positive prompts, you can use phrases like "smooth motion" or "in the wind" to enhance dynamism. Additionally, it is advisable to avoid using dynamic terms like "motion" in negative prompts. + +## Adding Control Signals to CogVideoX-FUN + +On the basis of the original CogVideoX-FUN V1.0, we replaced the original mask signal with Pose control signals. The control signals are encoded using VAE and used as Guidance, along with latent data entering the patch processing flow. + +We filtered the 0.48 million dataset, selecting around 20,000 videos and images containing portraits for pose extraction, which served as condition control signals for training. + +During the training process, the videos are scaled according to different Token lengths. The entire training process is divided into two phases, with each phase comprising 13,312 (corresponding to 512x512x49 videos) and 53,248 (corresponding to 1024x1024x49 videos). + +Taking CogVideoX-Fun-V1.1-5b-Pose as an example: +- In the 13312 phase, the batch size is 128, with 2.4k training steps. +- In the 53248 phase, the batch size is 128, with 1.2k training steps. + +The working principle diagram is shown below: +ui diff --git a/VideoX-Fun/reports/cogvideox_fun/report_v1_1_zh-CN.md b/VideoX-Fun/reports/cogvideox_fun/report_v1_1_zh-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..81dfb7a902a71421b1baec0b3054a09a7a9b2b81 --- /dev/null +++ b/VideoX-Fun/reports/cogvideox_fun/report_v1_1_zh-CN.md @@ -0,0 +1,31 @@ +# CogVideoX FUN v1.1 Report + +在CogVideoX-FUN v1.1中，我们在之前的数据集中再次做了筛选，选出其中动作幅度较大，而不是静止画面移动的视频，数量大约为0.48m。模型依然支持图片与视频预测，支持像素值从512x512x49、768x768x49、1024x1024x49与不同纵横比的视频生成。我们支持图像到视频的生成与视频到视频的重建。 + +另外，我们还发布了添加控制信号的训练代码与预测代码，并发布了初版的Control模型。 + +对比V1.0版本，CogVideoX-FUN V1.1突出了以下功能： + +- 在5b模型中，给参考图片添加了Noise，增加了视频的运动幅度。 +- 发布了添加控制信号的训练代码与预测代码，并发布了初版的Control模型。 + +## 参考图片添加Noise +在原本CogVideoX-FUN V1.0的基础上，我们参考[CogVideoX](https://github.com/THUDM/CogVideo/)和[SVD](https://github.com/Stability-AI/generative-models)，在非0的参考图向上添加Noise以破环原图，追求更大的运动幅度。 + +我们5b模型中添加了Noise，2b模型仅使用了新数据进行了finetune，因为我们在2b模型中尝试添加Noise之后，生成的视频运动幅度过大导致结果变形，破坏了生成结果，而5b模型因为更为的强大生成能力，在运动中也保持了较为稳定的输出。 + +另外，提示词对生成结果影响较大，请尽量描写动作以增加动态性。如果不知道怎么写正向提示词，可以使用smooth motion or in the wind来增加动态性。并且尽量避免在负向提示词中出现motion等表示动态的词汇。 + +## 添加控制信号的CogVideoX-Fun +在原本CogVideoX-FUN V1.0的基础上，我们使用Pose控制信号替代了原本的mask信号，将控制信号使用VAE编码后作为Guidance与latent一起进入patch流程， + +我们在0.48m数据中进行了筛选，选择出大约20000包含人像的视频与图片进行pose提取，作为condition控制信号进行训练。 + +在进行训练时，我们根据不同Token长度，对视频进行缩放后进行训练。整个训练过程分为两个阶段，每个阶段的13312（对应512x512x49的视频），53248（对应1024x1024x49的视频）。 + +以CogVideoX-Fun-V1.1-5b-Pose为例子，其中： +- 13312阶段，Batch size为128，训练步数为2.4k +- 53248阶段，Batch size为128，训练步数为1.2k。 + +工作原理图如下： +ui diff --git a/VideoX-Fun/reports/cogvideox_fun/report_v1_zh-CN.md b/VideoX-Fun/reports/cogvideox_fun/report_v1_zh-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..2d42f123eac46efcd8648fbee9bd4542e96924dd --- /dev/null +++ b/VideoX-Fun/reports/cogvideox_fun/report_v1_zh-CN.md @@ -0,0 +1,43 @@ +# CogVideoX FUN v1 Report + +在CogVideoX-FUN中，我们基于CogVideoX在大约1.2m的数据上进行了训练，支持图片与视频预测，支持像素值从512x512x49、768x768x49、1024x1024x49与不同纵横比的视频生成。另外，我们支持图像到视频的生成与视频到视频的重建。 + +对比与CogVideoX，CogVideoX FUN还突出了以下功能： + +- 引入InPaint模型，实现图生视频功能，可以通过首尾图指定视频生成。 +- 基于Token长度的模型训练。达成不同大小多分辨率在同一模型中的实现。 + +## InPaint模型 +我们以[CogVideoX](https://github.com/THUDM/CogVideo/)作为基础结构，参考[EasyAnimate](https://github.com/aigc-apps/EasyAnimate)进行图生视频的模型训练。 + +在进行视频生成的时候，将**参考视频**使用VAE进行encode，**上图黑色的部分代表需要重建的部分，白色的部分代表首图**，与噪声Latents一起堆叠后输入到Transformer中进行视频生成。 + +我们对**被Mask的区域**进行3D Resize，直接Resize到需要重建的视频的画布大小。 + +然后将Latent、Encode后的参考视频、被Mask的区域，concat后输入到DiT中进行噪声预测。获得最终的视频。 + +CogVideoX FUN的Pipeline结构如下： +ui + +## 基于Token长度的模型训练 +我们收集了大约高质量的1.2m数据进行CogVideoX-Fun的训练。 + +在进行训练时，我们根据不同Token长度，对视频进行缩放后进行训练。整个训练过程分为三个阶段，每个阶段的13312（对应512x512x49的视频），29952（对应768x768x49的视频），53248（对应1024x1024x49的视频）。 + +以CogVideoX-Fun-2B为例子，其中： +- 13312阶段，Batch size为128，训练步数为7k +- 29952阶段，Batch size为256，训练步数为6.5k。 +- 53248阶段，Batch size为128，训练步数为5k。 + +训练时我们采用高低分辨率结合训练，因此模型支持从512到1280任意分辨率的视频生成，以13312 token长度为例： +- 在512x512分辨率下，视频帧数为49； +- 在768x768分辨率下，视频帧数为21； +- 在1024x1024分辨率下，视频帧数为9； +这些分辨率与对应长度混合训练，模型可以完成不同大小分辨率的视频生成。 + +## Resize 3D Embedding +在适配CogVideoX-2B到CogVideoX-Fun框架的途中，发现源码是以截断的方式去得到3D Embedding的，这样的方式只能适配单一分辨率，当分辨率发生变化时，Embedding也应当发生变化。 + +ui + +参考Pixart-Sigma，上图来自于Pixart-Sigma论文，我们采用Positional Embeddings Interpolation（PE Interpolation）对3D embedding进行Resize，PE Interpolation相比于直接生成不同分辨率的Cos Sin Embedding更易收敛。 \ No newline at end of file diff --git a/VideoX-Fun/reports/wan2_1_fun/report_v1_1.md b/VideoX-Fun/reports/wan2_1_fun/report_v1_1.md new file mode 100644 index 0000000000000000000000000000000000000000..3e724f5cf4095bcb726412714f718a33371521c1 --- /dev/null +++ b/VideoX-Fun/reports/wan2_1_fun/report_v1_1.md @@ -0,0 +1,31 @@ +# Wan Fun v1.1 Report + +In Wan-Fun v1.1, we updated six models: the 14B Inpaint model, Control model, and Control-Camera model; as well as the 1.3B Inpaint model, Control model, and Control-Camera model. + +Compared to the previous version, the Inpaint model has been trained with a larger batch size, resulting in more stable performance. The Control model now includes a reference image model to achieve effects similar to Animate Anyone. While retaining its original functionality, it can also accept both a reference image and a control video as inputs for generation. Finally, we provide a camera control model that supports pan-and-tilt movements (left, right, up, down). + +Additionally, we have released training and inference code for adding reference control signals, as well as training and inference code for adding camera control signals. + +Compared to V1.0, Wan Fun V1.1 highlights the following features: + +- A more stable Inpaint model. +- On top of the original control scheme, we’ve implemented a new control approach combining reference images and control videos. +- Added support for a camera control model. + +## Implementation of Reference Image + Control Video +In Wan-Fun V1.0, we already supported multiple control signals such as Canny, Depth, Pose, and MLSD, and implemented two control schemes: initial-image plus trajectory control, and control-video-guided generation. + +To further enhance the usability of the control model, we developed a new control scheme that combines reference images with control videos, akin to Animate Anyone. This feature takes a reference image as input and generates output based on control signals like Openpose (though it is not limited to Openpose—Depth signals also yield impressive results). Previously, methods like Unimate or Animate Anyone typically required the skeleton of the reference image to closely align with the control video. However, in Wan-Fun V1.1 Control, even if there is some misalignment, the system still produces acceptable results, though better alignment naturally leads to higher similarity. + +We encode the reference image using VAE, then tile the latent representation and concatenate the tiled features with the video features for generation. To ensure that this does not interfere with the model's original functionality, during training, we randomly initialize the reference image latent as all zeros to simulate cases where no reference image is provided. The overall workflow of the model is shown in the figure below: + +![Control_Ref](https://github.com/user-attachments/assets/8987a3b5-e691-4c49-a83c-10cad0fe13f3) + +## Camera Control Model +Building upon Wan-Fun V1.0, we now support additional camera information inputs for camera control. + +Inspired by [CameraCtrl](https://github.com/hehao13/CameraCtrl) and [EasyAnimate](https://github.com/aigc-apps/EasyAnimate), instead of directly resizing inputs like EasyAnimate does to input camera trajectories, we first use PixelUnshuffle to convert temporal information into channel information. Then, using an adapter mechanism, we transform the camera trajectory into high-level semantic information before adding it to the video features post-Conv. This allows us to achieve precise control over the camera lens movement. + +The overall framework of the model is shown in the figure below: + +![Control_Camera](https://github.com/user-attachments/assets/0fdb129f-7c74-48e6-9fbd-9fef23ef446e) \ No newline at end of file diff --git a/VideoX-Fun/reports/wan2_1_fun/report_v1_1_zh-CN.md b/VideoX-Fun/reports/wan2_1_fun/report_v1_1_zh-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..0c816116f5dc4c46ae2e1a8a4466c57fd6f0ccc1 --- /dev/null +++ b/VideoX-Fun/reports/wan2_1_fun/report_v1_1_zh-CN.md @@ -0,0 +1,31 @@ +# Wan Fun v1.1 Report + +在Wan-Fun v1.1中，我们更新了6个模型，分别是：14B的Inpaint模型，Control模型、Control-Camera模型；1.3B的Inpaint模型，Control模型、Control-Camera模型。 + +相比于上一个版本，Inpaint模型经过了更大batch size的训练，模型效果的稳定性更优秀；Control模型则新增加了一个参考图模型，以实现类似于Animate Anyone的效果，在保留之前功能的基础上，我们可以同时传入参考图片和控制视频，以实现生成；最后我们提供了镜头控制模型，可以实现上下左右的镜头控制。 + +另外，我们还发布了添加参考控制信号的训练代码与预测代码，添加镜头控制信号的训练代码和预测代码。 + +对比V1.0版本，Wan Fun V1.1突出了以下功能： + +- 更为稳定的Inpaint模型。 +- 在原控制方案的基础上，实现了参考图加上控制视频的控制方案。 +- 实现了镜头控制模型。 + +## 参考图加上控制视频的实现 +在原本Wan-Fun V1.0的中，我们已经支持了多种控制信号，如Canny、Depth、Pose、MLSD；实现了两种控制方案，如首图+轨迹控制、控制视频指导生成。 + +为了进一步提高控制模型的可用性，我们进一步开发了参考图加上控制视频的控制方案，该功能animate anyone，输入一张参考图片，然后根据Openpose（并不局限于Openpose，Depth信号也有非常亮眼的效果）之类的控制实现生成。此前，Unimate、animate anyone之类的方案一般要求参考图和控制视频骨架基本对齐。在Wan-Fun V1.1 Control中，对齐可以有更好的相似度，不对齐也可以有一定的参考生成效果。 + +我们将参考图使用VAE Encode之后，将latent平铺，然后将平铺后的特征与视频特征进行concat实现生成，为了不影响模型的原功能，我们在训练中随机将参考图latent初始化为全0，代表没有参考图输入，整体模型的工作框架如图所示： + +![Control_Ref](https://github.com/user-attachments/assets/8987a3b5-e691-4c49-a83c-10cad0fe13f3) + +## 镜头控制模型 +在原本Wan-Fun V1.0的基础上，我们支持进一步输入Camera信息，以进行镜头控制。 + +参考[CameraCtrl](https://github.com/hehao13/CameraCtrl)与[EasyAnimate](https://github.com/aigc-apps/EasyAnimate)，我们没有选择类似于EasyAnimate那种直接Resize的方式输入相机镜头的控制轨迹，而是先使用PixelUnshuffle将时序信息转换成通道信息，然后使用Adapter的方式，将相机镜头的轨迹转换成高层语义信息后再与Conv in后的视频特征相加，从而实现了视频镜头的控制。 + +整体模型的工作框架如图所示： + +![Control_Camera](https://github.com/user-attachments/assets/0fdb129f-7c74-48e6-9fbd-9fef23ef446e) \ No newline at end of file diff --git a/VideoX-Fun/requirements.txt b/VideoX-Fun/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..ff393ffe1ea8974ee36ec32c0ec5403c7d39c473 --- /dev/null +++ b/VideoX-Fun/requirements.txt @@ -0,0 +1,28 @@ +Pillow +einops +safetensors +timm +tomesd +librosa +torch>=2.1.2 +torchdiffeq +torchsde +decord +datasets +numpy +scikit-image +opencv-python +omegaconf +SentencePiece +albumentations +imageio[ffmpeg] +imageio[pyav] +tensorboard +beautifulsoup4 +ftfy +func_timeout +onnxruntime +accelerate>=0.25.0 +gradio>=3.41.2 +diffusers>=0.30.1 +transformers>=4.46.2 diff --git a/VideoX-Fun/scripts/README_DEMO.md b/VideoX-Fun/scripts/README_DEMO.md new file mode 100644 index 0000000000000000000000000000000000000000..0d0032cc84e71b0813e3415545960c3ea7d1f486 --- /dev/null +++ b/VideoX-Fun/scripts/README_DEMO.md @@ -0,0 +1,37 @@ +## Demo + +Image generation video corresponding images and prompts. + +If you don't know how to write positive prompts, you can use "smooth motion" or "in the wind" to add dynamism. + +| Image | Prompt | +|--|--| +| ![1.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/1.png) | closeup face photo of man is smiling in black clothes, night city street, bokeh, fireworks in background | +| ![2.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/2.png) | sunset, orange sky, warm lighting, fishing boats, ocean waves, seagulls, rippling water, wharf, silhouette, serene atmosphere, dusk, evening glow, golden hour, coastal landscape, seaside scenery | +| ![3.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/3.png) | a man in an astronaut suit playing a guitar | +| ![4.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/4.png) | time-lapse of a blooming flower with leaves and a stem, blossom | +| ![5.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/5.png) | fireworks display over night city | +| ![6.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/6.png) | a beautiful woman with long hair and a dress blowing in the wind | +| ![7.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/7.png) | the dog is shaking head | +| ![8.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/8.png) | a robot is walking through a destroyed city | +| ![9.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/9.png) | a group of penguins walking on a beach | +| ![10.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/10.png) | a bonfire is lit in the middle of a field | +| ![11.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/11.png) | a boat traveling on the ocean | +| ![12.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/12.png) | pouring honey onto some slices of bread | +| ![13.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/13.png) | a sailboat sailing in rough seas with a dramatic sunset | +| ![14.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/14.png) | a boat traveling on the ocean | +| ![15.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/15.png) | a scenic view of a lake with several seagulls flying above the water. In the foreground, there is a person wearing a red garment, possibly a jacket or a shawl, observing the scenery. The lake has clear blue water, and there's a structure that appears to be a wooden pavilion or boathouse on stilts situated in the water. In the background, hills or mountains can be seen under a clear blue sky, enhancing the tranquil and picturesque setting | +| ![16.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/16.png) | A man's body shimmered with golden light in the wind | +| ![17.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/17.png) | a buried broken emerald cross glazed by the sun emitting smoke, backlit, forgotten, atmospheric AF, detailed, 8k | +| ![18.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/18.png) | A beautiful woman is smiling in the wind | +| ![19.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/19.png) | A beautiful woman is smiling in the wind | +| ![20.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/20.png) | A beautiful woman is smiling in the wind | +| ![21.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/21.png) | A beautiful woman smiles in the heavy snow | +| ![22.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/22.png) | cats smiling taking a selfie with a super wide angle lenses, opening mouth. | +| ![23.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/23.png) | The sturdy sailboat in 'Temperamental Tides', masterfully navigating the restless, pulsating waves of the deep navy sea, maintaining balance on the surging storm grey crests | +| ![24.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/24.png) | The sturdy sailboat in 'Temperamental Tides', masterfully navigating the restless, pulsating waves of the deep navy sea, maintaining balance on the surging storm grey crests | +| ![25.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/25.png) | a beach with waves crashing against it and a sunset in the background a brigantine, a sailboat in the distance, 4k | +| ![26.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/26.png) | Create an illustration that captures the essence of water. The scene should be a tranquil beach at sunrise, with the calm ocean stretching out to the horizon. The sky is painted in soft hues of pink and orange as the sun begins to rise. Gentle waves lap against the sandy shore, creating delicate ripples. The water is crystal clear, reflecting the colors of the sky, and small, glistening seashells are scattered along the shoreline. In the distance, a small sailboat with white sails drifts peacefully on the water. The overall mood of the illustration should be serene and calming, emphasizing the fluid and reflective nature of water.glowneon, glowing, sparks, lightning | +| ![27.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/27.png) | a Lighthouse battered by high winds, huge crashing waves, realistic northern lights, behind lighthouse, realistic stormy seas, high quality image, photographic, mist, and sea spray, storm clouds, angry sky, dusk, peninsula, winter, almost dark, storm, gales, elevated view point, high up perspective, night time, lighthouse light beams, position lighthouse to left of image, view from on high | +| ![28.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/28.png) | Two racing cars racing towards the camera, desert dune in the background, hyperrealistic, driver turning the wheel, more details, speed of light, a trail of intense light follows the cars, image evokes the sensation of speed, frozen movement, insane intricate detail, (masterpiece, best quality), high resolution, (ultra detailed), | +| ![29.png](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/i2v_images/29.png) | one long eared dog, beagle, making goofy faces under water, lie the wind is blowing in his open mouth bubbles wide. an annoyed goldfish swims by | \ No newline at end of file diff --git a/VideoX-Fun/scripts/build_split_models.sh b/VideoX-Fun/scripts/build_split_models.sh new file mode 100644 index 0000000000000000000000000000000000000000..8c8bba74a634bd9e051ea174111ea0125a0447be --- /dev/null +++ b/VideoX-Fun/scripts/build_split_models.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Compile every sliced ONNX model and collect the resulting AXModels. +set -euo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +PROJECT_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) + +ONNX_DIR="${PROJECT_ROOT}/compiled_subgraph_from_onnx/frontend" +OUTPUT_ROOT="${PROJECT_ROOT}/compiled_slice_quant_onnx" +LOG_ROOT="${OUTPUT_ROOT}/logs" +CONFIG_FILE="${PROJECT_ROOT}/pulsar2_configs/transformers.json" +TARGET_HW="AX650" +NPU_MODE="NPU3" +SKIP_FILES=("optimized.onnx" "optimized_quant_axmodel.onnx") + +mkdir -p "${OUTPUT_ROOT}" "${LOG_ROOT}" + +if [[ ! -d "${ONNX_DIR}" ]]; then + echo "未找到切分后的 ONNX 目录: ${ONNX_DIR}" >&2 + exit 1 +fi + +mapfile -t ONNX_FILES < <(find "${ONNX_DIR}" -maxdepth 1 -type f -name '*.onnx' -printf '%f\n' | sort) + +if [[ ${#ONNX_FILES[@]} -eq 0 ]]; then + echo "目录 ${ONNX_DIR} 中没有 ONNX 文件" >&2 + exit 1 +fi + +echo "即将编译 ${#ONNX_FILES[@]} 个子模型" + +for filename in "${ONNX_FILES[@]}"; do + skip=false + for banned in "${SKIP_FILES[@]}"; do + if [[ "${filename}" == "${banned}" ]]; then + skip=true + break + fi + done + if [[ "${skip}" == true ]]; then + echo "跳过 ${filename}" + continue + fi + + stem="${filename%.onnx}" + input_path="${ONNX_DIR}/${filename}" + output_dir="${OUTPUT_ROOT}/${stem}" + output_name="${stem}.axmodel" + log_path="${LOG_ROOT}/${stem}.log" + + mkdir -p "${output_dir}" + + echo "[pulsar2] ${filename} -> ${output_name}" + if ! pulsar2 build \ + --input "${input_path}" \ + --model_type QuantAxModel \ + --output_dir "${output_dir}" \ + --output_name "${output_name}" \ + --config "${CONFIG_FILE}" \ + --target_hardware "${TARGET_HW}" \ + --npu_mode "${NPU_MODE}" \ + --debug.dump_frontend_graph true \ + 2>&1 | tee "${log_path}"; then + echo "编译 ${filename} 失败，日志已保存到 ${log_path}" >&2 + continue + fi + + cp -f "${output_dir}/${output_name}" "${OUTPUT_ROOT}/" + echo "已复制 ${output_name} 到 ${OUTPUT_ROOT}" +done + +echo "全部子模型编译完成。AXModel 文件已汇总至 ${OUTPUT_ROOT}" \ No newline at end of file diff --git a/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN.md b/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..b58198b80c028f2b8f48c9b5ddd4ed413a066741 --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN.md @@ -0,0 +1,163 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use deepspeed and fsdp in CogVideoX-Fun, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since CogVideoX-Fun uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +CogVideoX-Fun without deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +CogVideoX-Fun with Deepspeed Zero-2: +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=4 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --use_deepspeed \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap CogVideoXBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --train_mode="inpaint" \ + --trainable_modules "." +``` diff --git a/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN_CONTROL.md b/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN_CONTROL.md new file mode 100644 index 0000000000000000000000000000000000000000..c63597c612962390423c7e0e3e1869259e0fc2b4 --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN_CONTROL.md @@ -0,0 +1,179 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use deepspeed and fsdp in CogVideoX-Fun, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in CogVideoX-Fun, you need to add a control_file_path, and [DWPose](https://github.com/IDEA-Research/DWPose) is suggested as tool to generate control file. + +```json +[ + { + "file_path": "train/00000001.mp4", + "control_file_path": "control/00000001.mp4", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "control_file_path": "control/00000002.jpg", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +CogVideoX-Fun without deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-Pose" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_control.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=4 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=50 \ + --seed=43 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --trainable_modules "." +``` + +CogVideoX-Fun with Deepspeed Zero-2: +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-Pose" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=4 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=50 \ + --seed=43 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --use_deepspeed \ + --trainable_modules "." +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-Pose" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap CogVideoXBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/cogvideox_fun/train_control.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=4 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=50 \ + --seed=43 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN_LORA.md b/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..187401f325b0b949189aa0132d5243373d11b1ca --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN_LORA.md @@ -0,0 +1,171 @@ +## Lora Training Code + +We can choose whether to use deepspeed and fsdp in CogVideoX-Fun, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since CogVideoX-Fun uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +CogVideoX-Fun without deepspeed: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +CogVideoX-Fun with Deepspeed Zero-2: +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/cogvideox_fun/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap CogVideoXBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/cogvideox_fun/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN_REWARD.md b/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN_REWARD.md new file mode 100644 index 0000000000000000000000000000000000000000..3754c7f3e85a0f9d50c2a40cce6dcbb77895be45 --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/README_TRAIN_REWARD.md @@ -0,0 +1,263 @@ +# Enhance CogVideoX-Fun with Reward Backpropagation (Preference Optimization) +We explore the Reward Backpropagation technique [1](#ref1) [2](#ref2) to optimized the generated videos by [CogVideoX-Fun-V1.1](https://github.com/aigc-apps/CogVideoX-Fun) for better alignment with human preferences. +We provide pre-trained models (i.e. LoRAs) along with the training script. You can use these LoRAs to enhance the corresponding base model as a plug-in or train your own reward LoRA. + +- [Enhance CogVideoX-Fun with Reward Backpropagation (Preference Optimization)](#enhance-cogvideox-fun-with-reward-backpropagation-preference-optimization) + - [Demo](#demo) + - [CogVideoX-Fun-V1.1-5B](#cogvideox-fun-v11-5b) + - [CogVideoX-Fun-V1.1-2B](#cogvideox-fun-v11-2b) + - [Model Zoo](#model-zoo) + - [Inference](#inference) + - [Training](#training) + - [Setup](#setup) + - [Important Args](#important-args) + - [Limitations](#limitations) + - [References](#references) + + +## Demo +### CogVideoX-Fun-V1.1-5B + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PromptCogVideoX-Fun-V1.1-5BCogVideoX-Fun-V1.1-5B
HPSv2.1 Reward LoRA		CogVideoX-Fun-V1.1-5B
MPS Reward LoRA
+ Pig with wings flying above a diamond mountain + + + + + + +
+ A dog runs through a field while a cat climbs a tree + + + + + + +
+ Crystal cake shimmering beside a metal apple + + + + + + +
+ Elderly artist with a white beard painting on a white canvas + + + + + + +
+ +### CogVideoX-Fun-V1.1-2B + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PromptCogVideoX-Fun-V1.1-2BCogVideoX-Fun-V1.1-2B
HPSv2.1 Reward LoRA		CogVideoX-Fun-V1.1-2B
MPS Reward LoRA
+ A blue car drives past a white picket fence on a sunny day + + + + + + +
+ Blue jay swooping near a red maple tree + + + + + + +
+ Yellow curtains swaying near a blue sofa + + + + + + +
+ White tractor plowing near a green farmhouse + + + + + + +
+ +> [!NOTE] +> The above test prompts are from T2V-CompBench. All videos are generated with lora weight 0.7. + +## Model Zoo +| Name | Base Model | Reward Model | Hugging Face | Description | +|--|--|--|--|--| +| CogVideoX-Fun-V1.1-5b-InP-HPS2.1.safetensors | CogVideoX-Fun-V1.1-5b | [HPS v2.1](https://github.com/tgxs002/HPSv2) | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs/resolve/main/CogVideoX-Fun-V1.1-5b-InP-HPS2.1.safetensors) | Official HPS v2.1 reward LoRA (`rank=128` and `network_alpha=64`) for CogVideoX-Fun-V1.1-5b-InP. It is trained with a batch size of 8 for 1,500 steps.| +| CogVideoX-Fun-V1.1-2b-InP-HPS2.1.safetensors | CogVideoX-Fun-V1.1-2b | [HPS v2.1](https://github.com/tgxs002/HPSv2) | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs/resolve/main/CogVideoX-Fun-V1.1-2b-InP-HPS2.1.safetensors) | Official HPS v2.1 reward LoRA (`rank=128` and `network_alpha=64`) for CogVideoX-Fun-V1.1-2b-InP. It is trained with a batch size of 8 for 3,000 steps.| +| CogVideoX-Fun-V1.1-5b-InP-MPS.safetensors | CogVideoX-Fun-V1.1-5b | [MPS](https://github.com/Kwai-Kolors/MPS) | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs/resolve/main/CogVideoX-Fun-V1.1-5b-InP-MPS.safetensors) | Official MPS reward LoRA (`rank=128` and `network_alpha=64`) for CogVideoX-Fun-V1.1-5b-InP. It is trained with a batch size of 8 for 5,500 steps.| +| CogVideoX-Fun-V1.1-2b-InP-MPS.safetensors | CogVideoX-Fun-V1.1-2b | [MPS](https://github.com/Kwai-Kolors/MPS) | [🤗Link](https://huggingface.co/alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs/resolve/main/CogVideoX-Fun-V1.1-2b-InP-MPS.safetensors) | Official MPS reward LoRA (`rank=128` and `network_alpha=64`) for CogVideoX-Fun-V1.1-2b-InP. It is trained with a batch size of 8 for 16,000 steps.| + +## Inference +We provide an example inference code to run CogVideoX-Fun-V1.1-5b-InP with its HPS2.1 reward LoRA. + +```python +import torch +from diffusers import CogVideoXDDIMScheduler + +from videox_fun.models.transformer3d import CogVideoXTransformer3DModel +from videox_fun.pipeline.pipeline_cogvideox_inpaint import CogVideoXFunInpaintPipeline +from videox_fun.utils.lora_utils import merge_lora +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +model_path = "alibaba-pai/CogVideoX-Fun-V1.1-5b-InP" +lora_path = "alibaba-pai/CogVideoX-Fun-V1.1-Reward-LoRAs/CogVideoX-Fun-V1.1-5b-InP-HPS2.1.safetensors" +lora_weight = 0.7 + +prompt = "Pig with wings flying above a diamond mountain" +sample_size = [512, 512] +video_length = 49 + +transformer = CogVideoXTransformer3DModel.from_pretrained(model_path, subfolder="transformer").to(torch.bfloat16) +scheduler = CogVideoXDDIMScheduler.from_pretrained(model_path, subfolder="scheduler") +pipeline = CogVideoXFunInpaintPipeline.from_pretrained( + model_path, transformer=transformer, scheduler=scheduler, torch_dtype=torch.bfloat16 +) +pipeline.enable_model_cpu_offload() +pipeline = merge_lora(pipeline, lora_path, lora_weight) + +generator = torch.Generator(device="cuda").manual_seed(42) +input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=sample_size) +sample = pipeline( + prompt, + num_frames = video_length, + negative_prompt = "bad detailed", + height = sample_size[0], + width = sample_size[1], + generator = generator, + guidance_scale = 7.0, + num_inference_steps = 50, + video = input_video, + mask_video = input_video_mask, +).videos + +save_videos_grid(sample, "samples/output.mp4", fps=8) +``` + +## Training +The [training code](./train_reward_lora.py) is based on [train_lora.py](./train_lora.py). +We provide [a shell script](./train_reward_lora.sh) to train the HPS v2.1 reward LoRA for CogVideoX-Fun-V1.1-2b-InP, +which can be trained on a single A10 with 24GB VRAM. To further reduce the VRAM requirement, please read [Important Args](#important-args). + +### Setup +Please read the [quick-start](https://github.com/aigc-apps/CogVideoX-Fun/blob/main/README.md#quick-start) section to setup the CogVideoX-Fun environment. +**If you're playing with HPS reward model**, please run the following script to install the dependencies: +```bash +# For HPS reward model only +pip install hpsv2 +site_packages=$(python -c "import site; print(site.getsitepackages()[0])") +wget -O $site_packages/hpsv2/src/open_clip/factory.py https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/package/patches/hpsv2_src_open_clip_factory_patches.py +wget -O $site_packages/hpsv2/src/open_clip/bpe_simple_vocab_16e6.txt.gz https://github.com/tgxs002/HPSv2/raw/refs/heads/master/hpsv2/src/open_clip/bpe_simple_vocab_16e6.txt.gz +``` + +> [!NOTE] +> Since some models will be downloaded automatically from HuggingFace, Please run `HF_ENDPOINT=https://hf-mirror.com sh scripts/cogvideox_fun/train_reward_lora.sh` if you cannot access to huggingface.com. + +### Important Args ++ `rank`: The size of LoRA model. The higher the LoRA rank, the more parameters it has, and the more it can learn (including some unnecessary information). +Bt default, we set the rank to 128. You can lower this value to reduce training GPU memory and the LoRA file size. ++ `network_alpha`: A scaling factor changes how the LoRA affect the base model weight. In general, it can be set to half of the `rank`. ++ `prompt_path`: The path to the prompt file (in txt format, each line is a prompt) for sampling training videos. +We randomly selected 701 prompts from [MovieGenBench](https://github.com/facebookresearch/MovieGenBench/blob/main/benchmark/MovieGenVideoBench.txt). ++ `train_sample_height` and `train_sample_width`: The resolution of the sampled training videos. We found +training at a 256x256 resolution can generalize to any other resolution. Reducing the resolution can save GPU memory +during training, but it is recommended that the resolution should be equal to or greater than the image input resolution of the reward model. +Due to the resize and crop preprocessing operations, we suggest using a 1:1 aspect ratio. ++ `reward_fn` and `reward_fn_kwargs`: The reward model name and its keyword arguments. All supported reward models +(Aesthetic Predictor [v2](https://github.com/christophschuhmann/improved-aesthetic-predictor)/[v2.5](https://github.com/discus0434/aesthetic-predictor-v2-5), +[HPS](https://github.com/tgxs002/HPSv2) v2/v2.1, [PickScore](https://github.com/yuvalkirstain/PickScore) and [MPS](https://github.com/Kwai-Kolors/MPS)) +can be found in [reward_fn.py](../cogvideox/reward/reward_fn.py). +You can also customize your own reward model (e.g., combining aesthetic predictor with HPS). ++ `num_decoded_latents` and `num_sampled_frames`: The number of decoded latents (for VAE) and sampled frames (for the reward model). +Since CogVideoX-Fun adopts the 3D casual VAE, we found decoding only the first latent to obtain the first frame for computing the reward +not only reduces training memory usage but also prevents excessive reward optimization and maintains the dynamics of generated videos. + +## Limitations +1. We observe after training to a certain extent, the reward continues to increase, but the quality of the generated videos does not further improve. + The model trickly learns some shortcuts (by adding artifacts in the background, i.e., adversarial patches) to increase the reward. +2. Currently, there is still a lack of suitable preference models for video generation. Directly using image preference models cannot + evaluate preferences along the temporal dimension (such as dynamism and consistency). Further more, We find using image preference models leads to a decrease + in the dynamism of generated videos. Although this can be mitigated by computing the reward using only the first frame of the decoded video, the impact still persists. + +## References +
    +
  1. Clark, Kevin, et al. "Directly fine-tuning diffusion models on differentiable rewards.". In ICLR 2024.
    2. +
  2. Prabhudesai, Mihir, et al. "Aligning text-to-image diffusion models with reward backpropagation." arXiv preprint arXiv:2310.03739 (2023).
    3. +
\ No newline at end of file diff --git a/VideoX-Fun/scripts/cogvideox_fun/train.py b/VideoX-Fun/scripts/cogvideox_fun/train.py new file mode 100644 index 0000000000000000000000000000000000000000..b199440514cd1b1058c4f13f4dc9732b16737bed --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/train.py @@ -0,0 +1,1839 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import EMAModel +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import T5EncoderModel, T5Tokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) +from videox_fun.pipeline import (CogVideoXFunPipeline, + CogVideoXFunControlPipeline, + CogVideoXFunInpaintPipeline) +from videox_fun.pipeline.pipeline_cogvideox_fun_inpaint import ( + add_noise_to_reference_video, get_3d_rotary_pos_embed, + get_resize_crop_region_for_grid) +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import create_network, merge_lora, unmerge_lora +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, save_videos_grid) + + +if is_wandb_available(): + import wandb + + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = CogVideoXTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer" + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = DDIMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler") + + if args.train_mode != "normal": + pipeline = CogVideoXFunInpaintPipeline.from_pretrained( + args.pretrained_model_name_or_path, + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + torch_dtype=weight_dtype + ) + else: + pipeline = CogVideoXFunPipeline.from_pretrained( + args.pretrained_model_name_or_path, + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + torch_dtype=weight_dtype + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=226, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"inpaint"`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler") + + tokenizer = T5Tokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer", revision=args.revision + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + text_encoder = T5EncoderModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision, variant=args.variant, + torch_dtype=weight_dtype + ) + + vae = AutoencoderKLCogVideoX.from_pretrained( + args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision, variant=args.variant + ) + + transformer3d = CogVideoXTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer" + ) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + ema_transformer3d = CogVideoXTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer" + ) + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=CogVideoXTransformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = CogVideoXTransformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = CogVideoXTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + ) + load_model = EMAModel(load_model.parameters(), model_cls=CogVideoXTransformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = CogVideoXTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + patch_size_t = accelerator.unwrap_model(transformer3d).config.patch_size_t + + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + if args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + + for example in examples: + if args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["text"].append(example["text"]) + + batch_video_length = int(min(batch_video_length, len(pixel_values))) + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (batch_video_length - 1) // sample_n_frames_bucket_interval + 1 + local_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + local_video_length -= additional_frames * sample_n_frames_bucket_interval + batch_video_length = local_video_length + + if batch_video_length <= 0: + batch_video_length = 1 + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example[:batch_video_length] for example in new_examples["mask"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids, + return_dict=False + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + mask_pixel_values, texts = batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (pixel_value, text) in enumerate(zip(mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t + patch_size_t = accelerator.unwrap_model(transformer3d).config.patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + temp_n_frames -= additional_frames * sample_n_frames_bucket_interval + if temp_n_frames <= 0: + temp_n_frames = 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t + patch_size_t = accelerator.unwrap_model(transformer3d).config.patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + actual_video_length -= additional_frames * sample_n_frames_bucket_interval + if actual_video_length <= 0: + actual_video_length = 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + if args.train_mode != "normal": + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = latents * vae.config.scaling_factor + + if args.train_mode != "normal": + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = 1 - mask + mask = resize_mask(mask, latents) + + if unwrap_model(transformer3d).config.add_noise_in_inpaint_model: + mask_pixel_values = add_noise_to_reference_video(mask_pixel_values) + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + inpaint_latents = inpaint_latents * vae.config.scaling_factor + inpaint_latents = rearrange(inpaint_latents, "b c f h w -> b f c h w") + + latents = rearrange(latents, "b c f h w -> b f c h w") + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + prompt_embeds = text_encoder( + prompt_ids.input_ids.to(latents.device), + return_dict=False + )[0] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz = latents.shape[0] + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + timesteps = idx_sampling(bsz, generator=torch_rng, device=latents.device) + timesteps = timesteps.long() + + def _prepare_rotary_positional_embeddings( + height: int, + width: int, + num_frames: int, + device: torch.device + ): + vae_scale_factor_spatial = ( + 2 ** (len(vae.config.block_out_channels) - 1) if vae is not None else 8 + ) + + p = unwrap_model(transformer3d).config.patch_size + p_t = unwrap_model(transformer3d).config.patch_size_t + + grid_height = height // (vae_scale_factor_spatial * p) + grid_width = width // (vae_scale_factor_spatial * p) + base_size_height = unwrap_model(transformer3d).config.sample_height // p + base_size_width = unwrap_model(transformer3d).config.sample_width // p + + if p_t is None: + # CogVideoX 1.0 + grid_crops_coords = get_resize_crop_region_for_grid( + (grid_height, grid_width), base_size_width, base_size_height + ) + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=unwrap_model(transformer3d).config.attention_head_dim, + crops_coords=grid_crops_coords, + grid_size=(grid_height, grid_width), + temporal_size=num_frames, + use_real=True, + ) + else: + # CogVideoX 1.5 + base_num_frames = (num_frames + p_t - 1) // p_t + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=unwrap_model(transformer3d).config.attention_head_dim, + crops_coords=None, + grid_size=(grid_height, grid_width), + temporal_size=base_num_frames, + grid_type="slice", + max_size=(base_size_height, base_size_width), + ) + freqs_cos = freqs_cos.to(device=device) + freqs_sin = freqs_sin.to(device=device) + return freqs_cos, freqs_sin + + height, width = batch["pixel_values"].size()[-2], batch["pixel_values"].size()[-1] + # 7. Create rotary embeds if required + image_rotary_emb = ( + _prepare_rotary_positional_embeddings(height, width, latents.size(1), latents.device) + if unwrap_model(transformer3d).config.use_rotary_positional_embeddings + else None + ) + prompt_embeds = prompt_embeds.to(device=latents.device) + + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + if noise_scheduler.config.prediction_type == "epsilon": + target = noise + elif noise_scheduler.config.prediction_type == "v_prediction": + target = noise_scheduler.get_velocity(latents, noise, timesteps) + else: + raise ValueError(f"Unknown prediction type {noise_scheduler.config.prediction_type}") + + # predict the noise residual + noise_pred = transformer3d( + hidden_states=noisy_latents, + encoder_hidden_states=prompt_embeds, + timestep=timesteps, + image_rotary_emb=image_rotary_emb, + return_dict=False, + inpaint_latents=inpaint_latents if args.train_mode != "normal" else None, + )[0] + + loss = F.mse_loss(noise_pred.float(), target.float(), reduction="mean") + + if args.motion_sub_loss and noise_pred.size()[1] > 2: + gt_sub_noise = noise_pred[:, 1:, :].float() - noise_pred[:, :-1, :].float() + pre_sub_noise = target[:, 1:, :].float() - target[:, :-1, :].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/scripts/cogvideox_fun/train.sh b/VideoX-Fun/scripts/cogvideox_fun/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..d3e75082f4d74d17297888cc162a6dcc83852b54 --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/train.sh @@ -0,0 +1,79 @@ +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --train_mode="inpaint" \ + --trainable_modules "." + +# Training command for CogVideoX-Fun-V1.5 +# export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-V1.5-5b-InP" +# export DATASET_NAME="datasets/internal_datasets/" +# export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +# NCCL_DEBUG=INFO + +# accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train.py \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --train_data_dir=$DATASET_NAME \ +# --train_data_meta=$DATASET_META_NAME \ +# --image_sample_size=512 \ +# --video_sample_size=512 \ +# --token_sample_size=512 \ +# --video_sample_stride=3 \ +# --video_sample_n_frames=85 \ +# --train_batch_size=1 \ +# --video_repeat=1 \ +# --gradient_accumulation_steps=1 \ +# --dataloader_num_workers=8 \ +# --num_train_epochs=100 \ +# --checkpointing_steps=50 \ +# --learning_rate=2e-05 \ +# --lr_scheduler="constant_with_warmup" \ +# --lr_warmup_steps=100 \ +# --seed=42 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --vae_mini_batch=1 \ +# --max_grad_norm=0.05 \ +# --random_hw_adapt \ +# --training_with_video_token_length \ +# --enable_bucket \ +# --train_mode="inpaint" \ +# --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/cogvideox_fun/train_control.py b/VideoX-Fun/scripts/cogvideox_fun/train_control.py new file mode 100644 index 0000000000000000000000000000000000000000..0bd223f3f67a85dce43b23991ab335737e504e02 --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/train_control.py @@ -0,0 +1,1726 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import AutoencoderKL, DDPMScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import EMAModel +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from packaging import version +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) +from videox_fun.pipeline import (CogVideoXFunPipeline, + CogVideoXFunControlPipeline, + CogVideoXFunInpaintPipeline) +from videox_fun.pipeline.pipeline_cogvideox_fun_inpaint import ( + add_noise_to_reference_video, get_3d_rotary_pos_embed, + get_resize_crop_region_for_grid) +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, save_videos_grid) + +if is_wandb_available(): + import wandb + + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = CogVideoXTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer" + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + pipeline = CogVideoXFunControlPipeline.from_pretrained( + args.pretrained_model_name_or_path, + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + torch_dtype=weight_dtype, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int(args.video_sample_n_frames // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(args.validation_paths[i], video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + control_video = input_video, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_paths", + type=str, + default=None, + nargs="+", + help=("A set of control videos evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--not_sigma_loss", action="store_true", help="Whether or not to not use sigma_loss." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=226, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler") + + tokenizer = T5Tokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer", revision=args.revision + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + text_encoder = T5EncoderModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + vae = AutoencoderKLCogVideoX.from_pretrained( + args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision, variant=args.variant + ) + vae = vae.eval() + + transformer3d = CogVideoXTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer" + ) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + ema_transformer3d = CogVideoXTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer" + ) + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=CogVideoXTransformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = CogVideoXTransformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = CogVideoXTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + ) + load_model = EMAModel(load_model.parameters(), model_cls=CogVideoXTransformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = CogVideoXTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + patch_size_t = accelerator.unwrap_model(transformer3d).config.patch_size_t + + train_dataset = ImageVideoControlDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + new_examples["control_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + if args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + + for example in examples: + if args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + new_examples["text"].append(example["text"]) + + batch_video_length = int(min(batch_video_length, len(pixel_values))) + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (batch_video_length - 1) // sample_n_frames_bucket_interval + 1 + local_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + local_video_length -= additional_frames * sample_n_frames_bucket_interval + batch_video_length = local_video_length + + if batch_video_length <= 0: + batch_video_length = 1 + + new_examples["pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids, + return_dict=False + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t + patch_size_t = accelerator.unwrap_model(transformer3d).config.patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + temp_n_frames -= additional_frames * sample_n_frames_bucket_interval + if temp_n_frames <= 0: + temp_n_frames = 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t + patch_size_t = accelerator.unwrap_model(transformer3d).config.patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + actual_video_length -= additional_frames * sample_n_frames_bucket_interval + if actual_video_length <= 0: + actual_video_length = 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = latents * vae.config.scaling_factor + + control_latents = _batch_encode_vae(control_pixel_values) + control_latents = control_latents * vae.config.scaling_factor + control_latents = rearrange(control_latents, "b c f h w -> b f c h w") + + latents = rearrange(latents, "b c f h w -> b f c h w") + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + prompt_embeds = text_encoder( + prompt_ids.input_ids.to(latents.device), + return_dict=False + )[0] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz = latents.shape[0] + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + timesteps = idx_sampling(bsz, generator=torch_rng, device=latents.device) + timesteps = timesteps.long() + + def _prepare_rotary_positional_embeddings( + height: int, + width: int, + num_frames: int, + device: torch.device + ): + vae_scale_factor_spatial = ( + 2 ** (len(vae.config.block_out_channels) - 1) if vae is not None else 8 + ) + + p = unwrap_model(transformer3d).config.patch_size + p_t = unwrap_model(transformer3d).config.patch_size_t + + grid_height = height // (vae_scale_factor_spatial * p) + grid_width = width // (vae_scale_factor_spatial * p) + base_size_height = unwrap_model(transformer3d).config.sample_height // p + base_size_width = unwrap_model(transformer3d).config.sample_width // p + + if p_t is None: + # CogVideoX 1.0 + grid_crops_coords = get_resize_crop_region_for_grid( + (grid_height, grid_width), base_size_width, base_size_height + ) + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=unwrap_model(transformer3d).config.attention_head_dim, + crops_coords=grid_crops_coords, + grid_size=(grid_height, grid_width), + temporal_size=num_frames, + use_real=True, + ) + else: + # CogVideoX 1.5 + base_num_frames = (num_frames + p_t - 1) // p_t + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=unwrap_model(transformer3d).config.attention_head_dim, + crops_coords=None, + grid_size=(grid_height, grid_width), + temporal_size=base_num_frames, + grid_type="slice", + max_size=(base_size_height, base_size_width), + ) + freqs_cos = freqs_cos.to(device=device) + freqs_sin = freqs_sin.to(device=device) + return freqs_cos, freqs_sin + + height, width = batch["pixel_values"].size()[-2], batch["pixel_values"].size()[-1] + # 7. Create rotary embeds if required + image_rotary_emb = ( + _prepare_rotary_positional_embeddings(height, width, latents.size(1), latents.device) + if unwrap_model(transformer3d).config.use_rotary_positional_embeddings + else None + ) + prompt_embeds = prompt_embeds.to(device=latents.device) + + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + if noise_scheduler.config.prediction_type == "epsilon": + target = noise + elif noise_scheduler.config.prediction_type == "v_prediction": + target = noise_scheduler.get_velocity(latents, noise, timesteps) + else: + raise ValueError(f"Unknown prediction type {noise_scheduler.config.prediction_type}") + + # predict the noise residual + noise_pred = transformer3d( + hidden_states=noisy_latents, + encoder_hidden_states=prompt_embeds, + timestep=timesteps, + image_rotary_emb=image_rotary_emb, + return_dict=False, + control_latents=control_latents, + )[0] + + loss = F.mse_loss(noise_pred.float(), target.float(), reduction="mean") + + if args.motion_sub_loss and noise_pred.size()[1] > 2: + gt_sub_noise = noise_pred[:, 1:, :].float() - noise_pred[:, :-1, :].float() + pre_sub_noise = target[:, 1:, :].float() - target[:, :-1, :].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/cogvideox_fun/train_control.sh b/VideoX-Fun/scripts/cogvideox_fun/train_control.sh new file mode 100644 index 0000000000000000000000000000000000000000..1ee000ea7b00000fa6b10fd9a4375a7f43818994 --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/train_control.sh @@ -0,0 +1,77 @@ +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-Pose" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_control.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=4 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=50 \ + --seed=43 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --trainable_modules "." + +# Training command for CogVideoX-Fun-V1.5 +# export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-V1.5-5b-Pose" +# export DATASET_NAME="datasets/internal_datasets/" +# export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +# NCCL_DEBUG=INFO + +# accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_control.py \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --train_data_dir=$DATASET_NAME \ +# --train_data_meta=$DATASET_META_NAME \ +# --image_sample_size=512 \ +# --video_sample_size=512 \ +# --token_sample_size=512 \ +# --video_sample_stride=3 \ +# --video_sample_n_frames=85 \ +# --train_batch_size=4 \ +# --video_repeat=1 \ +# --gradient_accumulation_steps=1 \ +# --dataloader_num_workers=8 \ +# --num_train_epochs=100 \ +# --checkpointing_steps=50 \ +# --learning_rate=2e-05 \ +# --lr_scheduler="constant_with_warmup" \ +# --lr_warmup_steps=50 \ +# --seed=43 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --vae_mini_batch=1 \ +# --max_grad_norm=0.05 \ +# --random_hw_adapt \ +# --training_with_video_token_length \ +# --enable_bucket \ +# --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/cogvideox_fun/train_lora.py b/VideoX-Fun/scripts/cogvideox_fun/train_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..8332585dde7f16503ab868d4186a1759270a0807 --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/train_lora.py @@ -0,0 +1,1874 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import copy +import gc +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, DDPMScheduler +from diffusers.optimization import get_scheduler +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) +from videox_fun.pipeline import (CogVideoXFunPipeline, + CogVideoXFunControlPipeline, + CogVideoXFunInpaintPipeline) +from videox_fun.pipeline.pipeline_cogvideox_fun_inpaint import ( + add_noise_to_reference_video, get_3d_rotary_pos_embed, + get_resize_crop_region_for_grid) +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import create_network, merge_lora, unmerge_lora +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, save_videos_grid) + +if is_wandb_available(): + import wandb + + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = CogVideoXTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = DDIMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler") + + if args.train_mode != "normal": + pipeline = CogVideoXFunInpaintPipeline.from_pretrained( + args.pretrained_model_name_or_path, + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + torch_dtype=weight_dtype, + ) + else: + pipeline = CogVideoXFunPipeline.from_pretrained( + args.pretrained_model_name_or_path, + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + torch_dtype=weight_dtype + ) + + pipeline = pipeline.to(accelerator.device) + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 7, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--noise_share_in_frames", action="store_true", help="Whether enable noise share in frames." + ) + parser.add_argument( + "--noise_share_in_frames_ratio", type=float, default=0.5, help="Noise share ratio.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--image_repeat_in_forward", + type=int, + default=0, + help="Num of repeat image in forward.", + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=226, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"inpaint"`.' + ), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler") + + tokenizer = T5Tokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer", revision=args.revision + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + text_encoder = T5EncoderModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision, variant=args.variant, + torch_dtype=weight_dtype + ) + + vae = AutoencoderKLCogVideoX.from_pretrained( + args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision, variant=args.variant + ) + + transformer3d = CogVideoXTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer" + ) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + patch_size_t = accelerator.unwrap_model(transformer3d).config.patch_size_t + + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + if args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + + for example in examples: + if args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["text"].append(example["text"]) + + batch_video_length = int(min(batch_video_length, len(pixel_values))) + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (batch_video_length - 1) // sample_n_frames_bucket_interval + 1 + local_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + local_video_length -= additional_frames * sample_n_frames_bucket_interval + batch_video_length = local_video_length + + if batch_video_length <= 0: + batch_video_length = 1 + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example[:batch_video_length] for example in new_examples["mask"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids, + return_dict=False + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=transformer3d.transformer_blocks) + transformer3d = shard_fn(transformer3d) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + mask_pixel_values, texts = batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (pixel_value, text) in enumerate(zip(mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t + patch_size_t = accelerator.unwrap_model(transformer3d).config.patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + temp_n_frames -= additional_frames * sample_n_frames_bucket_interval + if temp_n_frames <= 0: + temp_n_frames = 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t + patch_size_t = accelerator.unwrap_model(transformer3d).config.patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + actual_video_length -= additional_frames * sample_n_frames_bucket_interval + if actual_video_length <= 0: + actual_video_length = 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + if args.train_mode != "normal": + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = latents * vae.config.scaling_factor + + if args.train_mode != "normal": + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = 1 - mask + mask = resize_mask(mask, latents) + + if unwrap_model(transformer3d).config.add_noise_in_inpaint_model: + mask_pixel_values = add_noise_to_reference_video(mask_pixel_values) + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + inpaint_latents = inpaint_latents * vae.config.scaling_factor + inpaint_latents = rearrange(inpaint_latents, "b c f h w -> b f c h w") + + latents = rearrange(latents, "b c f h w -> b f c h w") + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + prompt_embeds = text_encoder( + prompt_ids.input_ids.to(latents.device), + return_dict=False + )[0] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz = latents.shape[0] + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + timesteps = idx_sampling(bsz, generator=torch_rng, device=latents.device) + timesteps = timesteps.long() + + def _prepare_rotary_positional_embeddings( + height: int, + width: int, + num_frames: int, + device: torch.device + ): + vae_scale_factor_spatial = ( + 2 ** (len(vae.config.block_out_channels) - 1) if vae is not None else 8 + ) + + p = unwrap_model(transformer3d).config.patch_size + p_t = unwrap_model(transformer3d).config.patch_size_t + + grid_height = height // (vae_scale_factor_spatial * p) + grid_width = width // (vae_scale_factor_spatial * p) + base_size_height = unwrap_model(transformer3d).config.sample_height // p + base_size_width = unwrap_model(transformer3d).config.sample_width // p + + if p_t is None: + # CogVideoX 1.0 + grid_crops_coords = get_resize_crop_region_for_grid( + (grid_height, grid_width), base_size_width, base_size_height + ) + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=unwrap_model(transformer3d).config.attention_head_dim, + crops_coords=grid_crops_coords, + grid_size=(grid_height, grid_width), + temporal_size=num_frames, + use_real=True, + ) + else: + # CogVideoX 1.5 + base_num_frames = (num_frames + p_t - 1) // p_t + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=unwrap_model(transformer3d).config.attention_head_dim, + crops_coords=None, + grid_size=(grid_height, grid_width), + temporal_size=base_num_frames, + grid_type="slice", + max_size=(base_size_height, base_size_width), + ) + freqs_cos = freqs_cos.to(device=device) + freqs_sin = freqs_sin.to(device=device) + return freqs_cos, freqs_sin + + height, width = batch["pixel_values"].size()[-2], batch["pixel_values"].size()[-1] + # 7. Create rotary embeds if required + image_rotary_emb = ( + _prepare_rotary_positional_embeddings(height, width, latents.size(1), latents.device) + if unwrap_model(transformer3d).config.use_rotary_positional_embeddings + else None + ) + prompt_embeds = prompt_embeds.to(device=latents.device) + + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + if noise_scheduler.config.prediction_type == "epsilon": + target = noise + elif noise_scheduler.config.prediction_type == "v_prediction": + target = noise_scheduler.get_velocity(latents, noise, timesteps) + else: + raise ValueError(f"Unknown prediction type {noise_scheduler.config.prediction_type}") + + # predict the noise residual + noise_pred = transformer3d( + hidden_states=noisy_latents, + encoder_hidden_states=prompt_embeds, + timestep=timesteps, + image_rotary_emb=image_rotary_emb, + return_dict=False, + inpaint_latents=inpaint_latents if args.train_mode != "normal" else None, + )[0] + + loss = F.mse_loss(noise_pred.float(), target.float(), reduction="mean") + + if args.motion_sub_loss and noise_pred.size()[1] > 2: + gt_sub_noise = noise_pred[:, 1:, :].float() - noise_pred[:, :-1, :].float() + pre_sub_noise = target[:, 1:, :].float() - target[:, :-1, :].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/scripts/cogvideox_fun/train_lora.sh b/VideoX-Fun/scripts/cogvideox_fun/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..19ec0acf310c66f8a270dd530953d34f8aa9133d --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/train_lora.sh @@ -0,0 +1,83 @@ +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-2b-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=512 \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=3 \ + --video_sample_n_frames=49 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="inpaint" + +# Training command for CogVideoX-Fun-V1.5 +# export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-V1.5-5b-InP" +# export DATASET_NAME="datasets/internal_datasets/" +# export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +# NCCL_DEBUG=INFO + +# accelerate launch --mixed_precision="bf16" scripts/cogvideox_fun/train_lora.py \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --train_data_dir=$DATASET_NAME \ +# --train_data_meta=$DATASET_META_NAME \ +# --image_sample_size=512 \ +# --video_sample_size=512 \ +# --token_sample_size=512 \ +# --video_sample_stride=3 \ +# --video_sample_n_frames=85 \ +# --train_batch_size=1 \ +# --video_repeat=1 \ +# --gradient_accumulation_steps=1 \ +# --dataloader_num_workers=8 \ +# --num_train_epochs=100 \ +# --checkpointing_steps=50 \ +# --learning_rate=1e-04 \ +# --seed=42 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --vae_mini_batch=1 \ +# --max_grad_norm=0.05 \ +# --random_hw_adapt \ +# --training_with_video_token_length \ +# --enable_bucket \ +# --rank=64 \ +# --network_alpha=32 \ +# --target_name="to_q,to_k,to_v,ff.0,ff.2" \ +# --use_peft_lora \ +# --low_vram \ +# --train_mode="inpaint" \ No newline at end of file diff --git a/VideoX-Fun/scripts/cogvideox_fun/train_reward_lora.py b/VideoX-Fun/scripts/cogvideox_fun/train_reward_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..81e78fadfaaf9ab03ae56522fba1f5824e8e77a1 --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/train_reward_lora.py @@ -0,0 +1,1416 @@ +"""Modified from VideoX-Fun/scripts/cogvideox_fun/train_lora.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import json +import logging +import math +import os +import random +import shutil +import sys +from contextlib import contextmanager +from typing import List, Optional, Union + +import accelerate +import diffusers +import numpy as np +import torch +import torch.utils.checkpoint +import torchvision +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from decord import VideoReader +from diffusers import CogVideoXDPMScheduler, DDIMScheduler +from diffusers.optimization import get_scheduler +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from packaging import version +from tqdm.auto import tqdm +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +import videox_fun.reward.reward_fn as reward_fn +from videox_fun.models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) +from videox_fun.pipeline.pipeline_cogvideox_fun_inpaint import (CogVideoXFunInpaintPipeline, + get_3d_rotary_pos_embed, + get_resize_crop_region_for_grid) +from videox_fun.utils.lora_utils import create_network, merge_lora +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + + +@contextmanager +def video_reader(*args, **kwargs): + """A context manager to solve the memory leak of decord. + """ + vr = VideoReader(*args, **kwargs) + try: + yield vr + finally: + del vr + gc.collect() + + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, + loss_fn, args, accelerator, weight_dtype, global_step, validation_prompts_idx +): + try: + logger.info("Running validation... ") + + transformer3d_val = CogVideoXTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = DDIMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler") + + if args.vae_gradient_checkpointing or args.low_vram: + # Initialize a new vae if gradient checkpointing is enabled. + vae = AutoencoderKLCogVideoX.from_pretrained( + args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision, variant=args.variant + ).to(weight_dtype) + pipeline = CogVideoXFunInpaintPipeline.from_pretrained( + args.pretrained_model_name_or_path, + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + torch_dtype=weight_dtype, + ) + if args.low_vram: + pipeline.enable_model_cpu_offload() + else: + pipeline = pipeline.to(device=accelerator.device) + lora_state_dict = accelerator.unwrap_model(network).state_dict() + pipeline = merge_lora(pipeline, None, 1, accelerator.device, state_dict=lora_state_dict, transformer_only=True) + + to_tensor = torchvision.transforms.ToTensor() + validation_loss, validation_reward = 0, 0 + for i in range(len(validation_prompts_idx)): + validation_idx, validation_prompt = validation_prompts_idx[i] + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + temporal_compression_ratio = vae.config.temporal_compression_ratio + video_length = 1 + if args.video_length != 1: + video_length += int((args.video_length - 1) // temporal_compression_ratio * temporal_compression_ratio) + sample_size = [args.validation_sample_height, args.validation_sample_width] + input_video, input_video_mask, _ = get_image_to_video_latent( + None, None, video_length=video_length, sample_size=sample_size + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + sample = pipeline( + validation_prompt, + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.validation_sample_height, + width = args.validation_sample_width, + guidance_scale = 7, + generator = generator, + video = input_video, + mask_video = input_video_mask, + ).videos + sample_saved_name = f"validation_sample/sample-{global_step}-{validation_idx}.mp4" + sample_saved_path = os.path.join(args.output_dir, sample_saved_name) + save_videos_grid(sample, sample_saved_path, fps=8) + + num_sampled_frames = 4 + sampled_frames_list = [] + with video_reader(sample_saved_path) as vr: + sampled_frame_idx_list = np.linspace(0, len(vr), num_sampled_frames, endpoint=False, dtype=int) + sampled_frame_list = vr.get_batch(sampled_frame_idx_list).asnumpy() + sampled_frames = torch.stack([to_tensor(frame) for frame in sampled_frame_list], dim=0) + sampled_frames_list.append(sampled_frames) + + sampled_frames = torch.stack(sampled_frames_list) + sampled_frames = rearrange(sampled_frames, "b t c h w -> b c t h w") + loss, reward = loss_fn(sampled_frames, [validation_prompt]) + validation_loss, validation_reward = validation_loss + loss, validation_reward + reward + + validation_loss = validation_loss / len(validation_prompts_idx) + validation_reward = validation_reward / len(validation_prompts_idx) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return validation_loss, validation_reward + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None, None + + +def load_prompts(prompt_path, prompt_column="prompt", start_idx=None, end_idx=None): + prompt_list = [] + if prompt_path.endswith(".txt"): + with open(prompt_path, "r") as f: + for line in f: + prompt_list.append(line.strip()) + elif prompt_path.endswith(".jsonl"): + with open(prompt_path, "r") as f: + for line in f.readlines(): + item = json.loads(line) + prompt_list.append(item[prompt_column]) + else: + raise ValueError("The prompt_path must end with .txt or .jsonl.") + prompt_list = prompt_list[start_idx:end_idx] + + return prompt_list + + +# Modified from cogvideox.pipeline.pipeline_cogvideox_inpaint.CogVideoXFunInpaintPipeline._get_t5_prompt_embeds +def get_t5_prompt_embeds( + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 226, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return prompt_embeds + + +# Modified from cogvideox.pipeline.pipeline_cogvideox_inpaint.CogVideoXFunInpaintPipeline.encode_prompt +def encode_prompt( + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 226, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +): + r""" + Encodes the prompt into text encoder hidden states. + """ + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + +# Modified from cogvideox.pipeline.pipeline_cogvideox_inpaint.CogVideoXFunInpaintPipeline.prepare_extra_step_kwargs +def prepare_extra_step_kwargs(scheduler, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + import inspect + + accepts_eta = "eta" in set(inspect.signature(scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + +# Modified from cogvideox.pipeline.pipeline_cogvideox_inpaint.CogVideoXFunInpaintPipeline._prepare_rotary_positional_embeddings +def prepare_rotary_positional_embeddings( + height: int, + width: int, + num_frames: int, + vae_scale_factor_spatial: int = 8, + patch_size: int = 2, + patch_size_t: int = 2, + attention_head_dim: int = 64, + sample_height: int = 720, + sample_width: int = 480, + device: torch.device = "cpu" +): + + grid_height = height // (vae_scale_factor_spatial * patch_size) + grid_width = width // (vae_scale_factor_spatial * patch_size) + base_size_height = sample_height // patch_size + base_size_width = sample_width // patch_size + + if patch_size_t is None: + # CogVideoX 1.0 + grid_crops_coords = get_resize_crop_region_for_grid( + (grid_height, grid_width), base_size_width, base_size_height + ) + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=attention_head_dim, + crops_coords=grid_crops_coords, + grid_size=(grid_height, grid_width), + temporal_size=num_frames, + use_real=True, + ) + else: + # CogVideoX 1.5 + base_num_frames = (num_frames + patch_size_t - 1) // patch_size_t + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=attention_head_dim, + crops_coords=None, + grid_size=(grid_height, grid_width), + temporal_size=base_num_frames, + grid_type="slice", + max_size=(base_size_height, base_size_width), + ) + freqs_cos = freqs_cos.to(device=device) + freqs_sin = freqs_sin.to(device=device) + return freqs_cos, freqs_sin + + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--validation_prompt_path", + type=str, + default=None, + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_batch_size", + type=int, + default=1, + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_sample_height", + type=int, + default=512, + help="The height of sampling videos in validation.", + ) + parser.add_argument( + "--validation_sample_width", + type=int, + default=512, + help="The width of sampling videos in validation.", + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=200) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing (for DiT) to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--vae_gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing (for VAE) to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + + parser.add_argument( + "--prompt_path", + type=str, + default="normal", + help="The path to the training prompt file.", + ) + parser.add_argument( + '--train_sample_height', + type=int, + default=384, + help='The height of sampling videos in training' + ) + parser.add_argument( + '--train_sample_width', + type=int, + default=672, + help='The width of sampling videos in training' + ) + parser.add_argument( + "--video_length", + type=int, + default=49, + help="The number of frames to generate in training and validation." + ) + parser.add_argument( + '--eta', + type=float, + default=0.0, + help='eta parameter for the DDIM sampler. this controls the amount of noise injected into the sampling process, ' + 'with 0.0 being fully deterministic and 1.0 being equivalent to the DDPM sampler.' + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=6.0, + help="The classifier-free diffusion guidance." + ) + parser.add_argument( + "--num_inference_steps", + type=int, + default=50, + help="The number of denoising steps in training and validation." + ) + parser.add_argument( + "--num_decoded_latents", + type=int, + default=3, + help="The number of latents to be decoded." + ) + parser.add_argument( + "--num_sampled_frames", + type=int, + default=None, + help="The number of sampled frames for the reward function." + ) + parser.add_argument( + "--reward_fn", + type=str, + default="HPSReward", + help='The reward function.' + ) + parser.add_argument( + "--reward_fn_kwargs", + type=str, + default=None, + help='The keyword arguments of the reward function.' + ) + parser.add_argument( + "--backprop", + action="store_true", + default=False, + help="Whether to use the reward backprop training mode.", + ) + parser.add_argument( + "--backprop_step_list", + nargs="+", + type=int, + default=None, + help="The preset step list for reward backprop. If provided, overrides `backprop_strategy`." + ) + parser.add_argument( + "--backprop_strategy", + choices=["last", "tail", "uniform", "random"], + default="last", + help="The strategy for reward backprop." + ) + parser.add_argument( + "--stop_latent_model_input_gradient", + action="store_true", + default=False, + help="Whether to stop the gradient of the latents during reward backprop.", + ) + parser.add_argument( + "--backprop_random_start_step", + type=int, + default=0, + help="The random start step for reward backprop. Only used when `backprop_strategy` is random." + ) + parser.add_argument( + "--backprop_random_end_step", + type=int, + default=50, + help="The random end step for reward backprop. Only used when `backprop_strategy` is random." + ) + parser.add_argument( + "--backprop_num_steps", + type=int, + default=5, + help="The number of steps for backprop. Only used when `backprop_strategy` is tail/uniform/random." + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + # Sanity check for validation + do_validation = (args.validation_prompt_path is not None or args.validation_prompts is not None) + if do_validation: + if not (os.path.exists(args.validation_prompt_path) or args.validation_prompt_path.endswith(".txt")): + raise ValueError("The `--validation_prompt_path` must be a txt file containing prompts.") + if args.validation_batch_size < accelerator.num_processes or args.validation_batch_size % accelerator.num_processes != 0: + raise ValueError("The `--validation_batch_size` must be divisible by the number of processes.") + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed, device_specific=True) + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + # Use DDIM instead of DDPM to sample training videos. + noise_scheduler = DDIMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler") + noise_scheduler.set_timesteps(args.num_inference_steps, device=accelerator.device) + + tokenizer = T5Tokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer", revision=args.revision + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + text_encoder = T5EncoderModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision, variant=args.variant, + torch_dtype=weight_dtype + ) + + vae = AutoencoderKLCogVideoX.from_pretrained( + args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision, variant=args.variant + ) + + transformer3d = CogVideoXTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer" + ) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + add_lora_in_attn_temporal=True, + ) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + vae_scale_factor_spatial = 2 ** (len(vae.config.block_out_channels) - 1) + vae_scale_factor_temporal = vae.config.temporal_compression_ratio + num_channels_latent = vae.config.latent_channels + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + accelerator.register_save_state_pre_hook(save_model_hook) + # accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + if args.vae_gradient_checkpointing: + # Since 3D casual VAE need a cache to decode all latents autoregressively, .Thus, gradient checkpointing can only be + # enabled when decoding the first batch (i.e. the first three) of latents, in which case the cache is not being used. + if args.num_decoded_latents > 3: + raise ValueError("The vae_gradient_checkpointing is not supported for num_decoded_latents > 3.") + vae.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # loss function + reward_fn_kwargs = {} + if args.reward_fn_kwargs is not None: + reward_fn_kwargs = json.loads(args.reward_fn_kwargs) + if accelerator.is_main_process: + # Check if the model is downloaded in the main process. + loss_fn = getattr(reward_fn, args.reward_fn)(device="cpu", dtype=weight_dtype, **reward_fn_kwargs) + accelerator.wait_for_everyone() + loss_fn = getattr(reward_fn, args.reward_fn)(device=accelerator.device, dtype=weight_dtype, **reward_fn_kwargs) + + # Get RL training prompts + prompt_list = load_prompts(args.prompt_path) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(prompt_list) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + network, optimizer, lr_scheduler = accelerator.prepare( + network, optimizer, lr_scheduler + ) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device, dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + text_encoder.to(accelerator.device) + + # Enable auto split process for vae + vae.enable_auto_split_process() + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(prompt_list) / args.train_batch_size / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(prompt_list)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + from safetensors.torch import load_file, safe_open + state_dict = load_file(os.path.join(os.path.join(args.output_dir, path), "lora_diffusion_pytorch_model.safetensors")) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + first_epoch = global_step // num_update_steps_per_epoch + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + train_reward = 0.0 + # In the following training loop, randomly select training prompts and use the + # `CogVideoXFunInpaintPipeline` to sample videos, calculate rewards, and update the network. + for _ in range(num_update_steps_per_epoch): + # train_prompt = random.sample(prompt_list, args.train_batch_size) + train_prompt = random.choices(prompt_list, k=args.train_batch_size) + logger.info(f"train_prompt: {train_prompt}") + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = args.guidance_scale > 1.0 + + # Reduce the vram by offload text encoders + if args.low_vram: + torch.cuda.empty_cache() + text_encoder.to(accelerator.device) + + # Encode input prompt + prompt_embeds, negative_prompt_embeds = encode_prompt( + tokenizer, + text_encoder, + train_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt="", + dtype=weight_dtype, + device=accelerator.device, + ) + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + + # Reduce the vram by offload text encoders + if args.low_vram: + text_encoder.to("cpu") + torch.cuda.empty_cache() + + # Prepare timesteps + timesteps = noise_scheduler.timesteps + + # Prepare latents + latent_shape = [ + len(train_prompt), + (args.video_length - 1) // vae_scale_factor_temporal + 1, + num_channels_latent, + args.train_sample_height // vae_scale_factor_spatial, + args.train_sample_width // vae_scale_factor_spatial, + ] + + with accelerator.accumulate(transformer3d): + with accelerator.autocast(): + latents = torch.randn(*latent_shape, device=accelerator.device, dtype=weight_dtype) + latents = latents * noise_scheduler.init_noise_sigma + + mask_latents = torch.zeros_like(latents)[:, :, :1].to(latents.device, latents.dtype) + masked_video_latents = torch.zeros_like(latents).to(latents.device, latents.dtype) + mask_input = torch.cat([mask_latents] * 2) if do_classifier_free_guidance else mask_latents + masked_video_latents_input = ( + torch.cat([masked_video_latents] * 2) if do_classifier_free_guidance else masked_video_latents + ) + inpaint_latents = torch.cat([mask_input, masked_video_latents_input], dim=2).to(latents.dtype) + + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + # Prepare extra step kwargs. + extra_step_kwargs = prepare_extra_step_kwargs(noise_scheduler, generator, args.eta) + + # Create rotary embeds if required + image_rotary_emb = ( + prepare_rotary_positional_embeddings( + height = args.train_sample_height, + width = args.train_sample_width, + num_frames = latents.size(1), + vae_scale_factor_spatial = vae_scale_factor_spatial, + patch_size = unwrap_model(transformer3d).config.patch_size, + patch_size_t = unwrap_model(transformer3d).config.patch_size_t, + attention_head_dim = unwrap_model(transformer3d).config.attention_head_dim, + sample_height = unwrap_model(transformer3d).config.sample_height, + sample_width = unwrap_model(transformer3d).config.sample_width, + device = accelerator.device + ) + if unwrap_model(transformer3d).config.use_rotary_positional_embeddings + else None + ) + + # Denoising loop + if args.backprop: + if args.backprop_step_list is None: + if args.backprop_strategy == "last": + backprop_step_list = [args.num_inference_steps - 1] + elif args.backprop_strategy == "tail": + backprop_step_list = list(range(args.num_inference_steps))[-args.backprop_num_steps:] + elif args.backprop_strategy == "uniform": + interval = args.num_inference_steps // args.backprop_num_steps + random_start = random.randint(0, interval) + backprop_step_list = [random_start + i * interval for i in range(args.backprop_num_steps)] + elif args.backprop_strategy == "random": + backprop_step_list = random.sample( + range(args.backprop_random_start_step, args.backprop_random_end_step + 1), args.backprop_num_steps + ) + else: + raise ValueError(f"Invalid backprop strategy: {args.backprop_strategy}.") + else: + backprop_step_list = args.backprop_step_list + + for i, t in enumerate(tqdm(timesteps)): + # for DPM-solver++ + old_pred_original_sample = None + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = noise_scheduler.scale_model_input(latent_model_input, t) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + # Whether to enable DRTune: https://arxiv.org/abs/2405.00760 + if args.stop_latent_model_input_gradient: + latent_model_input = latent_model_input.detach() + + # predict noise model_output + noise_pred = transformer3d( + hidden_states=latent_model_input, + encoder_hidden_states=prompt_embeds, + timestep=timestep, + image_rotary_emb=image_rotary_emb, + return_dict=False, + inpaint_latents=inpaint_latents + )[0] + noise_pred = noise_pred.float() + + # Optimize the denoising results only for the specified steps. + if i in backprop_step_list: + noise_pred = noise_pred + else: + noise_pred = noise_pred.detach() + + # perform guidance + guidance_scale = args.guidance_scale + # if args.use_dynamic_cfg: + # guidance_scale = 1 + guidance_scale * ( + # (1 - math.cos(math.pi * ((args.num_inference_steps - t.item()) / args.num_inference_steps) ** 5.0)) / 2 + # ) + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + if not isinstance(noise_scheduler, CogVideoXDPMScheduler): + latents = noise_scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + else: + latents, old_pred_original_sample = noise_scheduler.step( + noise_pred, + old_pred_original_sample, + t, + timesteps[i - 1] if i > 0 else None, + latents, + **extra_step_kwargs, + return_dict=False, + ) + latents = latents.to(prompt_embeds.dtype) + + # decode latents (tensor) + latents = latents.permute(0, 2, 1, 3, 4) # [B, C, T, H, W] + # Since the casual VAE decoding consumes a large amount of VRAM, and we need to keep the decoding + # operation within the computational graph. Thus, we only decode the first args.num_decoded_latents + # to calculate the reward. + sampled_frame_indices = list(range(args.num_decoded_latents)) + sampled_latents = latents[:, :, sampled_frame_indices, :, :] + sampled_latents = 1 / vae.config.scaling_factor * sampled_latents + sampled_frames = vae.decode(sampled_latents).sample + sampled_frames = (sampled_frames / 2 + 0.5).clamp(0, 1) # [-1, 1] -> [0, 1] + + if global_step % args.checkpointing_steps == 0: + saved_file = f"sample-{global_step}-{accelerator.process_index}.mp4" + save_videos_grid( + sampled_frames.to(torch.float32).detach().cpu(), + os.path.join(args.output_dir, "train_sample", saved_file), + fps=8 + ) + + if args.num_sampled_frames is not None: + num_frames = sampled_frames.size(2) - 1 + sampled_frames_indices = torch.linspace(0, num_frames, steps=args.num_sampled_frames).long() + sampled_frames = sampled_frames[:, :, sampled_frames_indices, :, :] + # compute loss and reward + loss, reward = loss_fn(sampled_frames, train_prompt) + + # Gather the losses and rewards across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + avg_reward = accelerator.gather(reward.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + train_reward += avg_reward.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + total_norm = accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + # If use_deepspeed, `total_norm` cannot be logged by accelerator. + if not args.use_deepspeed: + accelerator.log({"total_norm": total_norm}, step=global_step) + else: + if hasattr(optimizer, "optimizer") and hasattr(optimizer.optimizer, "_global_grad_norm"): + accelerator.log({"total_norm": optimizer.optimizer._global_grad_norm}, step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss, "train_reward": train_reward}, step=global_step) + train_loss = 0.0 + train_reward = 0.0 + + if global_step % args.checkpointing_steps == 0: + # DeepSpeed requires saving weights on every device; saving weights only on the main process would cause issues. + if args.use_deepspeed or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + # Validation (distributed) + if do_validation and (global_step % args.validation_steps) == 0: + if args.validation_prompts is None and args.validation_prompt_path.endswith(".txt"): + validation_prompts = [] + with open(args.validation_prompt_path, "r") as f: + for line in f: + validation_prompts.append(line.strip()) + # Do not select randomly to ensure that `args.validation_prompts` is the same for each process. + args.validation_prompts = validation_prompts[:args.validation_batch_size] + + validation_prompts_idx = [(i, p) for i, p in enumerate(args.validation_prompts)] + + if hasattr(vae, "enable_cache_in_vae"): + vae.enable_cache_in_vae() + accelerator.wait_for_everyone() + with accelerator.split_between_processes(validation_prompts_idx) as splitted_prompts_idx: + validation_loss, validation_reward = log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + loss_fn, + args, + accelerator, + weight_dtype, + global_step, + splitted_prompts_idx + ) + avg_validation_loss = accelerator.gather(validation_loss).mean() + avg_validation_reward = accelerator.gather(validation_reward).mean() + if accelerator.is_main_process: + accelerator.log({"validation_loss": avg_validation_loss, "validation_reward": avg_validation_reward}, step=global_step) + accelerator.wait_for_everyone() + + logs = {"step_loss": loss.detach().item(), "step_reward": reward.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/cogvideox_fun/train_reward_lora.sh b/VideoX-Fun/scripts/cogvideox_fun/train_reward_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..626e4ed2d9b2d6090be75189f353c029b449044b --- /dev/null +++ b/VideoX-Fun/scripts/cogvideox_fun/train_reward_lora.sh @@ -0,0 +1,63 @@ +export MODEL_NAME="models/Diffusion_Transformer/CogVideoX-Fun-V1.1-2b-InP" +export TRAIN_PROMPT_PATH="MovieGenVideoBench_train.txt" +# Performing validation simultaneously with training will increase time and GPU memory usage. +export VALIDATION_PROMPT_PATH="MovieGenVideoBench_val.txt" +# Use 49 for V1 and V1.1; Use 85 for V1.5. +export VIDEO_LENGTH=49 + +accelerate launch --num_processes=1 --mixed_precision="bf16" scripts/cogvideox_fun/train_reward_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --rank=32 \ + --network_alpha=16 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --max_train_steps=10000 \ + --checkpointing_steps=100 \ + --learning_rate=1e-05 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --vae_gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --max_grad_norm=0.3 \ + --prompt_path=$TRAIN_PROMPT_PATH \ + --train_sample_height=224 \ + --train_sample_width=224 \ + --video_length=$VIDEO_LENGTH \ + --num_decoded_latents=1 \ + --num_sampled_frames=1 \ + --reward_fn="HPSReward" \ + --reward_fn_kwargs='{"version": "v2.1"}' \ + --backprop + +# Training command for CogVideoX-Fun-V1.1-2b-InP-HPS2.1.safetensors (with 8 A100 GPUs) +# accelerate launch --num_processes=8 --mixed_precision="bf16" --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json scripts/cogvideox_fun/train_reward_lora.py \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --rank=128 \ +# --network_alpha=64 \ +# --train_batch_size=1 \ +# --gradient_accumulation_steps=1 \ +# --max_train_steps=10000 \ +# --checkpointing_steps=100 \ +# --learning_rate=1e-05 \ +# --seed=42 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --max_grad_norm=0.3 \ +# --prompt_path=$TRAIN_PROMPT_PATH \ +# --train_sample_height=256 \ +# --train_sample_width=256 \ +# --video_length=$VIDEO_LENGTH \ +# --validation_prompt_path=$VALIDATION_PROMPT_PATH \ +# --validation_steps=100 \ +# --validation_batch_size=8 \ +# --num_decoded_latents=1 \ +# --num_sampled_frames=1 \ +# --reward_fn="HPSReward" \ +# --reward_fn_kwargs='{"version": "v2.1"}' \ +# --backprop \ No newline at end of file diff --git a/VideoX-Fun/scripts/fantasytalking/README_TRAIN.md b/VideoX-Fun/scripts/fantasytalking/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..138c56969690cefb48b671b82fb9e5783982b509 --- /dev/null +++ b/VideoX-Fun/scripts/fantasytalking/README_TRAIN.md @@ -0,0 +1,233 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use fsdp in FantasyTalking, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in FantasyTalking, you need to add a audio_path. + +```json +[ + { + "file_path": "train/00000001.mp4", + "audio_path": "wav/00000001.wav", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the videos at the center, but instead, it trains the videos after grouping them into buckets based on resolution. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for videos. When `random_hw_adapt` is enabled, for training videos, the height and width will be set to `video_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=768`, the resolution of video inputs for training is `512x512x49`, `768x768x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training videos, the height and width will be set to `video_sample_size` as the maximum and `256` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=768`, the resolution of video inputs for training is `256x256x49`, `512x512x49`, `768x768x21`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +FantasyTalking without deepspeed: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/fantasytalking/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=1 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --transformer_path="models/FantasyTalking/fantasytalking_model.ckpt" \ + --trainable_modules "processor." "proj_model." +``` + +FantasyTalking with Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/fantasytalking/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=1 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --transformer_path="models/FantasyTalking/fantasytalking_model.ckpt" \ + --trainable_modules "processor." "proj_model." +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +FantasyTalking with DeepSpeed Zero-3: + +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/fantasytalking/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=1 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --transformer_path="models/FantasyTalking/fantasytalking_model.ckpt" \ + --trainable_modules "processor." "proj_model." +``` + +FantasyTalking with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=AudioAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/fantasytalking/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=1 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --transformer_path="models/FantasyTalking/fantasytalking_model.ckpt" \ + --trainable_modules "processor." "proj_model." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/fantasytalking/train.py b/VideoX-Fun/scripts/fantasytalking/train.py new file mode 100644 index 0000000000000000000000000000000000000000..6787b993f02353a4f95f0137902dde98560dfce2 --- /dev/null +++ b/VideoX-Fun/scripts/fantasytalking/train.py @@ -0,0 +1,1908 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import ImageVideoSampler, get_random_mask +from videox_fun.data.dataset_video import VideoSpeechDataset +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, FantasyTalkingAudioEncoder, + FantasyTalkingTransformer3DModel) +from videox_fun.pipeline import FantasyTalkingPipeline, WanFunPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = FantasyTalkingTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + pipeline = FantasyTalkingPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int(args.video_sample_n_frames // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, clip_image = get_image_to_video_latent(args.validation_image_paths[i], None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + audio_path = args.validation_audio_paths[i] + + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + audio_path = audio_path, + shift = 5, + fps = 16 + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_image_paths", + type=str, + default=None, + nargs="+", + help=("A set of images evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_audio_paths", + type=str, + default=None, + nargs="+", + help=("A set of audios evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae = vae.eval() + # Get Clip Image Encoder + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + audio_encoder = FantasyTalkingAudioEncoder( + os.path.join(args.pretrained_model_name_or_path, "audio_encoder") + ) + + # Get Transformer + transformer3d = FantasyTalkingTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + clip_image_encoder.requires_grad_(False) + audio_encoder.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + + if "audio_processor" in state_dict: + audio_processor_dict = state_dict["audio_processor"] if "audio_processor" in state_dict else state_dict + m, u = transformer3d.load_state_dict(audio_processor_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + proj_model_dict = state_dict["proj_model"] if "proj_model" in state_dict else state_dict + proj_model_dict = {"proj_model." + k : v for k, v in proj_model_dict.items()} + m, u = transformer3d.load_state_dict(proj_model_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + else: + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = FantasyTalkingTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=FantasyTalkingTransformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = FantasyTalkingTransformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = FantasyTalkingTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=FantasyTalkingTransformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = FantasyTalkingTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = VideoSpeechDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + enable_bucket=args.enable_bucket, enable_inpaint=True, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.video_sample_size > 256: + sample_sizes = list(range(256, args.video_sample_size + 1, 128)) + + if sample_sizes[-1] != args.video_sample_size: + sample_sizes.append(args.video_sample_size) + else: + sample_sizes = [args.video_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + new_examples["audio"] = [] + new_examples["sample_rate"] = [] + + # Used in Inpaint mode + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + f, h, w, c = np.shape(pixel_value) + + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + local_video_sample_size = np.random.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + audio_length = np.shape(example["audio"])[0] + batch_audio_length = int(audio_length / pixel_values.size()[0] * batch_video_length) + new_examples["audio"].append(example["audio"][:batch_audio_length]) + new_examples["sample_rate"].append(example["sample_rate"]) + + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["audio"] = torch.stack([example for example in new_examples["audio"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + audio_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=torch.float32) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + audio = batch["audio"] + sample_rate = batch["sample_rate"] + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + audio = audio * 4 + sample_rate = sample_rate * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + audio = audio * 2 + sample_rate = sample_rate * 2 + + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + clip_image_encoder.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask = resize_mask(1 - mask, latents) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + + if rng is None: + zero_init_clip_in = np.random.choice([True, False], p=[0.1, 0.9]) + else: + zero_init_clip_in = rng.choice([True, False], p=[0.1, 0.9]) + clip_context.append(_clip_context if not zero_init_clip_in else torch.zeros_like(_clip_context)) + + clip_context = torch.cat(clip_context) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + with torch.no_grad(): + # Extract audio emb + audio_wav2vec_fea = [] + for index in range(len(audio)): + _audio_wav2vec_fea = audio_encoder.extract_audio_feat_without_file_load(audio[index], sample_rate[index]) + audio_wav2vec_fea.append(_audio_wav2vec_fea) + audio_wav2vec_fea = torch.cat(audio_wav2vec_fea).to(weight_dtype) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Predict the noise residual + if rng is None: + audio_scale = np.random.choice([1, 0], p=[0.9, 0.1]) + else: + audio_scale = rng.choice([1, 0], p=[0.9, 0.1]) + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=inpaint_latents, + clip_fea=clip_context, + audio_wav2vec_fea=audio_wav2vec_fea, + audio_scale=audio_scale, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/fantasytalking/train.sh b/VideoX-Fun/scripts/fantasytalking/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..b7eac022244ac90941f101b1a2151d3d1b1ccb7a --- /dev/null +++ b/VideoX-Fun/scripts/fantasytalking/train.sh @@ -0,0 +1,40 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/fantasytalking/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=512 \ + --token_sample_size=512 \ + --video_sample_stride=1 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --transformer_path="models/FantasyTalking/fantasytalking_model.ckpt" \ + --trainable_modules "processor." "proj_model." \ No newline at end of file diff --git a/VideoX-Fun/scripts/flux/README_TRAIN.md b/VideoX-Fun/scripts/flux/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..3308bff72f0edb9f019c347de0c49d51ec9b5133 --- /dev/null +++ b/VideoX-Fun/scripts/flux/README_TRAIN.md @@ -0,0 +1,133 @@ +## Training Code + +We can choose whether to use deepspeed or fsdp in flux, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images at the center, but instead, it trains the entire images after grouping them into buckets based on resolution. +- `random_hw_adapt` is used to enable automatic height and width scaling for images. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024` +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Without deepspeed: + +Training flux without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/flux/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` + +With Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/flux/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap FluxSingleTransformerBlock,FluxTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/flux/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/flux/README_TRAIN_LORA.md b/VideoX-Fun/scripts/flux/README_TRAIN_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..75f98a412c2a5895a92917307027b4904dfd6f0c --- /dev/null +++ b/VideoX-Fun/scripts/flux/README_TRAIN_LORA.md @@ -0,0 +1,140 @@ +## Lora Training Code + +We can choose whether to use deepspeed or fsdp in flux, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images at the center, but instead, it trains the entire images after grouping them into buckets based on resolution. +- `random_hw_adapt` is used to enable automatic height and width scaling for images. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024` +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Without deepspeed: + +Training flux without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/flux/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling +``` + +With Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/flux/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap FluxSingleTransformerBlock,FluxTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/flux/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/flux/train.py b/VideoX-Fun/scripts/flux/train.py new file mode 100644 index 0000000000000000000000000000000000000000..1bb94d3a1751965c4ca38a3c660e6995d1b7e688 --- /dev/null +++ b/VideoX-Fun/scripts/flux/train.py @@ -0,0 +1,1701 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys +from typing import List, NamedTuple, Optional, Union + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoencoderKLWan, + CLIPImageProcessor, CLIPTextModel, + CLIPTokenizer, CLIPVisionModelWithProjection, + FluxTransformer2DModel, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, QwenImageTransformer2DModel, + T5EncoderModel, T5TokenizerFast) +from videox_fun.pipeline import FluxPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + return latents + +def _extract_masked_hidden(hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + +def _prepare_latent_image_ids(batch_size, height, width, device, dtype): + latent_image_ids = torch.zeros(height, width, 3) + latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height)[:, None] + latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width)[None, :] + + latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape + + latent_image_ids = latent_image_ids.reshape( + latent_image_id_height * latent_image_id_width, latent_image_id_channels + ) + + return latent_image_ids.to(device=device, dtype=dtype) + +def _get_t5_prompt_embeds( + prompt = None, + max_sequence_length = 512, + tokenizer_2 = None, + text_encoder_2 = None, + device = None, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = tokenizer_2( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_length=False, + return_overflowing_tokens=False, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_embeds = text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + + dtype = text_encoder_2.dtype + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + _, seq_len, _ = prompt_embeds.shape + + return prompt_embeds + +def _get_clip_prompt_embeds( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + tokenizer = None, + text_encoder = None, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_overflowing_tokens=False, + return_length=False, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=False) + + # Use pooled output of CLIPTextModel + prompt_embeds = prompt_embeds.pooler_output + prompt_embeds = prompt_embeds.to(dtype=text_encoder.dtype, device=device) + + return prompt_embeds + +def encode_prompt( + prompt: Union[str, List[str]], + prompt_2: Optional[Union[str, List[str]]] = None, + device: Optional[torch.device] = None, + dtype = None, + max_sequence_length: int = 512, + text_encoder = None, + tokenizer = None, + text_encoder_2 = None, + tokenizer_2 = None, +): + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + prompt = [prompt] if isinstance(prompt, str) else prompt + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # We only use the pooled prompt output from the CLIPTextModel + pooled_prompt_embeds = _get_clip_prompt_embeds( + prompt=prompt, + device=device, + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + prompt_embeds = _get_t5_prompt_embeds( + prompt=prompt_2, + max_sequence_length=max_sequence_length, + device=device, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + ) + + text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=dtype) + return prompt_embeds, pooled_prompt_embeds, text_ids + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, text_encoder_2, tokenizer, tokenizer_2, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = FluxTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + pipeline = FluxPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + text_encoder_2=accelerator.unwrap_model(text_encoder_2), + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=3.5, + help="the FLUX.1 dev variant is a guidance distilled model", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = CLIPTokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + tokenizer_2 = T5TokenizerFast.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer_2" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = CLIPTextModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder_2 = T5EncoderModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder_2", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + text_encoder_2 = text_encoder_2.eval() + # Get Vae + vae = AutoencoderKL.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + + # Get Transformer + transformer3d = FluxTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + text_encoder_2.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = FluxTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=FluxTransformer2DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = FluxTransformer2DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = FluxTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + ) + load_model = EMAModel(load_model.parameters(), model_cls=FluxTransformer2DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = FluxTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["text"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_embeds, pooled_prompt_embeds, text_ids = encode_prompt( + batch['text'], dtype=weight_dtype, device="cpu", + text_encoder=text_encoder, + tokenizer=tokenizer, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + ) + + new_examples['pooled_prompt_embeds'] = pooled_prompt_embeds + new_examples['prompt_embeds'] = prompt_embeds + new_examples['text_ids'] = text_ids + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.text_model.encoder.layers) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + text_encoder_2.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need】 + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + text_encoder_2.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = pixel_values.squeeze(1) + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = ((latents - vae.config.shift_factor) * vae.config.scaling_factor).to(dtype=weight_dtype) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + text_encoder_2.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['prompt_embeds'].to(dtype=latents.dtype, device=accelerator.device) + pooled_prompt_embeds = batch['pooled_prompt_embeds'].to(dtype=latents.dtype, device=accelerator.device) + text_ids = batch['text_ids'] + else: + with torch.no_grad(): + prompt_embeds, pooled_prompt_embeds, text_ids = encode_prompt( + batch['text'], dtype=latents.dtype, device=accelerator.device, + text_encoder=text_encoder, + tokenizer=tokenizer, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + ) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + text_encoder_2.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, height, width = latents.size() + latents = _pack_latents(latents, bsz, channel, height, width) + latent_image_ids = _prepare_latent_image_ids(bsz, height // 2, width // 2, latents.device, weight_dtype) + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + # handle guidance + if unwrap_model(transformer3d).config.guidance_embeds: + guidance = torch.tensor([args.guidance_scale], device=accelerator.device) + guidance = guidance.expand(latents.shape[0]) + else: + guidance = None + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + sigmas = np.linspace(1.0, 1 / args.train_sampling_steps, args.train_sampling_steps) + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.set_timesteps(sigmas=sigmas, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + hidden_states=noisy_latents, + timestep=timesteps / 1000, + guidance=guidance, + encoder_hidden_states=prompt_embeds, + pooled_projections=pooled_prompt_embeds, + txt_ids=text_ids, + img_ids=latent_image_ids, + return_dict=False, + )[0] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + text_encoder_2, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + text_encoder_2, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/flux/train.sh b/VideoX-Fun/scripts/flux/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..5488c309441bfd3030247dad0c9297123713a8c0 --- /dev/null +++ b/VideoX-Fun/scripts/flux/train.sh @@ -0,0 +1,32 @@ +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap FluxSingleTransformerBlock,FluxTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/flux/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/flux/train_lora.py b/VideoX-Fun/scripts/flux/train_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..ad71f90490f57c78a88c97c2c8198c324724b33e --- /dev/null +++ b/VideoX-Fun/scripts/flux/train_lora.py @@ -0,0 +1,1737 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import copy +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys +from typing import List, NamedTuple, Optional, Union + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoencoderKLWan, + CLIPImageProcessor, CLIPTextModel, + CLIPTokenizer, CLIPVisionModelWithProjection, + FluxTransformer2DModel, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, QwenImageTransformer2DModel, + T5EncoderModel, T5TokenizerFast) +from videox_fun.pipeline import FluxPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +def _prepare_latent_image_ids(batch_size, height, width, device, dtype): + latent_image_ids = torch.zeros(height, width, 3) + latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height)[:, None] + latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width)[None, :] + + latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape + + latent_image_ids = latent_image_ids.reshape( + latent_image_id_height * latent_image_id_width, latent_image_id_channels + ) + + return latent_image_ids.to(device=device, dtype=dtype) + +def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + return latents + +def _extract_masked_hidden(hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + +def _get_t5_prompt_embeds( + prompt = None, + max_sequence_length = 512, + tokenizer_2 = None, + text_encoder_2 = None, + device = None, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = tokenizer_2( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_length=False, + return_overflowing_tokens=False, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_embeds = text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + + dtype = text_encoder_2.dtype + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + _, seq_len, _ = prompt_embeds.shape + + return prompt_embeds + +def _get_clip_prompt_embeds( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + tokenizer = None, + text_encoder = None, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_overflowing_tokens=False, + return_length=False, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=False) + + # Use pooled output of CLIPTextModel + prompt_embeds = prompt_embeds.pooler_output + prompt_embeds = prompt_embeds.to(dtype=text_encoder.dtype, device=device) + + return prompt_embeds + +def encode_prompt( + prompt: Union[str, List[str]], + prompt_2: Optional[Union[str, List[str]]] = None, + device: Optional[torch.device] = None, + dtype = None, + max_sequence_length: int = 512, + text_encoder = None, + tokenizer = None, + text_encoder_2 = None, + tokenizer_2 = None, +): + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + prompt = [prompt] if isinstance(prompt, str) else prompt + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # We only use the pooled prompt output from the CLIPTextModel + pooled_prompt_embeds = _get_clip_prompt_embeds( + prompt=prompt, + device=device, + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + prompt_embeds = _get_t5_prompt_embeds( + prompt=prompt_2, + max_sequence_length=max_sequence_length, + device=device, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + ) + + text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=dtype) + return prompt_embeds, pooled_prompt_embeds, text_ids + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, text_encoder_2, tokenizer, tokenizer_2, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = FluxTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + pipeline = FluxPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + text_encoder_2=accelerator.unwrap_model(text_encoder_2), + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=3.5, + help="the FLUX.1 dev variant is a guidance distilled model", + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = CLIPTokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + tokenizer_2 = T5TokenizerFast.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer_2" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = CLIPTextModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder_2 = T5EncoderModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder_2", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + text_encoder_2 = text_encoder_2.eval() + # Get Vae + vae = AutoencoderKL.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + + # Get Transformer + transformer3d = FluxTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + text_encoder_2.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import (LoraConfig, get_peft_model_state_dict, + inject_adapter_in_model) + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["text"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_embeds, pooled_prompt_embeds, text_ids = encode_prompt( + batch['text'], dtype=weight_dtype, device="cpu", + text_encoder=text_encoder, + tokenizer=tokenizer, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + ) + + new_examples['pooled_prompt_embeds'] = pooled_prompt_embeds + new_examples['prompt_embeds'] = prompt_embeds + new_examples['text_ids'] = text_ids + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=list(transformer3d.transformer_blocks) + list(transformer3d.single_transformer_blocks)) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.text_model.encoder.layers) + text_encoder = shard_fn(text_encoder) + # shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder_2.encoder.block) + # text_encoder_2 = shard_fn(text_encoder_2) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + text_encoder_2.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need】 + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + text_encoder_2.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = pixel_values.squeeze(1) + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = ((latents - vae.config.shift_factor) * vae.config.scaling_factor).to(dtype=weight_dtype) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + text_encoder_2.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['prompt_embeds'].to(dtype=latents.dtype, device=accelerator.device) + pooled_prompt_embeds = batch['pooled_prompt_embeds'].to(dtype=latents.dtype, device=accelerator.device) + text_ids = batch['text_ids'] + else: + with torch.no_grad(): + prompt_embeds, pooled_prompt_embeds, text_ids = encode_prompt( + batch['text'], dtype=latents.dtype, device=accelerator.device, + text_encoder=text_encoder, + tokenizer=tokenizer, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + ) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + text_encoder_2.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, height, width = latents.size() + latents = _pack_latents(latents, bsz, channel, height, width) + latent_image_ids = _prepare_latent_image_ids(bsz, height // 2, width // 2, latents.device, weight_dtype) + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + # handle guidance + if unwrap_model(transformer3d).config.guidance_embeds: + guidance = torch.tensor([args.guidance_scale], device=accelerator.device) + guidance = guidance.expand(latents.shape[0]) + else: + guidance = None + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + sigmas = np.linspace(1.0, 1 / args.train_sampling_steps, args.train_sampling_steps) + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.set_timesteps(sigmas=sigmas, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + hidden_states=noisy_latents, + timestep=timesteps / 1000, + guidance=guidance, + encoder_hidden_states=prompt_embeds, + pooled_projections=pooled_prompt_embeds, + txt_ids=text_ids, + img_ids=latent_image_ids, + return_dict=False, + )[0] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + text_encoder_2, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + text_encoder_2, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/flux/train_lora.sh b/VideoX-Fun/scripts/flux/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..5080571ca2d7ddc4344a98a2598aa881144b18bc --- /dev/null +++ b/VideoX-Fun/scripts/flux/train_lora.sh @@ -0,0 +1,33 @@ +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/flux/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling \ No newline at end of file diff --git a/VideoX-Fun/scripts/flux2/README_TRAIN.md b/VideoX-Fun/scripts/flux2/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..c68371283c13eedd7a55fccad37ecbd1e627fe0a --- /dev/null +++ b/VideoX-Fun/scripts/flux2/README_TRAIN.md @@ -0,0 +1,133 @@ +## Training Code + +We can choose whether to use deepspeed or fsdp in flux2, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images at the center, but instead, it trains the entire images after grouping them into buckets based on resolution. +- `random_hw_adapt` is used to enable automatic height and width scaling for images. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024` +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Without deepspeed: + +Training flux2 without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/flux2/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` + +With Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/flux2/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.1-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap Flux2SingleTransformerBlock,Flux2TransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/flux2/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/flux2/README_TRAIN_LORA.md b/VideoX-Fun/scripts/flux2/README_TRAIN_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..a577ed825a2e243b011dec16e8676de94cd4a500 --- /dev/null +++ b/VideoX-Fun/scripts/flux2/README_TRAIN_LORA.md @@ -0,0 +1,140 @@ +## Lora Training Code + +We can choose whether to use deepspeed or fsdp in flux2, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images at the center, but instead, it trains the entire images after grouping them into buckets based on resolution. +- `random_hw_adapt` is used to enable automatic height and width scaling for images. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024` +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Without deepspeed: + +Training flux2 without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.2-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/flux2/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling +``` + +With Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.2-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/flux2/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/FLUX.2-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap Flux2SingleTransformerBlock,Flux2TransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/flux2/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/flux2/train.py b/VideoX-Fun/scripts/flux2/train.py new file mode 100644 index 0000000000000000000000000000000000000000..23455e488742f7d68fb513bd27c8d3bcbbeb33c5 --- /dev/null +++ b/VideoX-Fun/scripts/flux2/train.py @@ -0,0 +1,1778 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys +from typing import List, NamedTuple, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLFlux2, + CLIPImageProcessor, Mistral3ForConditionalGeneration, + PixtralProcessor, CLIPVisionModelWithProjection, + Flux2Transformer2DModel, + Mistral3ForConditionalGeneration, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, QwenImageTransformer2DModel) +from videox_fun.pipeline import Flux2Pipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float: + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return float(mu) + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +def _prepare_latent_ids( + latents: torch.Tensor, # (B, C, H, W) +): + r""" + Generates 4D position coordinates (T, H, W, L) for latent tensors. + + Args: + latents (torch.Tensor): + Latent tensor of shape (B, C, H, W) + + Returns: + torch.Tensor: + Position IDs tensor of shape (B, H*W, 4) All batches share the same coordinate structure: T=0, + H=[0..H-1], W=[0..W-1], L=0 + """ + + batch_size, _, height, width = latents.shape + + t = torch.arange(1) # [0] - time dimension + h = torch.arange(height) + w = torch.arange(width) + l = torch.arange(1) # [0] - layer dimension + + # Create position IDs: (H*W, 4) + latent_ids = torch.cartesian_prod(t, h, w, l) + + # Expand to batch: (B, H*W, 4) + latent_ids = latent_ids.unsqueeze(0).expand(batch_size, -1, -1) + + return latent_ids + +def _patchify_latents(latents): + batch_size, num_channels_latents, height, width = latents.shape + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 1, 3, 5, 2, 4) + latents = latents.reshape(batch_size, num_channels_latents * 4, height // 2, width // 2) + return latents + +def _pack_latents(latents): + """ + pack latents: (batch_size, num_channels, height, width) -> (batch_size, height * width, num_channels) + """ + + batch_size, num_channels, height, width = latents.shape + latents = latents.reshape(batch_size, num_channels, height * width).permute(0, 2, 1) + + return latents + +def format_text_input(prompts: List[str], system_message: str = None): + # Remove [IMG] tokens from prompts to avoid Pixtral validation issues + # when truncation is enabled. The processor counts [IMG] tokens and fails + # if the count changes after truncation. + cleaned_txt = [prompt.replace("[IMG]", "") for prompt in prompts] + + return [ + [ + { + "role": "system", + "content": [{"type": "text", "text": system_message}], + }, + {"role": "user", "content": [{"type": "text", "text": prompt}]}, + ] + for prompt in cleaned_txt + ] + +def _get_mistral_3_small_prompt_embeds( + text_encoder: Mistral3ForConditionalGeneration, + tokenizer: PixtralProcessor, + prompt: Union[str, List[str]], + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + max_sequence_length: int = 512, + # fmt: off + system_message: str = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation.", + # fmt: on + hidden_states_layers: List[int] = (10, 20, 30), +): + dtype = text_encoder.dtype if dtype is None else dtype + device = text_encoder.device if device is None else device + + prompt = [prompt] if isinstance(prompt, str) else prompt + + # Format input messages + messages_batch = format_text_input(prompts=prompt, system_message=system_message) + + # Process all messages at once + inputs = tokenizer.apply_chat_template( + messages_batch, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=max_sequence_length, + ) + + # Move to device + input_ids = inputs["input_ids"].to(device) + attention_mask = inputs["attention_mask"].to(device) + + # Forward pass through the model + output = text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + use_cache=False, + ) + + # Only use outputs from intermediate layers and stack them + out = torch.stack([output.hidden_states[k] for k in hidden_states_layers], dim=1) + out = out.to(dtype=dtype, device=device) + + batch_size, num_channels, seq_len, hidden_dim = out.shape + prompt_embeds = out.permute(0, 2, 1, 3).reshape(batch_size, seq_len, num_channels * hidden_dim) + + return prompt_embeds + +def _prepare_text_ids( + x: torch.Tensor, # (B, L, D) or (L, D) + t_coord: Optional[torch.Tensor] = None, +): + B, L, _ = x.shape + out_ids = [] + + for i in range(B): + t = torch.arange(1) if t_coord is None else t_coord[i] + h = torch.arange(1) + w = torch.arange(1) + l = torch.arange(L) + + coords = torch.cartesian_prod(t, h, w, l) + out_ids.append(coords) + + return torch.stack(out_ids) + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + text_encoder=None, + tokenizer=None, + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + text_encoder_out_layers: Tuple[int] = (10, 20, 30), + system_message = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation." +): + if prompt is None: + prompt = "" + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt_embeds is None: + prompt_embeds = _get_mistral_3_small_prompt_embeds( + text_encoder=text_encoder, + tokenizer=tokenizer, + prompt=prompt, + device=device, + max_sequence_length=max_sequence_length, + system_message=system_message, + hidden_states_layers=text_encoder_out_layers, + ) + + batch_size, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + text_ids = _prepare_text_ids(prompt_embeds) + text_ids = text_ids.to(device) + return prompt_embeds, text_ids + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = Flux2Transformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + pipeline = Flux2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--prompt_template_encode", + type=str, + default="<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", + help=( + 'The prompt template for text encoder.' + ), + ) + parser.add_argument( + "--prompt_template_encode_start_idx", + type=int, + default=34, + help=( + 'The start idx for prompt template.' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=3.5, + help="the FLUX.1 dev variant is a guidance distilled model", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = PixtralProcessor.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So Mistral3ForConditionalGeneration and AutoencoderKLFlux2 will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = Mistral3ForConditionalGeneration.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLFlux2.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + latents_bn_mean = vae.bn.running_mean.view(1, -1, 1, 1).to(accelerator.device, weight_dtype) + latents_bn_std = torch.sqrt(vae.bn.running_var.view(1, -1, 1, 1) + vae.config.batch_norm_eps).to(accelerator.device, weight_dtype) + + # Get Transformer + transformer3d = Flux2Transformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = Flux2Transformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=Flux2Transformer2DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = Flux2Transformer2DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = Flux2Transformer2DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + ) + load_model = EMAModel(load_model.parameters(), model_cls=Flux2Transformer2DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = Flux2Transformer2DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["text"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_embeds, text_ids = encode_prompt( + batch['text'], device="cpu", + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + new_examples['prompt_embeds'] = prompt_embeds + new_examples['text_ids'] = text_ids + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need】 + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = pixel_values.squeeze(1) + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['prompt_embeds'].to(dtype=latents.dtype, device=accelerator.device) + text_ids = batch['text_ids'] + else: + with torch.no_grad(): + prompt_embeds, text_ids = encode_prompt( + batch['text'], device=accelerator.device, + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, height, width = latents.size() + latents = _patchify_latents(latents) + latent_image_ids = _prepare_latent_ids(latents) + latents = ((latents - latents_bn_mean) / latents_bn_std).to(dtype=weight_dtype) + latents = _pack_latents(latents) + + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + # handle guidance + guidance = torch.tensor([args.guidance_scale], device=accelerator.device) + guidance = guidance.expand(latents.shape[0]) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + sigmas = np.linspace(1.0, 1 / args.train_sampling_steps, args.train_sampling_steps) + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.set_timesteps(sigmas=sigmas, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + hidden_states=noisy_latents, + timestep=timesteps / 1000, + guidance=guidance, + encoder_hidden_states=prompt_embeds, + txt_ids=text_ids, + img_ids=latent_image_ids, + return_dict=False, + )[0] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/flux2/train.sh b/VideoX-Fun/scripts/flux2/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..84271f6e5a72824dd22a949ffd3d39e9800a8322 --- /dev/null +++ b/VideoX-Fun/scripts/flux2/train.sh @@ -0,0 +1,32 @@ +export MODEL_NAME="models/Diffusion_Transformer/FLUX.2-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/flux2/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/flux2/train_lora.py b/VideoX-Fun/scripts/flux2/train_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..45a9028b442f67a4f35c3144c4aa40f796d297c0 --- /dev/null +++ b/VideoX-Fun/scripts/flux2/train_lora.py @@ -0,0 +1,1724 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys +from typing import List, NamedTuple, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKLFlux2, + CLIPImageProcessor, Mistral3ForConditionalGeneration, + PixtralProcessor, CLIPVisionModelWithProjection, + Flux2Transformer2DModel, + Mistral3ForConditionalGeneration, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, QwenImageTransformer2DModel) +from videox_fun.pipeline import Flux2Pipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float: + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return float(mu) + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +def _prepare_latent_ids( + latents: torch.Tensor, # (B, C, H, W) +): + r""" + Generates 4D position coordinates (T, H, W, L) for latent tensors. + + Args: + latents (torch.Tensor): + Latent tensor of shape (B, C, H, W) + + Returns: + torch.Tensor: + Position IDs tensor of shape (B, H*W, 4) All batches share the same coordinate structure: T=0, + H=[0..H-1], W=[0..W-1], L=0 + """ + + batch_size, _, height, width = latents.shape + + t = torch.arange(1) # [0] - time dimension + h = torch.arange(height) + w = torch.arange(width) + l = torch.arange(1) # [0] - layer dimension + + # Create position IDs: (H*W, 4) + latent_ids = torch.cartesian_prod(t, h, w, l) + + # Expand to batch: (B, H*W, 4) + latent_ids = latent_ids.unsqueeze(0).expand(batch_size, -1, -1) + + return latent_ids + +def _patchify_latents(latents): + batch_size, num_channels_latents, height, width = latents.shape + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 1, 3, 5, 2, 4) + latents = latents.reshape(batch_size, num_channels_latents * 4, height // 2, width // 2) + return latents + +def _pack_latents(latents): + """ + pack latents: (batch_size, num_channels, height, width) -> (batch_size, height * width, num_channels) + """ + + batch_size, num_channels, height, width = latents.shape + latents = latents.reshape(batch_size, num_channels, height * width).permute(0, 2, 1) + + return latents + +def format_text_input(prompts: List[str], system_message: str = None): + # Remove [IMG] tokens from prompts to avoid Pixtral validation issues + # when truncation is enabled. The processor counts [IMG] tokens and fails + # if the count changes after truncation. + cleaned_txt = [prompt.replace("[IMG]", "") for prompt in prompts] + + return [ + [ + { + "role": "system", + "content": [{"type": "text", "text": system_message}], + }, + {"role": "user", "content": [{"type": "text", "text": prompt}]}, + ] + for prompt in cleaned_txt + ] + +def _get_mistral_3_small_prompt_embeds( + text_encoder: Mistral3ForConditionalGeneration, + tokenizer: PixtralProcessor, + prompt: Union[str, List[str]], + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + max_sequence_length: int = 512, + # fmt: off + system_message: str = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation.", + # fmt: on + hidden_states_layers: List[int] = (10, 20, 30), +): + dtype = text_encoder.dtype if dtype is None else dtype + device = text_encoder.device if device is None else device + + prompt = [prompt] if isinstance(prompt, str) else prompt + + # Format input messages + messages_batch = format_text_input(prompts=prompt, system_message=system_message) + + # Process all messages at once + inputs = tokenizer.apply_chat_template( + messages_batch, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=max_sequence_length, + ) + + # Move to device + input_ids = inputs["input_ids"].to(device) + attention_mask = inputs["attention_mask"].to(device) + + # Forward pass through the model + output = text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + use_cache=False, + ) + + # Only use outputs from intermediate layers and stack them + out = torch.stack([output.hidden_states[k] for k in hidden_states_layers], dim=1) + out = out.to(dtype=dtype, device=device) + + batch_size, num_channels, seq_len, hidden_dim = out.shape + prompt_embeds = out.permute(0, 2, 1, 3).reshape(batch_size, seq_len, num_channels * hidden_dim) + + return prompt_embeds + +def _prepare_text_ids( + x: torch.Tensor, # (B, L, D) or (L, D) + t_coord: Optional[torch.Tensor] = None, +): + B, L, _ = x.shape + out_ids = [] + + for i in range(B): + t = torch.arange(1) if t_coord is None else t_coord[i] + h = torch.arange(1) + w = torch.arange(1) + l = torch.arange(L) + + coords = torch.cartesian_prod(t, h, w, l) + out_ids.append(coords) + + return torch.stack(out_ids) + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + text_encoder=None, + tokenizer=None, + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + text_encoder_out_layers: Tuple[int] = (10, 20, 30), + system_message = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation." +): + if prompt is None: + prompt = "" + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt_embeds is None: + prompt_embeds = _get_mistral_3_small_prompt_embeds( + text_encoder=text_encoder, + tokenizer=tokenizer, + prompt=prompt, + device=device, + max_sequence_length=max_sequence_length, + system_message=system_message, + hidden_states_layers=text_encoder_out_layers, + ) + + batch_size, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + text_ids = _prepare_text_ids(prompt_embeds) + text_ids = text_ids.to(device) + return prompt_embeds, text_ids + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = Flux2Transformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + pipeline = Flux2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=3.5, + help="the FLUX.1 dev variant is a guidance distilled model", + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = PixtralProcessor.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So Mistral3ForConditionalGeneration and AutoencoderKLFlux2 will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = Mistral3ForConditionalGeneration.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLFlux2.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + latents_bn_mean = vae.bn.running_mean.view(1, -1, 1, 1).to(accelerator.device, weight_dtype) + latents_bn_std = torch.sqrt(vae.bn.running_var.view(1, -1, 1, 1) + vae.config.batch_norm_eps).to(accelerator.device, weight_dtype) + + # Get Transformer + transformer3d = Flux2Transformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import (LoraConfig, get_peft_model_state_dict, + inject_adapter_in_model) + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["text"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_embeds, text_ids = encode_prompt( + batch['text'], device="cpu", + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + new_examples['prompt_embeds'] = prompt_embeds + new_examples['text_ids'] = text_ids + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=list(transformer3d.transformer_blocks) + list(transformer3d.single_transformer_blocks)) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need】 + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = pixel_values.squeeze(1) + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['prompt_embeds'].to(dtype=latents.dtype, device=accelerator.device) + text_ids = batch['text_ids'] + else: + with torch.no_grad(): + prompt_embeds, text_ids = encode_prompt( + batch['text'], device=accelerator.device, + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, height, width = latents.size() + latents = _patchify_latents(latents) + latent_image_ids = _prepare_latent_ids(latents) + latents = ((latents - latents_bn_mean) / latents_bn_std).to(dtype=weight_dtype) + latents = _pack_latents(latents) + + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + # handle guidance + guidance = torch.tensor([args.guidance_scale], device=accelerator.device) + guidance = guidance.expand(latents.shape[0]) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + sigmas = np.linspace(1.0, 1 / args.train_sampling_steps, args.train_sampling_steps) + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.set_timesteps(sigmas=sigmas, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + hidden_states=noisy_latents, + timestep=timesteps / 1000, + guidance=guidance, + encoder_hidden_states=prompt_embeds, + txt_ids=text_ids, + img_ids=latent_image_ids, + return_dict=False, + )[0] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/flux2/train_lora.sh b/VideoX-Fun/scripts/flux2/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..5b1078d1612dc9977613c98bcfa3e38875584920 --- /dev/null +++ b/VideoX-Fun/scripts/flux2/train_lora.sh @@ -0,0 +1,33 @@ +export MODEL_NAME="models/Diffusion_Transformer/FLUX.2-dev" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/flux2/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling \ No newline at end of file diff --git a/VideoX-Fun/scripts/generate_subgraph_configs.py b/VideoX-Fun/scripts/generate_subgraph_configs.py new file mode 100644 index 0000000000000000000000000000000000000000..576bb4a03a27d6f167c4d506faec2e734a93bde0 --- /dev/null +++ b/VideoX-Fun/scripts/generate_subgraph_configs.py @@ -0,0 +1,184 @@ +#!/usr/bin/env python3 +"""为每个子图ONNX生成量化配置JSON文件""" +import argparse +import json +import os +from pathlib import Path + +# 路径配置默认值 +REPO_ROOT = Path(__file__).parent.parent +DEFAULT_TAR_LIST_FILE = REPO_ROOT / "onnx-calibration-subgraphs" / "subgraph_calibration_paths.txt" +DEFAULT_OUTPUT_CONFIG_DIR = REPO_ROOT / "pulsar2_configs" / "subgraphs" +DEFAULT_TEMPLATE_CONFIG = REPO_ROOT / "pulsar2_configs" / "transformers.json" + +# JSON模板配置 +CONFIG_TEMPLATE = { + "model_type": "ONNX", + "npu_mode": "NPU3", + "quant": { + "input_configs": [ + { + "tensor_name": "DEFAULT", + "calibration_dataset": "", # 将被替换 + "calibration_size": -1, + "calibration_format": "NumpyObject" + } + ], + "calibration_method": "MinMax", + "precision_analysis": True, + "precision_analysis_method": "PerLayer", + "enable_smooth_quant": True, + "conv_bias_data_type": "FP32", + "layer_configs": [ + { + "start_tensor_names": ["DEFAULT"], + "end_tensor_names": ["DEFAULT"], + "data_type": "U16" + } + ] + }, + "input_processors": [ + { + "tensor_name": "DEFAULT" + } + ], + "compiler": { + "check": 0 + } +} + + +def load_template_config(template_path: Path) -> dict: + """加载模板配置文件""" + if template_path.exists(): + with open(template_path, 'r', encoding='utf-8') as f: + return json.load(f) + return CONFIG_TEMPLATE + + +def extract_subgraph_name(tar_path: str) -> str: + """从tar文件路径提取子图名称 + 例如: /path/to/cfg_00.tar -> cfg_00 + """ + return Path(tar_path).stem + + +def create_config_for_subgraph(tar_path: str, template: dict, output_dir: Path) -> Path: + """为单个子图创建配置文件""" + subgraph_name = extract_subgraph_name(tar_path) + + # 深拷贝模板 + config = json.loads(json.dumps(template)) + + # 修改 calibration_dataset 字段 + config["quant"]["input_configs"][0]["calibration_dataset"] = tar_path + + # 生成输出文件路径 + config_file = output_dir / f"{subgraph_name}.json" + + # 保存配置文件 + with open(config_file, 'w', encoding='utf-8') as f: + json.dump(config, f, indent=2, ensure_ascii=False) + + return config_file + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="为子图ONNX生成量化配置JSON文件") + parser.add_argument( + "--tar-list-file", + type=Path, + default=DEFAULT_TAR_LIST_FILE, + help="包含tar路径的列表文件，一行一个；如果同时提供 --tar，将忽略此文件", + ) + parser.add_argument( + "--tar", + dest="tar_paths", + action="append", + help="直接指定tar文件路径，可重复", + ) + parser.add_argument( + "--output-config-dir", + type=Path, + default=DEFAULT_OUTPUT_CONFIG_DIR, + help="生成的配置文件输出目录", + ) + parser.add_argument( + "--template-config", + type=Path, + default=DEFAULT_TEMPLATE_CONFIG, + help="配置模板文件路径，默认使用 pulsar2_configs/transformers.json", + ) + return parser.parse_args() + + +def main(): + args = parse_args() + + print("=" * 80) + print("为子图ONNX生成量化配置文件") + print("=" * 80) + + # 读取tar文件路径 + if args.tar_paths: + tar_paths = args.tar_paths + print(f"\n使用命令行提供的 {len(tar_paths)} 个tar文件路径") + else: + if not args.tar_list_file.exists(): + print(f"错误: 找不到tar列表文件: {args.tar_list_file}") + print("请先运行 collect_subgraph_inputs.py 生成校准数据或使用 --tar 指定tar路径") + return + + print(f"\n读取tar文件列表: {args.tar_list_file}") + with open(args.tar_list_file, 'r') as f: + tar_paths = [line.strip() for line in f if line.strip()] + print(f"找到 {len(tar_paths)} 个tar文件") + + # 加载模板配置 + print(f"\n加载配置模板: {args.template_config}") + template = load_template_config(args.template_config) + + # 创建输出目录 + args.output_config_dir.mkdir(parents=True, exist_ok=True) + print(f"输出目录: {args.output_config_dir}") + + # 为每个tar文件生成配置 + print(f"\n生成配置文件...") + created_configs = [] + + for tar_path in tar_paths: + if not os.path.exists(tar_path): + print(f" 警告: tar文件不存在: {tar_path}") + continue + + try: + config_file = create_config_for_subgraph(tar_path, template, args.output_config_dir) + created_configs.append(config_file) + print(f" ✓ {config_file.name}") + except Exception as e: + print(f" ✗ 生成配置失败 ({extract_subgraph_name(tar_path)}): {e}") + + # 生成一个索引文件，列出所有配置文件路径 + index_file = args.output_config_dir / "subgraph_configs_list.txt" + with open(index_file, 'w') as f: + for config_file in sorted(created_configs): + f.write(str(config_file.absolute()) + '\n') + + print(f"\n配置文件索引已保存: {index_file}") + + print("\n" + "=" * 80) + print(f"完成! 共生成 {len(created_configs)} 个配置文件") + print(f"配置文件目录: {args.output_config_dir}") + print(f"配置文件列表: {index_file}") + print("=" * 80) + + # 显示示例配置 + if created_configs: + print(f"\n示例配置 ({created_configs[0].name}):") + print("-" * 80) + with open(created_configs[0], 'r') as f: + print(f.read()) + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/hunyuanvideo/README_TRAIN.md b/VideoX-Fun/scripts/hunyuanvideo/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..efd9e5f3b07f7113dfd96f1db2dbeec4aae3a362 --- /dev/null +++ b/VideoX-Fun/scripts/hunyuanvideo/README_TRAIN.md @@ -0,0 +1,227 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use DeepSpeed and FSDP in HunyuanVideo, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since HunyuanVideo uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +HunyuanVideo T2V without deepspeed: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/hunyuanvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +Hun yuan T2V with Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/hunyuanvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +HunyuanVideo T2V with FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=HunyuanVideoTransformerBlock,HunyuanVideoSingleTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/hunyuanvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +HunyuanVideo I2V with FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo-I2V" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=HunyuanVideoTokenReplaceTransformerBlock,HunyuanVideoTokenReplaceSingleTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/hunyuanvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir2" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="i2v" \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/hunyuanvideo/README_TRAIN_LORA.md b/VideoX-Fun/scripts/hunyuanvideo/README_TRAIN_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..c969fdf717185855b2f5990b433687edd86260cc --- /dev/null +++ b/VideoX-Fun/scripts/hunyuanvideo/README_TRAIN_LORA.md @@ -0,0 +1,235 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use DeepSpeed and FSDP in HunyuanVideo, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since HunyuanVideo uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +HunyuanVideo T2V without deepspeed: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/internaldatasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/hunyuanvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +Hun yuan T2V with Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/hunyuanvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +HunyuanVideo T2V with FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=HunyuanVideoTransformerBlock,HunyuanVideoSingleTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/hunyuanvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" +``` + +HunyuanVideo I2V with FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo-I2V" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=HunyuanVideoTokenReplaceTransformerBlock,HunyuanVideoTokenReplaceSingleTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/hunyuanvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="i2v" +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/hunyuanvideo/train.py b/VideoX-Fun/scripts/hunyuanvideo/train.py new file mode 100644 index 0000000000000000000000000000000000000000..a9626ca5a42962510aab12f2d80a3f12cbea69f2 --- /dev/null +++ b/VideoX-Fun/scripts/hunyuanvideo/train.py @@ -0,0 +1,2126 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLHunyuanVideo, CLIPImageProcessor, + CLIPTextModel, CLIPTokenizer, + HunyuanVideoTransformer3DModel, LlamaModel, + LlamaTokenizerFast, + LlavaForConditionalGeneration) +from videox_fun.pipeline import HunyuanVideoPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, text_encoder_2, tokenizer, tokenizer_2, transformer3d, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = HunyuanVideoTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'transformer'), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + if args.train_mode != "normal": + raise NotImplementedError("train_mode is not implemented") + else: + pipeline = HunyuanVideoPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + text_encoder_2=accelerator.unwrap_model(text_encoder_2), + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + raise NotImplementedError("train_mode is not implemented") + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +DEFAULT_PROMPT_TEMPLATE = { + "template": ( + "<|start_header_id|>system<|end_header_id|>\n\nDescribe the video by detailing the following aspects: " + "1. The main content and theme of the video." + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects." + "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects." + "4. background environment, light, style and atmosphere." + "5. camera angles, movements, and transitions used in the video:<|eot_id|>" + "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + ), + "crop_start": 95, +} + +DEFAULT_PROMPT_TEMPLATE_I2V = { + "template": ( + "<|start_header_id|>system<|end_header_id|>\n\n\nDescribe the video by detailing the following aspects according to the reference image: " + "1. The main content and theme of the video." + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects." + "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects." + "4. background environment, light, style and atmosphere." + "5. camera angles, movements, and transitions used in the video:<|eot_id|>\n\n" + "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + "<|start_header_id|>assistant<|end_header_id|>\n\n" + ), + "crop_start": 103, + "image_emb_start": 5, + "image_emb_end": 581, + "image_emb_len": 576, + "double_return_token_id": 271, +} + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def _expand_input_ids_with_image_tokens( + text_input_ids, + prompt_attention_mask, + max_sequence_length, + image_token_index, + image_emb_len, + image_emb_start, + image_emb_end, + pad_token_id, +): + special_image_token_mask = text_input_ids == image_token_index + num_special_image_tokens = torch.sum(special_image_token_mask, dim=-1) + batch_indices, non_image_indices = torch.where(text_input_ids != image_token_index) + + max_expanded_length = max_sequence_length + (num_special_image_tokens.max() * (image_emb_len - 1)) + new_token_positions = torch.cumsum((special_image_token_mask * (image_emb_len - 1) + 1), -1) - 1 + text_to_overwrite = new_token_positions[batch_indices, non_image_indices] + + expanded_input_ids = torch.full( + (text_input_ids.shape[0], max_expanded_length), + pad_token_id, + dtype=text_input_ids.dtype, + device=text_input_ids.device, + ) + expanded_input_ids[batch_indices, text_to_overwrite] = text_input_ids[batch_indices, non_image_indices] + expanded_input_ids[batch_indices, image_emb_start:image_emb_end] = image_token_index + + expanded_attention_mask = torch.zeros( + (text_input_ids.shape[0], max_expanded_length), + dtype=prompt_attention_mask.dtype, + device=prompt_attention_mask.device, + ) + attn_batch_indices, attention_indices = torch.where(expanded_input_ids != pad_token_id) + expanded_attention_mask[attn_batch_indices, attention_indices] = 1.0 + expanded_attention_mask = expanded_attention_mask.to(prompt_attention_mask.dtype) + position_ids = (expanded_attention_mask.cumsum(-1) - 1).masked_fill_((expanded_attention_mask == 0), 1) + + return { + "input_ids": expanded_input_ids, + "attention_mask": expanded_attention_mask, + "position_ids": position_ids, + } + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=256, + help='Max length of tokenizer' + ) + parser.add_argument( + '--num_hidden_layers_to_skip', + type=int, + default=2, + help='Number of hidden layers to skip' + ) + parser.add_argument( + '--image_embed_interleave', + type=int, + default=4, + help='Image embed interleave' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=6, + help="the hunyuan variant is a guidance distilled model", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'scheduler'), + ) + + # Get Tokenizer + tokenizer = LlamaTokenizerFast.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'tokenizer'), + ) + tokenizer_2 = CLIPTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'tokenizer_2'), + ) + if args.train_mode != "normal": + image_processor = CLIPImageProcessor.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'image_processor'), + ) + + from contextlib import contextmanager + @contextmanager + def zero3_init_context_manager(deepspeed_plugin, enable=False): + old = deepspeed_plugin.zero3_init_flag + if old == enable: + yield + else: + deepspeed_plugin.zero3_init_flag = enable + deepspeed_plugin.dschf = None + yield + deepspeed_plugin.zero3_init_flag = old + deepspeed_plugin.dschf = None + deepspeed_plugin.set_deepspeed_weakref() + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [zero3_init_context_manager(deepspeed_plugin, enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + if args.train_mode != "normal": + text_encoder = LlavaForConditionalGeneration.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'text_encoder'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + else: + text_encoder = LlamaModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'text_encoder'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder_2 = CLIPTextModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'text_encoder_2'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + text_encoder_2 = text_encoder_2.eval() + # Get Vae + vae = AutoencoderKLHunyuanVideo.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'vae'), + ) + vae.eval() + + # Get Transformer + transformer3d = HunyuanVideoTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'transformer'), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + text_encoder_2.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = HunyuanVideoTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'transformer'), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=HunyuanVideoTransformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = HunyuanVideoTransformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = HunyuanVideoTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + ) + load_model = EMAModel(load_model.parameters(), model_cls=HunyuanVideoTransformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = HunyuanVideoTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + if args.train_mode != "normal": + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + else: + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.layers) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + text_encoder_2.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + clip_pixel_values = batch["clip_pixel_values"] + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + text_encoder_2.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = latents * vae.config.scaling_factor + + if args.train_mode != "normal": + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values)[:, :, :1] + mask_latents = mask_latents * vae.config.scaling_factor + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + text_encoder_2.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + raise ValueError("The enable_text_encoder_in_dataloader is not implemented") + else: + with torch.no_grad(): + if args.train_mode != "normal": + prompt = [DEFAULT_PROMPT_TEMPLATE_I2V["template"].format(p) for p in batch['text']] + crop_start = DEFAULT_PROMPT_TEMPLATE_I2V.get("crop_start", None) + + image_emb_len = DEFAULT_PROMPT_TEMPLATE_I2V.get("image_emb_len", 576) + image_emb_start = DEFAULT_PROMPT_TEMPLATE_I2V.get("image_emb_start", 5) + image_emb_end = DEFAULT_PROMPT_TEMPLATE_I2V.get("image_emb_end", 581) + double_return_token_id = DEFAULT_PROMPT_TEMPLATE_I2V.get("double_return_token_id", 271) + + if crop_start is None: + prompt_template_input = tokenizer( + DEFAULT_PROMPT_TEMPLATE_I2V["template"], + padding="max_length", + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=False, + ) + crop_start = prompt_template_input["input_ids"].shape[-1] + # Remove <|start_header_id|>, <|end_header_id|>, assistant, <|eot_id|>, and placeholder {} + crop_start -= 5 + + max_sequence_length = args.tokenizer_max_length + crop_start + num_hidden_layers_to_skip = args.num_hidden_layers_to_skip + image_embed_interleave = args.image_embed_interleave + text_inputs = tokenizer( + prompt, + max_length=max_sequence_length, + padding="max_length", + truncation=True, + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=True, + ) + text_input_ids = text_inputs.input_ids.to(device=accelerator.device) + prompt_attention_mask = text_inputs.attention_mask.to(device=accelerator.device) + + image_embeds = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + _image_embeds = image_processor(clip_image, return_tensors="pt").pixel_values.to(accelerator.device) + print(_image_embeds.size()) + image_embeds.append(_image_embeds) + image_embeds = torch.cat(image_embeds, dim=0) + + image_token_index = text_encoder.config.image_token_index + pad_token_id = text_encoder.config.pad_token_id + expanded_inputs = _expand_input_ids_with_image_tokens( + text_input_ids, + prompt_attention_mask, + max_sequence_length, + image_token_index, + image_emb_len, + image_emb_start, + image_emb_end, + pad_token_id, + ) + prompt_embeds = text_encoder( + **expanded_inputs, + pixel_values=image_embeds, + output_hidden_states=True, + ).hidden_states[-(num_hidden_layers_to_skip + 1)] + prompt_embeds = prompt_embeds.to(dtype=weight_dtype) + + if crop_start is not None and crop_start > 0: + text_crop_start = crop_start - 1 + image_emb_len + batch_indices, last_double_return_token_indices = torch.where(text_input_ids == double_return_token_id) + + if last_double_return_token_indices.shape[0] == 3: + # in case the prompt is too long + last_double_return_token_indices = torch.cat( + (last_double_return_token_indices, torch.tensor([text_input_ids.shape[-1]])) + ) + batch_indices = torch.cat((batch_indices, torch.tensor([0]))) + + last_double_return_token_indices = last_double_return_token_indices.reshape(text_input_ids.shape[0], -1)[ + :, -1 + ] + batch_indices = batch_indices.reshape(text_input_ids.shape[0], -1)[:, -1] + assistant_crop_start = last_double_return_token_indices - 1 + image_emb_len - 4 + assistant_crop_end = last_double_return_token_indices - 1 + image_emb_len + attention_mask_assistant_crop_start = last_double_return_token_indices - 4 + attention_mask_assistant_crop_end = last_double_return_token_indices + + prompt_embed_list = [] + prompt_attention_mask_list = [] + image_embed_list = [] + image_attention_mask_list = [] + + for i in range(text_input_ids.shape[0]): + prompt_embed_list.append( + torch.cat( + [ + prompt_embeds[i, text_crop_start : assistant_crop_start[i].item()], + prompt_embeds[i, assistant_crop_end[i].item() :], + ] + ) + ) + prompt_attention_mask_list.append( + torch.cat( + [ + prompt_attention_mask[i, crop_start : attention_mask_assistant_crop_start[i].item()], + prompt_attention_mask[i, attention_mask_assistant_crop_end[i].item() :], + ] + ) + ) + image_embed_list.append(prompt_embeds[i, image_emb_start:image_emb_end]) + image_attention_mask_list.append( + torch.ones(image_embed_list[-1].shape[0]).to(prompt_embeds.device).to(prompt_attention_mask.dtype) + ) + + prompt_embed_list = torch.stack(prompt_embed_list) + prompt_attention_mask_list = torch.stack(prompt_attention_mask_list) + image_embed_list = torch.stack(image_embed_list) + image_attention_mask_list = torch.stack(image_attention_mask_list) + + if 0 < image_embed_interleave < 6: + image_embed_list = image_embed_list[:, ::image_embed_interleave, :] + image_attention_mask_list = image_attention_mask_list[:, ::image_embed_interleave] + + assert ( + prompt_embed_list.shape[0] == prompt_attention_mask_list.shape[0] + and image_embed_list.shape[0] == image_attention_mask_list.shape[0] + ) + + prompt_embeds = torch.cat([image_embed_list, prompt_embed_list], dim=1) + prompt_attention_mask = torch.cat([image_attention_mask_list, prompt_attention_mask_list], dim=1) + + else: + prompt = [DEFAULT_PROMPT_TEMPLATE["template"].format(p) for p in batch['text']] + crop_start = DEFAULT_PROMPT_TEMPLATE.get("crop_start", None) + if crop_start is None: + prompt_template_input = tokenizer( + DEFAULT_PROMPT_TEMPLATE["template"], + padding="max_length", + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=False, + ) + crop_start = prompt_template_input["input_ids"].shape[-1] + # Remove <|eot_id|> token and placeholder {} + crop_start -= 2 + + max_sequence_length = args.tokenizer_max_length + crop_start + num_hidden_layers_to_skip = args.num_hidden_layers_to_skip + text_inputs = tokenizer( + prompt, + max_length=max_sequence_length, + padding="max_length", + truncation=True, + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=True, + ) + text_input_ids = text_inputs.input_ids.to(accelerator.device) + prompt_attention_mask = text_inputs.attention_mask.to(accelerator.device) + + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_attention_mask, + output_hidden_states=True, + ).hidden_states[-(num_hidden_layers_to_skip + 1)] + prompt_embeds = prompt_embeds.to(dtype=weight_dtype) + + if crop_start is not None and crop_start > 0: + prompt_embeds = prompt_embeds[:, crop_start:] + prompt_attention_mask = prompt_attention_mask[:, crop_start:] + + text_inputs_2 = tokenizer_2( + batch['text'], + padding="max_length", + max_length=77, + truncation=True, + return_tensors="pt", + ) + text_inputs_2_ids = text_inputs_2.input_ids + prompt_embeds_2 = text_encoder_2(text_inputs_2_ids.to(accelerator.device), output_hidden_states=False).pooler_output + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + text_encoder_2.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + # Predict the noise residual + guidance = torch.tensor([args.guidance_scale] * latents.shape[0], dtype=weight_dtype, device=accelerator.device) * 1000.0 + + # Make first frame input to start image. + if args.train_mode != "normal": + noisy_latents[:, :, :1] = mask_latents + + # Go Go Go + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + hidden_states=noisy_latents, + timestep=timesteps, + encoder_hidden_states=prompt_embeds, + encoder_attention_mask=prompt_attention_mask, + pooled_projections=prompt_embeds_2, + guidance=guidance, + )[0] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + if target.shape[2] > 1: + loss = custom_mse_loss(noise_pred.float()[:, :, 1:], target.float()[:, :, 1:], weighting.float()) + else: + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) * 0 + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + text_encoder_2, + tokenizer, + tokenizer_2, + transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + text_encoder_2, + tokenizer, + tokenizer_2, + transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/hunyuanvideo/train.sh b/VideoX-Fun/scripts/hunyuanvideo/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..2ba2234555aec74a389dca8f00eb571cb401d252 --- /dev/null +++ b/VideoX-Fun/scripts/hunyuanvideo/train.sh @@ -0,0 +1,41 @@ +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/internaldatasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/hunyuanvideo/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/hunyuanvideo/train_lora.py b/VideoX-Fun/scripts/hunyuanvideo/train_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..d1fa84cedf7cd3e93ae2ccc077616d366b28fc72 --- /dev/null +++ b/VideoX-Fun/scripts/hunyuanvideo/train_lora.py @@ -0,0 +1,2081 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLHunyuanVideo, CLIPImageProcessor, + CLIPTextModel, CLIPTokenizer, + HunyuanVideoTransformer3DModel, LlamaModel, + LlamaTokenizerFast, + LlavaForConditionalGeneration) +from videox_fun.pipeline import HunyuanVideoPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import (get_image, get_image_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, text_encoder_2, tokenizer, tokenizer_2, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = HunyuanVideoTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'transformer'), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + if args.train_mode != "normal": + raise NotImplementedError("train_mode is not implemented") + else: + pipeline = HunyuanVideoPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + text_encoder_2=accelerator.unwrap_model(text_encoder_2), + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + raise NotImplementedError("train_mode is not implemented") + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +DEFAULT_PROMPT_TEMPLATE = { + "template": ( + "<|start_header_id|>system<|end_header_id|>\n\nDescribe the video by detailing the following aspects: " + "1. The main content and theme of the video." + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects." + "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects." + "4. background environment, light, style and atmosphere." + "5. camera angles, movements, and transitions used in the video:<|eot_id|>" + "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + ), + "crop_start": 95, +} + +DEFAULT_PROMPT_TEMPLATE_I2V = { + "template": ( + "<|start_header_id|>system<|end_header_id|>\n\n\nDescribe the video by detailing the following aspects according to the reference image: " + "1. The main content and theme of the video." + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects." + "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects." + "4. background environment, light, style and atmosphere." + "5. camera angles, movements, and transitions used in the video:<|eot_id|>\n\n" + "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + "<|start_header_id|>assistant<|end_header_id|>\n\n" + ), + "crop_start": 103, + "image_emb_start": 5, + "image_emb_end": 581, + "image_emb_len": 576, + "double_return_token_id": 271, +} + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def _expand_input_ids_with_image_tokens( + text_input_ids, + prompt_attention_mask, + max_sequence_length, + image_token_index, + image_emb_len, + image_emb_start, + image_emb_end, + pad_token_id, +): + special_image_token_mask = text_input_ids == image_token_index + num_special_image_tokens = torch.sum(special_image_token_mask, dim=-1) + batch_indices, non_image_indices = torch.where(text_input_ids != image_token_index) + + max_expanded_length = max_sequence_length + (num_special_image_tokens.max() * (image_emb_len - 1)) + new_token_positions = torch.cumsum((special_image_token_mask * (image_emb_len - 1) + 1), -1) - 1 + text_to_overwrite = new_token_positions[batch_indices, non_image_indices] + + expanded_input_ids = torch.full( + (text_input_ids.shape[0], max_expanded_length), + pad_token_id, + dtype=text_input_ids.dtype, + device=text_input_ids.device, + ) + expanded_input_ids[batch_indices, text_to_overwrite] = text_input_ids[batch_indices, non_image_indices] + expanded_input_ids[batch_indices, image_emb_start:image_emb_end] = image_token_index + + expanded_attention_mask = torch.zeros( + (text_input_ids.shape[0], max_expanded_length), + dtype=prompt_attention_mask.dtype, + device=prompt_attention_mask.device, + ) + attn_batch_indices, attention_indices = torch.where(expanded_input_ids != pad_token_id) + expanded_attention_mask[attn_batch_indices, attention_indices] = 1.0 + expanded_attention_mask = expanded_attention_mask.to(prompt_attention_mask.dtype) + position_ids = (expanded_attention_mask.cumsum(-1) - 1).masked_fill_((expanded_attention_mask == 0), 1) + + return { + "input_ids": expanded_input_ids, + "attention_mask": expanded_attention_mask, + "position_ids": position_ids, + } + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=256, + help='Max length of tokenizer' + ) + parser.add_argument( + '--num_hidden_layers_to_skip', + type=int, + default=2, + help='Number of hidden layers to skip' + ) + parser.add_argument( + '--image_embed_interleave', + type=int, + default=4, + help='Image embed interleave' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=6, + help="the hunyuan variant is a guidance distilled model", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'scheduler'), + ) + + # Get Tokenizer + tokenizer = LlamaTokenizerFast.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'tokenizer'), + ) + tokenizer_2 = CLIPTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'tokenizer_2'), + ) + if args.train_mode != "normal": + image_processor = CLIPImageProcessor.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'image_processor'), + ) + + from contextlib import contextmanager + @contextmanager + def zero3_init_context_manager(deepspeed_plugin, enable=False): + old = deepspeed_plugin.zero3_init_flag + if old == enable: + yield + else: + deepspeed_plugin.zero3_init_flag = enable + deepspeed_plugin.dschf = None + yield + deepspeed_plugin.zero3_init_flag = old + deepspeed_plugin.dschf = None + deepspeed_plugin.set_deepspeed_weakref() + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [zero3_init_context_manager(deepspeed_plugin, enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + if args.train_mode != "normal": + text_encoder = LlavaForConditionalGeneration.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'text_encoder'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + else: + text_encoder = LlamaModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'text_encoder'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder_2 = CLIPTextModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'text_encoder_2'), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + text_encoder_2 = text_encoder_2.eval() + # Get Vae + vae = AutoencoderKLHunyuanVideo.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'vae'), + ) + vae.eval() + + # Get Transformer + transformer3d = HunyuanVideoTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, 'transformer'), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + text_encoder_2.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import (LoraConfig, get_peft_model_state_dict, + inject_adapter_in_model) + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + raise ValueError("The enable_text_encoder_in_dataloader is not implemented") + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=list(transformer3d.transformer_blocks) + list(transformer3d.single_transformer_blocks)) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + if args.train_mode != "normal": + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + else: + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.layers) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + text_encoder_2.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + clip_pixel_values = batch["clip_pixel_values"] + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + text_encoder_2.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = latents * vae.config.scaling_factor + + if args.train_mode != "normal": + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values)[:, :, :1] + mask_latents = mask_latents * vae.config.scaling_factor + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + text_encoder_2.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + raise ValueError("The enable_text_encoder_in_dataloader is not implemented") + else: + with torch.no_grad(): + if args.train_mode != "normal": + prompt = [DEFAULT_PROMPT_TEMPLATE_I2V["template"].format(p) for p in batch['text']] + crop_start = DEFAULT_PROMPT_TEMPLATE_I2V.get("crop_start", None) + + image_emb_len = DEFAULT_PROMPT_TEMPLATE_I2V.get("image_emb_len", 576) + image_emb_start = DEFAULT_PROMPT_TEMPLATE_I2V.get("image_emb_start", 5) + image_emb_end = DEFAULT_PROMPT_TEMPLATE_I2V.get("image_emb_end", 581) + double_return_token_id = DEFAULT_PROMPT_TEMPLATE_I2V.get("double_return_token_id", 271) + + if crop_start is None: + prompt_template_input = tokenizer( + DEFAULT_PROMPT_TEMPLATE_I2V["template"], + padding="max_length", + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=False, + ) + crop_start = prompt_template_input["input_ids"].shape[-1] + # Remove <|start_header_id|>, <|end_header_id|>, assistant, <|eot_id|>, and placeholder {} + crop_start -= 5 + + max_sequence_length = args.tokenizer_max_length + crop_start + num_hidden_layers_to_skip = args.num_hidden_layers_to_skip + image_embed_interleave = args.image_embed_interleave + text_inputs = tokenizer( + prompt, + max_length=max_sequence_length, + padding="max_length", + truncation=True, + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=True, + ) + text_input_ids = text_inputs.input_ids.to(device=accelerator.device) + prompt_attention_mask = text_inputs.attention_mask.to(device=accelerator.device) + + image_embeds = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + _image_embeds = image_processor(clip_image, return_tensors="pt").pixel_values.to(accelerator.device) + image_embeds.append(_image_embeds) + image_embeds = torch.cat(image_embeds, dim=0) + + image_token_index = text_encoder.config.image_token_index + pad_token_id = text_encoder.config.pad_token_id + expanded_inputs = _expand_input_ids_with_image_tokens( + text_input_ids, + prompt_attention_mask, + max_sequence_length, + image_token_index, + image_emb_len, + image_emb_start, + image_emb_end, + pad_token_id, + ) + prompt_embeds = text_encoder( + **expanded_inputs, + pixel_values=image_embeds, + output_hidden_states=True, + ).hidden_states[-(num_hidden_layers_to_skip + 1)] + prompt_embeds = prompt_embeds.to(dtype=weight_dtype) + + if crop_start is not None and crop_start > 0: + text_crop_start = crop_start - 1 + image_emb_len + batch_indices, last_double_return_token_indices = torch.where(text_input_ids == double_return_token_id) + + if last_double_return_token_indices.shape[0] == 3: + # in case the prompt is too long + last_double_return_token_indices = torch.cat( + (last_double_return_token_indices, torch.tensor([text_input_ids.shape[-1]])) + ) + batch_indices = torch.cat((batch_indices, torch.tensor([0]))) + + last_double_return_token_indices = last_double_return_token_indices.reshape(text_input_ids.shape[0], -1)[ + :, -1 + ] + batch_indices = batch_indices.reshape(text_input_ids.shape[0], -1)[:, -1] + assistant_crop_start = last_double_return_token_indices - 1 + image_emb_len - 4 + assistant_crop_end = last_double_return_token_indices - 1 + image_emb_len + attention_mask_assistant_crop_start = last_double_return_token_indices - 4 + attention_mask_assistant_crop_end = last_double_return_token_indices + + prompt_embed_list = [] + prompt_attention_mask_list = [] + image_embed_list = [] + image_attention_mask_list = [] + + for i in range(text_input_ids.shape[0]): + prompt_embed_list.append( + torch.cat( + [ + prompt_embeds[i, text_crop_start : assistant_crop_start[i].item()], + prompt_embeds[i, assistant_crop_end[i].item() :], + ] + ) + ) + prompt_attention_mask_list.append( + torch.cat( + [ + prompt_attention_mask[i, crop_start : attention_mask_assistant_crop_start[i].item()], + prompt_attention_mask[i, attention_mask_assistant_crop_end[i].item() :], + ] + ) + ) + image_embed_list.append(prompt_embeds[i, image_emb_start:image_emb_end]) + image_attention_mask_list.append( + torch.ones(image_embed_list[-1].shape[0]).to(prompt_embeds.device).to(prompt_attention_mask.dtype) + ) + + prompt_embed_list = torch.stack(prompt_embed_list) + prompt_attention_mask_list = torch.stack(prompt_attention_mask_list) + image_embed_list = torch.stack(image_embed_list) + image_attention_mask_list = torch.stack(image_attention_mask_list) + + if 0 < image_embed_interleave < 6: + image_embed_list = image_embed_list[:, ::image_embed_interleave, :] + image_attention_mask_list = image_attention_mask_list[:, ::image_embed_interleave] + + assert ( + prompt_embed_list.shape[0] == prompt_attention_mask_list.shape[0] + and image_embed_list.shape[0] == image_attention_mask_list.shape[0] + ) + + prompt_embeds = torch.cat([image_embed_list, prompt_embed_list], dim=1) + prompt_attention_mask = torch.cat([image_attention_mask_list, prompt_attention_mask_list], dim=1) + + else: + prompt = [DEFAULT_PROMPT_TEMPLATE["template"].format(p) for p in batch['text']] + crop_start = DEFAULT_PROMPT_TEMPLATE.get("crop_start", None) + if crop_start is None: + prompt_template_input = tokenizer( + DEFAULT_PROMPT_TEMPLATE["template"], + padding="max_length", + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=False, + ) + crop_start = prompt_template_input["input_ids"].shape[-1] + # Remove <|eot_id|> token and placeholder {} + crop_start -= 2 + + max_sequence_length = args.tokenizer_max_length + crop_start + num_hidden_layers_to_skip = args.num_hidden_layers_to_skip + text_inputs = tokenizer( + prompt, + max_length=max_sequence_length, + padding="max_length", + truncation=True, + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=True, + ) + text_input_ids = text_inputs.input_ids.to(accelerator.device) + prompt_attention_mask = text_inputs.attention_mask.to(accelerator.device) + + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_attention_mask, + output_hidden_states=True, + ).hidden_states[-(num_hidden_layers_to_skip + 1)] + prompt_embeds = prompt_embeds.to(dtype=weight_dtype) + + if crop_start is not None and crop_start > 0: + prompt_embeds = prompt_embeds[:, crop_start:] + prompt_attention_mask = prompt_attention_mask[:, crop_start:] + + text_inputs_2 = tokenizer_2( + batch['text'], + padding="max_length", + max_length=77, + truncation=True, + return_tensors="pt", + ) + text_inputs_2_ids = text_inputs_2.input_ids + prompt_embeds_2 = text_encoder_2(text_inputs_2_ids.to(accelerator.device), output_hidden_states=False).pooler_output + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + text_encoder_2.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + # Predict the noise residual + guidance = torch.tensor([args.guidance_scale] * latents.shape[0], dtype=weight_dtype, device=accelerator.device) * 1000.0 + + # Make first frame input to start image. + if args.train_mode != "normal": + noisy_latents[:, :, :1] = mask_latents + + # Go Go Go + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + hidden_states=noisy_latents, + timestep=timesteps, + encoder_hidden_states=prompt_embeds, + encoder_attention_mask=prompt_attention_mask, + pooled_projections=prompt_embeds_2, + guidance=guidance, + )[0] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + if target.shape[2] > 1: + loss = custom_mse_loss(noise_pred.float()[:, :, 1:], target.float()[:, :, 1:], weighting.float()) + else: + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) * 0 + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + text_encoder_2, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + text_encoder_2, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/hunyuanvideo/train_lora.sh b/VideoX-Fun/scripts/hunyuanvideo/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..62007501a4c7ef88c1bcb3c48aa8af0e86f2fe35 --- /dev/null +++ b/VideoX-Fun/scripts/hunyuanvideo/train_lora.sh @@ -0,0 +1,42 @@ +export MODEL_NAME="models/Diffusion_Transformer/HunyuanVideo" +export DATASET_NAME="datasets/internaldatasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/hunyuanvideo/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --low_vram \ + --train_mode="normal" \ No newline at end of file diff --git a/VideoX-Fun/scripts/qwenimage/README_TRAIN.md b/VideoX-Fun/scripts/qwenimage/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..6b2de6b9f56a5cbc5fe36cdd775dbad64b5d3c49 --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/README_TRAIN.md @@ -0,0 +1,178 @@ +## Training Code + +We can choose whether to use deepspeed or fsdp in qwen-image, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images at the center, but instead, it trains the entire images after grouping them into buckets based on resolution. +- `random_hw_adapt` is used to enable automatic height and width scaling for images. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024` +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Without deepspeed: + +Training qwen-image without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/qwenimage/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` + +With Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/qwenimage/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +DeepSpeed Zero-3: + +After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/qwenimage/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=QwenImageTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/qwenimage/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/qwenimage/README_TRAIN_EDIT.md b/VideoX-Fun/scripts/qwenimage/README_TRAIN_EDIT.md new file mode 100644 index 0000000000000000000000000000000000000000..a5db00abe35300a53de4db6eab43e9acdd220898 --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/README_TRAIN_EDIT.md @@ -0,0 +1,208 @@ +## Training Code + +We can choose whether to use deepspeed or fsdp in qwen-image, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in Qwen-Image, you need to add a source_file_path. + +Qwen-Image-Edit needs only one file in source_file_path. +Qwen-Image-Edit-2509 needs only one file or more files in source_file_path. + +```json +[ + { + "file_path": "train/00000001.mp4", + "source_file_path": ["source/00000001.mp4"], + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "source_file_path": ["source/00000002.jpg"], + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images at the center, but instead, it trains the entire images after grouping them into buckets based on resolution. +- `random_hw_adapt` is used to enable automatic height and width scaling for images. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024` +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `train_mode` is used to specify the training mode, which can be either qwen_image_edit or qwen_image_edit_plus. + - `qwen_image_edit` is for Qwen-Image-Edit. + - `qwen_image_edit_plus` is for Qwen-Image-Edit-2509 + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Without deepspeed: + +Training qwen-image-edit without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/qwenimage/train_edit.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." \ + --train_mode "qwen_image_edit" +``` + +With Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/qwenimage/train_edit.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." \ + --train_mode "qwen_image_edit" +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +DeepSpeed Zero-3: + +After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/qwenimage/train_edit.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." \ + --train_mode "qwen_image_edit" +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=QwenImageTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/qwenimage/train_edit.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." \ + --train_mode "qwen_image_edit" +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/qwenimage/README_TRAIN_LORA.md b/VideoX-Fun/scripts/qwenimage/README_TRAIN_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..628ec41cfee940b3b76baf7ca403eed7c80335ae --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/README_TRAIN_LORA.md @@ -0,0 +1,184 @@ +## Lora Training Code + +We can choose whether to use deepspeed or fsdp in qwen-image, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images at the center, but instead, it trains the entire images after grouping them into buckets based on resolution. +- `random_hw_adapt` is used to enable automatic height and width scaling for images. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024` +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Without deepspeed: + +Training qwen-image without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/qwenimage/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,img_mod.1,txt_mod.1,img_mlp.0,img_mlp.2,txt_mlp.0,txt_mlp.2" \ + --use_peft_lora \ + --uniform_sampling +``` + +With Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/qwenimage/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,img_mod.1,txt_mod.1,img_mlp.0,img_mlp.2,txt_mlp.0,txt_mlp.2" \ + --use_peft_lora \ + --uniform_sampling +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is not compatible with PEFT. + +DeepSpeed Zero-3: + +After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/qwenimage/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=QwenImageTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/qwenimage/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,img_mod.1,txt_mod.1,img_mlp.0,img_mlp.2,txt_mlp.0,txt_mlp.2" \ + --use_peft_lora \ + --uniform_sampling +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/qwenimage/train.py b/VideoX-Fun/scripts/qwenimage/train.py new file mode 100644 index 0000000000000000000000000000000000000000..c943e5dc29fb397a08ee59548339b91afac08a31 --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/train.py @@ -0,0 +1,1616 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLQwenImage, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, QwenImageTransformer2DModel) +from videox_fun.pipeline import QwenImagePipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + return latents + +def _extract_masked_hidden(hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = QwenImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + pipeline = QwenImagePipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=1024, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--prompt_template_encode", + type=str, + default="<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", + help=( + 'The prompt template for text encoder.' + ), + ) + parser.add_argument( + "--prompt_template_encode_start_idx", + type=int, + default=34, + help=( + 'The start idx for prompt template.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = Qwen2Tokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLQwenImage.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + latents_mean = (torch.tensor(vae.config.latents_mean).view(1, vae.config.z_dim, 1, 1, 1)).to(accelerator.device) + latents_std = 1.0 / torch.tensor(vae.config.latents_std).view(1, vae.config.z_dim, 1, 1, 1).to(accelerator.device) + + # Get Transformer + transformer3d = QwenImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = QwenImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=QwenImageTransformer2DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = QwenImageTransformer2DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = QwenImageTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + ) + load_model = EMAModel(load_model.parameters(), model_cls=QwenImageTransformer2DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = QwenImageTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["text"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + template = args.prompt_template_encode + drop_idx = args.prompt_template_encode_start_idx + + txt = [template.format(e) for e in batch['text']] + txt_tokens = tokenizer( + txt, max_length=args.tokenizer_max_length + drop_idx, padding=True, truncation=True, return_tensors="pt" + ).to(accelerator.device) + encoder_hidden_states = text_encoder( + input_ids=txt_tokens.input_ids, + attention_mask=txt_tokens.attention_mask, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = _extract_masked_hidden(hidden_states, txt_tokens.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=latents.dtype, device=accelerator.device) + + new_examples['encoder_attention_mask'] = encoder_attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = ((latents - latents_mean) * latents_std).to(dtype=weight_dtype) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + encoder_attention_mask = batch['encoder_attention_mask'] + else: + with torch.no_grad(): + template = args.prompt_template_encode + drop_idx = args.prompt_template_encode_start_idx + + txt = [template.format(e) for e in batch['text']] + txt_tokens = tokenizer( + txt, max_length=args.tokenizer_max_length + drop_idx, padding=True, truncation=True, return_tensors="pt" + ).to(accelerator.device) + encoder_hidden_states = text_encoder( + input_ids=txt_tokens.input_ids, + attention_mask=txt_tokens.attention_mask, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = _extract_masked_hidden(hidden_states, txt_tokens.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=latents.dtype, device=accelerator.device) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frame, height, width = latents.size() + latents = _pack_latents(latents, bsz, channel, height, width) + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + sigmas = np.linspace(1.0, 1 / args.train_sampling_steps, args.train_sampling_steps) + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.set_timesteps(sigmas=sigmas, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + img_shapes = [[(1, height // 2, width // 2)]] * latents.size(0) + txt_seq_lens = encoder_attention_mask.sum(dim=1).tolist() if encoder_attention_mask is not None else None + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + hidden_states=noisy_latents, + timestep=timesteps / 1000, + encoder_hidden_states_mask=encoder_attention_mask, + encoder_hidden_states=prompt_embeds, + img_shapes=img_shapes, + txt_seq_lens=txt_seq_lens, + return_dict=False, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/qwenimage/train.sh b/VideoX-Fun/scripts/qwenimage/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..2e81f318f40605d8e1f58ee3ef5fe623172e3995 --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/train.sh @@ -0,0 +1,32 @@ +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/qwenimage/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/qwenimage/train_edit.py b/VideoX-Fun/scripts/qwenimage/train_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..787eaae853598ff8e678d968a868c212983e5f28 --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/train_edit.py @@ -0,0 +1,1792 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoSampler, + get_random_mask) +from videox_fun.data.dataset_image import ImageEditDataset +from videox_fun.models import (AutoencoderKLQwenImage, Qwen2VLProcessor, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, QwenImageTransformer2DModel) +from videox_fun.pipeline import QwenImageEditPipeline, QwenImageEditPlusPipeline +from videox_fun.pipeline.pipeline_qwenimage_edit import PREFERRED_QWENIMAGE_RESOLUTIONS, calculate_dimensions +from videox_fun.pipeline.pipeline_qwenimage_edit_plus import CONDITION_IMAGE_SIZE, VAE_IMAGE_SIZE +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid, get_image + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + return latents + +def _extract_masked_hidden(hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = QwenImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + if args.train_mode == "qwen_image_edit": + pipeline = QwenImageEditPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + else: + pipeline = QwenImageEditPlusPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode == "qwen_image_edit": + image = get_image(args.validation_image_paths[i]) + else: + image = [get_image(args.validation_image_paths[i])] + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator, + image = image + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_image_paths", + type=str, + default=None, + nargs="+", + help=("A set of images evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=1024, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="qwen_image_edit", + help=( + 'The format of training data. Support `"qwen_image_edit"`' + ' (default), `"qwen_image_edit_plus"`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + if args.train_batch_size >= 2: + raise ValueError("This code does not support args.train_batch_size >= 2 now.") + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = Qwen2Tokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + + # Get processor + processor = Qwen2VLProcessor.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="processor" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + if args.train_mode == "qwen_image_edit": + template = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" + drop_idx = 64 + else: + template = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" + drop_idx = 64 + + # Get Vae + vae = AutoencoderKLQwenImage.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + latents_mean = (torch.tensor(vae.config.latents_mean).view(1, vae.config.z_dim, 1, 1, 1)).to(accelerator.device) + latents_std = 1.0 / torch.tensor(vae.config.latents_std).view(1, vae.config.z_dim, 1, 1, 1).to(accelerator.device) + vae_scale_factor = 2 ** len(vae.temperal_downsample) + + # Get Transformer + transformer3d = QwenImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = QwenImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=QwenImageTransformer2DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = QwenImageTransformer2DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = QwenImageTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + ) + load_model = EMAModel(load_model.parameters(), model_cls=QwenImageTransformer2DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = QwenImageTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageEditDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["source_pixel_values"] = [] + new_examples["text"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + source_pixel_values = examples[0]["source_pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + source_pixel_values = [] + for _source_pixel_value in example["source_pixel_values"]: + source_pixel_values.append(np.array(_source_pixel_value)) + + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["source_pixel_values"].append(source_pixel_values) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + txt = [template.format(e) for e in batch['text']] + prompt_in_pixel_values = [ + Image.fromarray(np.array((source_pixel_value.float().cpu().permute(0, 2, 3, 1) * 0.5 + 0.5)[0], np.uint8)) for source_pixel_value in source_pixel_values + ] + model_inputs = processor( + text=txt, + images=prompt_in_pixel_values, + padding=True, + return_tensors="pt", + ) + encoder_hidden_states = text_encoder( + input_ids=model_inputs.input_ids, + attention_mask=model_inputs.attention_mask, + pixel_values=model_inputs.pixel_values, + image_grid_thw=model_inputs.image_grid_thw, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = _extract_masked_hidden(hidden_states, model_inputs.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=latents.dtype) + + new_examples['encoder_attention_mask'] = encoder_attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + source_pixel_values = batch['source_pixel_values'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, source_pixel_value, text) in enumerate(zip(pixel_values, source_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + for local_index, _source_pixel_value in enumerate(source_pixel_value): + _source_pixel_value = Image.fromarray(np.uint8(_source_pixel_value)) + _source_pixel_value.save(f"{args.output_dir}/sanity_check/source_{local_index}_{gif_name[:10]}.jpg") + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + source_pixel_values = batch["source_pixel_values"] + + condition_image_sizes = [] + condition_images = [] + vae_image_sizes = [] + vae_images = [] + + if args.train_mode == "qwen_image_edit": + for source_pixel_value in source_pixel_values: + _prompt_in_pixel_values = [] + _prompt_in_sizes = [] + for _source_pixel_value in source_pixel_value: + source_image_height, source_image_width = np.shape(_source_pixel_value)[-2:] + aspect_ratio = source_image_width / source_image_height + _, source_image_width, source_image_height = min( + (abs(aspect_ratio - w / h), w, h) for w, h in PREFERRED_QWENIMAGE_RESOLUTIONS + ) + multiple_of = vae_scale_factor * 2 + source_image_width = source_image_width // multiple_of * multiple_of + source_image_height = source_image_height // multiple_of * multiple_of + + _source_pixel_value = Image.fromarray(np.array(_source_pixel_value, np.uint8)) + _source_pixel_value = _source_pixel_value.resize((source_image_width, source_image_height), resample=Image.LANCZOS) + + _prompt_in_pixel_values.append(_source_pixel_value) + _prompt_in_sizes.append([source_image_width, source_image_height]) + + condition_images.append(_prompt_in_pixel_values) + condition_image_sizes.append(_prompt_in_sizes) + vae_images.append(_prompt_in_pixel_values) + vae_image_sizes.append(_prompt_in_sizes) + else: + for source_pixel_value in source_pixel_values: + _condition_images = [] + _condition_image_sizes = [] + _vae_images = [] + _vae_image_sizes = [] + for _source_pixel_value in source_pixel_value: + _source_pixel_value = Image.fromarray(np.array(_source_pixel_value, np.uint8)) + + image_width, image_height = _source_pixel_value.size + vae_width, vae_height = calculate_dimensions(VAE_IMAGE_SIZE, image_width / image_height) + _vae_image_sizes.append((vae_width, vae_height)) + _vae_images.append(_source_pixel_value.resize((vae_width, vae_height), resample=Image.LANCZOS)) + + condition_width, condition_height = calculate_dimensions(CONDITION_IMAGE_SIZE, image_width / image_height) + _condition_image_sizes.append((condition_width, condition_height)) + _condition_images.append(_source_pixel_value.resize((condition_width, condition_height), resample=Image.LANCZOS)) + + condition_images.append(_condition_images) + condition_image_sizes.append(_condition_image_sizes) + vae_images.append(_vae_images) + vae_image_sizes.append(_vae_image_sizes) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = ((latents - latents_mean) * latents_std).to(dtype=weight_dtype) + + source_latents = [] + source_exist = False + for _vae_images in vae_images: + _source_latents = [] + for _vae_image in _vae_images: + _vae_image = torch.from_numpy(np.expand_dims(np.array(_vae_image), 0)).to(dtype=weight_dtype, device=vae.device) + _vae_image = _vae_image.permute(0, 3, 1, 2).contiguous().unsqueeze(0) + _vae_image = _vae_image / 255. + _vae_image = (_vae_image - 0.5) / 0.5 + + _source_latent = _batch_encode_vae(_vae_image) + _source_latent = ((_source_latent - latents_mean) * latents_std).to(dtype=weight_dtype) + + bsz, source_channel, _, source_latent_height, source_latent_width = _source_latent.size() + _source_latent = _pack_latents(_source_latent, bsz, source_channel, source_latent_height, source_latent_width) + _source_latents.append(_source_latent) + if len(_source_latents) != 0: + _source_latents = torch.cat(_source_latents, dim = 1) + source_exist = True + source_latents.append(_source_latents) + if source_exist: + source_latents = torch.cat(source_latents, dim = 0) + else: + source_latents = None + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + encoder_attention_mask = batch['encoder_attention_mask'] + else: + with torch.no_grad(): + if args.train_mode == "qwen_image_edit": + if source_exist: + # Take the first batch condition_images + prompt_in_pixel_values = condition_images[0][:1] + else: + prompt_in_pixel_values = None + base_img_prompt = "" + txt = [template.format(e) for e in batch['text']] + else: + if source_exist: + # Take the first batch condition_images + prompt_in_pixel_values = condition_images[0] + else: + prompt_in_pixel_values = None + + img_prompt_template = "Picture {}: <|vision_start|><|image_pad|><|vision_end|>" + if isinstance(prompt_in_pixel_values, list): + base_img_prompt = "" + for i, img in enumerate(prompt_in_pixel_values): + base_img_prompt += img_prompt_template.format(i + 1) + elif prompt_in_pixel_values is not None: + base_img_prompt = img_prompt_template.format(1) + else: + base_img_prompt = "" + + txt = [template.format(base_img_prompt + e) for e in batch['text']] + + model_inputs = processor( + text=txt, + images=prompt_in_pixel_values, + padding=True, + return_tensors="pt", + ).to(accelerator.device) + if prompt_in_pixel_values is None: + encoder_hidden_states = text_encoder( + input_ids=model_inputs.input_ids, + attention_mask=model_inputs.attention_mask, + output_hidden_states=True, + ) + else: + encoder_hidden_states = text_encoder( + input_ids=model_inputs.input_ids, + attention_mask=model_inputs.attention_mask, + pixel_values=model_inputs.pixel_values, + image_grid_thw=model_inputs.image_grid_thw, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = _extract_masked_hidden(hidden_states, model_inputs.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=latents.dtype, device=accelerator.device) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frame, height, width = latents.size() + latents = _pack_latents(latents, bsz, channel, height, width) + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + sigmas = np.linspace(1.0, 1 / args.train_sampling_steps, args.train_sampling_steps) + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.set_timesteps(sigmas=sigmas, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + img_shapes = [ + [ + (1, height // 2, width // 2), + *[ + (1, vae_height // vae_scale_factor // 2, vae_width // vae_scale_factor // 2) + for vae_width, vae_height in vae_image_sizes[0] + ], + ] + ] * latents.size(0) + txt_seq_lens = encoder_attention_mask.sum(dim=1).tolist() if encoder_attention_mask is not None else None + + if source_latents is not None: + noisy_latents_and_image_latents = torch.cat([noisy_latents, source_latents], dim=1) + else: + noisy_latents_and_image_latents = noisy_latents + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + hidden_states=noisy_latents_and_image_latents, + timestep=timesteps / 1000, + encoder_hidden_states_mask=encoder_attention_mask, + encoder_hidden_states=prompt_embeds, + img_shapes=img_shapes, + txt_seq_lens=txt_seq_lens, + return_dict=False, + ) + noise_pred = noise_pred[:, : noisy_latents.size(1)] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/qwenimage/train_edit.sh b/VideoX-Fun/scripts/qwenimage/train_edit.sh new file mode 100644 index 0000000000000000000000000000000000000000..50c6f5153606c99be2497ec404113985f99ca32b --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/train_edit.sh @@ -0,0 +1,33 @@ +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/qwenimage/train_edit.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." \ + --train_mode "qwen_image_edit" \ No newline at end of file diff --git a/VideoX-Fun/scripts/qwenimage/train_edit_lora.py b/VideoX-Fun/scripts/qwenimage/train_edit_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..f9c4181ceb8b896c31f39ecc44e1cc8b37046a67 --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/train_edit_lora.py @@ -0,0 +1,1832 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image import ImageEditDataset +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLQwenImage, AutoencoderKLWan, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, Qwen2VLProcessor, + QwenImageTransformer2DModel) +from videox_fun.pipeline import QwenImageEditPipeline, QwenImageEditPlusPipeline +from videox_fun.pipeline.pipeline_qwenimage_edit import ( + PREFERRED_QWENIMAGE_RESOLUTIONS, calculate_dimensions) +from videox_fun.pipeline.pipeline_qwenimage_edit_plus import ( + CONDITION_IMAGE_SIZE, VAE_IMAGE_SIZE) +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid, get_image + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + return latents + +def _extract_masked_hidden(hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = QwenImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + if args.train_mode == "qwen_image_edit": + pipeline = QwenImageEditPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + else: + pipeline = QwenImageEditPlusPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode == "qwen_image_edit": + image = get_image(args.validation_image_paths[i]) + else: + image = [get_image(args.validation_image_paths[i])] + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator, + image = image + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_image_paths", + type=str, + default=None, + nargs="+", + help=("A set of images evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=1024, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="qwen_image_edit", + help=( + 'The format of training data. Support `"qwen_image_edit"`' + ' (default), `"qwen_image_edit_plus"`.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + if args.train_batch_size >= 2: + raise ValueError("This code does not support args.train_batch_size >= 2 now.") + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = Qwen2Tokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + + # Get processor + processor = Qwen2VLProcessor.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="processor" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + if args.train_mode == "qwen_image_edit": + template = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" + drop_idx = 64 + else: + template = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" + drop_idx = 64 + + # Get Vae + vae = AutoencoderKLQwenImage.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + latents_mean = (torch.tensor(vae.config.latents_mean).view(1, vae.config.z_dim, 1, 1, 1)).to(accelerator.device) + latents_std = 1.0 / torch.tensor(vae.config.latents_std).view(1, vae.config.z_dim, 1, 1, 1).to(accelerator.device) + vae_scale_factor = 2 ** len(vae.temperal_downsample) + + # Get Transformer + transformer3d = QwenImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageEditDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["source_pixel_values"] = [] + new_examples["text"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + source_pixel_values = examples[0]["source_pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + source_pixel_values = [] + for _source_pixel_value in example["source_pixel_values"]: + source_pixel_values.append(np.array(_source_pixel_value)) + + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["source_pixel_values"].append(source_pixel_values) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + txt = [template.format(e) for e in batch['text']] + prompt_in_pixel_values = [ + Image.fromarray(np.array((source_pixel_value.float().cpu().permute(0, 2, 3, 1) * 0.5 + 0.5)[0], np.uint8)) for source_pixel_value in source_pixel_values + ] + model_inputs = processor( + text=txt, + images=prompt_in_pixel_values, + padding=True, + return_tensors="pt", + ) + encoder_hidden_states = text_encoder( + input_ids=model_inputs.input_ids, + attention_mask=model_inputs.attention_mask, + pixel_values=model_inputs.pixel_values, + image_grid_thw=model_inputs.image_grid_thw, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = _extract_masked_hidden(hidden_states, model_inputs.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=latents.dtype) + + new_examples['encoder_attention_mask'] = encoder_attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=transformer3d.transformer_blocks) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + source_pixel_values = batch['source_pixel_values'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, source_pixel_value, text) in enumerate(zip(pixel_values, source_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + for local_index, _source_pixel_value in enumerate(source_pixel_value): + _source_pixel_value = Image.fromarray(np.uint8(_source_pixel_value)) + _source_pixel_value.save(f"{args.output_dir}/sanity_check/source_{local_index}_{gif_name[:10]}.jpg") + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + source_pixel_values = batch["source_pixel_values"] + + condition_image_sizes = [] + condition_images = [] + vae_image_sizes = [] + vae_images = [] + + if args.train_mode == "qwen_image_edit": + for source_pixel_value in source_pixel_values: + _prompt_in_pixel_values = [] + _prompt_in_sizes = [] + for _source_pixel_value in source_pixel_value: + source_image_height, source_image_width = np.shape(_source_pixel_value)[-2:] + aspect_ratio = source_image_width / source_image_height + _, source_image_width, source_image_height = min( + (abs(aspect_ratio - w / h), w, h) for w, h in PREFERRED_QWENIMAGE_RESOLUTIONS + ) + multiple_of = vae_scale_factor * 2 + source_image_width = source_image_width // multiple_of * multiple_of + source_image_height = source_image_height // multiple_of * multiple_of + + _source_pixel_value = Image.fromarray(np.array(_source_pixel_value, np.uint8)) + _source_pixel_value = _source_pixel_value.resize((source_image_width, source_image_height), resample=Image.LANCZOS) + + _prompt_in_pixel_values.append(_source_pixel_value) + _prompt_in_sizes.append([source_image_width, source_image_height]) + + condition_images.append(_prompt_in_pixel_values) + condition_image_sizes.append(_prompt_in_sizes) + vae_images.append(_prompt_in_pixel_values) + vae_image_sizes.append(_prompt_in_sizes) + else: + for source_pixel_value in source_pixel_values: + _condition_images = [] + _condition_image_sizes = [] + _vae_images = [] + _vae_image_sizes = [] + for _source_pixel_value in source_pixel_value: + _source_pixel_value = Image.fromarray(np.array(_source_pixel_value, np.uint8)) + + image_width, image_height = _source_pixel_value.size + vae_width, vae_height = calculate_dimensions(VAE_IMAGE_SIZE, image_width / image_height) + _vae_image_sizes.append((vae_width, vae_height)) + _vae_images.append(_source_pixel_value.resize((vae_width, vae_height), resample=Image.LANCZOS)) + + condition_width, condition_height = calculate_dimensions(CONDITION_IMAGE_SIZE, image_width / image_height) + _condition_image_sizes.append((condition_width, condition_height)) + _condition_images.append(_source_pixel_value.resize((condition_width, condition_height), resample=Image.LANCZOS)) + + condition_images.append(_condition_images) + condition_image_sizes.append(_condition_image_sizes) + vae_images.append(_vae_images) + vae_image_sizes.append(_vae_image_sizes) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = ((latents - latents_mean) * latents_std).to(dtype=weight_dtype) + + source_latents = [] + source_exist = False + for _vae_images in vae_images: + _source_latents = [] + for _vae_image in _vae_images: + _vae_image = torch.from_numpy(np.expand_dims(np.array(_vae_image), 0)).to(dtype=weight_dtype, device=vae.device) + _vae_image = _vae_image.permute(0, 3, 1, 2).contiguous().unsqueeze(0) + _vae_image = _vae_image / 255. + _vae_image = (_vae_image - 0.5) / 0.5 + + _source_latent = _batch_encode_vae(_vae_image) + _source_latent = ((_source_latent - latents_mean) * latents_std).to(dtype=weight_dtype) + + bsz, source_channel, _, source_latent_height, source_latent_width = _source_latent.size() + _source_latent = _pack_latents(_source_latent, bsz, source_channel, source_latent_height, source_latent_width) + _source_latents.append(_source_latent) + if len(_source_latents) != 0: + _source_latents = torch.cat(_source_latents, dim = 1) + source_exist = True + source_latents.append(_source_latents) + if source_exist: + source_latents = torch.cat(source_latents, dim = 0) + else: + source_latents = None + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + encoder_attention_mask = batch['encoder_attention_mask'] + else: + with torch.no_grad(): + if args.train_mode == "qwen_image_edit": + if source_exist: + # Take the first batch condition_images + prompt_in_pixel_values = condition_images[0][:1] + else: + prompt_in_pixel_values = None + base_img_prompt = "" + txt = [template.format(e) for e in batch['text']] + else: + if source_exist: + # Take the first batch condition_images + prompt_in_pixel_values = condition_images[0] + else: + prompt_in_pixel_values = None + + img_prompt_template = "Picture {}: <|vision_start|><|image_pad|><|vision_end|>" + if isinstance(prompt_in_pixel_values, list): + base_img_prompt = "" + for i, img in enumerate(prompt_in_pixel_values): + base_img_prompt += img_prompt_template.format(i + 1) + elif prompt_in_pixel_values is not None: + base_img_prompt = img_prompt_template.format(1) + else: + base_img_prompt = "" + + txt = [template.format(base_img_prompt + e) for e in batch['text']] + + model_inputs = processor( + text=txt, + images=prompt_in_pixel_values, + padding=True, + return_tensors="pt", + ).to(accelerator.device) + if prompt_in_pixel_values is None: + encoder_hidden_states = text_encoder( + input_ids=model_inputs.input_ids, + attention_mask=model_inputs.attention_mask, + output_hidden_states=True, + ) + else: + encoder_hidden_states = text_encoder( + input_ids=model_inputs.input_ids, + attention_mask=model_inputs.attention_mask, + pixel_values=model_inputs.pixel_values, + image_grid_thw=model_inputs.image_grid_thw, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = _extract_masked_hidden(hidden_states, model_inputs.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=latents.dtype, device=accelerator.device) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frame, height, width = latents.size() + latents = _pack_latents(latents, bsz, channel, height, width) + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + sigmas = np.linspace(1.0, 1 / args.train_sampling_steps, args.train_sampling_steps) + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.set_timesteps(sigmas=sigmas, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + img_shapes = [ + [ + (1, height // 2, width // 2), + *[ + (1, vae_height // vae_scale_factor // 2, vae_width // vae_scale_factor // 2) + for vae_width, vae_height in vae_image_sizes[0] + ], + ] + ] * latents.size(0) + txt_seq_lens = encoder_attention_mask.sum(dim=1).tolist() if encoder_attention_mask is not None else None + + if source_latents is not None: + noisy_latents_and_image_latents = torch.cat([noisy_latents, source_latents], dim=1) + else: + noisy_latents_and_image_latents = noisy_latents + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + hidden_states=noisy_latents_and_image_latents, + timestep=timesteps / 1000, + encoder_hidden_states_mask=encoder_attention_mask, + encoder_hidden_states=prompt_embeds, + img_shapes=img_shapes, + txt_seq_lens=txt_seq_lens, + return_dict=False, + ) + noise_pred = noise_pred[:, : noisy_latents.size(1)] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/qwenimage/train_edit_lora.sh b/VideoX-Fun/scripts/qwenimage/train_edit_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..ec17fe700774a9029015e871159d697e6b70193d --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/train_edit_lora.sh @@ -0,0 +1,34 @@ +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/qwenimage/train_edit_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,img_mod.1,txt_mod.1,img_mlp.0,img_mlp.2,txt_mlp.0,txt_mlp.2" \ + --use_peft_lora \ + --train_mode "qwen_image_edit" \ No newline at end of file diff --git a/VideoX-Fun/scripts/qwenimage/train_lora.py b/VideoX-Fun/scripts/qwenimage/train_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..0055b529cf1c97b279a4c662f28981ec55fdd93c --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/train_lora.py @@ -0,0 +1,1651 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLQwenImage, AutoencoderKLWan, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, QwenImageTransformer2DModel) +from videox_fun.pipeline import QwenImagePipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + return latents + +def _extract_masked_hidden(hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = QwenImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + pipeline = QwenImagePipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=1024, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--prompt_template_encode", + type=str, + default="<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", + help=( + 'The prompt template for text encoder.' + ), + ) + parser.add_argument( + "--prompt_template_encode_start_idx", + type=int, + default=34, + help=( + 'The start idx for prompt template.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = Qwen2Tokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = Qwen2_5_VLForConditionalGeneration.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLQwenImage.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + latents_mean = (torch.tensor(vae.config.latents_mean).view(1, vae.config.z_dim, 1, 1, 1)).to(accelerator.device) + latents_std = 1.0 / torch.tensor(vae.config.latents_std).view(1, vae.config.z_dim, 1, 1, 1).to(accelerator.device) + + # Get Transformer + transformer3d = QwenImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["text"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + template = args.prompt_template_encode + drop_idx = args.prompt_template_encode_start_idx + + txt = [template.format(e) for e in batch['text']] + txt_tokens = tokenizer( + txt, max_length=args.tokenizer_max_length + drop_idx, padding=True, truncation=True, return_tensors="pt" + ).to(accelerator.device) + encoder_hidden_states = text_encoder( + input_ids=txt_tokens.input_ids, + attention_mask=txt_tokens.attention_mask, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = _extract_masked_hidden(hidden_states, txt_tokens.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=latents.dtype, device=accelerator.device) + + new_examples['encoder_attention_mask'] = encoder_attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=transformer3d.transformer_blocks) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.language_model.layers) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need】 + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + latents = ((latents - latents_mean) * latents_std).to(dtype=weight_dtype) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + encoder_attention_mask = batch['encoder_attention_mask'] + else: + with torch.no_grad(): + template = args.prompt_template_encode + drop_idx = args.prompt_template_encode_start_idx + + txt = [template.format(e) for e in batch['text']] + txt_tokens = tokenizer( + txt, max_length=args.tokenizer_max_length + drop_idx, padding=True, truncation=True, return_tensors="pt" + ).to(accelerator.device) + encoder_hidden_states = text_encoder( + input_ids=txt_tokens.input_ids, + attention_mask=txt_tokens.attention_mask, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = _extract_masked_hidden(hidden_states, txt_tokens.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=latents.dtype, device=accelerator.device) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frame, height, width = latents.size() + latents = _pack_latents(latents, bsz, channel, height, width) + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + sigmas = np.linspace(1.0, 1 / args.train_sampling_steps, args.train_sampling_steps) + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.set_timesteps(sigmas=sigmas, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + img_shapes = [[(1, height // 2, width // 2)]] * latents.size(0) + txt_seq_lens = encoder_attention_mask.sum(dim=1).tolist() if encoder_attention_mask is not None else None + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + hidden_states=noisy_latents, + timestep=timesteps / 1000, + encoder_hidden_states_mask=encoder_attention_mask, + encoder_hidden_states=prompt_embeds, + img_shapes=img_shapes, + txt_seq_lens=txt_seq_lens, + return_dict=False, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/qwenimage/train_lora.sh b/VideoX-Fun/scripts/qwenimage/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..7a35068058477798df649d1527ca4deba2b39c9f --- /dev/null +++ b/VideoX-Fun/scripts/qwenimage/train_lora.sh @@ -0,0 +1,33 @@ +export MODEL_NAME="models/Diffusion_Transformer/Qwen-Image" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/qwenimage/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,img_mod.1,txt_mod.1,img_mlp.0,img_mlp.2,txt_mlp.0,txt_mlp.2" \ + --use_peft_lora \ + --uniform_sampling \ No newline at end of file diff --git a/VideoX-Fun/scripts/split_onnx_by_subconfig.py b/VideoX-Fun/scripts/split_onnx_by_subconfig.py new file mode 100644 index 0000000000000000000000000000000000000000..c37d7dbae07c851b0ade9522fbb4fe002d612580 --- /dev/null +++ b/VideoX-Fun/scripts/split_onnx_by_subconfig.py @@ -0,0 +1,748 @@ +#!/usr/bin/env python3 +"""基于 tensor 名称的通用 ONNX 子图切分工具。 + +相较于 split_quant_onnx_by_subconfigs.py，本脚本额外提供： +1. 为每个子模型执行 onnx checker 与 shape inference（可关闭）。 +2. 支持从 .npz/.npy（包含 dict 或单数组）加载验证数据。 +3. 可用 onnxruntime 串联执行全部子模型，既可校验精度，也可单独输出流水线结果。 +""" +from __future__ import annotations + +import argparse +import json +import logging +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Sequence, Set + +import numpy as np +import onnx +from onnx import TensorProto, checker, helper, shape_inference, utils as onnx_utils + +try: # pragma: no cover - 可选依赖 + import onnxruntime as ort +except ImportError: # pragma: no cover + ort = None + + +@dataclass +class SubGraphSpec: + label: str + start: List[str] + end: List[str] + node_names: Set[str] + source: str + output_path: Optional[Path] = None + + +@dataclass +class GraphIndex: + tensor_to_producer: Dict[str, str] + tensor_to_consumers: Dict[str, List[str]] + node_inputs: Dict[str, List[str]] + node_outputs: Dict[str, List[str]] + graph_inputs: Set[str] + graph_outputs: Set[str] + initializer_names: Set[str] + node_order: List[str] + + +def sanitize(name: str) -> str: + keep = [c if c.isalnum() else "_" for c in name] if name else ["anon"] + sanitized = "".join(keep).strip("_") + return sanitized or "tensor" + + +def build_graph_index(model: onnx.ModelProto) -> GraphIndex: + tensor_to_producer: Dict[str, str] = {} + tensor_to_consumers: Dict[str, List[str]] = defaultdict(list) + node_inputs: Dict[str, List[str]] = {} + node_outputs: Dict[str, List[str]] = {} + node_order: List[str] = [] + + used_names: Set[str] = set() + for idx, node in enumerate(model.graph.node): + base = node.name.strip() if node.name else "" + candidate = base or f"node_{idx}" + while candidate in used_names: + candidate = f"{candidate}_{idx}" + used_names.add(candidate) + node_order.append(candidate) + node_inputs[candidate] = [x for x in node.input if x] + node_outputs[candidate] = [y for y in node.output if y] + for out_name in node_outputs[candidate]: + tensor_to_producer[out_name] = candidate + for inp_name in node_inputs[candidate]: + tensor_to_consumers[inp_name].append(candidate) + + graph_inputs = {vi.name for vi in model.graph.input} + graph_outputs = {vi.name for vi in model.graph.output} + initializer_names = {init.name for init in model.graph.initializer} + + return GraphIndex( + tensor_to_producer=tensor_to_producer, + tensor_to_consumers=tensor_to_consumers, + node_inputs=node_inputs, + node_outputs=node_outputs, + graph_inputs=graph_inputs, + graph_outputs=graph_outputs, + initializer_names=initializer_names, + node_order=node_order, + ) + + +def ensure_value_infos(model: onnx.ModelProto, tensor_names: Iterable[str]) -> None: + existing = {vi.name for vi in model.graph.value_info} + source_map = {} + for vi in list(model.graph.input) + list(model.graph.value_info) + list(model.graph.output): + source_map[vi.name] = vi + + added: List[str] = [] + for name in tensor_names: + if name in existing: + continue + src = source_map.get(name) + if src is not None: + vi = onnx.ValueInfoProto() + vi.CopyFrom(src) + else: + vi = helper.make_tensor_value_info(name, TensorProto.UNDEFINED, None) + model.graph.value_info.append(vi) + existing.add(name) + added.append(name) + if added: + logging.debug("已为以下 tensor 补充 ValueInfo: %s", added) + + +def ensure_extractor_value_infos( + extractor: onnx_utils.Extractor, + tensor_names: Iterable[str], + source_model: onnx.ModelProto, +) -> None: + existing_inputs = {vi.name for vi in extractor.graph.input} + existing_outputs = {vi.name for vi in extractor.graph.output} + existing_vi = {vi.name for vi in extractor.graph.value_info} + source_map = {} + for vi in ( + list(source_model.graph.input) + + list(source_model.graph.value_info) + + list(source_model.graph.output) + ): + source_map[vi.name] = vi + + added: List[str] = [] + for name in tensor_names: + if name in existing_inputs or name in existing_outputs or name in existing_vi: + continue + src = source_map.get(name) + if src is not None: + vi = onnx.ValueInfoProto() + vi.CopyFrom(src) + else: + vi = helper.make_tensor_value_info(name, TensorProto.UNDEFINED, None) + extractor.graph.value_info.append(vi) + extractor.vimap[name] = vi + existing_vi.add(name) + added.append(name) + if added: + logging.debug("Extractor 侧补充 ValueInfo: %s", added) + + +def trace_nodes_between(spec: SubGraphSpec, index: GraphIndex) -> Set[str]: + boundary = set(spec.start) | index.graph_inputs | index.initializer_names + visited_tensors: Set[str] = set() + stack = list(spec.end) + discovered_nodes: Set[str] = set() + + while stack: + tensor = stack.pop() + if tensor in visited_tensors: + continue + visited_tensors.add(tensor) + if tensor in boundary: + continue + producer = index.tensor_to_producer.get(tensor) + if not producer or producer in discovered_nodes: + continue + discovered_nodes.add(producer) + for upstream in index.node_inputs.get(producer, []): + if upstream and upstream not in boundary: + stack.append(upstream) + return discovered_nodes + + +def untouched_components( + all_nodes: Sequence[str], + covered_nodes: Set[str], + index: GraphIndex, +) -> List[Set[str]]: + remaining = [n for n in all_nodes if n not in covered_nodes] + if not remaining: + return [] + adjacency: Dict[str, Set[str]] = {name: set() for name in remaining} + rem_set = set(remaining) + + for node in remaining: + for out_name in index.node_outputs.get(node, []): + for consumer in index.tensor_to_consumers.get(out_name, []): + if consumer in rem_set: + adjacency[node].add(consumer) + adjacency[consumer].add(node) + for inp_name in index.node_inputs.get(node, []): + producer = index.tensor_to_producer.get(inp_name) + if producer in rem_set: + adjacency[node].add(producer) + adjacency[producer].add(node) + + components: List[Set[str]] = [] + visited: Set[str] = set() + for node in remaining: + if node in visited: + continue + stack = [node] + comp: Set[str] = set() + while stack: + cur = stack.pop() + if cur in visited: + continue + visited.add(cur) + comp.add(cur) + stack.extend(adjacency[cur] - visited) + components.append(comp) + return components + + +def derive_interface(nodes: Set[str], index: GraphIndex) -> (List[str], List[str]): + produced = set() + for node in nodes: + produced.update(index.node_outputs.get(node, [])) + + start: Set[str] = set() + for node in nodes: + for inp in index.node_inputs.get(node, []): + producer = index.tensor_to_producer.get(inp) + if producer is None and inp not in index.initializer_names: + start.add(inp) + elif producer not in nodes and inp not in index.initializer_names: + start.add(inp) + + end: Set[str] = set() + for node in nodes: + for out in index.node_outputs.get(node, []): + consumers = index.tensor_to_consumers.get(out, []) + if not consumers: + if out in index.graph_outputs: + end.add(out) + continue + if any(consumer not in nodes for consumer in consumers): + end.add(out) + end.update(index.graph_outputs & produced) + + if not end and produced: + end = produced.copy() + + return sorted(start), sorted(end) + + +def ordered_specs(specs: Sequence[SubGraphSpec], index: GraphIndex) -> List[SubGraphSpec]: + available = set(index.graph_inputs) | index.initializer_names + pending = list(specs) + ordered: List[SubGraphSpec] = [] + while pending: + progressed = False + for spec in list(pending): + if set(spec.start).issubset(available): + ordered.append(spec) + available.update(spec.end) + pending.remove(spec) + progressed = True + if not progressed: + missing = {spec.label: sorted(set(spec.start) - available) for spec in pending} + raise RuntimeError(f"无法解析子图拓扑，缺少以下张量: {missing}") + return ordered + + +def is_valid_onnx_model(model_path: Path) -> bool: + """检查 ONNX 模型文件是否有效（包含必需的 opset_import）""" + try: + # 检查文件大小 + if model_path.stat().st_size == 0: + logging.warning(f"模型 {model_path.name} 是空文件") + return False + + model = onnx.load(model_path.as_posix(), load_external_data=False) + + # 检查模型是否为 None + if model is None: + logging.warning(f"模型 {model_path.name} 加载后为 None") + return False + + # 检查是否有 graph + if not hasattr(model, 'graph') or model.graph is None: + logging.warning(f"模型 {model_path.name} 缺少 graph") + return False + + # 检查是否有 opset_import + if len(model.opset_import) == 0: + logging.warning(f"模型 {model_path.name} 缺少 opset_import 信息") + return False + + return True + except Exception as e: + logging.warning(f"无法加载模型 {model_path.name}: {e}") + return False + + +def extract_model_file( + source_model: onnx.ModelProto, + spec: SubGraphSpec, + output_dir: Path, + suffix: str, + run_checker: bool, + run_shape_infer: bool, + skip_existing: bool = True, +) -> Path: + head = sanitize(spec.start[0]) if spec.start else "const" + tail = sanitize(spec.end[0]) if spec.end else "out" + filename = f"{spec.label}_{head}_to_{tail}_{suffix}.onnx" + destination = output_dir / filename + + # 如果文件已存在且 skip_existing=True，检查文件是否有效 + if skip_existing and destination.exists(): + if is_valid_onnx_model(destination): + logging.info("跳过已存在的子图 %s (文件: %s)", spec.label, destination.name) + return destination + else: + logging.warning("子图 %s 的文件无效，将重新生成", spec.label) + + # 手动构建子图，不使用 Extractor + # 1. 创建新的空图 + sub_graph = helper.make_graph( + nodes=[], + name=f"{spec.label}_subgraph", + inputs=[], + outputs=[], + initializer=[] + ) + + # 2. 从原始模型复制需要的节点 + # 注意：GraphIndex 为重复/空名字节点生成了唯一的候选名，这里必须使用同样的规则 + node_map = {} + used_names: Set[str] = set() + for idx, node in enumerate(source_model.graph.node): + base = node.name.strip() if node.name else "" + candidate = base or f"node_{idx}" + while candidate in used_names: + candidate = f"{candidate}_{idx}" + used_names.add(candidate) + node_map[candidate] = node + + missing_nodes: List[str] = [] + for node_name in spec.node_names: + target = node_map.get(node_name) + if target is None: + missing_nodes.append(node_name) + continue + new_node = onnx.NodeProto() + new_node.CopyFrom(target) + sub_graph.node.append(new_node) + + if missing_nodes: + logging.warning("子图 %s: 有 %d 个节点未匹配到源模型，将被跳过: %s", spec.label, len(missing_nodes), missing_nodes[:5]) + + # 3. 收集所有需要的张量名称 + node_inputs = set() + node_outputs = set() + for node in sub_graph.node: + for inp in node.input: + if inp: + node_inputs.add(inp) + for out in node.output: + if out: + node_outputs.add(out) + + # 4. 从原始模型收集 value_info + source_value_info_map = {} + for vi in list(source_model.graph.input) + list(source_model.graph.value_info) + list(source_model.graph.output): + source_value_info_map[vi.name] = vi + + # 5. 从原始模型收集 initializers + source_init_map = {init.name: init for init in source_model.graph.initializer} + + # 6. 添加输入：从 spec.start 和需要但不是节点输出的张量 + input_tensor_names = set(spec.start) + for tensor_name in node_inputs: + if tensor_name not in node_outputs and tensor_name not in source_init_map: + input_tensor_names.add(tensor_name) + + for tensor_name in sorted(input_tensor_names): + if tensor_name in source_value_info_map: + vi = onnx.ValueInfoProto() + vi.CopyFrom(source_value_info_map[tensor_name]) + sub_graph.input.append(vi) + else: + vi = helper.make_tensor_value_info(tensor_name, TensorProto.UNDEFINED, None) + sub_graph.input.append(vi) + + # 7. 添加 initializers + for tensor_name in node_inputs: + if tensor_name in source_init_map: + init = onnx.TensorProto() + init.CopyFrom(source_init_map[tensor_name]) + sub_graph.initializer.append(init) + + # 8. 添加输出：从 spec.end + for tensor_name in spec.end: + if tensor_name in source_value_info_map: + vi = onnx.ValueInfoProto() + vi.CopyFrom(source_value_info_map[tensor_name]) + sub_graph.output.append(vi) + else: + vi = helper.make_tensor_value_info(tensor_name, TensorProto.UNDEFINED, None) + sub_graph.output.append(vi) + + # 9. 检查输出是否都有对应的产生节点，如果没有则添加 Identity + for out_name in spec.end: + if out_name not in node_outputs: + # 这个输出没有被任何节点产生 + if out_name in input_tensor_names or out_name in source_init_map: + # 添加 Identity 节点 + identity_node = helper.make_node( + 'Identity', + inputs=[out_name], + outputs=[out_name], + name=f'passthrough_{sanitize(out_name)}' + ) + sub_graph.node.append(identity_node) + logging.info(f"子图 {spec.label}: 为输出 {out_name} 添加 Identity 节点") + else: + logging.error(f"子图 {spec.label}: 输出 {out_name} 无法产生（不在输入/initializer/节点输出中）") + + # 10. 创建模型 + sub_model = helper.make_model(sub_graph) + + # 11. 复制元数据 + sub_model.ir_version = source_model.ir_version + sub_model.producer_name = source_model.producer_name + sub_model.producer_version = source_model.producer_version + sub_model.domain = source_model.domain + sub_model.model_version = source_model.model_version + sub_model.doc_string = source_model.doc_string + + # 12. 复制 opset imports + while len(sub_model.opset_import) > 0: + sub_model.opset_import.pop() + + if len(source_model.opset_import) > 0: + for opset in source_model.opset_import: + opset_import = sub_model.opset_import.add() + opset_import.CopyFrom(opset) + else: + # 如果源模型没有 opset_import，添加默认的 opset + logging.warning(f"源模型缺少 opset_import，为子图 {spec.label} 添加默认 opset 17") + opset_import = sub_model.opset_import.add() + opset_import.domain = "" + opset_import.version = 17 # 使用 ONNX opset 17 作为默认值 + + # 验证 opset_import 是否正确设置 + if len(sub_model.opset_import) == 0: + raise RuntimeError(f"子图 {spec.label} 缺少 opset_import 信息") + + # 13. Shape inference 和 checker + if run_shape_infer: + try: + sub_model = shape_inference.infer_shapes(sub_model) + except Exception as e: + logging.warning(f"子图 {spec.label} shape inference 失败: {e}") + + if run_checker: + try: + checker.check_model(sub_model) + except Exception as e: + logging.warning(f"子图 {spec.label} checker 验证失败: {e}") + + # 14. 保存 + onnx.save(sub_model, destination.as_posix()) + logging.info( + "保存子图 %s (start=%s, end=%s, 节点数=%d, checker=%s, infer_shape=%s)", + destination.name, + spec.start, + spec.end, + len(sub_graph.node), + bool(run_checker), + bool(run_shape_infer), + ) + return destination + + + +def load_numpy_inputs(path: Path, expected_inputs: Iterable[str]) -> Dict[str, np.ndarray]: + suffix = path.suffix.lower() + expected = list(expected_inputs) + if suffix == ".npz": + data = np.load(path, allow_pickle=False) + return {key: data[key] for key in data.files} + if suffix == ".npy": + arr = np.load(path, allow_pickle=True) + if isinstance(arr, np.ndarray) and arr.shape == () and isinstance(arr.item(), dict): + return {str(k): np.asarray(v) for k, v in arr.item().items()} + if isinstance(arr, np.ndarray) and arr.dtype.names: + return {name: arr[name] for name in arr.dtype.names} + if len(expected) == 1: + return {expected[0]: np.asarray(arr)} + raise ValueError("多输入模型需要字典格式的 .npy/.npz 数据。") + raise ValueError("仅支持 .npz 或 .npy 输入数据。") + + +def run_full_model(model_path: Path, feed_dict: Dict[str, np.ndarray], providers: List[str]): + if ort is None: + raise RuntimeError("需要 onnxruntime 才能执行验证。") + session = ort.InferenceSession(model_path.as_posix(), providers=providers) + outputs = session.run(None, feed_dict) + names = [meta.name for meta in session.get_outputs()] + return dict(zip(names, outputs)) + + +def run_split_pipeline( + ordered_subgraphs: Sequence[SubGraphSpec], + feed_dict: Dict[str, np.ndarray], + providers: List[str], +) -> Dict[str, np.ndarray]: + if ort is None: + raise RuntimeError("需要 onnxruntime 才能执行验证。") + tensor_store = dict(feed_dict) + for spec in ordered_subgraphs: + if spec.output_path is None: + raise RuntimeError(f"子图 {spec.label} 尚未生成 ONNX 文件。") + if not spec.output_path.exists(): + raise RuntimeError(f"子图 {spec.label} 的输出文件不存在: {spec.output_path}") + + logging.info("运行子图 %s (输入: %s, 输出: %s)", spec.label, spec.start, spec.end) + session = ort.InferenceSession(spec.output_path.as_posix(), providers=providers) + + # 获取实际的输入名称 + actual_inputs = [inp.name for inp in session.get_inputs()] + logging.debug("子图 %s 实际输入: %s", spec.label, actual_inputs) + + fetch_inputs = {} + for name in actual_inputs: + if name not in tensor_store: + # 尝试从 spec.start 中查找 + if name in spec.start: + logging.warning(f"子图 {spec.label} 缺少输入张量 {name}，尝试从 feed_dict 查找") + if name in feed_dict: + tensor_store[name] = feed_dict[name] + else: + available = list(tensor_store.keys()) + raise KeyError(f"子图 {spec.label} 缺少输入张量 {name}。当前可用: {available}") + else: + available = list(tensor_store.keys()) + raise KeyError(f"子图 {spec.label} 缺少输入张量 {name}。当前可用: {available}") + fetch_inputs[name] = tensor_store[name] + + results = session.run(None, fetch_inputs) + for meta, value in zip(session.get_outputs(), results): + tensor_store[meta.name] = value + logging.debug("子图 %s 产生输出: %s (shape=%s)", spec.label, meta.name, value.shape) + return tensor_store + + +def verify( + model_path: Path, + ordered_subgraphs: Sequence[SubGraphSpec], + feed_dict: Dict[str, np.ndarray], + providers: List[str], + rtol: float, + atol: float, +) -> None: + """验证切分后的子图流水线与原始模型输出是否一致。 + + 验证流程： + 1. 运行完整的原始模型，获得所有输出 + 2. 按拓扑顺序依次运行所有子图（前一个子图的输出作为后一个子图的输入） + 3. 比较原始模型的最终输出与子图流水线的最终输出 + 4. 如果所有输出在指定的误差范围内一致，则验证通过 + """ + logging.info("开始验证：运行原始模型...") + full_outputs = run_full_model(model_path, feed_dict, providers) + logging.info("原始模型运行完成，产生 %d 个输出", len(full_outputs)) + + logging.info("开始验证：运行子图流水线...") + split_store = run_split_pipeline(ordered_subgraphs, feed_dict, providers) + logging.info("子图流水线运行完成") + + logging.info("比较输出...") + for name, ref in full_outputs.items(): + cand = split_store.get(name) + if cand is None: + raise AssertionError(f"切分流水线未产生模型输出 {name}") + if not np.allclose(ref, cand, rtol=rtol, atol=atol): + diff = float(np.max(np.abs(ref - cand))) + raise AssertionError(f"输出 {name} 不匹配，最大偏差 {diff:.3e}") + logging.info("✓ 输出 %s 验证通过 (shape=%s)", name, ref.shape) + logging.info("✓ 切分模型与原始模型输出一致 (rtol=%g, atol=%g)。", rtol, atol) + + +def save_outputs(outputs: Dict[str, np.ndarray], destination: Optional[Path]) -> None: + if destination is None: + return + destination.parent.mkdir(parents=True, exist_ok=True) + np.savez(destination, **outputs) + logging.info("流水线输出已保存至 %s", destination) + + +def load_sub_configs(config_path: Path) -> List[dict]: + with config_path.open("r", encoding="utf-8") as f: + config = json.load(f) + sub_configs = config.get("compiler", {}).get("sub_configs") + if not sub_configs: + sub_configs = config.get("sub_configs") + if not sub_configs: + raise ValueError("配置文件中未找到 sub_configs。") + return sub_configs + + +def main() -> None: + parser = argparse.ArgumentParser(description="根据 tensor 名切分 ONNX 子图") + parser.add_argument("--model", required=True, type=Path, help="原始 ONNX 文件") + parser.add_argument("--config", required=True, type=Path, help="包含 sub_configs 的 JSON") + parser.add_argument("--output-dir", default="./split-onnx", type=Path, help="子模型输出目录") + parser.add_argument("--providers", nargs="*", default=["CPUExecutionProvider"], help="onnxruntime providers 顺序") + parser.add_argument("--verify", action="store_true", help="比较原始模型与流水线输出") + parser.add_argument("--run-pipeline", action="store_true", help="只运行切分流水线并输出结果") + parser.add_argument("--input-data", type=Path, help=".npz/.npy 格式的输入数据") + parser.add_argument("--pipeline-output", type=Path, help="保存流水线输出为 npz") + parser.add_argument("--rtol", type=float, default=1e-4, help="验证 rtol") + parser.add_argument("--atol", type=float, default=1e-5, help="验证 atol") + parser.add_argument("--skip-checker", action="store_true", help="跳过 onnx checker") + parser.add_argument("--skip-shape-infer", action="store_true", help="跳过 shape inference") + parser.add_argument("--skip-existing", action="store_true", help="跳过已存在的子图文件") + parser.add_argument("--log", default="INFO", help="日志等级") + args = parser.parse_args() + + logging.basicConfig(level=getattr(logging, args.log.upper(), logging.INFO)) + + # 必须加载完整数据（包括外部数据文件） + # 虽然模型很大，但这是必须的，因为我们需要将权重完全内嵌到子模型中 + # 否则子模型会引用原始的外部数据路径，导致在新目录下无法找到数据文件 + logging.info("加载完整模型（包括外部数据文件）...这可能需要一些时间和内存") + model = onnx.load(args.model.as_posix(), load_external_data=True) + + # 跳过对原始巨大模型的 checker，只对切分后的小模型进行 checker + # if not args.skip_checker: + # checker.check_model(args.model.as_posix()) + logging.info("跳过对原始模型的 checker（模型过大），将只对切分后的子模型进行验证") + + graph_index = build_graph_index(model) + + sub_configs = load_sub_configs(args.config) + specs: List[SubGraphSpec] = [] + covered_nodes: Set[str] = set() + + for idx, entry in enumerate(sub_configs): + start = [name for name in entry.get("start_tensor_names", []) if name] + end = [name for name in entry.get("end_tensor_names", []) if name] + if not start or not end: + raise ValueError(f"sub_config[{idx}] 缺少 start/end tensor name。") + spec = SubGraphSpec( + label=f"cfg_{idx:02d}", + start=start, + end=end, + node_names=set(), + source="config", + ) + nodes = trace_nodes_between(spec, graph_index) + spec.node_names = nodes + covered_nodes.update(nodes) + specs.append(spec) + + leftovers = untouched_components(graph_index.node_order, covered_nodes, graph_index) + for idx, component in enumerate(leftovers): + start, end = derive_interface(component, graph_index) + if not end: + continue + spec = SubGraphSpec( + label=f"auto_{idx:02d}", + start=start, + end=end, + node_names=component, + source="auto", + ) + specs.append(spec) + logging.info( + "自动补充子图 %s: start=%s end=%s (节点数=%d)", + spec.label, + spec.start, + spec.end, + len(component), + ) + + ordered = ordered_specs(specs, graph_index) + + required_tensors: Set[str] = set() + for spec in ordered: + required_tensors.update(spec.start) + required_tensors.update(spec.end) + ensure_value_infos(model, required_tensors) + + args.output_dir.mkdir(parents=True, exist_ok=True) + + # 首先检查所有现有的子图文件，收集需要重新生成的 + corrupted_specs = [] + for spec in ordered: + head = sanitize(spec.start[0]) if spec.start else "const" + tail = sanitize(spec.end[0]) if spec.end else "out" + filename = f"{spec.label}_{head}_to_{tail}_{spec.source}.onnx" + potential_path = args.output_dir / filename + + if potential_path.exists() and not is_valid_onnx_model(potential_path): + corrupted_specs.append(spec.label) + logging.warning(f"检测到损坏的子图: {spec.label}, 将重新生成") + + if corrupted_specs: + logging.info(f"发现 {len(corrupted_specs)} 个损坏的子图，将重新生成: {corrupted_specs}") + + # 生成或更新子图 + for spec in ordered: + spec.output_path = extract_model_file( + model, + spec, + args.output_dir, + spec.source, + run_checker=not args.skip_checker, + run_shape_infer=not args.skip_shape_infer, + skip_existing=args.skip_existing, + ) + + need_inputs = args.verify or args.run_pipeline + if need_inputs: + if args.input_data is None: + raise ValueError("verify/run-pipeline 模式需要 --input-data 提供输入。") + feed = load_numpy_inputs(args.input_data, graph_index.graph_inputs) + missing_inputs = graph_index.graph_inputs - feed.keys() + if missing_inputs: + raise ValueError(f"输入数据缺少以下张量: {sorted(missing_inputs)}") + else: + feed = {} + + if args.verify: + verify(args.model, ordered, feed, args.providers, args.rtol, args.atol) + elif args.run_pipeline: + outputs = run_split_pipeline(ordered, feed, args.providers) + save_outputs(outputs, args.pipeline_output) + else: + logging.info("子模型已生成，如需验证请添加 --verify 或 --run-pipeline。") + + +if __name__ == "__main__": + """ + python ./scripts/split_onnx_by_subconfig.py \ + --model ./onnx-models/z_image_transformer_body_only_simp_slim.onnx \ + --config ./pulsar2_configs/transformers_subgraph.json \ + --output-dir ./transformers_body_only_split_onnx \ + --verify \ + --input-data ./onnx-calibration-no-controlnet/transformer_inputs_prompt000_step00.npy \ + --providers CPUExecutionProvider + """ + main() diff --git a/VideoX-Fun/scripts/split_onnx_by_subconfig.py.old b/VideoX-Fun/scripts/split_onnx_by_subconfig.py.old new file mode 100644 index 0000000000000000000000000000000000000000..d9db1d2f0ca8a2fc689dfed26dfa44f8b3dc7dd9 --- /dev/null +++ b/VideoX-Fun/scripts/split_onnx_by_subconfig.py.old @@ -0,0 +1,623 @@ +#!/usr/bin/env python3 +"""基于 tensor 名称的通用 ONNX 子图切分工具。 + +相较于 split_quant_onnx_by_subconfigs.py，本脚本额外提供： +1. 为每个子模型执行 onnx checker 与 shape inference（可关闭）。 +2. 支持从 .npz/.npy（包含 dict 或单数组）加载验证数据。 +3. 可用 onnxruntime 串联执行全部子模型，既可校验精度，也可单独输出流水线结果。 +""" +from __future__ import annotations + +import argparse +import json +import logging +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Sequence, Set + +import numpy as np +import onnx +from onnx import TensorProto, checker, helper, shape_inference, utils as onnx_utils + +try: # pragma: no cover - 可选依赖 + import onnxruntime as ort +except ImportError: # pragma: no cover + ort = None + + +@dataclass +class SubGraphSpec: + label: str + start: List[str] + end: List[str] + node_names: Set[str] + source: str + output_path: Optional[Path] = None + + +@dataclass +class GraphIndex: + tensor_to_producer: Dict[str, str] + tensor_to_consumers: Dict[str, List[str]] + node_inputs: Dict[str, List[str]] + node_outputs: Dict[str, List[str]] + graph_inputs: Set[str] + graph_outputs: Set[str] + initializer_names: Set[str] + node_order: List[str] + + +def sanitize(name: str) -> str: + keep = [c if c.isalnum() else "_" for c in name] if name else ["anon"] + sanitized = "".join(keep).strip("_") + return sanitized or "tensor" + + +def build_graph_index(model: onnx.ModelProto) -> GraphIndex: + tensor_to_producer: Dict[str, str] = {} + tensor_to_consumers: Dict[str, List[str]] = defaultdict(list) + node_inputs: Dict[str, List[str]] = {} + node_outputs: Dict[str, List[str]] = {} + node_order: List[str] = [] + + used_names: Set[str] = set() + for idx, node in enumerate(model.graph.node): + base = node.name.strip() if node.name else "" + candidate = base or f"node_{idx}" + while candidate in used_names: + candidate = f"{candidate}_{idx}" + used_names.add(candidate) + node_order.append(candidate) + node_inputs[candidate] = [x for x in node.input if x] + node_outputs[candidate] = [y for y in node.output if y] + for out_name in node_outputs[candidate]: + tensor_to_producer[out_name] = candidate + for inp_name in node_inputs[candidate]: + tensor_to_consumers[inp_name].append(candidate) + + graph_inputs = {vi.name for vi in model.graph.input} + graph_outputs = {vi.name for vi in model.graph.output} + initializer_names = {init.name for init in model.graph.initializer} + + return GraphIndex( + tensor_to_producer=tensor_to_producer, + tensor_to_consumers=tensor_to_consumers, + node_inputs=node_inputs, + node_outputs=node_outputs, + graph_inputs=graph_inputs, + graph_outputs=graph_outputs, + initializer_names=initializer_names, + node_order=node_order, + ) + + +def ensure_value_infos(model: onnx.ModelProto, tensor_names: Iterable[str]) -> None: + existing = {vi.name for vi in model.graph.value_info} + source_map = {} + for vi in list(model.graph.input) + list(model.graph.value_info) + list(model.graph.output): + source_map[vi.name] = vi + + added: List[str] = [] + for name in tensor_names: + if name in existing: + continue + src = source_map.get(name) + if src is not None: + vi = onnx.ValueInfoProto() + vi.CopyFrom(src) + else: + vi = helper.make_tensor_value_info(name, TensorProto.UNDEFINED, None) + model.graph.value_info.append(vi) + existing.add(name) + added.append(name) + if added: + logging.debug("已为以下 tensor 补充 ValueInfo: %s", added) + + +def ensure_extractor_value_infos( + extractor: onnx_utils.Extractor, + tensor_names: Iterable[str], + source_model: onnx.ModelProto, +) -> None: + existing_inputs = {vi.name for vi in extractor.graph.input} + existing_outputs = {vi.name for vi in extractor.graph.output} + existing_vi = {vi.name for vi in extractor.graph.value_info} + source_map = {} + for vi in ( + list(source_model.graph.input) + + list(source_model.graph.value_info) + + list(source_model.graph.output) + ): + source_map[vi.name] = vi + + added: List[str] = [] + for name in tensor_names: + if name in existing_inputs or name in existing_outputs or name in existing_vi: + continue + src = source_map.get(name) + if src is not None: + vi = onnx.ValueInfoProto() + vi.CopyFrom(src) + else: + vi = helper.make_tensor_value_info(name, TensorProto.UNDEFINED, None) + extractor.graph.value_info.append(vi) + extractor.vimap[name] = vi + existing_vi.add(name) + added.append(name) + if added: + logging.debug("Extractor 侧补充 ValueInfo: %s", added) + + +def trace_nodes_between(spec: SubGraphSpec, index: GraphIndex) -> Set[str]: + boundary = set(spec.start) | index.graph_inputs | index.initializer_names + visited_tensors: Set[str] = set() + stack = list(spec.end) + discovered_nodes: Set[str] = set() + + while stack: + tensor = stack.pop() + if tensor in visited_tensors: + continue + visited_tensors.add(tensor) + if tensor in boundary: + continue + producer = index.tensor_to_producer.get(tensor) + if not producer or producer in discovered_nodes: + continue + discovered_nodes.add(producer) + for upstream in index.node_inputs.get(producer, []): + if upstream and upstream not in boundary: + stack.append(upstream) + return discovered_nodes + + +def untouched_components( + all_nodes: Sequence[str], + covered_nodes: Set[str], + index: GraphIndex, +) -> List[Set[str]]: + remaining = [n for n in all_nodes if n not in covered_nodes] + if not remaining: + return [] + adjacency: Dict[str, Set[str]] = {name: set() for name in remaining} + rem_set = set(remaining) + + for node in remaining: + for out_name in index.node_outputs.get(node, []): + for consumer in index.tensor_to_consumers.get(out_name, []): + if consumer in rem_set: + adjacency[node].add(consumer) + adjacency[consumer].add(node) + for inp_name in index.node_inputs.get(node, []): + producer = index.tensor_to_producer.get(inp_name) + if producer in rem_set: + adjacency[node].add(producer) + adjacency[producer].add(node) + + components: List[Set[str]] = [] + visited: Set[str] = set() + for node in remaining: + if node in visited: + continue + stack = [node] + comp: Set[str] = set() + while stack: + cur = stack.pop() + if cur in visited: + continue + visited.add(cur) + comp.add(cur) + stack.extend(adjacency[cur] - visited) + components.append(comp) + return components + + +def derive_interface(nodes: Set[str], index: GraphIndex) -> (List[str], List[str]): + produced = set() + for node in nodes: + produced.update(index.node_outputs.get(node, [])) + + start: Set[str] = set() + for node in nodes: + for inp in index.node_inputs.get(node, []): + producer = index.tensor_to_producer.get(inp) + if producer is None and inp not in index.initializer_names: + start.add(inp) + elif producer not in nodes and inp not in index.initializer_names: + start.add(inp) + + end: Set[str] = set() + for node in nodes: + for out in index.node_outputs.get(node, []): + consumers = index.tensor_to_consumers.get(out, []) + if not consumers: + if out in index.graph_outputs: + end.add(out) + continue + if any(consumer not in nodes for consumer in consumers): + end.add(out) + end.update(index.graph_outputs & produced) + + if not end and produced: + end = produced.copy() + + return sorted(start), sorted(end) + + +def ordered_specs(specs: Sequence[SubGraphSpec], index: GraphIndex) -> List[SubGraphSpec]: + available = set(index.graph_inputs) | index.initializer_names + pending = list(specs) + ordered: List[SubGraphSpec] = [] + while pending: + progressed = False + for spec in list(pending): + if set(spec.start).issubset(available): + ordered.append(spec) + available.update(spec.end) + pending.remove(spec) + progressed = True + if not progressed: + missing = {spec.label: sorted(set(spec.start) - available) for spec in pending} + raise RuntimeError(f"无法解析子图拓扑，缺少以下张量: {missing}") + return ordered + + +def extract_model_file( + source_model: onnx.ModelProto, + spec: SubGraphSpec, + output_dir: Path, + suffix: str, + run_checker: bool, + run_shape_infer: bool, +) -> Path: + head = sanitize(spec.start[0]) if spec.start else "const" + tail = sanitize(spec.end[0]) if spec.end else "out" + filename = f"{spec.label}_{head}_to_{tail}_{suffix}.onnx" + destination = output_dir / filename + + # 手动构建子图，不使用 Extractor + # 1. 创建新的空图 + sub_graph = helper.make_graph( + nodes=[], + name=f"{spec.label}_subgraph", + inputs=[], + outputs=[], + initializer=[] + ) + + # 2. 从原始模型复制需要的节点 + node_map = {node.name or f"node_{i}": node for i, node in enumerate(source_model.graph.node)} + for node_name in spec.node_names: + if node_name in node_map: + new_node = onnx.NodeProto() + new_node.CopyFrom(node_map[node_name]) + sub_graph.node.append(new_node) + + # 3. 收集所有需要的张量名称 + node_inputs = set() + node_outputs = set() + for node in sub_graph.node: + for inp in node.input: + if inp: + node_inputs.add(inp) + for out in node.output: + if out: + node_outputs.add(out) + + # 4. 从原始模型收集 value_info + source_value_info_map = {} + for vi in list(source_model.graph.input) + list(source_model.graph.value_info) + list(source_model.graph.output): + source_value_info_map[vi.name] = vi + + # 5. 从原始模型收集 initializers + source_init_map = {init.name: init for init in source_model.graph.initializer} + + # 6. 添加输入：从 spec.start 和需要但不是节点输出的张量 + input_tensor_names = set(spec.start) + for tensor_name in node_inputs: + if tensor_name not in node_outputs and tensor_name not in source_init_map: + input_tensor_names.add(tensor_name) + + for tensor_name in sorted(input_tensor_names): + if tensor_name in source_value_info_map: + vi = onnx.ValueInfoProto() + vi.CopyFrom(source_value_info_map[tensor_name]) + sub_graph.input.append(vi) + else: + vi = helper.make_tensor_value_info(tensor_name, TensorProto.UNDEFINED, None) + sub_graph.input.append(vi) + + # 7. 添加 initializers + for tensor_name in node_inputs: + if tensor_name in source_init_map: + init = onnx.TensorProto() + init.CopyFrom(source_init_map[tensor_name]) + sub_graph.initializer.append(init) + + # 8. 添加输出：从 spec.end + for tensor_name in spec.end: + if tensor_name in source_value_info_map: + vi = onnx.ValueInfoProto() + vi.CopyFrom(source_value_info_map[tensor_name]) + sub_graph.output.append(vi) + else: + vi = helper.make_tensor_value_info(tensor_name, TensorProto.UNDEFINED, None) + sub_graph.output.append(vi) + + # 9. 检查输出是否都有对应的产生节点，如果没有则添加 Identity + for out_name in spec.end: + if out_name not in node_outputs: + # 这个输出没有被任何节点产生 + if out_name in input_tensor_names or out_name in source_init_map: + # 添加 Identity 节点 + identity_node = helper.make_node( + 'Identity', + inputs=[out_name], + outputs=[out_name], + name=f'passthrough_{sanitize(out_name)}' + ) + sub_graph.node.append(identity_node) + logging.info(f"子图 {spec.label}: 为输出 {out_name} 添加 Identity 节点") + else: + logging.error(f"子图 {spec.label}: 输出 {out_name} 无法产生（不在输入/initializer/节点输出中）") + + # 10. 创建模型 + sub_model = helper.make_model(sub_graph) + + # 11. 复制元数据 + sub_model.ir_version = source_model.ir_version + sub_model.producer_name = source_model.producer_name + sub_model.producer_version = source_model.producer_version + sub_model.domain = source_model.domain + sub_model.model_version = source_model.model_version + sub_model.doc_string = source_model.doc_string + + # 12. 复制 opset imports + while len(sub_model.opset_import) > 0: + sub_model.opset_import.pop() + for opset in source_model.opset_import: + opset_import = sub_model.opset_import.add() + opset_import.CopyFrom(opset) + + # 13. Shape inference 和 checker + if run_shape_infer: + try: + sub_model = shape_inference.infer_shapes(sub_model) + except Exception as e: + logging.warning(f"子图 {spec.label} shape inference 失败: {e}") + + if run_checker: + try: + checker.check_model(sub_model) + except Exception as e: + logging.warning(f"子图 {spec.label} checker 验证失败: {e}") + + # 14. 保存 + onnx.save(sub_model, destination.as_posix()) + logging.info( + "保存子图 %s (start=%s, end=%s, 节点数=%d, checker=%s, infer_shape=%s)", + destination.name, + spec.start, + spec.end, + len(sub_graph.node), + bool(run_checker), + bool(run_shape_infer), + ) + return destination + + + +def load_numpy_inputs(path: Path, expected_inputs: Iterable[str]) -> Dict[str, np.ndarray]: + suffix = path.suffix.lower() + expected = list(expected_inputs) + if suffix == ".npz": + data = np.load(path, allow_pickle=False) + return {key: data[key] for key in data.files} + if suffix == ".npy": + arr = np.load(path, allow_pickle=True) + if isinstance(arr, np.ndarray) and arr.shape == () and isinstance(arr.item(), dict): + return {str(k): np.asarray(v) for k, v in arr.item().items()} + if isinstance(arr, np.ndarray) and arr.dtype.names: + return {name: arr[name] for name in arr.dtype.names} + if len(expected) == 1: + return {expected[0]: np.asarray(arr)} + raise ValueError("多输入模型需要字典格式的 .npy/.npz 数据。") + raise ValueError("仅支持 .npz 或 .npy 输入数据。") + + +def run_full_model(model_path: Path, feed_dict: Dict[str, np.ndarray], providers: List[str]): + if ort is None: + raise RuntimeError("需要 onnxruntime 才能执行验证。") + session = ort.InferenceSession(model_path.as_posix(), providers=providers) + outputs = session.run(None, feed_dict) + names = [meta.name for meta in session.get_outputs()] + return dict(zip(names, outputs)) + + +def run_split_pipeline( + ordered_subgraphs: Sequence[SubGraphSpec], + feed_dict: Dict[str, np.ndarray], + providers: List[str], +) -> Dict[str, np.ndarray]: + if ort is None: + raise RuntimeError("需要 onnxruntime 才能执行验证。") + tensor_store = dict(feed_dict) + for spec in ordered_subgraphs: + if spec.output_path is None: + raise RuntimeError(f"子图 {spec.label} 尚未生成 ONNX 文件。") + session = ort.InferenceSession(spec.output_path.as_posix(), providers=providers) + fetch_inputs = {} + for name in spec.start: + if name not in tensor_store: + raise KeyError(f"子图 {spec.label} 缺少输入张量 {name}。") + fetch_inputs[name] = tensor_store[name] + results = session.run(None, fetch_inputs) + for meta, value in zip(session.get_outputs(), results): + tensor_store[meta.name] = value + return tensor_store + + +def verify( + model_path: Path, + ordered_subgraphs: Sequence[SubGraphSpec], + feed_dict: Dict[str, np.ndarray], + providers: List[str], + rtol: float, + atol: float, +) -> None: + full_outputs = run_full_model(model_path, feed_dict, providers) + split_store = run_split_pipeline(ordered_subgraphs, feed_dict, providers) + for name, ref in full_outputs.items(): + cand = split_store.get(name) + if cand is None: + raise AssertionError(f"切分流水线未产生模型输出 {name}") + if not np.allclose(ref, cand, rtol=rtol, atol=atol): + diff = float(np.max(np.abs(ref - cand))) + raise AssertionError(f"输出 {name} 不匹配，最大偏差 {diff:.3e}") + logging.info("切分模型与原始模型输出一致 (rtol=%g, atol=%g)。", rtol, atol) + + +def save_outputs(outputs: Dict[str, np.ndarray], destination: Optional[Path]) -> None: + if destination is None: + return + destination.parent.mkdir(parents=True, exist_ok=True) + np.savez(destination, **outputs) + logging.info("流水线输出已保存至 %s", destination) + + +def load_sub_configs(config_path: Path) -> List[dict]: + with config_path.open("r", encoding="utf-8") as f: + config = json.load(f) + sub_configs = config.get("compiler", {}).get("sub_configs") + if not sub_configs: + sub_configs = config.get("sub_configs") + if not sub_configs: + raise ValueError("配置文件中未找到 sub_configs。") + return sub_configs + + +def main() -> None: + parser = argparse.ArgumentParser(description="根据 tensor 名切分 ONNX 子图") + parser.add_argument("--model", required=True, type=Path, help="原始 ONNX 文件") + parser.add_argument("--config", required=True, type=Path, help="包含 sub_configs 的 JSON") + parser.add_argument("--output-dir", default="./split-onnx", type=Path, help="子模型输出目录") + parser.add_argument("--providers", nargs="*", default=["CPUExecutionProvider"], help="onnxruntime providers 顺序") + parser.add_argument("--verify", action="store_true", help="比较原始模型与流水线输出") + parser.add_argument("--run-pipeline", action="store_true", help="只运行切分流水线并输出结果") + parser.add_argument("--input-data", type=Path, help=".npz/.npy 格式的输入数据") + parser.add_argument("--pipeline-output", type=Path, help="保存流水线输出为 npz") + parser.add_argument("--rtol", type=float, default=1e-4, help="验证 rtol") + parser.add_argument("--atol", type=float, default=1e-5, help="验证 atol") + parser.add_argument("--skip-checker", action="store_true", help="跳过 onnx checker") + parser.add_argument("--skip-shape-infer", action="store_true", help="跳过 shape inference") + parser.add_argument("--log", default="INFO", help="日志等级") + args = parser.parse_args() + + logging.basicConfig(level=getattr(logging, args.log.upper(), logging.INFO)) + + # 必须加载完整数据（包括外部数据文件） + # 虽然模型很大，但这是必须的，因为我们需要将权重完全内嵌到子模型中 + # 否则子模型会引用原始的外部数据路径，导致在新目录下无法找到数据文件 + logging.info("加载完整模型（包括外部数据文件）...这可能需要一些时间和内存") + model = onnx.load(args.model.as_posix(), load_external_data=True) + + # 跳过对原始巨大模型的 checker，只对切分后的小模型进行 checker + # if not args.skip_checker: + # checker.check_model(args.model.as_posix()) + logging.info("跳过对原始模型的 checker（模型过大），将只对切分后的子模型进行验证") + + graph_index = build_graph_index(model) + + sub_configs = load_sub_configs(args.config) + specs: List[SubGraphSpec] = [] + covered_nodes: Set[str] = set() + + for idx, entry in enumerate(sub_configs): + start = [name for name in entry.get("start_tensor_names", []) if name] + end = [name for name in entry.get("end_tensor_names", []) if name] + if not start or not end: + raise ValueError(f"sub_config[{idx}] 缺少 start/end tensor name。") + spec = SubGraphSpec( + label=f"cfg_{idx:02d}", + start=start, + end=end, + node_names=set(), + source="config", + ) + nodes = trace_nodes_between(spec, graph_index) + spec.node_names = nodes + covered_nodes.update(nodes) + specs.append(spec) + + leftovers = untouched_components(graph_index.node_order, covered_nodes, graph_index) + for idx, component in enumerate(leftovers): + start, end = derive_interface(component, graph_index) + if not end: + continue + spec = SubGraphSpec( + label=f"auto_{idx:02d}", + start=start, + end=end, + node_names=component, + source="auto", + ) + specs.append(spec) + logging.info( + "自动补充子图 %s: start=%s end=%s (节点数=%d)", + spec.label, + spec.start, + spec.end, + len(component), + ) + + ordered = ordered_specs(specs, graph_index) + + required_tensors: Set[str] = set() + for spec in ordered: + required_tensors.update(spec.start) + required_tensors.update(spec.end) + ensure_value_infos(model, required_tensors) + + args.output_dir.mkdir(parents=True, exist_ok=True) + for spec in ordered: + spec.output_path = extract_model_file( + model, + spec, + args.output_dir, + spec.source, + run_checker=not args.skip_checker, + run_shape_infer=not args.skip_shape_infer, + ) + + need_inputs = args.verify or args.run_pipeline + if need_inputs: + if args.input_data is None: + raise ValueError("verify/run-pipeline 模式需要 --input-data 提供输入。") + feed = load_numpy_inputs(args.input_data, graph_index.graph_inputs) + missing_inputs = graph_index.graph_inputs - feed.keys() + if missing_inputs: + raise ValueError(f"输入数据缺少以下张量: {sorted(missing_inputs)}") + else: + feed = {} + + if args.verify: + verify(args.model, ordered, feed, args.providers, args.rtol, args.atol) + elif args.run_pipeline: + outputs = run_split_pipeline(ordered, feed, args.providers) + save_outputs(outputs, args.pipeline_output) + else: + logging.info("子模型已生成，如需验证请添加 --verify 或 --run-pipeline。") + + +if __name__ == "__main__": + """ + python ./scripts/split_onnx_by_subconfig.py \ + --model ./onnx-models/z_image_transformer_body_only_simp_slim.onnx \ + --config ./pulsar2_configs/transformers_subgraph.json \ + --output-dir ./transformers_body_only_split_onnx \ + --verify \ + --input-data ./onnx-calibration-no-controlnet/transformer_inputs_prompt000_step00.npy \ + --providers CPUExecutionProvider + """ + main() diff --git a/VideoX-Fun/scripts/split_quant_onnx_by_subconfigs.py b/VideoX-Fun/scripts/split_quant_onnx_by_subconfigs.py new file mode 100644 index 0000000000000000000000000000000000000000..8cba999cf53144c2c4c140514e8fea6c95104038 --- /dev/null +++ b/VideoX-Fun/scripts/split_quant_onnx_by_subconfigs.py @@ -0,0 +1,414 @@ +#!/usr/bin/env python3 +"""Split an ONNX graph into smaller sub-models driven by sub_config rules. + +This script reads a JSON config file (matching the pulsar2 sub_config layout), +extracts the requested subgraphs, and optionally emits any leftover parts of the +model as independent ONNX graphs. A verification utility can run the original +model and the stitched micro-model pipeline to make sure their outputs match. +""" +from __future__ import annotations + +import argparse +import json +import logging +from collections import defaultdict, deque +from dataclasses import dataclass, field +from pathlib import Path +from typing import Dict, Iterable, List, Optional, Sequence, Set + +import numpy as np +import onnx +from onnx import utils as onnx_utils + +try: + import onnxruntime as ort +except ImportError: # pragma: no cover - optional dependency + ort = None + + +@dataclass +class SubGraphSpec: + """Describes a single subgraph to extract from the full model.""" + + label: str + start: List[str] + end: List[str] + node_names: Set[str] + source: str + output_path: Optional[Path] = None + + +@dataclass +class GraphIndex: + """Caches helpful lookups for traversing an ONNX graph.""" + + tensor_to_producer: Dict[str, str] + tensor_to_consumers: Dict[str, List[str]] + node_inputs: Dict[str, List[str]] + node_outputs: Dict[str, List[str]] + graph_inputs: Set[str] + graph_outputs: Set[str] + initializer_names: Set[str] + node_order: List[str] + + +def sanitize(name: str) -> str: + keep = [c if c.isalnum() else "_" for c in name] if name else ["anon"] + sanitized = "".join(keep).strip("_") + return sanitized or "tensor" + + +def build_graph_index(model: onnx.ModelProto) -> GraphIndex: + tensor_to_producer: Dict[str, str] = {} + tensor_to_consumers: Dict[str, List[str]] = defaultdict(list) + node_inputs: Dict[str, List[str]] = {} + node_outputs: Dict[str, List[str]] = {} + node_order: List[str] = [] + + used_names: Set[str] = set() + for idx, node in enumerate(model.graph.node): + base = node.name.strip() if node.name else "" + candidate = base or f"node_{idx}" + while candidate in used_names: + candidate = f"{candidate}_{idx}" + used_names.add(candidate) + node_name = candidate + node_order.append(node_name) + node_inputs[node_name] = [x for x in node.input if x] + node_outputs[node_name] = [y for y in node.output if y] + for out_name in node_outputs[node_name]: + tensor_to_producer[out_name] = node_name + for inp_name in node_inputs[node_name]: + tensor_to_consumers[inp_name].append(node_name) + + graph_inputs = {vi.name for vi in model.graph.input} + graph_outputs = {vi.name for vi in model.graph.output} + initializer_names = {init.name for init in model.graph.initializer} + + return GraphIndex( + tensor_to_producer=tensor_to_producer, + tensor_to_consumers=tensor_to_consumers, + node_inputs=node_inputs, + node_outputs=node_outputs, + graph_inputs=graph_inputs, + graph_outputs=graph_outputs, + initializer_names=initializer_names, + node_order=node_order, + ) + + +def trace_nodes_between( + spec: SubGraphSpec, + index: GraphIndex, +) -> Set[str]: + boundary = set(spec.start) | index.graph_inputs | index.initializer_names + visited_tensors: Set[str] = set() + stack = list(spec.end) + discovered_nodes: Set[str] = set() + + while stack: + tensor = stack.pop() + if tensor in visited_tensors: + continue + visited_tensors.add(tensor) + if tensor in boundary: + continue + producer = index.tensor_to_producer.get(tensor) + if not producer: + continue + if producer in discovered_nodes: + continue + discovered_nodes.add(producer) + for upstream in index.node_inputs.get(producer, []): + if upstream and upstream not in boundary: + stack.append(upstream) + return discovered_nodes + + +def untouched_components( + all_nodes: Sequence[str], + covered_nodes: Set[str], + index: GraphIndex, +) -> List[Set[str]]: + remaining = [n for n in all_nodes if n not in covered_nodes] + if not remaining: + return [] + adjacency: Dict[str, Set[str]] = {name: set() for name in remaining} + rem_set = set(remaining) + for node in remaining: + for out_name in index.node_outputs.get(node, []): + for consumer in index.tensor_to_consumers.get(out_name, []): + if consumer in rem_set: + adjacency[node].add(consumer) + adjacency[consumer].add(node) + for inp_name in index.node_inputs.get(node, []): + producer = index.tensor_to_producer.get(inp_name) + if producer in rem_set: + adjacency[node].add(producer) + adjacency[producer].add(node) + + components: List[Set[str]] = [] + visited: Set[str] = set() + for node in remaining: + if node in visited: + continue + stack = [node] + comp: Set[str] = set() + while stack: + cur = stack.pop() + if cur in visited: + continue + visited.add(cur) + comp.add(cur) + stack.extend(adjacency[cur] - visited) + components.append(comp) + return components + + +def derive_interface( + nodes: Set[str], + index: GraphIndex, +) -> (List[str], List[str]): + produced = set() + for node in nodes: + produced.update(index.node_outputs.get(node, [])) + + start: Set[str] = set() + for node in nodes: + for inp in index.node_inputs.get(node, []): + producer = index.tensor_to_producer.get(inp) + if producer is None and inp not in index.initializer_names: + start.add(inp) + elif producer not in nodes and inp not in index.initializer_names: + start.add(inp) + + end: Set[str] = set() + for node in nodes: + for out in index.node_outputs.get(node, []): + consumers = index.tensor_to_consumers.get(out, []) + if not consumers: + if out in index.graph_outputs: + end.add(out) + continue + if any(consumer not in nodes for consumer in consumers): + end.add(out) + end.update(index.graph_outputs & produced) + + if not end and produced: + end = produced.copy() + + return sorted(start), sorted(end) + + +def extract_model_file( + model_path: Path, + spec: SubGraphSpec, + output_dir: Path, + suffix: str, +) -> Path: + head = sanitize(spec.start[0]) if spec.start else "const" + tail = sanitize(spec.end[0]) if spec.end else "out" + filename = f"{spec.label}_{head}_to_{tail}_{suffix}.onnx" + destination = output_dir / filename + onnx_utils.extract_model( + model_path.as_posix(), + destination.as_posix(), + input_names=spec.start, + output_names=spec.end, + check_model=False, + ) + logging.info("Saved %s (start=%s, end=%s)", destination.name, spec.start, spec.end) + return destination + + +def ordered_specs( + specs: Sequence[SubGraphSpec], + index: GraphIndex, +) -> List[SubGraphSpec]: + available = set(index.graph_inputs) | index.initializer_names + pending = list(specs) + ordered: List[SubGraphSpec] = [] + while pending: + progressed = False + for spec in list(pending): + if set(spec.start).issubset(available): + ordered.append(spec) + available.update(spec.end) + pending.remove(spec) + progressed = True + if not progressed: + missing = {spec.label: sorted(set(spec.start) - available) for spec in pending} + raise RuntimeError( + "无法解析子图拓扑，缺少以下张量: %s" % missing + ) + return ordered + + +def run_full_model(model_path: Path, feed_dict: Dict[str, np.ndarray], providers: List[str]): + if ort is None: + raise RuntimeError("需要 onnxruntime 才能执行验证。") + session = ort.InferenceSession(model_path.as_posix(), providers=providers) + outputs = session.run(None, feed_dict) + names = [meta.name for meta in session.get_outputs()] + return dict(zip(names, outputs)) + + +def run_split_pipeline( + ordered_subgraphs: Sequence[SubGraphSpec], + feed_dict: Dict[str, np.ndarray], + providers: List[str], +) -> Dict[str, np.ndarray]: + if ort is None: + raise RuntimeError("需要 onnxruntime 才能执行验证。") + tensor_store = dict(feed_dict) + for spec in ordered_subgraphs: + if spec.output_path is None: + raise RuntimeError(f"子图 {spec.label} 尚未生成 ONNX 文件。") + session = ort.InferenceSession(spec.output_path.as_posix(), providers=providers) + fetch_inputs = {} + for name in spec.start: + if name not in tensor_store: + raise KeyError( + f"子图 {spec.label} 缺少输入张量 {name}，请确认切分顺序。" + ) + fetch_inputs[name] = tensor_store[name] + results = session.run(None, fetch_inputs) + for meta, value in zip(session.get_outputs(), results): + tensor_store[meta.name] = value + return tensor_store + + +def verify( + model_path: Path, + ordered_subgraphs: Sequence[SubGraphSpec], + feed_dict: Dict[str, np.ndarray], + providers: List[str], + rtol: float, + atol: float, +) -> None: + full_outputs = run_full_model(model_path, feed_dict, providers) + split_store = run_split_pipeline(ordered_subgraphs, feed_dict, providers) + for name, ref in full_outputs.items(): + cand = split_store.get(name) + if cand is None: + raise AssertionError(f"切分流水线未产生模型输出 {name}") + if not np.allclose(ref, cand, rtol=rtol, atol=atol): + diff = np.max(np.abs(ref - cand)) + raise AssertionError( + f"输出 {name} 不匹配，最大偏差 {diff:.3e}" + ) + logging.info("切分模型与原始模型输出一致 (rtol=%g, atol=%g)。", rtol, atol) + + +def load_npz_inputs(npz_path: Path) -> Dict[str, np.ndarray]: + data = np.load(npz_path, allow_pickle=False) + return {key: data[key] for key in data.files} + + +def main() -> None: + parser = argparse.ArgumentParser(description="根据 sub_config 切分 ONNX 模型。") + parser.add_argument("--model", required=True, type=Path, help="原始 ONNX 路径") + parser.add_argument("--config", required=True, type=Path, help="pulsar2 配置 JSON") + parser.add_argument("--output-dir", required=False, default="./split-onnx", type=Path, help="保存子模型的目录") + parser.add_argument( + "--verify", + action="store_true", + help="生成后立即用 onnxruntime 校验输出是否一致", + ) + parser.add_argument( + "--input-npz", + type=Path, + help="包含模型所有输入张量的 npz 文件 (verify 模式需要)", + ) + parser.add_argument( + "--providers", + nargs="*", + default=["CPUExecutionProvider"], + help="onnxruntime 推理后端顺序", + ) + parser.add_argument("--rtol", type=float, default=1e-4, help="验证 rtol") + parser.add_argument("--atol", type=float, default=1e-5, help="验证 atol") + parser.add_argument("--log", default="INFO", help="日志等级") + args = parser.parse_args() + + logging.basicConfig(level=getattr(logging, args.log.upper(), logging.INFO)) + + model = onnx.load(args.model.as_posix()) + graph_index = build_graph_index(model) + + with args.config.open("r", encoding="utf-8") as f: + config = json.load(f) + + sub_configs = config.get("compiler", {}).get("sub_configs", []) + if not sub_configs: + raise ValueError("配置文件中未找到 compiler.sub_configs。") + + specs: List[SubGraphSpec] = [] + covered_nodes: Set[str] = set() + + for idx, entry in enumerate(sub_configs): + start = [name for name in entry.get("start_tensor_names", []) if name] + end = [name for name in entry.get("end_tensor_names", []) if name] + if not start or not end: + raise ValueError(f"sub_config[{idx}] 缺少 start/end tensor name。") + spec = SubGraphSpec( + label=f"cfg_{idx:02d}", + start=start, + end=end, + node_names=set(), + source="config", + ) + nodes = trace_nodes_between(spec, graph_index) + spec.node_names = nodes + covered_nodes.update(nodes) + specs.append(spec) + + leftovers = untouched_components(graph_index.node_order, covered_nodes, graph_index) + for idx, component in enumerate(leftovers): + start, end = derive_interface(component, graph_index) + if not end: + logging.warning("自动发现的剩余子图 %d 没有输出，跳过。", idx) + continue + spec = SubGraphSpec( + label=f"auto_{idx:02d}", + start=start, + end=end, + node_names=component, + source="auto", + ) + specs.append(spec) + logging.info( + "自动补充子图 %s: start=%s end=%s (节点数=%d)", + spec.label, + spec.start, + spec.end, + len(component), + ) + + ordered = ordered_specs(specs, graph_index) + + args.output_dir.mkdir(parents=True, exist_ok=True) + for spec in ordered: + spec.output_path = extract_model_file(args.model, spec, args.output_dir, spec.source) + + if args.verify: + if args.input_npz is None: + raise ValueError("verify 模式需要 --input-npz 提供输入数据。") + feed = load_npz_inputs(args.input_npz) + missing_inputs = graph_index.graph_inputs - feed.keys() + if missing_inputs: + raise ValueError(f"npz 中缺少以下模型输入: {sorted(missing_inputs)}") + verify(args.model, ordered, feed, args.providers, args.rtol, args.atol) + + +if __name__ == "__main__": + """ + 用法示例: + python python/VideoX-Fun/scripts/split_onnx_by_subconfigs.py \ + --model /path/to/full.onnx \ + --config python/VideoX-Fun/pulsar2_configs/transformers_subgraph.json \ + --output-dir /tmp/sliced_models \ + --verify \ + --input-npz /path/to/inputs.npz + """ + main() diff --git a/VideoX-Fun/scripts/verify_and_fix_subgraphs.py b/VideoX-Fun/scripts/verify_and_fix_subgraphs.py new file mode 100644 index 0000000000000000000000000000000000000000..fc1f4307d6ab56689a9aabc2a99e49ded184d1cb --- /dev/null +++ b/VideoX-Fun/scripts/verify_and_fix_subgraphs.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +""" +验证并修复损坏的子图文件 +""" +import argparse +import logging +from pathlib import Path +import onnx + + +def is_valid_onnx_model(model_path: Path) -> tuple[bool, str]: + """检查 ONNX 模型文件是否有效，返回 (是否有效, 错误信息)""" + try: + # 检查文件大小 + if model_path.stat().st_size == 0: + return False, "文件为空" + + model = onnx.load(model_path.as_posix(), load_external_data=False) + + # 检查模型是否为 None + if model is None: + return False, "模型加载后为 None" + + # 检查是否有 graph + if not hasattr(model, 'graph') or model.graph is None: + return False, "缺少 graph" + + # 检查是否有 opset_import + if len(model.opset_import) == 0: + return False, "缺少 opset_import 信息" + + return True, "OK" + except Exception as e: + return False, str(e) + + +def main(): + parser = argparse.ArgumentParser(description="验证子图文件的有效性") + parser.add_argument("--dir", type=Path, required=True, help="包含子图的目录") + parser.add_argument("--remove-corrupted", action="store_true", help="删除损坏的文件") + parser.add_argument("--log", default="INFO", help="日志等级") + args = parser.parse_args() + + logging.basicConfig( + level=getattr(logging, args.log.upper(), logging.INFO), + format='%(levelname)s: %(message)s' + ) + + if not args.dir.exists(): + logging.error(f"目录不存在: {args.dir}") + return 1 + + # 获取所有 .onnx 文件 + onnx_files = sorted(args.dir.glob("*.onnx")) + + if not onnx_files: + logging.warning(f"目录 {args.dir} 中没有找到 ONNX 文件") + return 0 + + logging.info(f"找到 {len(onnx_files)} 个 ONNX 文件") + logging.info("=" * 80) + + valid_count = 0 + corrupted_files = [] + + for idx, onnx_file in enumerate(onnx_files, 1): + is_valid, error_msg = is_valid_onnx_model(onnx_file) + + if is_valid: + logging.info(f"[{idx}/{len(onnx_files)}] ✓ {onnx_file.name}") + valid_count += 1 + else: + logging.error(f"[{idx}/{len(onnx_files)}] ✗ {onnx_file.name}: {error_msg}") + corrupted_files.append(onnx_file) + + # 总结 + logging.info("=" * 80) + logging.info(f"总计: {len(onnx_files)} 个文件") + logging.info(f"有效: {valid_count} 个文件") + logging.info(f"损坏: {len(corrupted_files)} 个文件") + + if corrupted_files: + logging.info("\n损坏的文件列表:") + for f in corrupted_files: + logging.info(f" - {f.name}") + + if args.remove_corrupted: + logging.info("\n删除损坏的文件...") + for f in corrupted_files: + f.unlink() + logging.info(f" 已删除: {f.name}") + logging.info(f"已删除 {len(corrupted_files)} 个损坏的文件") + else: + logging.info("\n使用 --remove-corrupted 选项来删除这些损坏的文件") + logging.info("然后重新运行 split_onnx_by_subconfig.py 来重新生成") + + return 0 if len(corrupted_files) == 0 else 1 + + +if __name__ == "__main__": + """ + python ./scripts/verify_and_fix_subgraphs.py \ + --dir ./transformers_body_only_1728_992_split_onnx + """ + exit(main()) diff --git a/VideoX-Fun/scripts/wan2.1/README_TRAIN.md b/VideoX-Fun/scripts/wan2.1/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..63c82f830f2deaba6df0d2aea917c2674bb37cff --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/README_TRAIN.md @@ -0,0 +1,246 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan T2V without deepspeed: + +Wan without DeepSpeed is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +Wan T2V with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="normal" \ + --trainable_modules "." +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan T2V with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="normal" \ + --trainable_modules "." +``` + +Wan T2V with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="normal" \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1/README_TRAIN_DISTILL.md b/VideoX-Fun/scripts/wan2.1/README_TRAIN_DISTILL.md new file mode 100644 index 0000000000000000000000000000000000000000..4adb31d569b8b70b99a3b6571a322bbc387e5a1c --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/README_TRAIN_DISTILL.md @@ -0,0 +1,247 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use DeepSpeed and FSDP in Wan distill, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan distill without deepspeed: + +Wan distill without DeepSpeed and FSDP is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +Wan distill with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan distill with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` + +Wan distill with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1/README_TRAIN_DISTILL_LORA.md b/VideoX-Fun/scripts/wan2.1/README_TRAIN_DISTILL_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..3c3c6eabe1398394b4d0dff79a633fa1183eb159 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/README_TRAIN_DISTILL_LORA.md @@ -0,0 +1,253 @@ +## Lora Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use DeepSpeed and FSDP in Wan distill, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan distill without deepspeed: + +Wan distill without DeepSpeed and FSDP is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +Wan distill with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is not compatible with PEFT. + +Wan distill with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --low_vram +``` + +Wan distill with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1/README_TRAIN_LORA.md b/VideoX-Fun/scripts/wan2.1/README_TRAIN_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..345ffb9eefc468aa1ae3cc7ea30da02543009c4f --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/README_TRAIN_LORA.md @@ -0,0 +1,243 @@ +## Lora Training Code + +We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan T2V without deepspeed: + +Wan without DeepSpeed is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +Wan T2V with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is not compatible with PEFT. + +Wan T2V with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram +``` + +Wan T2V with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1/train.py b/VideoX-Fun/scripts/wan2.1/train.py new file mode 100644 index 0000000000000000000000000000000000000000..9e0a954b60369de265cdd22e1299629d1e57553f --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/train.py @@ -0,0 +1,1938 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedStateDictConfig, ShardedOptimStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.pipeline import WanPipeline, WanI2VPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = WanI2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + else: + pipeline = WanPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--selective_ac", + type=float, + default=0, + help="Rate for transformer block apply checkpointing.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Clip Image Encoder + if args.train_mode != "normal": + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + # Get Transformer + transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + if args.train_mode != "normal": + clip_image_encoder.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=WanTransformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = WanTransformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = WanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=WanTransformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = WanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + elif args.selective_ac > 0: + from videox_fun.utils.ac_handle import apply_checkpointing, partial + from videox_fun.models.wan_transformer3d import WanAttentionBlock + apply_selective_ac = partial(apply_checkpointing, block=WanAttentionBlock) + apply_selective_ac(transformer3d, p=args.selective_ac) + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if args.train_mode != "normal": + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + if args.train_mode != "normal": + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if args.train_mode != "normal": + clip_image_encoder.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "normal": + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask = resize_mask(1 - mask, latents) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + clip_context.append(_clip_context) + clip_context = torch.cat(clip_context) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + if args.train_mode != "normal": + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1/train.sh b/VideoX-Fun/scripts/wan2.1/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..0afb828dc93d633846323ed2ea82cc1395f64242 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/train.sh @@ -0,0 +1,85 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." + +# # Training command for I2V +# export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" +# export DATASET_NAME="datasets/internal_datasets/" +# export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +# NCCL_DEBUG=INFO + +# accelerate launch --mixed_precision="bf16" scripts/wan2.1/train.py \ +# --config_path="config/wan2.1/wan_civitai.yaml" \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --train_data_dir=$DATASET_NAME \ +# --train_data_meta=$DATASET_META_NAME \ +# --image_sample_size=640 \ +# --video_sample_size=640 \ +# --token_sample_size=640 \ +# --video_sample_stride=2 \ +# --video_sample_n_frames=81 \ +# --train_batch_size=1 \ +# --video_repeat=1 \ +# --gradient_accumulation_steps=1 \ +# --dataloader_num_workers=8 \ +# --num_train_epochs=100 \ +# --checkpointing_steps=50 \ +# --learning_rate=2e-05 \ +# --lr_scheduler="constant_with_warmup" \ +# --lr_warmup_steps=100 \ +# --seed=42 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --vae_mini_batch=1 \ +# --max_grad_norm=0.05 \ +# --random_hw_adapt \ +# --training_with_video_token_length \ +# --enable_bucket \ +# --uniform_sampling \ +# --low_vram \ +# --train_mode="i2v" \ +# --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1/train_distill.py b/VideoX-Fun/scripts/wan2.1/train_distill.py new file mode 100644 index 0000000000000000000000000000000000000000..2123167422f18d944762f4465f35c4fbf0f0c3f9 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/train_distill.py @@ -0,0 +1,2323 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import contextlib +import gc +import json +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.distributed as dist +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import Dataset, RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + BatchSampler, RandomSampler, + get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + TextDataset, get_random_mask) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.pipeline import WanI2VPipeline, WanPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = WanI2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + else: + pipeline = WanPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + num_inference_steps = len(args.denoising_step_list), + guidance_scale = 1.0, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + num_inference_steps = len(args.denoising_step_list), + guidance_scale = 1.0, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + num_inference_steps = len(args.denoising_step_list), + guidance_scale = 1.0, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--negative_prompt", + type=str, + default="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + help=("The negative prompt of cfg distill"), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--learning_rate_critic", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--gen_update_interval", + type=int, + default=5, + help="The ratio to update transformer3d.", + ) + parser.add_argument( + "--fake_guidance_scale", + type=float, + default=0.0, + help="The cfg scale for fake iscore.", + ) + parser.add_argument( + "--real_guidance_scale", + type=float, + default=6.0, + help="The cfg scale for real score.", + ) + parser.add_argument( + '--denoising_step_indices_list', + nargs='+', + default=[1000, 750, 500, 250], + help="The denoising step list.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + accelerator_fake_score_transformer3d = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Clip Image Encoder + if args.train_mode != "normal": + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + # Get Transformer + generator_transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + real_score_transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + fake_score_transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set generator_transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + generator_transformer3d.requires_grad_(False) + real_score_transformer3d.requires_grad_(False) + fake_score_transformer3d.requires_grad_(False) + if args.train_mode != "normal": + clip_image_encoder.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = generator_transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + generator_transformer3d.train() + fake_score_transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in generator_transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + for name, param in fake_score_transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = WanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + accelerator_fake_score_transformer3d.register_save_state_pre_hook(save_model_hook) + accelerator_fake_score_transformer3d.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + generator_transformer3d.enable_gradient_checkpointing() + fake_score_transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, generator_transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in generator_transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + fake_trainable_params = list(filter(lambda p: p.requires_grad, fake_score_transformer3d.parameters())) + fake_trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in fake_score_transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + fake_trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + fake_trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + critic_optimizer = optimizer_cls( + fake_trainable_params_optim, + lr=args.learning_rate_critic, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + critic_optimizer = optimizer_cls( + fake_trainable_params_optim, + lr=args.learning_rate_critic, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + if args.train_mode != "normal": + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + else: + train_dataset = TextDataset( + args.train_data_meta + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + if args.enable_bucket and args.train_mode != "normal": + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to("cpu"), attention_mask=prompt_attention_mask.to("cpu"))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to("cpu"), attention_mask=neg_prompt_attention_mask.to("cpu"))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + new_examples['neg_encoder_attention_mask'] = neg_prompt_ids.attention_mask + new_examples['neg_encoder_hidden_states'] = neg_prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + elif args.train_mode == "normal": + def collate_fn(examples): + new_examples = {} + new_examples["text"] = [] + for example in examples: + new_examples["text"].append(example["text"]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to("cpu"), attention_mask=prompt_attention_mask.to("cpu"))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to("cpu"), attention_mask=neg_prompt_attention_mask.to("cpu"))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + new_examples['neg_encoder_attention_mask'] = neg_prompt_ids.attention_mask + new_examples['neg_encoder_hidden_states'] = neg_prompt_embeds + + return new_examples + + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = BatchSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), batch_size=args.train_batch_size, drop_last=True) + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + fake_score_lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + generator_transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + generator_transformer3d, optimizer, train_dataloader, lr_scheduler + ) + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler= accelerator_fake_score_transformer3d.prepare( + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler + ) + if fsdp_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + real_score_transformer3d = shard_fn(real_score_transformer3d) + if fsdp_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + real_score_transformer3d.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu") + if args.train_mode != "normal": + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + fake_score_path = os.path.join(path, "fake_score") + accelerator.load_state(os.path.join(args.output_dir, path)) + accelerator_fake_score_transformer3d.load_state(os.path.join(args.output_dir, fake_score_path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + denoising_step_list = noise_scheduler.timesteps[args.train_sampling_steps - torch.tensor(args.denoising_step_indices_list)] + + for epoch in range(first_epoch, args.num_train_epochs): + train_dmd_loss = 0.0 + train_denoising_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if args.train_mode != "normal" and epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + if args.train_mode != "normal": + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + batch['neg_encoder_hidden_states'] = torch.tile(batch['neg_encoder_hidden_states'], (4, 1, 1)) + batch['neg_encoder_attention_mask'] = torch.tile(batch['neg_encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + batch['neg_encoder_hidden_states'] = torch.tile(batch['neg_encoder_hidden_states'], (2, 1, 1)) + batch['neg_encoder_attention_mask'] = torch.tile(batch['neg_encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + clip_image_encoder.to(accelerator.device) + real_score_transformer3d = real_score_transformer3d.to("cpu") + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask = resize_mask(1 - mask, mask_latents) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + clip_context.append(_clip_context) + clip_context = torch.cat(clip_context) + + target_shape = mask_latents.size() + else: + text = batch['text'] + if args.fix_sample_size is not None: + local_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + num_frames = args.video_sample_n_frames + else: + if args.random_hw_adapt and args.training_with_video_token_length: + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + if rng is None: + local_length = np.random.choice(list(length_to_frame_num.keys())) + else: + local_length = rng.choice(list(length_to_frame_num.keys())) + num_frames = length_to_frame_num[local_length] + + aspect_ratio_sample_size = {key : [x / 512 * local_length for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if rng is None: + aspect_ratio_key = np.random.choice(list(aspect_ratio_sample_size.keys())) + else: + aspect_ratio_key = rng.choice(list(aspect_ratio_sample_size.keys())) + local_sample_size = aspect_ratio_sample_size[aspect_ratio_key] + else: + num_frames = args.video_sample_n_frames + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if rng is None: + aspect_ratio_key = np.random.choice(list(aspect_ratio_sample_size.keys())) + else: + aspect_ratio_key = rng.choice(list(aspect_ratio_sample_size.keys())) + local_sample_size = aspect_ratio_sample_size[aspect_ratio_key] + local_sample_size = [int(x / 16) * 16 for x in local_sample_size] + + target_shape = ( + len(text), + vae.latent_channels, + int((num_frames - 1) // vae.temporal_compression_ratio + 1), + int(local_sample_size[0] // vae.spatial_compression_ratio), + int(local_sample_size[1] // vae.spatial_compression_ratio), + ) + + if args.low_vram: + vae.to('cpu') + real_score_transformer3d = real_score_transformer3d.to("cpu") + if args.train_mode != "normal": + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=accelerator.device) + neg_prompt_embeds = batch['neg_encoder_hidden_states'].to(device=accelerator.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(accelerator.device), attention_mask=prompt_attention_mask.to(accelerator.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to(accelerator.device), attention_mask=neg_prompt_attention_mask.to(accelerator.device))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + if args.low_vram: + generator_transformer3d = generator_transformer3d.to(accelerator.device) + real_score_transformer3d = real_score_transformer3d.to(accelerator.device) + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + with accelerator.accumulate(generator_transformer3d): + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + + step_indices = [ + torch.argmin(torch.abs(schedule_timesteps - t)).item() + for t in timesteps + ] + step_indices = torch.tensor(step_indices, device=accelerator.device) + sigma = sigmas[step_indices].flatten() + + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + def add_noise(latents, noise, timesteps): + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + return (1.0 - sigmas) * latents + sigmas * noise + + def generate_and_sync_list(num_denoising_steps, device): + indices = torch.randint(low=0, high=num_denoising_steps, size=(1,), generator=torch_rng, device=device) + if dist.is_initialized(): + dist.broadcast(indices, src=0) + return indices.tolist() + + def convert_flow_pred_to_x0( + scheduler, + flow_pred: torch.Tensor, + xt: torch.Tensor, + timestep: torch.Tensor + ) -> torch.Tensor: + """ + Convert flow matching's prediction to x0 prediction. + Supports both 4D [B, C, H, W] and 5D [B, C, F, H, W] inputs. + """ + original_dtype = flow_pred.dtype + device = flow_pred.device + + flow_pred = flow_pred.double() + xt = xt.double() + timesteps = scheduler.timesteps.to(device).double() + sigmas = scheduler.sigmas.to(device).double() + timestep = timestep.to(device).double() + + timestep_id = torch.argmin((timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1) + sigma_t = sigmas[timestep_id] + + ndim = flow_pred.ndim + if ndim == 4: + sigma_t = sigma_t.view(-1, 1, 1, 1) + elif ndim == 5: + sigma_t = sigma_t.view(-1, 1, 1, 1, 1) + else: + raise ValueError(f"Expected 4D or 5D input, got {ndim}D tensor.") + + x0_pred = xt - sigma_t * flow_pred + return x0_pred.to(original_dtype) + + # --- Main Training Logic --- + bsz, channel, num_frames, height, width = target_shape + if step % args.gen_update_interval == 0: + generator_noise = torch.randn(target_shape, device=accelerator.device, generator=torch_rng, dtype=weight_dtype) + num_denoising_steps = len(denoising_step_list) + final_step_index = generate_and_sync_list(num_denoising_steps, device=generator_noise.device)[0] + + # Precompute seq_len once (same for all steps) + patch_h, patch_w = accelerator.unwrap_model(generator_transformer3d).config.patch_size[1:] + seq_len = math.ceil((width * height) / (patch_h * patch_w) * num_frames) + + for index, current_timestep in enumerate(denoising_step_list): + is_final_step = (index == final_step_index) + timestep = torch.full( + generator_noise.shape[:1], + current_timestep, + device=generator_noise.device, + dtype=torch.int64 + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + context_manager = torch.no_grad() if not is_final_step else contextlib.nullcontext() + + with context_manager: + generator_pred = generator_transformer3d( + x=generator_noise, + context=prompt_embeds, + t=timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + generator_pred = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=generator_pred, + xt=generator_noise, + timestep=timestep + ) + + if is_final_step: + break + + next_timestep = denoising_step_list[index + 1] * torch.ones( + generator_noise.shape[:1], dtype=torch.long, device=generator_noise.device + ) + generator_noise = add_noise( + generator_pred, + torch.randn(generator_pred.shape, dtype=generator_pred.dtype, device=generator_pred.device, generator=torch_rng), + next_timestep + ) + + indices = idx_sampling(bsz, generator=torch_rng, device=accelerator.device).long().cpu() + generator_timestep = noise_scheduler.timesteps[indices].to(device=accelerator.device) + generator_denoised_input = add_noise( + generator_pred, + torch.randn(generator_pred.shape, dtype=generator_pred.dtype, device=generator_pred.device, generator=torch_rng), + generator_timestep + ).detach().to(accelerator.device, dtype=weight_dtype) + + # Compute fake score + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device), torch.no_grad(): + fake_score_main_cond = fake_score_transformer3d( + x=generator_denoised_input, + context=prompt_embeds, + t=generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + fake_score_main_cond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_main_cond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + if args.fake_guidance_scale != 0.0: + fake_score_main_uncond = fake_score_transformer3d( + x=generator_denoised_input, + context=neg_prompt_embeds, + t=generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + fake_score_main_uncond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_main_uncond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + fake_score_main = fake_score_main_uncond + ( + fake_score_main_cond - fake_score_main_uncond + ) * args.fake_guidance_scale + else: + fake_score_main = fake_score_main_cond + + # Compute real score + real_score_main_cond = real_score_transformer3d( + x=generator_denoised_input, + context=prompt_embeds, + t=generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + real_score_main_cond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=real_score_main_cond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + real_score_main_uncond = real_score_transformer3d( + x=generator_denoised_input, + context=neg_prompt_embeds, + t=generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + real_score_main_uncond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=real_score_main_uncond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + real_score_main = real_score_main_uncond + ( + real_score_main_cond - real_score_main_uncond + ) * args.real_guidance_scale + + # DMD loss + fake_to_real_grad = fake_score_main - real_score_main + generator_to_real_norm = generator_pred - real_score_main + normalizer = torch.abs(generator_to_real_norm).mean(dim=[1, 2, 3, 4], keepdim=True) + fake_to_real_grad = fake_to_real_grad / normalizer + fake_to_real_grad = torch.nan_to_num(fake_to_real_grad) + + dmd_loss = 0.5 * F.mse_loss( + generator_pred.double(), + (generator_pred.double() - fake_to_real_grad.double()).detach(), + reduction="mean" + ) + avg_dmd_loss = accelerator.gather(dmd_loss.repeat(args.train_batch_size)).mean() + train_dmd_loss += avg_dmd_loss.item() / args.gradient_accumulation_steps + + if args.low_vram: + real_score_transformer3d = real_score_transformer3d.to("cpu") + fake_score_transformer3d = fake_score_transformer3d.to("cpu") + torch.cuda.empty_cache() + + accelerator.backward(dmd_loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + if args.low_vram: + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + torch.cuda.empty_cache() + + with accelerator_fake_score_transformer3d.accumulate(fake_score_transformer3d): + # --- Fake Critic Denoising Loss --- + with torch.no_grad(): + fake_score_critic_noise = torch.randn(target_shape, device=accelerator.device, generator=torch_rng, dtype=weight_dtype) + num_denoising_steps = len(denoising_step_list) + final_step_index = generate_and_sync_list(num_denoising_steps, device=fake_score_critic_noise.device)[0] + + patch_h, patch_w = accelerator.unwrap_model(generator_transformer3d).config.patch_size[1:] + seq_len = math.ceil((width * height) / (patch_h * patch_w) * num_frames) + + for index, current_timestep in enumerate(denoising_step_list): + is_final_step = (index == final_step_index) + timestep = torch.full( + fake_score_critic_noise.shape[:1], + current_timestep, + device=fake_score_critic_noise.device, + dtype=torch.int64 + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + fake_score_denoised_pred = generator_transformer3d( + x=fake_score_critic_noise, + context=prompt_embeds, + t=timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + fake_score_denoised_pred = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_denoised_pred, + xt=fake_score_critic_noise, + timestep=timestep + ) + + if is_final_step: + break + + next_timestep = denoising_step_list[index + 1] * torch.ones( + fake_score_critic_noise.shape[:1], + dtype=torch.long, + device=fake_score_critic_noise.device + ) + + fake_score_critic_noise = add_noise( + fake_score_denoised_pred, + torch.randn(fake_score_denoised_pred.shape, dtype=fake_score_denoised_pred.dtype, device=fake_score_denoised_pred.device, generator=torch_rng), + next_timestep + ) + + indices = idx_sampling(bsz, generator=torch_rng, device=accelerator.device).long().cpu() + critic_timestep = noise_scheduler.timesteps[indices].to(device=accelerator.device) + critic_noise = torch.randn(fake_score_denoised_pred.shape, dtype=fake_score_denoised_pred.dtype, device=fake_score_denoised_pred.device, generator=torch_rng) + + fake_score_denoised_input = add_noise( + fake_score_denoised_pred, + critic_noise, + critic_timestep + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + fake_score_denoised_output = fake_score_transformer3d( + x=fake_score_denoised_input, + context=prompt_embeds, + t=critic_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + denoising_loss = custom_mse_loss(fake_score_denoised_output, critic_noise - fake_score_denoised_pred) + avg_denoising_loss = accelerator.gather(denoising_loss.repeat(args.train_batch_size)).mean() + train_denoising_loss += avg_denoising_loss.item() / args.gradient_accumulation_steps + + accelerator_fake_score_transformer3d.backward(denoising_loss) + if accelerator_fake_score_transformer3d.sync_gradients: + accelerator_fake_score_transformer3d.clip_grad_norm_(fake_trainable_params, args.max_grad_norm) + critic_optimizer.step() + fake_score_lr_scheduler.step() + critic_optimizer.zero_grad() + + if args.low_vram: + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + generator_transformer3d = generator_transformer3d.to(accelerator.device) + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_denoising_loss": train_denoising_loss, "train_dmd_loss": train_dmd_loss}, step=global_step) + train_dmd_loss = 0.0 + train_denoising_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + fake_score_save_path = os.path.join(save_path, "fake_score") + accelerator.save_state(save_path) + accelerator_fake_score_transformer3d.save_state(fake_score_save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + generator_transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"denoising_loss": denoising_loss.detach().item(), "dmd_loss": dmd_loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + generator_transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + generator_transformer3d = unwrap_model(generator_transformer3d) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + fake_score_save_path = os.path.join(save_path, "fake_score") + accelerator.save_state(save_path) + accelerator_fake_score_transformer3d.save_state(fake_score_save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1/train_distill.sh b/VideoX-Fun/scripts/wan2.1/train_distill.sh new file mode 100644 index 0000000000000000000000000000000000000000..f1b5dc25b165cf8914f546cfeababbb1ce35ca17 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/train_distill.sh @@ -0,0 +1,43 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_distill.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="normal" \ + --trainable_modules "." \ + --low_vram diff --git a/VideoX-Fun/scripts/wan2.1/train_distill_lora.py b/VideoX-Fun/scripts/wan2.1/train_distill_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..62ea9c3410ea7c8cdd9f2dde5df2a0349da5786b --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/train_distill_lora.py @@ -0,0 +1,2406 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import contextlib +import gc +import json +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.distributed as dist +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import Dataset, RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + BatchSampler, RandomSampler, + get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + TextDataset, get_random_mask) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.pipeline import WanI2VPipeline, WanPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, network, config, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = WanI2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + else: + pipeline = WanPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--negative_prompt", + type=str, + default="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + help=("The negative prompt of cfg distill"), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--learning_rate_critic", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--gen_update_interval", + type=int, + default=5, + help="The ratio to update transformer3d.", + ) + parser.add_argument( + "--fake_guidance_scale", + type=float, + default=0.0, + help="The cfg scale for fake iscore.", + ) + parser.add_argument( + "--real_guidance_scale", + type=float, + default=6.0, + help="The cfg scale for real score.", + ) + parser.add_argument( + '--denoising_step_indices_list', + nargs='+', + default=[1000, 750, 500, 250], + help="The denoising step list.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + accelerator_fake_score_transformer3d = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Clip Image Encoder + if args.train_mode != "normal": + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + # Get Transformer + generator_transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + ).to(weight_dtype) + real_score_transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + ).to(weight_dtype) + fake_score_transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set generator_transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + generator_transformer3d.requires_grad_(False) + real_score_transformer3d.requires_grad_(False) + fake_score_transformer3d.requires_grad_(False) + if args.train_mode != "normal": + clip_image_encoder.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + generator_transformer3d = inject_adapter_in_model(lora_config, generator_transformer3d) + + fake_score_lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + fake_score_transformer3d = inject_adapter_in_model(fake_score_lora_config, fake_score_transformer3d) + + network = None + fake_score_network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + generator_transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network.apply_to(text_encoder, generator_transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + fake_score_network = create_network( + 1.0, + args.rank, + args.network_alpha, + None, + fake_score_transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + fake_score_network.apply_to(None, fake_score_transformer3d, False, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = generator_transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + accelerator_fake_score_transformer3d.register_save_state_pre_hook(save_model_hook) + accelerator_fake_score_transformer3d.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + generator_transformer3d.enable_gradient_checkpointing() + fake_score_transformer3d.enable_gradient_checkpointing() + real_score_transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, generator_transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, generator_transformer3d.parameters())) + + logging.info("Add fake score peft parameters") + fake_trainable_params = list(filter(lambda p: p.requires_grad, fake_score_transformer3d.parameters())) + fake_trainable_params_optim = list(filter(lambda p: p.requires_grad, fake_score_transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + logging.info("Add fake_score_network parameters") + fake_trainable_params = list(filter(lambda p: p.requires_grad, fake_score_network.parameters())) + fake_trainable_params_optim = fake_score_network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + critic_optimizer = optimizer_cls( + fake_trainable_params_optim, + lr=args.learning_rate_critic, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + critic_optimizer = optimizer_cls( + fake_trainable_params_optim, + lr=args.learning_rate_critic, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + if args.train_mode != "normal": + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + else: + train_dataset = TextDataset( + args.train_data_meta + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + if args.enable_bucket and args.train_mode != "normal": + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to("cpu"), attention_mask=prompt_attention_mask.to("cpu"))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to("cpu"), attention_mask=neg_prompt_attention_mask.to("cpu"))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + new_examples['neg_encoder_attention_mask'] = neg_prompt_ids.attention_mask + new_examples['neg_encoder_hidden_states'] = neg_prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + elif args.train_mode == "normal": + def collate_fn(examples): + new_examples = {} + new_examples["text"] = [] + for example in examples: + new_examples["text"].append(example["text"]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to("cpu"), attention_mask=prompt_attention_mask.to("cpu"))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to("cpu"), attention_mask=neg_prompt_attention_mask.to("cpu"))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + new_examples['neg_encoder_attention_mask'] = neg_prompt_ids.attention_mask + new_examples['neg_encoder_hidden_states'] = neg_prompt_embeds + + return new_examples + + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = BatchSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), batch_size=args.train_batch_size, drop_last=True) + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + fake_score_lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + generator_transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + generator_transformer3d, optimizer, train_dataloader, lr_scheduler + ) + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler = accelerator_fake_score_transformer3d.prepare( + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler + ) + elif fsdp_stage != 0: + generator_transformer3d.network = network + generator_transformer3d = generator_transformer3d.to(dtype=weight_dtype) + generator_transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + generator_transformer3d, optimizer, train_dataloader, lr_scheduler + ) + fake_score_transformer3d.network = fake_score_network + fake_score_transformer3d = fake_score_transformer3d.to(dtype=weight_dtype) + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler = accelerator_fake_score_transformer3d.prepare( + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + fake_score_network, critic_optimizer, fake_score_lr_scheduler= accelerator_fake_score_transformer3d.prepare( + fake_score_network, critic_optimizer, fake_score_lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + generator_transformer3d = shard_fn(generator_transformer3d) + fake_score_transformer3d = shard_fn(fake_score_transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + real_score_transformer3d = shard_fn(real_score_transformer3d) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + generator_transformer3d.to(accelerator.device, dtype=weight_dtype) + fake_score_transformer3d.to(accelerator.device, dtype=weight_dtype) + real_score_transformer3d.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu") + if args.train_mode != "normal": + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + fake_score_path = os.path.join(path, "fake_score") + accelerator.load_state(os.path.join(args.output_dir, path)) + accelerator_fake_score_transformer3d.load_state(os.path.join(args.output_dir, fake_score_path)) + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + denoising_step_list = noise_scheduler.timesteps[args.train_sampling_steps - torch.tensor(args.denoising_step_indices_list)] + + for epoch in range(first_epoch, args.num_train_epochs): + train_dmd_loss = 0.0 + train_denoising_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if args.train_mode != "normal" and epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + if args.train_mode != "normal": + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + batch['neg_encoder_hidden_states'] = torch.tile(batch['neg_encoder_hidden_states'], (4, 1, 1)) + batch['neg_encoder_attention_mask'] = torch.tile(batch['neg_encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + batch['neg_encoder_hidden_states'] = torch.tile(batch['neg_encoder_hidden_states'], (2, 1, 1)) + batch['neg_encoder_attention_mask'] = torch.tile(batch['neg_encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + clip_image_encoder.to(accelerator.device) + real_score_transformer3d = real_score_transformer3d.to("cpu") + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask = resize_mask(1 - mask, mask_latents) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + clip_context.append(_clip_context) + clip_context = torch.cat(clip_context) + + target_shape = mask_latents.size() + else: + text = batch['text'] + if args.fix_sample_size is not None: + local_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + num_frames = args.video_sample_n_frames + else: + if args.random_hw_adapt and args.training_with_video_token_length: + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + if rng is None: + local_length = np.random.choice(list(length_to_frame_num.keys())) + else: + local_length = rng.choice(list(length_to_frame_num.keys())) + num_frames = length_to_frame_num[local_length] + + aspect_ratio_sample_size = {key : [x / 512 * local_length for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if rng is None: + aspect_ratio_key = np.random.choice(list(aspect_ratio_sample_size.keys())) + else: + aspect_ratio_key = rng.choice(list(aspect_ratio_sample_size.keys())) + local_sample_size = aspect_ratio_sample_size[aspect_ratio_key] + else: + num_frames = args.video_sample_n_frames + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if rng is None: + aspect_ratio_key = np.random.choice(list(aspect_ratio_sample_size.keys())) + else: + aspect_ratio_key = rng.choice(list(aspect_ratio_sample_size.keys())) + local_sample_size = aspect_ratio_sample_size[aspect_ratio_key] + local_sample_size = [int(x / 16) * 16 for x in local_sample_size] + + target_shape = ( + len(text), + vae.latent_channels, + int((num_frames - 1) // vae.temporal_compression_ratio + 1), + int(local_sample_size[0] // vae.spatial_compression_ratio), + int(local_sample_size[1] // vae.spatial_compression_ratio), + ) + + if args.low_vram: + vae.to('cpu') + real_score_transformer3d = real_score_transformer3d.to("cpu") + if args.train_mode != "normal": + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=accelerator.device) + neg_prompt_embeds = batch['neg_encoder_hidden_states'].to(device=accelerator.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(accelerator.device), attention_mask=prompt_attention_mask.to(accelerator.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to(accelerator.device), attention_mask=neg_prompt_attention_mask.to(accelerator.device))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + if args.low_vram: + generator_transformer3d = generator_transformer3d.to(accelerator.device) + real_score_transformer3d = real_score_transformer3d.to(accelerator.device) + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + with accelerator.accumulate(generator_transformer3d): + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + + step_indices = [ + torch.argmin(torch.abs(schedule_timesteps - t)).item() + for t in timesteps + ] + step_indices = torch.tensor(step_indices, device=accelerator.device) + sigma = sigmas[step_indices].flatten() + + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + def add_noise(latents, noise, timesteps): + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + return (1.0 - sigmas) * latents + sigmas * noise + + def generate_and_sync_list(num_denoising_steps, device): + indices = torch.randint(low=0, high=num_denoising_steps, size=(1,), generator=torch_rng, device=device) + if dist.is_initialized(): + dist.broadcast(indices, src=0) + return indices.tolist() + + def convert_flow_pred_to_x0( + scheduler, + flow_pred: torch.Tensor, + xt: torch.Tensor, + timestep: torch.Tensor + ) -> torch.Tensor: + """ + Convert flow matching's prediction to x0 prediction. + Supports both 4D [B, C, H, W] and 5D [B, C, F, H, W] inputs. + """ + original_dtype = flow_pred.dtype + device = flow_pred.device + + flow_pred = flow_pred.double() + xt = xt.double() + timesteps = scheduler.timesteps.to(device).double() + sigmas = scheduler.sigmas.to(device).double() + timestep = timestep.to(device).double() + + timestep_id = torch.argmin((timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1) + sigma_t = sigmas[timestep_id] + + ndim = flow_pred.ndim + if ndim == 4: + sigma_t = sigma_t.view(-1, 1, 1, 1) + elif ndim == 5: + sigma_t = sigma_t.view(-1, 1, 1, 1, 1) + else: + raise ValueError(f"Expected 4D or 5D input, got {ndim}D tensor.") + + x0_pred = xt - sigma_t * flow_pred + return x0_pred.to(original_dtype) + + # --- Main Training Logic --- + bsz, channel, num_frames, height, width = target_shape + if step % args.gen_update_interval == 0: + generator_noise = torch.randn(target_shape, device=accelerator.device, generator=torch_rng, dtype=weight_dtype) + num_denoising_steps = len(denoising_step_list) + final_step_index = generate_and_sync_list(num_denoising_steps, device=generator_noise.device)[0] + + # Precompute seq_len once (same for all steps) + patch_h, patch_w = accelerator.unwrap_model(generator_transformer3d).config.patch_size[1:] + seq_len = math.ceil((width * height) / (patch_h * patch_w) * num_frames) + + for index, current_timestep in enumerate(denoising_step_list): + is_final_step = (index == final_step_index) + timestep = torch.full( + generator_noise.shape[:1], + current_timestep, + device=generator_noise.device, + dtype=torch.int64 + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + context_manager = torch.no_grad() if not is_final_step else contextlib.nullcontext() + + with context_manager: + generator_pred = generator_transformer3d( + x=generator_noise, + context=prompt_embeds, + t=timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + generator_pred = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=generator_pred, + xt=generator_noise, + timestep=timestep + ) + + if is_final_step: + break + + next_timestep = denoising_step_list[index + 1] * torch.ones( + generator_noise.shape[:1], dtype=torch.long, device=generator_noise.device + ) + generator_noise = add_noise( + generator_pred, + torch.randn(generator_pred.shape, dtype=generator_pred.dtype, device=generator_pred.device, generator=torch_rng), + next_timestep + ) + + indices = idx_sampling(bsz, generator=torch_rng, device=accelerator.device).long().cpu() + generator_timestep = noise_scheduler.timesteps[indices].to(device=accelerator.device) + generator_denoised_input = add_noise( + generator_pred, + torch.randn(generator_pred.shape, dtype=generator_pred.dtype, device=generator_pred.device, generator=torch_rng), + generator_timestep + ).detach().to(accelerator.device, dtype=weight_dtype) + + # Compute fake score + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device), torch.no_grad(): + fake_score_main_cond = fake_score_transformer3d( + x=generator_denoised_input, + context=prompt_embeds, + t=generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + fake_score_main_cond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_main_cond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + if args.fake_guidance_scale != 0.0: + fake_score_main_uncond = fake_score_transformer3d( + x=generator_denoised_input, + context=neg_prompt_embeds, + t=generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + fake_score_main_uncond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_main_uncond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + fake_score_main = fake_score_main_uncond + ( + fake_score_main_cond - fake_score_main_uncond + ) * args.fake_guidance_scale + else: + fake_score_main = fake_score_main_cond + + # Compute real score + real_score_main_cond = real_score_transformer3d( + x=generator_denoised_input, + context=prompt_embeds, + t=generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + real_score_main_cond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=real_score_main_cond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + real_score_main_uncond = real_score_transformer3d( + x=generator_denoised_input, + context=neg_prompt_embeds, + t=generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + real_score_main_uncond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=real_score_main_uncond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + real_score_main = real_score_main_uncond + ( + real_score_main_cond - real_score_main_uncond + ) * args.real_guidance_scale + + # DMD loss + fake_to_real_grad = fake_score_main - real_score_main + generator_to_real_norm = generator_pred - real_score_main + normalizer = torch.abs(generator_to_real_norm).mean(dim=[1, 2, 3, 4], keepdim=True) + fake_to_real_grad = fake_to_real_grad / normalizer + fake_to_real_grad = torch.nan_to_num(fake_to_real_grad) + + dmd_loss = 0.5 * F.mse_loss( + generator_pred.double(), + (generator_pred.double() - fake_to_real_grad.double()).detach(), + reduction="mean" + ) + avg_dmd_loss = accelerator.gather(dmd_loss.repeat(args.train_batch_size)).mean() + train_dmd_loss += avg_dmd_loss.item() / args.gradient_accumulation_steps + + if args.low_vram: + real_score_transformer3d = real_score_transformer3d.to("cpu") + fake_score_transformer3d = fake_score_transformer3d.to("cpu") + torch.cuda.empty_cache() + + accelerator.backward(dmd_loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + if args.low_vram: + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + torch.cuda.empty_cache() + + with accelerator_fake_score_transformer3d.accumulate(fake_score_transformer3d): + # --- Fake Critic Denoising Loss --- + with torch.no_grad(): + fake_score_critic_noise = torch.randn(target_shape, device=accelerator.device, generator=torch_rng, dtype=weight_dtype) + num_denoising_steps = len(denoising_step_list) + final_step_index = generate_and_sync_list(num_denoising_steps, device=fake_score_critic_noise.device)[0] + + patch_h, patch_w = accelerator.unwrap_model(generator_transformer3d).config.patch_size[1:] + seq_len = math.ceil((width * height) / (patch_h * patch_w) * num_frames) + + for index, current_timestep in enumerate(denoising_step_list): + is_final_step = (index == final_step_index) + timestep = torch.full( + fake_score_critic_noise.shape[:1], + current_timestep, + device=fake_score_critic_noise.device, + dtype=torch.int64 + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + fake_score_denoised_pred = generator_transformer3d( + x=fake_score_critic_noise, + context=prompt_embeds, + t=timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + fake_score_denoised_pred = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_denoised_pred, + xt=fake_score_critic_noise, + timestep=timestep + ) + + if is_final_step: + break + + next_timestep = denoising_step_list[index + 1] * torch.ones( + fake_score_critic_noise.shape[:1], + dtype=torch.long, + device=fake_score_critic_noise.device + ) + + fake_score_critic_noise = add_noise( + fake_score_denoised_pred, + torch.randn(fake_score_denoised_pred.shape, dtype=fake_score_denoised_pred.dtype, device=fake_score_denoised_pred.device, generator=torch_rng), + next_timestep + ) + + indices = idx_sampling(bsz, generator=torch_rng, device=accelerator.device).long().cpu() + critic_timestep = noise_scheduler.timesteps[indices].to(device=accelerator.device) + critic_noise = torch.randn(fake_score_denoised_pred.shape, dtype=fake_score_denoised_pred.dtype, device=fake_score_denoised_pred.device, generator=torch_rng) + + fake_score_denoised_input = add_noise( + fake_score_denoised_pred, + critic_noise, + critic_timestep + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + fake_score_denoised_output = fake_score_transformer3d( + x=fake_score_denoised_input, + context=prompt_embeds, + t=critic_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + denoising_loss = custom_mse_loss(fake_score_denoised_output, critic_noise - fake_score_denoised_pred) + avg_denoising_loss = accelerator_fake_score_transformer3d.gather(denoising_loss.repeat(args.train_batch_size)).mean() + train_denoising_loss += avg_denoising_loss.item() / args.gradient_accumulation_steps + + if args.low_vram: + generator_transformer3d = generator_transformer3d.to("cpu") + torch.cuda.empty_cache() + + accelerator_fake_score_transformer3d.backward(denoising_loss) + if accelerator_fake_score_transformer3d.sync_gradients: + accelerator_fake_score_transformer3d.clip_grad_norm_(fake_trainable_params, args.max_grad_norm) + critic_optimizer.step() + fake_score_lr_scheduler.step() + critic_optimizer.zero_grad() + + if args.low_vram: + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + generator_transformer3d = generator_transformer3d.to(accelerator.device) + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_denoising_loss": train_denoising_loss, "train_dmd_loss": train_dmd_loss}, step=global_step) + train_dmd_loss = 0.0 + train_denoising_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(generator_transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}-fake_score.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(fake_score_transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}-fake_score.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(fake_score_network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + fake_score_save_path = os.path.join(save_path, "fake_score") + accelerator.save_state(save_path) + accelerator_fake_score_transformer3d.save_state(fake_score_save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + generator_transformer3d, + network, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"denoising_loss": denoising_loss.detach().item(), "dmd_loss": dmd_loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + generator_transformer3d, + network, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + generator_transformer3d = unwrap_model(generator_transformer3d) + fake_score_transformer3d = unwrap_model(fake_score_transformer3d) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(generator_transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}-fake_score.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(fake_score_transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}-fake_score.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(fake_score_network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + fake_score_save_path = os.path.join(save_path, "fake_score") + accelerator.save_state(save_path) + accelerator_fake_score_transformer3d.save_state(fake_score_save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1/train_distill_lora.sh b/VideoX-Fun/scripts/wan2.1/train_distill_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..2d55cb19ee717d04737433e5b74831b23a95b9b0 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/train_distill_lora.sh @@ -0,0 +1,45 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_distill_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram + diff --git a/VideoX-Fun/scripts/wan2.1/train_lora.py b/VideoX-Fun/scripts/wan2.1/train_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..2ff6a448269e25f564ae625fa1fb28acd1ff2941 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/train_lora.py @@ -0,0 +1,1956 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.pipeline import WanPipeline, WanI2VPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import create_network, merge_lora, unmerge_lora +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, network, config, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = WanI2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + else: + pipeline = WanPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--noise_share_in_frames", action="store_true", help="Whether enable noise share in frames." + ) + parser.add_argument( + "--noise_share_in_frames_ratio", type=float, default=0.5, help="Noise share ratio.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Clip Image Encoder + if args.train_mode != "normal": + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + # Get Transformer + transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + if args.train_mode != "normal": + clip_image_encoder.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + enable_inpaint=True if args.train_mode != "normal" else False, + ) + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if args.train_mode != "normal": + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + if args.train_mode != "normal": + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if args.train_mode != "normal": + clip_image_encoder.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "normal": + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask = resize_mask(1 - mask, latents) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + clip_context.append(_clip_context) + clip_context = torch.cat(clip_context) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + if args.train_mode != "normal": + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1/train_lora.sh b/VideoX-Fun/scripts/wan2.1/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..be0ece95575876dbd8b0425e5334aaef9d3cabe0 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/train_lora.sh @@ -0,0 +1,86 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram + +# # Training command for I2V +# export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-I2V-14B-720P" +# export DATASET_NAME="datasets/internal_datasets/" +# export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +# NCCL_DEBUG=INFO + +# accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_lora.py \ +# --config_path="config/wan2.1/wan_civitai.yaml" \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --train_data_dir=$DATASET_NAME \ +# --train_data_meta=$DATASET_META_NAME \ +# --image_sample_size=640 \ +# --video_sample_size=640 \ +# --token_sample_size=640 \ +# --video_sample_stride=2 \ +# --video_sample_n_frames=81 \ +# --train_batch_size=1 \ +# --video_repeat=1 \ +# --gradient_accumulation_steps=1 \ +# --dataloader_num_workers=8 \ +# --num_train_epochs=100 \ +# --checkpointing_steps=50 \ +# --learning_rate=1e-04 \ +# --seed=42 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --vae_mini_batch=1 \ +# --max_grad_norm=0.05 \ +# --random_hw_adapt \ +# --training_with_video_token_length \ +# --enable_bucket \ +# --uniform_sampling \ +# --rank=64 \ +# --network_alpha=32 \ +# --target_name="q,k,v,ffn.0,ffn.2" \ +# --use_peft_lora \ +# --low_vram \ +# --train_mode="i2v" \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1/train_reward_lora.py b/VideoX-Fun/scripts/wan2.1/train_reward_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..76712675ecaf519555ef6950c0e2578400d6f3b3 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/train_reward_lora.py @@ -0,0 +1,1400 @@ +"""Modified from EasyAnimate/scripts/train_lora.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import json +import logging +import math +import os +import random +import shutil +import sys +from contextlib import contextmanager +from typing import List, Optional + +import accelerate +import diffusers +import numpy as np +import torch +import torch.utils.checkpoint +import torchvision.transforms as transforms +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from decord import VideoReader +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.utils import check_min_version, is_wandb_available +from diffusers.utils.import_utils import is_xformers_available +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +import videox_fun.reward.reward_fn as reward_fn +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.pipeline import WanPipeline, WanI2VPipeline +from videox_fun.utils.lora_utils import create_network, merge_lora +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +@contextmanager +def video_reader(*args, **kwargs): + """A context manager to solve the memory leak of decord. + """ + vr = VideoReader(*args, **kwargs) + try: + yield vr + finally: + del vr + gc.collect() + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + + +def log_validation( + vae, text_encoder, tokenizer, transformer3d, network, + loss_fn, config, args, accelerator, weight_dtype, global_step, validation_prompts_idx +): + try: + logger.info("Running validation... ") + + transformer3d_val = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + # Initialize a new vae if gradient checkpointing is enabled. + if args.vae_gradient_checkpointing: + # Get Vae + vae = WanTransformer3DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision, variant=args.variant + ).to(weight_dtype) + + pipeline = WanPipeline( + vae=vae if args.vae_gradient_checkpointing else accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(dtype=weight_dtype) + if args.low_vram: + pipeline.enable_model_cpu_offload() + else: + pipeline = pipeline.to(device=accelerator.device) + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + to_tensor = transforms.ToTensor() + validation_loss, validation_reward = 0, 0 + + for i in range(len(validation_prompts_idx)): + validation_idx, validation_prompt = validation_prompts_idx[i] + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + sample_size = [args.validation_sample_height, args.validation_sample_width] + input_video, input_video_mask, clip_image = get_image_to_video_latent( + None, None, video_length=args.video_length, sample_size=sample_size + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + sample = pipeline( + validation_prompt, + video_length = video_length, + negative_prompt = "bad detailed", + height = args.validation_sample_height, + width = args.validation_sample_width, + guidance_scale = 6, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + ).frames + sample_saved_path = os.path.join(args.output_dir, f"validation_sample/sample-{global_step}-{validation_idx}.mp4") + save_videos_grid(sample, sample_saved_path, fps=8) + + num_sampled_frames = 4 + sampled_frames_list = [] + with video_reader(sample_saved_path) as vr: + sampled_frame_idx_list = np.linspace(0, len(vr), num_sampled_frames, endpoint=False, dtype=int) + sampled_frame_list = vr.get_batch(sampled_frame_idx_list).asnumpy() + sampled_frames = torch.stack([to_tensor(frame) for frame in sampled_frame_list], dim=0) + sampled_frames_list.append(sampled_frames) + + sampled_frames = torch.stack(sampled_frames_list) + sampled_frames = rearrange(sampled_frames, "b t c h w -> b c t h w") + loss, reward = loss_fn(sampled_frames, [validation_prompt]) + validation_loss, validation_reward = validation_loss + loss, validation_reward + reward + + validation_loss = validation_loss / len(validation_prompts_idx) + validation_reward = validation_reward / len(validation_prompts_idx) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return validation_loss, validation_reward + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None, None + + +def load_prompts(prompt_path, prompt_column="prompt", start_idx=None, end_idx=None): + prompt_list = [] + if prompt_path.endswith(".txt"): + with open(prompt_path, "r") as f: + for line in f: + prompt_list.append(line.strip()) + elif prompt_path.endswith(".jsonl"): + with open(prompt_path, "r") as f: + for line in f.readlines(): + item = json.loads(line) + prompt_list.append(item[prompt_column]) + else: + raise ValueError("The prompt_path must end with .txt or .jsonl.") + prompt_list = prompt_list[start_idx:end_idx] + + return prompt_list + + +def _get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + +def encode_prompt( + tokenizer, + text_encoder, + prompt, + negative_prompt, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = _get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = _get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + +# Modified from EasyAnimateInpaintPipeline.prepare_extra_step_kwargs +def prepare_extra_step_kwargs(scheduler, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + import inspect + + accepts_eta = "eta" in set(inspect.signature(scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--validation_prompt_path", + type=str, + default=None, + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_batch_size", + type=int, + default=1, + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_sample_height", + type=int, + default=512, + help="The height of sampling videos in validation.", + ) + parser.add_argument( + "--validation_sample_width", + type=int, + default=512, + help="The width of sampling videos in validation.", + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing (for DiT) to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--vae_gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing (for VAE) to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + + parser.add_argument( + "--prompt_path", + type=str, + default="normal", + help="The path to the training prompt file.", + ) + parser.add_argument( + '--train_sample_height', + type=int, + default=384, + help='The height of sampling videos in training' + ) + parser.add_argument( + '--train_sample_width', + type=int, + default=672, + help='The width of sampling videos in training' + ) + parser.add_argument( + "--video_length", + type=int, + default=49, + help="The number of frames to generate in training and validation." + ) + parser.add_argument( + '--eta', + type=float, + default=0.0, + help='eta parameter for the DDIM sampler. this controls the amount of noise injected into the sampling process, ' + 'with 0.0 being fully deterministic and 1.0 being equivalent to the DDPM sampler.' + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=6.0, + help="The classifier-free diffusion guidance." + ) + parser.add_argument( + "--num_inference_steps", + type=int, + default=50, + help="The number of denoising steps in training and validation." + ) + parser.add_argument( + "--num_decoded_latents", + type=int, + default=3, + help="The number of latents to be decoded." + ) + parser.add_argument( + "--num_sampled_frames", + type=int, + default=None, + help="The number of sampled frames for the reward function." + ) + parser.add_argument( + "--reward_fn", + type=str, + default="aesthetic_loss_fn", + help='The reward function.' + ) + parser.add_argument( + "--reward_fn_kwargs", + type=str, + default=None, + help='The keyword arguments of the reward function.' + ) + parser.add_argument( + "--backprop", + action="store_true", + default=False, + help="Whether to use the reward backprop training mode.", + ) + parser.add_argument( + "--backprop_step_list", + nargs="+", + type=int, + default=None, + help="The preset step list for reward backprop. If provided, overrides `backprop_strategy`." + ) + parser.add_argument( + "--backprop_strategy", + choices=["last", "tail", "uniform", "random"], + default="last", + help="The strategy for reward backprop." + ) + parser.add_argument( + "--stop_latent_model_input_gradient", + action="store_true", + default=False, + help="Whether to stop the gradient of the latents during reward backprop.", + ) + parser.add_argument( + "--backprop_random_start_step", + type=int, + default=0, + help="The random start step for reward backprop. Only used when `backprop_strategy` is random." + ) + parser.add_argument( + "--backprop_random_end_step", + type=int, + default=50, + help="The random end step for reward backprop. Only used when `backprop_strategy` is random." + ) + parser.add_argument( + "--backprop_num_steps", + type=int, + default=5, + help="The number of steps for backprop. Only used when `backprop_strategy` is tail/uniform/random." + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # Sanity check for validation + do_validation = (args.validation_prompt_path is not None or args.validation_prompts is not None) + if do_validation: + if not (os.path.exists(args.validation_prompt_path) or args.validation_prompt_path.endswith(".txt")): + raise ValueError("The `--validation_prompt_path` must be a txt file containing prompts.") + if args.validation_batch_size < accelerator.num_processes or args.validation_batch_size % accelerator.num_processes != 0: + raise ValueError("The `--validation_batch_size` must be divisible by the number of processes.") + + # Sanity check for validation + if args.backprop: + if args.backprop_step_list is not None: + logger.warning( + f"The backprop_strategy {args.backprop_strategy} will be ignored " + f"when using backprop_step_list {args.backprop_step_list}." + ) + assert any(step <= args.num_inference_steps - 1 for step in args.backprop_step_list) + else: + if args.backprop_strategy in set(["tail", "uniform", "random"]): + assert args.backprop_num_steps <= args.num_inference_steps - 1 + if args.backprop_strategy == "random": + assert args.backprop_random_start_step <= args.backprop_random_end_step + assert args.backprop_random_end_step <= args.num_inference_steps - 1 + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed, device_specific=True) + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + + # Get Transformer + transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ) + + # if args.train_mode != "normal": + # # Get Clip Image Encoder + # clip_image_encoder = CLIPModel.from_pretrained( + # os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + # ) + # clip_image_encoder = clip_image_encoder.eval() + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + vae.eval() + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + # clip_image_encoder.requires_grad_(False) + + # Lora will work with this... + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + add_lora_in_attn_temporal=True, + ) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + # Load transformer and vae from path if it needs. + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + accelerator.register_save_state_pre_hook(save_model_hook) + # Save the model weights directly before save_state instead of using a hook. + # accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + if args.vae_gradient_checkpointing: + # Since 3D casual VAE need a cache to decode all latents autoregressively, .Thus, gradient checkpointing can only be + # enabled when decoding the first batch (i.e. the first three) of latents, in which case the cache is not being used. + + # num_decoded_latents > 3 is support in EasyAnimate now. + # if args.num_decoded_latents > 3: + # raise ValueError("The vae_gradient_checkpointing is not supported for num_decoded_latents > 3.") + vae.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + # Init optimizer + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # loss function + reward_fn_kwargs = {} + if args.reward_fn_kwargs is not None: + reward_fn_kwargs = json.loads(args.reward_fn_kwargs) + if accelerator.is_main_process: + # Check if the model is downloaded in the main process. + loss_fn = getattr(reward_fn, args.reward_fn)(device="cpu", dtype=weight_dtype, **reward_fn_kwargs) + accelerator.wait_for_everyone() + loss_fn = getattr(reward_fn, args.reward_fn)(device=accelerator.device, dtype=weight_dtype, **reward_fn_kwargs) + + # Get RL training prompts + prompt_list = load_prompts(args.prompt_path) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(prompt_list) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + network, optimizer, lr_scheduler = accelerator.prepare(network, optimizer, lr_scheduler) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device, dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + text_encoder.to(accelerator.device) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(prompt_list) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(prompt_list)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + from safetensors.torch import load_file, safe_open + state_dict = load_file(os.path.join(os.path.join(args.output_dir, path), "lora_diffusion_pytorch_model.safetensors")) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + train_reward = 0.0 + + # In the following training loop, randomly select training prompts and use the + # `EasyAnimatePipelineInpaint` to sample videos, calculate rewards, and update the network. + for _ in range(num_update_steps_per_epoch): + # train_prompt = random.sample(prompt_list, args.train_batch_size) + train_prompt = random.choices(prompt_list, k=args.train_batch_size) + logger.info(f"train_prompt: {train_prompt}") + + # default height and width + height = int(args.train_sample_height // 16 * 16) + width = int(args.train_sample_width // 16 * 16) + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = args.guidance_scale > 1.0 + + # Reduce the vram by offload text encoders + if args.low_vram: + torch.cuda.empty_cache() + text_encoder.to(accelerator.device) + + # Encode input prompt + ( + prompt_embeds, + negative_prompt_embeds + ) = encode_prompt( + tokenizer, + text_encoder, + train_prompt, + negative_prompt=[""] * len(train_prompt), + device=accelerator.device, + dtype=weight_dtype, + do_classifier_free_guidance=do_classifier_free_guidance, + ) + if do_classifier_free_guidance: + prompt_embeds = negative_prompt_embeds + prompt_embeds + + # Reduce the vram by offload text encoders + if args.low_vram: + text_encoder.to("cpu") + torch.cuda.empty_cache() + + # Prepare timesteps + if hasattr(noise_scheduler, "use_dynamic_shifting") and noise_scheduler.use_dynamic_shifting: + noise_scheduler.set_timesteps(args.num_inference_steps, device=accelerator.device, mu=1) + else: + noise_scheduler.set_timesteps(args.num_inference_steps, device=accelerator.device) + timesteps = noise_scheduler.timesteps + + # Prepare latent variables + vae_scale_factor = vae.spatial_compression_ratio + latent_shape = [ + args.train_batch_size, + vae.config.latent_channels, + int((args.video_length - 1) // vae.temporal_compression_ratio + 1) if args.video_length != 1 else 1, + args.train_sample_height // vae_scale_factor, + args.train_sample_width // vae_scale_factor, + ] + + with accelerator.accumulate(transformer3d): + latents = torch.randn(*latent_shape, device=accelerator.device, dtype=weight_dtype) + + if hasattr(noise_scheduler, "init_noise_sigma"): + latents = latents * noise_scheduler.init_noise_sigma + + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + # Prepare extra step kwargs. + extra_step_kwargs = prepare_extra_step_kwargs(noise_scheduler, generator, args.eta) + + bsz, channel, num_frames, height, width = latents.size() + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Denoising loop + if args.backprop: + if args.backprop_step_list is None: + if args.backprop_strategy == "last": + backprop_step_list = [args.num_inference_steps - 1] + elif args.backprop_strategy == "tail": + backprop_step_list = list(range(args.num_inference_steps))[-args.backprop_num_steps:] + elif args.backprop_strategy == "uniform": + interval = args.num_inference_steps // args.backprop_num_steps + random_start = random.randint(0, interval) + backprop_step_list = [random_start + i * interval for i in range(args.backprop_num_steps)] + elif args.backprop_strategy == "random": + backprop_step_list = random.sample( + range(args.backprop_random_start_step, args.backprop_random_end_step + 1), args.backprop_num_steps + ) + else: + raise ValueError(f"Invalid backprop strategy: {args.backprop_strategy}.") + else: + backprop_step_list = args.backprop_step_list + + for i, t in enumerate(tqdm(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(noise_scheduler, "scale_model_input"): + latent_model_input = noise_scheduler.scale_model_input(latent_model_input, t) + + # expand scalar t to 1-D tensor to match the 1st dim of latent_model_input + t_expand = torch.tensor([t] * latent_model_input.shape[0], device=accelerator.device).to( + dtype=latent_model_input.dtype + ) + + # predict the noise residual + if args.stop_latent_model_input_gradient: + # See https://arxiv.org/abs/2405.00760 + latent_model_input = latent_model_input.detach() + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype): + noise_pred = transformer3d( + x=latent_model_input, + context=prompt_embeds, + t=t_expand, + seq_len=seq_len, + ) + + # Optimize the denoising results only for the specified steps. + if i in backprop_step_list: + noise_pred = noise_pred + else: + noise_pred = noise_pred.detach() + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred[0], noise_pred[1] + noise_pred = noise_pred_uncond + args.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = noise_scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + # decode latents (tensor) + # latents = latents.permute(0, 2, 1, 3, 4) # [B, C, T, H, W] + # Since the casual VAE decoding consumes a large amount of VRAM, and we need to keep the decoding + # operation within the computational graph. Thus, we only decode the first args.num_decoded_latents + # to calculate the reward. + # TODO: Decode all latents but keep a portion of the decoding operation within the computational graph. + sampled_latent_indices = list(range(args.num_decoded_latents)) + sampled_latents = latents[:, :, sampled_latent_indices, :, :] + sampled_frames = vae.decode(sampled_latents.to(vae.device, vae.dtype))[0] + sampled_frames = sampled_frames.clamp(-1, 1) + sampled_frames = (sampled_frames / 2 + 0.5).clamp(0, 1) # [-1, 1] -> [0, 1] + + if global_step % args.checkpointing_steps == 0: + saved_file = f"sample-{global_step}-{accelerator.process_index}.mp4" + save_videos_grid( + sampled_frames.to(torch.float32).detach().cpu(), + os.path.join(args.output_dir, "train_sample", saved_file), + fps=8 + ) + + if args.num_sampled_frames is not None: + num_frames = sampled_frames.size(2) - 1 + sampled_frames_indices = torch.linspace(0, num_frames, steps=args.num_sampled_frames).long() + sampled_frames = sampled_frames[:, :, sampled_frames_indices, :, :] + # compute loss and reward + loss, reward = loss_fn(sampled_frames, train_prompt) + + # Gather the losses and rewards across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + avg_reward = accelerator.gather(reward.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + train_reward += avg_reward.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + total_norm = accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + # If use_deepspeed, `total_norm` cannot be logged by accelerator. + if not args.use_deepspeed: + accelerator.log({"total_norm": total_norm}, step=global_step) + else: + if hasattr(optimizer, "optimizer") and hasattr(optimizer.optimizer, "_global_grad_norm"): + accelerator.log({"total_norm": optimizer.optimizer._global_grad_norm}, step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss, "train_reward": train_reward}, step=global_step) + train_loss = 0.0 + train_reward = 0.0 + + if global_step % args.checkpointing_steps == 0: + # DeepSpeed requires saving weights on every device; saving weights only on the main process would cause issues. + if args.use_deepspeed or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + # Validation (distributed) + if do_validation and (global_step % args.validation_steps) == 0: + if args.validation_prompts is None and args.validation_prompt_path.endswith(".txt"): + validation_prompts = [] + with open(args.validation_prompt_path, "r") as f: + for line in f: + validation_prompts.append(line.strip()) + # Do not select randomly to ensure that `args.validation_prompts` is the same for each process. + args.validation_prompts = validation_prompts[:args.validation_batch_size] + validation_prompts_idx = [(i, p) for i, p in enumerate(args.validation_prompts)] + + if hasattr(vae, "enable_cache_in_vae"): + vae.enable_cache_in_vae() + accelerator.wait_for_everyone() + with accelerator.split_between_processes(validation_prompts_idx) as splitted_prompts_idx: + validation_loss, validation_reward = log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + loss_fn, + config, + args, + accelerator, + weight_dtype, + global_step, + splitted_prompts_idx + ) + if validation_loss is not None and validation_reward is not None: + avg_validation_loss = accelerator.gather(validation_loss).mean() + avg_validation_reward = accelerator.gather(validation_reward).mean() + accelerator.print(avg_validation_loss, avg_validation_reward) + if accelerator.is_main_process: + accelerator.log( + {"validation_loss": avg_validation_loss, "validation_reward": avg_validation_reward}, + step=global_step + ) + + accelerator.wait_for_everyone() + + logs = {"step_loss": loss.detach().item(), "step_reward": reward.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1/train_reward_lora.sh b/VideoX-Fun/scripts/wan2.1/train_reward_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..5c149603cc69f610dee8adc09d84cf426294556e --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1/train_reward_lora.sh @@ -0,0 +1,37 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-T2V-1.3B" +export TRAIN_PROMPT_PATH="MovieGenVideoBench_train.txt" +# Performing validation simultaneously with training will increase time and GPU memory usage. +export VALIDATION_PROMPT_PATH="MovieGenVideoBench_val.txt" +# Set 1 for Wan2.1-T2V-14B +export BACKPROP_NUM_STEPS=5 + +accelerate launch --mixed_precision="bf16" scripts/wan2.1/train_reward_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --rank=32 \ + --network_alpha=16 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --max_train_steps=10000 \ + --checkpointing_steps=100 \ + --learning_rate=1e-05 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --vae_gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --max_grad_norm=0.3 \ + --low_vram \ + --prompt_path=$TRAIN_PROMPT_PATH \ + --train_sample_height=256 \ + --train_sample_width=256 \ + --num_inference_steps=30 \ + --video_length=81 \ + --num_decoded_latents=1 \ + --reward_fn="HPSReward" \ + --reward_fn_kwargs='{"version": "v2.1"}' \ + --backprop_strategy "tail" \ + --backprop_num_steps=$BACKPROP_NUM_STEPS \ + --backprop \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN.md b/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..d3ddd7b6c52bf364678c6760aa6778d22bdd3e60 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN.md @@ -0,0 +1,244 @@ +## Training Code + +We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or inpaint. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan T2V without deepspeed: + +Wan without DeepSpeed is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +Wan T2V with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan T2V with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +Wan T2V with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="inpaint" \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_CONTROL.md b/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_CONTROL.md new file mode 100644 index 0000000000000000000000000000000000000000..00fea70faf53e11f7dc0c9f3ca4e8df47849fbd6 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_CONTROL.md @@ -0,0 +1,427 @@ +## Training Code + +We can choose whether to use DeepSpeed and FSDP in Wan-Fun, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in Wan-Fun, you need to add a control_file_path, and [DWPose](https://github.com/IDEA-Research/DWPose) is suggested as tool to generate control file. + +```json +[ + { + "file_path": "train/00000001.mp4", + "control_file_path": "control/00000001.mp4", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "control_file_path": "control/00000002.jpg", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `train_mode` is used to set the training mode. + - The models named `Wan2.1-Fun-*-Control` are trained in the `control_ref` mode. + - The models named `Wan2.1-Fun-*-Control-Camera` are trained in the `control_ref_camera` mode. +- `control_ref_image` is used to specify the type of control image. The available options are `first_frame` and `random`. + - `first_frame` is used in V1.0 because V1.0 supports using a specified start frame as the control image. The Control-Camera models use the first frame as the control image. + - `random` is used in V1.1 because V1.1 supports both using a specified start frame and a reference image as the control image. +- `add_full_ref_image_in_self_attention` determines whether to include the reference image in self-attention. This option is used in V1.1, as it supports using a reference image as the control image. It should not be used in V1.0 and Control-Camera models. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/wan2.1_fun/xxx.py +``` + +Wan-Fun-Control-V1.1 without deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_control.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_full_ref_image_in_self_attention \ + --trainable_modules "." +``` + +Wan-Fun-Control-V1.1 with deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_control.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_full_ref_image_in_self_attention \ + --trainable_modules "." +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan-Fun-Control-V1.1 with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_control.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_full_ref_image_in_self_attention \ + --trainable_modules "." +``` + +Wan-Fun-Control-V1.1 with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1_fun/train_control.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_full_ref_image_in_self_attention \ + --trainable_modules "." +``` + +
+ (Obsolete) V1.0: + +Wan-Fun-Control-V1.0 without deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_control.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="control_ref" \ + --control_ref_image="first_frame" \ + --trainable_modules "." +``` + +Wan-Fun-Control-V1.0 with deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_control.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="first_frame" \ + --trainable_modules "." +``` + +Wan-Fun-Control-V1.0 with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_control.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="first_frame" \ + --trainable_modules "." +``` +
diff --git a/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_CONTROL_LORA.md b/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_CONTROL_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..9e85ee789f0f1571a2a896b7af14e9dcb5a25cf8 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_CONTROL_LORA.md @@ -0,0 +1,427 @@ +## Training Code + +We can choose whether to use DeepSpeed and FSDP in Wan-Fun, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in Wan-Fun, you need to add a control_file_path, and [DWPose](https://github.com/IDEA-Research/DWPose) is suggested as tool to generate control file. + +```json +[ + { + "file_path": "train/00000001.mp4", + "control_file_path": "control/00000001.mp4", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "control_file_path": "control/00000002.jpg", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint and set the `save_state` to `True`. +- `train_mode` is used to set the training mode. + - The models named `Wan2.1-Fun-*-Control` are trained in the `control_ref` mode. + - The models named `Wan2.1-Fun-*-Control-Camera` are trained in the `control_ref_camera` mode. +- `control_ref_image` is used to specify the type of control image. The available options are `first_frame` and `random`. + - `first_frame` is used in V1.0 because V1.0 supports using a specified start frame as the control image. The Control-Camera models use the first frame as the control image. + - `random` is used in V1.1 because V1.1 supports both using a specified start frame and a reference image as the control image. +- `add_full_ref_image_in_self_attention` determines whether to include the reference image in self-attention. This option is used in V1.1, as it supports using a reference image as the control image. It should not be used in V1.0 and Control-Camera models. +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/wan2.1_fun/xxx.py +``` + +Wan-Fun-Control-V1.1 without deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_control_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_full_ref_image_in_self_attention \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +Wan-Fun-Control-V1.1 with deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_control_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_full_ref_image_in_self_attention \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is not compatible with PEFT. + +Wan-Fun-Control-V1.1 with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_control_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --save_state \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_full_ref_image_in_self_attention \ + --low_vram +``` + +Wan-Fun-Control-V1.1 with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1_fun/train_control_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --save_state \ + --use_fsdp \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_full_ref_image_in_self_attention \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +
+ (Obsolete) V1.0: + +Wan-Fun-Control-V1.0 without deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_control_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="control_ref" \ + --control_ref_image="first_frame" \ + --low_vram +``` + +Wan-Fun-Control-V1.0 with deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_control_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="first_frame" \ + --low_vram +``` + +Wan-Fun-Control-V1.0 with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_control_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --save_state \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="first_frame" \ + --low_vram +``` +
\ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_LORA.md b/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..19c138592335b3b70d69acfe948b79968ac6081d --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_LORA.md @@ -0,0 +1,252 @@ +## Lora Training Code + +We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or inpaint. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan T2V without deepspeed: + +Wan without DeepSpeed is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +Wan T2V with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --use_deepspeed \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is not compatible with PEFT. + +Wan T2V with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --save_state \ + --use_deepspeed \ + --train_mode="inpaint" \ + --low_vram +``` + +Wan T2V with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --save_state \ + --use_deepspeed \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_REWARD.md b/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_REWARD.md new file mode 100644 index 0000000000000000000000000000000000000000..d7a18a128ce3aa30efefd622f5f33a345740f7ca --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/README_TRAIN_REWARD.md @@ -0,0 +1,209 @@ +# Wan2.1-Fun-Reward-LoRAs +## Introduction +We explore the Reward Backpropagation technique [1](#ref1) [2](#ref2) to optimized the generated videos by [Wan2.1-Fun](https://github.com/aigc-apps/VideoX-Fun) for better alignment with human preferences. +We provide the following pre-trained models (i.e. LoRAs) along with [the training script](https://github.com/aigc-apps/VideoX-Fun/blob/main/scripts/wan2.1_fun/train_reward_lora.py). You can use these LoRAs to enhance the corresponding base model as a plug-in or train your own reward LoRA. + +For more details, please refer to our [GitHub repo](https://github.com/aigc-apps/VideoX-Fun). + +| Name | Base Model | Reward Model | Hugging Face | Description | +|--|--|--|--|--| +| Wan2.1-Fun-1.3B-InP-HPS2.1.safetensors | [Wan2.1-Fun-1.3B-InP](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-InP) | [HPS v2.1](https://github.com/tgxs002/HPSv2) | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-Reward-LoRAs/resolve/main/Wan2.1-Fun-1.3B-InP-HPS2.1.safetensors) | Official HPS v2.1 reward LoRA (`rank=128` and `network_alpha=64`) for Wan2.1-Fun-1.3B-InP. It is trained with a batch size of 8 for 5,000 steps.| +| Wan2.1-Fun-1.3B-InP-MPS.safetensors | [Wan2.1-Fun-1.3B-InP](https://huggingface.co/alibaba-pai/Wan2.1-Fun-1.3B-InP) | [MPS](https://github.com/Kwai-Kolors/MPS) | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-Reward-LoRAs/resolve/main/Wan2.1-Fun-1.3B-InP-MPS.safetensors) | Official MPS reward LoRA (`rank=128` and `network_alpha=64`) for Wan2.1-Fun-1.3B-InP. It is trained with a batch size of 8 for 7,500 steps.| +| Wan2.1-Fun-14B-InP-HPS2.1.safetensors | [Wan2.1-Fun-14B-InP](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-InP) | [HPS v2.1](https://github.com/tgxs002/HPSv2) | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-Reward-LoRAs/resolve/main/Wan2.1-Fun-14B-InP-HPS2.1.safetensors) | Official HPS v2.1 reward LoRA (`rank=128` and `network_alpha=64`) for Wan2.1-Fun-14B-InP. It is trained with a batch size of 32 for 3,000 steps.| +| Wan2.1-Fun-14B-InP-MPS.safetensors | [Wan2.1-Fun-14B-InP](https://huggingface.co/alibaba-pai/Wan2.1-Fun-14B-InP) | [MPS](https://github.com/Kwai-Kolors/MPS) | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-Reward-LoRAs/resolve/main/Wan2.1-Fun-14B-InP-MPS.safetensors) | Official MPS reward LoRA (`rank=128` and `network_alpha=64`) for Wan2.1-Fun-14B-InP. It is trained with a batch size of 8 for 4,500 steps.| + +## Demo +### Wan2.1-Fun-1.3B-InP + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PromptWan2.1-Fun-1.3B-InPWan2.1-Fun-1.3B-InP
HPSv2.1 Reward LoRA		Wan2.1-Fun-1.3B-InP
MPS Reward LoRA
+ A kangaroo bounds across the plain and a cow grazes +
+ Expanded +

In a vast, sun-drenched Australian plain, a lively kangaroo bounds with powerful leaps across the dry grass, its shadow following closely. Nearby, a serene brown and white cow grazes leisurely, its tail swishing gently in the warm breeze. The sky is a vibrant blue, dotted with fluffy clouds.

+
+ 		+ + + + + +
+ A penguin waddles on the ice, a camel treks by +
+ Expanded +

A small penguin waddles slowly across a vast, icy surface under a clear blue sky. The penguin's short, flipper-like wings sway at its sides as it moves. Nearby, a camel treks steadily, its long legs navigating the snowy terrain with ease. The camel's fur is thick, providing warmth in the cold environment.

+
+ 		+ + + + + +
+ Porcelain rabbit hopping by a golden cactus +
+ Expanded +

A delicate porcelain rabbit, with intricate painted details, hops gracefully across a sandy desert floor. Nearby, a golden cactus stands tall, its metallic surface glimmering in the sunlight. The backdrop features rolling sand dunes under a clear blue sky, casting gentle shadows.

+
+ 		+ + + + + +
+ Pig with wings flying above a diamond mountain +
+ Expanded +

A whimsical pig, complete with delicate feathered wings, soars gracefully above a shimmering diamond mountain. The pig's pink skin glistens in the sunlight as it flaps its wings. The mountain below sparkles with countless facets, reflecting brilliant rays of light into the clear blue sky.

+
+ 		+ + + + + +
+ +> [!NOTE] +> The above test prompts are from T2V-CompBench and expanded into detailed prompts by Llama-3.3. +> Videos are generated with HPSv2.1 Reward LoRA weight 0.5 and MPS Reward LoRA weight 0.7. + +### Wan2.1-Fun-14B-InP + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
PromptWan2.1-Fun-1.3B-InPWan2.1-Fun-1.3B-InP
HPSv2.1 Reward LoRA		Wan2.1-Fun-1.3B-InP
MPS Reward LoRA
+ A panda eats bamboo while a monkey swings from branch to branch +
+ Expanded +

In a lush green forest, a panda sits comfortably against a tree, leisurely munching on bamboo stalks. Nearby, a lively monkey swings energetically from branch to branch, its tail curling around the limbs. Sunlight filters through the canopy, casting dappled shadows on the forest floor.

+
+ 		+ + + + + +
+ A dog runs through a field while a cat climbs a tree +
+ Expanded +

In a sunlit, expansive green field surrounded by tall trees, a playful golden retriever sprints energetically across the grass, its fur gleaming in the afternoon sun. Nearby, a nimble tabby cat gracefully climbs a sturdy tree, its claws gripping the bark effortlessly. The sky is clear blue with occasional birds flying.

+
+ 		+ + + + + +
+ Elderly artist with a white beard painting on a white canvas +
+ Expanded +

An elderly artist with a long white beard stands in a sunlit studio surrounded by art supplies. He wears a paint-splattered apron over a casual shirt. His hand moves gracefully as he paints vibrant colors on a large white canvas positioned on an easel. The studio is filled with natural light streaming through tall windows, highlighting the textures of his work.

+
+ 		+ + + + + +
+ Pig with wings flying above a diamond mountain +
+ Expanded +

A whimsical pig, complete with delicate feathered wings, soars gracefully above a shimmering diamond mountain. The pig's pink skin glistens in the sunlight as it flaps its wings. The mountain below sparkles with countless facets, reflecting brilliant rays of light into the clear blue sky.

+
+ 		+ + + + + +
+ + +> [!NOTE] +> The above test prompts are from T2V-CompBench and expanded into detailed prompts by Llama-3.3. +> Videos are generated with HPSv2.1 Reward LoRA weight 0.7 and MPS Reward LoRA weight 0.7. + +## Quick Start +Set `lora_path` and `lora_weight` in [examples/wan2.1_fun/predict_t2v.py](https://github.com/aigc-apps/VideoX-Fun/blob/main/examples/wan2.1_fun/predict_t2v.py). + +## Limitations +1. We observe after training to a certain extent, the reward continues to increase, but the quality of the generated videos does not further improve. + The model trickly learns some shortcuts (by adding artifacts in the background, i.e., adversarial patches) to increase the reward. +2. Currently, there is still a lack of suitable preference models for video generation. Directly using image preference models cannot + evaluate preferences along the temporal dimension (such as dynamism and consistency). Further more, We find using image preference models leads to a decrease + in the dynamism of generated videos. Although this can be mitigated by computing the reward using only the first frame of the decoded video, the impact still persists. + +## Reference +
    +
  1. Clark, Kevin, et al. "Directly fine-tuning diffusion models on differentiable rewards.". In ICLR 2024.
    2. +
  2. Prabhudesai, Mihir, et al. "Aligning text-to-image diffusion models with reward backpropagation." arXiv preprint arXiv:2310.03739 (2023).
    3. +
diff --git a/VideoX-Fun/scripts/wan2.1_fun/train.py b/VideoX-Fun/scripts/wan2.1_fun/train.py new file mode 100644 index 0000000000000000000000000000000000000000..7c2a4faa4dc3dc4ad8b5d30f9129dd2bd78e40bb --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/train.py @@ -0,0 +1,1941 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.pipeline import WanFunInpaintPipeline, WanFunPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = WanFunInpaintPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + else: + pipeline = WanFunPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Clip Image Encoder + if args.train_mode != "normal": + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + # Get Transformer + transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + if args.train_mode != "normal": + clip_image_encoder.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=WanTransformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = WanTransformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = WanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=WanTransformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = WanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size]) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + local_video_sample_size = np.random.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if args.train_mode != "normal": + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + if args.train_mode != "normal": + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if args.train_mode != "normal": + clip_image_encoder.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "normal": + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask = resize_mask(1 - mask, latents) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + + if rng is None: + zero_init_clip_in = np.random.choice([True, False], p=[0.1, 0.9]) + else: + zero_init_clip_in = rng.choice([True, False], p=[0.1, 0.9]) + clip_context.append(_clip_context if not zero_init_clip_in else torch.zeros_like(_clip_context)) + + clip_context = torch.cat(clip_context) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + if args.train_mode != "normal": + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1_fun/train.sh b/VideoX-Fun/scripts/wan2.1_fun/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..385f03390224e31cc2f184a45c57d8f472809658 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/train.sh @@ -0,0 +1,85 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." + +# # Training command for T2V +# export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +# export DATASET_NAME="datasets/internal_datasets/" +# export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +# NCCL_DEBUG=INFO + +# accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train.py \ +# --config_path="config/wan2.1/wan_civitai.yaml" \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --train_data_dir=$DATASET_NAME \ +# --train_data_meta=$DATASET_META_NAME \ +# --image_sample_size=640 \ +# --video_sample_size=640 \ +# --token_sample_size=640 \ +# --video_sample_stride=2 \ +# --video_sample_n_frames=81 \ +# --train_batch_size=1 \ +# --video_repeat=1 \ +# --gradient_accumulation_steps=1 \ +# --dataloader_num_workers=8 \ +# --num_train_epochs=100 \ +# --checkpointing_steps=50 \ +# --learning_rate=2e-05 \ +# --lr_scheduler="constant_with_warmup" \ +# --lr_warmup_steps=100 \ +# --seed=42 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --vae_mini_batch=1 \ +# --max_grad_norm=0.05 \ +# --random_hw_adapt \ +# --training_with_video_token_length \ +# --enable_bucket \ +# --uniform_sampling \ +# --low_vram \ +# --train_mode="normal" \ +# --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1_fun/train_control.py b/VideoX-Fun/scripts/wan2.1_fun/train_control.py new file mode 100644 index 0000000000000000000000000000000000000000..85a05a2303810c84f91672f890b84affa8140d61 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/train_control.py @@ -0,0 +1,1978 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask, + process_pose_file, + process_pose_params) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.pipeline import WanFunControlPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + pipeline = WanFunControlPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int(args.video_sample_n_frames // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(args.validation_paths[i], video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + control_video = input_video, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_paths", + type=str, + default=None, + nargs="+", + help=("A set of control videos evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="control", + help=( + 'The format of training data. Support `"control"`' + ' (default), `"control_ref"`, `"control_camera_ref"`.' + ), + ) + parser.add_argument( + "--control_ref_image", + type=str, + default="first_frame", + help=( + 'The format of training data. Support `"first_frame"`' + ' (default), `"random"`.' + ), + ) + parser.add_argument( + "--add_full_ref_image_in_self_attention", + action="store_true", + help=( + 'Whether enable add full ref image in self attention.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Clip Image Encoder + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + # Get Transformer + transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + clip_image_encoder.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=WanTransformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = WanTransformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = WanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=WanTransformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = WanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoControlDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + enable_camera_info=args.train_mode == "control_camera_ref" + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Control Mode + new_examples["control_pixel_values"] = [] + # Used in Control Ref Mode + if args.train_mode != "control": + new_examples["ref_pixel_values"] = [] + new_examples["clip_pixel_values"] = [] + new_examples["clip_idx"] = [] + # Used in Control Camera Ref Mode + if args.train_mode == "control_camera_ref": + new_examples["control_camera_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size]) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + + def get_random_downsample_probability(choice_list, token_sample_size): + length = len(choice_list) + if length == 1: + return [1.0] # If there's only one element, it gets all the probability + + # Find the index of the closest value to token_sample_size + closest_index = min(range(length), key=lambda i: abs(choice_list[i] - token_sample_size)) + + # Assign 50% to the closest index + first_element = 0.50 + remaining_sum = 1.0 - first_element + + # Distribute the remaining 50% evenly among the other elements + other_elements_value = remaining_sum / (length - 1) if length > 1 else 0.0 + + # Construct the probability distribution + probability_list = [other_elements_value] * length + probability_list[closest_index] = first_element + + return probability_list + + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + probabilities = get_random_downsample_probability(choice_list, args.token_sample_size) + local_video_sample_size = np.random.choice(choice_list, p=probabilities) + + random_downsample_ratio = args.video_sample_size / local_video_sample_size + batch_video_length = length_to_frame_num[local_video_sample_size] + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + + if args.train_mode == "control_camera_ref": + control_camera_values = example.get("control_camera_values", None) + if control_camera_values is None: + control_camera_values_size = ( + new_examples["control_pixel_values"][-1].size()[0], + 6, + new_examples["control_pixel_values"][-1].size()[2], + new_examples["control_pixel_values"][-1].size()[3] + ) + local_control_camera_values = torch.zeros(control_camera_values_size) + new_examples["control_camera_values"].append(local_control_camera_values) + else: + local_control_camera_values = process_pose_params(example["control_camera_values"], height=resize_size[0], width=resize_size[1]).permute(0, 3, 1, 2).contiguous() + new_examples["control_camera_values"].append(transform_no_normalize(local_control_camera_values)) + + new_examples["text"].append(example["text"]) + + if args.train_mode != "control": + if args.control_ref_image == "first_frame": + clip_index = 0 + else: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.40 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + number_list_prob = np.array(_create_special_list(len(new_examples["pixel_values"][-1]))) + clip_index = np.random.choice(list(range(len(new_examples["pixel_values"][-1]))), p = number_list_prob) + new_examples["clip_idx"].append(clip_index) + + ref_pixel_values = new_examples["pixel_values"][-1][clip_index].unsqueeze(0) + new_examples["ref_pixel_values"].append(ref_pixel_values) + + clip_pixel_values = new_examples["pixel_values"][-1][clip_index].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + if args.train_mode != "control": + new_examples["ref_pixel_values"] = torch.stack([example for example in new_examples["ref_pixel_values"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["clip_idx"] = torch.tensor(new_examples["clip_idx"]) + if args.train_mode == "control_camera_ref": + new_examples["control_camera_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_camera_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + control_pixel_values = batch["control_pixel_values"].cpu() + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, control_pixel_value, text) in enumerate(zip(pixel_values, control_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + control_pixel_value = control_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + save_videos_grid(control_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_control.gif", rescale=True) + + if args.train_mode != "control": + ref_pixel_values = batch["ref_pixel_values"].cpu() + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + for idx, (ref_pixel_value, text) in enumerate(zip(ref_pixel_values, texts)): + ref_pixel_value = ref_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(ref_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_ref.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + if args.train_mode == "control_camera_ref": + control_camera_values = batch["control_camera_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + if args.train_mode == "control_camera_ref": + control_camera_values = torch.tile(control_camera_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + if args.train_mode == "control_camera_ref": + control_camera_values = torch.tile(control_camera_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "control": + ref_pixel_values = batch["ref_pixel_values"].to(weight_dtype) + clip_pixel_values = batch["clip_pixel_values"] + clip_idx = batch["clip_idx"] + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (4, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (4,)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (2, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (2,)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + clip_image_encoder.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "control_camera_ref": + control_latents = _batch_encode_vae(control_pixel_values) + # Make control latents to zero + for bs_index in range(control_latents.size()[0]): + if rng is None: + zero_init_control_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_control_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if zero_init_control_latents_conv_in: + control_latents[bs_index] = control_latents[bs_index] * 0 + control_camera_latents = None + else: + control_latents = None + control_camera_latents = rearrange(control_camera_values, "b f c h w -> b c f h w") + control_camera_latents = torch.concat( + [ + torch.repeat_interleave(control_camera_latents[:, :, 0:1], repeats=4, dim=2), + control_camera_latents[:, :, 1:] + ], dim=2 + ).transpose(1, 2).contiguous() + control_camera_latents = control_camera_latents.view(control_camera_latents.shape[0], control_camera_latents.shape[1] // 4, 4, control_camera_latents.shape[2], control_camera_latents.shape[3], control_camera_latents.shape[4]) + control_camera_latents = control_camera_latents.transpose(2, 3).contiguous() + control_camera_latents = control_camera_latents.view(control_camera_latents.shape[0], control_camera_latents.shape[1], control_camera_latents.shape[2] * 4, control_camera_latents.shape[4], control_camera_latents.shape[5]) + control_camera_latents = control_camera_latents.transpose(1, 2) + + if args.train_mode != "control": + ref_latents = _batch_encode_vae(ref_pixel_values) + if args.add_full_ref_image_in_self_attention: + full_ref = ref_latents[:, :, 0].clone() + + ref_latents_conv_in = torch.zeros_like(latents).to(ref_latents.device, ref_latents.dtype) + ref_latents_conv_in[:, :, :1] = ref_latents + for bs_index in range(ref_latents.size()[0]): + if rng is None: + zero_init_ref_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_ref_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if clip_idx[bs_index] != 0 or (zero_init_ref_latents_conv_in and latents.size()[1] != 1): + ref_latents_conv_in[bs_index, :, :1] = ref_latents_conv_in[bs_index, :, :1] * 0 + + if args.add_full_ref_image_in_self_attention: + if rng is None: + zero_init_full_ref_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_full_ref_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + if clip_idx[bs_index] == 0 or zero_init_full_ref_conv_in: + full_ref[bs_index] = full_ref[bs_index] * 0 + if control_latents is None: + control_latents = ref_latents_conv_in + else: + control_latents = torch.cat([control_latents, ref_latents_conv_in], dim = 1) + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + + if rng is None: + zero_init_clip_in = np.random.choice([True, False], p=[0.1, 0.9]) + else: + zero_init_clip_in = rng.choice([True, False], p=[0.1, 0.9]) + clip_context.append(_clip_context if not zero_init_clip_in else torch.zeros_like(_clip_context)) + + clip_context = torch.cat(clip_context) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=control_latents, + y_camera=control_camera_latents if args.train_mode == "control_camera_ref" else None, + clip_fea=clip_context, + full_ref=full_ref if args.add_full_ref_image_in_self_attention else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1_fun/train_control.sh b/VideoX-Fun/scripts/wan2.1_fun/train_control.sh new file mode 100644 index 0000000000000000000000000000000000000000..4cf7fbd2b48881693e9ae17f0f5b2224f547ffd3 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/train_control.sh @@ -0,0 +1,44 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_control.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_full_ref_image_in_self_attention \ + --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1_fun/train_control_lora.py b/VideoX-Fun/scripts/wan2.1_fun/train_control_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..804c543ac934eacb5008918d3b4e1660e977e42a --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/train_control_lora.py @@ -0,0 +1,2005 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask, + process_pose_file, + process_pose_params) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.pipeline import WanFunControlPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, network, config, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + pipeline = WanFunControlPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int(args.video_sample_n_frames // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(args.validation_paths[i], video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + control_video = input_video, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_paths", + type=str, + default=None, + nargs="+", + help=("A set of control videos evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="control", + help=( + 'The format of training data. Support `"control"`' + ' (default), `"control_ref"`, `"control_camera_ref"`.' + ), + ) + parser.add_argument( + "--control_ref_image", + type=str, + default="first_frame", + help=( + 'The format of training data. Support `"first_frame"`' + ' (default), `"random"`.' + ), + ) + parser.add_argument( + "--add_full_ref_image_in_self_attention", + action="store_true", + help=( + 'Whether enable add full ref image in self attention.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Clip Image Encoder + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + # Get Transformer + transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + clip_image_encoder.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoControlDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + enable_camera_info=args.train_mode == "control_camera_ref" + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Control Mode + new_examples["control_pixel_values"] = [] + # Used in Control Ref Mode + if args.train_mode != "control": + new_examples["ref_pixel_values"] = [] + new_examples["clip_pixel_values"] = [] + new_examples["clip_idx"] = [] + # Used in Control Camera Ref Mode + if args.train_mode == "control_camera_ref": + new_examples["control_camera_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size]) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + + def get_random_downsample_probability(choice_list, token_sample_size): + length = len(choice_list) + if length == 1: + return [1.0] # If there's only one element, it gets all the probability + + # Find the index of the closest value to token_sample_size + closest_index = min(range(length), key=lambda i: abs(choice_list[i] - token_sample_size)) + + # Assign 50% to the closest index + first_element = 0.50 + remaining_sum = 1.0 - first_element + + # Distribute the remaining 50% evenly among the other elements + other_elements_value = remaining_sum / (length - 1) if length > 1 else 0.0 + + # Construct the probability distribution + probability_list = [other_elements_value] * length + probability_list[closest_index] = first_element + + return probability_list + + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + probabilities = get_random_downsample_probability(choice_list, args.token_sample_size) + local_video_sample_size = np.random.choice(choice_list, p=probabilities) + + random_downsample_ratio = args.video_sample_size / local_video_sample_size + batch_video_length = length_to_frame_num[local_video_sample_size] + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + + if args.train_mode == "control_camera_ref": + control_camera_values = example.get("control_camera_values", None) + if control_camera_values is None: + control_camera_values_size = ( + new_examples["control_pixel_values"][-1].size()[0], + 6, + new_examples["control_pixel_values"][-1].size()[2], + new_examples["control_pixel_values"][-1].size()[3] + ) + local_control_camera_values = torch.zeros(control_camera_values_size) + new_examples["control_camera_values"].append(local_control_camera_values) + else: + local_control_camera_values = process_pose_params(example["control_camera_values"], height=resize_size[0], width=resize_size[1]).permute(0, 3, 1, 2).contiguous() + new_examples["control_camera_values"].append(transform_no_normalize(local_control_camera_values)) + + new_examples["text"].append(example["text"]) + + if args.train_mode != "control": + if args.control_ref_image == "first_frame": + clip_index = 0 + else: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.40 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + number_list_prob = np.array(_create_special_list(len(new_examples["pixel_values"][-1]))) + clip_index = np.random.choice(list(range(len(new_examples["pixel_values"][-1]))), p = number_list_prob) + new_examples["clip_idx"].append(clip_index) + + ref_pixel_values = new_examples["pixel_values"][-1][clip_index].unsqueeze(0) + new_examples["ref_pixel_values"].append(ref_pixel_values) + + clip_pixel_values = new_examples["pixel_values"][-1][clip_index].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + if args.train_mode != "control": + new_examples["ref_pixel_values"] = torch.stack([example for example in new_examples["ref_pixel_values"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["clip_idx"] = torch.tensor(new_examples["clip_idx"]) + if args.train_mode == "control_camera_ref": + new_examples["control_camera_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_camera_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + else: + vae_stream_1 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + control_pixel_values = batch["control_pixel_values"].cpu() + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, control_pixel_value, text) in enumerate(zip(pixel_values, control_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + control_pixel_value = control_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + save_videos_grid(control_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_control.gif", rescale=True) + + if args.train_mode != "control": + ref_pixel_values = batch["ref_pixel_values"].cpu() + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + for idx, (ref_pixel_value, text) in enumerate(zip(ref_pixel_values, texts)): + ref_pixel_value = ref_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(ref_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_ref.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + if args.train_mode == "control_camera_ref": + control_camera_values = batch["control_camera_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + if args.train_mode == "control_camera_ref": + control_camera_values = torch.tile(control_camera_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + if args.train_mode == "control_camera_ref": + control_camera_values = torch.tile(control_camera_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "control": + ref_pixel_values = batch["ref_pixel_values"].to(weight_dtype) + clip_pixel_values = batch["clip_pixel_values"] + clip_idx = batch["clip_idx"] + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (4, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (4,)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (2, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (2,)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + clip_image_encoder.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "control_camera_ref": + control_latents = _batch_encode_vae(control_pixel_values) + # Make control latents to zero + for bs_index in range(control_latents.size()[0]): + if rng is None: + zero_init_control_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_control_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if zero_init_control_latents_conv_in: + control_latents[bs_index] = control_latents[bs_index] * 0 + control_camera_latents = None + else: + control_latents = None + control_camera_latents = rearrange(control_camera_values, "b f c h w -> b c f h w") + control_camera_latents = torch.concat( + [ + torch.repeat_interleave(control_camera_latents[:, :, 0:1], repeats=4, dim=2), + control_camera_latents[:, :, 1:] + ], dim=2 + ).transpose(1, 2).contiguous() + control_camera_latents = control_camera_latents.view(control_camera_latents.shape[0], control_camera_latents.shape[1] // 4, 4, control_camera_latents.shape[2], control_camera_latents.shape[3], control_camera_latents.shape[4]) + control_camera_latents = control_camera_latents.transpose(2, 3).contiguous() + control_camera_latents = control_camera_latents.view(control_camera_latents.shape[0], control_camera_latents.shape[1], control_camera_latents.shape[2] * 4, control_camera_latents.shape[4], control_camera_latents.shape[5]) + control_camera_latents = control_camera_latents.transpose(1, 2) + + if args.train_mode != "control": + ref_latents = _batch_encode_vae(ref_pixel_values) + if args.add_full_ref_image_in_self_attention: + full_ref = ref_latents[:, :, 0].clone() + + ref_latents_conv_in = torch.zeros_like(latents).to(ref_latents.device, ref_latents.dtype) + ref_latents_conv_in[:, :, :1] = ref_latents + for bs_index in range(ref_latents.size()[0]): + if rng is None: + zero_init_ref_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_ref_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if clip_idx[bs_index] != 0 or (zero_init_ref_latents_conv_in and latents.size()[1] != 1): + ref_latents_conv_in[bs_index, :, :1] = ref_latents_conv_in[bs_index, :, :1] * 0 + + if args.add_full_ref_image_in_self_attention: + if rng is None: + zero_init_full_ref_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_full_ref_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + if clip_idx[bs_index] == 0 or zero_init_full_ref_conv_in: + full_ref[bs_index] = full_ref[bs_index] * 0 + if control_latents is None: + control_latents = ref_latents_conv_in + else: + control_latents = torch.cat([control_latents, ref_latents_conv_in], dim = 1) + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + + if rng is None: + zero_init_clip_in = np.random.choice([True, False], p=[0.1, 0.9]) + else: + zero_init_clip_in = rng.choice([True, False], p=[0.1, 0.9]) + clip_context.append(_clip_context if not zero_init_clip_in else torch.zeros_like(_clip_context)) + + clip_context = torch.cat(clip_context) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=control_latents, + y_camera=control_camera_latents if args.train_mode == "control_camera_ref" else None, + clip_fea=clip_context, + full_ref=full_ref if args.add_full_ref_image_in_self_attention else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1_fun/train_control_lora.sh b/VideoX-Fun/scripts/wan2.1_fun/train_control_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..a5df6dd0fee39354e76ef0453409dfc4f0fae312 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/train_control_lora.sh @@ -0,0 +1,45 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_control_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_full_ref_image_in_self_attention \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1_fun/train_lora.py b/VideoX-Fun/scripts/wan2.1_fun/train_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..6fc114f5ceebdbbf5d13891e968ac7978ceec7f7 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/train_lora.py @@ -0,0 +1,1963 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.pipeline import WanFunInpaintPipeline, WanFunPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, network, config, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = WanFunInpaintPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + else: + pipeline = WanFunPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"inpaint"`.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Clip Image Encoder + if args.train_mode != "normal": + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + # Get Transformer + transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + if args.train_mode != "normal": + clip_image_encoder.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size]) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + local_video_sample_size = np.random.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if args.train_mode != "normal": + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + if args.train_mode != "normal": + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if args.train_mode != "normal": + clip_image_encoder.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "normal": + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask = resize_mask(1 - mask, latents) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + + if rng is None: + zero_init_clip_in = np.random.choice([True, False], p=[0.1, 0.9]) + else: + zero_init_clip_in = rng.choice([True, False], p=[0.1, 0.9]) + clip_context.append(_clip_context if not zero_init_clip_in else torch.zeros_like(_clip_context)) + + clip_context = torch.cat(clip_context) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + if args.train_mode != "normal": + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + clip_fea=clip_context if args.train_mode != "normal" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1_fun/train_lora.sh b/VideoX-Fun/scripts/wan2.1_fun/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..4d0aa0c27b59cb4b86e46c12a3c4d56f7b64b6f4 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/train_lora.sh @@ -0,0 +1,87 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="inpaint" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram + +# # Training command for T2V +# export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-14B-InP" +# export DATASET_NAME="datasets/internal_datasets/" +# export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +# NCCL_DEBUG=INFO + +# accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_lora.py \ +# --config_path="config/wan2.1/wan_civitai.yaml" \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --train_data_dir=$DATASET_NAME \ +# --train_data_meta=$DATASET_META_NAME \ +# --image_sample_size=640 \ +# --video_sample_size=640 \ +# --token_sample_size=640 \ +# --video_sample_stride=2 \ +# --video_sample_n_frames=81 \ +# --train_batch_size=1 \ +# --video_repeat=1 \ +# --gradient_accumulation_steps=1 \ +# --dataloader_num_workers=8 \ +# --num_train_epochs=100 \ +# --checkpointing_steps=50 \ +# --learning_rate=1e-04 \ +# --seed=42 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --vae_mini_batch=1 \ +# --max_grad_norm=0.05 \ +# --random_hw_adapt \ +# --training_with_video_token_length \ +# --enable_bucket \ +# --uniform_sampling \ +# --rank=64 \ +# --network_alpha=32 \ +# --target_name="q,k,v,ffn.0,ffn.2" \ +# --use_peft_lora \ +# --low_vram \ +# --train_mode="normal" \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1_fun/train_reward_lora.py b/VideoX-Fun/scripts/wan2.1_fun/train_reward_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..9e24adad9a44d297ac067f9caab62c92f4ef3735 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/train_reward_lora.py @@ -0,0 +1,1438 @@ +"""Modified from VideoX-Fun/scripts/wan2.1_fun/train_lora.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import json +import logging +import math +import os +import random +import shutil +import sys +from contextlib import contextmanager +from typing import List, Optional + +import accelerate +import diffusers +import numpy as np +import torch +import torch.utils.checkpoint +import torchvision.transforms as transforms +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from decord import VideoReader +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.utils import check_min_version, is_wandb_available +from diffusers.utils.import_utils import is_xformers_available +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +import videox_fun.reward.reward_fn as reward_fn +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, + WanTransformer3DModel) +from videox_fun.pipeline import WanFunInpaintPipeline +from videox_fun.utils.lora_utils import create_network, merge_lora +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +@contextmanager +def video_reader(*args, **kwargs): + """A context manager to solve the memory leak of decord. + """ + vr = VideoReader(*args, **kwargs) + try: + yield vr + finally: + del vr + gc.collect() + + +def log_validation( + vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, network, + loss_fn, config, args, accelerator, weight_dtype, global_step, validation_prompts_idx +): + try: + logger.info("Running validation... ") + + transformer3d_val = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Initialize a new vae if gradient checkpointing or model cpu offload is enabled. + if args.vae_gradient_checkpointing or args.low_vram: + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Initialize a new image encoder if model cpu offload is enabled. + if args.low_vram: + image_encoder_subpath = config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder') + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, image_encoder_subpath), + ) + clip_image_encoder.eval() + pipeline = WanFunInpaintPipeline( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + pipeline = pipeline.to(dtype=weight_dtype) + if args.low_vram: + pipeline.enable_model_cpu_offload() + else: + pipeline = pipeline.to(device=accelerator.device) + lora_state_dict = accelerator.unwrap_model(network).state_dict() + pipeline = merge_lora(pipeline, None, 1, accelerator.device, state_dict=lora_state_dict, transformer_only=True) + + to_tensor = transforms.ToTensor() + validation_loss, validation_reward = 0, 0 + for i in range(len(validation_prompts_idx)): + validation_idx, validation_prompt = validation_prompts_idx[i] + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + temporal_compression_ratio = vae.config.temporal_compression_ratio + video_length = 1 + if args.video_length != 1: + video_length += int((args.video_length - 1) // temporal_compression_ratio * temporal_compression_ratio) + sample_size = [args.validation_sample_height, args.validation_sample_width] + input_video, input_video_mask, clip_image = get_image_to_video_latent( + None, None, video_length=video_length, sample_size=sample_size + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + sample = pipeline( + validation_prompt, + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.validation_sample_height, + width = args.validation_sample_width, + guidance_scale = 7, + generator = generator, + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + ).videos + sample_saved_name = f"validation_sample/sample-{global_step}-{validation_idx}.mp4" + sample_saved_path = os.path.join(args.output_dir, sample_saved_name) + save_videos_grid(sample, sample_saved_path, fps=16) + + num_sampled_frames = 4 + sampled_frames_list = [] + with video_reader(sample_saved_path) as vr: + sampled_frame_idx_list = np.linspace(0, len(vr), num_sampled_frames, endpoint=False, dtype=int) + sampled_frame_list = vr.get_batch(sampled_frame_idx_list).asnumpy() + sampled_frames = torch.stack([to_tensor(frame) for frame in sampled_frame_list], dim=0) + sampled_frames_list.append(sampled_frames) + + sampled_frames = torch.stack(sampled_frames_list) + sampled_frames = rearrange(sampled_frames, "b t c h w -> b c t h w") + loss, reward = loss_fn(sampled_frames, [validation_prompt]) + validation_loss, validation_reward = validation_loss + loss, validation_reward + reward + + validation_loss = validation_loss / len(validation_prompts_idx) + validation_reward = validation_reward / len(validation_prompts_idx) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return validation_loss, validation_reward + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None, None + + +def load_prompts(prompt_path, prompt_column="prompt", start_idx=None, end_idx=None): + prompt_list = [] + if prompt_path.endswith(".txt"): + with open(prompt_path, "r") as f: + for line in f: + prompt_list.append(line.strip()) + elif prompt_path.endswith(".jsonl"): + with open(prompt_path, "r") as f: + for line in f.readlines(): + item = json.loads(line) + prompt_list.append(item[prompt_column]) + else: + raise ValueError("The prompt_path must end with .txt or .jsonl.") + prompt_list = prompt_list[start_idx:end_idx] + + return prompt_list + + +def _get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + +def encode_prompt( + tokenizer, + text_encoder, + prompt, + negative_prompt, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = _get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = _get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + + +# Modified from EasyAnimateInpaintPipeline.prepare_extra_step_kwargs +def prepare_extra_step_kwargs(scheduler, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + import inspect + + accepts_eta = "eta" in set(inspect.signature(scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--validation_prompt_path", + type=str, + default=None, + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_batch_size", + type=int, + default=1, + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_sample_height", + type=int, + default=512, + help="The height of sampling videos in validation.", + ) + parser.add_argument( + "--validation_sample_width", + type=int, + default=512, + help="The width of sampling videos in validation.", + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing (for DiT) to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--vae_gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing (for VAE) to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + + parser.add_argument( + "--prompt_path", + type=str, + default="normal", + help="The path to the training prompt file.", + ) + parser.add_argument( + '--train_sample_height', + type=int, + default=384, + help='The height of sampling videos in training' + ) + parser.add_argument( + '--train_sample_width', + type=int, + default=672, + help='The width of sampling videos in training' + ) + parser.add_argument( + "--video_length", + type=int, + default=49, + help="The number of frames to generate in training and validation." + ) + parser.add_argument( + '--eta', + type=float, + default=0.0, + help='eta parameter for the DDIM sampler. this controls the amount of noise injected into the sampling process, ' + 'with 0.0 being fully deterministic and 1.0 being equivalent to the DDPM sampler.' + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=6.0, + help="The classifier-free diffusion guidance." + ) + parser.add_argument( + "--num_inference_steps", + type=int, + default=50, + help="The number of denoising steps in training and validation." + ) + parser.add_argument( + "--num_decoded_latents", + type=int, + default=3, + help="The number of latents to be decoded." + ) + parser.add_argument( + "--num_sampled_frames", + type=int, + default=None, + help="The number of sampled frames for the reward function." + ) + parser.add_argument( + "--reward_fn", + type=str, + default="aesthetic_loss_fn", + help='The reward function.' + ) + parser.add_argument( + "--reward_fn_kwargs", + type=str, + default=None, + help='The keyword arguments of the reward function.' + ) + parser.add_argument( + "--backprop", + action="store_true", + default=False, + help="Whether to use the reward backprop training mode.", + ) + parser.add_argument( + "--backprop_step_list", + nargs="+", + type=int, + default=None, + help="The preset step list for reward backprop. If provided, overrides `backprop_strategy`." + ) + parser.add_argument( + "--backprop_strategy", + choices=["last", "tail", "uniform", "random"], + default="last", + help="The strategy for reward backprop." + ) + parser.add_argument( + "--stop_latent_model_input_gradient", + action="store_true", + default=False, + help="Whether to stop the gradient of the latents during reward backprop.", + ) + parser.add_argument( + "--backprop_random_start_step", + type=int, + default=0, + help="The random start step for reward backprop. Only used when `backprop_strategy` is random." + ) + parser.add_argument( + "--backprop_random_end_step", + type=int, + default=50, + help="The random end step for reward backprop. Only used when `backprop_strategy` is random." + ) + parser.add_argument( + "--backprop_num_steps", + type=int, + default=5, + help="The number of steps for backprop. Only used when `backprop_strategy` is tail/uniform/random." + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # Sanity check for validation + do_validation = (args.validation_prompt_path is not None or args.validation_prompts is not None) + if do_validation: + if not (os.path.exists(args.validation_prompt_path) or args.validation_prompt_path.endswith(".txt")): + raise ValueError("The `--validation_prompt_path` must be a txt file containing prompts.") + if args.validation_batch_size < accelerator.num_processes or args.validation_batch_size % accelerator.num_processes != 0: + raise ValueError("The `--validation_batch_size` must be divisible by the number of processes.") + + # Sanity check for validation + if args.backprop: + if args.backprop_step_list is not None: + logger.warning( + f"The backprop_strategy {args.backprop_strategy} will be ignored " + f"when using backprop_step_list {args.backprop_step_list}." + ) + assert any(step <= args.num_inference_steps - 1 for step in args.backprop_step_list) + else: + if args.backprop_strategy in set(["tail", "uniform", "random"]): + assert args.backprop_num_steps <= args.num_inference_steps - 1 + if args.backprop_strategy == "random": + assert args.backprop_random_start_step <= args.backprop_random_end_step + assert args.backprop_random_end_step <= args.num_inference_steps - 1 + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed, device_specific=True) + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + transformer3d = WanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ) + + # Get Clip Image Encoder + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder.eval() + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + clip_image_encoder.requires_grad_(False) + + # Lora will work with this... + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + add_lora_in_attn_temporal=True, + ) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + # Load transformer and vae from path if it needs. + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + accelerator.register_save_state_pre_hook(save_model_hook) + # Save the model weights directly before save_state instead of using a hook. + # accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + if args.vae_gradient_checkpointing: + # Since 3D casual VAE need a cache to decode all latents autoregressively, .Thus, gradient checkpointing can only be + # enabled when decoding the first batch (i.e. the first three) of latents, in which case the cache is not being used. + + # num_decoded_latents > 3 is support in EasyAnimate now. + # if args.num_decoded_latents > 3: + # raise ValueError("The vae_gradient_checkpointing is not supported for num_decoded_latents > 3.") + vae.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + # Init optimizer + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # loss function + reward_fn_kwargs = {} + if args.reward_fn_kwargs is not None: + reward_fn_kwargs = json.loads(args.reward_fn_kwargs) + if accelerator.is_main_process: + # Check if the model is downloaded in the main process. + loss_fn = getattr(reward_fn, args.reward_fn)(device="cpu", dtype=weight_dtype, **reward_fn_kwargs) + accelerator.wait_for_everyone() + loss_fn = getattr(reward_fn, args.reward_fn)(device=accelerator.device, dtype=weight_dtype, **reward_fn_kwargs) + + # Get RL training prompts + prompt_list = load_prompts(args.prompt_path) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(prompt_list) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + network, optimizer, lr_scheduler = accelerator.prepare(network, optimizer, lr_scheduler) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device, dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + text_encoder.to(accelerator.device) + clip_image_encoder.to(accelerator.device) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(prompt_list) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(prompt_list)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + from safetensors.torch import load_file, safe_open + state_dict = load_file(os.path.join(os.path.join(args.output_dir, path), "lora_diffusion_pytorch_model.safetensors")) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + train_reward = 0.0 + + # In the following training loop, randomly select training prompts and use the + # `EasyAnimatePipelineInpaint` to sample videos, calculate rewards, and update the network. + for _ in range(num_update_steps_per_epoch): + # train_prompt = random.sample(prompt_list, args.train_batch_size) + train_prompt = random.choices(prompt_list, k=args.train_batch_size) + logger.info(f"train_prompt: {train_prompt}") + + # default height and width + height = int(args.train_sample_height // 16 * 16) + width = int(args.train_sample_width // 16 * 16) + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = args.guidance_scale > 1.0 + + # Reduce the vram by offload text encoders + if args.low_vram: + torch.cuda.empty_cache() + text_encoder.to(accelerator.device) + + # Encode input prompt + ( + prompt_embeds, + negative_prompt_embeds + ) = encode_prompt( + tokenizer, + text_encoder, + train_prompt, + negative_prompt=[""] * len(train_prompt), + device=accelerator.device, + dtype=weight_dtype, + do_classifier_free_guidance=do_classifier_free_guidance, + ) + if do_classifier_free_guidance: + prompt_embeds = negative_prompt_embeds + prompt_embeds + + # Reduce the vram by offload text encoders + if args.low_vram: + text_encoder.to("cpu") + torch.cuda.empty_cache() + + # Prepare timesteps + if hasattr(noise_scheduler, "use_dynamic_shifting") and noise_scheduler.use_dynamic_shifting: + noise_scheduler.set_timesteps(args.num_inference_steps, device=accelerator.device, mu=1) + else: + noise_scheduler.set_timesteps(args.num_inference_steps, device=accelerator.device) + timesteps = noise_scheduler.timesteps + + # Prepare latent variables + vae_scale_factor = vae.spatial_compression_ratio + latent_shape = [ + args.train_batch_size, + vae.config.latent_channels, + int((args.video_length - 1) // vae.temporal_compression_ratio + 1) if args.video_length != 1 else 1, + args.train_sample_height // vae_scale_factor, + args.train_sample_width // vae_scale_factor, + ] + + with accelerator.accumulate(transformer3d): + latents = torch.randn(*latent_shape, device=accelerator.device, dtype=weight_dtype) + + # Prepare inpaint latents if it needs. + # Use zero latents if we want to t2v. + mask_latents = torch.zeros_like(latents)[:, :4].to(latents.device, latents.dtype) + masked_video_latents = torch.zeros_like(latents).to(latents.device, latents.dtype) + + mask_input = torch.cat([mask_latents] * 2) if do_classifier_free_guidance else mask_latents + masked_video_latents_input = ( + torch.cat([masked_video_latents] * 2) if do_classifier_free_guidance else masked_video_latents + ) + inpaint_latents = torch.cat([mask_input, masked_video_latents_input], dim=1).to(latents.dtype) + + if hasattr(noise_scheduler, "init_noise_sigma"): + latents = latents * noise_scheduler.init_noise_sigma + + clip_context = [] + for _ in range(args.train_batch_size): + clip_image = Image.new("RGB", (512, 512), color=(0, 0, 0)) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(latents.device, latents.dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + clip_context.append(torch.zeros_like(_clip_context)) + clip_context = torch.cat(clip_context) + clip_context = ( + torch.cat([clip_context] * 2) if do_classifier_free_guidance else clip_context + ) + clip_context = torch.zeros_like(clip_context) + + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + # Prepare extra step kwargs. + extra_step_kwargs = prepare_extra_step_kwargs(noise_scheduler, generator, args.eta) + + bsz, channel, num_frames, height, width = latents.size() + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Denoising loop + if args.backprop: + if args.backprop_step_list is None: + if args.backprop_strategy == "last": + backprop_step_list = [args.num_inference_steps - 1] + elif args.backprop_strategy == "tail": + backprop_step_list = list(range(args.num_inference_steps))[-args.backprop_num_steps:] + elif args.backprop_strategy == "uniform": + interval = args.num_inference_steps // args.backprop_num_steps + random_start = random.randint(0, interval) + backprop_step_list = [random_start + i * interval for i in range(args.backprop_num_steps)] + elif args.backprop_strategy == "random": + backprop_step_list = random.sample( + range(args.backprop_random_start_step, args.backprop_random_end_step + 1), args.backprop_num_steps + ) + else: + raise ValueError(f"Invalid backprop strategy: {args.backprop_strategy}.") + else: + backprop_step_list = args.backprop_step_list + + for i, t in enumerate(tqdm(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(noise_scheduler, "scale_model_input"): + latent_model_input = noise_scheduler.scale_model_input(latent_model_input, t) + + # expand scalar t to 1-D tensor to match the 1st dim of latent_model_input + t_expand = torch.tensor([t] * latent_model_input.shape[0], device=accelerator.device).to( + dtype=latent_model_input.dtype + ) + + # Whether to enable DRTune: https://arxiv.org/abs/2405.00760 + if args.stop_latent_model_input_gradient: + latent_model_input = latent_model_input.detach() + + # predict noise model_output + with accelerator.autocast(): + noise_pred = transformer3d( + x=latent_model_input, + context=prompt_embeds, + t=t_expand, + seq_len=seq_len, + y=inpaint_latents, + clip_fea=clip_context + ) + + # Optimize the denoising results only for the specified steps. + if i in backprop_step_list: + noise_pred = noise_pred + else: + noise_pred = noise_pred.detach() + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred[0], noise_pred[1] + noise_pred = noise_pred_uncond + args.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = noise_scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + # decode latents (tensor) + # latents = latents.permute(0, 2, 1, 3, 4) # [B, C, T, H, W] + # Since the casual VAE decoding consumes a large amount of VRAM, and we need to keep the decoding + # operation within the computational graph. Thus, we only decode the first args.num_decoded_latents + # to calculate the reward. + # TODO: Decode all latents but keep a portion of the decoding operation within the computational graph. + sampled_latent_indices = list(range(args.num_decoded_latents)) + sampled_latents = latents[:, :, sampled_latent_indices, :, :] + sampled_frames = vae.decode(sampled_latents.to(vae.device, vae.dtype))[0] + sampled_frames = sampled_frames.clamp(-1, 1) + sampled_frames = (sampled_frames / 2 + 0.5).clamp(0, 1) # [-1, 1] -> [0, 1] + + if global_step % args.checkpointing_steps == 0: + saved_file = f"sample-{global_step}-{accelerator.process_index}.mp4" + save_videos_grid( + sampled_frames.to(torch.float32).detach().cpu(), + os.path.join(args.output_dir, "train_sample", saved_file), + fps=16 + ) + + if args.num_sampled_frames is not None: + num_frames = sampled_frames.size(2) - 1 + sampled_frames_indices = torch.linspace(0, num_frames, steps=args.num_sampled_frames).long() + sampled_frames = sampled_frames[:, :, sampled_frames_indices, :, :] + # compute loss and reward + loss, reward = loss_fn(sampled_frames, train_prompt) + + # Gather the losses and rewards across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + avg_reward = accelerator.gather(reward.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + train_reward += avg_reward.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + total_norm = accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + # If use_deepspeed, `total_norm` cannot be logged by accelerator. + if not args.use_deepspeed: + accelerator.log({"total_norm": total_norm}, step=global_step) + else: + if hasattr(optimizer, "optimizer") and hasattr(optimizer.optimizer, "_global_grad_norm"): + accelerator.log({"total_norm": optimizer.optimizer._global_grad_norm}, step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss, "train_reward": train_reward}, step=global_step) + train_loss = 0.0 + train_reward = 0.0 + + if global_step % args.checkpointing_steps == 0: + # DeepSpeed requires saving weights on every device; saving weights only on the main process would cause issues. + if args.use_deepspeed or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + # Validation (distributed) + if do_validation and (global_step % args.validation_steps) == 0: + if args.validation_prompts is None and args.validation_prompt_path.endswith(".txt"): + validation_prompts = [] + with open(args.validation_prompt_path, "r") as f: + for line in f: + validation_prompts.append(line.strip()) + # Do not select randomly to ensure that `args.validation_prompts` is the same for each process. + args.validation_prompts = validation_prompts[:args.validation_batch_size] + validation_prompts_idx = [(i, p) for i, p in enumerate(args.validation_prompts)] + + if hasattr(vae, "enable_cache_in_vae"): + vae.enable_cache_in_vae() + accelerator.wait_for_everyone() + with accelerator.split_between_processes(validation_prompts_idx) as splitted_prompts_idx: + validation_loss, validation_reward = log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + network, + loss_fn, + config, + args, + accelerator, + weight_dtype, + global_step, + splitted_prompts_idx + ) + if validation_loss is not None and validation_reward is not None: + avg_validation_loss = accelerator.gather(validation_loss).mean() + avg_validation_reward = accelerator.gather(validation_reward).mean() + accelerator.print(avg_validation_loss, avg_validation_reward) + if accelerator.is_main_process: + accelerator.log( + {"validation_loss": avg_validation_loss, "validation_reward": avg_validation_reward}, + step=global_step + ) + + accelerator.wait_for_everyone() + + logs = {"step_loss": loss.detach().item(), "step_reward": reward.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1_fun/train_reward_lora.sh b/VideoX-Fun/scripts/wan2.1_fun/train_reward_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..f222bfff727af678dba855d36a69f46bce6bafcc --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_fun/train_reward_lora.sh @@ -0,0 +1,37 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-Fun-V1.1-1.3B-InP" +export TRAIN_PROMPT_PATH="MovieGenVideoBench_train.txt" +# Performing validation simultaneously with training will increase time and GPU memory usage. +export VALIDATION_PROMPT_PATH="MovieGenVideoBench_val.txt" +# Set 1 for Wan2.1-Fun-14B-InP +export BACKPROP_NUM_STEPS=5 + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_fun/train_reward_lora.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --rank=32 \ + --network_alpha=16 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --max_train_steps=10000 \ + --checkpointing_steps=100 \ + --learning_rate=1e-05 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --vae_gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --max_grad_norm=0.3 \ + --low_vram \ + --prompt_path=$TRAIN_PROMPT_PATH \ + --train_sample_height=256 \ + --train_sample_width=256 \ + --num_inference_steps=30 \ + --video_length=81 \ + --num_decoded_latents=1 \ + --reward_fn="HPSReward" \ + --reward_fn_kwargs='{"version": "v2.1"}' \ + --backprop_strategy="tail" \ + --backprop_num_steps=$BACKPROP_NUM_STEPS \ + --backprop \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1_vace/README_TRAIN.md b/VideoX-Fun/scripts/wan2.1_vace/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..d04424f96ec1bce5a693ef0fd83666a8ffe1730f --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_vace/README_TRAIN.md @@ -0,0 +1,263 @@ +## Training Code + +We can choose whether to use deepspeed or fsdp in Wan-Fun, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in Wan-Fun, you need to add a control_file_path, and [DWPose](https://github.com/IDEA-Research/DWPose) is suggested as tool to generate control file. + +```json +[ + { + "file_path": "train/00000001.mp4", + "control_file_path": "control/00000001.mp4", + "object_file_path": ["object/1.jpg", "object/2.jpg"], + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "control_file_path": "control/00000002.jpg", + "object_file_path": ["object/3.jpg", "object/4.jpg"], + "text": "Ba Da Ba Ba Ba Ba.", + "type": "image" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `train_mode` is used to set the training mode. + - The models named `Wan2.1-Fun-*-Control` are trained in the `control_ref` mode. + - The models named `Wan2.1-Fun-*-Control-Camera` are trained in the `control_ref_camera` mode. +- `control_ref_image` is used to specify the type of control image. The available options are `first_frame` and `random`. + - `first_frame` is used in V1.0 because V1.0 supports using a specified start frame as the control image. The Control-Camera models use the first frame as the control image. + - `random` is used in V1.1 because V1.1 supports both using a specified start frame and a reference image as the control image. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan-Fun-Control without deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-VACE-1.3B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_vace/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --control_ref_image="random" \ + --trainable_modules "vace" +``` + +Wan-Fun-Control with deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-VACE-1.3B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_vace/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --control_ref_image="random" \ + --trainable_modules "vace" +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan-Fun-Control with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-VACE-1.3B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.1_vace/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --control_ref_image="random" \ + --trainable_modules "vace" +``` + +Wan-Fun-Control with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-VACE-1.3B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=VaceWanAttentionBlock,BaseWanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.1_vace/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --control_ref_image="random" \ + --trainable_modules "vace" +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.1_vace/train.py b/VideoX-Fun/scripts/wan2.1_vace/train.py new file mode 100644 index 0000000000000000000000000000000000000000..b47913e4504090e0e7a35b89ee3e4c0579a111e1 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_vace/train.py @@ -0,0 +1,2073 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask, + padding_image, + process_pose_file, + process_pose_params) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, + VaceWanTransformer3DModel, WanT5EncoderModel) +from videox_fun.pipeline import WanVacePipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = VaceWanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + pipeline = WanVacePipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int(args.video_sample_n_frames // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + control_video, _, _, _ = get_video_to_video_latent(args.validation_paths[i], video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = None, + vace_context_scale = 1, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_paths", + type=str, + default=None, + nargs="+", + help=("A set of control videos evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--control_ref_image", + type=str, + default="first_frame", + help=( + 'The format of training data. Support `"first_frame"`' + ' (default), `"random"`.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + if args.train_batch_size >= 2: + raise ValueError("This code does not support args.train_batch_size >= 2 now.") + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKLWan.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + transformer3d = VaceWanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = VaceWanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=VaceWanTransformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = VaceWanTransformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = VaceWanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=VaceWanTransformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = VaceWanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoControlDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + enable_inpaint=False, + enable_camera_info=False, + enable_subject_info=True + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Control Mode + new_examples["control_pixel_values"] = [] + # Used in Control Ref Mode + new_examples["ref_pixel_values"] = [] + new_examples["clip_pixel_values"] = [] + new_examples["clip_idx"] = [] + + # Used in Inpaint mode + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + + new_examples["subject_images"] = [] + new_examples["subject_flags"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size]) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + + def get_random_downsample_probability(choice_list, token_sample_size): + length = len(choice_list) + if length == 1: + return [1.0] # If there's only one element, it gets all the probability + + # Find the index of the closest value to token_sample_size + closest_index = min(range(length), key=lambda i: abs(choice_list[i] - token_sample_size)) + + # Assign 50% to the closest index + first_element = 0.50 + remaining_sum = 1.0 - first_element + + # Distribute the remaining 50% evenly among the other elements + other_elements_value = remaining_sum / (length - 1) if length > 1 else 0.0 + + # Construct the probability distribution + probability_list = [other_elements_value] * length + probability_list[closest_index] = first_element + + return probability_list + + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + probabilities = get_random_downsample_probability(choice_list, args.token_sample_size) + local_video_sample_size = np.random.choice(choice_list, p=probabilities) + + random_downsample_ratio = args.video_sample_size / local_video_sample_size + batch_video_length = length_to_frame_num[local_video_sample_size] + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + if args.control_ref_image == "first_frame": + clip_index = 0 + else: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.40 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + number_list_prob = np.array(_create_special_list(len(pixel_values))) + clip_index = np.random.choice(list(range(len(pixel_values))), p = number_list_prob) + new_examples["clip_idx"].append(clip_index) + + ref_pixel_values = pixel_values[clip_index].permute(1, 2, 0).contiguous() + ref_pixel_values = Image.fromarray(np.uint8(ref_pixel_values * 255)) + ref_pixel_values = padding_image(ref_pixel_values, closest_size[1], closest_size[0]) + ref_pixel_values = (torch.tensor(np.array(ref_pixel_values)).unsqueeze(0).permute(0, 3, 1, 2).contiguous() / 255 - 0.5)/0.5 + new_examples["ref_pixel_values"].append(ref_pixel_values) + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + _, channel, h, w = pixel_values.size() + new_subject_image = torch.zeros(4, channel, h, w) + num_subject = len(example["subject_image"]) + if num_subject != 0: + subject_image = torch.from_numpy(example["subject_image"]).permute(0, 3, 1, 2).contiguous() + new_subject_image[:num_subject] = subject_image + subject_image = new_subject_image / 255. + subject_flag = torch.from_numpy(np.array([1] * num_subject + [0] * (4 - num_subject))) + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + + new_examples["text"].append(example["text"]) + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = int( + min( + batch_video_length, + (len(pixel_values) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1, + ) + ) + if batch_video_length == 0: + batch_video_length = 1 + + clip_pixel_values = new_examples["pixel_values"][-1][clip_index].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + new_examples["subject_images"].append(transform(subject_image)) + new_examples["subject_flags"].append(subject_flag) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + new_examples["ref_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["ref_pixel_values"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["clip_idx"] = torch.tensor(new_examples["clip_idx"]) + new_examples["mask_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example[:batch_video_length] for example in new_examples["mask"]]) + new_examples["subject_images"] = torch.stack([example for example in new_examples["subject_images"]]) + new_examples["subject_flags"] = torch.stack([example for example in new_examples["subject_flags"]]) + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + idx_sampling = DiscreteSampling(args.train_sampling_steps, start_num_idx=0, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + control_pixel_values = batch["control_pixel_values"].cpu() + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, control_pixel_value, text) in enumerate(zip(pixel_values, control_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + control_pixel_value = control_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + save_videos_grid(control_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_control.gif", rescale=True) + + ref_pixel_values = batch["ref_pixel_values"].cpu() + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + for idx, (ref_pixel_value, text) in enumerate(zip(ref_pixel_values, texts)): + ref_pixel_value = ref_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(ref_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_ref.gif", rescale=True) + + subject_images = batch["subject_images"].cpu() + subject_images = rearrange(subject_images, "b f c h w -> b c f h w") + for idx, (subject_image, text) in enumerate(zip(subject_images, texts)): + subject_image = subject_image[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(subject_image, f"{args.output_dir}/sanity_check/{gif_name[:10]}_subject.gif", rescale=True) + + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + ref_pixel_values = batch["ref_pixel_values"].to(weight_dtype) + clip_pixel_values = batch["clip_pixel_values"] + subject_images = batch["subject_images"].to(weight_dtype) + subject_flags = batch["subject_flags"].to(weight_dtype) + clip_idx = batch["clip_idx"] + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + subject_images = torch.tile(subject_images, (4, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (4, 1, 1, 1, 1)) + subject_flags = torch.tile(subject_flags, (4, 1)) + clip_idx = torch.tile(clip_idx, (4,)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + subject_images = torch.tile(subject_images, (2, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (2, 1, 1, 1, 1)) + subject_flags = torch.tile(subject_flags, (2, 1)) + clip_idx = torch.tile(clip_idx, (2,)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + def vace_encode_frames(frames, ref_images, masks=None): + weight_dtype = frames.dtype + if ref_images is None: + ref_images = [None] * len(frames) + else: + assert len(frames) == len(ref_images) + + if masks is None: + latents = vae.encode(frames)[0].mode() + else: + masks = [torch.where(m > 0.5, 1.0, 0.0).to(weight_dtype) for m in masks] + inactive = [i * (1 - m) + 0 * m for i, m in zip(frames, masks)] + reactive = [i * m + 0 * (1 - m) for i, m in zip(frames, masks)] + inactive = vae.encode(inactive)[0].mode() + reactive = vae.encode(reactive)[0].mode() + latents = [torch.cat((u, c), dim=0) for u, c in zip(inactive, reactive)] + + cat_latents = [] + for latent, refs in zip(latents, ref_images): + if refs is not None: + if masks is None: + ref_latent = vae.encode(refs)[0].mode() + else: + ref_latent = vae.encode(refs)[0].mode() + ref_latent = [torch.cat((u, torch.zeros_like(u)), dim=0) for u in ref_latent] + assert all([x.shape[1] == 1 for x in ref_latent]) + latent = torch.cat([*ref_latent, latent], dim=1) + cat_latents.append(latent) + return cat_latents + + def vace_encode_masks(masks, ref_images=None, vae_stride=[4, 8, 8]): + if ref_images is None: + ref_images = [None] * len(masks) + else: + assert len(masks) == len(ref_images) + + result_masks = [] + for mask, refs in zip(masks, ref_images): + c, depth, height, width = mask.shape + new_depth = int((depth + 3) // vae_stride[0]) + height = 2 * (int(height) // (vae_stride[1] * 2)) + width = 2 * (int(width) // (vae_stride[2] * 2)) + + # reshape + mask = mask[0, :, :, :] + mask = mask.view( + depth, height, vae_stride[1], width, vae_stride[1] + ) # depth, height, 8, width, 8 + mask = mask.permute(2, 4, 0, 1, 3) # 8, 8, depth, height, width + mask = mask.reshape( + vae_stride[1] * vae_stride[2], depth, height, width + ) # 8*8, depth, height, width + + # interpolation + mask = F.interpolate(mask.unsqueeze(0), size=(new_depth, height, width), mode='nearest-exact').squeeze(0) + + if refs is not None: + length = len(refs) + c, depth, height, width = mask.shape + mask_pad = mask.new_zeros(c, length, height, width) + mask = torch.cat((mask_pad, mask), dim=1) + result_masks.append(mask) + return result_masks + + def vace_latent(z, m): + return [torch.cat([zz, mm], dim=0) for zz, mm in zip(z, m)] + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if rng is None: + subject_images_num = np.random.choice([0, 1, 2, 3, 4]) + else: + subject_images_num = rng.choice([0, 1, 2, 3, 4]) + + if rng is None: + use_full_photo_ref_flag = np.random.choice([True, False], p=[0.25, 0.75]) + else: + use_full_photo_ref_flag = rng.choice([True, False], p=[0.25, 0.75]) + + if not use_full_photo_ref_flag: + if subject_images_num == 0: + subject_ref_images = None + else: + subject_ref_images = rearrange(subject_images, "b f c h w -> b c f h w") + subject_ref_images = subject_ref_images[:, :, :subject_images_num] + + bs, c, f, h, w = subject_ref_images.size() + new_subject_ref_images = [] + for i in range(bs): + act_subject_images_num = min(subject_images_num, int(torch.sum(subject_flags[i]))) + + if act_subject_images_num == 0: + new_subject_ref_images.append(None) + else: + new_subject_ref_images.append([]) + for j in range(act_subject_images_num): + new_subject_ref_images[i].append(subject_ref_images[i, :, j:j+1]) + subject_ref_images = new_subject_ref_images + else: + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + + bs, c, f, h, w = ref_pixel_values.size() + new_ref_pixel_values = [] + for i in range(bs): + new_ref_pixel_values.append([]) + for j in range(1): + new_ref_pixel_values[i].append(ref_pixel_values[i, :, j:j+1]) + subject_ref_images = new_ref_pixel_values + + if rng is None: + inpaint_flag = np.random.choice([True, False], p=[0.75, 0.25]) + else: + inpaint_flag = rng.choice([True, False], p=[0.75, 0.25]) + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.tile(mask, [1, 3, 1, 1, 1]) + if inpaint_flag or (control_pixel_values == -1).all(): + if rng is None: + do_not_use_ref_images = np.random.choice([True, False], p=[0.50, 0.50]) + else: + do_not_use_ref_images = rng.choice([True, False], p=[0.50, 0.50]) + if do_not_use_ref_images: + subject_ref_images = None + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + vace_latents = vace_encode_frames(mask_pixel_values, subject_ref_images, mask) + else: + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + vace_latents = vace_encode_frames(control_pixel_values, subject_ref_images, mask) + mask = torch.ones_like(mask) + + mask_latents = vace_encode_masks(mask, subject_ref_images) + vace_context = torch.stack(vace_latent(vace_latents, mask_latents)) + + if subject_ref_images is not None: + for i in range(len(subject_ref_images)): + if subject_ref_images[i] is not None: + + subject_ref_images[i] = torch.cat( + [subject_ref_image.unsqueeze(0) for subject_ref_image in subject_ref_images[i]], 2 + ) + subject_ref_images[i] = torch.cat( + [vae.encode(subject_ref_images[i][:, :, j:j+1])[0].sample() for j in range(subject_ref_images[i].size(2))], 2 + ) + + if subject_ref_images[0] is not None: + subject_ref_images = torch.cat(subject_ref_images) + latents = torch.cat( + [subject_ref_images, latents], dim=2 + ) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn( + (bsz, channel, num_frames, height, width), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, vace_latents[0].size(1), width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + vace_context=vace_context, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.1_vace/train.sh b/VideoX-Fun/scripts/wan2.1_vace/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..867964fa5f4aa84e7389c98e2048a5631658c2fd --- /dev/null +++ b/VideoX-Fun/scripts/wan2.1_vace/train.sh @@ -0,0 +1,41 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.1-VACE-1.3B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 + +accelerate launch --mixed_precision="bf16" scripts/wan2.1_vace/train.py \ + --config_path="config/wan2.1/wan_civitai.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --control_ref_image="random" \ + --trainable_modules "vace" \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/README_TRAIN.md b/VideoX-Fun/scripts/wan2.2/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..aa76febd6a061dd69f2a13307e38a5c5aca9e9e4 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/README_TRAIN.md @@ -0,0 +1,300 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal, i2v or ti2v. The t2v is used for 14B T2V model. The i2v is used for 14B I2V model. The ti2v is used in 5B TI2V model. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). + +If you want to train 5B Wan2.2 TI2V model, please set config to `config/wan2.2/wan_civitai_5b.yaml`, set train_mode to `ti2v` and set boundary_type to `full`. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan2.2 T2V without deepspeed: + +Training 14B Wan2.2 without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +Wan T2V with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan T2V with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --use_deepspeed \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +Wan T2V with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." +``` + +If you want to train 5B Wan2.2 TI2V model, please set config to `config/wan2.2/wan_civitai_5b.yaml`, set train_mode to `ti2v` and set boundary_type to `full`. Training shell command is as follows: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-TI2V-5B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="full" \ + --train_mode="ti2v" \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/README_TRAIN_ANIMATE.md b/VideoX-Fun/scripts/wan2.2/README_TRAIN_ANIMATE.md new file mode 100644 index 0000000000000000000000000000000000000000..08d7ea32bd1bef8738aacd37f9bcd670b953339d --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/README_TRAIN_ANIMATE.md @@ -0,0 +1,256 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use DeepSpeed and FSDP in Wan-Animate, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in Wan, you need to add some new paths in json. + +- Animate tiem: a control_file_path, a face_file_path and a ref_file_path. +- Replace item: a control_file_path, a face_file_path, a ref_file_path, a mask_file_path and a background_file_path. + +You can use + +```json +[ + { + "file_path": "train/00000001.mp4", + "control_file_path": "control/00000001_src_pose.mp4", + "face_file_path": "face/00000001_src_face.mp4", + "ref_file_path": "ref/00000001_src_ref.png", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.mp4", + "control_file_path": "control/00000001_src_pose.mp4", + "face_file_path": "face/00000001_src_face.mp4", + "ref_file_path": "ref/00000001_src_ref.png", + "background_file_path": "bg/00000002_src_bg.png", + "mask_file_path": "mask/00000002_src_mask.mp4", + "text": "视频中的人在做动作", + "type": "video", + "height": 480, + "width": 832 + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the videos at the center, but instead, it trains the videos after grouping them into buckets based on resolution. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for videos. When `random_hw_adapt` is enabled, for training videos, the height and width will be set to `video_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=768`, the resolution of video inputs for training is `512x512x49`, `768x768x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training videos, the height and width will be set to `video_sample_size` as the maximum and `256` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=768`, the resolution of video inputs for training is `256x256x49`, `512x512x49`, `768x768x21`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan-Animate without deepspeed: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Animate-14B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_animate.py \ + --config_path="config/wan2.2/wan_civitai_animate.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --low_vram \ + --trainable_modules "." +``` + +Wan-Animate with Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Animate-14B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_animate.py \ + --config_path="config/wan2.2/wan_civitai_animate.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --low_vram \ + --trainable_modules "." +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan-Animate with DeepSpeed Zero-3: + +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Animate-14B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_animate.py \ + --config_path="config/wan2.2/wan_civitai_animate.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --low_vram \ + --trainable_modules "." +``` + +Wan-Animate with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Animate-14B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_animate.py \ + --config_path="config/wan2.2/wan_civitai_animate.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --low_vram \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/README_TRAIN_DISTILL.md b/VideoX-Fun/scripts/wan2.2/README_TRAIN_DISTILL.md new file mode 100644 index 0000000000000000000000000000000000000000..bfffc5923d7c1c65c5d5d061854c1f110bba455f --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/README_TRAIN_DISTILL.md @@ -0,0 +1,252 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use DeepSpeed and FSDP in Wan distill, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan distill without deepspeed: + +Wan distill without DeepSpeed and FSDP is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07\ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="i2v" \ + --trainable_modules "." \ + --low_vram +``` + +Wan distill with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07\ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="i2v" \ + --trainable_modules "." \ + --low_vram +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan distill with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07\ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="i2v" \ + --trainable_modules "." \ + --low_vram +``` + +Wan distill with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07\ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="i2v" \ + --trainable_modules "." \ + --low_vram +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/README_TRAIN_DISTILL_LORA.md b/VideoX-Fun/scripts/wan2.2/README_TRAIN_DISTILL_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..8dc6a1929669c52c9e9161a77ad7a3c03df5c252 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/README_TRAIN_DISTILL_LORA.md @@ -0,0 +1,258 @@ +## Lora Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use DeepSpeed and FSDP in Wan distill, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or i2v. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan distill without deepspeed: + +Wan distill without DeepSpeed and FSDP is more suitable for 1.3B Wan, as using it with 14B Wan may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="i2v" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +Wan distill with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training 1.3B Wan and 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="i2v" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is not compatible with PEFT. + +Wan distill with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="i2v" \ + --low_vram +``` + +Wan distill with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="i2v" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/README_TRAIN_LORA.md b/VideoX-Fun/scripts/wan2.2/README_TRAIN_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..69aa36ecef5f7f70eb0dd3e8614b1375cf620c74 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/README_TRAIN_LORA.md @@ -0,0 +1,303 @@ +## Lora Training Code + +We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal, i2v or ti2v. The t2v is used for 14B T2V model. The i2v is used for 14B I2V model. The ti2v is used in 5B TI2V model. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +If you want to train 5B Wan2.2 TI2V model, please set config to `config/wan2.2/wan_civitai_5b.yaml`, set train_mode to `ti2v` and set boundary_type to `full`. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan2.2 T2V without deepspeed: + +Training 14B Wan2.2 without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +Wan T2V with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is not compatible with PEFT. + +Wan T2V with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="normal" \ + --low_vram +``` + +Wan T2V with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram +``` + +If you want to train 5B Wan2.2 TI2V model, please set config to `config/wan2.2/wan_civitai_5b.yaml`, set train_mode to `ti2v` and set boundary_type to `full`. Training shell command is as follows: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="ti2v" \ + --low_vram +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/README_TRAIN_S2V.md b/VideoX-Fun/scripts/wan2.2/README_TRAIN_S2V.md new file mode 100644 index 0000000000000000000000000000000000000000..4c377b554de80a92c9b1945185b2b6545e8fe041 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/README_TRAIN_S2V.md @@ -0,0 +1,230 @@ +## Training Code + +The default training commands for the different versions are as follows: + +We can choose whether to use DeepSpeed and FSDP in Wan-S2V, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in Wan, you need to add a audio_path. + +```json +[ + { + "file_path": "train/00000001.mp4", + "audio_path": "wav/00000001.wav", + "control_file_path": "control/00000001.mp4", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the videos at the center, but instead, it trains the videos after grouping them into buckets based on resolution. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for videos. When `random_hw_adapt` is enabled, for training videos, the height and width will be set to `video_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=768`, the resolution of video inputs for training is `512x512x49`, `768x768x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training videos, the height and width will be set to `video_sample_size` as the maximum and `256` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=768`, the resolution of video inputs for training is `256x256x49`, `512x512x49`, `768x768x21`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan-S2V without deepspeed: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_s2v.py \ + --config_path="config/wan2.2/wan_civitai_s2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=1 \ + --video_sample_n_frames=80 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --trainable_modules "." +``` + +Wan-S2V with Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_s2v.py \ + --config_path="config/wan2.2/wan_civitai_s2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=1 \ + --video_sample_n_frames=80 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --trainable_modules "." +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan-S2V with DeepSpeed Zero-3: + +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2/train_s2v.py \ + --config_path="config/wan2.2/wan_civitai_s2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=1 \ + --video_sample_n_frames=80 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --trainable_modules "." +``` + +Wan-S2V with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanS2VAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2/train_s2v.py \ + --config_path="config/wan2.2/wan_civitai_s2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=1 \ + --video_sample_n_frames=80 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/train.py b/VideoX-Fun/scripts/wan2.2/train.py new file mode 100644 index 0000000000000000000000000000000000000000..e42adf93b53a8b26e63fb30517d596ce8db1b88f --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train.py @@ -0,0 +1,1991 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedStateDictConfig, ShardedOptimStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, WanT5EncoderModel, + Wan2_2Transformer3DModel) +from videox_fun.pipeline import Wan2_2Pipeline, Wan2_2I2VPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = Wan2_2I2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + else: + pipeline = Wan2_2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=Wan2_2Transformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = Wan2_2Transformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = Wan2_2Transformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=Wan2_2Transformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = Wan2_2Transformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + if args.boundary_type == "high" or args.boundary_type == "low": + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + if args.train_mode != "normal": + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "normal": + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + + if args.train_mode != "ti2v": + mask = resize_mask(1 - mask, latents) + else: + mask = F.interpolate(mask[:, :1], size=latents.size()[-3:], mode='trilinear', align_corners=True).to(accelerator.device, weight_dtype) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + if args.train_mode != "ti2v": + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + else: + inpaint_latents = mask_latents + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + if args.train_mode == "ti2v": + if rng is None: + t2v_in_ti2v = np.random.choice([0, 1], p = [0.50, 0.50]) + else: + t2v_in_ti2v = rng.choice([0, 1], p = [0.50, 0.50]) + + mask_bs = mask.size()[0] + if t2v_in_ti2v: + noisy_latents = (1 - mask) * inpaint_latents + mask * noisy_latents + + temp_ts = (mask[:, 0, :, ::2, ::2] * timesteps[:, None, None, None]).flatten(1) + timesteps = torch.cat([temp_ts, temp_ts.new_ones(mask_bs, seq_len - temp_ts.size(1)) * timesteps[:, None,]], dim = 1) + else: + timesteps = mask.new_ones(mask_bs, seq_len) * timesteps[:, None,] + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2/train.sh b/VideoX-Fun/scripts/wan2.2/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..1224fb63fa786f5bdbd98b03d324b92b74261bc0 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train.sh @@ -0,0 +1,90 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="normal" \ + --trainable_modules "." + +# The Training Shell Code for Image to Video +# You need to use "config/wan2.2/wan_civitai_i2v.yaml" +# +# export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +# export DATASET_NAME="datasets/internal_datasets/" +# export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# # export NCCL_IB_DISABLE=1 +# # export NCCL_P2P_DISABLE=1 +# NCCL_DEBUG=INFO + +# accelerate launch --mixed_precision="bf16" scripts/wan2.2/train.py \ +# --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --train_data_dir=$DATASET_NAME \ +# --train_data_meta=$DATASET_META_NAME \ +# --image_sample_size=640 \ +# --video_sample_size=640 \ +# --token_sample_size=640 \ +# --video_sample_stride=2 \ +# --video_sample_n_frames=81 \ +# --train_batch_size=1 \ +# --video_repeat=1 \ +# --gradient_accumulation_steps=1 \ +# --dataloader_num_workers=8 \ +# --num_train_epochs=100 \ +# --checkpointing_steps=50 \ +# --learning_rate=2e-05 \ +# --lr_scheduler="constant_with_warmup" \ +# --lr_warmup_steps=100 \ +# --seed=42 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --vae_mini_batch=1 \ +# --max_grad_norm=0.05 \ +# --random_hw_adapt \ +# --training_with_video_token_length \ +# --enable_bucket \ +# --uniform_sampling \ +# --low_vram \ +# --boundary_type="low" \ +# --train_mode="i2v" \ +# --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/train_animate.py b/VideoX-Fun/scripts/wan2.2/train_animate.py new file mode 100644 index 0000000000000000000000000000000000000000..aa5874d68bbd24ee8e3b9ece2141b60ad778fe2f --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_animate.py @@ -0,0 +1,2023 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data import (VideoAnimateDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask, + process_pose_file, + process_pose_params) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, CLIPModel, WanT5EncoderModel, + Wan2_2Transformer3DModel_Animate) +from videox_fun.pipeline import Wan2_2FunControlPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +def get_i2v_mask(lat_t, lat_h, lat_w, mask_len=1, mask_pixel_values=None, device="cuda"): + if mask_pixel_values is None: + msk = torch.zeros(1, (lat_t-1) * 4 + 1, lat_h, lat_w, device=device) + else: + msk = mask_pixel_values.clone() + msk[:, :mask_len] = 1 + msk = torch.concat([torch.repeat_interleave(msk[:, 0:1], repeats=4, dim=1), msk[:, 1:]], dim=1) + msk = msk.view(1, msk.shape[1] // 4, 4, lat_h, lat_w) + msk = msk.transpose(1, 2) + return msk + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + pipeline = Wan2_2FunControlPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int(args.video_sample_n_frames // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(args.validation_paths[i], video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + control_video = input_video, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_paths", + type=str, + default=None, + nargs="+", + help=("A set of control videos evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Clip Image Encoder + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + clip_image_encoder.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=Wan2_2Transformer3DModel_Animate, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = Wan2_2Transformer3DModel_Animate.load_config(ema_path, return_unused_kwargs=True) + load_model = Wan2_2Transformer3DModel_Animate.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=Wan2_2Transformer3DModel_Animate, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = Wan2_2Transformer3DModel_Animate.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = VideoAnimateDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.video_sample_size > 256: + sample_sizes = list(range(256, args.video_sample_size + 1, 128)) + + if sample_sizes[-1] != args.video_sample_size: + sample_sizes.append(args.video_sample_size) + else: + sample_sizes = [args.video_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Control pixel values + new_examples["control_pixel_values"] = [] + # Used in Face pixel values + new_examples["face_pixel_values"] = [] + # Used in Ref pixel values + new_examples["ref_pixel_values"] = [] + # Used in Background pixel values + new_examples["background_pixel_values"] = [] + # Used in Mask pixel values + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + local_video_sample_size = np.random.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + face_pixel_values = torch.from_numpy(example["face_pixel_values"]).permute(0, 3, 1, 2).contiguous() + face_pixel_values = face_pixel_values / 255. + + ref_pixel_values = torch.from_numpy(example["ref_pixel_values"]).unsqueeze(0).permute(0, 3, 1, 2).contiguous() + ref_pixel_values = ref_pixel_values / 255. + + background_pixel_values = torch.from_numpy(example["background_pixel_values"]).permute(0, 3, 1, 2).contiguous() + background_pixel_values = background_pixel_values / 255. + + mask = torch.from_numpy(example["mask"]).permute(0, 3, 1, 2).contiguous() + mask = mask / 255. + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + ]) + + transform_512 = transforms.Compose([ + transforms.Resize((512, 512), interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop((512, 512)), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + new_examples["face_pixel_values"].append(transform_512(face_pixel_values)[:batch_video_length]) + new_examples["ref_pixel_values"].append(transform(ref_pixel_values)) + new_examples["background_pixel_values"].append(transform(background_pixel_values)[:batch_video_length]) + new_examples["mask"].append(transform_no_normalize(mask)[:batch_video_length]) + new_examples["clip_pixel_values"].append(torch.from_numpy(example["clip_pixel_values"])) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + new_examples["face_pixel_values"] = torch.stack([example for example in new_examples["face_pixel_values"]]) + new_examples["ref_pixel_values"] = torch.stack([example for example in new_examples["ref_pixel_values"]]) + new_examples["background_pixel_values"] = torch.stack([example for example in new_examples["background_pixel_values"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + control_pixel_values = batch["control_pixel_values"].cpu() + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, control_pixel_value, text) in enumerate(zip(pixel_values, control_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + control_pixel_value = control_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + save_videos_grid(control_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_control.gif", rescale=True) + + face_pixel_values = batch["face_pixel_values"].cpu() + face_pixel_values = rearrange(face_pixel_values, "b f c h w -> b c f h w") + for idx, (face_pixel_value, text) in enumerate(zip(face_pixel_values, texts)): + face_pixel_value = face_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(face_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_face.gif", rescale=True) + + ref_pixel_values = batch["ref_pixel_values"].cpu() + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + for idx, (ref_pixel_value, text) in enumerate(zip(ref_pixel_values, texts)): + ref_pixel_value = ref_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(ref_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_ref.gif", rescale=True) + + background_pixel_values = batch["background_pixel_values"].cpu() + background_pixel_values = rearrange(background_pixel_values, "b f c h w -> b c f h w") + for idx, (bg_pixel_value, text) in enumerate(zip(background_pixel_values, texts)): + bg_pixel_value = bg_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(bg_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_bg.gif", rescale=True) + + clip_pixel_values, mask, texts = batch['clip_pixel_values'].cpu(), batch['mask'].cpu(), batch['text'] + mask = rearrange(mask, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + face_pixel_values = batch["face_pixel_values"].to(weight_dtype) + ref_pixel_values = batch["ref_pixel_values"].to(weight_dtype) + background_pixel_values = batch["background_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + face_pixel_values = torch.tile(face_pixel_values, (4, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (4, 1, 1, 1, 1)) + background_pixel_values = torch.tile(background_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + face_pixel_values = torch.tile(face_pixel_values, (2, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (2, 1, 1, 1, 1)) + background_pixel_values = torch.tile(background_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + face_pixel_values = face_pixel_values[:, :temp_n_frames, :, :] + background_pixel_values = background_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + face_pixel_values = face_pixel_values[:, :actual_video_length, :, :] + background_pixel_values = background_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + clip_image_encoder.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + control_latents = _batch_encode_vae(control_pixel_values) + ref_latents = _batch_encode_vae(ref_pixel_values) + + if rng is None: + refert_num = np.random.choice([0, 1], p = [0.75, 0.25]) + else: + refert_num = rng.choice([0, 1], p = [0.75, 0.25]) + + background_pixel_values[:, :refert_num, :] = pixel_values[:, :refert_num, :] + mask[:, :refert_num, :] = torch.zeros_like(mask[:, :refert_num, :]) + for bs_index in range(background_pixel_values.size()[0]): + if rng is None: + zero_init_background_pixel_values_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_background_pixel_values_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if zero_init_background_pixel_values_conv_in: + background_pixel_values[bs_index] = background_pixel_values[bs_index] * 0 + mask[bs_index] = torch.ones_like(mask[bs_index]) + background_latents = _batch_encode_vae(background_pixel_values) + + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask = resize_mask(1 - mask, latents) + y_reft = torch.concat([mask, background_latents], dim=1).to(device=accelerator.device, dtype=weight_dtype) + + mask_ref = get_i2v_mask(1, latents.size(-2), latents.size(-1), 1, device=accelerator.device) + y_ref = torch.concat([mask_ref, ref_latents], dim=1).to(device=accelerator.device, dtype=weight_dtype) + y = torch.concat([y_ref, y_reft], dim=2) + + face_pixel_values = rearrange(face_pixel_values, "b c f h w -> b f c h w") + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + clip_context.append(_clip_context) + clip_context = torch.cat(clip_context) + + latents = torch.cat([ref_latents, latents], dim=2) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn( + latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype + ) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=y, + clip_fea=clip_context, + pose_latents=control_latents, + face_pixel_values=face_pixel_values, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2/train_animate.sh b/VideoX-Fun/scripts/wan2.2/train_animate.sh new file mode 100644 index 0000000000000000000000000000000000000000..ac27520115aded1a17417115ee8acaa40fa6440c --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_animate.sh @@ -0,0 +1,41 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Animate-14B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_animate.py \ + --config_path="config/wan2.2/wan_civitai_animate.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --low_vram \ + --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/train_animate_lora.py b/VideoX-Fun/scripts/wan2.2/train_animate_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..98b908612b689d01389e62a5335a35bbf7ab673a --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_animate_lora.py @@ -0,0 +1,1997 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data import (ImageVideoDataset, ImageVideoSampler, + VideoAnimateDataset, get_random_mask, + process_pose_file, process_pose_params) +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + CLIPModel, Wan2_2Transformer3DModel, + Wan2_2Transformer3DModel_Animate, + WanT5EncoderModel) +from videox_fun.pipeline import (Wan2_2FunControlPipeline, Wan2_2I2VPipeline, + Wan2_2Pipeline) +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +def get_i2v_mask(lat_t, lat_h, lat_w, mask_len=1, mask_pixel_values=None, device="cuda"): + if mask_pixel_values is None: + msk = torch.zeros(1, (lat_t-1) * 4 + 1, lat_h, lat_w, device=device) + else: + msk = mask_pixel_values.clone() + msk[:, :mask_len] = 1 + msk = torch.concat([torch.repeat_interleave(msk[:, 0:1], repeats=4, dim=1), msk[:, 1:]], dim=1) + msk = msk.view(1, msk.shape[1] // 4, 4, lat_h, lat_w) + msk = msk.transpose(1, 2) + return msk + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + pipeline = Wan2_2FunControlPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int(args.video_sample_n_frames // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(args.validation_paths[i], video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + control_video = input_video, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_paths", + type=str, + default=None, + nargs="+", + help=("A set of control videos evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Clip Image Encoder + clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ) + clip_image_encoder = clip_image_encoder.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = Wan2_2Transformer3DModel_Animate.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + clip_image_encoder.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = VideoAnimateDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.video_sample_size > 256: + sample_sizes = list(range(256, args.video_sample_size + 1, 128)) + + if sample_sizes[-1] != args.video_sample_size: + sample_sizes.append(args.video_sample_size) + else: + sample_sizes = [args.video_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Control pixel values + new_examples["control_pixel_values"] = [] + # Used in Face pixel values + new_examples["face_pixel_values"] = [] + # Used in Ref pixel values + new_examples["ref_pixel_values"] = [] + # Used in Background pixel values + new_examples["background_pixel_values"] = [] + # Used in Mask pixel values + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + local_video_sample_size = np.random.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + face_pixel_values = torch.from_numpy(example["face_pixel_values"]).permute(0, 3, 1, 2).contiguous() + face_pixel_values = face_pixel_values / 255. + + ref_pixel_values = torch.from_numpy(example["ref_pixel_values"]).unsqueeze(0).permute(0, 3, 1, 2).contiguous() + ref_pixel_values = ref_pixel_values / 255. + + background_pixel_values = torch.from_numpy(example["background_pixel_values"]).permute(0, 3, 1, 2).contiguous() + background_pixel_values = background_pixel_values / 255. + + mask = torch.from_numpy(example["mask"]).permute(0, 3, 1, 2).contiguous() + mask = mask / 255. + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + ]) + + transform_512 = transforms.Compose([ + transforms.Resize((512, 512), interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop((512, 512)), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + new_examples["face_pixel_values"].append(transform_512(face_pixel_values)[:batch_video_length]) + new_examples["ref_pixel_values"].append(transform(ref_pixel_values)) + new_examples["background_pixel_values"].append(transform(background_pixel_values)[:batch_video_length]) + new_examples["mask"].append(transform_no_normalize(mask)[:batch_video_length]) + new_examples["clip_pixel_values"].append(torch.from_numpy(example["clip_pixel_values"])) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + new_examples["face_pixel_values"] = torch.stack([example for example in new_examples["face_pixel_values"]]) + new_examples["ref_pixel_values"] = torch.stack([example for example in new_examples["ref_pixel_values"]]) + new_examples["background_pixel_values"] = torch.stack([example for example in new_examples["background_pixel_values"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + clip_image_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + control_pixel_values = batch["control_pixel_values"].cpu() + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, control_pixel_value, text) in enumerate(zip(pixel_values, control_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + control_pixel_value = control_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + save_videos_grid(control_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_control.gif", rescale=True) + + face_pixel_values = batch["face_pixel_values"].cpu() + face_pixel_values = rearrange(face_pixel_values, "b f c h w -> b c f h w") + for idx, (face_pixel_value, text) in enumerate(zip(face_pixel_values, texts)): + face_pixel_value = face_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(face_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_face.gif", rescale=True) + + ref_pixel_values = batch["ref_pixel_values"].cpu() + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + for idx, (ref_pixel_value, text) in enumerate(zip(ref_pixel_values, texts)): + ref_pixel_value = ref_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(ref_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_ref.gif", rescale=True) + + background_pixel_values = batch["background_pixel_values"].cpu() + background_pixel_values = rearrange(background_pixel_values, "b f c h w -> b c f h w") + for idx, (bg_pixel_value, text) in enumerate(zip(background_pixel_values, texts)): + bg_pixel_value = bg_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(bg_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_bg.gif", rescale=True) + + clip_pixel_values, mask, texts = batch['clip_pixel_values'].cpu(), batch['mask'].cpu(), batch['text'] + mask = rearrange(mask, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + face_pixel_values = batch["face_pixel_values"].to(weight_dtype) + ref_pixel_values = batch["ref_pixel_values"].to(weight_dtype) + background_pixel_values = batch["background_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + face_pixel_values = torch.tile(face_pixel_values, (4, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (4, 1, 1, 1, 1)) + background_pixel_values = torch.tile(background_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + face_pixel_values = torch.tile(face_pixel_values, (2, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (2, 1, 1, 1, 1)) + background_pixel_values = torch.tile(background_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + face_pixel_values = face_pixel_values[:, :temp_n_frames, :, :] + background_pixel_values = background_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + face_pixel_values = face_pixel_values[:, :actual_video_length, :, :] + background_pixel_values = background_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + clip_image_encoder.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + control_latents = _batch_encode_vae(control_pixel_values) + ref_latents = _batch_encode_vae(ref_pixel_values) + + if rng is None: + refert_num = np.random.choice([0, 1], p = [0.75, 0.25]) + else: + refert_num = rng.choice([0, 1], p = [0.75, 0.25]) + + background_pixel_values[:, :refert_num, :] = pixel_values[:, :refert_num, :] + mask[:, :refert_num, :] = torch.zeros_like(mask[:, :refert_num, :]) + for bs_index in range(background_pixel_values.size()[0]): + if rng is None: + zero_init_background_pixel_values_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_background_pixel_values_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if zero_init_background_pixel_values_conv_in: + background_pixel_values[bs_index] = background_pixel_values[bs_index] * 0 + mask[bs_index] = torch.ones_like(mask[bs_index]) + background_latents = _batch_encode_vae(background_pixel_values) + + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask = resize_mask(1 - mask, latents) + y_reft = torch.concat([mask, background_latents], dim=1).to(device=accelerator.device, dtype=weight_dtype) + + mask_ref = get_i2v_mask(1, latents.size(-2), latents.size(-1), 1, device=accelerator.device) + y_ref = torch.concat([mask_ref, ref_latents], dim=1).to(device=accelerator.device, dtype=weight_dtype) + y = torch.concat([y_ref, y_reft], dim=2) + + face_pixel_values = rearrange(face_pixel_values, "b c f h w -> b f c h w") + + clip_context = [] + for clip_pixel_value in clip_pixel_values: + clip_image = Image.fromarray(np.uint8(clip_pixel_value.float().cpu().numpy())) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(clip_image_encoder.device, weight_dtype) + _clip_context = clip_image_encoder([clip_image[:, None, :, :]]) + clip_context.append(_clip_context) + clip_context = torch.cat(clip_context) + + latents = torch.cat([ref_latents, latents], dim=2) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + clip_image_encoder.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn( + latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype + ) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=y, + clip_fea=clip_context, + pose_latents=control_latents, + face_pixel_values=face_pixel_values, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2/train_animate_lora.sh b/VideoX-Fun/scripts/wan2.2/train_animate_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..e38a4b1f89be612dbe2af29d3cfa8704d1f9aa5b --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_animate_lora.sh @@ -0,0 +1,42 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Animate-14B/" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_animate_lora.py \ + --config_path="config/wan2.2/wan_civitai_animate.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/train_distill.py b/VideoX-Fun/scripts/wan2.2/train_distill.py new file mode 100644 index 0000000000000000000000000000000000000000..98634836489da3d1f9ad4ea408ccaf076fa8a26d --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_distill.py @@ -0,0 +1,2423 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import contextlib +import gc +import json +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.distributed as dist +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import Dataset, RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + BatchSampler, RandomSampler, + get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + TextDataset, get_random_mask) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, AutoencoderKLWan3_8, + Wan2_2Transformer3DModel) +from videox_fun.pipeline import WanI2VPipeline, WanPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = Wan2_2I2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + else: + pipeline = Wan2_2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--negative_prompt", + type=str, + default="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + help=("The negative prompt of cfg distill"), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--learning_rate_critic", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--gen_update_interval", + type=int, + default=5, + help="The ratio to update transformer3d.", + ) + parser.add_argument( + "--fake_guidance_scale", + type=float, + default=0.0, + help="The cfg scale for fake iscore.", + ) + parser.add_argument( + "--real_guidance_scale", + type=float, + default=6.0, + help="The cfg scale for real score.", + ) + parser.add_argument( + '--denoising_step_indices_list', + nargs='+', + default=[1000, 750, 500, 250], + help="The denoising step list.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + accelerator_fake_score_transformer3d = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + + # Get Transformer + generator_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + real_score_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + fake_score_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set generator_transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + generator_transformer3d.requires_grad_(False) + real_score_transformer3d.requires_grad_(False) + fake_score_transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = generator_transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + generator_transformer3d.train() + fake_score_transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in generator_transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + for name, param in fake_score_transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = Wan2_2Transformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + accelerator_fake_score_transformer3d.register_save_state_pre_hook(save_model_hook) + accelerator_fake_score_transformer3d.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + generator_transformer3d.enable_gradient_checkpointing() + fake_score_transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, generator_transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in generator_transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + fake_trainable_params = list(filter(lambda p: p.requires_grad, fake_score_transformer3d.parameters())) + fake_trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in fake_score_transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + fake_trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + fake_trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + critic_optimizer = optimizer_cls( + fake_trainable_params_optim, + lr=args.learning_rate_critic, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + critic_optimizer = optimizer_cls( + fake_trainable_params_optim, + lr=args.learning_rate_critic, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + if args.train_mode != "normal": + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + else: + train_dataset = TextDataset( + args.train_data_meta + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + if args.enable_bucket and args.train_mode != "normal": + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to("cpu"), attention_mask=prompt_attention_mask.to("cpu"))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to("cpu"), attention_mask=neg_prompt_attention_mask.to("cpu"))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + new_examples['neg_encoder_attention_mask'] = neg_prompt_ids.attention_mask + new_examples['neg_encoder_hidden_states'] = neg_prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + elif args.train_mode == "normal": + def collate_fn(examples): + new_examples = {} + new_examples["text"] = [] + for example in examples: + new_examples["text"].append(example["text"]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to("cpu"), attention_mask=prompt_attention_mask.to("cpu"))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to("cpu"), attention_mask=neg_prompt_attention_mask.to("cpu"))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + new_examples['neg_encoder_attention_mask'] = neg_prompt_ids.attention_mask + new_examples['neg_encoder_hidden_states'] = neg_prompt_embeds + + return new_examples + + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = BatchSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), batch_size=args.train_batch_size, drop_last=True) + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + fake_score_lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + generator_transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + generator_transformer3d, optimizer, train_dataloader, lr_scheduler + ) + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler= accelerator_fake_score_transformer3d.prepare( + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler + ) + if fsdp_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + real_score_transformer3d = shard_fn(real_score_transformer3d) + if fsdp_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + real_score_transformer3d.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu") + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + fake_score_path = os.path.join(path, "fake_score") + accelerator.load_state(os.path.join(args.output_dir, path)) + accelerator_fake_score_transformer3d.load_state(os.path.join(args.output_dir, fake_score_path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + if args.boundary_type == "high" or args.boundary_type == "low": + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + denoising_step_list = noise_scheduler.timesteps[args.train_sampling_steps - torch.tensor(args.denoising_step_indices_list)] + + for epoch in range(first_epoch, args.num_train_epochs): + train_dmd_loss = 0.0 + train_denoising_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if args.train_mode != "normal" and epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + if args.train_mode != "normal": + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + batch['neg_encoder_hidden_states'] = torch.tile(batch['neg_encoder_hidden_states'], (4, 1, 1)) + batch['neg_encoder_attention_mask'] = torch.tile(batch['neg_encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + batch['neg_encoder_hidden_states'] = torch.tile(batch['neg_encoder_hidden_states'], (2, 1, 1)) + batch['neg_encoder_attention_mask'] = torch.tile(batch['neg_encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + real_score_transformer3d = real_score_transformer3d.to("cpu") + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + + if args.train_mode != "ti2v": + mask = resize_mask(1 - mask, mask_latents) + else: + mask = F.interpolate(mask[:, :1], size=mask_latents.size()[-3:], mode='trilinear', align_corners=True).to(accelerator.device, weight_dtype) + + if args.train_mode != "ti2v": + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + else: + inpaint_latents = mask_latents + + target_shape = mask_latents.size() + else: + text = batch['text'] + if args.fix_sample_size is not None: + local_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + num_frames = args.video_sample_n_frames + else: + if args.random_hw_adapt and args.training_with_video_token_length: + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + if rng is None: + local_length = np.random.choice(list(length_to_frame_num.keys())) + else: + local_length = rng.choice(list(length_to_frame_num.keys())) + num_frames = length_to_frame_num[local_length] + + aspect_ratio_sample_size = {key : [x / 512 * local_length for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if rng is None: + aspect_ratio_key = np.random.choice(list(aspect_ratio_sample_size.keys())) + else: + aspect_ratio_key = rng.choice(list(aspect_ratio_sample_size.keys())) + local_sample_size = aspect_ratio_sample_size[aspect_ratio_key] + else: + num_frames = args.video_sample_n_frames + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if rng is None: + aspect_ratio_key = np.random.choice(list(aspect_ratio_sample_size.keys())) + else: + aspect_ratio_key = rng.choice(list(aspect_ratio_sample_size.keys())) + local_sample_size = aspect_ratio_sample_size[aspect_ratio_key] + local_sample_size = [int(x / 16) * 16 for x in local_sample_size] + + target_shape = ( + len(text), + vae.latent_channels, + int((num_frames - 1) // vae.temporal_compression_ratio + 1), + int(local_sample_size[0] // vae.spatial_compression_ratio), + int(local_sample_size[1] // vae.spatial_compression_ratio), + ) + + if args.low_vram: + vae.to('cpu') + real_score_transformer3d = real_score_transformer3d.to("cpu") + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=accelerator.device) + neg_prompt_embeds = batch['neg_encoder_hidden_states'].to(device=accelerator.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(accelerator.device), attention_mask=prompt_attention_mask.to(accelerator.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to(accelerator.device), attention_mask=neg_prompt_attention_mask.to(accelerator.device))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + if args.low_vram: + generator_transformer3d = generator_transformer3d.to(accelerator.device) + real_score_transformer3d = real_score_transformer3d.to(accelerator.device) + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + if args.train_mode == "ti2v": + if rng is None: + t2v_in_ti2v = np.random.choice([0, 1], p = [0.50, 0.50]) + else: + t2v_in_ti2v = rng.choice([0, 1], p = [0.50, 0.50]) + + with accelerator.accumulate(generator_transformer3d): + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + + step_indices = [ + torch.argmin(torch.abs(schedule_timesteps - t)).item() + for t in timesteps + ] + step_indices = torch.tensor(step_indices, device=accelerator.device) + sigma = sigmas[step_indices].flatten() + + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + def add_noise(latents, noise, timesteps): + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + return (1.0 - sigmas) * latents + sigmas * noise + + def generate_and_sync_list(num_denoising_steps, device): + indices = torch.randint(low=0, high=num_denoising_steps, size=(1,), generator=torch_rng, device=device) + if dist.is_initialized(): + dist.broadcast(indices, src=0) + return indices.tolist() + + def convert_flow_pred_to_x0( + scheduler, + flow_pred: torch.Tensor, + xt: torch.Tensor, + timestep: torch.Tensor + ) -> torch.Tensor: + """ + Convert flow matching's prediction to x0 prediction. + Supports both 4D [B, C, H, W] and 5D [B, C, F, H, W] inputs. + """ + original_dtype = flow_pred.dtype + device = flow_pred.device + + flow_pred = flow_pred.double() + xt = xt.double() + timesteps = scheduler.timesteps.to(device).double() + sigmas = scheduler.sigmas.to(device).double() + timestep = timestep.to(device).double() + + timestep_id = torch.argmin((timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1) + sigma_t = sigmas[timestep_id] + + ndim = flow_pred.ndim + if ndim == 4: + sigma_t = sigma_t.view(-1, 1, 1, 1) + elif ndim == 5: + sigma_t = sigma_t.view(-1, 1, 1, 1, 1) + else: + raise ValueError(f"Expected 4D or 5D input, got {ndim}D tensor.") + + x0_pred = xt - sigma_t * flow_pred + return x0_pred.to(original_dtype) + + # --- Main Training Logic --- + bsz, channel, num_frames, height, width = target_shape + if step % args.gen_update_interval == 0: + generator_noise = torch.randn(target_shape, device=accelerator.device, generator=torch_rng, dtype=weight_dtype) + num_denoising_steps = len(denoising_step_list) + final_step_index = generate_and_sync_list(num_denoising_steps, device=generator_noise.device)[0] + + # Precompute seq_len once (same for all steps) + patch_h, patch_w = accelerator.unwrap_model(generator_transformer3d).config.patch_size[1:] + seq_len = math.ceil((width * height) / (patch_h * patch_w) * num_frames) + + for index, current_timestep in enumerate(denoising_step_list): + is_final_step = (index == final_step_index) + timestep = torch.full( + generator_noise.shape[:1], + current_timestep, + device=generator_noise.device, + dtype=torch.int64 + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + context_manager = torch.no_grad() if not is_final_step else contextlib.nullcontext() + + with context_manager: + if args.train_mode == "ti2v": + mask_bs = mask.size()[0] + if t2v_in_ti2v: + _generator_noise = (1 - mask) * inpaint_latents + mask * generator_noise + + temp_ts = (mask[:, 0, :, ::2, ::2] * timestep[:, None, None, None]).flatten(1) + _timestep = torch.cat([temp_ts, temp_ts.new_ones(mask_bs, seq_len - temp_ts.size(1)) * timestep[:, None,]], dim = 1) + else: + _timestep = mask.new_ones(mask_bs, seq_len) * timestep[:, None,] + _generator_noise = generator_noise + else: + _timestep = timestep + _generator_noise = generator_noise + generator_pred = generator_transformer3d( + x=_generator_noise, + context=prompt_embeds, + t=_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + generator_pred = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=generator_pred, + xt=generator_noise, + timestep=timestep + ) + + if is_final_step: + break + + next_timestep = denoising_step_list[index + 1] * torch.ones( + generator_noise.shape[:1], dtype=torch.long, device=generator_noise.device + ) + generator_noise = add_noise( + generator_pred, + torch.randn(generator_pred.shape, dtype=generator_pred.dtype, device=generator_pred.device, generator=torch_rng), + next_timestep + ) + + indices = idx_sampling(bsz, generator=torch_rng, device=accelerator.device).long().cpu() + generator_timestep = noise_scheduler.timesteps[indices].to(device=accelerator.device) + generator_denoised_input = add_noise( + generator_pred, + torch.randn(generator_pred.shape, dtype=generator_pred.dtype, device=generator_pred.device, generator=torch_rng), + generator_timestep + ).detach().to(accelerator.device, dtype=weight_dtype) + + # Compute fake score + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device), torch.no_grad(): + if args.train_mode == "ti2v": + mask_bs = mask.size()[0] + if t2v_in_ti2v: + _generator_denoised_input = (1 - mask) * inpaint_latents + mask * generator_denoised_input + + temp_ts = (mask[:, 0, :, ::2, ::2] * generator_timestep[:, None, None, None]).flatten(1) + _generator_timestep = torch.cat([temp_ts, temp_ts.new_ones(mask_bs, seq_len - temp_ts.size(1)) * generator_timestep[:, None,]], dim = 1) + else: + _generator_timestep = mask.new_ones(mask_bs, seq_len) * generator_timestep[:, None,] + _generator_denoised_input = generator_denoised_input + else: + _generator_timestep = generator_timestep + _generator_denoised_input = generator_denoised_input + fake_score_main_cond = fake_score_transformer3d( + x=_generator_denoised_input, + context=prompt_embeds, + t=_generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + fake_score_main_cond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_main_cond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + if args.fake_guidance_scale != 0.0: + fake_score_main_uncond = fake_score_transformer3d( + x=_generator_denoised_input, + context=neg_prompt_embeds, + t=_generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + fake_score_main_uncond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_main_uncond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + fake_score_main = fake_score_main_uncond + ( + fake_score_main_cond - fake_score_main_uncond + ) * args.fake_guidance_scale + else: + fake_score_main = fake_score_main_cond + + # Compute real score + real_score_main_cond = real_score_transformer3d( + x=_generator_denoised_input, + context=prompt_embeds, + t=_generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + real_score_main_cond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=real_score_main_cond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + real_score_main_uncond = real_score_transformer3d( + x=_generator_denoised_input, + context=neg_prompt_embeds, + t=_generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + real_score_main_uncond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=real_score_main_uncond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + real_score_main = real_score_main_uncond + ( + real_score_main_cond - real_score_main_uncond + ) * args.real_guidance_scale + + # DMD loss + fake_to_real_grad = fake_score_main - real_score_main + generator_to_real_norm = generator_pred - real_score_main + normalizer = torch.abs(generator_to_real_norm).mean(dim=[1, 2, 3, 4], keepdim=True) + fake_to_real_grad = fake_to_real_grad / normalizer + fake_to_real_grad = torch.nan_to_num(fake_to_real_grad) + + dmd_loss = 0.5 * F.mse_loss( + generator_pred.double(), + (generator_pred.double() - fake_to_real_grad.double()).detach(), + reduction="mean" + ) + avg_dmd_loss = accelerator.gather(dmd_loss.repeat(args.train_batch_size)).mean() + train_dmd_loss += avg_dmd_loss.item() / args.gradient_accumulation_steps + + if args.low_vram: + real_score_transformer3d = real_score_transformer3d.to("cpu") + fake_score_transformer3d = fake_score_transformer3d.to("cpu") + torch.cuda.empty_cache() + + accelerator.backward(dmd_loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + if args.low_vram: + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + torch.cuda.empty_cache() + + with accelerator_fake_score_transformer3d.accumulate(fake_score_transformer3d): + # --- Fake Critic Denoising Loss --- + with torch.no_grad(): + fake_score_critic_noise = torch.randn(target_shape, device=accelerator.device, generator=torch_rng, dtype=weight_dtype) + num_denoising_steps = len(denoising_step_list) + final_step_index = generate_and_sync_list(num_denoising_steps, device=fake_score_critic_noise.device)[0] + + patch_h, patch_w = accelerator.unwrap_model(generator_transformer3d).config.patch_size[1:] + seq_len = math.ceil((width * height) / (patch_h * patch_w) * num_frames) + + for index, current_timestep in enumerate(denoising_step_list): + is_final_step = (index == final_step_index) + timestep = torch.full( + fake_score_critic_noise.shape[:1], + current_timestep, + device=fake_score_critic_noise.device, + dtype=torch.int64 + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + if args.train_mode == "ti2v": + mask_bs = mask.size()[0] + if t2v_in_ti2v: + _fake_score_critic_noise = (1 - mask) * inpaint_latents + mask * fake_score_critic_noise + + temp_ts = (mask[:, 0, :, ::2, ::2] * timestep[:, None, None, None]).flatten(1) + _timestep = torch.cat([temp_ts, temp_ts.new_ones(mask_bs, seq_len - temp_ts.size(1)) * timestep[:, None,]], dim = 1) + else: + _timestep = mask.new_ones(mask_bs, seq_len) * timestep[:, None,] + _fake_score_critic_noise = fake_score_critic_noise + else: + _timestep = timestep + _fake_score_critic_noise = fake_score_critic_noise + fake_score_denoised_pred = generator_transformer3d( + x=_fake_score_critic_noise, + context=prompt_embeds, + t=_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + fake_score_denoised_pred = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_denoised_pred, + xt=fake_score_critic_noise, + timestep=timestep + ) + + if is_final_step: + break + + next_timestep = denoising_step_list[index + 1] * torch.ones( + fake_score_critic_noise.shape[:1], + dtype=torch.long, + device=fake_score_critic_noise.device + ) + + fake_score_critic_noise = add_noise( + fake_score_denoised_pred, + torch.randn(fake_score_denoised_pred.shape, dtype=fake_score_denoised_pred.dtype, device=fake_score_denoised_pred.device, generator=torch_rng), + next_timestep + ) + + indices = idx_sampling(bsz, generator=torch_rng, device=accelerator.device).long().cpu() + critic_timestep = noise_scheduler.timesteps[indices].to(device=accelerator.device) + critic_noise = torch.randn(fake_score_denoised_pred.shape, dtype=fake_score_denoised_pred.dtype, device=fake_score_denoised_pred.device, generator=torch_rng) + + fake_score_denoised_input = add_noise( + fake_score_denoised_pred, + critic_noise, + critic_timestep + ) + if args.train_mode == "ti2v": + mask_bs = mask.size()[0] + if t2v_in_ti2v: + fake_score_denoised_input = (1 - mask) * inpaint_latents + mask * fake_score_denoised_input + + temp_ts = (mask[:, 0, :, ::2, ::2] * critic_timestep[:, None, None, None]).flatten(1) + _critic_timestep = torch.cat([temp_ts, temp_ts.new_ones(mask_bs, seq_len - temp_ts.size(1)) * critic_timestep[:, None,]], dim = 1) + else: + _critic_timestep = mask.new_ones(mask_bs, seq_len) * critic_timestep[:, None,] + else: + _critic_timestep = critic_timestep + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + fake_score_denoised_output = fake_score_transformer3d( + x=fake_score_denoised_input, + context=prompt_embeds, + t=_critic_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + denoising_loss = custom_mse_loss(fake_score_denoised_output, critic_noise - fake_score_denoised_pred) + avg_denoising_loss = accelerator.gather(denoising_loss.repeat(args.train_batch_size)).mean() + train_denoising_loss += avg_denoising_loss.item() / args.gradient_accumulation_steps + + accelerator_fake_score_transformer3d.backward(denoising_loss) + if accelerator_fake_score_transformer3d.sync_gradients: + accelerator_fake_score_transformer3d.clip_grad_norm_(fake_trainable_params, args.max_grad_norm) + critic_optimizer.step() + fake_score_lr_scheduler.step() + critic_optimizer.zero_grad() + + if args.low_vram: + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + generator_transformer3d = generator_transformer3d.to(accelerator.device) + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_denoising_loss": train_denoising_loss, "train_dmd_loss": train_dmd_loss}, step=global_step) + train_dmd_loss = 0.0 + train_denoising_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + fake_score_save_path = os.path.join(save_path, "fake_score") + accelerator.save_state(save_path) + accelerator_fake_score_transformer3d.save_state(fake_score_save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + generator_transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"denoising_loss": denoising_loss.detach().item(), "dmd_loss": dmd_loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + generator_transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + generator_transformer3d = unwrap_model(generator_transformer3d) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + fake_score_save_path = os.path.join(save_path, "fake_score") + accelerator.save_state(save_path) + accelerator_fake_score_transformer3d.save_state(fake_score_save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2/train_distill.sh b/VideoX-Fun/scripts/wan2.2/train_distill.sh new file mode 100644 index 0000000000000000000000000000000000000000..d9af9aa9eb6c0aa6a8a474bce4d742f97139c51f --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_distill.sh @@ -0,0 +1,44 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-06 \ + --learning_rate_critic=2e-07\ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="i2v" \ + --trainable_modules "." \ + --low_vram \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/train_distill_lora.py b/VideoX-Fun/scripts/wan2.2/train_distill_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..f7d21d6bea1ff0ac59de240b2432bf0b72e32ec1 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_distill_lora.py @@ -0,0 +1,2510 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import contextlib +import gc +import json +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.distributed as dist +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import Dataset, RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + BatchSampler, RandomSampler, + get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + TextDataset, get_random_mask) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, WanT5EncoderModel, AutoencoderKLWan3_8, + Wan2_2Transformer3DModel) +from videox_fun.pipeline import WanI2VPipeline, WanPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, config, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = Wan2_2I2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + else: + pipeline = Wan2_2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--negative_prompt", + type=str, + default="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走", + help=("The negative prompt of cfg distill"), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--learning_rate_critic", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--gen_update_interval", + type=int, + default=5, + help="The ratio to update transformer3d.", + ) + parser.add_argument( + "--fake_guidance_scale", + type=float, + default=0.0, + help="The cfg scale for fake iscore.", + ) + parser.add_argument( + "--real_guidance_scale", + type=float, + default=6.0, + help="The cfg scale for real score.", + ) + parser.add_argument( + '--denoising_step_indices_list', + nargs='+', + default=[1000, 750, 500, 250], + help="The denoising step list.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + accelerator_fake_score_transformer3d = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + + # Get Transformer + generator_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + real_score_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + fake_score_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set generator_transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + generator_transformer3d.requires_grad_(False) + real_score_transformer3d.requires_grad_(False) + fake_score_transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + generator_transformer3d = inject_adapter_in_model(lora_config, generator_transformer3d) + + fake_score_lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + fake_score_transformer3d = inject_adapter_in_model(fake_score_lora_config, fake_score_transformer3d) + + network = None + fake_score_network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + generator_transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network.apply_to(text_encoder, generator_transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + fake_score_network = create_network( + 1.0, + args.rank, + args.network_alpha, + None, + fake_score_transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + fake_score_network.apply_to(None, fake_score_transformer3d, False, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = generator_transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + accelerator_fake_score_transformer3d.register_save_state_pre_hook(save_model_hook) + accelerator_fake_score_transformer3d.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + generator_transformer3d.enable_gradient_checkpointing() + fake_score_transformer3d.enable_gradient_checkpointing() + real_score_transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, generator_transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, generator_transformer3d.parameters())) + + logging.info("Add fake score peft parameters") + fake_trainable_params = list(filter(lambda p: p.requires_grad, fake_score_transformer3d.parameters())) + fake_trainable_params_optim = list(filter(lambda p: p.requires_grad, fake_score_transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + logging.info("Add fake_score_network parameters") + fake_trainable_params = list(filter(lambda p: p.requires_grad, fake_score_network.parameters())) + fake_trainable_params_optim = fake_score_network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + critic_optimizer = optimizer_cls( + fake_trainable_params_optim, + lr=args.learning_rate_critic, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + critic_optimizer = optimizer_cls( + fake_trainable_params_optim, + lr=args.learning_rate_critic, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + if args.train_mode != "normal": + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + else: + train_dataset = TextDataset( + args.train_data_meta + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + if args.enable_bucket and args.train_mode != "normal": + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to("cpu"), attention_mask=prompt_attention_mask.to("cpu"))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to("cpu"), attention_mask=neg_prompt_attention_mask.to("cpu"))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + new_examples['neg_encoder_attention_mask'] = neg_prompt_ids.attention_mask + new_examples['neg_encoder_hidden_states'] = neg_prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + elif args.train_mode == "normal": + def collate_fn(examples): + new_examples = {} + new_examples["text"] = [] + for example in examples: + new_examples["text"].append(example["text"]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to("cpu"), attention_mask=prompt_attention_mask.to("cpu"))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = prompt_embeds + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to("cpu"), attention_mask=neg_prompt_attention_mask.to("cpu"))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + new_examples['neg_encoder_attention_mask'] = neg_prompt_ids.attention_mask + new_examples['neg_encoder_hidden_states'] = neg_prompt_embeds + + return new_examples + + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = BatchSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), batch_size=args.train_batch_size, drop_last=True) + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + fake_score_lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + generator_transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + generator_transformer3d, optimizer, train_dataloader, lr_scheduler + ) + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler = accelerator_fake_score_transformer3d.prepare( + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler + ) + elif fsdp_stage != 0: + generator_transformer3d.network = network + generator_transformer3d = generator_transformer3d.to(dtype=weight_dtype) + generator_transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + generator_transformer3d, optimizer, train_dataloader, lr_scheduler + ) + fake_score_transformer3d.network = fake_score_network + fake_score_transformer3d = fake_score_transformer3d.to(dtype=weight_dtype) + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler = accelerator_fake_score_transformer3d.prepare( + fake_score_transformer3d, critic_optimizer, fake_score_lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + fake_score_network, critic_optimizer, fake_score_lr_scheduler= accelerator_fake_score_transformer3d.prepare( + fake_score_network, critic_optimizer, fake_score_lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + generator_transformer3d = shard_fn(generator_transformer3d) + fake_score_transformer3d = shard_fn(fake_score_transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + real_score_transformer3d = shard_fn(real_score_transformer3d) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + generator_transformer3d.to(accelerator.device, dtype=weight_dtype) + fake_score_transformer3d.to(accelerator.device, dtype=weight_dtype) + real_score_transformer3d.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu") + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + fake_score_path = os.path.join(path, "fake_score") + accelerator.load_state(os.path.join(args.output_dir, path)) + accelerator_fake_score_transformer3d.load_state(os.path.join(args.output_dir, fake_score_path)) + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + if args.boundary_type == "high" or args.boundary_type == "low": + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + denoising_step_list = noise_scheduler.timesteps[args.train_sampling_steps - torch.tensor(args.denoising_step_indices_list)] + + for epoch in range(first_epoch, args.num_train_epochs): + train_dmd_loss = 0.0 + train_denoising_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if args.train_mode != "normal" and epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + if args.train_mode != "normal": + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + batch['neg_encoder_hidden_states'] = torch.tile(batch['neg_encoder_hidden_states'], (4, 1, 1)) + batch['neg_encoder_attention_mask'] = torch.tile(batch['neg_encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + batch['neg_encoder_hidden_states'] = torch.tile(batch['neg_encoder_hidden_states'], (2, 1, 1)) + batch['neg_encoder_attention_mask'] = torch.tile(batch['neg_encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + real_score_transformer3d = real_score_transformer3d.to("cpu") + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + + if args.train_mode != "ti2v": + mask = resize_mask(1 - mask, mask_latents) + else: + mask = F.interpolate(mask[:, :1], size=mask_latents.size()[-3:], mode='trilinear', align_corners=True).to(accelerator.device, weight_dtype) + + if args.train_mode != "ti2v": + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + else: + inpaint_latents = mask_latents + + target_shape = mask_latents.size() + else: + text = batch['text'] + if args.fix_sample_size is not None: + local_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + num_frames = args.video_sample_n_frames + else: + if args.random_hw_adapt and args.training_with_video_token_length: + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + if rng is None: + local_length = np.random.choice(list(length_to_frame_num.keys())) + else: + local_length = rng.choice(list(length_to_frame_num.keys())) + num_frames = length_to_frame_num[local_length] + + aspect_ratio_sample_size = {key : [x / 512 * local_length for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if rng is None: + aspect_ratio_key = np.random.choice(list(aspect_ratio_sample_size.keys())) + else: + aspect_ratio_key = rng.choice(list(aspect_ratio_sample_size.keys())) + local_sample_size = aspect_ratio_sample_size[aspect_ratio_key] + else: + num_frames = args.video_sample_n_frames + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if rng is None: + aspect_ratio_key = np.random.choice(list(aspect_ratio_sample_size.keys())) + else: + aspect_ratio_key = rng.choice(list(aspect_ratio_sample_size.keys())) + local_sample_size = aspect_ratio_sample_size[aspect_ratio_key] + local_sample_size = [int(x / 16) * 16 for x in local_sample_size] + + target_shape = ( + len(text), + vae.latent_channels, + int((num_frames - 1) // vae.temporal_compression_ratio + 1), + int(local_sample_size[0] // vae.spatial_compression_ratio), + int(local_sample_size[1] // vae.spatial_compression_ratio), + ) + + if args.low_vram: + vae.to('cpu') + real_score_transformer3d = real_score_transformer3d.to("cpu") + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=accelerator.device) + neg_prompt_embeds = batch['neg_encoder_hidden_states'].to(device=accelerator.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(accelerator.device), attention_mask=prompt_attention_mask.to(accelerator.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + neg_txt = [ + "色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走" for text in batch['text'] + ] + neg_prompt_ids = tokenizer( + neg_txt, + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + neg_text_input_ids = neg_prompt_ids.input_ids + neg_prompt_attention_mask = neg_prompt_ids.attention_mask + + neg_seq_lens = neg_prompt_attention_mask.gt(0).sum(dim=1).long() + neg_prompt_embeds = text_encoder(neg_text_input_ids.to(accelerator.device), attention_mask=neg_prompt_attention_mask.to(accelerator.device))[0] + neg_prompt_embeds = [u[:v] for u, v in zip(neg_prompt_embeds, neg_seq_lens)] + + if args.low_vram: + generator_transformer3d = generator_transformer3d.to(accelerator.device) + real_score_transformer3d = real_score_transformer3d.to(accelerator.device) + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + if args.train_mode == "ti2v": + if rng is None: + t2v_in_ti2v = np.random.choice([0, 1], p = [0.50, 0.50]) + else: + t2v_in_ti2v = rng.choice([0, 1], p = [0.50, 0.50]) + + with accelerator.accumulate(generator_transformer3d): + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + + step_indices = [ + torch.argmin(torch.abs(schedule_timesteps - t)).item() + for t in timesteps + ] + step_indices = torch.tensor(step_indices, device=accelerator.device) + sigma = sigmas[step_indices].flatten() + + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + def add_noise(latents, noise, timesteps): + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + return (1.0 - sigmas) * latents + sigmas * noise + + def generate_and_sync_list(num_denoising_steps, device): + indices = torch.randint(low=0, high=num_denoising_steps, size=(1,), generator=torch_rng, device=device) + if dist.is_initialized(): + dist.broadcast(indices, src=0) + return indices.tolist() + + def convert_flow_pred_to_x0( + scheduler, + flow_pred: torch.Tensor, + xt: torch.Tensor, + timestep: torch.Tensor + ) -> torch.Tensor: + """ + Convert flow matching's prediction to x0 prediction. + Supports both 4D [B, C, H, W] and 5D [B, C, F, H, W] inputs. + """ + original_dtype = flow_pred.dtype + device = flow_pred.device + + flow_pred = flow_pred.double() + xt = xt.double() + timesteps = scheduler.timesteps.to(device).double() + sigmas = scheduler.sigmas.to(device).double() + timestep = timestep.to(device).double() + + timestep_id = torch.argmin((timesteps.unsqueeze(0) - timestep.unsqueeze(1)).abs(), dim=1) + sigma_t = sigmas[timestep_id] + + ndim = flow_pred.ndim + if ndim == 4: + sigma_t = sigma_t.view(-1, 1, 1, 1) + elif ndim == 5: + sigma_t = sigma_t.view(-1, 1, 1, 1, 1) + else: + raise ValueError(f"Expected 4D or 5D input, got {ndim}D tensor.") + + x0_pred = xt - sigma_t * flow_pred + return x0_pred.to(original_dtype) + + # --- Main Training Logic --- + bsz, channel, num_frames, height, width = target_shape + if step % args.gen_update_interval == 0: + generator_noise = torch.randn(target_shape, device=accelerator.device, generator=torch_rng, dtype=weight_dtype) + num_denoising_steps = len(denoising_step_list) + final_step_index = generate_and_sync_list(num_denoising_steps, device=generator_noise.device)[0] + + # Precompute seq_len once (same for all steps) + patch_h, patch_w = accelerator.unwrap_model(generator_transformer3d).config.patch_size[1:] + seq_len = math.ceil((width * height) / (patch_h * patch_w) * num_frames) + + for index, current_timestep in enumerate(denoising_step_list): + is_final_step = (index == final_step_index) + timestep = torch.full( + generator_noise.shape[:1], + current_timestep, + device=generator_noise.device, + dtype=torch.int64 + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + context_manager = torch.no_grad() if not is_final_step else contextlib.nullcontext() + + with context_manager: + if args.train_mode == "ti2v": + mask_bs = mask.size()[0] + if t2v_in_ti2v: + _generator_noise = (1 - mask) * inpaint_latents + mask * generator_noise + + temp_ts = (mask[:, 0, :, ::2, ::2] * timestep[:, None, None, None]).flatten(1) + _timestep = torch.cat([temp_ts, temp_ts.new_ones(mask_bs, seq_len - temp_ts.size(1)) * timestep[:, None,]], dim = 1) + else: + _timestep = mask.new_ones(mask_bs, seq_len) * timestep[:, None,] + _generator_noise = generator_noise + else: + _timestep = timestep + _generator_noise = generator_noise + generator_pred = generator_transformer3d( + x=_generator_noise, + context=prompt_embeds, + t=_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + generator_pred = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=generator_pred, + xt=generator_noise, + timestep=timestep + ) + + if is_final_step: + break + + next_timestep = denoising_step_list[index + 1] * torch.ones( + generator_noise.shape[:1], dtype=torch.long, device=generator_noise.device + ) + generator_noise = add_noise( + generator_pred, + torch.randn(generator_pred.shape, dtype=generator_pred.dtype, device=generator_pred.device, generator=torch_rng), + next_timestep + ) + + indices = idx_sampling(bsz, generator=torch_rng, device=accelerator.device).long().cpu() + generator_timestep = noise_scheduler.timesteps[indices].to(device=accelerator.device) + generator_denoised_input = add_noise( + generator_pred, + torch.randn(generator_pred.shape, dtype=generator_pred.dtype, device=generator_pred.device, generator=torch_rng), + generator_timestep + ).detach().to(accelerator.device, dtype=weight_dtype) + + # Compute fake score + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device), torch.no_grad(): + if args.train_mode == "ti2v": + mask_bs = mask.size()[0] + if t2v_in_ti2v: + _generator_denoised_input = (1 - mask) * inpaint_latents + mask * generator_denoised_input + + temp_ts = (mask[:, 0, :, ::2, ::2] * generator_timestep[:, None, None, None]).flatten(1) + _generator_timestep = torch.cat([temp_ts, temp_ts.new_ones(mask_bs, seq_len - temp_ts.size(1)) * generator_timestep[:, None,]], dim = 1) + else: + _generator_timestep = mask.new_ones(mask_bs, seq_len) * generator_timestep[:, None,] + _generator_denoised_input = generator_denoised_input + else: + _generator_timestep = generator_timestep + _generator_denoised_input = generator_denoised_input + fake_score_main_cond = fake_score_transformer3d( + x=_generator_denoised_input, + context=prompt_embeds, + t=_generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + fake_score_main_cond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_main_cond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + if args.fake_guidance_scale != 0.0: + fake_score_main_uncond = fake_score_transformer3d( + x=_generator_denoised_input, + context=neg_prompt_embeds, + t=_generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + fake_score_main_uncond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_main_uncond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + fake_score_main = fake_score_main_uncond + ( + fake_score_main_cond - fake_score_main_uncond + ) * args.fake_guidance_scale + else: + fake_score_main = fake_score_main_cond + + # Compute real score + real_score_main_cond = real_score_transformer3d( + x=_generator_denoised_input, + context=prompt_embeds, + t=_generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + real_score_main_cond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=real_score_main_cond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + real_score_main_uncond = real_score_transformer3d( + x=_generator_denoised_input, + context=neg_prompt_embeds, + t=_generator_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + real_score_main_uncond = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=real_score_main_uncond, + xt=generator_denoised_input, + timestep=generator_timestep + ) + + real_score_main = real_score_main_uncond + ( + real_score_main_cond - real_score_main_uncond + ) * args.real_guidance_scale + + # DMD loss + fake_to_real_grad = fake_score_main - real_score_main + generator_to_real_norm = generator_pred - real_score_main + normalizer = torch.abs(generator_to_real_norm).mean(dim=[1, 2, 3, 4], keepdim=True) + fake_to_real_grad = fake_to_real_grad / normalizer + fake_to_real_grad = torch.nan_to_num(fake_to_real_grad) + + dmd_loss = 0.5 * F.mse_loss( + generator_pred.double(), + (generator_pred.double() - fake_to_real_grad.double()).detach(), + reduction="mean" + ) + avg_dmd_loss = accelerator.gather(dmd_loss.repeat(args.train_batch_size)).mean() + train_dmd_loss += avg_dmd_loss.item() / args.gradient_accumulation_steps + + if args.low_vram: + real_score_transformer3d = real_score_transformer3d.to("cpu") + fake_score_transformer3d = fake_score_transformer3d.to("cpu") + torch.cuda.empty_cache() + + accelerator.backward(dmd_loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + if args.low_vram: + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + torch.cuda.empty_cache() + + with accelerator_fake_score_transformer3d.accumulate(fake_score_transformer3d): + # --- Fake Critic Denoising Loss --- + with torch.no_grad(): + fake_score_critic_noise = torch.randn(target_shape, device=accelerator.device, generator=torch_rng, dtype=weight_dtype) + num_denoising_steps = len(denoising_step_list) + final_step_index = generate_and_sync_list(num_denoising_steps, device=fake_score_critic_noise.device)[0] + + patch_h, patch_w = accelerator.unwrap_model(generator_transformer3d).config.patch_size[1:] + seq_len = math.ceil((width * height) / (patch_h * patch_w) * num_frames) + + for index, current_timestep in enumerate(denoising_step_list): + is_final_step = (index == final_step_index) + timestep = torch.full( + fake_score_critic_noise.shape[:1], + current_timestep, + device=fake_score_critic_noise.device, + dtype=torch.int64 + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + if args.train_mode == "ti2v": + mask_bs = mask.size()[0] + if t2v_in_ti2v: + _fake_score_critic_noise = (1 - mask) * inpaint_latents + mask * fake_score_critic_noise + + temp_ts = (mask[:, 0, :, ::2, ::2] * timestep[:, None, None, None]).flatten(1) + _timestep = torch.cat([temp_ts, temp_ts.new_ones(mask_bs, seq_len - temp_ts.size(1)) * timestep[:, None,]], dim = 1) + else: + _timestep = mask.new_ones(mask_bs, seq_len) * timestep[:, None,] + _fake_score_critic_noise = fake_score_critic_noise + else: + _timestep = timestep + _fake_score_critic_noise = fake_score_critic_noise + fake_score_denoised_pred = generator_transformer3d( + x=_fake_score_critic_noise, + context=prompt_embeds, + t=_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + fake_score_denoised_pred = convert_flow_pred_to_x0( + scheduler=noise_scheduler, + flow_pred=fake_score_denoised_pred, + xt=fake_score_critic_noise, + timestep=timestep + ) + + if is_final_step: + break + + next_timestep = denoising_step_list[index + 1] * torch.ones( + fake_score_critic_noise.shape[:1], + dtype=torch.long, + device=fake_score_critic_noise.device + ) + + fake_score_critic_noise = add_noise( + fake_score_denoised_pred, + torch.randn(fake_score_denoised_pred.shape, dtype=fake_score_denoised_pred.dtype, device=fake_score_denoised_pred.device, generator=torch_rng), + next_timestep + ) + + indices = idx_sampling(bsz, generator=torch_rng, device=accelerator.device).long().cpu() + critic_timestep = noise_scheduler.timesteps[indices].to(device=accelerator.device) + critic_noise = torch.randn(fake_score_denoised_pred.shape, dtype=fake_score_denoised_pred.dtype, device=fake_score_denoised_pred.device, generator=torch_rng) + + fake_score_denoised_input = add_noise( + fake_score_denoised_pred, + critic_noise, + critic_timestep + ) + if args.train_mode == "ti2v": + mask_bs = mask.size()[0] + if t2v_in_ti2v: + fake_score_denoised_input = (1 - mask) * inpaint_latents + mask * fake_score_denoised_input + + temp_ts = (mask[:, 0, :, ::2, ::2] * critic_timestep[:, None, None, None]).flatten(1) + _critic_timestep = torch.cat([temp_ts, temp_ts.new_ones(mask_bs, seq_len - temp_ts.size(1)) * critic_timestep[:, None,]], dim = 1) + else: + _critic_timestep = mask.new_ones(mask_bs, seq_len) * critic_timestep[:, None,] + else: + _critic_timestep = critic_timestep + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + fake_score_denoised_output = fake_score_transformer3d( + x=fake_score_denoised_input, + context=prompt_embeds, + t=_critic_timestep, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + denoising_loss = custom_mse_loss(fake_score_denoised_output, critic_noise - fake_score_denoised_pred) + avg_denoising_loss = accelerator_fake_score_transformer3d.gather(denoising_loss.repeat(args.train_batch_size)).mean() + train_denoising_loss += avg_denoising_loss.item() / args.gradient_accumulation_steps + + if args.low_vram: + generator_transformer3d = generator_transformer3d.to("cpu") + torch.cuda.empty_cache() + + accelerator_fake_score_transformer3d.backward(denoising_loss) + if accelerator_fake_score_transformer3d.sync_gradients: + accelerator_fake_score_transformer3d.clip_grad_norm_(fake_trainable_params, args.max_grad_norm) + critic_optimizer.step() + fake_score_lr_scheduler.step() + critic_optimizer.zero_grad() + + if args.low_vram: + fake_score_transformer3d = fake_score_transformer3d.to(accelerator.device) + generator_transformer3d = generator_transformer3d.to(accelerator.device) + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_denoising_loss": train_denoising_loss, "train_dmd_loss": train_dmd_loss}, step=global_step) + train_dmd_loss = 0.0 + train_denoising_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(generator_transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}-fake_score.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(fake_score_transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}-fake_score.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(fake_score_network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + fake_score_save_path = os.path.join(save_path, "fake_score") + accelerator.save_state(save_path) + accelerator_fake_score_transformer3d.save_state(fake_score_save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + generator_transformer3d, + network, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"denoising_loss": denoising_loss.detach().item(), "dmd_loss": dmd_loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + generator_transformer3d, + network, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + generator_transformer3d = unwrap_model(generator_transformer3d) + fake_score_transformer3d = unwrap_model(fake_score_transformer3d) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(generator_transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}-fake_score.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(fake_score_transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}-fake_score.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(fake_score_network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + fake_score_save_path = os.path.join(save_path, "fake_score") + accelerator.save_state(save_path) + accelerator_fake_score_transformer3d.save_state(fake_score_save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2/train_distill_lora.sh b/VideoX-Fun/scripts/wan2.2/train_distill_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..241daa32553578ad74c600db752be58c505692dd --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_distill_lora.sh @@ -0,0 +1,45 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_distill_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-05 \ + --learning_rate_critic=1e-06 \ + --seed=42 \ + --output_dir="output_dir_distill" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="i2v" \ + --low_vram \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2/train_lora.py b/VideoX-Fun/scripts/wan2.2/train_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..f608a4eeda707a654e1519e77568f8b3756e2aa5 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_lora.py @@ -0,0 +1,2022 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, WanT5EncoderModel, + Wan2_2Transformer3DModel) +from videox_fun.pipeline import Wan2_2Pipeline, Wan2_2I2VPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import create_network, merge_lora, unmerge_lora +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.75 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, config, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = Wan2_2I2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + else: + pipeline = Wan2_2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--noise_share_in_frames", action="store_true", help="Whether enable noise share in frames." + ) + parser.add_argument( + "--noise_share_in_frames_ratio", type=float, default=0.5, help="Noise share ratio.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + enable_inpaint=True if args.train_mode != "normal" else False, + ) + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def collate_fn(examples): + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size], rng=rng) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + if rng is None: + local_video_sample_size = np.random.choice(choice_list) + else: + local_video_sample_size = rng.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio( + args.video_sample_size, rng=rng) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size(), image_start_only=True) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + if args.boundary_type == "high" or args.boundary_type == "low": + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + if args.train_mode != "normal": + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "normal": + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + + if args.train_mode != "ti2v": + mask = resize_mask(1 - mask, latents) + else: + mask = F.interpolate(mask[:, :1], size=latents.size()[-3:], mode='trilinear', align_corners=True).to(accelerator.device, weight_dtype) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + if args.train_mode != "ti2v": + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + else: + inpaint_latents = mask_latents + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + if args.train_mode == "ti2v": + if rng is None: + t2v_in_ti2v = np.random.choice([0, 1], p = [0.50, 0.50]) + else: + t2v_in_ti2v = rng.choice([0, 1], p = [0.50, 0.50]) + + mask_bs = mask.size()[0] + if t2v_in_ti2v: + noisy_latents = (1 - mask) * inpaint_latents + mask * noisy_latents + + temp_ts = (mask[:, 0, :, ::2, ::2] * timesteps[:, None, None, None]).flatten(1) + timesteps = torch.cat([temp_ts, temp_ts.new_ones(mask_bs, seq_len - temp_ts.size(1)) * timesteps[:, None,]], dim = 1) + else: + timesteps = mask.new_ones(mask_bs, seq_len) * timesteps[:, None,] + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" and args.train_mode != "ti2v" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2/train_lora.sh b/VideoX-Fun/scripts/wan2.2/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..112b669be4f2cae585543ead3ff83a327bb47104 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_lora.sh @@ -0,0 +1,92 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-T2V-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="normal" \ + --low_vram + +# The Training Shell Code for Image to Video +# You need to use "config/wan2.2/wan_civitai_i2v.yaml" +# +# export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-I2V-A14B" +# export DATASET_NAME="datasets/internal_datasets/" +# export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# # NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# # export NCCL_IB_DISABLE=1 +# # export NCCL_P2P_DISABLE=1 +# NCCL_DEBUG=INFO + +# accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_lora.py \ +# --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --train_data_dir=$DATASET_NAME \ +# --train_data_meta=$DATASET_META_NAME \ +# --image_sample_size=640 \ +# --video_sample_size=640 \ +# --token_sample_size=640 \ +# --video_sample_stride=2 \ +# --video_sample_n_frames=81 \ +# --train_batch_size=1 \ +# --video_repeat=1 \ +# --gradient_accumulation_steps=1 \ +# --dataloader_num_workers=8 \ +# --num_train_epochs=100 \ +# --checkpointing_steps=50 \ +# --learning_rate=1e-04 \ +# --seed=42 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --vae_mini_batch=1 \ +# --max_grad_norm=0.05 \ +# --random_hw_adapt \ +# --training_with_video_token_length \ +# --enable_bucket \ +# --uniform_sampling \ +# --boundary_type="low" \ +# --rank=64 \ +# --network_alpha=32 \ +# --target_name="q,k,v,ffn.0,ffn.2" \ +# --use_peft_lora \ +# --train_mode="i2v" \ +# --low_vram diff --git a/VideoX-Fun/scripts/wan2.2/train_s2v.py b/VideoX-Fun/scripts/wan2.2/train_s2v.py new file mode 100644 index 0000000000000000000000000000000000000000..84b31831a2560d76a7c4cf35b7ef41b070676e89 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_s2v.py @@ -0,0 +1,2076 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +import random +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedStateDictConfig, ShardedOptimStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.data.dataset_video import VideoSpeechControlDataset +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, WanT5EncoderModel, WanAudioEncoder, + Wan2_2Transformer3DModel_S2V) +from videox_fun.pipeline import Wan2_2S2VPipeline, Wan2_2I2VPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + pipeline = Wan2_2S2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--motion_frames", + type=int, + default=73, + help="Motion frames of s2v.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--control_ref_image", + type=str, + default="first_frame", + help=( + 'The format of training data. Support `"first_frame"`' + ' (default), `"random"`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Audio Encoder + audio_encoder = WanAudioEncoder( + os.path.join(args.pretrained_model_name_or_path, config['audio_encoder_kwargs'].get('audio_encoder_subpath', 'audio_encoder')), + "cpu" + ) + audio_encoder.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=Wan2_2Transformer3DModel_S2V, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = Wan2_2Transformer3DModel_S2V.load_config(ema_path, return_unused_kwargs=True) + load_model = Wan2_2Transformer3DModel_S2V.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=Wan2_2Transformer3DModel_S2V, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = Wan2_2Transformer3DModel_S2V.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = VideoSpeechControlDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + enable_bucket=args.enable_bucket, enable_inpaint=True, enable_motion_info=True, motion_frames=args.motion_frames + ) + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.video_sample_size > 256: + sample_sizes = list(range(256, args.video_sample_size + 1, 128)) + + if sample_sizes[-1] != args.video_sample_size: + sample_sizes.append(args.video_sample_size) + else: + sample_sizes = [args.video_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["motion_pixel_values"] = [] + new_examples["text"] = [] + # Used in Control Mode + new_examples["control_pixel_values"] = [] + new_examples["ref_pixel_values"] = [] + new_examples["clip_idx"] = [] + + new_examples["audio"] = [] + new_examples["sample_rate"] = [] + new_examples["fps"] = [] + + # Used in Inpaint mode + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + f, h, w, c = np.shape(pixel_value) + + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + local_video_sample_size = np.random.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n. + batch_video_length = (batch_video_length) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + motion_pixel_values = torch.from_numpy(example["motion_pixel_values"]).permute(0, 3, 1, 2).contiguous() + motion_pixel_values = motion_pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["motion_pixel_values"].append(transform(motion_pixel_values)) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + new_examples["text"].append(example["text"]) + + if args.control_ref_image == "first_frame": + clip_index = 0 + else: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.40 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + number_list_prob = np.array(_create_special_list(len(new_examples["pixel_values"][-1]))) + clip_index = np.random.choice(list(range(len(new_examples["pixel_values"][-1]))), p = number_list_prob) + + ref_pixel_values = new_examples["pixel_values"][-1][clip_index].unsqueeze(0) + new_examples["ref_pixel_values"].append(ref_pixel_values) + + clip_pixel_values = new_examples["pixel_values"][-1][clip_index].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + new_examples["clip_idx"].append(clip_index) + + audio_length = np.shape(example["audio"])[0] + batch_audio_length = int(audio_length / pixel_values.size()[0] * batch_video_length) + new_examples["audio"].append(example["audio"][:batch_audio_length]) + new_examples["sample_rate"].append(example["sample_rate"]) + new_examples["fps"].append(example["fps"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["motion_pixel_values"] = torch.stack([example for example in new_examples["motion_pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + new_examples["ref_pixel_values"] = torch.stack([example for example in new_examples["ref_pixel_values"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["clip_idx"] = torch.tensor(new_examples["clip_idx"]) + new_examples["audio"] = torch.stack([example for example in new_examples["audio"]]) + new_examples["sample_rate"] = new_examples["sample_rate"] + new_examples["fps"] = new_examples["fps"] + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + audio_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=torch.float32) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + if args.boundary_type == "high" or args.boundary_type == "low": + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + ref_pixel_values = batch["ref_pixel_values"].cpu() + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + for idx, (ref_pixel_value, text) in enumerate(zip(ref_pixel_values, texts)): + ref_pixel_value = ref_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(ref_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_ref.gif", rescale=True) + + motion_pixel_values = batch["motion_pixel_values"].cpu() + motion_pixel_values = rearrange(motion_pixel_values, "b f c h w -> b c f h w") + for idx, (motion_pixel_value, text) in enumerate(zip(motion_pixel_values, texts)): + motion_pixel_value = motion_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(motion_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_motion.gif", rescale=True) + + clip_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['text'] + for idx, (clip_pixel_value, text) in enumerate(zip(clip_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + motion_pixel_values = batch["motion_pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + ref_pixel_values = batch["ref_pixel_values"].to(weight_dtype) + clip_idx = batch["clip_idx"] + audio = batch["audio"] + sample_rate = batch["sample_rate"] + fps = batch["fps"] + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + motion_pixel_values = torch.tile(motion_pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (4, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (4,)) + audio = torch.tile(audio, (4, 1)) + sample_rate = sample_rate * 4 + fps = fps * 4 + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + motion_pixel_values = torch.tile(motion_pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (2, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (2,)) + audio = torch.tile(audio, (2, 1)) + sample_rate = sample_rate * 2 + fps = fps * 2 + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n. + temp_n_frames = (temp_n_frames) // sample_n_frames_bucket_interval + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + )) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n. + actual_video_length = (actual_video_length) // sample_n_frames_bucket_interval + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + audio_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + + if rng is None: + zero_tail_frames = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_tail_frames = rng.choice([0, 1], p = [0.90, 0.10]) + if zero_tail_frames: + if rng is None: + zero_frames_num = np.random.randint(1, control_pixel_values.size()[1]) + else: + zero_frames_num = rng.integers(1, control_pixel_values.size()[1]) + control_pixel_values[:, zero_frames_num:] = torch.ones_like(control_pixel_values[:, zero_frames_num:]) * -1 + + # Make control pixel values to zero + for bs_index in range(control_pixel_values.size()[0]): + if rng is None: + zero_init_control_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_control_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if zero_init_control_latents_conv_in: + control_pixel_values[bs_index] = torch.ones_like(control_pixel_values[bs_index]) * -1 + # Encode control latents + pad_control_pixel_values = torch.cat([control_pixel_values[:, 0:1, :].repeat(1, 1, 1, 1, 1), control_pixel_values], dim=1) + control_latents = _batch_encode_vae(pad_control_pixel_values)[:, :, 1:] + + # Encode Reference latents + ref_latents = _batch_encode_vae(ref_pixel_values) + + # Encode Motion latents + if rng is None: + zero_motion_pixel_values = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_motion_pixel_values = rng.choice([0, 1], p = [0.90, 0.10]) + if zero_motion_pixel_values: + height, width = control_pixel_values.size()[-2], control_pixel_values.size()[-1] + motion_pixel_values = torch.zeros([1, args.motion_frames, 3, height, width], dtype=control_latents.dtype, device=control_latents.device) + + has_motion_pixel_values = torch.sum(motion_pixel_values) == 0 + if torch.sum(clip_idx) != 0: + init_first_frame = False + else: + if rng is None: + init_first_frame = np.random.choice([0, 1], p = [0.50, 0.50]) + else: + init_first_frame = rng.choice([0, 1], p = [0.50, 0.50]) + if init_first_frame or has_motion_pixel_values: + if not has_motion_pixel_values: + motion_pixel_values[:, -6:, :] = ref_pixel_values + + motion_frames_latents_length = int((args.motion_frames - 1) / sample_n_frames_bucket_interval + 1) + local_pixel_values = torch.cat([motion_pixel_values, pixel_values], dim = 1) + local_latents = _batch_encode_vae(local_pixel_values) + latents = local_latents[:, :, motion_frames_latents_length:] + motion_latents = local_latents[:, :, :motion_frames_latents_length] + drop_motion_frames = False + else: + local_pixel_values = torch.cat([ref_pixel_values, pixel_values], dim = 1) + latents = _batch_encode_vae(local_pixel_values) + latents = latents[:, :, 1:] + motion_latents = _batch_encode_vae(motion_pixel_values) + drop_motion_frames = True + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + audio_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + with torch.no_grad(): + # Extract audio emb + new_audio_wav2vec_fea = [] + for index in range(len(audio)): + _audio_wav2vec_fea = audio_encoder.extract_audio_feat_without_file_load( + audio[index], sample_rate[index], return_all_layers=True + ) + new_audio_wav2vec_fea.append(_audio_wav2vec_fea) + audio_wav2vec_fea = torch.stack(new_audio_wav2vec_fea).to(device=accelerator.device, dtype=weight_dtype) + + new_audio_wav2vec_fea = [] + for index in range(len(audio)): + _audio_wav2vec_fea, num_repeat = audio_encoder.get_audio_embed_bucket_fps( + audio_wav2vec_fea[index], fps=fps[index], batch_frames=control_pixel_values.size()[1], m=0) + new_audio_wav2vec_fea.append(_audio_wav2vec_fea) + audio_wav2vec_fea = torch.stack(new_audio_wav2vec_fea).to(device=accelerator.device, dtype=weight_dtype) + + if len(audio_wav2vec_fea.shape) == 3: + audio_wav2vec_fea = audio_wav2vec_fea.permute(0, 2, 1) + elif len(audio_wav2vec_fea.shape) == 4: + audio_wav2vec_fea = audio_wav2vec_fea.permute(0, 2, 3, 1) + + for bs_index in range(audio_wav2vec_fea.size()[0]): + if rng is None: + zero_init_control_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_control_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if zero_init_control_latents_conv_in: + audio_wav2vec_fea[bs_index] = torch.ones_like(audio_wav2vec_fea[bs_index]) * 0 + + if zero_tail_frames: + audio_wav2vec_fea[..., zero_frames_num:] = torch.zeros_like(audio_wav2vec_fea[..., zero_frames_num:]) + # audio_wav2vec_fea = audio_wav2vec_fea[..., :control_pixel_values.size()[1]] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + audio_encoder.to("cpu") + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + cond_states=control_latents, + motion_latents=motion_latents, + ref_latents=ref_latents, + audio_input=audio_wav2vec_fea, + motion_frames=[[args.motion_frames, (args.motion_frames + 3) // 4]] * bsz, + drop_motion_frames=drop_motion_frames, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2/train_s2v.sh b/VideoX-Fun/scripts/wan2.2/train_s2v.sh new file mode 100644 index 0000000000000000000000000000000000000000..a8d2d952a79bf2afa4ff68b5b39a584e878e5272 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_s2v.sh @@ -0,0 +1,39 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_s2v.py \ + --config_path="config/wan2.2/wan_civitai_s2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=1 \ + --video_sample_n_frames=80 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --trainable_modules "." diff --git a/VideoX-Fun/scripts/wan2.2/train_s2v_lora.py b/VideoX-Fun/scripts/wan2.2/train_s2v_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..af48a47055cab3eaf9e373839ee76daf9e4119a2 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_s2v_lora.py @@ -0,0 +1,2069 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + + +from videox_fun.data import (ImageVideoDataset, ImageVideoSampler, + VideoAnimateDataset, get_random_mask, + process_pose_file, process_pose_params) +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.data.dataset_video import VideoSpeechControlDataset +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, + CLIPModel, Wan2_2Transformer3DModel, + Wan2_2Transformer3DModel_Animate, + Wan2_2Transformer3DModel_S2V, WanAudioEncoder, + WanT5EncoderModel) +from videox_fun.pipeline import (Wan2_2FunControlPipeline, Wan2_2I2VPipeline, + Wan2_2Pipeline, Wan2_2S2VPipeline) +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + pipeline = Wan2_2S2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--motion_frames", + type=int, + default=73, + help="Motion frames of s2v.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--control_ref_image", + type=str, + default="first_frame", + help=( + 'The format of training data. Support `"first_frame"`' + ' (default), `"random"`.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + # Get Audio Encoder + audio_encoder = WanAudioEncoder( + os.path.join(args.pretrained_model_name_or_path, config['audio_encoder_kwargs'].get('audio_encoder_subpath', 'audio_encoder')), + "cpu" + ) + audio_encoder.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = Wan2_2Transformer3DModel_S2V.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = VideoSpeechControlDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + enable_bucket=args.enable_bucket, enable_inpaint=True, enable_motion_info=True, motion_frames=args.motion_frames + ) + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.video_sample_size > 256: + sample_sizes = list(range(256, args.video_sample_size + 1, 128)) + + if sample_sizes[-1] != args.video_sample_size: + sample_sizes.append(args.video_sample_size) + else: + sample_sizes = [args.video_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["motion_pixel_values"] = [] + new_examples["text"] = [] + # Used in Control Mode + new_examples["control_pixel_values"] = [] + new_examples["ref_pixel_values"] = [] + new_examples["clip_idx"] = [] + + new_examples["audio"] = [] + new_examples["sample_rate"] = [] + new_examples["fps"] = [] + + # Used in Inpaint mode + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + f, h, w, c = np.shape(pixel_value) + + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + local_video_sample_size = np.random.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n. + batch_video_length = (batch_video_length) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + motion_pixel_values = torch.from_numpy(example["motion_pixel_values"]).permute(0, 3, 1, 2).contiguous() + motion_pixel_values = motion_pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["motion_pixel_values"].append(transform(motion_pixel_values)) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + new_examples["text"].append(example["text"]) + + if args.control_ref_image == "first_frame": + clip_index = 0 + else: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.40 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + number_list_prob = np.array(_create_special_list(len(new_examples["pixel_values"][-1]))) + clip_index = np.random.choice(list(range(len(new_examples["pixel_values"][-1]))), p = number_list_prob) + + ref_pixel_values = new_examples["pixel_values"][-1][clip_index].unsqueeze(0) + new_examples["ref_pixel_values"].append(ref_pixel_values) + + clip_pixel_values = new_examples["pixel_values"][-1][clip_index].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + new_examples["clip_idx"].append(clip_index) + + audio_length = np.shape(example["audio"])[0] + batch_audio_length = int(audio_length / pixel_values.size()[0] * batch_video_length) + new_examples["audio"].append(example["audio"][:batch_audio_length]) + new_examples["sample_rate"].append(example["sample_rate"]) + new_examples["fps"].append(example["fps"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["motion_pixel_values"] = torch.stack([example for example in new_examples["motion_pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + new_examples["ref_pixel_values"] = torch.stack([example for example in new_examples["ref_pixel_values"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["clip_idx"] = torch.tensor(new_examples["clip_idx"]) + new_examples["audio"] = torch.stack([example for example in new_examples["audio"]]) + new_examples["sample_rate"] = new_examples["sample_rate"] + new_examples["fps"] = new_examples["fps"] + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + audio_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=torch.float32) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + if args.boundary_type == "high" or args.boundary_type == "low": + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + ref_pixel_values = batch["ref_pixel_values"].cpu() + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + for idx, (ref_pixel_value, text) in enumerate(zip(ref_pixel_values, texts)): + ref_pixel_value = ref_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(ref_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_ref.gif", rescale=True) + + motion_pixel_values = batch["motion_pixel_values"].cpu() + motion_pixel_values = rearrange(motion_pixel_values, "b f c h w -> b c f h w") + for idx, (motion_pixel_value, text) in enumerate(zip(motion_pixel_values, texts)): + motion_pixel_value = motion_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(motion_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_motion.gif", rescale=True) + + clip_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['text'] + for idx, (clip_pixel_value, text) in enumerate(zip(clip_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + motion_pixel_values = batch["motion_pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + ref_pixel_values = batch["ref_pixel_values"].to(weight_dtype) + clip_idx = batch["clip_idx"] + audio = batch["audio"] + sample_rate = batch["sample_rate"] + fps = batch["fps"] + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + motion_pixel_values = torch.tile(motion_pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (4, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (4,)) + audio = torch.tile(audio, (4, 1)) + sample_rate = sample_rate * 4 + fps = fps * 4 + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + motion_pixel_values = torch.tile(motion_pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (2, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (2,)) + audio = torch.tile(audio, (2, 1)) + sample_rate = sample_rate * 2 + fps = fps * 2 + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + # clip_pixel_values = batch["clip_pixel_values"].to(weight_dtype) + # # Increase the batch size when the length of the latent sequence of the current sample is small + # if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + # if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + # clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + # elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + # clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n. + temp_n_frames = (temp_n_frames) // sample_n_frames_bucket_interval + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + )) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n. + actual_video_length = (actual_video_length) // sample_n_frames_bucket_interval + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + audio_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + + if rng is None: + zero_tail_frames = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_tail_frames = rng.choice([0, 1], p = [0.90, 0.10]) + if zero_tail_frames: + if rng is None: + zero_frames_num = np.random.randint(1, control_pixel_values.size()[1]) + else: + zero_frames_num = rng.integers(1, control_pixel_values.size()[1]) + control_pixel_values[:, zero_frames_num:] = torch.ones_like(control_pixel_values[:, zero_frames_num:]) * -1 + + # Make control pixel values to zero + for bs_index in range(control_pixel_values.size()[0]): + if rng is None: + zero_init_control_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_control_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if zero_init_control_latents_conv_in: + control_pixel_values[bs_index] = torch.ones_like(control_pixel_values[bs_index]) * -1 + # Encode control latents + pad_control_pixel_values = torch.cat([control_pixel_values[:, 0:1, :].repeat(1, 1, 1, 1, 1), control_pixel_values], dim=1) + control_latents = _batch_encode_vae(pad_control_pixel_values)[:, :, 1:] + + # Encode Reference latents + ref_latents = _batch_encode_vae(ref_pixel_values) + + # Encode Motion latents + if rng is None: + zero_motion_pixel_values = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_motion_pixel_values = rng.choice([0, 1], p = [0.90, 0.10]) + if zero_motion_pixel_values: + height, width = control_pixel_values.size()[-2], control_pixel_values.size()[-1] + motion_pixel_values = torch.zeros([1, args.motion_frames, 3, height, width], dtype=control_latents.dtype, device=control_latents.device) + + has_motion_pixel_values = torch.sum(motion_pixel_values) == 0 + if torch.sum(clip_idx) != 0: + init_first_frame = False + else: + if rng is None: + init_first_frame = np.random.choice([0, 1], p = [0.50, 0.50]) + else: + init_first_frame = rng.choice([0, 1], p = [0.50, 0.50]) + if init_first_frame or has_motion_pixel_values: + if not has_motion_pixel_values: + motion_pixel_values[:, -6:, :] = ref_pixel_values + + motion_frames_latents_length = int((args.motion_frames - 1) / sample_n_frames_bucket_interval + 1) + local_pixel_values = torch.cat([motion_pixel_values, pixel_values], dim = 1) + local_latents = _batch_encode_vae(local_pixel_values) + latents = local_latents[:, :, motion_frames_latents_length:] + motion_latents = local_latents[:, :, :motion_frames_latents_length] + drop_motion_frames = False + else: + local_pixel_values = torch.cat([ref_pixel_values, pixel_values], dim = 1) + latents = _batch_encode_vae(local_pixel_values) + latents = latents[:, :, 1:] + motion_latents = _batch_encode_vae(motion_pixel_values) + drop_motion_frames = True + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + audio_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + with torch.no_grad(): + # Extract audio emb + new_audio_wav2vec_fea = [] + for index in range(len(audio)): + _audio_wav2vec_fea = audio_encoder.extract_audio_feat_without_file_load( + audio[index], sample_rate[index], return_all_layers=True + ) + new_audio_wav2vec_fea.append(_audio_wav2vec_fea) + audio_wav2vec_fea = torch.stack(new_audio_wav2vec_fea).to(device=accelerator.device, dtype=weight_dtype) + + new_audio_wav2vec_fea = [] + for index in range(len(audio)): + _audio_wav2vec_fea, num_repeat = audio_encoder.get_audio_embed_bucket_fps( + audio_wav2vec_fea[index], fps=fps[index], batch_frames=control_pixel_values.size()[1], m=0) + new_audio_wav2vec_fea.append(_audio_wav2vec_fea) + audio_wav2vec_fea = torch.stack(new_audio_wav2vec_fea).to(device=accelerator.device, dtype=weight_dtype) + + if len(audio_wav2vec_fea.shape) == 3: + audio_wav2vec_fea = audio_wav2vec_fea.permute(0, 2, 1) + elif len(audio_wav2vec_fea.shape) == 4: + audio_wav2vec_fea = audio_wav2vec_fea.permute(0, 2, 3, 1) + + for bs_index in range(audio_wav2vec_fea.size()[0]): + if rng is None: + zero_init_control_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_control_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if zero_init_control_latents_conv_in: + audio_wav2vec_fea[bs_index] = torch.ones_like(audio_wav2vec_fea[bs_index]) * 0 + + if zero_tail_frames: + audio_wav2vec_fea[..., zero_frames_num:] = torch.zeros_like(audio_wav2vec_fea[..., zero_frames_num:]) + # audio_wav2vec_fea = audio_wav2vec_fea[..., :control_pixel_values.size()[1]] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + audio_encoder.to("cpu") + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + cond_states=control_latents, + motion_latents=motion_latents, + ref_latents=ref_latents, + audio_input=audio_wav2vec_fea, + motion_frames=[[args.motion_frames, (args.motion_frames + 3) // 4]] * bsz, + drop_motion_frames=drop_motion_frames, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2/train_s2v_lora.sh b/VideoX-Fun/scripts/wan2.2/train_s2v_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..f52c617f53dfd590d04669286e233c3594f0e5b4 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2/train_s2v_lora.sh @@ -0,0 +1,43 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-S2V-14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata_control.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2/train_s2v_lora.py \ + --config_path="config/wan2.2/wan_civitai_s2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --control_ref_image="random" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN.md b/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..a938759acbfbd53b303b6b07e5f97ecca11a252d --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN.md @@ -0,0 +1,298 @@ +## Training Code + +We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or inpaint. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). + +If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/wan2.2_fun/xxx.py +``` + +Wan T2V without deepspeed: + +Training 14B Wan2.2 without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --low_vram \ + --train_mode="normal" \ + --trainable_modules "." +``` + +Wan T2V with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --low_vram \ + --use_deepspeed \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan T2V with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --low_vram \ + --use_deepspeed \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +Wan T2V with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." +``` + +If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. Training shell command is as follows: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-5B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --low_vram \ + --train_mode="inpaint" \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_CONTROL.md b/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_CONTROL.md new file mode 100644 index 0000000000000000000000000000000000000000..ad85200ba58edb44589f26a896578cdf23f0560d --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_CONTROL.md @@ -0,0 +1,335 @@ +## Training Code + +We can choose whether to use DeepSpeed and FSDP in Wan-Fun, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in Wan-Fun, you need to add a control_file_path, and [DWPose](https://github.com/IDEA-Research/DWPose) is suggested as tool to generate control file. + +```json +[ + { + "file_path": "train/00000001.mp4", + "control_file_path": "control/00000001.mp4", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "control_file_path": "control/00000002.jpg", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `train_mode` is used to set the training mode. + - The models named `Wan2.1-Fun-*-Control` are trained in the `control_ref` mode. + - The models named `Wan2.1-Fun-*-Control-Camera` are trained in the `control_ref_camera` mode. +- `control_ref_image` is used to specify the type of control image. The available options are `first_frame` and `random`. + - `first_frame` is used in V1.0 because V1.0 supports using a specified start frame as the control image. The Control-Camera models use the first frame as the control image. + - `random` is used in V1.1 because V1.1 supports both using a specified start frame and a reference image as the control image. +- `add_full_ref_image_in_self_attention` determines whether to include the reference image in self-attention. This option is used in V1.1, as it supports using a reference image as the control image. It should not be used in V1.0 and Control-Camera models. +- `add_inpaint_info` determines whether to incorporate inpaint information into the model training. When enabled, this allows the model to support specifying starting and ending images in the controls during generation. +- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). + +If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/wan2.2_fun/xxx.py +``` + +Wan-Fun-Control without deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train_control.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --low_vram \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --trainable_modules "." +``` + +Wan-Fun-Control with deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_control.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --low_vram \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --trainable_modules "." +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan-Fun-Control with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_control.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --low_vram \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --trainable_modules "." +``` + +Wan-Fun-Control with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train_control.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --low_vram \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --trainable_modules "." +``` + +If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. Training shell command is as follows: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-5B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train_control.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --low_vram \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_CONTROL_LORA.md b/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_CONTROL_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..2c28cf8e11f1bc2c4548454ac37c9a617578fdf7 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_CONTROL_LORA.md @@ -0,0 +1,347 @@ +## Training Code + +We can choose whether to use DeepSpeed and FSDP in Wan-Fun, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in Wan-Fun, you need to add a control_file_path, and [DWPose](https://github.com/IDEA-Research/DWPose) is suggested as tool to generate control file. + +```json +[ + { + "file_path": "train/00000001.mp4", + "control_file_path": "control/00000001.mp4", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "control_file_path": "control/00000002.jpg", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint and set the `save_state` to `True`. +- `train_mode` is used to set the training mode. + - The models named `Wan2.1-Fun-*-Control` are trained in the `control_ref` mode. + - The models named `Wan2.1-Fun-*-Control-Camera` are trained in the `control_ref_camera` mode. +- `control_ref_image` is used to specify the type of control image. The available options are `first_frame` and `random`. + - `first_frame` is used in V1.0 because V1.0 supports using a specified start frame as the control image. The Control-Camera models use the first frame as the control image. + - `random` is used in V1.1 because V1.1 supports both using a specified start frame and a reference image as the control image. +- `add_full_ref_image_in_self_attention` determines whether to include the reference image in self-attention. This option is used in V1.1, as it supports using a reference image as the control image. It should not be used in V1.0 and Control-Camera models. +- `add_inpaint_info` determines whether to incorporate inpaint information into the model training. When enabled, this allows the model to support specifying starting and ending images in the controls during generation. +- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/wan2.2_fun/xxx.py +``` + +Wan-Fun-Control without deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train_control_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +Wan-Fun-Control with deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_control_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is not compatible with PEFT. + +Wan-Fun-Control with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_control_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --save_state \ + --use_deepspeed \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --low_vram +``` + +Wan-Fun-Control with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train_control_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --save_state \ + --use_fsdp \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` + +If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. Training shell command is as follows: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-5B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train_control_lora.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --save_state \ + --use_fsdp \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_LORA.md b/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..018b768112bc0a34fddefdf3fa69eadd8bf71dd1 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_LORA.md @@ -0,0 +1,303 @@ +## Lora Training Code + +We can choose whether to use DeepSpeed and FSDP in Wan, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `train_mode` is used to specify the training mode, which can be either normal or inpaint. Since Wan uses the inpaint model to achieve image-to-video generation, the default is set to inpaint mode. If you only wish to achieve text-to-video generation, you can remove this line, and it will default to the text-to-video mode. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/wan2.2_fun/xxx.py +``` + +Wan T2V without deepspeed: + +Training 14B Wan2.2 without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="inpaint" \ + --low_vram +``` + +Wan T2V with Deepspeed Zero-2: + +Wan with Deepspeed Zero-2 is suitable for training 14B Wan at low resolutions, but training 14B Wan at high resolutions may still result in insufficient GPU memory. + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="inpaint" \ + --low_vram +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +It is known that DeepSpeed Zero-3 is not compatible with PEFT. + +Wan T2V with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. You must set save_state to True to save the model. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="inpaint" \ + --low_vram +``` + +Wan T2V with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="inpaint" \ + --low_vram +``` + +If you want to train 5B Wan2.2 model, please set config to `config/wan2.2/wan_civitai_5b.yaml` and set boundary_type to `full`. Training shell command is as follows: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-5B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=WanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_5b.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="full" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --train_mode="inpaint" \ + --low_vram +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_REWARD.md b/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_REWARD.md new file mode 100644 index 0000000000000000000000000000000000000000..8ea5cc39b41b88b1cf5460b111be0a776e5d5282 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/README_TRAIN_REWARD.md @@ -0,0 +1,41 @@ +# Wan2.2-Fun-Reward-LoRAs +## Introduction +We explore the Reward Backpropagation technique [1](#ref1) [2](#ref2) to optimized the generated videos by [Wan2.2-Fun](https://github.com/aigc-apps/VideoX-Fun) for better alignment with human preferences. +We provide the following pre-trained models (i.e. LoRAs) along with [the training script](https://github.com/aigc-apps/VideoX-Fun/blob/main/scripts/wan2.2_fun/train_reward_lora.py). You can use these LoRAs to enhance the corresponding base model as a plug-in or train your own reward LoRA. + +For more details, please refer to our [GitHub repo](https://github.com/aigc-apps/VideoX-Fun). + +| Name | Base Model | Reward Model | Hugging Face | Description | +|--|--|--|--|--| +| Wan2.2-Fun-A14B-InP-high-noise-HPS2.1.safetensors | [Wan2.2-Fun-A14B-InP (high noise)](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-InP/tree/main/high_noise_model) | [HPS v2.1](https://github.com/tgxs002/HPSv2) | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-Reward-LoRAs/resolve/main/Wan2.2-Fun-A14B-InP-high-noise-HPS2.1.safetensors) | Official HPS v2.1 reward LoRA (`rank=128` and `network_alpha=64`) for Wan2.2-Fun-A14B-InP (high noise). It is trained with a batch size of 8 for 5,000 steps.| +| Wan2.2-Fun-A14B-InP-low-noise-HPS2.1.safetensors | [Wan2.2-Fun-A14B-InP (low noise)](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-InP/tree/main/low_noise_model) | [MPS](https://github.com/Kwai-Kolors/MPS) | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-Reward-LoRAs/resolve/main/Wan2.2-Fun-A14B-InP-low-noise-HPS2.1.safetensors) | Official HPS v2.1 reward LoRA (`rank=128` and `network_alpha=64`) for Wan2.2-Fun-A14B-InP (low noise). It is trained with a batch size of 8 for 2,700 steps.| +| Wan2.2-Fun-A14B-InP-high-noise-MPS.safetensors | [Wan2.2-Fun-A14B-InP (high noise)](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-InP/tree/main/high_noise_model) | [HPS v2.1](https://github.com/tgxs002/HPSv2) | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.2-Fun-Reward-LoRAs/resolve/main/Wan2.2-Fun-A14B-InP-high-noise-MPS.safetensors) | Official MPS reward LoRA (`rank=128` and `network_alpha=64`) for Wan2.2-Fun-A14B-InP (high noise). It is trained with a batch size of 8 for 5,000 steps.| +| Wan2.2-Fun-A14B-InP-low-noise-MPS.safetensors | [Wan2.2-Fun-A14B-InP (low noise)](https://huggingface.co/alibaba-pai/Wan2.2-Fun-A14B-InP/tree/main/low_noise_model) | [MPS](https://github.com/Kwai-Kolors/MPS) | [🤗Link](https://huggingface.co/alibaba-pai/Wan2.1-Fun-Reward-LoRAs/resolve/main/Wan2.1-Fun-14B-InP-MPS.safetensors) | Official MPS reward LoRA (`rank=128` and `network_alpha=64`) for Wan2.2-Fun-A14B-InP (low noise). It is trained with a batch size of 8 for xxx steps.| + +> [!NOTE] +> We found that, MPS reward LoRA for the low-noise model converges significantly more slowly than on the other models, and may not deliver satisfactory results. Therefore, for the low-noise model, we recommend using HPSv2.1 reward LoRA. + +## Demo +Please refer to [here](https://huggingface.co/alibaba-pai/Wan2.2-Fun-Reward-LoRAs#demo). + +## Quick Start +Set `lora_path` along with `lora_weight` for the low noise reward LoRA, while specifying `lora_high_path` and `lora_high_weight` for high noise reward LoRA in [examples/wan2.2_fun/predict_t2v.py](https://github.com/aigc-apps/VideoX-Fun/blob/main/examples/wan2.1_fun/predict_t2v.py). + +## Training +The training code is based on [train_lora.py](./train_lora.py). We provide a shell script to train the HPS v2.1 reward LoRA for the low noise model of Wan2.2-Fun-A14B-InP, which can be trained on a single 8*A100 node with 80GB VRAM. To train reward LoRA for the high noise model, Deepspeed Zero3 with CPU offload is required. + +Please refer to [Setup](https://github.com/aigc-apps/VideoX-Fun/blob/main/scripts/cogvideox_fun/README_TRAIN_REWARD.md#setup) and [Important Args](https://github.com/aigc-apps/VideoX-Fun/blob/main/scripts/cogvideox_fun/README_TRAIN_REWARD.md#important-args) before training. + + +## Limitations +1. We observe after training to a certain extent, the reward continues to increase, but the quality of the generated videos does not further improve. + The model trickly learns some shortcuts (by adding artifacts in the background, i.e., adversarial patches) to increase the reward. +2. Currently, there is still a lack of suitable preference models for video generation. Directly using image preference models cannot + evaluate preferences along the temporal dimension (such as dynamism and consistency). Further more, We find using image preference models leads to a decrease + in the dynamism of generated videos. Although this can be mitigated by computing the reward using only the first frame of the decoded video, the impact still persists. + +## Reference +
    +
  1. Clark, Kevin, et al. "Directly fine-tuning diffusion models on differentiable rewards.". In ICLR 2024.
    2. +
  2. Prabhudesai, Mihir, et al. "Aligning text-to-image diffusion models with reward backpropagation." arXiv preprint arXiv:2310.03739 (2023).
    3. +
diff --git a/VideoX-Fun/scripts/wan2.2_fun/train.py b/VideoX-Fun/scripts/wan2.2_fun/train.py new file mode 100644 index 0000000000000000000000000000000000000000..42d311da17a3eb36f5240d90855a89ee98c2a91d --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/train.py @@ -0,0 +1,1996 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.distributed.fsdp.fully_sharded_data_parallel import ( + FullOptimStateDictConfig, FullStateDictConfig, ShardedOptimStateDictConfig, + ShardedStateDictConfig) +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, CLIPModel, WanT5EncoderModel, + Wan2_2Transformer3DModel) +from videox_fun.pipeline import Wan2_2Pipeline, Wan2_2I2VPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = Wan2_2I2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + else: + pipeline = Wan2_2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"i2v"`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=Wan2_2Transformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = Wan2_2Transformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = Wan2_2Transformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=Wan2_2Transformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = Wan2_2Transformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size]) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + local_video_sample_size = np.random.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + if args.train_mode != "normal": + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "normal": + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask_conditions = F.interpolate(mask[:, :1], size=latents.size()[-3:], mode='trilinear', align_corners=True).to(accelerator.device, weight_dtype) + mask = resize_mask(1 - mask, latents) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + if spatial_compression_ratio >= 16: + mask_conditions_bs = mask_conditions.size()[0] + mask_conditions[:, :, 1:, :, :] = 1 + if not mask_conditions[:, :, 0, :, :].any(): + noisy_latents = (1 - mask_conditions) * inpaint_latents[:, -vae.latent_channels:] + mask_conditions * noisy_latents + + temp_ts = (mask_conditions[:, 0, :, ::2, ::2] * timesteps[:, None, None, None]).flatten(1) + timesteps = torch.cat([temp_ts, temp_ts.new_ones(mask_conditions_bs, seq_len - temp_ts.size(1)) * timesteps[:, None,]], dim = 1) + else: + timesteps = mask_conditions.new_ones(mask_conditions_bs, seq_len) * timesteps[:, None,] + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2_fun/train.sh b/VideoX-Fun/scripts/wan2.2_fun/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..dc1efb141515c4f6dc2bdfd41813f30ea0c40495 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/train.sh @@ -0,0 +1,43 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --boundary_type="low" \ + --train_mode="inpaint" \ + --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2_fun/train_control.py b/VideoX-Fun/scripts/wan2.2_fun/train_control.py new file mode 100644 index 0000000000000000000000000000000000000000..6985e729004bb03f52c091e8ccc9d85da458ffde --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/train_control.py @@ -0,0 +1,2143 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask, + process_pose_file, + process_pose_params) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, CLIPModel, WanT5EncoderModel, + Wan2_2Transformer3DModel) +from videox_fun.pipeline import Wan2_2FunControlPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + pipeline = Wan2_2FunControlPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int(args.video_sample_n_frames // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(args.validation_paths[i], video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + control_video = input_video, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_paths", + type=str, + default=None, + nargs="+", + help=("A set of control videos evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="control", + help=( + 'The format of training data. Support `"control"`' + ' (default), `"control_ref"`, `"control_camera_ref"`.' + ), + ) + parser.add_argument( + "--control_ref_image", + type=str, + default="first_frame", + help=( + 'The format of training data. Support `"first_frame"`' + ' (default), `"random"`.' + ), + ) + parser.add_argument( + "--add_full_ref_image_in_self_attention", + action="store_true", + help=( + 'Whether enable add full ref image in self attention.' + ), + ) + parser.add_argument( + "--add_inpaint_info", + action="store_true", + help=( + 'Whether enable add inpaint info in self attention.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=Wan2_2Transformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = Wan2_2Transformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = Wan2_2Transformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=Wan2_2Transformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = Wan2_2Transformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoControlDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + enable_camera_info=args.train_mode == "control_camera_ref" + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Control Mode + new_examples["control_pixel_values"] = [] + # Used in Control Ref Mode + if args.train_mode != "control": + new_examples["ref_pixel_values"] = [] + new_examples["clip_pixel_values"] = [] + new_examples["clip_idx"] = [] + # Used in Control Camera Ref Mode + if args.train_mode == "control_camera_ref": + new_examples["control_camera_values"] = [] + + # Used in Inpaint mode + if args.add_inpaint_info: + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size]) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + + def get_random_downsample_probability(choice_list, token_sample_size): + length = len(choice_list) + if length == 1: + return [1.0] # If there's only one element, it gets all the probability + + # Find the index of the closest value to token_sample_size + closest_index = min(range(length), key=lambda i: abs(choice_list[i] - token_sample_size)) + + # Assign 50% to the closest index + first_element = 0.50 + remaining_sum = 1.0 - first_element + + # Distribute the remaining 50% evenly among the other elements + other_elements_value = remaining_sum / (length - 1) if length > 1 else 0.0 + + # Construct the probability distribution + probability_list = [other_elements_value] * length + probability_list[closest_index] = first_element + + return probability_list + + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + probabilities = get_random_downsample_probability(choice_list, args.token_sample_size) + local_video_sample_size = np.random.choice(choice_list, p=probabilities) + + random_downsample_ratio = args.video_sample_size / local_video_sample_size + batch_video_length = length_to_frame_num[local_video_sample_size] + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + + if args.train_mode == "control_camera_ref": + control_camera_values = example.get("control_camera_values", None) + if control_camera_values is None: + control_camera_values_size = ( + new_examples["control_pixel_values"][-1].size()[0], + 6, + new_examples["control_pixel_values"][-1].size()[2], + new_examples["control_pixel_values"][-1].size()[3] + ) + local_control_camera_values = torch.zeros(control_camera_values_size) + new_examples["control_camera_values"].append(local_control_camera_values) + else: + local_control_camera_values = process_pose_params(example["control_camera_values"], height=resize_size[0], width=resize_size[1]).permute(0, 3, 1, 2).contiguous() + new_examples["control_camera_values"].append(transform_no_normalize(local_control_camera_values)) + + new_examples["text"].append(example["text"]) + + if args.train_mode != "control": + if args.control_ref_image == "first_frame": + clip_index = 0 + else: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.40 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + number_list_prob = np.array(_create_special_list(len(new_examples["pixel_values"][-1]))) + clip_index = np.random.choice(list(range(len(new_examples["pixel_values"][-1]))), p = number_list_prob) + new_examples["clip_idx"].append(clip_index) + + ref_pixel_values = new_examples["pixel_values"][-1][clip_index].unsqueeze(0) + new_examples["ref_pixel_values"].append(ref_pixel_values) + + clip_pixel_values = new_examples["pixel_values"][-1][clip_index].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + if args.add_inpaint_info: + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + if args.train_mode != "control": + new_examples["ref_pixel_values"] = torch.stack([example for example in new_examples["ref_pixel_values"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["clip_idx"] = torch.tensor(new_examples["clip_idx"]) + if args.train_mode == "control_camera_ref": + new_examples["control_camera_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_camera_values"]]) + if args.add_inpaint_info: + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + control_pixel_values = batch["control_pixel_values"].cpu() + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, control_pixel_value, text) in enumerate(zip(pixel_values, control_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + control_pixel_value = control_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + save_videos_grid(control_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_control.gif", rescale=True) + + if args.train_mode != "control": + ref_pixel_values = batch["ref_pixel_values"].cpu() + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + for idx, (ref_pixel_value, text) in enumerate(zip(ref_pixel_values, texts)): + ref_pixel_value = ref_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(ref_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_ref.gif", rescale=True) + + if args.add_inpaint_info: + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + if args.train_mode == "control_camera_ref": + control_camera_values = batch["control_camera_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + if args.train_mode == "control_camera_ref": + control_camera_values = torch.tile(control_camera_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + if args.train_mode == "control_camera_ref": + control_camera_values = torch.tile(control_camera_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "control": + ref_pixel_values = batch["ref_pixel_values"].to(weight_dtype) + clip_idx = batch["clip_idx"] + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + ref_pixel_values = torch.tile(ref_pixel_values, (4, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (4,)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + ref_pixel_values = torch.tile(ref_pixel_values, (2, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (2,)) + + if args.add_inpaint_info: + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and not zero_stage == 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "control_camera_ref": + control_latents = _batch_encode_vae(control_pixel_values) + # Make control latents to zero + for bs_index in range(control_latents.size()[0]): + if rng is None: + zero_init_control_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_control_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if zero_init_control_latents_conv_in: + control_latents[bs_index] = control_latents[bs_index] * 0 + control_camera_latents = None + else: + control_latents = None + control_camera_latents = rearrange(control_camera_values, "b f c h w -> b c f h w") + control_camera_latents = torch.concat( + [ + torch.repeat_interleave(control_camera_latents[:, :, 0:1], repeats=4, dim=2), + control_camera_latents[:, :, 1:] + ], dim=2 + ).transpose(1, 2).contiguous() + control_camera_latents = control_camera_latents.view(control_camera_latents.shape[0], control_camera_latents.shape[1] // 4, 4, control_camera_latents.shape[2], control_camera_latents.shape[3], control_camera_latents.shape[4]) + control_camera_latents = control_camera_latents.transpose(2, 3).contiguous() + control_camera_latents = control_camera_latents.view(control_camera_latents.shape[0], control_camera_latents.shape[1], control_camera_latents.shape[2] * 4, control_camera_latents.shape[4], control_camera_latents.shape[5]) + control_camera_latents = control_camera_latents.transpose(1, 2) + + if args.train_mode != "control": + ref_latents = _batch_encode_vae(ref_pixel_values) + if args.add_full_ref_image_in_self_attention: + full_ref = ref_latents[:, :, 0].clone() + + ref_latents_conv_in = torch.zeros_like(latents).to(ref_latents.device, ref_latents.dtype) + ref_latents_conv_in[:, :, :1] = ref_latents + for bs_index in range(ref_latents.size()[0]): + if rng is None: + zero_init_ref_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_ref_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if clip_idx[bs_index] != 0 or (zero_init_ref_latents_conv_in and latents.size()[1] != 1): + ref_latents_conv_in[bs_index, :, :1] = ref_latents_conv_in[bs_index, :, :1] * 0 + + if args.add_full_ref_image_in_self_attention: + if rng is None: + zero_init_full_ref_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_full_ref_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + if clip_idx[bs_index] == 0 or zero_init_full_ref_conv_in: + full_ref[bs_index] = full_ref[bs_index] * 0 + + if args.add_inpaint_info: + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask_conditions = F.interpolate(mask[:, :1], size=latents.size()[-3:], mode='trilinear', align_corners=True).to(accelerator.device, weight_dtype) + mask = resize_mask(1 - mask, latents) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + else: + inpaint_latents = None + + if control_latents is None: + if inpaint_latents is None: + control_latents = ref_latents_conv_in + else: + control_latents = inpaint_latents + else: + if inpaint_latents is None: + control_latents = torch.cat([control_latents, ref_latents_conv_in], dim = 1) + else: + control_latents = torch.cat([control_latents, inpaint_latents], dim = 1) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + if spatial_compression_ratio >= 16: + mask_conditions_bs = mask_conditions.size()[0] + mask_conditions[:, :, 1:, :, :] = 1 + if not mask_conditions[:, :, 0, :, :].any(): + noisy_latents = (1 - mask_conditions) * control_latents[:, -vae.latent_channels:] + mask_conditions * noisy_latents + + temp_ts = (mask_conditions[:, 0, :, ::2, ::2] * timesteps[:, None, None, None]).flatten(1) + timesteps = torch.cat([temp_ts, temp_ts.new_ones(mask_conditions_bs, seq_len - temp_ts.size(1)) * timesteps[:, None,]], dim = 1) + else: + timesteps = mask_conditions.new_ones(mask_conditions_bs, seq_len) * timesteps[:, None,] + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=control_latents, + y_camera=control_camera_latents if args.train_mode == "control_camera_ref" else None, + full_ref=full_ref if args.add_full_ref_image_in_self_attention else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2_fun/train_control.sh b/VideoX-Fun/scripts/wan2.2_fun/train_control.sh new file mode 100644 index 0000000000000000000000000000000000000000..0ae5eab0bdd965e5df2b8d43ab421a9508fde2d7 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/train_control.sh @@ -0,0 +1,46 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train_control.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --boundary_type="low" \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --low_vram \ + --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2_fun/train_control_lora.py b/VideoX-Fun/scripts/wan2.2_fun/train_control_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..1f456304e1b18e36929ef716fbec8494eb37844a --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/train_control_lora.py @@ -0,0 +1,2172 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask, + process_pose_file, + process_pose_params) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, CLIPModel, WanT5EncoderModel, + Wan2_2Transformer3DModel) +from videox_fun.pipeline import Wan2_2FunControlPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, config, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + pipeline = Wan2_2FunControlPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int(args.video_sample_n_frames // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(args.validation_paths[i], video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + control_video = input_video, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_paths", + type=str, + default=None, + nargs="+", + help=("A set of control videos evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="control", + help=( + 'The format of training data. Support `"control"`' + ' (default), `"control_ref"`, `"control_camera_ref"`.' + ), + ) + parser.add_argument( + "--control_ref_image", + type=str, + default="first_frame", + help=( + 'The format of training data. Support `"first_frame"`' + ' (default), `"random"`.' + ), + ) + parser.add_argument( + "--add_full_ref_image_in_self_attention", + action="store_true", + help=( + 'Whether enable add full ref image in self attention.' + ), + ) + parser.add_argument( + "--add_inpaint_info", + action="store_true", + help=( + 'Whether enable add inpaint info in self attention.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoControlDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + enable_camera_info=args.train_mode == "control_camera_ref" + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Control Mode + new_examples["control_pixel_values"] = [] + # Used in Control Ref Mode + if args.train_mode != "control": + new_examples["ref_pixel_values"] = [] + new_examples["clip_pixel_values"] = [] + new_examples["clip_idx"] = [] + # Used in Control Camera Ref Mode + if args.train_mode == "control_camera_ref": + new_examples["control_camera_values"] = [] + + # Used in Inpaint mode + if args.add_inpaint_info: + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size]) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + + def get_random_downsample_probability(choice_list, token_sample_size): + length = len(choice_list) + if length == 1: + return [1.0] # If there's only one element, it gets all the probability + + # Find the index of the closest value to token_sample_size + closest_index = min(range(length), key=lambda i: abs(choice_list[i] - token_sample_size)) + + # Assign 50% to the closest index + first_element = 0.50 + remaining_sum = 1.0 - first_element + + # Distribute the remaining 50% evenly among the other elements + other_elements_value = remaining_sum / (length - 1) if length > 1 else 0.0 + + # Construct the probability distribution + probability_list = [other_elements_value] * length + probability_list[closest_index] = first_element + + return probability_list + + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + probabilities = get_random_downsample_probability(choice_list, args.token_sample_size) + local_video_sample_size = np.random.choice(choice_list, p=probabilities) + + random_downsample_ratio = args.video_sample_size / local_video_sample_size + batch_video_length = length_to_frame_num[local_video_sample_size] + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + + if args.train_mode == "control_camera_ref": + control_camera_values = example.get("control_camera_values", None) + if control_camera_values is None: + control_camera_values_size = ( + new_examples["control_pixel_values"][-1].size()[0], + 6, + new_examples["control_pixel_values"][-1].size()[2], + new_examples["control_pixel_values"][-1].size()[3] + ) + local_control_camera_values = torch.zeros(control_camera_values_size) + new_examples["control_camera_values"].append(local_control_camera_values) + else: + local_control_camera_values = process_pose_params(example["control_camera_values"], height=resize_size[0], width=resize_size[1]).permute(0, 3, 1, 2).contiguous() + new_examples["control_camera_values"].append(transform_no_normalize(local_control_camera_values)) + + new_examples["text"].append(example["text"]) + + if args.train_mode != "control": + if args.control_ref_image == "first_frame": + clip_index = 0 + else: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.40 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + number_list_prob = np.array(_create_special_list(len(new_examples["pixel_values"][-1]))) + clip_index = np.random.choice(list(range(len(new_examples["pixel_values"][-1]))), p = number_list_prob) + new_examples["clip_idx"].append(clip_index) + + ref_pixel_values = new_examples["pixel_values"][-1][clip_index].unsqueeze(0) + new_examples["ref_pixel_values"].append(ref_pixel_values) + + clip_pixel_values = new_examples["pixel_values"][-1][clip_index].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + if args.add_inpaint_info: + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + if args.train_mode != "control": + new_examples["ref_pixel_values"] = torch.stack([example for example in new_examples["ref_pixel_values"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["clip_idx"] = torch.tensor(new_examples["clip_idx"]) + if args.train_mode == "control_camera_ref": + new_examples["control_camera_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_camera_values"]]) + if args.add_inpaint_info: + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + else: + vae_stream_1 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + control_pixel_values = batch["control_pixel_values"].cpu() + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, control_pixel_value, text) in enumerate(zip(pixel_values, control_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + control_pixel_value = control_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + save_videos_grid(control_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_control.gif", rescale=True) + + if args.train_mode != "control": + ref_pixel_values = batch["ref_pixel_values"].cpu() + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + for idx, (ref_pixel_value, text) in enumerate(zip(ref_pixel_values, texts)): + ref_pixel_value = ref_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(ref_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_ref.gif", rescale=True) + + if args.add_inpaint_info: + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + if args.train_mode == "control_camera_ref": + control_camera_values = batch["control_camera_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and not zero_stage == 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + if args.train_mode == "control_camera_ref": + control_camera_values = torch.tile(control_camera_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + if args.train_mode == "control_camera_ref": + control_camera_values = torch.tile(control_camera_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "control": + ref_pixel_values = batch["ref_pixel_values"].to(weight_dtype) + clip_idx = batch["clip_idx"] + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and not zero_stage == 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + ref_pixel_values = torch.tile(ref_pixel_values, (4, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (4,)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + ref_pixel_values = torch.tile(ref_pixel_values, (2, 1, 1, 1, 1)) + clip_idx = torch.tile(clip_idx, (2,)) + + if args.add_inpaint_info: + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and not zero_stage == 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "control_camera_ref": + control_latents = _batch_encode_vae(control_pixel_values) + # Make control latents to zero + for bs_index in range(control_latents.size()[0]): + if rng is None: + zero_init_control_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_control_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if zero_init_control_latents_conv_in: + control_latents[bs_index] = control_latents[bs_index] * 0 + control_camera_latents = None + else: + control_latents = None + control_camera_latents = rearrange(control_camera_values, "b f c h w -> b c f h w") + control_camera_latents = torch.concat( + [ + torch.repeat_interleave(control_camera_latents[:, :, 0:1], repeats=4, dim=2), + control_camera_latents[:, :, 1:] + ], dim=2 + ).transpose(1, 2).contiguous() + control_camera_latents = control_camera_latents.view(control_camera_latents.shape[0], control_camera_latents.shape[1] // 4, 4, control_camera_latents.shape[2], control_camera_latents.shape[3], control_camera_latents.shape[4]) + control_camera_latents = control_camera_latents.transpose(2, 3).contiguous() + control_camera_latents = control_camera_latents.view(control_camera_latents.shape[0], control_camera_latents.shape[1], control_camera_latents.shape[2] * 4, control_camera_latents.shape[4], control_camera_latents.shape[5]) + control_camera_latents = control_camera_latents.transpose(1, 2) + + if args.train_mode != "control": + ref_latents = _batch_encode_vae(ref_pixel_values) + if args.add_full_ref_image_in_self_attention: + full_ref = ref_latents[:, :, 0].clone() + + ref_latents_conv_in = torch.zeros_like(latents).to(ref_latents.device, ref_latents.dtype) + ref_latents_conv_in[:, :, :1] = ref_latents + for bs_index in range(ref_latents.size()[0]): + if rng is None: + zero_init_ref_latents_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_ref_latents_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + + if clip_idx[bs_index] != 0 or (zero_init_ref_latents_conv_in and latents.size()[1] != 1): + ref_latents_conv_in[bs_index, :, :1] = ref_latents_conv_in[bs_index, :, :1] * 0 + + if args.add_full_ref_image_in_self_attention: + if rng is None: + zero_init_full_ref_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_full_ref_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + if clip_idx[bs_index] == 0 or zero_init_full_ref_conv_in: + full_ref[bs_index] = full_ref[bs_index] * 0 + + if args.add_inpaint_info: + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask_conditions = F.interpolate(mask[:, :1], size=latents.size()[-3:], mode='trilinear', align_corners=True).to(accelerator.device, weight_dtype) + mask = resize_mask(1 - mask, latents) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + else: + inpaint_latents = None + + if control_latents is None: + if inpaint_latents is None: + control_latents = ref_latents_conv_in + else: + control_latents = inpaint_latents + else: + if inpaint_latents is None: + control_latents = torch.cat([control_latents, ref_latents_conv_in], dim = 1) + else: + control_latents = torch.cat([control_latents, inpaint_latents], dim = 1) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + if spatial_compression_ratio >= 16: + mask_conditions_bs = mask_conditions.size()[0] + mask_conditions[:, :, 1:, :, :] = 1 + if not mask_conditions[:, :, 0, :, :].any(): + noisy_latents = (1 - mask_conditions) * control_latents[:, -vae.latent_channels:] + mask_conditions * noisy_latents + + temp_ts = (mask_conditions[:, 0, :, ::2, ::2] * timesteps[:, None, None, None]).flatten(1) + timesteps = torch.cat([temp_ts, temp_ts.new_ones(mask_conditions_bs, seq_len - temp_ts.size(1)) * timesteps[:, None,]], dim = 1) + else: + timesteps = mask_conditions.new_ones(mask_conditions_bs, seq_len) * timesteps[:, None,] + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=control_latents, + y_camera=control_camera_latents if args.train_mode == "control_camera_ref" else None, + full_ref=full_ref if args.add_full_ref_image_in_self_attention else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2_fun/train_control_lora.sh b/VideoX-Fun/scripts/wan2.2_fun/train_control_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..ebafdefd4ffae85014a5599e35549a889c2e088b --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/train_control_lora.sh @@ -0,0 +1,47 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-Control" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train_control_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="control_ref" \ + --control_ref_image="random" \ + --add_inpaint_info \ + --add_full_ref_image_in_self_attention \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2_fun/train_lora.py b/VideoX-Fun/scripts/wan2.2_fun/train_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..9eb6823d766c5f60e57d9cd991ad1967ab829850 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/train_lora.py @@ -0,0 +1,2011 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, WanT5EncoderModel, + Wan2_2Transformer3DModel) +from videox_fun.pipeline import Wan2_2Pipeline, Wan2_2I2VPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, config, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = Wan2_2I2VPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + else: + pipeline = Wan2_2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"normal"`' + ' (default), `"inpaint"`.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import LoraConfig, inject_adapter_in_model, get_peft_model_state_dict + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, enable_inpaint=True if args.train_mode != "normal" else False, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Inpaint mode + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + new_examples["clip_pixel_values"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size]) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + # The video will be resized to a lower resolution than its own. + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + local_video_sample_size = np.random.choice(choice_list) + batch_video_length = length_to_frame_num[local_video_sample_size] + random_downsample_ratio = args.video_sample_size / local_video_sample_size + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in closest_size] + + min_example_length = min( + [example["pixel_values"].shape[0] for example in examples] + ) + batch_video_length = int(min(batch_video_length, min_example_length)) + + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = (batch_video_length - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + + if batch_video_length <= 0: + batch_video_length = 1 + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)[:batch_video_length]) + new_examples["text"].append(example["text"]) + + if args.train_mode != "normal": + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + clip_pixel_values = new_examples["pixel_values"][-1][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + if args.train_mode != "normal": + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream and args.train_mode != "normal": + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + if args.train_mode != "normal": + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + if args.train_mode != "normal": + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + if args.train_mode != "normal": + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + # Make the inpaint latents to be zeros. + if args.train_mode != "normal": + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if args.train_mode != "normal": + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.concat( + [ + torch.repeat_interleave(mask[:, :, 0:1], repeats=4, dim=2), + mask[:, :, 1:] + ], dim=2 + ) + mask = mask.view(mask.shape[0], mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]) + mask = mask.transpose(1, 2) + mask_conditions = F.interpolate(mask[:, :1], size=latents.size()[-3:], mode='trilinear', align_corners=True).to(accelerator.device, weight_dtype) + mask = resize_mask(1 - mask, latents) + + # Encode inpaint latents. + mask_latents = _batch_encode_vae(mask_pixel_values) + if vae_stream_2 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_2) + + inpaint_latents = torch.concat([mask, mask_latents], dim=1) + inpaint_latents = t2v_flag[:, None, None, None, None] * inpaint_latents + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + if spatial_compression_ratio >= 16: + mask_conditions_bs = mask_conditions.size()[0] + mask_conditions[:, :, 1:, :, :] = 1 + if not mask_conditions[:, :, 0, :, :].any(): + noisy_latents = (1 - mask_conditions) * inpaint_latents[:, -vae.latent_channels:] + mask_conditions * noisy_latents + + temp_ts = (mask_conditions[:, 0, :, ::2, ::2] * timesteps[:, None, None, None]).flatten(1) + timesteps = torch.cat([temp_ts, temp_ts.new_ones(mask_conditions_bs, seq_len - temp_ts.size(1)) * timesteps[:, None,]], dim = 1) + else: + timesteps = mask_conditions.new_ones(mask_conditions_bs, seq_len) * timesteps[:, None,] + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + y=inpaint_latents if args.train_mode != "normal" else None, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2_fun/train_lora.sh b/VideoX-Fun/scripts/wan2.2_fun/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..3ca2d05b3bc2bcc719d8939eea4d239334d157c4 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/train_lora.sh @@ -0,0 +1,44 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_fun/train_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --train_mode="inpaint" \ + --boundary_type="low" \ + --rank=64 \ + --network_alpha=32 \ + --target_name="q,k,v,ffn.0,ffn.2" \ + --use_peft_lora \ + --low_vram diff --git a/VideoX-Fun/scripts/wan2.2_fun/train_reward_lora.py b/VideoX-Fun/scripts/wan2.2_fun/train_reward_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..855a8092fc8e975fd5737c753ba5ca022de95239 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/train_reward_lora.py @@ -0,0 +1,1721 @@ +"""Modified from VideoX-Fun/scripts/wan2.2_fun/train_lora.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import json +import logging +import math +import os +import pickle +import random +import shutil +import sys +from contextlib import contextmanager +from typing import List, Optional, Union + +import accelerate +import diffusers +import torch +import torch.utils.checkpoint +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from decord import VideoReader +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from torch.utils.tensorboard import SummaryWriter +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +import videox_fun.reward.reward_fn as reward_fn +from videox_fun.models import (AutoencoderKLWan, AutoencoderKLWan3_8, WanT5EncoderModel, + Wan2_2Transformer3DModel) +from videox_fun.pipeline import WanFunInpaintPipeline, WanFunPipeline +from videox_fun.utils.lora_utils import create_network, merge_lora +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +@contextmanager +def video_reader(*args, **kwargs): + """A context manager to solve the memory leak of decord. + """ + vr = VideoReader(*args, **kwargs) + try: + yield vr + finally: + del vr + gc.collect() + + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, config, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + if args.train_mode != "normal": + pipeline = WanFunInpaintPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + else: + pipeline = WanFunPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + if args.train_mode != "normal": + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int((args.video_sample_n_frames - 1) // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + video_length = 1 + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + guidance_scale = 6.0, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + else: + with torch.autocast("cuda", dtype=weight_dtype): + sample = pipeline( + args.validation_prompts[i], + num_frames = args.video_sample_n_frames, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + sample = pipeline( + args.validation_prompts[i], + num_frames = 1, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + + +def load_prompts(prompt_path, prompt_column="prompt", start_idx=None, end_idx=None): + prompt_list = [] + if prompt_path.endswith(".txt"): + with open(prompt_path, "r") as f: + for line in f: + prompt_list.append(line.strip()) + elif prompt_path.endswith(".jsonl"): + with open(prompt_path, "r") as f: + for line in f.readlines(): + item = json.loads(line) + prompt_list.append(item[prompt_column]) + else: + raise ValueError("The prompt_path must end with .txt or .jsonl.") + prompt_list = prompt_list[start_idx:end_idx] + + return prompt_list + + +def _get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +): + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + +def encode_prompt( + tokenizer, + text_encoder, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, +): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + # device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = _get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = _get_t5_prompt_embeds( + tokenizer, + text_encoder, + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs +def prepare_extra_step_kwargs(scheduler, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + import inspect + + accepts_eta = "eta" in set(inspect.signature(scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--validation_prompt_path", + type=str, + default=None, + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_batch_size", + type=int, + default=1, + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_sample_height", + type=int, + default=512, + help="The height of sampling videos in validation.", + ) + parser.add_argument( + "--validation_sample_width", + type=int, + default=512, + help="The width of sampling videos in validation.", + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing (for DiT) to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + + parser.add_argument( + "--prompt_path", + type=str, + default="normal", + help="The path to the training prompt file.", + ) + parser.add_argument( + '--train_sample_height', + type=int, + default=384, + help='The height of sampling videos in training' + ) + parser.add_argument( + '--train_sample_width', + type=int, + default=672, + help='The width of sampling videos in training' + ) + parser.add_argument( + "--video_length", + type=int, + default=49, + help="The number of frames to generate in training and validation." + ) + parser.add_argument( + '--eta', + type=float, + default=0.0, + help='eta parameter for the DDIM sampler. this controls the amount of noise injected into the sampling process, ' + 'with 0.0 being fully deterministic and 1.0 being equivalent to the DDPM sampler.' + ) + parser.add_argument( + "--guidance_scale", + type=float, + default=6.0, + help="The classifier-free diffusion guidance." + ) + parser.add_argument( + "--num_inference_steps", + type=int, + default=50, + help="The number of denoising steps in training and validation." + ) + parser.add_argument( + "--num_decoded_latents", + type=int, + default=3, + help="The number of latents to be decoded." + ) + parser.add_argument( + "--num_sampled_frames", + type=int, + default=None, + help="The number of sampled frames for the reward function." + ) + parser.add_argument( + "--reward_fn", + type=str, + default="aesthetic_loss_fn", + help='The reward function.' + ) + parser.add_argument( + "--reward_fn_kwargs", + type=str, + default=None, + help='The keyword arguments of the reward function.' + ) + parser.add_argument( + "--backprop", + action="store_true", + default=False, + help="Whether to use the reward backprop training mode.", + ) + parser.add_argument( + "--backprop_step_list", + nargs="+", + type=int, + default=None, + help="The preset step list for reward backprop. If provided, overrides `backprop_strategy`." + ) + parser.add_argument( + "--backprop_strategy", + choices=["last", "tail", "uniform", "random"], + default="last", + help="The strategy for reward backprop." + ) + parser.add_argument( + "--stop_latent_model_input_gradient", + action="store_true", + default=False, + help="Whether to stop the gradient of the latents during reward backprop.", + ) + parser.add_argument( + "--backprop_random_start_step", + type=int, + default=0, + help="The random start step for reward backprop. Only used when `backprop_strategy` is random." + ) + parser.add_argument( + "--backprop_random_end_step", + type=int, + default=50, + help="The random end step for reward backprop. Only used when `backprop_strategy` is random." + ) + parser.add_argument( + "--backprop_num_steps", + type=int, + default=5, + help="The number of steps for backprop. Only used when `backprop_strategy` is tail/uniform/random." + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # Sanity check for validation + do_validation = (args.validation_prompt_path is not None or args.validation_prompts is not None) + assert do_validation == False, "The `log_validation` is not supported currently." + if do_validation: + if not (os.path.exists(args.validation_prompt_path) or args.validation_prompt_path.endswith(".txt")): + raise ValueError("The `--validation_prompt_path` must be a txt file containing prompts.") + if args.validation_batch_size < accelerator.num_processes or args.validation_batch_size % accelerator.num_processes != 0: + raise ValueError("The `--validation_batch_size` must be divisible by the number of processes.") + # Sanity check for backpropagation + if args.backprop: + if args.backprop_step_list is not None: + logger.warning( + f"The backprop_strategy {args.backprop_strategy} will be ignored " + f"when using backprop_step_list {args.backprop_step_list}." + ) + assert any(step <= args.num_inference_steps - 1 for step in args.backprop_step_list) + else: + if args.backprop_strategy in set(["tail", "uniform", "random"]): + assert args.backprop_num_steps <= args.num_inference_steps - 1 + if args.backprop_strategy == "random": + assert args.backprop_random_start_step <= args.backprop_random_end_step + assert args.backprop_random_end_step <= args.num_inference_steps - 1 + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed, device_specific=True) + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + # TODO: support other noise schedulers (FlowUniPCMultistepScheduler and FlowDPMSolverMultistepScheduler) + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Some reward models comes from `transformers`. We also need `unset_hf_deepspeed_config` to prevent the + # model from being partitioned during `zero.Init`. + from transformers.integrations.deepspeed import unset_hf_deepspeed_config + unset_hf_deepspeed_config() + + # loss function + reward_fn_kwargs = {} + if args.reward_fn_kwargs is not None: + reward_fn_kwargs = json.loads(args.reward_fn_kwargs) + if accelerator.is_main_process: + # Check if the model is downloaded in the main process. + loss_fn = getattr(reward_fn, args.reward_fn)(device="cpu", dtype=weight_dtype, **reward_fn_kwargs) + accelerator.wait_for_everyone() + loss_fn = getattr(reward_fn, args.reward_fn)(device=accelerator.device, dtype=weight_dtype, **reward_fn_kwargs) + + # Get Transformer + # To unify the LoRA training for the 5/14B (including the high and low noise model), + # the script contains numerous transformer3d variables. + # `transformer3d` refers to the (sub)model currently being trained. + # `low_transformer3d` is an instance of the 14B low noise model. + # `high_transformer3d` is an instance of the 14B high noise model. + # `local_transformer` refers to the (sub)model actually used during the denoising process. + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + if args.boundary_type != "full": + # 14B + sub_path_2 = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + low_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True + ).to(weight_dtype) + high_transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path_2), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True + ).to(weight_dtype) + else: + # 5B + transformer3d = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + low_cpu_mem_usage=True + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + if args.boundary_type != "full": + low_transformer3d.requires_grad_(False) + high_transformer3d.requires_grad_(False) + if args.boundary_type == "low": + transformer3d = low_transformer3d + else: + transformer3d = high_transformer3d + else: + transformer3d.requires_grad_(False) + + # Lora will work with this... + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + skip_name=args.lora_skip_name, + ) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + elif zero_stage == 3: + def save_model_hook(models, weights, output_dir): + pass + else: + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + accelerator.register_save_state_pre_hook(save_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + # When training the high noise reward lora, the low noise transformer is still in the computation graph. + if args.boundary_type == "high": + low_transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get RL training prompts + prompt_list = load_prompts(args.prompt_path) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(prompt_list) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + if fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(weight_dtype) + transformer3d, optimizer, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, lr_scheduler + ) + else: + network, optimizer, lr_scheduler = accelerator.prepare( + network, optimizer, lr_scheduler + ) + + if zero_stage == 3: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + + if args.boundary_type != "full": + low_transformer3d = shard_fn(low_transformer3d) + if args.boundary_type == "high": + high_transformer3d = shard_fn(high_transformer3d) + else: + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0: + from functools import partial + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device, dtype=weight_dtype) + if args.boundary_type != "full": + high_transformer3d.to(accelerator.device, dtype=weight_dtype) + low_transformer3d.to(accelerator.device, dtype=weight_dtype) + else: + transformer3d.to(accelerator.device, dtype=weight_dtype) + text_encoder.to(accelerator.device) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(prompt_list) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(prompt_list)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + checkpoint_folder_path = os.path.join(args.output_dir, path) + pkl_path = os.path.join(checkpoint_folder_path, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + if zero_stage != 3 and not args.use_fsdp: + from safetensors.torch import load_file + state_dict = load_file(os.path.join(checkpoint_folder_path, "lora_diffusion_pytorch_model.safetensors"), device=str(accelerator.device)) + m, u = accelerator.unwrap_model(network).load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + optimizer_file_pt = os.path.join(checkpoint_folder_path, "optimizer.pt") + optimizer_file_bin = os.path.join(checkpoint_folder_path, "optimizer.bin") + optimizer_file_to_load = None + + if os.path.exists(optimizer_file_pt): + optimizer_file_to_load = optimizer_file_pt + elif os.path.exists(optimizer_file_bin): + optimizer_file_to_load = optimizer_file_bin + + if optimizer_file_to_load: + try: + accelerator.print(f"Loading optimizer state from {optimizer_file_to_load}") + optimizer_state = torch.load(optimizer_file_to_load, map_location=accelerator.device) + optimizer.load_state_dict(optimizer_state) + accelerator.print("Optimizer state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load optimizer state from {optimizer_file_to_load}: {e}") + + scheduler_file_pt = os.path.join(checkpoint_folder_path, "scheduler.pt") + scheduler_file_bin = os.path.join(checkpoint_folder_path, "scheduler.bin") + scheduler_file_to_load = None + + if os.path.exists(scheduler_file_pt): + scheduler_file_to_load = scheduler_file_pt + elif os.path.exists(scheduler_file_bin): + scheduler_file_to_load = scheduler_file_bin + + if scheduler_file_to_load: + try: + accelerator.print(f"Loading scheduler state from {scheduler_file_to_load}") + scheduler_state = torch.load(scheduler_file_to_load, map_location=accelerator.device) + lr_scheduler.load_state_dict(scheduler_state) + accelerator.print("Scheduler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load scheduler state from {scheduler_file_to_load}: {e}") + + if hasattr(accelerator, 'scaler') and accelerator.scaler is not None: + scaler_file = os.path.join(checkpoint_folder_path, "scaler.pt") + if os.path.exists(scaler_file): + try: + accelerator.print(f"Loading GradScaler state from {scaler_file}") + scaler_state = torch.load(scaler_file, map_location=accelerator.device) + accelerator.scaler.load_state_dict(scaler_state) + accelerator.print("GradScaler state loaded successfully.") + except Exception as e: + accelerator.print(f"Failed to load GradScaler state: {e}") + + else: + accelerator.load_state(checkpoint_folder_path) + accelerator.print("accelerator.load_state() completed for zero_stage 3.") + + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + boundary = config["transformer_additional_kwargs"].get("boundary", 0.900) + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}.") + + from diffusers.image_processor import VaeImageProcessor + image_processor = VaeImageProcessor(vae_scale_factor=vae.config.spatial_compression_ratio) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + train_reward = 0.0 + + # In the following training loop, randomly select training prompts and use the + # `Wan2_2FunInpaintPipeline` to sample videos, calculate rewards, and update the network. + for _ in range(num_update_steps_per_epoch): + # train_prompt = random.sample(prompt_list, args.train_batch_size) + train_prompt = random.choices(prompt_list, k=args.train_batch_size) + logger.info(f"train_prompt: {train_prompt}") + + # default height and width + height = int(args.train_sample_height // 16 * 16) + width = int(args.train_sample_width // 16 * 16) + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = args.guidance_scale > 1.0 + + # Reduce the vram by offload text encoders + if args.low_vram: + torch.cuda.empty_cache() + text_encoder.to(accelerator.device) + + # Encode input prompt + prompt_embeds, negative_prompt_embeds = encode_prompt( + tokenizer, + text_encoder, + train_prompt, + negative_prompt=[""] * len(train_prompt), + device=accelerator.device, + dtype=weight_dtype, + do_classifier_free_guidance=do_classifier_free_guidance, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # Reduce the vram by offload text encoders + if args.low_vram: + text_encoder.to("cpu") + torch.cuda.empty_cache() + + # Prepare timesteps + # TODO: support other noise schedulers (FlowUniPCMultistepScheduler and FlowDPMSolverMultistepScheduler) + noise_scheduler.set_timesteps(args.num_inference_steps, device=accelerator.device, mu=1) + timesteps = noise_scheduler.timesteps + + # Prepare latent variables + vae_scale_factor = vae.config.spatial_compression_ratio + latent_shape = [ + args.train_batch_size, + vae.config.latent_channels, + int((args.video_length - 1) // vae.config.temporal_compression_ratio + 1) if args.video_length != 1 else 1, + args.train_sample_height // vae_scale_factor, + args.train_sample_width // vae_scale_factor, + ] + + with accelerator.accumulate(transformer3d): + sample_size = [args.train_sample_height, args.train_sample_width] + input_video, input_video_mask, _ = get_image_to_video_latent(None, None, video_length=args.video_length, sample_size=sample_size) + if input_video is not None: + video_length = input_video.shape[2] + init_video = image_processor.preprocess(rearrange(input_video, "b c f h w -> (b f) c h w"), height=height, width=width) + init_video = init_video.to(dtype=torch.float32) + init_video = rearrange(init_video, "(b f) c h w -> b c f h w", f=video_length) + else: + init_video = None + + latents = torch.randn(*latent_shape, device=accelerator.device, dtype=weight_dtype) + + # Prepare mask latent variables + mask_latents = torch.tile( + torch.zeros_like(latents)[:, :1].to(accelerator.device, weight_dtype), [1, 4, 1, 1, 1] + ) + masked_video_latents = torch.zeros_like(latents).to(accelerator.device, weight_dtype) + if vae.config.spatial_compression_ratio >= 16: + mask = torch.ones_like(latents).to(accelerator.device, weight_dtype)[:, :1].to(accelerator.device, weight_dtype) + + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + # Prepare extra step kwargs. + extra_step_kwargs = prepare_extra_step_kwargs(noise_scheduler, generator, args.eta) + + bsz, channel, num_frames, height, width = latents.size() + target_shape = (vae.latent_channels, num_frames, width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + num_inference_steps_for_backprop = args.num_inference_steps + if args.boundary_type == "high": + # When training the high noise reward lora, the denoising steps in the high noise are eligible for backpropagation. + num_inference_steps_for_backprop = 0 + for i, t in enumerate(tqdm(noise_scheduler.timesteps)): + if t >= boundary * noise_scheduler.config.num_train_timesteps: + num_inference_steps_for_backprop += 1 + + # Denoising loop + if args.backprop: + if args.backprop_step_list is None: + if args.backprop_strategy == "last": + backprop_step_list = [num_inference_steps_for_backprop - 1] + elif args.backprop_strategy == "tail": + backprop_step_list = list(range(num_inference_steps_for_backprop))[-args.backprop_num_steps:] + elif args.backprop_strategy == "uniform": + interval = num_inference_steps_for_backprop // args.backprop_num_steps + random_start = random.randint(0, interval) + backprop_step_list = [random_start + i * interval for i in range(args.backprop_num_steps)] + elif args.backprop_strategy == "random": + backprop_step_list = random.sample( + range(args.backprop_random_start_step, args.backprop_random_end_step + 1), args.backprop_num_steps + ) + else: + raise ValueError(f"Invalid backprop strategy: {args.backprop_strategy}.") + else: + backprop_step_list = args.backprop_step_list + + # Sanity check for backpropagation (of 14B models) + if args.boundary_type != "full": + high_step_list = [] + low_step_list = [] + for i, t in enumerate(tqdm(timesteps)): + if t >= boundary * noise_scheduler.config.num_train_timesteps: + high_step_list.append(i) + else: + low_step_list.append(i) + if args.boundary_type == "high": + assert all(step in high_step_list for step in backprop_step_list), \ + f"{backprop_step_list} is not in {high_step_list}" + else: + assert all(step in low_step_list for step in backprop_step_list), \ + f"{backprop_step_list} is not in {low_step_list}" + + for i, t in enumerate(tqdm(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(noise_scheduler, "scale_model_input"): + latent_model_input = noise_scheduler.scale_model_input(latent_model_input, t) + + if init_video is not None: + mask_input = torch.cat([mask_latents] * 2) if do_classifier_free_guidance else mask_latents + masked_video_latents_input = ( + torch.cat([masked_video_latents] * 2) if do_classifier_free_guidance else masked_video_latents + ) + y = torch.cat([mask_input, masked_video_latents_input], dim=1).to(accelerator.device, weight_dtype) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + if vae.config.spatial_compression_ratio >= 16 and init_video is not None: + temp_ts = ((mask[0][0][:, ::2, ::2]) * t).flatten() + temp_ts = torch.cat([ + temp_ts, + temp_ts.new_ones(seq_len - temp_ts.size(0)) * t + ]) + temp_ts = temp_ts.unsqueeze(0) + timestep = temp_ts.expand(latent_model_input.shape[0], temp_ts.size(1)) + else: + timestep = t.expand(latent_model_input.shape[0]) + + if args.boundary_type != "full": + if t >= boundary * noise_scheduler.config.num_train_timesteps: + if args.low_vram and args.boundary_type == "low": + torch.cuda.empty_cache() + high_transformer3d.to(accelerator.device) + local_transformer = high_transformer3d + else: + local_transformer = low_transformer3d + else: + local_transformer = transformer3d + + # Whether to enable DRTune: https://arxiv.org/abs/2405.00760 + if args.stop_latent_model_input_gradient: + latent_model_input = latent_model_input.detach() + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = local_transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + y=y + ) + + if t >= boundary * noise_scheduler.config.num_train_timesteps: + pass + else: + # When training the high noise reward lora, we can not directly offload the + # high noise transformer to CPU as it need to be kept in the computation graph. + # It's recommended to use DeepSpeed ZeRO3 CPU offload to save VRAM. + if args.low_vram and args.boundary_type == "low" and (not zero_stage == 3): + high_transformer3d.to("cpu") + torch.cuda.empty_cache() + + # Optimize the denoising results only for the specified steps. + if i in backprop_step_list: + noise_pred = noise_pred + else: + noise_pred = noise_pred.detach() + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred[0], noise_pred[1] + noise_pred = noise_pred_uncond + args.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = noise_scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if vae.config.spatial_compression_ratio >= 16 and not mask[:, :, 0, :, :].any(): + latents = (1 - mask) * masked_video_latents + mask * latents + + # decode latents (tensor) + # latents = latents.permute(0, 2, 1, 3, 4) # [B, C, T, H, W] + # Since the casual VAE decoding consumes a large amount of VRAM, and we need to keep the decoding + # operation within the computational graph. Thus, we only decode the first args.num_decoded_latents + # to calculate the reward. + # TODO: Decode all latents but keep a portion of the decoding operation within the computational graph. + sampled_latent_indices = list(range(args.num_decoded_latents)) + sampled_latents = latents[:, :, sampled_latent_indices, :, :] + sampled_frames = vae.decode(sampled_latents.to(vae.device, vae.dtype))[0] + sampled_frames = sampled_frames.clamp(-1, 1) + sampled_frames = (sampled_frames / 2 + 0.5).clamp(0, 1) # [-1, 1] -> [0, 1] + + if global_step % args.checkpointing_steps == 0: + saved_file = f"sample-{global_step}-{accelerator.process_index}.mp4" + save_videos_grid( + sampled_frames.to(torch.float32).detach().cpu(), + os.path.join(args.output_dir, "train_sample", saved_file), + fps=16 + ) + + if args.num_sampled_frames is not None: + num_frames = sampled_frames.size(2) - 1 + sampled_frames_indices = torch.linspace(0, num_frames, steps=args.num_sampled_frames).long() + sampled_frames = sampled_frames[:, :, sampled_frames_indices, :, :] + # compute loss and reward + loss, reward = loss_fn(sampled_frames, train_prompt) + + # Gather the losses and rewards across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + avg_reward = accelerator.gather(reward.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + train_reward += avg_reward.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + total_norm = accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + # If use_deepspeed, `total_norm` cannot be logged by accelerator. + if not args.use_deepspeed: + accelerator.log({"total_norm": total_norm}, step=global_step) + else: + if hasattr(optimizer, "optimizer") and hasattr(optimizer.optimizer, "_global_grad_norm"): + accelerator.log({"total_norm": optimizer.optimizer._global_grad_norm}, step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss, "train_reward": train_reward}, step=global_step) + train_loss = 0.0 + train_reward = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "step_reward": reward.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + config, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2_fun/train_reward_lora.sh b/VideoX-Fun/scripts/wan2.2_fun/train_reward_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..ad58d2cf2de3a5a3ce0773d5f68ec024646ec39c --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_fun/train_reward_lora.sh @@ -0,0 +1,65 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-Fun-A14B-InP" +export TRAIN_PROMPT_PATH="MovieGenVideoBench_train.txt" + +# Train HPSv2.1 reward LoRA for the low noise model of Wan2.2-Fun-A14B-InP +accelerate launch --mixed_precision="bf16" --num-processes=8 --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json scripts/wan2.2_fun/train_reward_lora.py \ + --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_batch_size=1 \ + --gradient_accumulation_steps=1 \ + --max_train_steps=10000 \ + --checkpointing_steps=100 \ + --learning_rate=1e-05 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --max_grad_norm=0.3 \ + --boundary_type="low" \ + --lora_skip_name="ffn" \ + --low_vram \ + --use_deepspeed \ + --prompt_path=$TRAIN_PROMPT_PATH \ + --train_sample_height=256 \ + --train_sample_width=256 \ + --num_inference_steps=40 \ + --video_length=81 \ + --num_decoded_latents=1 \ + --reward_fn="HPSReward" \ + --reward_fn_kwargs='{"version": "v2.1"}' \ + --backprop_strategy="tail" \ + --backprop_num_steps=1 \ + --backprop + +# Train MPS reward LoRA for the high noise model of Wan2.2-Fun-A14B-InP +# accelerate launch --mixed_precision="bf16" --num-processes=8 --use_deepspeed --deepspeed_config_file config/zero_stage3_config_cpu_offload.json scripts/wan2.2_fun/train_reward_lora.py \ +# --config_path="config/wan2.2/wan_civitai_i2v.yaml" \ +# --pretrained_model_name_or_path=$MODEL_NAME \ +# --train_batch_size=1 \ +# --gradient_accumulation_steps=1 \ +# --max_train_steps=10000 \ +# --checkpointing_steps=100 \ +# --learning_rate=1e-05 \ +# --seed=42 \ +# --output_dir="output_dir" \ +# --gradient_checkpointing \ +# --mixed_precision="bf16" \ +# --adam_weight_decay=3e-2 \ +# --adam_epsilon=1e-10 \ +# --max_grad_norm=0.3 \ +# --boundary_type="high" \ +# --lora_skip_name="ffn" \ +# --low_vram \ +# --use_deepspeed \ +# --prompt_path=$TRAIN_PROMPT_PATH \ +# --train_sample_height=256 \ +# --train_sample_width=256 \ +# --num_inference_steps=40 \ +# --video_length=81 \ +# --num_decoded_latents=1 \ +# --reward_fn="MPSReward" \ +# --backprop_strategy="tail" \ +# --backprop_num_steps=1 \ +# --backprop \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2_vace_fun/README_TRAIN.md b/VideoX-Fun/scripts/wan2.2_vace_fun/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..221e035f5bb823c6b299d9b270d41f6f6b55aad0 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_vace_fun/README_TRAIN.md @@ -0,0 +1,268 @@ +## Training Code + +We can choose whether to use deepspeed or fsdp in Wan-Fun, which can save a lot of video memory. + +The metadata_control.json is a little different from normal json in Wan-Fun, you need to add a control_file_path, and [DWPose](https://github.com/IDEA-Research/DWPose) is suggested as tool to generate control file. + +```json +[ + { + "file_path": "train/00000001.mp4", + "control_file_path": "control/00000001.mp4", + "object_file_path": ["object/1.jpg", "object/2.jpg"], + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "video" + }, + { + "file_path": "train/00000002.jpg", + "control_file_path": "control/00000002.jpg", + "object_file_path": ["object/3.jpg", "object/4.jpg"], + "text": "Ba Da Ba Ba Ba Ba.", + "type": "image" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images and videos at the center, but instead, it trains the entire images and videos after grouping them into buckets based on resolution. +- Sample size Configuration Guide + - `video_sample_size` represents the resolution size of videos; when `random_hw_adapt` is True, it represents the minimum value between video and image resolutions. + - `image_sample_size` represents the resolution size of images; when `random_hw_adapt` is True, it represents the maximum value between video and image resolutions. + - `token_sample_size` represents the resolution corresponding to the maximum token length when `training_with_video_token_length` is True. + - Due to potential confusion in configuration, **if you don't require arbitrary resolution for finetuning**, it is recommended to set `video_sample_size`, `image_sample_size`, and `token_sample_size` to the same fixed value, such as **(320, 480, 512, 640, 960)**. + - **All set to 320** represents **240P**. + - **All set to 480** represents **320P**. + - **All set to 640** represents **480P**. + - **All set to 960** represents **720P**. +- `random_frame_crop` is used for random cropping on video frames to simulate videos with different frame counts. +- `random_hw_adapt` is used to enable automatic height and width scaling for images and videos. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. For training videos, the height and width will be set to `image_sample_size` as the maximum and `min(video_sample_size, 512)` as the minimum. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=1024`, and `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024`, and the resolution of video inputs for training is `512x512x49` to `1024x1024x49`. + - For example, when `random_hw_adapt` is enabled, with `video_sample_n_frames=49`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49`. +- `training_with_video_token_length` specifies training the model according to token length. For training images and videos, the height and width will be set to `image_sample_size` as the maximum and `video_sample_size` as the minimum. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=1024`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x49`. + - For example, when `training_with_video_token_length` is enabled, with `video_sample_n_frames=49`, `token_sample_size=512`, `video_sample_size=256`, and `image_sample_size=1024`, the resolution of image inputs for training is `256x256` to `1024x1024`, and the resolution of video inputs for training is `256x256x49` to `1024x1024x9`. + - The token length for a video with dimensions 512x512 and 49 frames is 13,312. We need to set the `token_sample_size = 512`. + - At 512x512 resolution, the number of video frames is 49 (~= 512 * 512 * 49 / 512 / 512). + - At 768x768 resolution, the number of video frames is 21 (~= 512 * 512 * 49 / 768 / 768). + - At 1024x1024 resolution, the number of video frames is 9 (~= 512 * 512 * 49 / 1024 / 1024). + - These resolutions combined with their corresponding lengths allow the model to generate videos of different sizes. +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `train_mode` is used to set the training mode. + - The models named `Wan2.1-Fun-*-Control` are trained in the `control_ref` mode. + - The models named `Wan2.1-Fun-*-Control-Camera` are trained in the `control_ref_camera` mode. +- `control_ref_image` is used to specify the type of control image. The available options are `first_frame` and `random`. + - `first_frame` is used in V1.0 because V1.0 supports using a specified start frame as the control image. The Control-Camera models use the first frame as the control image. + - `random` is used in V1.1 because V1.1 supports both using a specified start frame and a reference image as the control image. +- `boundary_type`: The Wan2.2 series includes two distinct models that handle different noise levels, specified via the `boundary_type` parameter. `low`: Corresponds to the **low noise model** (low_noise_model). `high`: Corresponds to the **high noise model**. (high_noise_model). `full`: Corresponds to the ti2v 5B model (single mode). + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Wan-Fun-Control without deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-VACE-Fun-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_vace_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --control_ref_image="random" \ + --boundary_type="low" \ + --trainable_modules "vace" +``` + +Wan-Fun-Control with deepspeed: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-VACE-Fun-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_vace_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --control_ref_image="random" \ + --boundary_type="low" \ + --trainable_modules "vace" +``` + +DeepSpeed Zero-3 is not highly recommended at the moment. In this repository, using FSDP has fewer errors and is more stable. + +Wan-Fun-Control with DeepSpeed Zero-3: + +Wan with DeepSpeed Zero-3 is suitable for 14B Wan at high resolutions. After training, you can use the following command to get the final model: +```sh +python scripts/zero_to_bf16.py output_dir/checkpoint-{our-num-steps} output_dir/checkpoint-{your-num-steps}-outputs --max_shard_size 80GB --safe_serialization +``` + +Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-VACE-Fun-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --zero_stage 3 --zero3_save_16bit_model true --zero3_init_flag true --use_deepspeed --deepspeed_config_file config/zero_stage3_config.json --deepspeed_multinode_launcher standard scripts/wan2.2_vace_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --control_ref_image="random" \ + --boundary_type="low" \ + --trainable_modules "vace" +``` + +Wan-Fun-Control with FSDP: + +Wan with FSDP is suitable for 14B Wan at high resolutions. Training shell command is as follows: +```sh +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-VACE-Fun-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap=VaceWanAttentionBlock,BaseWanAttentionBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/wan2.2_vace_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --control_ref_image="random" \ + --boundary_type="low" \ + --trainable_modules "vace" +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/wan2.2_vace_fun/train.py b/VideoX-Fun/scripts/wan2.2_vace_fun/train.py new file mode 100644 index 0000000000000000000000000000000000000000..684bf82132fdf10a66ec44b515508684a203b4c1 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_vace_fun/train.py @@ -0,0 +1,2144 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator, FullyShardedDataParallelPlugin +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask, + padding_image, + process_pose_file, + process_pose_params) +from videox_fun.models import (AutoencoderKLWan, CLIPModel, AutoencoderKLWan3_8, + VaceWanTransformer3DModel, WanT5EncoderModel) +from videox_fun.pipeline import Wan2_2VaceFunPipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.lora_utils import (create_network, merge_lora, + unmerge_lora) +from videox_fun.utils.utils import (get_image_to_video_latent, + get_video_to_video_latent, + save_videos_grid) + +if is_wandb_available(): + import wandb + + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, clip_image_encoder, transformer3d, args, config, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + if args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + transformer3d_2_val = None + else: + if args.boundary_type == "low": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + + transformer3d_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d_2_val = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + transformer3d_2_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + + scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + pipeline = Wan2_2VaceFunPipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + transformer_2=transformer3d_2_val, + scheduler=scheduler, + clip_image_encoder=clip_image_encoder, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + with torch.autocast("cuda", dtype=weight_dtype): + video_length = int(args.video_sample_n_frames // vae.config.temporal_compression_ratio * vae.config.temporal_compression_ratio) + 1 if args.video_sample_n_frames != 1 else 1 + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(None, None, video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + control_video, _, _, _ = get_video_to_video_latent(args.validation_paths[i], video_length=video_length, sample_size=[args.video_sample_size, args.video_sample_size]) + sample = pipeline( + args.validation_prompts[i], + num_frames = video_length, + negative_prompt = "bad detailed", + height = args.video_sample_size, + width = args.video_sample_size, + generator = generator, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = control_video, + subject_ref_images = None, + vace_context_scale = 1, + ).videos + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + save_videos_grid(sample, os.path.join(args.output_dir, f"sample/sample-{global_step}-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + return images + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + return None + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--validation_paths", + type=str, + default=None, + nargs="+", + help=("A set of control videos evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_frame_crop", action="store_true", help="Whether enable random frame crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--training_with_video_token_length", action="store_true", help="The training stage of the model in training.", + ) + parser.add_argument( + "--auto_tile_batch_size", action="store_true", help="Whether to auto tile batch size.", + ) + parser.add_argument( + "--motion_sub_loss", action="store_true", help="Whether enable motion sub loss." + ) + parser.add_argument( + "--motion_sub_loss_ratio", type=float, default=0.25, help="The ratio of motion sub loss." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--keep_all_node_same_token_length", + action="store_true", + help="Reference of the length token.", + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--video_sample_stride", + type=int, + default=4, + help="Sample stride of the video.", + ) + parser.add_argument( + "--video_sample_n_frames", + type=int, + default=17, + help="Num frame of video.", + ) + parser.add_argument( + "--video_repeat", + type=int, + default=0, + help="Num of repeat video.", + ) + parser.add_argument( + "--config_path", + type=str, + default=None, + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--boundary_type", + type=str, + default="low", + help=( + 'The format of training data. Support `"low"` and `"high"`' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--control_ref_image", + type=str, + default="first_frame", + help=( + 'The format of training data. Support `"first_frame"`' + ' (default), `"random"`.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + if args.train_batch_size >= 2: + raise ValueError("This code does not support args.train_batch_size >= 2 now.") + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + config = OmegaConf.load(args.config_path) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler( + **filter_kwargs(FlowMatchEulerDiscreteScheduler, OmegaConf.to_container(config['scheduler_kwargs'])) + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So CLIPTextModel and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=weight_dtype, + ) + text_encoder = text_encoder.eval() + # Get Vae + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(config['vae_kwargs']), + ) + vae.eval() + + # Get Transformer + if args.boundary_type == "low" or args.boundary_type == "full": + sub_path = config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer') + else: + sub_path = config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer') + transformer3d = VaceWanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, sub_path), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = VaceWanTransformer3DModel.from_pretrained( + os.path.join(args.pretrained_model_name_or_path, config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=VaceWanTransformer3DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = VaceWanTransformer3DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = VaceWanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']) + ) + load_model = EMAModel(load_model.parameters(), model_cls=VaceWanTransformer3DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = VaceWanTransformer3DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + sample_n_frames_bucket_interval = vae.config.temporal_compression_ratio + spatial_compression_ratio = vae.config.spatial_compression_ratio + + if args.fix_sample_size is not None and args.enable_bucket: + args.video_sample_size = max(max(args.fix_sample_size), args.video_sample_size) + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.training_with_video_token_length = False + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoControlDataset( + args.train_data_meta, args.train_data_dir, + video_sample_size=args.video_sample_size, video_sample_stride=args.video_sample_stride, video_sample_n_frames=args.video_sample_n_frames, + video_repeat=args.video_repeat, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + enable_inpaint=False, + enable_camera_info=False, + enable_subject_info=True + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_length_to_frame_num(token_length): + if args.image_sample_size > args.video_sample_size: + sample_sizes = list(range(args.video_sample_size, args.image_sample_size + 1, 128)) + + if sample_sizes[-1] != args.image_sample_size: + sample_sizes.append(args.image_sample_size) + else: + sample_sizes = [args.image_sample_size] + + length_to_frame_num = { + sample_size: min(token_length / sample_size / sample_size, args.video_sample_n_frames) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 for sample_size in sample_sizes + } + + return length_to_frame_num + + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Get token length + target_token_length = args.video_sample_n_frames * args.token_sample_size * args.token_sample_size + length_to_frame_num = get_length_to_frame_num(target_token_length) + + # Create new output + new_examples = {} + new_examples["target_token_length"] = target_token_length + new_examples["pixel_values"] = [] + new_examples["text"] = [] + # Used in Control Mode + new_examples["control_pixel_values"] = [] + # Used in Control Ref Mode + new_examples["ref_pixel_values"] = [] + new_examples["clip_pixel_values"] = [] + new_examples["clip_idx"] = [] + + # Used in Inpaint mode + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + + new_examples["subject_images"] = [] + new_examples["subject_flags"] = [] + + # Get downsample ratio in image and videos + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + if data_type == 'image': + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size, image_ratio=[args.image_sample_size / args.video_sample_size]) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + if args.random_hw_adapt: + if args.training_with_video_token_length: + local_min_size = np.min(np.array([np.mean(np.array([np.shape(example["pixel_values"])[1], np.shape(example["pixel_values"])[2]])) for example in examples])) + + def get_random_downsample_probability(choice_list, token_sample_size): + length = len(choice_list) + if length == 1: + return [1.0] # If there's only one element, it gets all the probability + + # Find the index of the closest value to token_sample_size + closest_index = min(range(length), key=lambda i: abs(choice_list[i] - token_sample_size)) + + # Assign 50% to the closest index + first_element = 0.50 + remaining_sum = 1.0 - first_element + + # Distribute the remaining 50% evenly among the other elements + other_elements_value = remaining_sum / (length - 1) if length > 1 else 0.0 + + # Construct the probability distribution + probability_list = [other_elements_value] * length + probability_list[closest_index] = first_element + + return probability_list + + choice_list = [length for length in list(length_to_frame_num.keys()) if length < local_min_size * 1.25] + if len(choice_list) == 0: + choice_list = list(length_to_frame_num.keys()) + probabilities = get_random_downsample_probability(choice_list, args.token_sample_size) + local_video_sample_size = np.random.choice(choice_list, p=probabilities) + + random_downsample_ratio = args.video_sample_size / local_video_sample_size + batch_video_length = length_to_frame_num[local_video_sample_size] + else: + random_downsample_ratio = get_random_downsample_ratio(args.video_sample_size) + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + else: + random_downsample_ratio = 1 + batch_video_length = args.video_sample_n_frames + sample_n_frames_bucket_interval + + aspect_ratio_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.video_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in closest_size] + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + if args.control_ref_image == "first_frame": + clip_index = 0 + else: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.40 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + number_list_prob = np.array(_create_special_list(len(pixel_values))) + clip_index = np.random.choice(list(range(len(pixel_values))), p = number_list_prob) + new_examples["clip_idx"].append(clip_index) + + ref_pixel_values = pixel_values[clip_index].permute(1, 2, 0).contiguous() + ref_pixel_values = Image.fromarray(np.uint8(ref_pixel_values * 255)) + ref_pixel_values = padding_image(ref_pixel_values, closest_size[1], closest_size[0]) + ref_pixel_values = (torch.tensor(np.array(ref_pixel_values)).unsqueeze(0).permute(0, 3, 1, 2).contiguous() / 255 - 0.5)/0.5 + new_examples["ref_pixel_values"].append(ref_pixel_values) + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + _, channel, h, w = pixel_values.size() + new_subject_image = torch.zeros(4, channel, h, w) + num_subject = len(example["subject_image"]) + if num_subject != 0: + subject_image = torch.from_numpy(example["subject_image"]).permute(0, 3, 1, 2).contiguous() + new_subject_image[:num_subject] = subject_image + subject_image = new_subject_image / 255. + subject_flag = torch.from_numpy(np.array([1] * num_subject + [0] * (4 - num_subject))) + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + transform_no_normalize = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + ]) + + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["control_pixel_values"].append(transform(control_pixel_values)) + + new_examples["text"].append(example["text"]) + # Magvae needs the number of frames to be 4n + 1. + batch_video_length = int( + min( + batch_video_length, + (len(pixel_values) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1, + ) + ) + if batch_video_length == 0: + batch_video_length = 1 + + clip_pixel_values = new_examples["pixel_values"][-1][clip_index].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + new_examples["clip_pixel_values"].append(clip_pixel_values) + + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + + # Wan 2.1 use 0 for masked pixels + # + torch.ones_like(new_examples["pixel_values"][-1]) * -1 * mask + new_examples["mask_pixel_values"].append(mask_pixel_values) + new_examples["mask"].append(mask) + + new_examples["subject_images"].append(transform(subject_image)) + new_examples["subject_flags"].append(subject_flag) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["control_pixel_values"]]) + new_examples["ref_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["ref_pixel_values"]]) + new_examples["clip_pixel_values"] = torch.stack([example for example in new_examples["clip_pixel_values"]]) + new_examples["clip_idx"] = torch.tensor(new_examples["clip_idx"]) + new_examples["mask_pixel_values"] = torch.stack([example[:batch_video_length] for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example[:batch_video_length] for example in new_examples["mask"]]) + new_examples["subject_images"] = torch.stack([example for example in new_examples["subject_images"]]) + new_examples["subject_flags"] = torch.stack([example for example in new_examples["subject_flags"]]) + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_ids = tokenizer( + new_examples['text'], + max_length=args.tokenizer_max_length, + padding="max_length", + add_special_tokens=True, + truncation=True, + return_tensors="pt" + ) + encoder_hidden_states = text_encoder( + prompt_ids.input_ids + )[0] + new_examples['encoder_attention_mask'] = prompt_ids.attention_mask + new_examples['encoder_hidden_states'] = encoder_hidden_states + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + text_encoder = shard_fn(text_encoder) + + # shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype) + # transformer3d = shard_fn(transformer3d) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + tracker_config.pop("validation_prompts") + tracker_config.pop("trainable_modules") + tracker_config.pop("trainable_modules_low_learning_rate") + tracker_config.pop("fix_sample_size") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need + boundary = config['transformer_additional_kwargs'].get('boundary', 0.900) + split_timesteps = args.train_sampling_steps * boundary + differences = torch.abs(noise_scheduler.timesteps - split_timesteps) + closest_index = torch.argmin(differences).item() + print(f"The boundary is {boundary} and the boundary_type is {args.boundary_type}. The closest_index we calculate is {closest_index}") + if args.boundary_type == "high": + start_num_idx = 0 + train_sampling_steps = closest_index + elif args.boundary_type == "low": + start_num_idx = closest_index + train_sampling_steps = args.train_sampling_steps - closest_index + else: + start_num_idx = 0 + train_sampling_steps = args.train_sampling_steps + idx_sampling = DiscreteSampling(train_sampling_steps, start_num_idx=start_num_idx, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + control_pixel_values = batch["control_pixel_values"].cpu() + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, control_pixel_value, text) in enumerate(zip(pixel_values, control_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + control_pixel_value = control_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + save_videos_grid(control_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_control.gif", rescale=True) + + ref_pixel_values = batch["ref_pixel_values"].cpu() + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + for idx, (ref_pixel_value, text) in enumerate(zip(ref_pixel_values, texts)): + ref_pixel_value = ref_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(ref_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_ref.gif", rescale=True) + + subject_images = batch["subject_images"].cpu() + subject_images = rearrange(subject_images, "b f c h w -> b c f h w") + for idx, (subject_image, text) in enumerate(zip(subject_images, texts)): + subject_image = subject_image[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(subject_image, f"{args.output_dir}/sanity_check/{gif_name[:10]}_subject.gif", rescale=True) + + clip_pixel_values, mask_pixel_values, texts = batch['clip_pixel_values'].cpu(), batch['mask_pixel_values'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + for idx, (clip_pixel_value, pixel_value, text) in enumerate(zip(clip_pixel_values, mask_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + Image.fromarray(np.uint8(clip_pixel_value)).save(f"{args.output_dir}/sanity_check/clip_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.png") + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (4, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (4, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (4, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (4, 1)) + else: + batch['text'] = batch['text'] * 4 + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + pixel_values = torch.tile(pixel_values, (2, 1, 1, 1, 1)) + control_pixel_values = torch.tile(control_pixel_values, (2, 1, 1, 1, 1)) + if args.enable_text_encoder_in_dataloader: + batch['encoder_hidden_states'] = torch.tile(batch['encoder_hidden_states'], (2, 1, 1)) + batch['encoder_attention_mask'] = torch.tile(batch['encoder_attention_mask'], (2, 1)) + else: + batch['text'] = batch['text'] * 2 + + ref_pixel_values = batch["ref_pixel_values"].to(weight_dtype) + clip_pixel_values = batch["clip_pixel_values"] + subject_images = batch["subject_images"].to(weight_dtype) + subject_flags = batch["subject_flags"].to(weight_dtype) + clip_idx = batch["clip_idx"] + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + + # Increase the batch size when the length of the latent sequence of the current sample is small + if args.auto_tile_batch_size and args.training_with_video_token_length and zero_stage != 3: + if args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 16 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (4, 1, 1, 1)) + subject_images = torch.tile(subject_images, (4, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (4, 1, 1, 1, 1)) + subject_flags = torch.tile(subject_flags, (4, 1)) + clip_idx = torch.tile(clip_idx, (4,)) + mask_pixel_values = torch.tile(mask_pixel_values, (4, 1, 1, 1, 1)) + mask = torch.tile(mask, (4, 1, 1, 1, 1)) + elif args.video_sample_n_frames * args.token_sample_size * args.token_sample_size // 4 >= pixel_values.size()[1] * pixel_values.size()[3] * pixel_values.size()[4]: + clip_pixel_values = torch.tile(clip_pixel_values, (2, 1, 1, 1)) + subject_images = torch.tile(subject_images, (2, 1, 1, 1, 1)) + ref_pixel_values = torch.tile(ref_pixel_values, (2, 1, 1, 1, 1)) + subject_flags = torch.tile(subject_flags, (2, 1)) + clip_idx = torch.tile(clip_idx, (2,)) + mask_pixel_values = torch.tile(mask_pixel_values, (2, 1, 1, 1, 1)) + mask = torch.tile(mask, (2, 1, 1, 1, 1)) + + if args.random_frame_crop: + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + last_element = 0.90 + remaining_sum = 1.0 - last_element + other_elements_value = remaining_sum / (length - 1) + special_list = [other_elements_value] * (length - 1) + [last_element] + return special_list + select_frames = [_tmp for _tmp in list(range(sample_n_frames_bucket_interval + 1, args.video_sample_n_frames + sample_n_frames_bucket_interval, sample_n_frames_bucket_interval))] + select_frames_prob = np.array(_create_special_list(len(select_frames))) + + if len(select_frames) != 0: + if rng is None: + temp_n_frames = np.random.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = rng.choice(select_frames, p = select_frames_prob) + else: + temp_n_frames = 1 + + # Magvae needs the number of frames to be 4n + 1. + temp_n_frames = (temp_n_frames - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :temp_n_frames, :, :] + control_pixel_values = control_pixel_values[:, :temp_n_frames, :, :] + mask_pixel_values = mask_pixel_values[:, :temp_n_frames, :, :] + mask = mask[:, :temp_n_frames, :, :] + + # Keep all node same token length to accelerate the traning when resolution grows. + if args.keep_all_node_same_token_length: + if args.token_sample_size > 256: + numbers_list = list(range(256, args.token_sample_size + 1, 128)) + + if numbers_list[-1] != args.token_sample_size: + numbers_list.append(args.token_sample_size) + else: + numbers_list = [256] + numbers_list = [_number * _number * args.video_sample_n_frames for _number in numbers_list] + + actual_token_length = index_rng.choice(numbers_list) + actual_video_length = (min( + actual_token_length / pixel_values.size()[-1] / pixel_values.size()[-2], args.video_sample_n_frames + ) - 1) // sample_n_frames_bucket_interval * sample_n_frames_bucket_interval + 1 + actual_video_length = int(max(actual_video_length, 1)) + + # Magvae needs the number of frames to be 4n + 1. + actual_video_length = (actual_video_length - 1) // sample_n_frames_bucket_interval + 1 + + pixel_values = pixel_values[:, :actual_video_length, :, :] + control_pixel_values = control_pixel_values[:, :actual_video_length, :, :] + mask_pixel_values = mask_pixel_values[:, :actual_video_length, :, :] + mask = mask[:, :actual_video_length, :, :] + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + def vace_encode_frames(frames, ref_images, masks=None): + weight_dtype = frames.dtype + if ref_images is None: + ref_images = [None] * len(frames) + else: + assert len(frames) == len(ref_images) + + if masks is None: + latents = vae.encode(frames)[0].mode() + else: + masks = [torch.where(m > 0.5, 1.0, 0.0).to(weight_dtype) for m in masks] + inactive = [i * (1 - m) + 0 * m for i, m in zip(frames, masks)] + reactive = [i * m + 0 * (1 - m) for i, m in zip(frames, masks)] + inactive = vae.encode(inactive)[0].mode() + reactive = vae.encode(reactive)[0].mode() + latents = [torch.cat((u, c), dim=0) for u, c in zip(inactive, reactive)] + + cat_latents = [] + for latent, refs in zip(latents, ref_images): + if refs is not None: + if masks is None: + ref_latent = vae.encode(refs)[0].mode() + else: + ref_latent = vae.encode(refs)[0].mode() + ref_latent = [torch.cat((u, torch.zeros_like(u)), dim=0) for u in ref_latent] + assert all([x.shape[1] == 1 for x in ref_latent]) + latent = torch.cat([*ref_latent, latent], dim=1) + cat_latents.append(latent) + return cat_latents + + def vace_encode_masks(masks, ref_images=None, vae_stride=[4, 8, 8]): + if ref_images is None: + ref_images = [None] * len(masks) + else: + assert len(masks) == len(ref_images) + + result_masks = [] + for mask, refs in zip(masks, ref_images): + c, depth, height, width = mask.shape + new_depth = int((depth + 3) // vae_stride[0]) + height = 2 * (int(height) // (vae_stride[1] * 2)) + width = 2 * (int(width) // (vae_stride[2] * 2)) + + # reshape + mask = mask[0, :, :, :] + mask = mask.view( + depth, height, vae_stride[1], width, vae_stride[1] + ) # depth, height, 8, width, 8 + mask = mask.permute(2, 4, 0, 1, 3) # 8, 8, depth, height, width + mask = mask.reshape( + vae_stride[1] * vae_stride[2], depth, height, width + ) # 8*8, depth, height, width + + # interpolation + mask = F.interpolate(mask.unsqueeze(0), size=(new_depth, height, width), mode='nearest-exact').squeeze(0) + + if refs is not None: + length = len(refs) + c, depth, height, width = mask.shape + mask_pad = mask.new_zeros(c, length, height, width) + mask = torch.cat((mask_pad, mask), dim=1) + result_masks.append(mask) + return result_masks + + def vace_latent(z, m): + return [torch.cat([zz, mm], dim=0) for zz, mm in zip(z, m)] + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values) + else: + latents = _batch_encode_vae(pixel_values) + + if rng is None: + subject_images_num = np.random.choice([0, 1, 2, 3, 4]) + else: + subject_images_num = rng.choice([0, 1, 2, 3, 4]) + + if rng is None: + use_full_photo_ref_flag = np.random.choice([True, False], p=[0.25, 0.75]) + else: + use_full_photo_ref_flag = rng.choice([True, False], p=[0.25, 0.75]) + + if not use_full_photo_ref_flag: + if subject_images_num == 0: + subject_ref_images = None + else: + subject_ref_images = rearrange(subject_images, "b f c h w -> b c f h w") + subject_ref_images = subject_ref_images[:, :, :subject_images_num] + + bs, c, f, h, w = subject_ref_images.size() + new_subject_ref_images = [] + for i in range(bs): + act_subject_images_num = min(subject_images_num, int(torch.sum(subject_flags[i]))) + + if act_subject_images_num == 0: + new_subject_ref_images.append(None) + else: + new_subject_ref_images.append([]) + for j in range(act_subject_images_num): + new_subject_ref_images[i].append(subject_ref_images[i, :, j:j+1]) + subject_ref_images = new_subject_ref_images + else: + ref_pixel_values = rearrange(ref_pixel_values, "b f c h w -> b c f h w") + + bs, c, f, h, w = ref_pixel_values.size() + new_ref_pixel_values = [] + for i in range(bs): + new_ref_pixel_values.append([]) + for j in range(1): + new_ref_pixel_values[i].append(ref_pixel_values[i, :, j:j+1]) + subject_ref_images = new_ref_pixel_values + + if rng is None: + inpaint_flag = np.random.choice([True, False], p=[0.75, 0.25]) + else: + inpaint_flag = rng.choice([True, False], p=[0.75, 0.25]) + mask = rearrange(mask, "b f c h w -> b c f h w") + mask = torch.tile(mask, [1, 3, 1, 1, 1]) + if inpaint_flag or (control_pixel_values == -1).all(): + if rng is None: + do_not_use_ref_images = np.random.choice([True, False], p=[0.50, 0.50]) + else: + do_not_use_ref_images = rng.choice([True, False], p=[0.50, 0.50]) + if do_not_use_ref_images: + subject_ref_images = None + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + vace_latents = vace_encode_frames(mask_pixel_values, subject_ref_images, mask) + else: + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + vace_latents = vace_encode_frames(control_pixel_values, subject_ref_images, mask) + mask = torch.ones_like(mask) + + mask_latents = vace_encode_masks(mask, subject_ref_images, vae_stride=[4, spatial_compression_ratio, spatial_compression_ratio]) + vace_context = torch.stack(vace_latent(vace_latents, mask_latents)) + + if subject_ref_images is not None: + for i in range(len(subject_ref_images)): + if subject_ref_images[i] is not None: + + subject_ref_images[i] = torch.cat( + [subject_ref_image.unsqueeze(0) for subject_ref_image in subject_ref_images[i]], 2 + ) + subject_ref_images[i] = torch.cat( + [vae.encode(subject_ref_images[i][:, :, j:j+1])[0].sample() for j in range(subject_ref_images[i].size(2))], 2 + ) + + if subject_ref_images[0] is not None: + subject_ref_images = torch.cat(subject_ref_images) + latents = torch.cat( + [subject_ref_images, latents], dim=2 + ) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['encoder_hidden_states'].to(device=latents.device) + else: + with torch.no_grad(): + prompt_ids = tokenizer( + batch['text'], + padding="max_length", + max_length=args.tokenizer_max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt" + ) + text_input_ids = prompt_ids.input_ids + prompt_attention_mask = prompt_ids.attention_mask + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = text_encoder(text_input_ids.to(latents.device), attention_mask=prompt_attention_mask.to(latents.device))[0] + prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, num_frames, height, width = latents.size() + noise = torch.randn( + (bsz, channel, num_frames, height, width), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + target_shape = (vae.latent_channels, vace_latents[0].size(1), width, height) + seq_len = math.ceil( + (target_shape[2] * target_shape[3]) / + (accelerator.unwrap_model(transformer3d).config.patch_size[1] * accelerator.unwrap_model(transformer3d).config.patch_size[2]) * + target_shape[1] + ) + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + context=prompt_embeds, + t=timesteps, + seq_len=seq_len, + vace_context=vace_context, + ) + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + if args.motion_sub_loss and noise_pred.size()[2] > 2: + gt_sub_noise = noise_pred[:, :, 1:].float() - noise_pred[:, :, :-1].float() + pre_sub_noise = target[:, :, 1:].float() - target[:, :, :-1].float() + sub_loss = F.mse_loss(gt_sub_noise, pre_sub_noise, reduction="mean") + loss = loss * (1 - args.motion_sub_loss_ratio) + sub_loss * args.motion_sub_loss_ratio + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + clip_image_encoder, + transformer3d, + args, + config, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + transformer3d = unwrap_model(transformer3d) + if args.use_ema: + ema_transformer3d.copy_to(transformer3d.parameters()) + + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/wan2.2_vace_fun/train.sh b/VideoX-Fun/scripts/wan2.2_vace_fun/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..26c59c7282bc15d133e8e43ba6d1bbf0005d7327 --- /dev/null +++ b/VideoX-Fun/scripts/wan2.2_vace_fun/train.sh @@ -0,0 +1,43 @@ +export MODEL_NAME="models/Diffusion_Transformer/Wan2.2-VACE-Fun-A14B" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/wan2.2_vace_fun/train.py \ + --config_path="config/wan2.2/wan_civitai_t2v.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --image_sample_size=640 \ + --video_sample_size=640 \ + --token_sample_size=640 \ + --video_sample_stride=2 \ + --video_sample_n_frames=81 \ + --train_batch_size=1 \ + --video_repeat=1 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --random_hw_adapt \ + --training_with_video_token_length \ + --enable_bucket \ + --uniform_sampling \ + --low_vram \ + --control_ref_image="random" \ + --boundary_type="low" \ + --trainable_modules "vace" \ No newline at end of file diff --git a/VideoX-Fun/scripts/z_image/README_TRAIN.md b/VideoX-Fun/scripts/z_image/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..81051b5c3e81b5e435abd215643fbb4f23b0bc7a --- /dev/null +++ b/VideoX-Fun/scripts/z_image/README_TRAIN.md @@ -0,0 +1,133 @@ +## Training Code + +We can choose whether to use deepspeed or fsdp in z_image, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images at the center, but instead, it trains the entire images after grouping them into buckets based on resolution. +- `random_hw_adapt` is used to enable automatic height and width scaling for images. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024` +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Without deepspeed: + +Training z_image without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/z_image/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` + +With Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/z_image/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap ZImageTransformer2DModel --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/z_image/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/z_image/README_TRAIN_LORA.md b/VideoX-Fun/scripts/z_image/README_TRAIN_LORA.md new file mode 100644 index 0000000000000000000000000000000000000000..b3e3a51a63ab490c4752af1fdcd09963019feddf --- /dev/null +++ b/VideoX-Fun/scripts/z_image/README_TRAIN_LORA.md @@ -0,0 +1,140 @@ +## Lora Training Code + +We can choose whether to use deepspeed or fsdp in z_image, which can save a lot of video memory. + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images at the center, but instead, it trains the entire images after grouping them into buckets based on resolution. +- `random_hw_adapt` is used to enable automatic height and width scaling for images. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024` +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. +- `target_name` represents the components/modules to which LoRA will be applied, separated by commas. +- `use_peft_lora` indicates whether to use the PEFT module for adding LoRA. Using this module will be more memory-efficient. +- `rank` means the dimension of the LoRA update matrices. +- `network_alpha` means the scale of the LoRA update matrices. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Without deepspeed: + +Training z_image without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/z_image/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling +``` + +With Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/z_image/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap ZImageTransformer2DModel --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/z_image/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=32 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/z_image/export_transformer_body_onnx.py b/VideoX-Fun/scripts/z_image/export_transformer_body_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..fad09a7173afa7701bc53eae8a6a479639c7b051 --- /dev/null +++ b/VideoX-Fun/scripts/z_image/export_transformer_body_onnx.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""Export the pure Z-Image transformer body (no ControlNet) to ONNX.""" + +import argparse +import logging +import os +import sys +from collections import OrderedDict +from typing import Dict, List, Optional, OrderedDict as OrderedDictType, Tuple + +import numpy as np +import torch +import onnx +from onnx import numpy_helper +import subprocess +from omegaconf import OmegaConf + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +from videox_fun.models import ZImageTransformer2DModel # noqa: E402 + +LOGGER = logging.getLogger("export_transformer_body_onnx") +logging.basicConfig(level=logging.INFO, format="[%(asctime)s] %(levelname)s: %(message)s") + +SEQ_MULTI_OF = 32 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Export the pure Z-Image transformer body to ONNX (no ControlNet)") + parser.add_argument("--config", default="config/z_image/z_image.yaml", help="Path to YAML config (if needed)") + parser.add_argument("--model-root", default="models/Diffusion_Transformer/Z-Image-Turbo/", help="Directory that stores the pretrained weights") + parser.add_argument("--checkpoint", default=None, help="Optional fine-tuned checkpoint to load") + parser.add_argument("--output", default="onnx-models/z_image_transformer_body_only.onnx", help="Target ONNX file path") + parser.add_argument("--height", type=int, default=864, help="Target image height used to derive latent resolution") + parser.add_argument("--width", type=int, default=496, help="Target image width used to derive latent resolution") + parser.add_argument("--batch-size", type=int, default=1, help="Batch size for the exported graph") + parser.add_argument("--sequence-length", type=int, default=512, help="Prompt embedding sequence length (must be a multiple of 32)") + parser.add_argument("--frames", type=int, default=1, help="Number of frames in the latent tensor") + parser.add_argument("--latent-downsample-factor", type=int, default=8, help="Downsampling ratio between spatial image size and latent size") + parser.add_argument("--latent-height", type=int, default=None, help="Override latent height (after downsampling)") + parser.add_argument("--latent-width", type=int, default=None, help="Override latent width (after downsampling)") + parser.add_argument("--dtype", choices=["fp16", "fp32"], default="fp16", help="Export precision") + parser.add_argument("--patch-size", type=int, default=2, help="Spatial patch size used by the transformer") + parser.add_argument("--f-patch-size", type=int, default=1, help="Frame patch size used by the transformer") + parser.add_argument("--opset", type=int, default=17, help="ONNX opset version") + parser.add_argument("--no-external-data", action="store_true", help="Disable external data format even if the model is larger than 2GB") + parser.add_argument("--skip-ort-check", action="store_true", help="Skip running an ONNX Runtime correctness check") + parser.add_argument("--ort-provider", default="CPUExecutionProvider", help="ONNX Runtime provider used during validation") + parser.add_argument("--save-calib-inputs", action="store_true", help="Dump ONNX input dictionaries as .npy for calibration") + parser.add_argument("--calib-dir", default="onnx-calibration", help="Directory for storing calibration npy files") + parser.add_argument("--dynamic-axes", action="store_true", help="Export ONNX with dynamic batch/seq/latent dims; default is static shape") + parser.add_argument("--skip-slim", action="store_true", help="Skip onnxslim simplification for faster debug export") + return parser.parse_args() + + +def run_onnxslim(input_file="vae.onnx", output_file="vae_slim.onnx") -> bool: + """调用 onnxslim，对大模型可能较慢，可用 --skip-slim 跳过。""" + try: + cmd = ["onnxslim", input_file, output_file] + print(f"执行命令: {' '.join(cmd)}") + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + universal_newlines=True, + ) + for line in process.stdout: + print(line, end="") + stdout, stderr = process.communicate() + if process.returncode != 0: + print(f"命令执行失败, 错误信息:\n{stderr}") + return False + print("ONNX模型压缩完成!") + return True + except FileNotFoundError: + print("错误: 未找到 onnxslim 命令, 请确保已安装 onnxslim (pip install onnx-simplifier)") + return False + except Exception as exc: # pragma: no cover + print(f"执行命令时发生错误: {exc}") + return False + + +def _resolve_path(path: str) -> str: + return os.path.abspath(os.path.join(REPO_ROOT, path)) if not os.path.isabs(path) else path + + +def load_transformer(args: argparse.Namespace, torch_dtype: torch.dtype, device: torch.device) -> ZImageTransformer2DModel: + model_root = _resolve_path(args.model_root) + checkpoint_path = _resolve_path(args.checkpoint) if args.checkpoint else None + + if not os.path.isdir(model_root): + raise FileNotFoundError(f"Model root not found: {model_root}") + + LOGGER.info("Loading transformer from %s", model_root) + transformer = ZImageTransformer2DModel.from_pretrained( + model_root, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=torch_dtype, + ) + transformer.eval() + transformer.to(device=device, dtype=torch_dtype) + + if checkpoint_path and os.path.exists(checkpoint_path): + LOGGER.info("Loading checkpoint %s", checkpoint_path) + if checkpoint_path.endswith(".safetensors"): + from safetensors.torch import load_file # type: ignore + + state_dict = load_file(checkpoint_path) + else: + state_dict = torch.load(checkpoint_path, map_location="cpu") + state_dict = state_dict.get("state_dict", state_dict) + missing, unexpected = transformer.load_state_dict(state_dict, strict=False) + LOGGER.info("Checkpoint loaded (missing=%d, unexpected=%d)", len(missing), len(unexpected)) + elif checkpoint_path: + raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}") + + return transformer + + +def _validate_sequence_length(seq_len: int) -> None: + if seq_len % SEQ_MULTI_OF != 0: + raise ValueError("sequence_length must be a multiple of 32 to satisfy transformer padding rules") + + +def _compute_latent_dims(args: argparse.Namespace) -> Dict[str, int]: + if args.latent_height is not None and args.latent_width is not None: + latent_h = args.latent_height + latent_w = args.latent_width + else: + if args.height % args.latent_downsample_factor != 0 or args.width % args.latent_downsample_factor != 0: + raise ValueError("height and width must be divisible by latent_downsample_factor") + latent_h = args.height // args.latent_downsample_factor + latent_w = args.width // args.latent_downsample_factor + if latent_h % args.patch_size != 0 or latent_w % args.patch_size != 0: + raise ValueError("latent dimensions must be divisible by patch_size") + if args.frames % args.f_patch_size != 0: + raise ValueError("frames must be divisible by f_patch_size") + return {"latent_h": latent_h, "latent_w": latent_w} + + +def build_dummy_inputs( + args: argparse.Namespace, + model: ZImageTransformer2DModel, + torch_dtype: torch.dtype, + device: torch.device, +) -> OrderedDictType[str, torch.Tensor]: + _validate_sequence_length(args.sequence_length) + dims = _compute_latent_dims(args) + batch = args.batch_size + in_channels = model.config.in_channels + cap_dim = model.config.cap_feat_dim + + latent = torch.randn( + batch, + in_channels, + args.frames, + dims["latent_h"], + dims["latent_w"], + dtype=torch_dtype, + device=device, + ) + timestep = torch.linspace(0.0, 1.0, steps=batch, dtype=torch.float32, device=device) + prompts = torch.randn( + batch, + args.sequence_length, + cap_dim, + dtype=torch_dtype, + device=device, + ) + + return OrderedDict( + latent_model_input=latent, + timestep=timestep, + prompt_embeds=prompts, + ) + + +def maybe_save_calibration_inputs(tag: str, inputs: OrderedDictType[str, torch.Tensor], args: argparse.Namespace) -> Optional[str]: + if not getattr(args, "save_calib_inputs", False): + return None + output_dir = _resolve_path(args.calib_dir) + os.makedirs(output_dir, exist_ok=True) + numpy_dict = {name: tensor.detach().cpu().numpy() for name, tensor in inputs.items()} + file_path = os.path.join(output_dir, f"{tag}_inputs.npy") + np.save(file_path, numpy_dict, allow_pickle=True) + LOGGER.info("Saved calibration inputs (%s) to %s", tag, file_path) + return file_path + + +def dump_initializer_parameters(model_path: str) -> str: + """Save all ONNX initializers into a standalone .npz file.""" + model_proto = onnx.load(model_path, load_external_data=True) + param_dict = {} + for initializer in model_proto.graph.initializer: + param_dict[initializer.name] = numpy_helper.to_array(initializer) + param_path = f"{model_path}.params.npz" + np.savez(param_path, **param_dict) + LOGGER.info("Saved %d parameters to %s", len(param_dict), param_path) + return param_path + + +class TransformerBodyOnlyWrapper(torch.nn.Module): + """轻量包装，暴露纯 transformer body 所需的输入。""" + + def __init__(self, model: ZImageTransformer2DModel, patch_size: int, f_patch_size: int): + super().__init__() + self.model = model + self.patch_size = patch_size + self.f_patch_size = f_patch_size + + def forward( + self, + latent_model_input: torch.Tensor, + timestep: torch.Tensor, + prompt_embeds: torch.Tensor, + ) -> torch.Tensor: + latents = list(latent_model_input.unbind(dim=0)) + prompts = list(prompt_embeds.unbind(dim=0)) + outputs = self.model( + latents, + timestep, + prompts, + patch_size=self.patch_size, + f_patch_size=self.f_patch_size, + ) + return outputs + + +def export_onnx( + wrapper: torch.nn.Module, + sample_inputs: OrderedDictType[str, torch.Tensor], + output_path: str, + output_names: List[str], + args: argparse.Namespace, +) -> Tuple[str, str]: + export_path = _resolve_path(output_path) + export_dir = os.path.dirname(export_path) + if export_dir: + os.makedirs(export_dir, exist_ok=True) + input_names = list(sample_inputs.keys()) + use_external = not args.no_external_data + wrapper.eval() + + dynamic_axes = None + if args.dynamic_axes: + dynamic_axes = { + "latent_model_input": {0: "batch", 2: "frames", 3: "latent_h", 4: "latent_w"}, + "prompt_embeds": {0: "batch", 1: "seq_len"}, + "timestep": {0: "batch"}, + "sample": {0: "batch", 2: "frames", 3: "latent_h", 4: "latent_w"}, + } + + LOGGER.info("Exporting ONNX to %s", export_path) + with torch.inference_mode(): + torch.onnx.export( + wrapper, + args=tuple(sample_inputs[name] for name in input_names), + f=export_path, + input_names=input_names, + output_names=output_names, + opset_version=args.opset, + do_constant_folding=True, + export_params=True, + dynamic_axes={k: v for k, v in dynamic_axes.items() if k in input_names + output_names} if dynamic_axes else None, + # use_external_data_format=use_external, + ) + + LOGGER.info("Raw ONNX export finished") + + trans_onnx = onnx.load(export_path) + simp_onnx_data = os.path.splitext(export_path)[0] + "_simp.onnx" + onnx.save( + trans_onnx, + simp_onnx_data, + save_as_external_data=True, + all_tensors_to_one_file=True, + ) + external_weight_file = simp_onnx_data + ".data" + LOGGER.info("Saved external-data ONNX to %s (weights -> %s)", simp_onnx_data, external_weight_file) + + if args.skip_slim: + LOGGER.info("Skip onnxslim as requested, using simplified external-data ONNX: %s", simp_onnx_data) + final_onnx = simp_onnx_data + else: + slim_onnx_path = os.path.splitext(simp_onnx_data)[0] + "_slim.onnx" + LOGGER.info("Transformer ONNX model exported, start to simplify via onnxslim") + success = run_onnxslim(simp_onnx_data, slim_onnx_path) + if not success: + raise RuntimeError("onnxslim simplification failed, please check logs") + final_onnx = slim_onnx_path + LOGGER.info("Transformer ONNX model exported successfully: %s", final_onnx) + + param_path = "" + # param_path = dump_initializer_parameters(final_onnx) # 如需单独保存参数可解开 + + return final_onnx, param_path + + +def run_ort_validation( + wrapper: torch.nn.Module, + sample_inputs: OrderedDictType[str, torch.Tensor], + onnx_path: str, + provider: str, +) -> None: + try: + import onnxruntime as ort + except ImportError: # pragma: no cover + LOGGER.warning("onnxruntime not installed, skip validation") + return + + wrapper.eval() + with torch.inference_mode(): + torch_output = wrapper(*sample_inputs.values()).detach().cpu().numpy() + + sess_options = ort.SessionOptions() + session = ort.InferenceSession(onnx_path, sess_options=sess_options, providers=[provider]) + ort_inputs = {name: tensor.detach().cpu().numpy() for name, tensor in sample_inputs.items()} + ort_output = session.run(None, ort_inputs)[0] + + abs_diff = np.max(np.abs(torch_output - ort_output)) + rel_diff = abs_diff / (np.maximum(1.0, np.max(np.abs(torch_output)))) + LOGGER.info("ONNX Runtime check done (abs=%.6f, rel=%.6f)", abs_diff, rel_diff) + + +def main() -> None: + args = parse_args() + device = torch.device("cpu") + torch_dtype = torch.float16 if args.dtype == "fp16" else torch.float32 + + torch.set_grad_enabled(False) + transformer = load_transformer(args, torch_dtype, device) + wrapper = TransformerBodyOnlyWrapper(transformer, args.patch_size, args.f_patch_size) + sample_inputs = build_dummy_inputs(args, transformer, torch_dtype, device) + + maybe_save_calibration_inputs("transformer_body_only", sample_inputs, args) + + transformer_model_path, _ = export_onnx( + wrapper, + sample_inputs, + args.output, + ["sample"], + args, + ) + + if not args.skip_ort_check: + try: + run_ort_validation(wrapper, sample_inputs, transformer_model_path, args.ort_provider) + except Exception as exc: # pragma: no cover + LOGGER.warning("ONNX Runtime validation failed: %s", exc) + + +if __name__ == "__main__": + """ + 示例: + python scripts/z_image/export_transformer_body_onnx.py \ + --output onnx-models/z_image_transformer_body_only.onnx \ + --height 512 --width 512 --sequence-length 128 \ + --latent-downsample-factor 8 \ + --dtype fp32 \ + --skip-slim + """ + main() diff --git a/VideoX-Fun/scripts/z_image/train.py b/VideoX-Fun/scripts/z_image/train.py new file mode 100644 index 0000000000000000000000000000000000000000..205d7a3271cf8deb0b5c7cf77fb279d4f9e00a57 --- /dev/null +++ b/VideoX-Fun/scripts/z_image/train.py @@ -0,0 +1,1616 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys +from typing import (Any, Callable, Dict, List, NamedTuple, Optional, Tuple, + Union) + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + CLIPImageProcessor, + CLIPVisionModelWithProjection, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, Qwen3ForCausalLM, + QwenImageTransformer2DModel, + ZImageTransformer2DModel) +from videox_fun.pipeline import Flux2Pipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float: + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return float(mu) + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + text_encoder = None, + tokenizer = None, + max_sequence_length: int = 512, +) -> List[torch.FloatTensor]: + if isinstance(prompt, str): + prompt = [prompt] + + for i, prompt_item in enumerate(prompt): + messages = [ + {"role": "user", "content": prompt_item}, + ] + prompt_item = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + prompt[i] = prompt_item + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] + + embeddings_list = [] + + for i in range(len(prompt_embeds)): + embeddings_list.append(prompt_embeds[i][prompt_masks[i]]) + + return embeddings_list + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = ZImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + pipeline = Flux2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--prompt_template_encode", + type=str, + default="<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", + help=( + 'The prompt template for text encoder.' + ), + ) + parser.add_argument( + "--prompt_template_encode_start_idx", + type=int, + default=34, + help=( + 'The start idx for prompt template.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So Qwen3ForCausalLM and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = Qwen3ForCausalLM.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKL.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + + # Get Transformer + transformer3d = ZImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = ZImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=ZImageTransformer2DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = ZImageTransformer2DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = ZImageTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + ) + load_model = EMAModel(load_model.parameters(), model_cls=ZImageTransformer2DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = ZImageTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["text"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_embeds = encode_prompt( + batch['text'], device="cpu", + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + new_examples['prompt_embeds'] = prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.model.layers) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need】 + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = pixel_values.squeeze(1) + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values).unsqueeze(2) + else: + latents = _batch_encode_vae(pixel_values).unsqueeze(2) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['prompt_embeds'].to(dtype=latents.dtype, device=accelerator.device) + else: + with torch.no_grad(): + prompt_embeds = encode_prompt( + batch['text'], device=accelerator.device, + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, f, height, width = latents.size() + latents = ((latents - vae.config.shift_factor) * vae.config.scaling_factor).to(dtype=weight_dtype) + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + sigmas = np.linspace(1.0, 1 / args.train_sampling_steps, args.train_sampling_steps) + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.set_timesteps(sigmas=sigmas, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + timesteps = (1000 - timesteps) / 1000 + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + t=timesteps, + cap_feats=prompt_embeds, + )[0] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/z_image/train.sh b/VideoX-Fun/scripts/z_image/train.sh new file mode 100644 index 0000000000000000000000000000000000000000..b34f13ec8a6306e164556a36300f0010b7589943 --- /dev/null +++ b/VideoX-Fun/scripts/z_image/train.sh @@ -0,0 +1,32 @@ +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/z_image/train.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1024 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_z_image" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --trainable_modules "." \ No newline at end of file diff --git a/VideoX-Fun/scripts/z_image/train_lora.py b/VideoX-Fun/scripts/z_image/train_lora.py new file mode 100644 index 0000000000000000000000000000000000000000..0dbdd6cab0484188a62585aa918029ce717f980f --- /dev/null +++ b/VideoX-Fun/scripts/z_image/train_lora.py @@ -0,0 +1,1569 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys +from typing import (Any, Callable, Dict, List, NamedTuple, Optional, Tuple, + Union) + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoDataset, + ImageVideoSampler, + get_random_mask) +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoTokenizer, + CLIPImageProcessor, + CLIPVisionModelWithProjection, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, Qwen3ForCausalLM, + QwenImageTransformer2DModel, + ZImageTransformer2DModel) +from videox_fun.pipeline import Flux2Pipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid + +if is_wandb_available(): + import wandb + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float: + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return float(mu) + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + text_encoder = None, + tokenizer = None, + max_sequence_length: int = 512, +) -> List[torch.FloatTensor]: + if isinstance(prompt, str): + prompt = [prompt] + + for i, prompt_item in enumerate(prompt): + messages = [ + {"role": "user", "content": prompt_item}, + ] + prompt_item = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + prompt[i] = prompt_item + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] + + embeddings_list = [] + + for i in range(len(prompt_embeds)): + embeddings_list.append(prompt_embeds[i][prompt_masks[i]]) + + return embeddings_list + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = ZImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + pipeline = Flux2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + pipeline = merge_lora( + pipeline, None, 1, accelerator.device, state_dict=accelerator.unwrap_model(network).state_dict(), transformer_only=True + ) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--rank", + type=int, + default=128, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--network_alpha", + type=int, + default=64, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--use_peft_lora", action="store_true", help="Whether or not to use peft lora." + ) + parser.add_argument( + "--train_text_encoder", + action="store_true", + help="Whether to train the text encoder. If set, the text encoder should be float32 precision.", + ) + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + parser.add_argument("--save_state", action="store_true", help="Whether or not to save state.") + + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + parser.add_argument( + "--lora_skip_name", + type=str, + default=None, + help=("The module is not trained in loras. "), + ) + parser.add_argument( + "--target_name", + type=str, + default=None, + help=("The module is trained in loras. "), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So Qwen3ForCausalLM and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = Qwen3ForCausalLM.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + # Get Vae + vae = AutoencoderKL.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + + # Get Transformer + transformer3d = ZImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + # Lora will work with this... + if args.use_peft_lora: + from peft import (LoraConfig, get_peft_model_state_dict, + inject_adapter_in_model) + lora_config = LoraConfig(r=args.rank, lora_alpha=args.network_alpha, target_modules=args.target_name.split(",")) + transformer3d = inject_adapter_in_model(lora_config, transformer3d) + + network = None + else: + network = create_network( + 1.0, + args.rank, + args.network_alpha, + text_encoder, + transformer3d, + neuron_dropout=None, + target_name=args.target_name, + skip_name=args.lora_skip_name, + ) + network = network.to(weight_dtype) + network.apply_to(text_encoder, transformer3d, args.train_text_encoder and not args.training_with_video_token_length, True) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = {} + for key in accelerate_state_dict: + if "network" in key: + network_state_dict[key.replace("network.", "")] = accelerate_state_dict[key].to(weight_dtype) + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + network_state_dict = get_peft_model_state_dict(accelerator.unwrap_model(models[-1]), accelerate_state_dict) + else: + network_state_dict = accelerate_state_dict + save_file(network_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + safetensor_save_path = os.path.join(output_dir, f"lora_diffusion_pytorch_model.safetensors") + if args.use_peft_lora: + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(models[-1]))) + else: + save_model(safetensor_save_path, accelerator.unwrap_model(models[-1])) + + if not args.use_deepspeed: + for _ in range(len(weights)): + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + if args.use_peft_lora: + logging.info("Add peft parameters") + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + else: + logging.info("Add network parameters") + trainable_params = list(filter(lambda p: p.requires_grad, network.parameters())) + trainable_params_optim = network.prepare_optimizer_params(args.learning_rate / 2, args.learning_rate, args.learning_rate) + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["text"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + if args.fix_sample_size is not None: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + new_examples["pixel_values"].append(transform(pixel_values)) + new_examples["text"].append(example["text"]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_embeds = encode_prompt( + batch['text'], device="cpu", + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + new_examples['prompt_embeds'] = prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + # Prepare everything with our `accelerator`. + if args.use_peft_lora: + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + elif fsdp_stage != 0: + transformer3d.network = network + transformer3d = transformer3d.to(dtype=weight_dtype) + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + else: + network, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + network, optimizer, train_dataloader, lr_scheduler + ) + + if zero_stage != 0 and not args.use_peft_lora: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=list(transformer3d.layers)) + transformer3d = shard_fn(transformer3d) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.model.layers) + text_encoder = shard_fn(text_encoder) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + transformer3d.to(accelerator.device, dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + # function for saving/removing + def save_model(ckpt_file, unwrapped_nw): + os.makedirs(args.output_dir, exist_ok=True) + accelerator.print(f"\nsaving checkpoint: {ckpt_file}") + if isinstance(unwrapped_nw, dict): + from safetensors.torch import save_file + save_file(unwrapped_nw, ckpt_file, metadata={"format": "pt"}) + return ckpt_file + unwrapped_nw.save_weights(ckpt_file, weight_dtype, None) + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need】 + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, text) in enumerate(zip(pixel_values, texts)): + pixel_value = pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = pixel_values.squeeze(1) + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values).unsqueeze(2) + else: + latents = _batch_encode_vae(pixel_values).unsqueeze(2) + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['prompt_embeds'].to(dtype=latents.dtype, device=accelerator.device) + else: + with torch.no_grad(): + prompt_embeds = encode_prompt( + batch['text'], device=accelerator.device, + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, f, height, width = latents.size() + latents = ((latents - vae.config.shift_factor) * vae.config.scaling_factor).to(dtype=weight_dtype) + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.sigma_min = 0.0 + noise_scheduler.set_timesteps(args.train_sampling_steps, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + timesteps = (1000 - timesteps) / 1000 + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + t=timesteps, + cap_feats=prompt_embeds, + )[0] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(-noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(trainable_params, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation( + vae, + text_encoder, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + if not args.save_state: + if args.use_peft_lora: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, get_peft_model_state_dict(accelerator.unwrap_model(transformer3d))) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + safetensor_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}.safetensors") + save_model(safetensor_save_path, accelerator.unwrap_model(network)) + logger.info(f"Saved safetensor to {safetensor_save_path}") + else: + accelerator_save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(accelerator_save_path) + logger.info(f"Saved state to {accelerator_save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/z_image/train_lora.sh b/VideoX-Fun/scripts/z_image/train_lora.sh new file mode 100644 index 0000000000000000000000000000000000000000..cf5f0fdbb39ce62bdf27f26ac9a3e78a253aba0f --- /dev/null +++ b/VideoX-Fun/scripts/z_image/train_lora.sh @@ -0,0 +1,33 @@ +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/z_image/train_lora.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=1000 \ + --checkpointing_steps=50 \ + --learning_rate=1e-04 \ + --seed=42 \ + --output_dir="output_dir_z_image_lora" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --rank=64 \ + --network_alpha=64 \ + --target_name="to_q,to_k,to_v,ff.0,ff.2,ff_context.0,ff_context.2" \ + --use_peft_lora \ + --uniform_sampling \ No newline at end of file diff --git a/VideoX-Fun/scripts/z_image_fun/README_TRAIN.md b/VideoX-Fun/scripts/z_image_fun/README_TRAIN.md new file mode 100644 index 0000000000000000000000000000000000000000..b00e1862062a1f5cc4ca7aca7bad4340f3e0858c --- /dev/null +++ b/VideoX-Fun/scripts/z_image_fun/README_TRAIN.md @@ -0,0 +1,153 @@ +## Training Code + +We can choose whether to use deepspeed or fsdp in z_image, which can save a lot of video memory +. +The metadata_control.json is a little different from normal json in Z-Image, you need to add a control_file_path, and [DWPose](https://github.com/IDEA-Research/DWPose) is suggested as tool to generate control file. + +```json +[ + { + "file_path": "train/00000002.jpg", + "control_file_path": "control/00000002.jpg", + "text": "A group of young men in suits and sunglasses are walking down a city street.", + "type": "image" + }, + ..... +] +``` + +Some parameters in the sh file can be confusing, and they are explained in this document: + +- `enable_bucket` is used to enable bucket training. When enabled, the model does not crop the images at the center, but instead, it trains the entire images after grouping them into buckets based on resolution. +- `random_hw_adapt` is used to enable automatic height and width scaling for images. When `random_hw_adapt` is enabled, the training images will have their height and width set to `image_sample_size` as the maximum and `512` as the minimum. + - For example, when `random_hw_adapt` is enabled, `image_sample_size=1024`, the resolution of image inputs for training is `512x512` to `1024x1024` +- `resume_from_checkpoint` is used to set the training should be resumed from a previous checkpoint. Use a path or `"latest"` to automatically select the last available checkpoint. + +When train model with multi machines, please set the params as follows: +```sh +export MASTER_ADDR="your master address" +export MASTER_PORT=10086 +export WORLD_SIZE=1 # The number of machines +export NUM_PROCESS=8 # The number of processes, such as WORLD_SIZE * 8 +export RANK=0 # The rank of this machine + +accelerate launch --mixed_precision="bf16" --main_process_ip=$MASTER_ADDR --main_process_port=$MASTER_PORT --num_machines=$WORLD_SIZE --num_processes=$NUM_PROCESS --machine_rank=$RANK scripts/xxx/xxx.py +``` + +Without deepspeed: + +Training z_image without DeepSpeed may result in insufficient GPU memory. +```sh +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/z_image_fun/train_control.py \ + --config_path="config/z_image/z_image_control.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_z_image_control" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --transformer_path="models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union.safetensors" \ + --trainable_modules "control" +``` + +With Deepspeed Zero-2: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --use_deepspeed --deepspeed_config_file config/zero_stage2_config.json --deepspeed_multinode_launcher standard scripts/z_image_fun/train_control.py \ + --config_path="config/z_image/z_image_control.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_z_image_control" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --transformer_path="models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union.safetensors" \ + --trainable_modules "control" +``` + +With FSDP: + +```sh +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" --use_fsdp --fsdp_auto_wrap_policy TRANSFORMER_BASED_WRAP --fsdp_transformer_layer_cls_to_wrap BaseZImageTransformerBlock,ZImageControlTransformerBlock --fsdp_sharding_strategy "FULL_SHARD" --fsdp_state_dict_type=SHARDED_STATE_DICT --fsdp_backward_prefetch "BACKWARD_PRE" --fsdp_cpu_ram_efficient_loading False scripts/z_image_fun/train_control.py \ + --config_path="config/z_image/z_image_control.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_z_image_control" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --transformer_path="models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union.safetensors" \ + --trainable_modules "control" +``` \ No newline at end of file diff --git a/VideoX-Fun/scripts/z_image_fun/export_transformer_onnx.py b/VideoX-Fun/scripts/z_image_fun/export_transformer_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..17d82a033e89721e3a52395754330782815d104e --- /dev/null +++ b/VideoX-Fun/scripts/z_image_fun/export_transformer_onnx.py @@ -0,0 +1,703 @@ +#!/usr/bin/env python3 +"""Export the Z-Image control transformer to ONNX for inference.""" + +import argparse +import logging +import os +import sys +from collections import OrderedDict +from typing import Any, Dict, List, Optional, OrderedDict as OrderedDictType, Tuple + +import numpy as np +import torch +from omegaconf import OmegaConf + +from loguru import logger +import onnx +from onnx import numpy_helper +import subprocess + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +from videox_fun.models import ZImageControlTransformer2DModel # noqa: E402 +from videox_fun.models.z_image_transformer2d import pad_stack # noqa: E402 + +LOGGER = logging.getLogger("export_transformer_onnx") +logging.basicConfig(level=logging.INFO, format="[%(asctime)s] %(levelname)s: %(message)s") + +SEQ_MULTI_OF = 32 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Export the Z-Image control transformer to ONNX") + parser.add_argument("--config", default="config/z_image/z_image_control.yaml", help="Path to the YAML config used to build the transformer") + parser.add_argument("--model-root", default="models/Diffusion_Transformer/Z-Image-Turbo/", help="Directory that stores the original diffusers weights") + parser.add_argument("--checkpoint", default="models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union.safetensors", help="Optional fine-tuned checkpoint to load") + parser.add_argument("--output", default="onnx-models/z_image_control_transformer.onnx", help="Target ONNX file path") + parser.add_argument("--body-output", default="onnx-models/z_image_transformer_body.onnx", help="Path for the body-only ONNX when --split-control is enabled") + parser.add_argument("--control-output", default="onnx-models/z_image_controlnet.onnx", help="Path for the control-only ONNX when --split-control is enabled") + parser.add_argument("--height", type=int, default=864, help="Target image height used to derive latent resolution") + parser.add_argument("--width", type=int, default=496, help="Target image width used to derive latent resolution") + parser.add_argument("--batch-size", type=int, default=1, help="Batch size for the exported graph") + parser.add_argument("--sequence-length", type=int, default=512, help="Prompt embedding sequence length (must be a multiple of 32)") + parser.add_argument("--frames", type=int, default=1, help="Number of frames in the latent tensor") + parser.add_argument("--latent-downsample-factor", type=int, default=8, help="Downsampling ratio between spatial image size and latent size") + parser.add_argument("--latent-height", type=int, default=None, help="Override latent height (after downsampling)") + parser.add_argument("--latent-width", type=int, default=None, help="Override latent width (after downsampling)") + parser.add_argument("--dtype", choices=["fp16", "fp32"], default="fp16", help="Export precision") + parser.add_argument("--control-scale", type=float, default=0.75, help="Default control context scale input") + parser.add_argument("--patch-size", type=int, default=2, help="Spatial patch size used by the transformer") + parser.add_argument("--f-patch-size", type=int, default=1, help="Frame patch size used by the transformer") + parser.add_argument("--opset", type=int, default=17, help="ONNX opset version") + parser.add_argument("--no-external-data", action="store_true", help="Disable external data format even if the model is larger than 2GB") + parser.add_argument("--skip-ort-check", action="store_true", help="Skip running an ONNX Runtime correctness check") + parser.add_argument("--ort-provider", default="CPUExecutionProvider", help="ONNX Runtime provider used during validation") + parser.add_argument("--split-control", action="store_true", help="Export transformer body and ControlNet separately instead of a fused model") + parser.add_argument("--save-calib-inputs", action="store_true", help="Dump ONNX input dictionaries as .npy for calibration") + parser.add_argument("--calib-dir", default="onnx-calibration", help="Directory for storing calibration npy files") + parser.add_argument("--dynamic-axes", action="store_true", help="Export ONNX with dynamic batch/seq/latent dims; default is static shape") + parser.add_argument("--skip-slim", action="store_true", help="Skip onnxslim simplification for faster debug export") + return parser.parse_args() + + +def run_onnxslim(input_file="vae.onnx", output_file="vae_slim.onnx"): + """ + 执行 onnxslim 命令压缩 ONNX 模型 + """ + try: + # 使用完整的命令路径（如果知道的话） + cmd = ["onnxslim", input_file, output_file] + + print(f"执行命令: {' '.join(cmd)}") + + # 执行命令, 实时输出 + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + universal_newlines=True + ) + + # 实时打印输出 + for line in process.stdout: + print(line, end='') + + # 等待命令完成 + stdout, stderr = process.communicate() + + if process.returncode != 0: + print(f"命令执行失败, 错误信息:\n{stderr}") + return False + else: + print("ONNX模型压缩完成!") + return True + + except FileNotFoundError: + print("错误: 未找到 onnxslim 命令, 请确保已安装 onnxslim") + print("安装方法: pip install onnx-simplifier") + return False + except Exception as e: + print(f"执行命令时发生错误: {e}") + return False + + +def _resolve_path(path: str) -> str: + return os.path.abspath(os.path.join(REPO_ROOT, path)) if not os.path.isabs(path) else path + + +def load_transformer(args: argparse.Namespace, torch_dtype: torch.dtype, device: torch.device) -> ZImageControlTransformer2DModel: + config_path = _resolve_path(args.config) + model_root = _resolve_path(args.model_root) + checkpoint_path = _resolve_path(args.checkpoint) if args.checkpoint else None + + if not os.path.exists(config_path): + raise FileNotFoundError(f"Config not found: {config_path}") + if not os.path.isdir(model_root): + raise FileNotFoundError(f"Model root not found: {model_root}") + + config = OmegaConf.load(config_path) + transformer_kwargs = OmegaConf.to_container(config.get("transformer_additional_kwargs", {}), resolve=True) + + LOGGER.info("Loading transformer from %s", model_root) + transformer = ZImageControlTransformer2DModel.from_pretrained( + model_root, + subfolder="transformer", + low_cpu_mem_usage=True, + torch_dtype=torch_dtype, + transformer_additional_kwargs=transformer_kwargs, + ) + transformer.eval() + transformer.to(device=device, dtype=torch_dtype) + + if checkpoint_path and os.path.exists(checkpoint_path): + LOGGER.info("Loading checkpoint %s", checkpoint_path) + if checkpoint_path.endswith(".safetensors"): + from safetensors.torch import load_file # type: ignore + + state_dict = load_file(checkpoint_path) + else: + state_dict = torch.load(checkpoint_path, map_location="cpu") + state_dict = state_dict.get("state_dict", state_dict) + missing, unexpected = transformer.load_state_dict(state_dict, strict=False) + LOGGER.info("Checkpoint loaded (missing=%d, unexpected=%d)", len(missing), len(unexpected)) + elif checkpoint_path: + raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}") + + return transformer + + +def _tensor_batch_to_list(batch_tensor: torch.Tensor) -> List[torch.Tensor]: + return [batch_tensor[i] for i in range(batch_tensor.shape[0])] + + +def _prepare_transformer_state( + model: ZImageControlTransformer2DModel, + latent_list: List[torch.Tensor], + prompt_list: List[torch.Tensor], + timestep: torch.Tensor, + patch_size: int, + f_patch_size: int, +) -> Dict[str, Any]: + bsz = len(latent_list) + device = latent_list[0].device + timestep = timestep.to(device=device, dtype=torch.float32) + t = timestep * model.t_scale + t = model.t_embedder(t) + + ( + x, + cap_feats, + x_size, + x_pos_ids, + cap_pos_ids, + x_inner_pad_mask, + cap_inner_pad_mask, + ) = model.patchify_and_embed(latent_list, prompt_list, patch_size, f_patch_size) + + # Latent tokens refinement + x_item_seqlens = [len(_) for _ in x] + assert all(_ % SEQ_MULTI_OF == 0 for _ in x_item_seqlens) + x_max_item_seqlen = max(x_item_seqlens) + + x = torch.cat(x, dim=0) + x = model.all_x_embedder[f"{patch_size}-{f_patch_size}"](x) + adaln_input = t.type_as(x) + + mask = torch.cat(x_inner_pad_mask) + if torch.onnx.is_in_onnx_export(): + if model.x_pad_token.dim() == 1: + x_pad_token_2d = model.x_pad_token.unsqueeze(0) + else: + x_pad_token_2d = model.x_pad_token + mask_2d = mask.unsqueeze(1) + x_pad_expanded = x_pad_token_2d.expand_as(x) + x = torch.where(mask_2d, x_pad_expanded, x) + else: + x[mask] = model.x_pad_token + + x = list(x.split(x_item_seqlens, dim=0)) + x_freqs_cis = list(model.rope_embedder(torch.cat(x_pos_ids, dim=0)).split(x_item_seqlens, dim=0)) + + x = pad_stack(x, x_max_item_seqlen, pad_value=0.0) + x_freqs_cis = pad_stack(x_freqs_cis, x_max_item_seqlen, pad_value=0.0) + x_attn_mask = torch.zeros((bsz, x_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(x_item_seqlens): + x_attn_mask[i, :seq_len] = 1 + + for layer in model.noise_refiner: + x = layer(x, x_attn_mask, x_freqs_cis, adaln_input) + + # Caption refinement + cap_item_seqlens = [len(_) for _ in cap_feats] + assert all(_ % SEQ_MULTI_OF == 0 for _ in cap_item_seqlens) + cap_max_item_seqlen = max(cap_item_seqlens) + + cap_feats = torch.cat(cap_feats, dim=0) + cap_feats = model.cap_embedder(cap_feats) + + cap_mask = torch.cat(cap_inner_pad_mask) + if torch.onnx.is_in_onnx_export(): + if model.cap_pad_token.dim() == 1: + cap_pad_token_2d = model.cap_pad_token.unsqueeze(0) + else: + cap_pad_token_2d = model.cap_pad_token + mask_2d = cap_mask.unsqueeze(1) + cap_pad_expanded = cap_pad_token_2d.expand_as(cap_feats) + cap_feats = torch.where(mask_2d, cap_pad_expanded, cap_feats) + else: + cap_feats[cap_mask] = model.cap_pad_token + + cap_feats = list(cap_feats.split(cap_item_seqlens, dim=0)) + cap_freqs_cis = list(model.rope_embedder(torch.cat(cap_pos_ids, dim=0)).split(cap_item_seqlens, dim=0)) + + cap_feats = pad_stack(cap_feats, cap_max_item_seqlen, pad_value=0.0) + cap_freqs_cis = pad_stack(cap_freqs_cis, cap_max_item_seqlen, pad_value=0.0) + cap_attn_mask = torch.zeros((bsz, cap_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(cap_item_seqlens): + cap_attn_mask[i, :seq_len] = 1 + + for layer in model.context_refiner: + cap_feats = layer(cap_feats, cap_attn_mask, cap_freqs_cis) + + # Context parallel handling + if model.sp_world_size > 1: + x = torch.chunk(x, model.sp_world_size, dim=1)[model.sp_world_rank] + x_item_seqlens = [len(_) for _ in x] + x_max_item_seqlen = max(x_item_seqlens) + x_attn_mask = torch.zeros((bsz, x_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(x_item_seqlens): + x_attn_mask[i, :seq_len] = 1 + + if x_freqs_cis is not None: + x_freqs_cis = torch.chunk(x_freqs_cis, model.sp_world_size, dim=1)[model.sp_world_rank] + + unified = [] + unified_freqs_cis = [] + for i in range(bsz): + x_len = x_item_seqlens[i] + cap_len = cap_item_seqlens[i] + unified.append(torch.cat([x[i][:x_len], cap_feats[i][:cap_len]])) + unified_freqs_cis.append(torch.cat([x_freqs_cis[i][:x_len], cap_freqs_cis[i][:cap_len]])) + unified_item_seqlens = [a + b for a, b in zip(cap_item_seqlens, x_item_seqlens)] + unified_max_item_seqlen = max(unified_item_seqlens) + + unified = pad_stack(unified, unified_max_item_seqlen, pad_value=0.0) + unified_freqs_cis = pad_stack(unified_freqs_cis, unified_max_item_seqlen, pad_value=0.0) + unified_attn_mask = torch.zeros((bsz, unified_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(unified_item_seqlens): + unified_attn_mask[i, :seq_len] = 1 + + kwargs = dict(attn_mask=unified_attn_mask, freqs_cis=unified_freqs_cis, adaln_input=adaln_input) + + return dict( + x=x, + cap_feats=cap_feats, + unified=unified, + kwargs=kwargs, + adaln_input=adaln_input, + time_embed=t, + x_item_seqlens=x_item_seqlens, + cap_item_seqlens=cap_item_seqlens, + x_size=x_size, + unified_attn_mask=unified_attn_mask, + unified_freqs_cis=unified_freqs_cis, + bsz=bsz, + ) + + +class TransformerOnnxWrapper(torch.nn.Module): + """Lightweight wrapper that exposes tensor inputs for ONNX export.""" + + def __init__(self, model: ZImageControlTransformer2DModel, patch_size: int, f_patch_size: int): + super().__init__() + self.model = model + self.patch_size = patch_size + self.f_patch_size = f_patch_size + + def forward( + self, + latent_model_input: torch.Tensor, + timestep: torch.Tensor, + prompt_embeds: torch.Tensor, + control_context: torch.Tensor, + control_context_scale: torch.Tensor, + ) -> torch.Tensor: + latents = list(latent_model_input.unbind(dim=0)) + prompts = list(prompt_embeds.unbind(dim=0)) + scale = control_context_scale.to(device=latent_model_input.device, dtype=latent_model_input.dtype) + outputs, _ = self.model( + latents, + timestep, + prompts, + patch_size=self.patch_size, + f_patch_size=self.f_patch_size, + control_context=control_context, + control_context_scale=scale, + ) + return outputs + + +class ControlNetWrapper(torch.nn.Module): + """Exports only the control branch so it can be invoked on demand.""" + + def __init__(self, model: ZImageControlTransformer2DModel, patch_size: int, f_patch_size: int): + super().__init__() + self.model = model + self.patch_size = patch_size + self.f_patch_size = f_patch_size + + def forward( + self, + latent_model_input: torch.Tensor, + timestep: torch.Tensor, + prompt_embeds: torch.Tensor, + control_context: torch.Tensor, + ) -> torch.Tensor: + latents = _tensor_batch_to_list(latent_model_input) + prompts = _tensor_batch_to_list(prompt_embeds) + control_list = _tensor_batch_to_list(control_context) + + state = _prepare_transformer_state( + self.model, + latents, + prompts, + timestep, + self.patch_size, + self.f_patch_size, + ) + + hints = self.model.forward_control( + state["unified"], + state["cap_feats"], + control_list, + state["kwargs"], + t=state["time_embed"], + patch_size=self.patch_size, + f_patch_size=self.f_patch_size, + ) + return torch.stack(hints, dim=0) + + +class TransformerBodyWrapper(torch.nn.Module): + """Exports the transformer body which consumes precomputed control hints.""" + + def __init__(self, model: ZImageControlTransformer2DModel, patch_size: int, f_patch_size: int): + super().__init__() + self.model = model + self.patch_size = patch_size + self.f_patch_size = f_patch_size + + def forward( + self, + latent_model_input: torch.Tensor, + timestep: torch.Tensor, + prompt_embeds: torch.Tensor, + control_hints: torch.Tensor, + control_context_scale: torch.Tensor, + ) -> torch.Tensor: + latents = _tensor_batch_to_list(latent_model_input) + prompts = _tensor_batch_to_list(prompt_embeds) + hints_list = list(torch.unbind(control_hints, dim=0)) + scale = control_context_scale.to(device=latent_model_input.device, dtype=latent_model_input.dtype) + + state = _prepare_transformer_state( + self.model, + latents, + prompts, + timestep, + self.patch_size, + self.f_patch_size, + ) + + unified = state["unified"] + for layer in self.model.layers: + layer_kwargs = dict( + attn_mask=state["unified_attn_mask"], + freqs_cis=state["unified_freqs_cis"], + adaln_input=state["adaln_input"], + hints=hints_list, + context_scale=scale, + ) + unified = layer(unified, **layer_kwargs) + + if self.model.sp_world_size > 1: + unified_out = [] + for i in range(state["bsz"]): + x_len = state["x_item_seqlens"][i] + unified_out.append(unified[i, :x_len]) + unified = torch.stack(unified_out) + unified = self.model.all_gather(unified, dim=1) + + final_layer = self.model.all_final_layer[f"{self.patch_size}-{self.f_patch_size}"] + unified = final_layer(unified, state["adaln_input"]) + unified = list(unified.unbind(dim=0)) + x = self.model.unpatchify(unified, state["x_size"], self.patch_size, self.f_patch_size) + x = torch.stack(x) + return x + + +def _validate_sequence_length(seq_len: int) -> None: + if seq_len % 32 != 0: + raise ValueError("sequence_length must be a multiple of 32 to satisfy transformer padding rules") + + +def _compute_latent_dims(args: argparse.Namespace) -> Dict[str, int]: + if args.latent_height is not None and args.latent_width is not None: + latent_h = args.latent_height + latent_w = args.latent_width + else: + if args.height % args.latent_downsample_factor != 0 or args.width % args.latent_downsample_factor != 0: + raise ValueError("height and width must be divisible by latent_downsample_factor") + latent_h = args.height // args.latent_downsample_factor + latent_w = args.width // args.latent_downsample_factor + if latent_h % args.patch_size != 0 or latent_w % args.patch_size != 0: + raise ValueError("latent dimensions must be divisible by patch_size") + if args.frames % args.f_patch_size != 0: + raise ValueError("frames must be divisible by f_patch_size") + return {"latent_h": latent_h, "latent_w": latent_w} + + +def build_dummy_inputs( + args: argparse.Namespace, + model: ZImageControlTransformer2DModel, + torch_dtype: torch.dtype, + device: torch.device, +) -> OrderedDictType[str, torch.Tensor]: + _validate_sequence_length(args.sequence_length) + dims = _compute_latent_dims(args) + batch = args.batch_size + in_channels = model.config.in_channels + cap_dim = model.config.cap_feat_dim + + latent = torch.randn( + batch, + in_channels, + args.frames, + dims["latent_h"], + dims["latent_w"], + dtype=torch_dtype, + device=device, + ) + timestep = torch.linspace(0.0, 1.0, steps=batch, dtype=torch.float32, device=device) + prompts = torch.randn( + batch, + args.sequence_length, + cap_dim, + dtype=torch_dtype, + device=device, + ) + control = torch.randn_like(latent) + control_scale = torch.full((1,), args.control_scale, dtype=torch.float32, device=device) + + return OrderedDict( + latent_model_input=latent, + timestep=timestep, + prompt_embeds=prompts, + control_context=control, + control_context_scale=control_scale, + ) + + +def maybe_save_calibration_inputs(tag: str, inputs: OrderedDictType[str, torch.Tensor], args: argparse.Namespace) -> Optional[str]: + if not getattr(args, "save_calib_inputs", False): + return None + output_dir = _resolve_path(args.calib_dir) + os.makedirs(output_dir, exist_ok=True) + numpy_dict = {name: tensor.detach().cpu().numpy() for name, tensor in inputs.items()} + file_path = os.path.join(output_dir, f"{tag}_inputs.npy") + np.save(file_path, numpy_dict, allow_pickle=True) + LOGGER.info("Saved calibration inputs (%s) to %s", tag, file_path) + return file_path + + +def dump_initializer_parameters(model_path: str) -> str: + """Save all ONNX initializers into a standalone .npz file.""" + model_proto = onnx.load(model_path, load_external_data=True) + param_dict = {} + for initializer in model_proto.graph.initializer: + param_dict[initializer.name] = numpy_helper.to_array(initializer) + param_path = f"{model_path}.params.npz" + np.savez(param_path, **param_dict) + LOGGER.info("Saved %d parameters to %s", len(param_dict), param_path) + return param_path + + +def export_onnx( + wrapper: torch.nn.Module, + sample_inputs: OrderedDictType[str, torch.Tensor], + output_path: str, + output_names: List[str], + args: argparse.Namespace, +) -> Tuple[str, str]: + export_path = _resolve_path(output_path) + export_dir = os.path.dirname(export_path) + if export_dir: + os.makedirs(export_dir, exist_ok=True) + input_names = list(sample_inputs.keys()) + use_external = not args.no_external_data + wrapper.eval() + + dynamic_axes = None + if args.dynamic_axes: + dynamic_axes = { + "latent_model_input": {0: "batch", 2: "frames", 3: "latent_h", 4: "latent_w"}, + "prompt_embeds": {0: "batch", 1: "seq_len"}, + "timestep": {0: "batch"}, + "control_context": {0: "batch", 2: "frames", 3: "latent_h", 4: "latent_w"}, + "control_hints": {0: "batch", 2: "frames", 3: "latent_h", 4: "latent_w"}, + "control_context_scale": {0: "scale_batch"}, + "sample": {0: "batch", 2: "frames", 3: "latent_h", 4: "latent_w"}, + "hints": {0: "batch", 2: "frames", 3: "latent_h", 4: "latent_w"}, + } + + LOGGER.info("Exporting ONNX to %s", export_path) + with torch.inference_mode(): + torch.onnx.export( + wrapper, + args=tuple(sample_inputs[name] for name in input_names), + f=export_path, + input_names=input_names, + output_names=output_names, + opset_version=args.opset, + do_constant_folding=True, + export_params=True, + dynamic_axes={k: v for k, v in dynamic_axes.items() if k in input_names + output_names} if dynamic_axes else None, + # use_external_data_format=use_external, + ) + + LOGGER.info("Raw ONNX export finished") + + trans_onnx = onnx.load(export_path) + simp_onnx_data = os.path.splitext(export_path)[0] + "_simp.onnx" + onnx.save( + trans_onnx, + simp_onnx_data, + save_as_external_data=True, + all_tensors_to_one_file=True, + ) + external_weight_file = simp_onnx_data + ".data" + LOGGER.info("Saved external-data ONNX to %s (weights -> %s)", simp_onnx_data, external_weight_file) + + if args.skip_slim: + LOGGER.info("Skip onnxslim as requested, using simplified external-data ONNX: %s", simp_onnx_data) + final_onnx = simp_onnx_data + else: + slim_onnx_path = os.path.splitext(simp_onnx_data)[0] + "_slim.onnx" + LOGGER.info("Transformer ONNX model exported, start to simplify via onnxslim") + success = run_onnxslim(simp_onnx_data, slim_onnx_path) + if not success: + raise RuntimeError("onnxslim simplification failed, please check logs") + final_onnx = slim_onnx_path + LOGGER.info("Transformer ONNX model exported successfully: %s", final_onnx) + + param_path = "" + # param_path = dump_initializer_parameters(final_onnx) # 当前看来无必要保存 + + return final_onnx, param_path + + +def run_ort_validation( + wrapper: torch.nn.Module, + sample_inputs: OrderedDictType[str, torch.Tensor], + onnx_path: str, + provider: str, +) -> None: + try: + import onnxruntime as ort + except ImportError: # pragma: no cover + LOGGER.warning("onnxruntime not installed, skip validation") + return + + wrapper.eval() + with torch.inference_mode(): + torch_output = wrapper(*sample_inputs.values()).detach().cpu().numpy() + + sess_options = ort.SessionOptions() + session = ort.InferenceSession(onnx_path, sess_options=sess_options, providers=[provider]) + ort_inputs = { + name: tensor.detach().cpu().numpy() + for name, tensor in sample_inputs.items() + } + ort_output = session.run(None, ort_inputs)[0] + + abs_diff = np.max(np.abs(torch_output - ort_output)) + rel_diff = abs_diff / (np.maximum(1.0, np.max(np.abs(torch_output)))) + LOGGER.info("ONNX Runtime check done (abs=%.6f, rel=%.6f)", abs_diff, rel_diff) + + +def main() -> None: + args = parse_args() + device = torch.device("cpu") + torch_dtype = torch.float16 if args.dtype == "fp16" else torch.float32 + + torch.set_grad_enabled(False) + transformer = load_transformer(args, torch_dtype, device) + sample_inputs = build_dummy_inputs(args, transformer, torch_dtype, device) + + if args.split_control: + control_wrapper = ControlNetWrapper(transformer, args.patch_size, args.f_patch_size) + body_wrapper = TransformerBodyWrapper(transformer, args.patch_size, args.f_patch_size) + + control_inputs = OrderedDict( + latent_model_input=sample_inputs["latent_model_input"], + timestep=sample_inputs["timestep"], + prompt_embeds=sample_inputs["prompt_embeds"], + control_context=sample_inputs["control_context"], + ) + + maybe_save_calibration_inputs("controlnet", control_inputs, args) + + with torch.inference_mode(): + control_hints_sample = control_wrapper(*control_inputs.values()).detach() + + control_model_path, _ = export_onnx( + control_wrapper, + control_inputs, + args.control_output, + ["hints"], + args, + ) + + body_inputs = OrderedDict( + latent_model_input=sample_inputs["latent_model_input"], + timestep=sample_inputs["timestep"], + prompt_embeds=sample_inputs["prompt_embeds"], + control_hints=control_hints_sample, + control_context_scale=sample_inputs["control_context_scale"], + ) + + maybe_save_calibration_inputs("transformer_body", body_inputs, args) + + body_model_path, _ = export_onnx( + body_wrapper, + body_inputs, + args.body_output, + ["sample"], + args, + ) + + if not args.skip_ort_check: + try: + run_ort_validation(control_wrapper, control_inputs, control_model_path, args.ort_provider) + run_ort_validation(body_wrapper, body_inputs, body_model_path, args.ort_provider) + except Exception as exc: # pragma: no cover + LOGGER.warning("ONNX Runtime validation failed: %s", exc) + else: + wrapper = TransformerOnnxWrapper(transformer, args.patch_size, args.f_patch_size) + + maybe_save_calibration_inputs("transformer", sample_inputs, args) + + transformer_model_path, _ = export_onnx( + wrapper, + sample_inputs, + args.output, + ["sample"], + args, + ) + + if not args.skip_ort_check: + try: + run_ort_validation(wrapper, sample_inputs, transformer_model_path, args.ort_provider) + except Exception as exc: # pragma: no cover + LOGGER.warning("ONNX Runtime validation failed: %s", exc) + + +if __name__ == "__main__": + """ + python scripts/z_image_fun/export_transformer_onnx.py \ + --split-control \ + --control-output onnx-models/z_image_controlnet.onnx \ + --body-output onnx-models/z_image_transformer_body.onnx \ + --save-calib-inputs \ + --height 512 \ + --width 512 \ + --sequence-length 128 \ + --latent-downsample-factor 8 \ + --skip-slim \ + --dtype fp32 + """ + main() diff --git a/VideoX-Fun/scripts/z_image_fun/export_vae_onnx.py b/VideoX-Fun/scripts/z_image_fun/export_vae_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..fb15ac33ef4e00b9cf59471dbe5fe2be11b5052b --- /dev/null +++ b/VideoX-Fun/scripts/z_image_fun/export_vae_onnx.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +"""导出 VAE Encoder/Decoder 的 ONNX 模型。""" + +import argparse +import logging +import os +import sys +from collections import OrderedDict +from typing import Any, Dict, Optional + +import numpy as np +import torch +from loguru import logger +import onnx +from onnx import numpy_helper +import subprocess + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) + +from videox_fun.models import AutoencoderKL # noqa: E402 + +LOGGER = logging.getLogger("export_vae_onnx") +logging.basicConfig(level=logging.INFO, format="[%(asctime)s] %(levelname)s: %(message)s") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Export VAE encoder/decoder to ONNX") + parser.add_argument("--model-root", default="models/Diffusion_Transformer/Z-Image-Turbo/", help="Diffusers 权重所在目录") + parser.add_argument("--checkpoint", default=None, help="可选的 VAE finetune checkpoint") + parser.add_argument("--encoder-output", default="onnx-models/vae_encoder.onnx", help="VAE Encoder ONNX 路径") + parser.add_argument("--decoder-output", default="onnx-models/vae_decoder.onnx", help="VAE Decoder ONNX 路径") + parser.add_argument("--height", type=int, default=864, help="导出时的图片高度") + parser.add_argument("--width", type=int, default=496, help="导出时的图片宽度") + parser.add_argument("--latent-downsample-factor", type=int, default=8, help="VAE 下采样倍数") + parser.add_argument("--batch-size", type=int, default=1, help="导出 batch 大小") + parser.add_argument("--dtype", choices=["fp16", "fp32"], default="fp16", help="导出精度") + parser.add_argument("--opset", type=int, default=17, help="ONNX opset 版本") + parser.add_argument("--dynamic-axes", action="store_true", help="是否导出动态维度") + parser.add_argument("--skip-ort-check", action="store_true", help="跳过 onnxruntime 结果校验") + parser.add_argument("--ort-provider", default="CPUExecutionProvider", help="onnxruntime provider") + parser.add_argument("--skip-slim", action="store_true", help="跳过 onnxslim") + parser.add_argument("--no-external-data", action="store_true", help="禁用外部数据格式保存") + parser.add_argument("--save-calib-inputs", action="store_true", help="保存校准输入 npy") + parser.add_argument("--calib-dir", default="onnx-calibration", help="校准输入保存目录") + return parser.parse_args() + + +def run_onnxslim(input_file: str, output_file: str) -> bool: + try: + cmd = ["onnxslim", input_file, output_file] + print(f"执行命令: {' '.join(cmd)}") + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + universal_newlines=True, + ) + for line in process.stdout: + print(line, end="") + _, stderr = process.communicate() + if process.returncode != 0: + print(f"命令执行失败, 错误信息:\n{stderr}") + return False + print("ONNX模型压缩完成!") + return True + except FileNotFoundError: + print("错误: 未找到 onnxslim 命令, 请确保已安装 onnxslim (pip install onnx-simplifier)") + return False + except Exception as exc: + print(f"执行命令时发生错误: {exc}") + return False + + +def _resolve_path(path: str) -> str: + return os.path.abspath(os.path.join(REPO_ROOT, path)) if not os.path.isabs(path) else path + + +def _check_image_dims(height: int, width: int, factor: int) -> None: + if height % factor != 0 or width % factor != 0: + raise ValueError("height 和 width 需要能被 latent_downsample_factor 整除") + + +def _compute_latent_dims(height: int, width: int, factor: int) -> Dict[str, int]: + _check_image_dims(height, width, factor) + return {"latent_h": height // factor, "latent_w": width // factor} + + +def load_vae(args: argparse.Namespace, torch_dtype: torch.dtype, device: torch.device) -> AutoencoderKL: + model_root = _resolve_path(args.model_root) + checkpoint_path = _resolve_path(args.checkpoint) if args.checkpoint else None + if not os.path.isdir(model_root): + raise FileNotFoundError(f"Model root not found: {model_root}") + + LOGGER.info("Loading VAE from %s", model_root) + vae = AutoencoderKL.from_pretrained( + model_root, + subfolder="vae", + torch_dtype=torch_dtype, + low_cpu_mem_usage=True, + ) + vae.to(device=device, dtype=torch_dtype) + vae.eval() + + if checkpoint_path: + if not os.path.exists(checkpoint_path): + raise FileNotFoundError(f"Checkpoint not found: {checkpoint_path}") + LOGGER.info("Loading checkpoint %s", checkpoint_path) + if checkpoint_path.endswith(".safetensors"): + from safetensors.torch import load_file # type: ignore + + state_dict = load_file(checkpoint_path) + else: + state_dict = torch.load(checkpoint_path, map_location="cpu") + state_dict = state_dict.get("state_dict", state_dict) + missing, unexpected = vae.load_state_dict(state_dict, strict=False) + LOGGER.info("Checkpoint loaded (missing=%d, unexpected=%d)", len(missing), len(unexpected)) + + return vae + + +def build_dummy_inputs(args: argparse.Namespace, vae: AutoencoderKL, torch_dtype: torch.dtype, device: torch.device) -> OrderedDict: + dims = _compute_latent_dims(args.height, args.width, args.latent_downsample_factor) + pixel_values = torch.randn( + args.batch_size, + 3, + args.height, + args.width, + dtype=torch_dtype, + device=device, + ) + latents = torch.randn( + args.batch_size, + vae.config.latent_channels, + dims["latent_h"], + dims["latent_w"], + dtype=torch_dtype, + device=device, + ) + return OrderedDict(pixel_values=pixel_values, latent=latents) + + +class VAEEncoderWrapper(torch.nn.Module): + def __init__(self, model: AutoencoderKL): + super().__init__() + self.model = model + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + latent_dist = self.model.encode(pixel_values)[0] + return latent_dist.mode() + + +class VAEDecoderWrapper(torch.nn.Module): + def __init__(self, model: AutoencoderKL): + super().__init__() + self.model = model + + def forward(self, latents: torch.Tensor) -> torch.Tensor: + image = self.model.decode(latents, return_dict=False)[0] + return image + + +def maybe_save_calibration_inputs(tag: str, inputs: OrderedDict, args: argparse.Namespace) -> Optional[str]: + if not getattr(args, "save_calib_inputs", False): + return None + output_dir = _resolve_path(args.calib_dir) + os.makedirs(output_dir, exist_ok=True) + numpy_dict = {name: tensor.detach().cpu().numpy() for name, tensor in inputs.items()} + file_path = os.path.join(output_dir, f"{tag}_inputs.npy") + np.save(file_path, numpy_dict, allow_pickle=True) + LOGGER.info("Saved calibration inputs (%s) to %s", tag, file_path) + return file_path + + +def dump_initializer_parameters(model_path: str) -> str: + model_proto = onnx.load(model_path, load_external_data=True) + param_dict: Dict[str, Any] = {} + for initializer in model_proto.graph.initializer: + param_dict[initializer.name] = numpy_helper.to_array(initializer) + param_path = f"{model_path}.params.npz" + np.savez(param_path, **param_dict) + LOGGER.info("Saved %d parameters to %s", len(param_dict), param_path) + return param_path + + +def export_onnx( + wrapper: torch.nn.Module, + sample_inputs: OrderedDict, + output_path: str, + output_names: list, + args: argparse.Namespace, +) -> str: + export_path = _resolve_path(output_path) + export_dir = os.path.dirname(export_path) + if export_dir: + os.makedirs(export_dir, exist_ok=True) + + input_names = list(sample_inputs.keys()) + wrapper.eval() + + dynamic_axes = None + if args.dynamic_axes: + dynamic_axes = { + "pixel_values": {0: "batch", 2: "height", 3: "width"}, + "latents": {0: "batch", 2: "latent_h", 3: "latent_w"}, + "images": {0: "batch", 2: "height", 3: "width"}, + } + + LOGGER.info("Exporting ONNX to %s", export_path) + with torch.inference_mode(): + torch.onnx.export( + wrapper, + args=tuple(sample_inputs[name] for name in input_names), + f=export_path, + input_names=input_names, + output_names=output_names, + opset_version=args.opset, + do_constant_folding=True, + export_params=True, + dynamic_axes={k: v for k, v in (dynamic_axes or {}).items() if k in input_names + output_names} if dynamic_axes else None, + # use_external_data_format=not args.no_external_data, + ) + + LOGGER.info("Raw ONNX export finished") + + onnx_model = onnx.load(export_path) + simp_onnx_path = os.path.splitext(export_path)[0] + "_simp.onnx" + onnx.save( + onnx_model, + simp_onnx_path, + save_as_external_data=True, + all_tensors_to_one_file=True, + ) + LOGGER.info("Saved external-data ONNX to %s", simp_onnx_path) + + if args.skip_slim: + LOGGER.info("Skip onnxslim as requested") + final_path = simp_onnx_path + else: + slim_path = os.path.splitext(simp_onnx_path)[0] + "_slim.onnx" + LOGGER.info("Start onnxslim simplification") + success = run_onnxslim(simp_onnx_path, slim_path) + if not success: + raise RuntimeError("onnxslim simplification failed") + final_path = slim_path + LOGGER.info("onnxslim done: %s", final_path) + + # dump_initializer_parameters(final_path) + return final_path + + +def run_ort_validation(wrapper: torch.nn.Module, sample_inputs: OrderedDict, onnx_path: str, provider: str) -> None: + try: + import onnxruntime as ort + except ImportError: + LOGGER.warning("onnxruntime not installed, skip validation") + return + + wrapper.eval() + with torch.inference_mode(): + torch_output = wrapper(*sample_inputs.values()).detach().cpu().numpy() + + session = ort.InferenceSession(onnx_path, providers=[provider]) + ort_inputs = {name: tensor.detach().cpu().numpy() for name, tensor in sample_inputs.items()} + ort_output = session.run(None, ort_inputs)[0] + + abs_diff = np.max(np.abs(torch_output - ort_output)) + rel_diff = abs_diff / max(1.0, float(np.max(np.abs(torch_output)))) + LOGGER.info("ONNX Runtime check done (abs=%.6f, rel=%.6f)", abs_diff, rel_diff) + + +def main() -> None: + args = parse_args() + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + torch_dtype = torch.float16 if args.dtype == "fp16" else torch.float32 + if torch_dtype == torch.float16 and device.type == "cpu": + LOGGER.warning("CPU 上不支持 fp16, 自动回退为 fp32") + torch_dtype = torch.float32 + + torch.set_grad_enabled(False) + vae = load_vae(args, torch_dtype, device) + sample_inputs = build_dummy_inputs(args, vae, torch_dtype, device) + + encoder_inputs = OrderedDict(pixel_values=sample_inputs["pixel_values"]) + encoder_wrapper = VAEEncoderWrapper(vae) + maybe_save_calibration_inputs("vae_encoder", encoder_inputs, args) + + with torch.inference_mode(): + latent_sample = encoder_wrapper(*encoder_inputs.values()).detach() + + encoder_onnx = export_onnx( + encoder_wrapper, + encoder_inputs, + args.encoder_output, + ["latents"], + args, + ) + + decoder_inputs = OrderedDict(latents=latent_sample) + decoder_wrapper = VAEDecoderWrapper(vae) + maybe_save_calibration_inputs("vae_decoder", decoder_inputs, args) + + decoder_onnx = export_onnx( + decoder_wrapper, + decoder_inputs, + args.decoder_output, + ["images"], + args, + ) + + if not args.skip_ort_check: + try: + run_ort_validation(encoder_wrapper, encoder_inputs, encoder_onnx, args.ort_provider) + run_ort_validation(decoder_wrapper, decoder_inputs, decoder_onnx, args.ort_provider) + except Exception as exc: + LOGGER.warning("ONNX Runtime validation failed: %s", exc) + + +if __name__ == "__main__": + """ + 示例: + python scripts/z_image_fun/export_vae_onnx.py \ + --model-root models/Diffusion_Transformer/Z-Image-Turbo/ \ + --height 512 --width 512 \ + --encoder-output onnx-models/vae_encoder.onnx \ + --decoder-output onnx-models/vae_decoder.onnx \ + --dtype fp32 \ + --save-calib-inputs \ + --skip-ort-check + """ + main() diff --git a/VideoX-Fun/scripts/z_image_fun/train_control.py b/VideoX-Fun/scripts/z_image_fun/train_control.py new file mode 100644 index 0000000000000000000000000000000000000000..86fde2292f49e502456c39c032dec85cf14a43e5 --- /dev/null +++ b/VideoX-Fun/scripts/z_image_fun/train_control.py @@ -0,0 +1,1733 @@ +"""Modified from https://github.com/huggingface/diffusers/blob/main/examples/text_to_image/train_text_to_image.py +""" +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +import argparse +import gc +import logging +import math +import os +import pickle +import random +import shutil +import sys +from typing import (Any, Callable, Dict, List, NamedTuple, Optional, Tuple, + Union) + +import accelerate +import diffusers +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import torchvision.transforms.functional as TF +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState +from accelerate.utils import ProjectConfiguration, set_seed +from diffusers import DDIMScheduler, FlowMatchEulerDiscreteScheduler +from diffusers.optimization import get_scheduler +from diffusers.training_utils import (EMAModel, + compute_density_for_timestep_sampling, + compute_loss_weighting_for_sd3) +from diffusers.utils import check_min_version, deprecate, is_wandb_available +from diffusers.utils.torch_utils import is_compiled_module +from einops import rearrange +from omegaconf import OmegaConf +from packaging import version +from PIL import Image +from torch.utils.data import RandomSampler +from torch.utils.tensorboard import SummaryWriter +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import AutoTokenizer +from transformers.utils import ContextManagers + +import datasets + +current_file_path = os.path.abspath(__file__) +project_roots = [os.path.dirname(current_file_path), os.path.dirname(os.path.dirname(current_file_path)), os.path.dirname(os.path.dirname(os.path.dirname(current_file_path)))] +for project_root in project_roots: + sys.path.insert(0, project_root) if project_root not in sys.path else None + +from qwen_vl_utils import process_vision_info + +from videox_fun.data.bucket_sampler import (ASPECT_RATIO_512, + ASPECT_RATIO_RANDOM_CROP_512, + ASPECT_RATIO_RANDOM_CROP_PROB, + AspectRatioBatchImageVideoSampler, + RandomSampler, get_closest_ratio) +from videox_fun.data.dataset_image_video import (ImageVideoControlDataset, + ImageVideoDataset, + ImageVideoSampler, + get_random_mask, + process_pose_file, + process_pose_params) +from videox_fun.dist import set_multi_gpus_devices, shard_model +from videox_fun.models import (AutoencoderKL, AutoProcessor, AutoTokenizer, + CLIPImageProcessor, + CLIPVisionModelWithProjection, Qwen2Tokenizer, + Qwen3ForCausalLM, QwenImageTransformer2DModel, + ZImageControlTransformer2DModel, + ZImageTransformer2DModel) +from videox_fun.pipeline import Flux2Pipeline +from videox_fun.utils.discrete_sampler import DiscreteSampling +from videox_fun.utils.utils import get_image_to_video_latent, save_videos_grid +from videox_fun.utils.utils_yolo import ObjectInstanceDetector + +if is_wandb_available(): + import wandb + +def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def linear_decay(initial_value, final_value, total_steps, current_step): + if current_step >= total_steps: + return final_value + current_step = max(0, current_step) + step_size = (final_value - initial_value) / total_steps + current_value = initial_value + step_size * current_step + return current_value + +def generate_timestep_with_lognorm(low, high, shape, device="cpu", generator=None): + u = torch.normal(mean=0.0, std=1.0, size=shape, device=device, generator=generator) + t = 1 / (1 + torch.exp(-u)) * (high - low) + low + return torch.clip(t.to(torch.int32), low, high - 1) + +def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float: + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return float(mu) + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + +def encode_prompt( + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + text_encoder = None, + tokenizer = None, + max_sequence_length: int = 512, +) -> List[torch.FloatTensor]: + if isinstance(prompt, str): + prompt = [prompt] + + for i, prompt_item in enumerate(prompt): + messages = [ + {"role": "user", "content": prompt_item}, + ] + prompt_item = tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + prompt[i] = prompt_item + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + + prompt_embeds = text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] + + embeddings_list = [] + + for i in range(len(prompt_embeds)): + embeddings_list.append(prompt_embeds[i][prompt_masks[i]]) + + return embeddings_list + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.18.0.dev0") + +logger = get_logger(__name__, log_level="INFO") + +def log_validation(vae, text_encoder, tokenizer, transformer3d, network, args, accelerator, weight_dtype, global_step): + try: + logger.info("Running validation... ") + + transformer3d_val = ZImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="transformer", torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + ).to(weight_dtype) + transformer3d_val.load_state_dict(accelerator.unwrap_model(transformer3d).state_dict()) + scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + transformer3d = transformer3d.to("cpu") + pipeline = Flux2Pipeline( + vae=accelerator.unwrap_model(vae).to(weight_dtype), + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + transformer=transformer3d_val, + scheduler=scheduler, + ) + pipeline = pipeline.to(accelerator.device) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.no_grad(): + sample = pipeline( + args.validation_prompts[i], + negative_prompt = "bad detailed", + height = args.image_sample_size, + width = args.image_sample_size, + generator = generator + ).images + os.makedirs(os.path.join(args.output_dir, "sample"), exist_ok=True) + image = sample[0].save(os.path.join(args.output_dir, f"sample/sample-{global_step}-image-{i}.gif")) + + del pipeline + del transformer3d_val + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + transformer3d = transformer3d.to(accelerator.device) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + print(f"Eval error with info {e}") + transformer3d = transformer3d.to(accelerator.device) + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--input_perturbation", type=float, default=0, help="The scale of input perturbation. Recommended 0.1." + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. " + ), + ) + parser.add_argument( + "--train_data_meta", + type=str, + default=None, + help=( + "A csv containing the training data. " + ), + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="sd-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--random_flip", + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--use_came", + action="store_true", + help="whether to use came", + ) + parser.add_argument( + "--multi_stream", + action="store_true", + help="whether to use cuda multi-stream", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--vae_mini_batch", type=int, default=32, help="mini batch size for vae." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--non_ema_revision", + type=str, + default=None, + required=False, + help=( + "Revision of pretrained non-ema model identifier. Must be a branch, tag or git identifier of the local or" + " remote repository specified with --pretrained_model_name_or_path." + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--prediction_type", + type=str, + default=None, + help="The prediction_type that shall be used for training. Choose between 'epsilon' or 'v_prediction' or leave `None`. If left to `None` the default prediction type of the scheduler: `noise_scheduler.config.prediciton_type` is chosen.", + ) + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--report_model_info", action="store_true", help="Whether or not to report more info about model (such as norm, grad)." + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument("--noise_offset", type=float, default=0, help="The scale of noise offset.") + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=2000, + help="Run validation every X steps.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + parser.add_argument( + "--snr_loss", action="store_true", help="Whether or not to use snr_loss." + ) + parser.add_argument( + "--uniform_sampling", action="store_true", help="Whether or not to use uniform_sampling." + ) + parser.add_argument( + "--enable_text_encoder_in_dataloader", action="store_true", help="Whether or not to use text encoder in dataloader." + ) + parser.add_argument( + "--enable_bucket", action="store_true", help="Whether enable bucket sample in datasets." + ) + parser.add_argument( + "--random_ratio_crop", action="store_true", help="Whether enable random ratio crop sample in datasets." + ) + parser.add_argument( + "--random_hw_adapt", action="store_true", help="Whether enable random adapt height and width in datasets." + ) + parser.add_argument( + "--token_sample_size", + type=int, + default=512, + help="Sample size of the token.", + ) + parser.add_argument( + "--train_sampling_steps", + type=int, + default=1000, + help="Run train_sampling_steps.", + ) + parser.add_argument( + "--video_sample_size", + type=int, + default=512, + help="Sample size of the video.", + ) + parser.add_argument( + "--image_sample_size", + type=int, + default=512, + help="Sample size of the image.", + ) + parser.add_argument( + "--fix_sample_size", + nargs=2, type=int, default=None, + help="Fix Sample size [height, width] when using bucket and collate_fn." + ) + parser.add_argument( + "--config_path", + type=str, + default="config/z_image/z_image_control.yaml", + help=( + "The config of the model in training." + ), + ) + parser.add_argument( + "--transformer_path", + type=str, + default=None, + help=("If you want to load the weight from other transformers, input its path."), + ) + parser.add_argument( + "--vae_path", + type=str, + default=None, + help=("If you want to load the weight from other vaes, input its path."), + ) + + parser.add_argument( + '--trainable_modules', + nargs='+', + help='Enter a list of trainable modules' + ) + parser.add_argument( + '--trainable_modules_low_learning_rate', + nargs='+', + default=[], + help='Enter a list of trainable modules with lower learning rate' + ) + parser.add_argument( + '--tokenizer_max_length', + type=int, + default=512, + help='Max length of tokenizer' + ) + parser.add_argument( + "--use_deepspeed", action="store_true", help="Whether or not to use deepspeed." + ) + parser.add_argument( + "--use_fsdp", action="store_true", help="Whether or not to use fsdp." + ) + parser.add_argument( + "--low_vram", action="store_true", help="Whether enable low_vram mode." + ) + parser.add_argument( + "--prompt_template_encode", + type=str, + default="<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n", + help=( + 'The prompt template for text encoder.' + ), + ) + parser.add_argument( + "--prompt_template_encode_start_idx", + type=int, + default=34, + help=( + 'The start idx for prompt template.' + ), + ) + parser.add_argument( + "--train_mode", + type=str, + default="normal", + help=( + 'The format of training data. Support `"control"`' + ' (default), `control_ref`.' + ), + ) + parser.add_argument( + "--abnormal_norm_clip_start", + type=int, + default=1000, + help=( + 'When do we start doing additional processing on abnormal gradients. ' + ), + ) + parser.add_argument( + "--initial_grad_norm_ratio", + type=int, + default=5, + help=( + 'The initial gradient is relative to the multiple of the max_grad_norm. ' + ), + ) + parser.add_argument( + "--add_inpaint_info", + action="store_true", + help=( + 'Whether enable add inpaint info in self attention.' + ), + ) + parser.add_argument( + "--weighting_scheme", + type=str, + default="none", + choices=["sigma_sqrt", "logit_normal", "mode", "cosmap", "none"], + help=('We default to the "none" weighting scheme for uniform sampling and uniform loss'), + ) + parser.add_argument( + "--logit_mean", type=float, default=0.0, help="mean to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--logit_std", type=float, default=1.0, help="std to use when using the `'logit_normal'` weighting scheme." + ) + parser.add_argument( + "--mode_scale", + type=float, + default=1.29, + help="Scale of mode weighting scheme. Only effective when using the `'mode'` as the `weighting_scheme`.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # default to using the same revision for the non-ema model if not specified + if args.non_ema_revision is None: + args.non_ema_revision = args.revision + + return args + + +def main(): + args = parse_args() + + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + if args.non_ema_revision is not None: + deprecate( + "non_ema_revision!=None", + "0.15.0", + message=( + "Downloading 'non_ema' weights from revision branches of the Hub is deprecated. Please make sure to" + " use `--variant=non_ema` instead." + ), + ) + logging_dir = os.path.join(args.output_dir, args.logging_dir) + + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + deepspeed_plugin = accelerator.state.deepspeed_plugin if hasattr(accelerator.state, "deepspeed_plugin") else None + fsdp_plugin = accelerator.state.fsdp_plugin if hasattr(accelerator.state, "fsdp_plugin") else None + if deepspeed_plugin is not None: + zero_stage = int(deepspeed_plugin.zero_stage) + fsdp_stage = 0 + print(f"Using DeepSpeed Zero stage: {zero_stage}") + + args.use_deepspeed = True + if zero_stage == 3: + print(f"Auto set save_state to True because zero_stage == 3") + args.save_state = True + elif fsdp_plugin is not None: + from torch.distributed.fsdp import ShardingStrategy + zero_stage = 0 + if fsdp_plugin.sharding_strategy is ShardingStrategy.FULL_SHARD: + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is None: # The fsdp_plugin.sharding_strategy is None in FSDP 2. + fsdp_stage = 3 + elif fsdp_plugin.sharding_strategy is ShardingStrategy.SHARD_GRAD_OP: + fsdp_stage = 2 + else: + fsdp_stage = 0 + print(f"Using FSDP stage: {fsdp_stage}") + + args.use_fsdp = True + if fsdp_stage == 3: + print(f"Auto set save_state to True because fsdp_stage == 3") + args.save_state = True + else: + zero_stage = 0 + fsdp_stage = 0 + print("DeepSpeed is not enabled.") + + if accelerator.is_main_process: + writer = SummaryWriter(log_dir=logging_dir) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + rng = np.random.default_rng(np.random.PCG64(args.seed + accelerator.process_index)) + torch_rng = torch.Generator(accelerator.device).manual_seed(args.seed + accelerator.process_index) + else: + rng = None + torch_rng = None + index_rng = np.random.default_rng(np.random.PCG64(43)) + print(f"Init rng with seed {args.seed + accelerator.process_index}. Process_index is {accelerator.process_index}") + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora transformer3d) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + # Load Config + config = OmegaConf.load(args.config_path) + + # Load scheduler, tokenizer and models. + noise_scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="scheduler" + ) + + # Get Tokenizer + tokenizer = AutoTokenizer.from_pretrained( + args.pretrained_model_name_or_path, subfolder="tokenizer" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if accelerate.state.is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + # Currently Accelerate doesn't know how to handle multiple models under Deepspeed ZeRO stage 3. + # For this to work properly all models must be run through `accelerate.prepare`. But accelerate + # will try to assign the same optimizer with the same weights to all models during + # `deepspeed.initialize`, which of course doesn't work. + # + # For now the following workaround will partially support Deepspeed ZeRO-3, by excluding the 2 + # frozen models from being partitioned during `zero.Init` which gets called during + # `from_pretrained` So Qwen3ForCausalLM and AutoencoderKL will not enjoy the parameter sharding + # across multiple gpus and only UNet2DConditionModel will get ZeRO sharded. + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + # Get Text encoder + text_encoder = Qwen3ForCausalLM.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ) + text_encoder = text_encoder.eval() + + # Get Vae + vae = AutoencoderKL.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="vae" + ).to(weight_dtype) + vae.eval() + + # Get Transformer + transformer3d = ZImageControlTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + low_cpu_mem_usage=True, + transformer_additional_kwargs=OmegaConf.to_container(config['transformer_additional_kwargs']), + ).to(weight_dtype) + yolo_instance = ObjectInstanceDetector(device=accelerator.device) + + # Freeze vae and text_encoder and set transformer3d to trainable + vae.requires_grad_(False) + text_encoder.requires_grad_(False) + transformer3d.requires_grad_(False) + + if args.transformer_path is not None: + print(f"From checkpoint: {args.transformer_path}") + if args.transformer_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.transformer_path) + else: + state_dict = torch.load(args.transformer_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = transformer3d.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + if args.vae_path is not None: + print(f"From checkpoint: {args.vae_path}") + if args.vae_path.endswith("safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(args.vae_path) + else: + state_dict = torch.load(args.vae_path, map_location="cpu") + state_dict = state_dict["state_dict"] if "state_dict" in state_dict else state_dict + + m, u = vae.load_state_dict(state_dict, strict=False) + print(f"missing keys: {len(m)}, unexpected keys: {len(u)}") + assert len(u) == 0 + + # A good trainable modules is showed below now. + # For 3D Patch: trainable_modules = ['ff.net', 'pos_embed', 'attn2', 'proj_out', 'timepositionalencoding', 'h_position', 'w_position'] + # For 2D Patch: trainable_modules = ['ff.net', 'attn2', 'timepositionalencoding', 'h_position', 'w_position'] + transformer3d.train() + if accelerator.is_main_process: + accelerator.print( + f"Trainable modules '{args.trainable_modules}'." + ) + for name, param in transformer3d.named_parameters(): + for trainable_module_name in args.trainable_modules + args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + param.requires_grad = True + break + + # Create EMA for the transformer3d. + if args.use_ema: + if zero_stage == 3: + raise NotImplementedError("FSDP does not support EMA.") + + ema_transformer3d = ZImageTransformer2DModel.from_pretrained( + args.pretrained_model_name_or_path, + subfolder="transformer", + torch_dtype=weight_dtype, + ).to(weight_dtype) + + ema_transformer3d = EMAModel(ema_transformer3d.parameters(), model_cls=ZImageTransformer2DModel, model_config=ema_transformer3d.config) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + if fsdp_stage != 0: + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + accelerate_state_dict = {k: v.to(dtype=weight_dtype) for k, v in accelerate_state_dict.items()} + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + elif zero_stage == 3: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + accelerate_state_dict = accelerator.get_state_dict(models[-1], unwrap=True) + if accelerator.is_main_process: + from safetensors.torch import save_file + safetensor_save_path = os.path.join(output_dir, f"diffusion_pytorch_model.safetensors") + save_file(accelerate_state_dict, safetensor_save_path, metadata={"format": "pt"}) + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + else: + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_transformer3d.save_pretrained(os.path.join(output_dir, "transformer_ema")) + + models[0].save_pretrained(os.path.join(output_dir, "transformer")) + if not args.use_deepspeed: + weights.pop() + + with open(os.path.join(output_dir, "sampler_pos_start.pkl"), 'wb') as file: + pickle.dump([batch_sampler.sampler._pos_start, first_epoch], file) + + def load_model_hook(models, input_dir): + if args.use_ema: + ema_path = os.path.join(input_dir, "transformer_ema") + _, ema_kwargs = ZImageTransformer2DModel.load_config(ema_path, return_unused_kwargs=True) + load_model = ZImageTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer_ema", + ) + load_model = EMAModel(load_model.parameters(), model_cls=ZImageTransformer2DModel, model_config=load_model.config) + load_model.load_state_dict(ema_kwargs) + + ema_transformer3d.load_state_dict(load_model.state_dict()) + ema_transformer3d.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = ZImageTransformer2DModel.from_pretrained( + input_dir, subfolder="transformer" + ) + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + pkl_path = os.path.join(input_dir, "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + loaded_number, _ = pickle.load(file) + batch_sampler.sampler._pos_start = max(loaded_number - args.dataloader_num_workers * accelerator.num_processes * 2, 0) + print(f"Load pkl from {pkl_path}. Get loaded_number = {loaded_number}.") + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + transformer3d.enable_gradient_checkpointing() + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + elif args.use_came: + try: + from came_pytorch import CAME + except: + raise ImportError( + "Please install came_pytorch to use CAME. You can do so by running `pip install came_pytorch`" + ) + + optimizer_cls = CAME + else: + optimizer_cls = torch.optim.AdamW + + trainable_params = list(filter(lambda p: p.requires_grad, transformer3d.parameters())) + trainable_params_optim = [ + {'params': [], 'lr': args.learning_rate}, + {'params': [], 'lr': args.learning_rate / 2}, + ] + in_already = [] + for name, param in transformer3d.named_parameters(): + high_lr_flag = False + if name in in_already: + continue + for trainable_module_name in args.trainable_modules: + if trainable_module_name in name: + in_already.append(name) + high_lr_flag = True + trainable_params_optim[0]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate}") + break + if high_lr_flag: + continue + for trainable_module_name in args.trainable_modules_low_learning_rate: + if trainable_module_name in name: + in_already.append(name) + trainable_params_optim[1]['params'].append(param) + if accelerator.is_main_process: + print(f"Set {name} to lr : {args.learning_rate / 2}") + break + + if args.use_came: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + # weight_decay=args.adam_weight_decay, + betas=(0.9, 0.999, 0.9999), + eps=(1e-30, 1e-16) + ) + else: + optimizer = optimizer_cls( + trainable_params_optim, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the training dataset + if args.fix_sample_size is not None and args.enable_bucket: + args.image_sample_size = max(max(args.fix_sample_size), args.image_sample_size) + args.random_hw_adapt = False + + # Get the dataset + train_dataset = ImageVideoControlDataset( + args.train_data_meta, args.train_data_dir, + image_sample_size=args.image_sample_size, + enable_bucket=args.enable_bucket, + enable_inpaint=args.add_inpaint_info, + enable_camera_info=False, + enable_subject_info=False, + ) + + def worker_init_fn(_seed): + _seed = _seed * 256 + def _worker_init_fn(worker_id): + print(f"worker_init_fn with {_seed + worker_id}") + np.random.seed(_seed + worker_id) + random.seed(_seed + worker_id) + return _worker_init_fn + + if args.enable_bucket: + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = AspectRatioBatchImageVideoSampler( + sampler=RandomSampler(train_dataset, generator=batch_sampler_generator), dataset=train_dataset.dataset, + batch_size=args.train_batch_size, train_folder = args.train_data_dir, drop_last=True, + aspect_ratios=aspect_ratio_sample_size, + ) + + def collate_fn(examples): + def get_random_downsample_ratio(sample_size, image_ratio=[], + all_choices=False, rng=None): + def _create_special_list(length): + if length == 1: + return [1.0] + if length >= 2: + first_element = 0.90 + remaining_sum = 1.0 - first_element + other_elements_value = remaining_sum / (length - 1) + special_list = [first_element] + [other_elements_value] * (length - 1) + return special_list + + if sample_size >= 1536: + number_list = [1, 1.25, 1.5, 2, 2.5, 3] + image_ratio + elif sample_size >= 1024: + number_list = [1, 1.25, 1.5, 2] + image_ratio + elif sample_size >= 768: + number_list = [1, 1.25, 1.5] + image_ratio + elif sample_size >= 512: + number_list = [1] + image_ratio + else: + number_list = [1] + + if all_choices: + return number_list + + number_list_prob = np.array(_create_special_list(len(number_list))) + if rng is None: + return np.random.choice(number_list, p = number_list_prob) + else: + return rng.choice(number_list, p = number_list_prob) + + # Create new output + new_examples = {} + new_examples["pixel_values"] = [] + new_examples["text"] = [] + + # Used in Control Mode + new_examples["control_pixel_values"] = [] + + # Used in Inpaint mode + new_examples["mask_pixel_values"] = [] + new_examples["mask"] = [] + + # Get downsample ratio in image + pixel_value = examples[0]["pixel_values"] + data_type = examples[0]["data_type"] + f, h, w, c = np.shape(pixel_value) + + random_downsample_ratio = 1 if not args.random_hw_adapt else get_random_downsample_ratio(args.image_sample_size) + + aspect_ratio_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + aspect_ratio_random_crop_sample_size = {key : [x / 512 * args.image_sample_size / random_downsample_ratio for x in ASPECT_RATIO_RANDOM_CROP_512[key]] for key in ASPECT_RATIO_RANDOM_CROP_512.keys()} + + if args.fix_sample_size is not None: + fix_sample_size = [int(x / 16) * 16 for x in args.fix_sample_size] + elif args.random_ratio_crop: + if rng is None: + random_sample_size = aspect_ratio_random_crop_sample_size[ + np.random.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + else: + random_sample_size = aspect_ratio_random_crop_sample_size[ + rng.choice(list(aspect_ratio_random_crop_sample_size.keys()), p = ASPECT_RATIO_RANDOM_CROP_PROB) + ] + random_sample_size = [int(x / 16) * 16 for x in random_sample_size] + else: + closest_size, closest_ratio = get_closest_ratio(h, w, ratios=aspect_ratio_sample_size) + closest_size = [int(x / 16) * 16 for x in closest_size] + + for example in examples: + # To 0~1 + pixel_values = torch.from_numpy(example["pixel_values"]).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + + control_pixel_values = torch.from_numpy(example["control_pixel_values"]).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + + if args.fix_sample_size is not None: + # Get adapt hw for resize + fix_sample_size = list(map(lambda x: int(x), fix_sample_size)) + transform = transforms.Compose([ + transforms.Resize(fix_sample_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(fix_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + elif args.random_ratio_crop: + # Get adapt hw for resize + b, c, h, w = pixel_values.size() + th, tw = random_sample_size + if th / tw > h / w: + nh = int(th) + nw = int(w / h * nh) + else: + nw = int(tw) + nh = int(h / w * nw) + + transform = transforms.Compose([ + transforms.Resize([nh, nw]), + transforms.CenterCrop([int(x) for x in random_sample_size]), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + else: + # Get adapt hw for resize + closest_size = list(map(lambda x: int(x), closest_size)) + if closest_size[0] / h > closest_size[1] / w: + resize_size = closest_size[0], int(w * closest_size[0] / h) + else: + resize_size = int(h * closest_size[1] / w), closest_size[1] + + transform = transforms.Compose([ + transforms.Resize(resize_size, interpolation=transforms.InterpolationMode.BILINEAR), # Image.BICUBIC + transforms.CenterCrop(closest_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + length = int(len(pixel_values) // 2) + new_examples["pixel_values"].append(transform(pixel_values)[length:length + 1]) + new_examples["control_pixel_values"].append(transform(control_pixel_values)[length:length + 1]) + + new_examples["text"].append(example["text"]) + + mask = get_random_mask(new_examples["pixel_values"][-1].size()) + mask_pixel_values = new_examples["pixel_values"][-1] * (1 - mask) + + new_examples["mask_pixel_values"].append(mask_pixel_values[:1]) + new_examples["mask"].append(mask[:1]) + + # Limit the number of frames to the same + new_examples["pixel_values"] = torch.stack([example for example in new_examples["pixel_values"]]) + new_examples["control_pixel_values"] = torch.stack([example for example in new_examples["control_pixel_values"]]) + new_examples["mask_pixel_values"] = torch.stack([example for example in new_examples["mask_pixel_values"]]) + new_examples["mask"] = torch.stack([example for example in new_examples["mask"]]) + + # Encode prompts when enable_text_encoder_in_dataloader=True + if args.enable_text_encoder_in_dataloader: + prompt_embeds = encode_prompt( + batch['text'], device="cpu", + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + new_examples['prompt_embeds'] = prompt_embeds + + return new_examples + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + collate_fn=collate_fn, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + else: + # DataLoaders creation: + batch_sampler_generator = torch.Generator().manual_seed(args.seed) + batch_sampler = ImageVideoSampler(RandomSampler(train_dataset, generator=batch_sampler_generator), train_dataset, args.train_batch_size) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + batch_sampler=batch_sampler, + persistent_workers=True if args.dataloader_num_workers != 0 else False, + num_workers=args.dataloader_num_workers, + worker_init_fn=worker_init_fn(args.seed + accelerator.process_index) + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + ) + + # Prepare everything with our `accelerator`. + transformer3d, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + transformer3d, optimizer, train_dataloader, lr_scheduler + ) + + if fsdp_stage != 0 or zero_stage != 0: + from functools import partial + + from videox_fun.dist import set_multi_gpus_devices, shard_model + shard_fn = partial(shard_model, device_id=accelerator.device, param_dtype=weight_dtype, module_to_wrapper=text_encoder.model.layers) + text_encoder = shard_fn(text_encoder) + + if args.use_ema: + ema_transformer3d.to(accelerator.device) + + # Move text_encode and vae to gpu and cast to weight_dtype + vae.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device if not args.low_vram else "cpu", dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + keys_to_pop = [k for k, v in tracker_config.items() if isinstance(v, list)] + for k in keys_to_pop: + tracker_config.pop(k) + print(f"Removed tracker_config['{k}']") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Function for unwrapping if model was compiled with `torch.compile`. + def unwrap_model(model): + model = accelerator.unwrap_model(model) + model = model._orig_mod if is_compiled_module(model) else model + return model + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + + pkl_path = os.path.join(os.path.join(args.output_dir, path), "sampler_pos_start.pkl") + if os.path.exists(pkl_path): + with open(pkl_path, 'rb') as file: + _, first_epoch = pickle.load(file) + else: + first_epoch = global_step // num_update_steps_per_epoch + print(f"Load pkl from {pkl_path}. Get first_epoch = {first_epoch}.") + + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + if args.multi_stream: + # create extra cuda streams to speedup inpaint vae computation + vae_stream_1 = torch.cuda.Stream() + vae_stream_2 = torch.cuda.Stream() + else: + vae_stream_1 = None + vae_stream_2 = None + + # Calculate the index we need】 + idx_sampling = DiscreteSampling(args.train_sampling_steps, uniform_sampling=args.uniform_sampling) + + for epoch in range(first_epoch, args.num_train_epochs): + train_loss = 0.0 + batch_sampler.sampler.generator = torch.Generator().manual_seed(args.seed + epoch) + for step, batch in enumerate(train_dataloader): + # Data batch sanity check + if epoch == first_epoch and step == 0: + pixel_values, texts = batch['pixel_values'].cpu(), batch['text'] + control_pixel_values = batch["control_pixel_values"].cpu() + pixel_values = rearrange(pixel_values, "b f c h w -> b c f h w") + control_pixel_values = rearrange(control_pixel_values, "b f c h w -> b c f h w") + os.makedirs(os.path.join(args.output_dir, "sanity_check"), exist_ok=True) + for idx, (pixel_value, control_pixel_value, text) in enumerate(zip(pixel_values, control_pixel_values, texts)): + pixel_value = pixel_value[None, ...] + control_pixel_value = control_pixel_value[None, ...] + gif_name = '-'.join(text.replace('/', '').split()[:10]) if not text == '' else f'{global_step}-{idx}' + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}.gif", rescale=True) + save_videos_grid(control_pixel_value, f"{args.output_dir}/sanity_check/{gif_name[:10]}_control.gif", rescale=True) + + mask_pixel_values, mask, texts = batch['mask_pixel_values'].cpu(), batch['mask'].cpu(), batch['text'] + mask_pixel_values = rearrange(mask_pixel_values, "b f c h w -> b c f h w") + mask = torch.tile(rearrange(mask, "b f c h w -> b c f h w"), [1, 3, 1, 1, 1]) + for idx, (pixel_value, _mask, text) in enumerate(zip(mask_pixel_values, mask, texts)): + pixel_value = pixel_value[None, ...] + _mask = _mask[None, ...] + save_videos_grid(pixel_value, f"{args.output_dir}/sanity_check/mask_pixel_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + save_videos_grid(_mask, f"{args.output_dir}/sanity_check/mask_{gif_name[:10] if not text == '' else f'{global_step}-{idx}'}.gif", rescale=True) + + with accelerator.accumulate(transformer3d): + # Convert images to latent space + pixel_values = batch["pixel_values"].to(weight_dtype) + control_pixel_values = batch["control_pixel_values"].to(weight_dtype) + mask_pixel_values = batch["mask_pixel_values"].to(weight_dtype) + mask = batch["mask"].to(weight_dtype) + + if args.low_vram: + torch.cuda.empty_cache() + vae.to(accelerator.device) + if not args.enable_text_encoder_in_dataloader: + text_encoder.to("cpu") + + with torch.no_grad(): + # This way is quicker when batch grows up + def _batch_encode_vae(pixel_values): + pixel_values = pixel_values.squeeze(1) + bs = args.vae_mini_batch + new_pixel_values = [] + for i in range(0, pixel_values.shape[0], bs): + pixel_values_bs = pixel_values[i : i + bs] + pixel_values_bs = vae.encode(pixel_values_bs)[0] + pixel_values_bs = pixel_values_bs.sample() + new_pixel_values.append(pixel_values_bs) + return torch.cat(new_pixel_values, dim = 0) + if vae_stream_1 is not None: + vae_stream_1.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(vae_stream_1): + latents = _batch_encode_vae(pixel_values).unsqueeze(2) + else: + latents = _batch_encode_vae(pixel_values).unsqueeze(2) + + control_latents = _batch_encode_vae(control_pixel_values).unsqueeze(2) + control_latents = ((control_latents - vae.config.shift_factor) * vae.config.scaling_factor).to(dtype=weight_dtype) + + for bs_index in range(control_latents.size()[0]): + if rng is None: + zero_init_control_conv_in = np.random.choice([0, 1], p = [0.90, 0.10]) + else: + zero_init_control_conv_in = rng.choice([0, 1], p = [0.90, 0.10]) + if zero_init_control_conv_in: + control_latents[bs_index] = control_latents[bs_index] * 0 + + if args.add_inpaint_info: + # Encode inpaint latents. + mask = mask.squeeze(1) + mask_conditions = F.interpolate(1 - mask[:, :1], size=control_latents.size()[-2:], mode='nearest').to(accelerator.device, weight_dtype) + mask_conditions = mask_conditions.unsqueeze(2) + + t2v_flag = [(_mask == 1).all() for _mask in mask] + new_t2v_flag = [] + for _mask in t2v_flag: + if _mask and np.random.rand() < 0.90: + new_t2v_flag.append(0) + else: + new_t2v_flag.append(1) + t2v_flag = torch.from_numpy(np.array(new_t2v_flag)).to(accelerator.device, dtype=weight_dtype) + + mask_latents = _batch_encode_vae(mask_pixel_values).unsqueeze(2) + mask_latents = ((mask_latents - vae.config.shift_factor) * vae.config.scaling_factor).to(dtype=weight_dtype) + mask_latents = t2v_flag[:, None, None] * mask_latents + + inpaint_latents = torch.cat([mask_conditions, mask_latents], dim=1) + control_context = torch.cat([control_latents, inpaint_latents], dim=1) + else: + control_context = control_latents + + # wait for latents = vae.encode(pixel_values) to complete + if vae_stream_1 is not None: + torch.cuda.current_stream().wait_stream(vae_stream_1) + + if args.low_vram: + vae.to('cpu') + torch.cuda.empty_cache() + if not args.enable_text_encoder_in_dataloader: + text_encoder.to(accelerator.device) + + if args.enable_text_encoder_in_dataloader: + prompt_embeds = batch['prompt_embeds'].to(dtype=latents.dtype, device=accelerator.device) + else: + with torch.no_grad(): + prompt_embeds = encode_prompt( + batch['text'], device=accelerator.device, + text_encoder=text_encoder, + tokenizer=tokenizer, + ) + + if args.low_vram and not args.enable_text_encoder_in_dataloader: + text_encoder.to('cpu') + torch.cuda.empty_cache() + + bsz, channel, f, height, width = latents.size() + latents = ((latents - vae.config.shift_factor) * vae.config.scaling_factor).to(dtype=weight_dtype) + noise = torch.randn(latents.size(), device=latents.device, generator=torch_rng, dtype=weight_dtype) + + if not args.uniform_sampling: + u = compute_density_for_timestep_sampling( + weighting_scheme=args.weighting_scheme, + batch_size=bsz, + logit_mean=args.logit_mean, + logit_std=args.logit_std, + mode_scale=args.mode_scale, + ) + indices = (u * noise_scheduler.config.num_train_timesteps).long() + else: + # Sample a random timestep for each image + # timesteps = generate_timestep_with_lognorm(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + # timesteps = torch.randint(0, args.train_sampling_steps, (bsz,), device=latents.device, generator=torch_rng) + indices = idx_sampling(bsz, generator=torch_rng, device=latents.device) + indices = indices.long().cpu() + + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + noise_scheduler.config.get("base_image_seq_len", 256), + noise_scheduler.config.get("max_image_seq_len", 4096), + noise_scheduler.config.get("base_shift", 0.5), + noise_scheduler.config.get("max_shift", 1.15), + ) + noise_scheduler.sigma_min = 0.0 + noise_scheduler.set_timesteps(args.train_sampling_steps, device=latents.device, mu=mu) + timesteps = noise_scheduler.timesteps[indices].to(device=latents.device) + + def get_sigmas(timesteps, n_dim=4, dtype=torch.float32): + sigmas = noise_scheduler.sigmas.to(device=accelerator.device, dtype=dtype) + schedule_timesteps = noise_scheduler.timesteps.to(accelerator.device) + timesteps = timesteps.to(accelerator.device) + step_indices = [(schedule_timesteps == t).nonzero().item() for t in timesteps] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < n_dim: + sigma = sigma.unsqueeze(-1) + return sigma + + # Add noise according to flow matching. + # zt = (1 - texp) * x + texp * z1 + sigmas = get_sigmas(timesteps, n_dim=latents.ndim, dtype=latents.dtype) + noisy_latents = (1.0 - sigmas) * latents + sigmas * noise + + # Add noise + target = noise - latents + + timesteps = (1000 - timesteps) / 1000 + + # Predict the noise residual + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=accelerator.device): + noise_pred = transformer3d( + x=noisy_latents, + t=timesteps, + cap_feats=prompt_embeds, + control_context=control_context, + )[0] + + def custom_mse_loss(noise_pred, target, weighting=None, threshold=50): + noise_pred = noise_pred.float() + target = target.float() + diff = noise_pred - target + mse_loss = F.mse_loss(noise_pred, target, reduction='none') + mask = (diff.abs() <= threshold).float() + masked_loss = mse_loss * mask + if weighting is not None: + masked_loss = masked_loss * weighting + final_loss = masked_loss.mean() + return final_loss + + weighting = compute_loss_weighting_for_sd3(weighting_scheme=args.weighting_scheme, sigmas=sigmas) + loss = custom_mse_loss(-noise_pred.float(), target.float(), weighting.float()) + loss = loss.mean() + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + if not args.use_deepspeed and not args.use_fsdp: + trainable_params_grads = [p.grad for p in trainable_params if p.grad is not None] + trainable_params_total_norm = torch.norm(torch.stack([torch.norm(g.detach(), 2) for g in trainable_params_grads]), 2) + max_grad_norm = linear_decay(args.max_grad_norm * args.initial_grad_norm_ratio, args.max_grad_norm, args.abnormal_norm_clip_start, global_step) + if trainable_params_total_norm / max_grad_norm > 5 and global_step > args.abnormal_norm_clip_start: + actual_max_grad_norm = max_grad_norm / min((trainable_params_total_norm / max_grad_norm), 10) + else: + actual_max_grad_norm = max_grad_norm + else: + actual_max_grad_norm = args.max_grad_norm + + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + if trainable_params_total_norm > 1 and global_step > args.abnormal_norm_clip_start: + for name, param in transformer3d.named_parameters(): + if param.requires_grad: + writer.add_scalar(f'gradients/before_clip_norm/{name}', param.grad.norm(), global_step=global_step) + + norm_sum = accelerator.clip_grad_norm_(trainable_params, actual_max_grad_norm) + if not args.use_deepspeed and not args.use_fsdp and args.report_model_info and accelerator.is_main_process: + writer.add_scalar(f'gradients/norm_sum', norm_sum, global_step=global_step) + writer.add_scalar(f'gradients/actual_max_grad_norm', actual_max_grad_norm, global_step=global_step) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + + if args.use_ema: + ema_transformer3d.step(transformer3d.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if accelerator.is_main_process: + if args.validation_prompts is not None and global_step % args.validation_steps == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_transformer3d.store(transformer3d.parameters()) + ema_transformer3d.copy_to(transformer3d.parameters()) + log_validation( + vae, + text_encoder, + tokenizer, + tokenizer_2, + transformer3d, + network, + args, + accelerator, + weight_dtype, + global_step, + ) + if args.use_ema: + # Switch back to the original transformer3d parameters. + ema_transformer3d.restore(transformer3d.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if args.use_deepspeed or args.use_fsdp or accelerator.is_main_process: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/scripts/z_image_fun/train_control.sh b/VideoX-Fun/scripts/z_image_fun/train_control.sh new file mode 100644 index 0000000000000000000000000000000000000000..82589a06e5eb685540a37581a37133a5b924e2ce --- /dev/null +++ b/VideoX-Fun/scripts/z_image_fun/train_control.sh @@ -0,0 +1,34 @@ +export MODEL_NAME="models/Diffusion_Transformer/Z-Image-Turbo" +export DATASET_NAME="datasets/internal_datasets/" +export DATASET_META_NAME="datasets/internal_datasets/metadata.json" +# NCCL_IB_DISABLE=1 and NCCL_P2P_DISABLE=1 are used in multi nodes without RDMA. +# export NCCL_IB_DISABLE=1 +# export NCCL_P2P_DISABLE=1 +NCCL_DEBUG=INFO + +accelerate launch --mixed_precision="bf16" scripts/z_image_fun/train_control.py \ + --config_path="config/z_image/z_image_control.yaml" \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATASET_NAME \ + --train_data_meta=$DATASET_META_NAME \ + --train_batch_size=1 \ + --image_sample_size=1328 \ + --gradient_accumulation_steps=1 \ + --dataloader_num_workers=8 \ + --num_train_epochs=100 \ + --checkpointing_steps=50 \ + --learning_rate=2e-05 \ + --lr_scheduler="constant_with_warmup" \ + --lr_warmup_steps=100 \ + --seed=42 \ + --output_dir="output_dir_z_image_control" \ + --gradient_checkpointing \ + --mixed_precision="bf16" \ + --adam_weight_decay=3e-2 \ + --adam_epsilon=1e-10 \ + --vae_mini_batch=1 \ + --max_grad_norm=0.05 \ + --enable_bucket \ + --uniform_sampling \ + --transformer_path="models/Personalized_Model/Z-Image-Turbo-Fun-Controlnet-Union.safetensors" \ + --trainable_modules "control" \ No newline at end of file diff --git a/VideoX-Fun/scripts/zero_to_bf16.py b/VideoX-Fun/scripts/zero_to_bf16.py new file mode 100644 index 0000000000000000000000000000000000000000..47623dc22a7a8316ac3fda6a1b9debec44f9ab96 --- /dev/null +++ b/VideoX-Fun/scripts/zero_to_bf16.py @@ -0,0 +1,788 @@ +#!/usr/bin/env python + +# Copyright (c) Microsoft Corporation. +# SPDX-License-Identifier: Apache-2.0 + +# DeepSpeed Team + +# This script extracts bf16 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets +# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in +# the future. Once extracted, the weights don't require DeepSpeed and can be used in any +# application. +# +# example: +# python zero_to_bf16.py . output_dir/ +# or +# python zero_to_bf16.py . output_dir/ --safe_serialization + +import argparse +import gc +import glob +import json +import math +import os +import queue +import re +from collections import OrderedDict +from dataclasses import dataclass +from threading import Thread + +import numpy as np +import torch +from deepspeed.checkpoint.constants import (BUFFER_NAMES, DS_VERSION, + FP32_FLAT_GROUPS, + FROZEN_PARAM_FRAGMENTS, + FROZEN_PARAM_SHAPES, + OPTIMIZER_STATE_DICT, PARAM_SHAPES, + PARTITION_COUNT, + SINGLE_PARTITION_OF_FP32_GROUPS, + ZERO_STAGE) +# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with +# DeepSpeed data structures it has to be available in the current python environment. +from deepspeed.utils import logger +from tqdm import tqdm + + +@dataclass +class zero_model_state: + buffers: dict() + param_shapes: dict() + shared_params: list + ds_version: int + frozen_param_shapes: dict() + frozen_param_fragments: dict() + + +debug = 0 + +# load to cpu +device = torch.device('cpu') + + +def atoi(text): + return int(text) if text.isdigit() else text + + +def natural_keys(text): + ''' + alist.sort(key=natural_keys) sorts in human order + http://nedbatchelder.com/blog/200712/human_sorting.html + (See Toothy's implementation in the comments) + ''' + return [atoi(c) for c in re.split(r'(\d+)', text)] + + +def get_model_state_file(checkpoint_dir, zero_stage): + if not os.path.isdir(checkpoint_dir): + raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist") + + # there should be only one file + if zero_stage <= 2: + file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt") + elif zero_stage == 3: + file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt") + + if not os.path.exists(file): + raise FileNotFoundError(f"can't find model states file at '{file}'") + + return file + + +def get_checkpoint_files(checkpoint_dir, glob_pattern): + # XXX: need to test that this simple glob rule works for multi-node setup too + ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys) + + if len(ckpt_files) == 0: + raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'") + + return ckpt_files + + +def get_optim_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt") + + +def get_model_state_files(checkpoint_dir): + return get_checkpoint_files(checkpoint_dir, "*_model_states.pt") + + +def parse_model_states(files): + zero_model_states = [] + for file in files: + state_dict = torch.load(file, map_location=device, weights_only=False) + + if BUFFER_NAMES not in state_dict: + raise ValueError(f"{file} is not a model state checkpoint") + buffer_names = state_dict[BUFFER_NAMES] + if debug: + print("Found buffers:", buffer_names) + + # recover just the buffers while restoring them to fp32 if they were saved in fp16 + buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names} + param_shapes = state_dict[PARAM_SHAPES] + + # collect parameters that are included in param_shapes + param_names = [] + for s in param_shapes: + for name in s.keys(): + param_names.append(name) + + # update with frozen parameters + frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None) + if frozen_param_shapes is not None: + if debug: + print(f"Found frozen_param_shapes: {frozen_param_shapes}") + param_names += list(frozen_param_shapes.keys()) + + # handle shared params + shared_params = [[k, v] for k, v in state_dict["shared_params"].items()] + + ds_version = state_dict.get(DS_VERSION, None) + + frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None) + + z_model_state = zero_model_state(buffers=buffers, + param_shapes=param_shapes, + shared_params=shared_params, + ds_version=ds_version, + frozen_param_shapes=frozen_param_shapes, + frozen_param_fragments=frozen_param_fragments) + zero_model_states.append(z_model_state) + + return zero_model_states + + +def parse_optim_states(files, ds_checkpoint_dir): + total_files = len(files) + state_dicts = [] + for f in tqdm(files, desc='Loading checkpoint shards'): + state_dict = torch.load(f, map_location=device, mmap=True, weights_only=False) + # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights + # and also handle the case where it was already removed by another helper script + state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None) + state_dicts.append(state_dict) + + if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]: + raise ValueError(f"{files[0]} is not a zero checkpoint") + zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE] + world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT] + + # For ZeRO-2 each param group can have different partition_count as data parallelism for expert + # parameters can be different from data parallelism for non-expert parameters. So we can just + # use the max of the partition_count to get the dp world_size. + + if type(world_size) is list: + world_size = max(world_size) + + if world_size != total_files: + raise ValueError( + f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. " + "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes." + ) + + # the groups are named differently in each stage + if zero_stage <= 2: + fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS + elif zero_stage == 3: + fp32_groups_key = FP32_FLAT_GROUPS + else: + raise ValueError(f"unknown zero stage {zero_stage}") + + fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))] + return zero_stage, world_size, fp32_flat_groups + + +def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters): + """ + Returns fp32 state_dict reconstructed from ds checkpoint + + Args: + - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are) + + """ + print(f"Processing zero checkpoint '{ds_checkpoint_dir}'") + + optim_files = get_optim_files(ds_checkpoint_dir) + zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir) + print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}") + + model_files = get_model_state_files(ds_checkpoint_dir) + + zero_model_states = parse_model_states(model_files) + print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}') + + if zero_stage <= 2: + return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + elif zero_stage == 3: + return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters) + + +def _zero2_merge_frozen_params(state_dict, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + frozen_param_fragments = zero_model_states[0].frozen_param_fragments + + if debug: + num_elem = sum(s.numel() for s in frozen_param_shapes.values()) + print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in frozen_param_fragments.values()]) + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + state_dict[name] = frozen_param_fragments[name] + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +def _has_callable(obj, fn): + attr = getattr(obj, fn, None) + return callable(attr) + + +def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + + # Reconstruction protocol: + # + # XXX: document this + + if debug: + for i in range(world_size): + for j in range(len(fp32_flat_groups[0])): + print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}") + + # XXX: memory usage doubles here (zero2) + num_param_groups = len(fp32_flat_groups[0]) + merged_single_partition_of_fp32_groups = [] + for i in range(num_param_groups): + merged_partitions = [sd[i] for sd in fp32_flat_groups] + full_single_fp32_vector = torch.cat(merged_partitions, 0) + merged_single_partition_of_fp32_groups.append(full_single_fp32_vector) + avail_numel = sum( + [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups]) + + if debug: + wanted_params = sum([len(shapes) for shapes in param_shapes]) + wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes]) + # not asserting if there is a mismatch due to possible padding + print(f"Have {avail_numel} numels to process.") + print(f"Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + total_numel = 0 + total_params = 0 + for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups): + offset = 0 + avail_numel = full_single_fp32_vector.numel() + for name, shape in shapes.items(): + + unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape) + total_numel += unpartitioned_numel + total_params += 1 + + if debug: + print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ") + state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape) + offset += unpartitioned_numel + + # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and + # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex + # paddings performed in the code it's almost impossible to predict the exact numbers w/o the + # live optimizer object, so we are checking that the numbers are within the right range + align_to = 2 * world_size + + def zero2_align(x): + return align_to * math.ceil(x / align_to) + + if debug: + print(f"original offset={offset}, avail_numel={avail_numel}") + + offset = zero2_align(offset) + avail_numel = zero2_align(avail_numel) + + if debug: + print(f"aligned offset={offset}, avail_numel={avail_numel}") + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero2_merge_frozen_params(state_dict, zero_model_states) + + _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def zero3_partitioned_param_info(unpartitioned_numel, world_size): + remainder = unpartitioned_numel % world_size + padding_numel = (world_size - remainder) if remainder else 0 + partitioned_numel = math.ceil(unpartitioned_numel / world_size) + return partitioned_numel, padding_numel + + +def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states): + if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0: + return + + if debug: + for i in range(world_size): + num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values()) + print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}') + + frozen_param_shapes = zero_model_states[0].frozen_param_shapes + wanted_params = len(frozen_param_shapes) + wanted_numel = sum(s.numel() for s in frozen_param_shapes.values()) + avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size + print(f'Frozen params: Have {avail_numel} numels to process.') + print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params') + + total_params = 0 + total_numel = 0 + for name, shape in zero_model_states[0].frozen_param_shapes.items(): + total_params += 1 + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + + param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states) + state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape) + + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements") + + +class GatheredTensor: + """ + A pseudo tensor that collects partitioned weights. + It is more memory efficient when there are multiple groups. + """ + + def __init__(self, flat_groups, flat_groups_offset, offset, partitioned_numel, shape): + self.flat_groups = flat_groups + self.flat_groups_offset = flat_groups_offset + self.offset = offset + self.partitioned_numel = partitioned_numel + self.shape = shape + self.dtype = self.flat_groups[0][0].dtype + + def contiguous(self): + """ + Merge partitioned weights from flat_groups into a single tensor. + """ + end_idx = self.offset + self.partitioned_numel + world_size = len(self.flat_groups) + pad_flat_param_chunks = [] + + for rank_i in range(world_size): + # for each rank, we need to collect weights from related group/groups + flat_groups_at_rank_i = self.flat_groups[rank_i] + start_group_id = None + end_group_id = None + for group_id in range(len(self.flat_groups_offset)): + if self.flat_groups_offset[group_id] <= self.offset < self.flat_groups_offset[group_id + 1]: + start_group_id = group_id + if self.flat_groups_offset[group_id] < end_idx <= self.flat_groups_offset[group_id + 1]: + end_group_id = group_id + break + # collect weights from related group/groups + for group_id in range(start_group_id, end_group_id + 1): + flat_tensor = flat_groups_at_rank_i[group_id] + start_offset = self.offset - self.flat_groups_offset[group_id] + end_offset = min(end_idx, self.flat_groups_offset[group_id + 1]) - self.flat_groups_offset[group_id] + pad_flat_param_chunks.append(flat_tensor[start_offset:end_offset]) + + # collect weights from all ranks + pad_flat_param = torch.cat(pad_flat_param_chunks, dim=0) + param = pad_flat_param[:self.shape.numel()].view(self.shape).contiguous() + return param + + +def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states): + param_shapes = zero_model_states[0].param_shapes + avail_numel = sum([flat_group.numel() for flat_group in fp32_flat_groups[0]]) * world_size + + # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each + # param, re-consolidating each param, while dealing with padding if any + + # merge list of dicts, preserving order + param_shapes = {k: v for d in param_shapes for k, v in d.items()} + + if debug: + for i in range(world_size): + print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}") + + wanted_params = len(param_shapes) + wanted_numel = sum(shape.numel() for shape in param_shapes.values()) + # not asserting if there is a mismatch due to possible padding + avail_numel = fp32_flat_groups[0].numel() * world_size + print(f"Trainable params: Have {avail_numel} numels to process.") + print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.") + + # params + # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support + # out-of-core computing solution + offset = 0 + total_numel = 0 + total_params = 0 + flat_groups_offset = [0] + list(np.cumsum([flat_tensor.numel() for flat_tensor in fp32_flat_groups[0]])) + for name, shape in tqdm(param_shapes.items(), desc='Gathering sharded weights'): + unpartitioned_numel = shape.numel() + total_numel += unpartitioned_numel + total_params += 1 + partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size) + + if debug: + print( + f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}" + ) + + # memory efficient tensor + tensor = GatheredTensor(fp32_flat_groups, flat_groups_offset, offset, partitioned_numel, shape) + state_dict[name] = tensor + offset += partitioned_numel + + offset *= world_size + + # Sanity check + if offset != avail_numel: + raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong") + + print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements") + + +def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states, + exclude_frozen_parameters): + state_dict = OrderedDict() + + # buffers + buffers = zero_model_states[0].buffers + state_dict.update(buffers) + if debug: + print(f"added {len(buffers)} buffers") + + if not exclude_frozen_parameters: + _zero3_merge_frozen_params(state_dict, world_size, zero_model_states) + + _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states) + + # recover shared parameters + for pair in zero_model_states[0].shared_params: + if pair[1] in state_dict: + state_dict[pair[0]] = state_dict[pair[1]] + + return state_dict + + +def to_torch_tensor(state_dict, return_empty_tensor=False): + """ + Convert state_dict of GatheredTensor to torch tensor + """ + torch_state_dict = {} + converted_tensors = {} + def convert_tensor(qin): + while True: + name, tensor = qin.get() + if name is None: + return + tensor_id = id(tensor) + if tensor_id in converted_tensors: + shared_tensor = torch_state_dict[converted_tensors[tensor_id]] + torch_state_dict[name] = shared_tensor.to(torch.bfloat16) + else: + converted_tensors[tensor_id] = name + if return_empty_tensor: + torch_state_dict[name] = torch.empty(tensor.shape, dtype=tensor.dtype).to(torch.bfloat16) + else: + torch_state_dict[name] = tensor.contiguous().to(torch.bfloat16) + + num_threads = 32 + qin = queue.Queue(num_threads) + threads = [Thread(target=convert_tensor, args=(qin, )) for _ in range(num_threads)] + [_.start() for _ in threads] + cnt = 0 + for name, tensor in state_dict.items(): + cnt += 1 + qin.put([name, tensor]) + if cnt % 1000 == 0: + print(f'{cnt} / {len(state_dict)}') + for _ in range(num_threads): + qin.put([None, None]) + [_.join() for _ in threads] + return torch_state_dict + + +def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag=None, + exclude_frozen_parameters=False, + lazy_mode=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with + ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example + via a model hub. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + - ``lazy_mode``: get state_dict in lazy mode. It returns a dict of pesduo tensor instead of torch tensor, which is more memory efficient. + Convert the pesduo tensor to torch tensor by ``.contiguous()`` + + Returns: + - pytorch ``state_dict`` + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + # do the training and checkpoint saving + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu + model = model.cpu() # move to cpu + model.load_state_dict(state_dict) + # submit to model hub or save the model to share with others + + In this example the ``model`` will no longer be usable in the deepspeed context of the same + application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead. + + Note: the above usage may not work if your application doesn't have sufficient free CPU memory. + You may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with + the checkpoint. Or you can load state_dict in lazy mode :: + + from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, lazy_mode=True) # not on cpu + for name, lazy_tensor in state_dict.item(): + tensor = lazy_tensor.contiguous() # to cpu + print(name, tensor) + # del tensor to release memory if it no longer in use + """ + if tag is None: + latest_path = os.path.join(checkpoint_dir, 'latest') + if os.path.isfile(latest_path): + with open(latest_path, 'r') as fd: + tag = fd.read().strip() + else: + raise ValueError(f"Unable to find 'latest' file at {latest_path}") + + ds_checkpoint_dir = os.path.join(checkpoint_dir, tag) + + if not os.path.isdir(ds_checkpoint_dir): + raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist") + + state_dict = _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters) + if lazy_mode: + return state_dict + else: + return to_torch_tensor(state_dict) + + +def convert_zero_checkpoint_to_bf16_state_dict(checkpoint_dir, + output_dir, + max_shard_size="5GB", + safe_serialization=False, + tag=None, + exclude_frozen_parameters=False): + """ + Convert ZeRO 2 or 3 checkpoint into a single bf16 consolidated ``state_dict`` file that can be + loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed. + + Args: + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``output_dir``: directory to the pytorch bf16 state_dict output files + - ``max_shard_size``: the maximum size for a checkpoint before being sharded, default value is 5GB + - ``safe_serialization``: whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + - ``exclude_frozen_parameters``: exclude frozen parameters + """ + + # Dependency pre-check + if safe_serialization: + try: + from safetensors.torch import save_file + except ImportError: + print('If you want to use `safe_serialization`, please `pip install safetensors`') + raise + if max_shard_size is not None: + try: + from huggingface_hub import split_torch_state_dict_into_shards + except ImportError: + print('If you want to use `max_shard_size`, please `pip install huggingface_hub`') + raise + + # Convert zero checkpoint to state_dict + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, + tag, + exclude_frozen_parameters, + lazy_mode=True) + + # Shard the model if it is too big. + weights_name = "diffusion_pytorch_model.safetensors" if safe_serialization else "pytorch_model.bin" + if max_shard_size is not None: + filename_pattern = weights_name.replace(".bin", "{suffix}.bin").replace(".safetensors", "{suffix}.safetensors") + # an memory-efficient approach for sharding + empty_state_dict = to_torch_tensor(state_dict, return_empty_tensor=True) + state_dict_split = split_torch_state_dict_into_shards(empty_state_dict, + filename_pattern=filename_pattern, + max_shard_size=max_shard_size) + else: + from collections import namedtuple + StateDictSplit = namedtuple("StateDictSplit", ["is_sharded", "filename_to_tensors"]) + state_dict_split = StateDictSplit(is_sharded=False, + filename_to_tensors={weights_name: list(state_dict.keys())}) + + # Save the model by shard + os.makedirs(output_dir, exist_ok=True) + filename_to_tensors = state_dict_split.filename_to_tensors.items() + for shard_file, tensors in tqdm(filename_to_tensors, desc="Saving checkpoint shards"): + shard_state_dict = {tensor_name: state_dict[tensor_name] for tensor_name in tensors} + shard_state_dict = to_torch_tensor(shard_state_dict) + # to bf16 + shard_state_dict = { + tensor_name: shard_state_dict[tensor_name].to(torch.bfloat16) for tensor_name in shard_state_dict + } + print('save shard_state_dict') + output_path = os.path.join(output_dir, shard_file) + if safe_serialization: + save_file(shard_state_dict, output_path, metadata={"format": "pt"}) + else: + torch.save(shard_state_dict, output_path) + # release the memory of current shard + for tensor_name in list(shard_state_dict.keys()): + del state_dict[tensor_name] + del shard_state_dict[tensor_name] + del shard_state_dict + gc.collect() + + # Save index if sharded + if state_dict_split.is_sharded: + index = { + "metadata": state_dict_split.metadata, + "weight_map": state_dict_split.tensor_to_filename, + } + save_index_file = "model.safetensors.index.json" if safe_serialization else "pytorch_model.bin.index.json" + save_index_file = os.path.join(output_dir, save_index_file) + with open(save_index_file, "w", encoding="utf-8") as f: + content = json.dumps(index, indent=2, sort_keys=True) + "\n" + f.write(content) + + +def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None): + """ + 1. Put the provided model to cpu + 2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` + 3. Load it into the provided model + + Args: + - ``model``: the model object to update + - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``) + - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14`` + + Returns: + - ``model`: modified model + + Make sure you have plenty of CPU memory available before you call this function. If you don't + have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it + conveniently placed for you in the checkpoint folder. + + A typical usage might be :: + + from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint + model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir) + # submit to model hub or save the model to share with others + + Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context + of the same application. i.e. you will need to re-initialize the deepspeed engine, since + ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it. + + """ + logger.info(f"Extracting fp32 weights") + state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag) + + logger.info(f"Overwriting model with fp32 weights") + model = model.cpu() + model.load_state_dict(state_dict, strict=False) + + return model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("checkpoint_dir", + type=str, + help="path to the desired checkpoint folder, e.g., path/checkpoint-12") + parser.add_argument("output_dir", + type=str, + help="directory to the pytorch bf16 state_dict output files" + "(e.g. path/checkpoint-12-output/)") + parser.add_argument( + "--max_shard_size", + type=str, + default="80GB", + help="The maximum size for a checkpoint before being sharded. Checkpoints shard will then be each of size" + "lower than this size. ") + parser.add_argument( + "--safe_serialization", + default=True, + action='store_true', + help="Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`).") + parser.add_argument("-t", + "--tag", + type=str, + default=None, + help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1") + parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters") + parser.add_argument("-d", "--debug", action='store_true', help="enable debug") + args = parser.parse_args() + + debug = args.debug + + convert_zero_checkpoint_to_bf16_state_dict(args.checkpoint_dir, + args.output_dir, + max_shard_size=args.max_shard_size, + safe_serialization=args.safe_serialization, + tag=args.tag, + exclude_frozen_parameters=args.exclude_frozen_parameters) diff --git a/VideoX-Fun/slim_all_onnx.py b/VideoX-Fun/slim_all_onnx.py new file mode 100644 index 0000000000000000000000000000000000000000..96dbebf05229e3c844b60fc735e8962bb8bac577 --- /dev/null +++ b/VideoX-Fun/slim_all_onnx.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +""" +Script to run onnxslim on all ONNX files in transformers_body_only_1728_992_split_onnx directory +""" +import os +import shutil +import subprocess +from pathlib import Path + +def main(): + # Directory containing ONNX files + onnx_dir = Path("transformers_body_only_1728_992_split_onnx") + + if not onnx_dir.exists(): + print(f"Error: Directory {onnx_dir} not found!") + return + + # Get all .onnx files + onnx_files = sorted(onnx_dir.glob("*.onnx")) + + if not onnx_files: + print(f"No ONNX files found in {onnx_dir}") + return + + print(f"Found {len(onnx_files)} ONNX files") + print("=" * 80) + + success_count = 0 + failed_files = [] + + for idx, onnx_file in enumerate(onnx_files, 1): + print(f"\n[{idx}/{len(onnx_files)}] Processing: {onnx_file.name}") + print(f"File size: {onnx_file.stat().st_size / (1024**3):.2f} GB") + + # Create backup before processing + backup_file = onnx_file.with_suffix(".onnx.backup") + shutil.copy2(onnx_file, backup_file) + + try: + # Run onnxslim: onnxslim input.onnx output.onnx (in-place) + cmd = ["onnxslim", str(onnx_file), str(onnx_file)] + print(f"Running: {' '.join(cmd)}") + + result = subprocess.run( + cmd, + check=True, + capture_output=True, + text=True + ) + + print(f"✓ Success: {onnx_file.name}") + if result.stdout: + print(f" Output: {result.stdout.strip()}") + success_count += 1 + + # Remove backup on success + backup_file.unlink() + + except subprocess.CalledProcessError as e: + print(f"✗ Failed: {onnx_file.name}") + print(f" Error: {e.stderr}") + failed_files.append(onnx_file.name) + + # Restore from backup + if backup_file.exists(): + shutil.copy2(backup_file, onnx_file) + backup_file.unlink() + print(f" Restored from backup") + + except Exception as e: + print(f"✗ Failed: {onnx_file.name}") + print(f" Error: {str(e)}") + failed_files.append(onnx_file.name) + + # Restore from backup + if backup_file.exists(): + shutil.copy2(backup_file, onnx_file) + backup_file.unlink() + print(f" Restored from backup") + + # Summary + print("\n" + "=" * 80) + print("SUMMARY") + print("=" * 80) + print(f"Total files: {len(onnx_files)}") + print(f"Successful: {success_count}") + print(f"Failed: {len(failed_files)}") + + if failed_files: + print("\nFailed files:") + for file in failed_files: + print(f" - {file}") + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/vae_decoder.axmodel b/VideoX-Fun/vae_decoder.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..1ea06e0a104479b4d88e16144ec011a533757ae9 --- /dev/null +++ b/VideoX-Fun/vae_decoder.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0798299fccdffae63be19d73323279e0f4b83883f634faaa88a44a9fe09ccc19 +size 71404143 diff --git a/VideoX-Fun/videox_fun/api/api.py b/VideoX-Fun/videox_fun/api/api.py new file mode 100644 index 0000000000000000000000000000000000000000..a3c0238a7ce3e8c87e61fd3c4e062b956a456a1f --- /dev/null +++ b/VideoX-Fun/videox_fun/api/api.py @@ -0,0 +1,226 @@ +import base64 +import gc +import hashlib +import io +import os +import tempfile +from io import BytesIO + +import gradio as gr +import requests +import torch +from fastapi import FastAPI +from PIL import Image + + +# Function to encode a file to Base64 +def encode_file_to_base64(file_path): + with open(file_path, "rb") as file: + # Encode the data to Base64 + file_base64 = base64.b64encode(file.read()) + return file_base64 + +def update_diffusion_transformer_api(_: gr.Blocks, app: FastAPI, controller): + @app.post("/videox_fun/update_diffusion_transformer") + def _update_diffusion_transformer_api( + datas: dict, + ): + diffusion_transformer_path = datas.get('diffusion_transformer_path', 'none') + + try: + controller.update_diffusion_transformer( + diffusion_transformer_path + ) + comment = "Success" + except Exception as e: + torch.cuda.empty_cache() + comment = f"Error. error information is {str(e)}" + + return {"message": comment} + +def download_from_url(url, timeout=10): + try: + response = requests.get(url, timeout=timeout) + response.raise_for_status() # 检查请求是否成功 + return response.content + except requests.exceptions.RequestException as e: + print(f"Error downloading from {url}: {e}") + return None + +def save_base64_video(base64_string): + video_data = base64.b64decode(base64_string) + + md5_hash = hashlib.md5(video_data).hexdigest() + filename = f"{md5_hash}.mp4" + + temp_dir = tempfile.gettempdir() + file_path = os.path.join(temp_dir, filename) + + with open(file_path, 'wb') as video_file: + video_file.write(video_data) + + return file_path + +def save_base64_image(base64_string): + video_data = base64.b64decode(base64_string) + + md5_hash = hashlib.md5(video_data).hexdigest() + filename = f"{md5_hash}.jpg" + + temp_dir = tempfile.gettempdir() + file_path = os.path.join(temp_dir, filename) + + with open(file_path, 'wb') as video_file: + video_file.write(video_data) + + return file_path + +def save_url_video(url): + video_data = download_from_url(url) + if video_data: + return save_base64_video(base64.b64encode(video_data)) + return None + +def save_url_image(url): + image_data = download_from_url(url) + if image_data: + return save_base64_image(base64.b64encode(image_data)) + return None + +def infer_forward_api(_: gr.Blocks, app: FastAPI, controller): + @app.post("/videox_fun/infer_forward") + def _infer_forward_api( + datas: dict, + ): + base_model_path = datas.get('base_model_path', 'none') + base_model_2_path = datas.get('base_model_2_path', 'none') + lora_model_path = datas.get('lora_model_path', 'none') + lora_model_2_path = datas.get('lora_model_2_path', 'none') + lora_alpha_slider = datas.get('lora_alpha_slider', 0.55) + prompt_textbox = datas.get('prompt_textbox', None) + negative_prompt_textbox = datas.get('negative_prompt_textbox', 'The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion. ') + sampler_dropdown = datas.get('sampler_dropdown', 'Euler') + sample_step_slider = datas.get('sample_step_slider', 30) + resize_method = datas.get('resize_method', "Generate by") + width_slider = datas.get('width_slider', 672) + height_slider = datas.get('height_slider', 384) + base_resolution = datas.get('base_resolution', 512) + is_image = datas.get('is_image', False) + generation_method = datas.get('generation_method', False) + length_slider = datas.get('length_slider', 49) + overlap_video_length = datas.get('overlap_video_length', 4) + partial_video_length = datas.get('partial_video_length', 72) + cfg_scale_slider = datas.get('cfg_scale_slider', 6) + start_image = datas.get('start_image', None) + end_image = datas.get('end_image', None) + validation_video = datas.get('validation_video', None) + validation_video_mask = datas.get('validation_video_mask', None) + control_video = datas.get('control_video', None) + denoise_strength = datas.get('denoise_strength', 0.70) + seed_textbox = datas.get("seed_textbox", 43) + + ref_image = datas.get('ref_image', None) + enable_teacache = datas.get('enable_teacache', True) + teacache_threshold = datas.get('teacache_threshold', 0.10) + num_skip_start_steps = datas.get('num_skip_start_steps', 1) + teacache_offload = datas.get('teacache_offload', False) + cfg_skip_ratio = datas.get('cfg_skip_ratio', 0) + enable_riflex = datas.get('enable_riflex', False) + riflex_k = datas.get('riflex_k', 6) + fps = datas.get('fps', None) + + generation_method = "Image Generation" if is_image else generation_method + + if start_image is not None: + if start_image.startswith('http'): + start_image = save_url_image(start_image) + start_image = [Image.open(start_image).convert("RGB")] + else: + start_image = base64.b64decode(start_image) + start_image = [Image.open(BytesIO(start_image)).convert("RGB")] + + if end_image is not None: + if end_image.startswith('http'): + end_image = save_url_image(end_image) + end_image = [Image.open(end_image).convert("RGB")] + else: + end_image = base64.b64decode(end_image) + end_image = [Image.open(BytesIO(end_image)).convert("RGB")] + + if validation_video is not None: + if validation_video.startswith('http'): + validation_video = save_url_video(validation_video) + else: + validation_video = save_base64_video(validation_video) + + if validation_video_mask is not None: + if validation_video_mask.startswith('http'): + validation_video_mask = save_url_image(validation_video_mask) + else: + validation_video_mask = save_base64_image(validation_video_mask) + + if control_video is not None: + if control_video.startswith('http'): + control_video = save_url_video(control_video) + else: + control_video = save_base64_video(control_video) + + if ref_image is not None: + if ref_image.startswith('http'): + ref_image = save_url_image(ref_image) + ref_image = [Image.open(ref_image).convert("RGB")] + else: + ref_image = base64.b64decode(ref_image) + ref_image = [Image.open(BytesIO(ref_image)).convert("RGB")] + + try: + save_sample_path, comment = controller.generate( + "", + base_model_path, + lora_model_path, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image = ref_image, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + base_model_2_dropdown = base_model_2_path, + lora_model_2_dropdown = lora_model_2_path, + fps = fps, + is_api = True, + ) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_sample_path = "" + comment = f"Error. error information is {str(e)}" + return {"message": comment, "save_sample_path": None, "base64_encoding": None} + + if save_sample_path != "": + return {"message": comment, "save_sample_path": save_sample_path, "base64_encoding": encode_file_to_base64(save_sample_path)} + else: + return {"message": comment, "save_sample_path": save_sample_path, "base64_encoding": None} \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/api/api_multi_nodes.py b/VideoX-Fun/videox_fun/api/api_multi_nodes.py new file mode 100644 index 0000000000000000000000000000000000000000..ebd98d9f8435127f216da441baf29b1b7c916689 --- /dev/null +++ b/VideoX-Fun/videox_fun/api/api_multi_nodes.py @@ -0,0 +1,320 @@ +# This file is modified from https://github.com/xdit-project/xDiT/blob/main/entrypoints/launch.py +import base64 +import gc +import hashlib +import io +import os +import tempfile +from io import BytesIO + +import gradio as gr +import requests +import torch +import torch.distributed as dist +from fastapi import FastAPI, HTTPException +from PIL import Image + +from .api import download_from_url, encode_file_to_base64 + +try: + import ray +except: + print("Ray is not installed. If you want to use multi gpus api. Please install it by running 'pip install ray'.") + ray = None + +def save_base64_video_dist(base64_string): + video_data = base64.b64decode(base64_string) + + md5_hash = hashlib.md5(video_data).hexdigest() + filename = f"{md5_hash}.mp4" + + temp_dir = tempfile.gettempdir() + file_path = os.path.join(temp_dir, filename) + + if dist.is_initialized(): + if dist.get_rank() == 0: + with open(file_path, 'wb') as video_file: + video_file.write(video_data) + dist.barrier() + else: + with open(file_path, 'wb') as video_file: + video_file.write(video_data) + return file_path + +def save_base64_image_dist(base64_string): + video_data = base64.b64decode(base64_string) + + md5_hash = hashlib.md5(video_data).hexdigest() + filename = f"{md5_hash}.jpg" + + temp_dir = tempfile.gettempdir() + file_path = os.path.join(temp_dir, filename) + + if dist.is_initialized(): + if dist.get_rank() == 0: + with open(file_path, 'wb') as video_file: + video_file.write(video_data) + dist.barrier() + else: + with open(file_path, 'wb') as video_file: + video_file.write(video_data) + return file_path + +def save_url_video_dist(url): + video_data = download_from_url(url) + if video_data: + return save_base64_video_dist(base64.b64encode(video_data)) + return None + +def save_url_image_dist(url): + image_data = download_from_url(url) + if image_data: + return save_base64_image_dist(base64.b64encode(image_data)) + return None + +if ray is not None: + @ray.remote(num_gpus=1) + class MultiNodesGenerator: + def __init__( + self, rank: int, world_size: int, Controller, + GPU_memory_mode, scheduler_dict, model_name=None, model_type="Inpaint", + config_path=None, ulysses_degree=1, ring_degree=1, + fsdp_dit=False, fsdp_text_encoder=False, compile_dit=False, + weight_dtype=None, savedir_sample=None, + ): + # Set PyTorch distributed environment variables + os.environ["RANK"] = str(rank) + os.environ["WORLD_SIZE"] = str(world_size) + os.environ["MASTER_ADDR"] = "127.0.0.1" + os.environ["MASTER_PORT"] = "29500" + + self.rank = rank + self.controller = Controller( + GPU_memory_mode, scheduler_dict, model_name=model_name, model_type=model_type, config_path=config_path, + ulysses_degree=ulysses_degree, ring_degree=ring_degree, + fsdp_dit=fsdp_dit, fsdp_text_encoder=fsdp_text_encoder, compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + def generate(self, datas): + try: + base_model_path = datas.get('base_model_path', 'none') + base_model_2_path = datas.get('base_model_2_path', 'none') + lora_model_path = datas.get('lora_model_path', 'none') + lora_model_2_path = datas.get('lora_model_2_path', 'none') + lora_alpha_slider = datas.get('lora_alpha_slider', 0.55) + prompt_textbox = datas.get('prompt_textbox', None) + negative_prompt_textbox = datas.get('negative_prompt_textbox', 'The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion. ') + sampler_dropdown = datas.get('sampler_dropdown', 'Euler') + sample_step_slider = datas.get('sample_step_slider', 30) + resize_method = datas.get('resize_method', "Generate by") + width_slider = datas.get('width_slider', 672) + height_slider = datas.get('height_slider', 384) + base_resolution = datas.get('base_resolution', 512) + is_image = datas.get('is_image', False) + generation_method = datas.get('generation_method', False) + length_slider = datas.get('length_slider', 49) + overlap_video_length = datas.get('overlap_video_length', 4) + partial_video_length = datas.get('partial_video_length', 72) + cfg_scale_slider = datas.get('cfg_scale_slider', 6) + start_image = datas.get('start_image', None) + end_image = datas.get('end_image', None) + validation_video = datas.get('validation_video', None) + validation_video_mask = datas.get('validation_video_mask', None) + control_video = datas.get('control_video', None) + denoise_strength = datas.get('denoise_strength', 0.70) + seed_textbox = datas.get("seed_textbox", 43) + + ref_image = datas.get('ref_image', None) + enable_teacache = datas.get('enable_teacache', True) + teacache_threshold = datas.get('teacache_threshold', 0.10) + num_skip_start_steps = datas.get('num_skip_start_steps', 1) + teacache_offload = datas.get('teacache_offload', False) + cfg_skip_ratio = datas.get('cfg_skip_ratio', 0) + enable_riflex = datas.get('enable_riflex', False) + riflex_k = datas.get('riflex_k', 6) + fps = datas.get('fps', None) + + generation_method = "Image Generation" if is_image else generation_method + + if start_image is not None: + if start_image.startswith('http'): + start_image = save_url_image_dist(start_image) + start_image = [Image.open(start_image).convert("RGB")] + else: + start_image = base64.b64decode(start_image) + start_image = [Image.open(BytesIO(start_image)).convert("RGB")] + + if end_image is not None: + if end_image.startswith('http'): + end_image = save_url_image_dist(end_image) + end_image = [Image.open(end_image).convert("RGB")] + else: + end_image = base64.b64decode(end_image) + end_image = [Image.open(BytesIO(end_image)).convert("RGB")] + + if validation_video is not None: + if validation_video.startswith('http'): + validation_video = save_url_video_dist(validation_video) + else: + validation_video = save_base64_video_dist(validation_video) + + if validation_video_mask is not None: + if validation_video_mask.startswith('http'): + validation_video_mask = save_url_image_dist(validation_video_mask) + else: + validation_video_mask = save_base64_image_dist(validation_video_mask) + + if control_video is not None: + if control_video.startswith('http'): + control_video = save_url_video_dist(control_video) + else: + control_video = save_base64_video_dist(control_video) + + if ref_image is not None: + if ref_image.startswith('http'): + ref_image = save_url_image_dist(ref_image) + ref_image = [Image.open(ref_image).convert("RGB")] + else: + ref_image = base64.b64decode(ref_image) + ref_image = [Image.open(BytesIO(ref_image)).convert("RGB")] + + try: + save_sample_path, comment = self.controller.generate( + "", + base_model_path, + lora_model_path, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image = ref_image, + enable_teacache = enable_teacache, + teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, + teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, + enable_riflex = enable_riflex, + riflex_k = riflex_k, + base_model_2_dropdown = base_model_2_path, + lora_model_2_dropdown = lora_model_2_path, + fps = fps, + is_api = True, + ) + except Exception as e: + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + save_sample_path = "" + comment = f"Error. error information is {str(e)}" + if dist.is_initialized(): + if dist.get_rank() == 0: + return {"message": comment, "save_sample_path": None, "base64_encoding": None} + else: + return None + else: + return {"message": comment, "save_sample_path": None, "base64_encoding": None} + + + if dist.is_initialized(): + if dist.get_rank() == 0: + if save_sample_path != "": + return {"message": comment, "save_sample_path": save_sample_path, "base64_encoding": encode_file_to_base64(save_sample_path)} + else: + return {"message": comment, "save_sample_path": None, "base64_encoding": None} + else: + return None + else: + if save_sample_path != "": + return {"message": comment, "save_sample_path": save_sample_path, "base64_encoding": encode_file_to_base64(save_sample_path)} + else: + return {"message": comment, "save_sample_path": None, "base64_encoding": None} + + except Exception as e: + print(f"Error generating: {str(e)}") + comment = f"Error generating: {str(e)}" + if dist.is_initialized(): + if dist.get_rank() == 0: + return {"message": comment, "save_sample_path": None, "base64_encoding": None} + else: + return None + else: + return {"message": comment, "save_sample_path": None, "base64_encoding": None} + + class MultiNodesEngine: + def __init__( + self, + world_size, + Controller, + GPU_memory_mode, + scheduler_dict, + model_name, + model_type, + config_path, + ulysses_degree=1, + ring_degree=1, + fsdp_dit=False, + fsdp_text_encoder=False, + compile_dit=False, + weight_dtype=torch.bfloat16, + savedir_sample="samples" + ): + # Ensure Ray is initialized + if not ray.is_initialized(): + ray.init() + + num_workers = world_size + self.workers = [ + MultiNodesGenerator.remote( + rank, world_size, Controller, + GPU_memory_mode, scheduler_dict, model_name=model_name, model_type=model_type, config_path=config_path, + ulysses_degree=ulysses_degree, ring_degree=ring_degree, + fsdp_dit=fsdp_dit, fsdp_text_encoder=fsdp_text_encoder, compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + for rank in range(num_workers) + ] + print("Update workers done") + + async def generate(self, data): + results = ray.get([ + worker.generate.remote(data) + for worker in self.workers + ]) + + return next(path for path in results if path is not None) + + def multi_nodes_infer_forward_api(_: gr.Blocks, app: FastAPI, engine): + + @app.post("/videox_fun/infer_forward") + async def _multi_nodes_infer_forward_api( + datas: dict, + ): + try: + result = await engine.generate(datas) + return result + except Exception as e: + if isinstance(e, HTTPException): + raise e + raise HTTPException(status_code=500, detail=str(e)) +else: + MultiNodesEngine = None + MultiNodesGenerator = None + multi_nodes_infer_forward_api = None \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/data/bucket_sampler.py b/VideoX-Fun/videox_fun/data/bucket_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..24b4160f3d2bbadca1d23e90c23e887ea6d15f70 --- /dev/null +++ b/VideoX-Fun/videox_fun/data/bucket_sampler.py @@ -0,0 +1,379 @@ +# Copyright (c) OpenMMLab. All rights reserved. +import os +from typing import (Generic, Iterable, Iterator, List, Optional, Sequence, + Sized, TypeVar, Union) + +import cv2 +import numpy as np +import torch +from PIL import Image +from torch.utils.data import BatchSampler, Dataset, Sampler + +ASPECT_RATIO_512 = { + '0.25': [256.0, 1024.0], '0.26': [256.0, 992.0], '0.27': [256.0, 960.0], '0.28': [256.0, 928.0], + '0.32': [288.0, 896.0], '0.33': [288.0, 864.0], '0.35': [288.0, 832.0], '0.4': [320.0, 800.0], + '0.42': [320.0, 768.0], '0.48': [352.0, 736.0], '0.5': [352.0, 704.0], '0.52': [352.0, 672.0], + '0.57': [384.0, 672.0], '0.6': [384.0, 640.0], '0.68': [416.0, 608.0], '0.72': [416.0, 576.0], + '0.78': [448.0, 576.0], '0.82': [448.0, 544.0], '0.88': [480.0, 544.0], '0.94': [480.0, 512.0], + '1.0': [512.0, 512.0], '1.07': [512.0, 480.0], '1.13': [544.0, 480.0], '1.21': [544.0, 448.0], + '1.29': [576.0, 448.0], '1.38': [576.0, 416.0], '1.46': [608.0, 416.0], '1.67': [640.0, 384.0], + '1.75': [672.0, 384.0], '2.0': [704.0, 352.0], '2.09': [736.0, 352.0], '2.4': [768.0, 320.0], + '2.5': [800.0, 320.0], '2.89': [832.0, 288.0], '3.0': [864.0, 288.0], '3.11': [896.0, 288.0], + '3.62': [928.0, 256.0], '3.75': [960.0, 256.0], '3.88': [992.0, 256.0], '4.0': [1024.0, 256.0] +} +ASPECT_RATIO_RANDOM_CROP_512 = { + '0.42': [320.0, 768.0], '0.5': [352.0, 704.0], + '0.57': [384.0, 672.0], '0.68': [416.0, 608.0], '0.78': [448.0, 576.0], '0.88': [480.0, 544.0], + '0.94': [480.0, 512.0], '1.0': [512.0, 512.0], '1.07': [512.0, 480.0], + '1.13': [544.0, 480.0], '1.29': [576.0, 448.0], '1.46': [608.0, 416.0], '1.75': [672.0, 384.0], + '2.0': [704.0, 352.0], '2.4': [768.0, 320.0] +} +ASPECT_RATIO_RANDOM_CROP_PROB = [ + 1, 2, + 4, 4, 4, 4, + 8, 8, 8, + 4, 4, 4, 4, + 2, 1 +] +ASPECT_RATIO_RANDOM_CROP_PROB = np.array(ASPECT_RATIO_RANDOM_CROP_PROB) / sum(ASPECT_RATIO_RANDOM_CROP_PROB) + +def get_closest_ratio(height: float, width: float, ratios: dict = ASPECT_RATIO_512): + aspect_ratio = height / width + closest_ratio = min(ratios.keys(), key=lambda ratio: abs(float(ratio) - aspect_ratio)) + return ratios[closest_ratio], float(closest_ratio) + +def get_image_size_without_loading(path): + with Image.open(path) as img: + return img.size # (width, height) + +class RandomSampler(Sampler[int]): + r"""Samples elements randomly. If without replacement, then sample from a shuffled dataset. + + If with replacement, then user can specify :attr:`num_samples` to draw. + + Args: + data_source (Dataset): dataset to sample from + replacement (bool): samples are drawn on-demand with replacement if ``True``, default=``False`` + num_samples (int): number of samples to draw, default=`len(dataset)`. + generator (Generator): Generator used in sampling. + """ + + data_source: Sized + replacement: bool + + def __init__(self, data_source: Sized, replacement: bool = False, + num_samples: Optional[int] = None, generator=None) -> None: + self.data_source = data_source + self.replacement = replacement + self._num_samples = num_samples + self.generator = generator + self._pos_start = 0 + + if not isinstance(self.replacement, bool): + raise TypeError(f"replacement should be a boolean value, but got replacement={self.replacement}") + + if not isinstance(self.num_samples, int) or self.num_samples <= 0: + raise ValueError(f"num_samples should be a positive integer value, but got num_samples={self.num_samples}") + + @property + def num_samples(self) -> int: + # dataset size might change at runtime + if self._num_samples is None: + return len(self.data_source) + return self._num_samples + + def __iter__(self) -> Iterator[int]: + n = len(self.data_source) + if self.generator is None: + seed = int(torch.empty((), dtype=torch.int64).random_().item()) + generator = torch.Generator() + generator.manual_seed(seed) + else: + generator = self.generator + + if self.replacement: + for _ in range(self.num_samples // 32): + yield from torch.randint(high=n, size=(32,), dtype=torch.int64, generator=generator).tolist() + yield from torch.randint(high=n, size=(self.num_samples % 32,), dtype=torch.int64, generator=generator).tolist() + else: + for _ in range(self.num_samples // n): + xx = torch.randperm(n, generator=generator).tolist() + if self._pos_start >= n: + self._pos_start = 0 + print("xx top 10", xx[:10], self._pos_start) + for idx in range(self._pos_start, n): + yield xx[idx] + self._pos_start = (self._pos_start + 1) % n + self._pos_start = 0 + yield from torch.randperm(n, generator=generator).tolist()[:self.num_samples % n] + + def __len__(self) -> int: + return self.num_samples + +class AspectRatioBatchImageSampler(BatchSampler): + """A sampler wrapper for grouping images with similar aspect ratio into a same batch. + + Args: + sampler (Sampler): Base sampler. + dataset (Dataset): Dataset providing data information. + batch_size (int): Size of mini-batch. + drop_last (bool): If ``True``, the sampler will drop the last batch if + its size would be less than ``batch_size``. + aspect_ratios (dict): The predefined aspect ratios. + """ + def __init__( + self, + sampler: Sampler, + dataset: Dataset, + batch_size: int, + train_folder: str = None, + aspect_ratios: dict = ASPECT_RATIO_512, + drop_last: bool = False, + config=None, + **kwargs + ) -> None: + if not isinstance(sampler, Sampler): + raise TypeError('sampler should be an instance of ``Sampler``, ' + f'but got {sampler}') + if not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError('batch_size should be a positive integer value, ' + f'but got batch_size={batch_size}') + self.sampler = sampler + self.dataset = dataset + self.train_folder = train_folder + self.batch_size = batch_size + self.aspect_ratios = aspect_ratios + self.drop_last = drop_last + self.config = config + # buckets for each aspect ratio + self._aspect_ratio_buckets = {ratio: [] for ratio in aspect_ratios} + # [str(k) for k, v in aspect_ratios] + self.current_available_bucket_keys = list(aspect_ratios.keys()) + + def __iter__(self): + for idx in self.sampler: + try: + image_dict = self.dataset[idx] + + width, height = image_dict.get("width", None), image_dict.get("height", None) + if width is None or height is None: + image_id, name = image_dict['file_path'], image_dict['text'] + if self.train_folder is None: + image_dir = image_id + else: + image_dir = os.path.join(self.train_folder, image_id) + + width, height = get_image_size_without_loading(image_dir) + + ratio = height / width # self.dataset[idx] + else: + height = int(height) + width = int(width) + ratio = height / width # self.dataset[idx] + except Exception as e: + print(e) + continue + # find the closest aspect ratio + closest_ratio = min(self.aspect_ratios.keys(), key=lambda r: abs(float(r) - ratio)) + if closest_ratio not in self.current_available_bucket_keys: + continue + bucket = self._aspect_ratio_buckets[closest_ratio] + bucket.append(idx) + # yield a batch of indices in the same aspect ratio group + if len(bucket) == self.batch_size: + yield bucket[:] + del bucket[:] + +class AspectRatioBatchSampler(BatchSampler): + """A sampler wrapper for grouping images with similar aspect ratio into a same batch. + + Args: + sampler (Sampler): Base sampler. + dataset (Dataset): Dataset providing data information. + batch_size (int): Size of mini-batch. + drop_last (bool): If ``True``, the sampler will drop the last batch if + its size would be less than ``batch_size``. + aspect_ratios (dict): The predefined aspect ratios. + """ + def __init__( + self, + sampler: Sampler, + dataset: Dataset, + batch_size: int, + video_folder: str = None, + train_data_format: str = "webvid", + aspect_ratios: dict = ASPECT_RATIO_512, + drop_last: bool = False, + config=None, + **kwargs + ) -> None: + if not isinstance(sampler, Sampler): + raise TypeError('sampler should be an instance of ``Sampler``, ' + f'but got {sampler}') + if not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError('batch_size should be a positive integer value, ' + f'but got batch_size={batch_size}') + self.sampler = sampler + self.dataset = dataset + self.video_folder = video_folder + self.train_data_format = train_data_format + self.batch_size = batch_size + self.aspect_ratios = aspect_ratios + self.drop_last = drop_last + self.config = config + # buckets for each aspect ratio + self._aspect_ratio_buckets = {ratio: [] for ratio in aspect_ratios} + # [str(k) for k, v in aspect_ratios] + self.current_available_bucket_keys = list(aspect_ratios.keys()) + + def __iter__(self): + for idx in self.sampler: + try: + video_dict = self.dataset[idx] + width, more = video_dict.get("width", None), video_dict.get("height", None) + + if width is None or height is None: + if self.train_data_format == "normal": + video_id, name = video_dict['file_path'], video_dict['text'] + if self.video_folder is None: + video_dir = video_id + else: + video_dir = os.path.join(self.video_folder, video_id) + else: + videoid, name, page_dir = video_dict['videoid'], video_dict['name'], video_dict['page_dir'] + video_dir = os.path.join(self.video_folder, f"{videoid}.mp4") + cap = cv2.VideoCapture(video_dir) + + # 获取视频尺寸 + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) # 浮点数转换为整数 + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # 浮点数转换为整数 + + ratio = height / width # self.dataset[idx] + else: + height = int(height) + width = int(width) + ratio = height / width # self.dataset[idx] + except Exception as e: + print(e, self.dataset[idx], "This item is error, please check it.") + continue + # find the closest aspect ratio + closest_ratio = min(self.aspect_ratios.keys(), key=lambda r: abs(float(r) - ratio)) + if closest_ratio not in self.current_available_bucket_keys: + continue + bucket = self._aspect_ratio_buckets[closest_ratio] + bucket.append(idx) + # yield a batch of indices in the same aspect ratio group + if len(bucket) == self.batch_size: + yield bucket[:] + del bucket[:] + +class AspectRatioBatchImageVideoSampler(BatchSampler): + """A sampler wrapper for grouping images with similar aspect ratio into a same batch. + + Args: + sampler (Sampler): Base sampler. + dataset (Dataset): Dataset providing data information. + batch_size (int): Size of mini-batch. + drop_last (bool): If ``True``, the sampler will drop the last batch if + its size would be less than ``batch_size``. + aspect_ratios (dict): The predefined aspect ratios. + """ + + def __init__(self, + sampler: Sampler, + dataset: Dataset, + batch_size: int, + train_folder: str = None, + aspect_ratios: dict = ASPECT_RATIO_512, + drop_last: bool = False + ) -> None: + if not isinstance(sampler, Sampler): + raise TypeError('sampler should be an instance of ``Sampler``, ' + f'but got {sampler}') + if not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError('batch_size should be a positive integer value, ' + f'but got batch_size={batch_size}') + self.sampler = sampler + self.dataset = dataset + self.train_folder = train_folder + self.batch_size = batch_size + self.aspect_ratios = aspect_ratios + self.drop_last = drop_last + + # buckets for each aspect ratio + self.current_available_bucket_keys = list(aspect_ratios.keys()) + self.bucket = { + 'image':{ratio: [] for ratio in aspect_ratios}, + 'video':{ratio: [] for ratio in aspect_ratios} + } + + def __iter__(self): + for idx in self.sampler: + content_type = self.dataset[idx].get('type', 'image') + if content_type == 'image': + try: + image_dict = self.dataset[idx] + + width, height = image_dict.get("width", None), image_dict.get("height", None) + if width is None or height is None: + image_id, name = image_dict['file_path'], image_dict['text'] + if self.train_folder is None: + image_dir = image_id + else: + image_dir = os.path.join(self.train_folder, image_id) + + width, height = get_image_size_without_loading(image_dir) + + ratio = height / width # self.dataset[idx] + else: + height = int(height) + width = int(width) + ratio = height / width # self.dataset[idx] + except Exception as e: + print(e, self.dataset[idx], "This item is error, please check it.") + continue + # find the closest aspect ratio + closest_ratio = min(self.aspect_ratios.keys(), key=lambda r: abs(float(r) - ratio)) + if closest_ratio not in self.current_available_bucket_keys: + continue + bucket = self.bucket['image'][closest_ratio] + bucket.append(idx) + # yield a batch of indices in the same aspect ratio group + if len(bucket) == self.batch_size: + yield bucket[:] + del bucket[:] + else: + try: + video_dict = self.dataset[idx] + width, height = video_dict.get("width", None), video_dict.get("height", None) + + if width is None or height is None: + video_id, name = video_dict['file_path'], video_dict['text'] + if self.train_folder is None: + video_dir = video_id + else: + video_dir = os.path.join(self.train_folder, video_id) + cap = cv2.VideoCapture(video_dir) + + # 获取视频尺寸 + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) # 浮点数转换为整数 + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) # 浮点数转换为整数 + + ratio = height / width # self.dataset[idx] + else: + height = int(height) + width = int(width) + ratio = height / width # self.dataset[idx] + except Exception as e: + print(e, self.dataset[idx], "This item is error, please check it.") + continue + # find the closest aspect ratio + closest_ratio = min(self.aspect_ratios.keys(), key=lambda r: abs(float(r) - ratio)) + if closest_ratio not in self.current_available_bucket_keys: + continue + bucket = self.bucket['video'][closest_ratio] + bucket.append(idx) + # yield a batch of indices in the same aspect ratio group + if len(bucket) == self.batch_size: + yield bucket[:] + del bucket[:] \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/data/dataset_image.py b/VideoX-Fun/videox_fun/data/dataset_image.py new file mode 100644 index 0000000000000000000000000000000000000000..18c672a9ca71a7f57f211a3aedd1f6a499b5e487 --- /dev/null +++ b/VideoX-Fun/videox_fun/data/dataset_image.py @@ -0,0 +1,191 @@ +import json +import os +import random + +import numpy as np +import torch +import torchvision.transforms as transforms +from PIL import Image +from torch.utils.data.dataset import Dataset + + +class CC15M(Dataset): + def __init__( + self, + json_path, + video_folder=None, + resolution=512, + enable_bucket=False, + ): + print(f"loading annotations from {json_path} ...") + self.dataset = json.load(open(json_path, 'r')) + self.length = len(self.dataset) + print(f"data scale: {self.length}") + + self.enable_bucket = enable_bucket + self.video_folder = video_folder + + resolution = tuple(resolution) if not isinstance(resolution, int) else (resolution, resolution) + self.pixel_transforms = transforms.Compose([ + transforms.Resize(resolution[0]), + transforms.CenterCrop(resolution), + transforms.ToTensor(), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + def get_batch(self, idx): + video_dict = self.dataset[idx] + video_id, name = video_dict['file_path'], video_dict['text'] + + if self.video_folder is None: + video_dir = video_id + else: + video_dir = os.path.join(self.video_folder, video_id) + + pixel_values = Image.open(video_dir).convert("RGB") + return pixel_values, name + + def __len__(self): + return self.length + + def __getitem__(self, idx): + while True: + try: + pixel_values, name = self.get_batch(idx) + break + except Exception as e: + print(e) + idx = random.randint(0, self.length-1) + + if not self.enable_bucket: + pixel_values = self.pixel_transforms(pixel_values) + else: + pixel_values = np.array(pixel_values) + + sample = dict(pixel_values=pixel_values, text=name) + return sample + +class ImageEditDataset(Dataset): + def __init__( + self, + ann_path, data_root=None, + image_sample_size=512, + text_drop_ratio=0.1, + enable_bucket=False, + enable_inpaint=False, + return_file_name=False, + ): + # Loading annotations from files + print(f"loading annotations from {ann_path} ...") + if ann_path.endswith('.csv'): + with open(ann_path, 'r') as csvfile: + dataset = list(csv.DictReader(csvfile)) + elif ann_path.endswith('.json'): + dataset = json.load(open(ann_path)) + + self.data_root = data_root + self.dataset = dataset + + self.length = len(self.dataset) + print(f"data scale: {self.length}") + # TODO: enable bucket training + self.enable_bucket = enable_bucket + self.text_drop_ratio = text_drop_ratio + self.enable_inpaint = enable_inpaint + self.return_file_name = return_file_name + + # Image params + self.image_sample_size = tuple(image_sample_size) if not isinstance(image_sample_size, int) else (image_sample_size, image_sample_size) + self.image_transforms = transforms.Compose([ + transforms.Resize(min(self.image_sample_size)), + transforms.CenterCrop(self.image_sample_size), + transforms.ToTensor(), + transforms.Normalize([0.5, 0.5, 0.5],[0.5, 0.5, 0.5]) + ]) + + def get_batch(self, idx): + data_info = self.dataset[idx % len(self.dataset)] + + image_path, text = data_info['file_path'], data_info['text'] + if self.data_root is not None: + image_path = os.path.join(self.data_root, image_path) + image = Image.open(image_path).convert('RGB') + + if not self.enable_bucket: + raise ValueError("Not enable_bucket is not supported now. ") + else: + image = np.expand_dims(np.array(image), 0) + + source_image_path = data_info.get('source_file_path', []) + source_image = [] + if isinstance(source_image_path, list): + for _source_image_path in source_image_path: + if self.data_root is not None: + _source_image_path = os.path.join(self.data_root, _source_image_path) + _source_image = Image.open(_source_image_path).convert('RGB') + source_image.append(_source_image) + else: + if self.data_root is not None: + _source_image_path = os.path.join(self.data_root, source_image_path) + _source_image = Image.open(_source_image_path).convert('RGB') + source_image.append(_source_image) + + if not self.enable_bucket: + raise ValueError("Not enable_bucket is not supported now. ") + else: + source_image = [np.array(_source_image) for _source_image in source_image] + + if random.random() < self.text_drop_ratio: + text = '' + return image, source_image, text, 'image', image_path + + def __len__(self): + return self.length + + def __getitem__(self, idx): + data_info = self.dataset[idx % len(self.dataset)] + data_type = data_info.get('type', 'image') + while True: + sample = {} + try: + data_info_local = self.dataset[idx % len(self.dataset)] + data_type_local = data_info_local.get('type', 'image') + if data_type_local != data_type: + raise ValueError("data_type_local != data_type") + + pixel_values, source_pixel_values, name, data_type, file_path = self.get_batch(idx) + sample["pixel_values"] = pixel_values + sample["source_pixel_values"] = source_pixel_values + sample["text"] = name + sample["data_type"] = data_type + sample["idx"] = idx + if self.return_file_name: + sample["file_name"] = os.path.basename(file_path) + + if len(sample) > 0: + break + except Exception as e: + print(e, self.dataset[idx % len(self.dataset)]) + idx = random.randint(0, self.length-1) + + if self.enable_inpaint and not self.enable_bucket: + mask = get_random_mask(pixel_values.size()) + mask_pixel_values = pixel_values * (1 - mask) + torch.ones_like(pixel_values) * -1 * mask + sample["mask_pixel_values"] = mask_pixel_values + sample["mask"] = mask + + clip_pixel_values = sample["pixel_values"][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + sample["clip_pixel_values"] = clip_pixel_values + + return sample + +if __name__ == "__main__": + dataset = CC15M( + csv_path="./cc15m_add_index.json", + resolution=512, + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size=4, num_workers=0,) + for idx, batch in enumerate(dataloader): + print(batch["pixel_values"].shape, len(batch["text"])) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/data/dataset_image_video.py b/VideoX-Fun/videox_fun/data/dataset_image_video.py new file mode 100644 index 0000000000000000000000000000000000000000..449a2f7b1df4a4c70e236658a93e435f6c1ff9d0 --- /dev/null +++ b/VideoX-Fun/videox_fun/data/dataset_image_video.py @@ -0,0 +1,657 @@ +import csv +import gc +import io +import json +import math +import os +import random +from contextlib import contextmanager +from random import shuffle +from threading import Thread + +import albumentations +import cv2 +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms as transforms +from decord import VideoReader +from einops import rearrange +from func_timeout import FunctionTimedOut, func_timeout +from packaging import version as pver +from PIL import Image +from safetensors.torch import load_file +from torch.utils.data import BatchSampler, Sampler +from torch.utils.data.dataset import Dataset + +from .utils import (VIDEO_READER_TIMEOUT, Camera, VideoReader_contextmanager, + custom_meshgrid, get_random_mask, get_relative_pose, + get_video_reader_batch, padding_image, process_pose_file, + process_pose_params, ray_condition, resize_frame, + resize_image_with_target_area) + + +class ImageVideoSampler(BatchSampler): + """A sampler wrapper for grouping images with similar aspect ratio into a same batch. + + Args: + sampler (Sampler): Base sampler. + dataset (Dataset): Dataset providing data information. + batch_size (int): Size of mini-batch. + drop_last (bool): If ``True``, the sampler will drop the last batch if + its size would be less than ``batch_size``. + aspect_ratios (dict): The predefined aspect ratios. + """ + + def __init__(self, + sampler: Sampler, + dataset: Dataset, + batch_size: int, + drop_last: bool = False + ) -> None: + if not isinstance(sampler, Sampler): + raise TypeError('sampler should be an instance of ``Sampler``, ' + f'but got {sampler}') + if not isinstance(batch_size, int) or batch_size <= 0: + raise ValueError('batch_size should be a positive integer value, ' + f'but got batch_size={batch_size}') + self.sampler = sampler + self.dataset = dataset + self.batch_size = batch_size + self.drop_last = drop_last + + # buckets for each aspect ratio + self.bucket = {'image':[], 'video':[]} + + def __iter__(self): + for idx in self.sampler: + content_type = self.dataset.dataset[idx].get('type', 'image') + self.bucket[content_type].append(idx) + + # yield a batch of indices in the same aspect ratio group + if len(self.bucket['video']) == self.batch_size: + bucket = self.bucket['video'] + yield bucket[:] + del bucket[:] + elif len(self.bucket['image']) == self.batch_size: + bucket = self.bucket['image'] + yield bucket[:] + del bucket[:] + + +class ImageVideoDataset(Dataset): + def __init__( + self, + ann_path, data_root=None, + video_sample_size=512, video_sample_stride=4, video_sample_n_frames=16, + image_sample_size=512, + video_repeat=0, + text_drop_ratio=0.1, + enable_bucket=False, + video_length_drop_start=0.0, + video_length_drop_end=1.0, + enable_inpaint=False, + return_file_name=False, + ): + # Loading annotations from files + print(f"loading annotations from {ann_path} ...") + if ann_path.endswith('.csv'): + with open(ann_path, 'r') as csvfile: + dataset = list(csv.DictReader(csvfile)) + elif ann_path.endswith('.json'): + dataset = json.load(open(ann_path)) + + self.data_root = data_root + + # It's used to balance num of images and videos. + if video_repeat > 0: + self.dataset = [] + for data in dataset: + if data.get('type', 'image') != 'video': + self.dataset.append(data) + + for _ in range(video_repeat): + for data in dataset: + if data.get('type', 'image') == 'video': + self.dataset.append(data) + else: + self.dataset = dataset + del dataset + + self.length = len(self.dataset) + print(f"data scale: {self.length}") + # TODO: enable bucket training + self.enable_bucket = enable_bucket + self.text_drop_ratio = text_drop_ratio + self.enable_inpaint = enable_inpaint + self.return_file_name = return_file_name + + self.video_length_drop_start = video_length_drop_start + self.video_length_drop_end = video_length_drop_end + + # Video params + self.video_sample_stride = video_sample_stride + self.video_sample_n_frames = video_sample_n_frames + self.video_sample_size = tuple(video_sample_size) if not isinstance(video_sample_size, int) else (video_sample_size, video_sample_size) + self.video_transforms = transforms.Compose( + [ + transforms.Resize(min(self.video_sample_size)), + transforms.CenterCrop(self.video_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ] + ) + + # Image params + self.image_sample_size = tuple(image_sample_size) if not isinstance(image_sample_size, int) else (image_sample_size, image_sample_size) + self.image_transforms = transforms.Compose([ + transforms.Resize(min(self.image_sample_size)), + transforms.CenterCrop(self.image_sample_size), + transforms.ToTensor(), + transforms.Normalize([0.5, 0.5, 0.5],[0.5, 0.5, 0.5]) + ]) + + self.larger_side_of_image_and_video = max(min(self.image_sample_size), min(self.video_sample_size)) + + def get_batch(self, idx): + data_info = self.dataset[idx % len(self.dataset)] + + if data_info.get('type', 'image')=='video': + video_id, text = data_info['file_path'], data_info['text'] + + if self.data_root is None: + video_dir = video_id + else: + video_dir = os.path.join(self.data_root, video_id) + + with VideoReader_contextmanager(video_dir, num_threads=2) as video_reader: + min_sample_n_frames = min( + self.video_sample_n_frames, + int(len(video_reader) * (self.video_length_drop_end - self.video_length_drop_start) // self.video_sample_stride) + ) + if min_sample_n_frames == 0: + raise ValueError(f"No Frames in video.") + + video_length = int(self.video_length_drop_end * len(video_reader)) + clip_length = min(video_length, (min_sample_n_frames - 1) * self.video_sample_stride + 1) + start_idx = random.randint(int(self.video_length_drop_start * video_length), video_length - clip_length) if video_length != clip_length else 0 + batch_index = np.linspace(start_idx, start_idx + clip_length - 1, min_sample_n_frames, dtype=int) + + try: + sample_args = (video_reader, batch_index) + pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + resized_frames = [] + for i in range(len(pixel_values)): + frame = pixel_values[i] + resized_frame = resize_frame(frame, self.larger_side_of_image_and_video) + resized_frames.append(resized_frame) + pixel_values = np.array(resized_frames) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + if not self.enable_bucket: + pixel_values = torch.from_numpy(pixel_values).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + del video_reader + else: + pixel_values = pixel_values + + if not self.enable_bucket: + pixel_values = self.video_transforms(pixel_values) + + # Random use no text generation + if random.random() < self.text_drop_ratio: + text = '' + return pixel_values, text, 'video', video_dir + else: + image_path, text = data_info['file_path'], data_info['text'] + if self.data_root is not None: + image_path = os.path.join(self.data_root, image_path) + image = Image.open(image_path).convert('RGB') + if not self.enable_bucket: + image = self.image_transforms(image).unsqueeze(0) + else: + image = np.expand_dims(np.array(image), 0) + if random.random() < self.text_drop_ratio: + text = '' + return image, text, 'image', image_path + + def __len__(self): + return self.length + + def __getitem__(self, idx): + data_info = self.dataset[idx % len(self.dataset)] + data_type = data_info.get('type', 'image') + while True: + sample = {} + try: + data_info_local = self.dataset[idx % len(self.dataset)] + data_type_local = data_info_local.get('type', 'image') + if data_type_local != data_type: + raise ValueError("data_type_local != data_type") + + pixel_values, name, data_type, file_path = self.get_batch(idx) + sample["pixel_values"] = pixel_values + sample["text"] = name + sample["data_type"] = data_type + sample["idx"] = idx + if self.return_file_name: + sample["file_name"] = os.path.basename(file_path) + + if len(sample) > 0: + break + except Exception as e: + print(e, self.dataset[idx % len(self.dataset)]) + idx = random.randint(0, self.length-1) + + if self.enable_inpaint and not self.enable_bucket: + mask = get_random_mask(pixel_values.size()) + mask_pixel_values = pixel_values * (1 - mask) + torch.ones_like(pixel_values) * -1 * mask + sample["mask_pixel_values"] = mask_pixel_values + sample["mask"] = mask + + clip_pixel_values = sample["pixel_values"][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + sample["clip_pixel_values"] = clip_pixel_values + + return sample + + +class ImageVideoControlDataset(Dataset): + def __init__( + self, + ann_path, data_root=None, + video_sample_size=512, video_sample_stride=4, video_sample_n_frames=16, + image_sample_size=512, + video_repeat=0, + text_drop_ratio=0.1, + enable_bucket=False, + video_length_drop_start=0.1, + video_length_drop_end=0.9, + enable_inpaint=False, + enable_camera_info=False, + return_file_name=False, + enable_subject_info=False, + padding_subject_info=True, + ): + # Loading annotations from files + print(f"loading annotations from {ann_path} ...") + if ann_path.endswith('.csv'): + with open(ann_path, 'r') as csvfile: + dataset = list(csv.DictReader(csvfile)) + elif ann_path.endswith('.json'): + dataset = json.load(open(ann_path)) + + self.data_root = data_root + + # It's used to balance num of images and videos. + if video_repeat > 0: + self.dataset = [] + for data in dataset: + if data.get('type', 'image') != 'video': + self.dataset.append(data) + + for _ in range(video_repeat): + for data in dataset: + if data.get('type', 'image') == 'video': + self.dataset.append(data) + else: + self.dataset = dataset + del dataset + + self.length = len(self.dataset) + print(f"data scale: {self.length}") + # TODO: enable bucket training + self.enable_bucket = enable_bucket + self.text_drop_ratio = text_drop_ratio + self.enable_inpaint = enable_inpaint + self.enable_camera_info = enable_camera_info + self.enable_subject_info = enable_subject_info + self.padding_subject_info = padding_subject_info + + self.video_length_drop_start = video_length_drop_start + self.video_length_drop_end = video_length_drop_end + + # Video params + self.video_sample_stride = video_sample_stride + self.video_sample_n_frames = video_sample_n_frames + self.video_sample_size = tuple(video_sample_size) if not isinstance(video_sample_size, int) else (video_sample_size, video_sample_size) + self.video_transforms = transforms.Compose( + [ + transforms.Resize(min(self.video_sample_size)), + transforms.CenterCrop(self.video_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ] + ) + if self.enable_camera_info: + self.video_transforms_camera = transforms.Compose( + [ + transforms.Resize(min(self.video_sample_size)), + transforms.CenterCrop(self.video_sample_size) + ] + ) + + # Image params + self.image_sample_size = tuple(image_sample_size) if not isinstance(image_sample_size, int) else (image_sample_size, image_sample_size) + self.image_transforms = transforms.Compose([ + transforms.Resize(min(self.image_sample_size)), + transforms.CenterCrop(self.image_sample_size), + transforms.ToTensor(), + transforms.Normalize([0.5, 0.5, 0.5],[0.5, 0.5, 0.5]) + ]) + + self.larger_side_of_image_and_video = max(min(self.image_sample_size), min(self.video_sample_size)) + + def get_batch(self, idx): + data_info = self.dataset[idx % len(self.dataset)] + video_id, text = data_info['file_path'], data_info['text'] + + if data_info.get('type', 'image')=='video': + if self.data_root is None: + video_dir = video_id + else: + video_dir = os.path.join(self.data_root, video_id) + + with VideoReader_contextmanager(video_dir, num_threads=2) as video_reader: + min_sample_n_frames = min( + self.video_sample_n_frames, + int(len(video_reader) * (self.video_length_drop_end - self.video_length_drop_start) // self.video_sample_stride) + ) + if min_sample_n_frames == 0: + raise ValueError(f"No Frames in video.") + + video_length = int(self.video_length_drop_end * len(video_reader)) + clip_length = min(video_length, (min_sample_n_frames - 1) * self.video_sample_stride + 1) + start_idx = random.randint(int(self.video_length_drop_start * video_length), video_length - clip_length) if video_length != clip_length else 0 + batch_index = np.linspace(start_idx, start_idx + clip_length - 1, min_sample_n_frames, dtype=int) + + try: + sample_args = (video_reader, batch_index) + pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + resized_frames = [] + for i in range(len(pixel_values)): + frame = pixel_values[i] + resized_frame = resize_frame(frame, self.larger_side_of_image_and_video) + resized_frames.append(resized_frame) + pixel_values = np.array(resized_frames) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + if not self.enable_bucket: + pixel_values = torch.from_numpy(pixel_values).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + del video_reader + else: + pixel_values = pixel_values + + if not self.enable_bucket: + pixel_values = self.video_transforms(pixel_values) + + # Random use no text generation + if random.random() < self.text_drop_ratio: + text = '' + + control_video_id = data_info['control_file_path'] + + if control_video_id is not None: + if self.data_root is None: + control_video_id = control_video_id + else: + control_video_id = os.path.join(self.data_root, control_video_id) + + if self.enable_camera_info: + if control_video_id.lower().endswith('.txt'): + if not self.enable_bucket: + control_pixel_values = torch.zeros_like(pixel_values) + + control_camera_values = process_pose_file(control_video_id, width=self.video_sample_size[1], height=self.video_sample_size[0]) + control_camera_values = torch.from_numpy(control_camera_values).permute(0, 3, 1, 2).contiguous() + control_camera_values = F.interpolate(control_camera_values, size=(len(video_reader), control_camera_values.size(3)), mode='bilinear', align_corners=True) + control_camera_values = self.video_transforms_camera(control_camera_values) + else: + control_pixel_values = np.zeros_like(pixel_values) + + control_camera_values = process_pose_file(control_video_id, width=self.video_sample_size[1], height=self.video_sample_size[0], return_poses=True) + control_camera_values = torch.from_numpy(np.array(control_camera_values)).unsqueeze(0).unsqueeze(0) + control_camera_values = F.interpolate(control_camera_values, size=(len(video_reader), control_camera_values.size(3)), mode='bilinear', align_corners=True)[0][0] + control_camera_values = np.array([control_camera_values[index] for index in batch_index]) + else: + if not self.enable_bucket: + control_pixel_values = torch.zeros_like(pixel_values) + control_camera_values = None + else: + control_pixel_values = np.zeros_like(pixel_values) + control_camera_values = None + else: + if control_video_id is not None: + with VideoReader_contextmanager(control_video_id, num_threads=2) as control_video_reader: + try: + sample_args = (control_video_reader, batch_index) + control_pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + resized_frames = [] + for i in range(len(control_pixel_values)): + frame = control_pixel_values[i] + resized_frame = resize_frame(frame, self.larger_side_of_image_and_video) + resized_frames.append(resized_frame) + control_pixel_values = np.array(resized_frames) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + if not self.enable_bucket: + control_pixel_values = torch.from_numpy(control_pixel_values).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + del control_video_reader + else: + control_pixel_values = control_pixel_values + + if not self.enable_bucket: + control_pixel_values = self.video_transforms(control_pixel_values) + else: + if not self.enable_bucket: + control_pixel_values = torch.zeros_like(pixel_values) + else: + control_pixel_values = np.zeros_like(pixel_values) + control_camera_values = None + + if self.enable_subject_info: + if not self.enable_bucket: + visual_height, visual_width = pixel_values.shape[-2:] + else: + visual_height, visual_width = pixel_values.shape[1:3] + + subject_id = data_info.get('object_file_path', []) + shuffle(subject_id) + subject_images = [] + for i in range(min(len(subject_id), 4)): + subject_image = Image.open(subject_id[i]) + width, height = subject_image.size + total_pixels = width * height + + if self.padding_subject_info: + img = padding_image(subject_image, visual_width, visual_height) + else: + img = resize_image_with_target_area(subject_image, 1024 * 1024) + + if random.random() < 0.5: + img = img.transpose(Image.FLIP_LEFT_RIGHT) + subject_images.append(np.array(img)) + if self.padding_subject_info: + subject_image = np.array(subject_images) + else: + subject_image = subject_images + else: + subject_image = None + + return pixel_values, control_pixel_values, subject_image, control_camera_values, text, "video" + else: + image_path, text = data_info['file_path'], data_info['text'] + if self.data_root is not None: + image_path = os.path.join(self.data_root, image_path) + image = Image.open(image_path).convert('RGB') + if not self.enable_bucket: + image = self.image_transforms(image).unsqueeze(0) + else: + image = np.expand_dims(np.array(image), 0) + + if random.random() < self.text_drop_ratio: + text = '' + + control_image_id = data_info['control_file_path'] + + if self.data_root is None: + control_image_id = control_image_id + else: + control_image_id = os.path.join(self.data_root, control_image_id) + + control_image = Image.open(control_image_id).convert('RGB') + if not self.enable_bucket: + control_image = self.image_transforms(control_image).unsqueeze(0) + else: + control_image = np.expand_dims(np.array(control_image), 0) + + if self.enable_subject_info: + if not self.enable_bucket: + visual_height, visual_width = image.shape[-2:] + else: + visual_height, visual_width = image.shape[1:3] + + subject_id = data_info.get('object_file_path', []) + shuffle(subject_id) + subject_images = [] + for i in range(min(len(subject_id), 4)): + subject_image = Image.open(subject_id[i]).convert('RGB') + width, height = subject_image.size + total_pixels = width * height + + if self.padding_subject_info: + img = padding_image(subject_image, visual_width, visual_height) + else: + img = resize_image_with_target_area(subject_image, 1024 * 1024) + + if random.random() < 0.5: + img = img.transpose(Image.FLIP_LEFT_RIGHT) + subject_images.append(np.array(img)) + if self.padding_subject_info: + subject_image = np.array(subject_images) + else: + subject_image = subject_images + else: + subject_image = None + + return image, control_image, subject_image, None, text, 'image' + + def __len__(self): + return self.length + + def __getitem__(self, idx): + data_info = self.dataset[idx % len(self.dataset)] + data_type = data_info.get('type', 'image') + while True: + sample = {} + try: + data_info_local = self.dataset[idx % len(self.dataset)] + data_type_local = data_info_local.get('type', 'image') + if data_type_local != data_type: + raise ValueError("data_type_local != data_type") + + pixel_values, control_pixel_values, subject_image, control_camera_values, name, data_type = self.get_batch(idx) + + sample["pixel_values"] = pixel_values + sample["control_pixel_values"] = control_pixel_values + sample["subject_image"] = subject_image + sample["text"] = name + sample["data_type"] = data_type + sample["idx"] = idx + + if self.enable_camera_info: + sample["control_camera_values"] = control_camera_values + + if len(sample) > 0: + break + except Exception as e: + print(e, self.dataset[idx % len(self.dataset)]) + idx = random.randint(0, self.length-1) + + if self.enable_inpaint and not self.enable_bucket: + mask = get_random_mask(pixel_values.size()) + mask_pixel_values = pixel_values * (1 - mask) + torch.zeros_like(pixel_values) * mask + sample["mask_pixel_values"] = mask_pixel_values + sample["mask"] = mask + + clip_pixel_values = sample["pixel_values"][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + sample["clip_pixel_values"] = clip_pixel_values + + return sample + + +class ImageVideoSafetensorsDataset(Dataset): + def __init__( + self, + ann_path, + data_root=None, + ): + # Loading annotations from files + print(f"loading annotations from {ann_path} ...") + if ann_path.endswith('.json'): + dataset = json.load(open(ann_path)) + + self.data_root = data_root + self.dataset = dataset + self.length = len(self.dataset) + print(f"data scale: {self.length}") + + def __len__(self): + return self.length + + def __getitem__(self, idx): + if self.data_root is None: + path = self.dataset[idx]["file_path"] + else: + path = os.path.join(self.data_root, self.dataset[idx]["file_path"]) + state_dict = load_file(path) + return state_dict + + +class TextDataset(Dataset): + def __init__(self, ann_path, text_drop_ratio=0.0): + print(f"loading annotations from {ann_path} ...") + with open(ann_path, 'r') as f: + self.dataset = json.load(f) + self.length = len(self.dataset) + print(f"data scale: {self.length}") + self.text_drop_ratio = text_drop_ratio + + def __len__(self): + return self.length + + def __getitem__(self, idx): + while True: + try: + item = self.dataset[idx] + text = item['text'] + + # Randomly drop text (for classifier-free guidance) + if random.random() < self.text_drop_ratio: + text = '' + + sample = { + "text": text, + "idx": idx + } + return sample + + except Exception as e: + print(f"Error at index {idx}: {e}, retrying with random index...") + idx = np.random.randint(0, self.length - 1) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/data/dataset_video.py b/VideoX-Fun/videox_fun/data/dataset_video.py new file mode 100644 index 0000000000000000000000000000000000000000..230528eaab912d3151ee5bbf663f1de544c99543 --- /dev/null +++ b/VideoX-Fun/videox_fun/data/dataset_video.py @@ -0,0 +1,901 @@ +import csv +import gc +import io +import json +import math +import os +import random +from contextlib import contextmanager +from threading import Thread + +import albumentations +import cv2 +import librosa +import numpy as np +import torch +import torchvision.transforms as transforms +from decord import VideoReader +from einops import rearrange +from func_timeout import FunctionTimedOut, func_timeout +from PIL import Image +from torch.utils.data import BatchSampler, Sampler +from torch.utils.data.dataset import Dataset + +from .utils import (VIDEO_READER_TIMEOUT, Camera, VideoReader_contextmanager, + custom_meshgrid, get_random_mask, get_relative_pose, + get_video_reader_batch, padding_image, process_pose_file, + process_pose_params, ray_condition, resize_frame, + resize_image_with_target_area) + + +class WebVid10M(Dataset): + def __init__( + self, + csv_path, video_folder, + sample_size=256, sample_stride=4, sample_n_frames=16, + enable_bucket=False, enable_inpaint=False, is_image=False, + ): + print(f"loading annotations from {csv_path} ...") + with open(csv_path, 'r') as csvfile: + self.dataset = list(csv.DictReader(csvfile)) + self.length = len(self.dataset) + print(f"data scale: {self.length}") + + self.video_folder = video_folder + self.sample_stride = sample_stride + self.sample_n_frames = sample_n_frames + self.enable_bucket = enable_bucket + self.enable_inpaint = enable_inpaint + self.is_image = is_image + + sample_size = tuple(sample_size) if not isinstance(sample_size, int) else (sample_size, sample_size) + self.pixel_transforms = transforms.Compose([ + transforms.Resize(sample_size[0]), + transforms.CenterCrop(sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ]) + + def get_batch(self, idx): + video_dict = self.dataset[idx] + videoid, name, page_dir = video_dict['videoid'], video_dict['name'], video_dict['page_dir'] + + video_dir = os.path.join(self.video_folder, f"{videoid}.mp4") + video_reader = VideoReader(video_dir) + video_length = len(video_reader) + + if not self.is_image: + clip_length = min(video_length, (self.sample_n_frames - 1) * self.sample_stride + 1) + start_idx = random.randint(0, video_length - clip_length) + batch_index = np.linspace(start_idx, start_idx + clip_length - 1, self.sample_n_frames, dtype=int) + else: + batch_index = [random.randint(0, video_length - 1)] + + if not self.enable_bucket: + pixel_values = torch.from_numpy(video_reader.get_batch(batch_index).asnumpy()).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + del video_reader + else: + pixel_values = video_reader.get_batch(batch_index).asnumpy() + + if self.is_image: + pixel_values = pixel_values[0] + return pixel_values, name + + def __len__(self): + return self.length + + def __getitem__(self, idx): + while True: + try: + pixel_values, name = self.get_batch(idx) + break + + except Exception as e: + print("Error info:", e) + idx = random.randint(0, self.length-1) + + if not self.enable_bucket: + pixel_values = self.pixel_transforms(pixel_values) + if self.enable_inpaint: + mask = get_random_mask(pixel_values.size()) + mask_pixel_values = pixel_values * (1 - mask) + torch.ones_like(pixel_values) * -1 * mask + sample = dict(pixel_values=pixel_values, mask_pixel_values=mask_pixel_values, mask=mask, text=name) + else: + sample = dict(pixel_values=pixel_values, text=name) + return sample + + +class VideoDataset(Dataset): + def __init__( + self, + ann_path, data_root=None, + sample_size=256, sample_stride=4, sample_n_frames=16, + enable_bucket=False, enable_inpaint=False + ): + print(f"loading annotations from {ann_path} ...") + self.dataset = json.load(open(ann_path, 'r')) + self.length = len(self.dataset) + print(f"data scale: {self.length}") + + self.data_root = data_root + self.sample_stride = sample_stride + self.sample_n_frames = sample_n_frames + self.enable_bucket = enable_bucket + self.enable_inpaint = enable_inpaint + + sample_size = tuple(sample_size) if not isinstance(sample_size, int) else (sample_size, sample_size) + self.pixel_transforms = transforms.Compose( + [ + transforms.Resize(sample_size[0]), + transforms.CenterCrop(sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ] + ) + + def get_batch(self, idx): + video_dict = self.dataset[idx] + video_id, text = video_dict['file_path'], video_dict['text'] + + if self.data_root is None: + video_dir = video_id + else: + video_dir = os.path.join(self.data_root, video_id) + + with VideoReader_contextmanager(video_dir, num_threads=2) as video_reader: + min_sample_n_frames = min( + self.video_sample_n_frames, + int(len(video_reader) * (self.video_length_drop_end - self.video_length_drop_start) // self.video_sample_stride) + ) + if min_sample_n_frames == 0: + raise ValueError(f"No Frames in video.") + + video_length = int(self.video_length_drop_end * len(video_reader)) + clip_length = min(video_length, (min_sample_n_frames - 1) * self.video_sample_stride + 1) + start_idx = random.randint(int(self.video_length_drop_start * video_length), video_length - clip_length) if video_length != clip_length else 0 + batch_index = np.linspace(start_idx, start_idx + clip_length - 1, min_sample_n_frames, dtype=int) + + try: + sample_args = (video_reader, batch_index) + pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + if not self.enable_bucket: + pixel_values = torch.from_numpy(pixel_values).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + del video_reader + else: + pixel_values = pixel_values + + if not self.enable_bucket: + pixel_values = self.video_transforms(pixel_values) + + # Random use no text generation + if random.random() < self.text_drop_ratio: + text = '' + return pixel_values, text + + def __len__(self): + return self.length + + def __getitem__(self, idx): + while True: + sample = {} + try: + pixel_values, name = self.get_batch(idx) + sample["pixel_values"] = pixel_values + sample["text"] = name + sample["idx"] = idx + if len(sample) > 0: + break + + except Exception as e: + print(e, self.dataset[idx % len(self.dataset)]) + idx = random.randint(0, self.length-1) + + if self.enable_inpaint and not self.enable_bucket: + mask = get_random_mask(pixel_values.size()) + mask_pixel_values = pixel_values * (1 - mask) + torch.zeros_like(pixel_values) * mask + sample["mask_pixel_values"] = mask_pixel_values + sample["mask"] = mask + + clip_pixel_values = sample["pixel_values"][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + sample["clip_pixel_values"] = clip_pixel_values + + return sample + + +class VideoSpeechDataset(Dataset): + def __init__( + self, + ann_path, data_root=None, + video_sample_size=512, video_sample_stride=4, video_sample_n_frames=16, + enable_bucket=False, enable_inpaint=False, + audio_sr=16000, # 新增：目标音频采样率 + text_drop_ratio=0.1 # 新增：文本丢弃概率 + ): + print(f"loading annotations from {ann_path} ...") + self.dataset = json.load(open(ann_path, 'r')) + self.length = len(self.dataset) + print(f"data scale: {self.length}") + + self.data_root = data_root + self.video_sample_stride = video_sample_stride + self.video_sample_n_frames = video_sample_n_frames + self.enable_bucket = enable_bucket + self.enable_inpaint = enable_inpaint + self.audio_sr = audio_sr + self.text_drop_ratio = text_drop_ratio + + video_sample_size = tuple(video_sample_size) if not isinstance(video_sample_size, int) else (video_sample_size, video_sample_size) + self.pixel_transforms = transforms.Compose( + [ + transforms.Resize(video_sample_size[0]), + transforms.CenterCrop(video_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ] + ) + + def get_batch(self, idx): + video_dict = self.dataset[idx] + video_id, text = video_dict['file_path'], video_dict['text'] + audio_id = video_dict['audio_path'] + + if self.data_root is None: + video_path = video_id + else: + video_path = os.path.join(self.data_root, video_id) + + if self.data_root is None: + audio_path = audio_id + else: + audio_path = os.path.join(self.data_root, audio_id) + + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found for {video_path}") + + with VideoReader_contextmanager(video_path, num_threads=2) as video_reader: + total_frames = len(video_reader) + fps = video_reader.get_avg_fps() # 获取原始视频帧率 + + # 计算实际采样的视频帧数（考虑边界） + max_possible_frames = (total_frames - 1) // self.video_sample_stride + 1 + actual_n_frames = min(self.video_sample_n_frames, max_possible_frames) + if actual_n_frames <= 0: + raise ValueError(f"Video too short: {video_path}") + + # 随机选择起始帧 + max_start = total_frames - (actual_n_frames - 1) * self.video_sample_stride - 1 + start_frame = random.randint(0, max_start) if max_start > 0 else 0 + frame_indices = [start_frame + i * self.video_sample_stride for i in range(actual_n_frames)] + + # 读取视频帧 + try: + sample_args = (video_reader, frame_indices) + pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + # 视频后处理 + if not self.enable_bucket: + pixel_values = torch.from_numpy(pixel_values).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + pixel_values = self.pixel_transforms(pixel_values) + + # === 新增：加载并截取对应音频 === + # 视频片段的起止时间（秒） + start_time = start_frame / fps + end_time = (start_frame + (actual_n_frames - 1) * self.video_sample_stride) / fps + duration = end_time - start_time + + # 使用 librosa 加载整个音频（或仅加载所需部分，但 librosa.load 不支持精确 seek，所以先加载再切） + audio_input, sample_rate = librosa.load(audio_path, sr=self.audio_sr) # 重采样到目标 sr + + # 转换为样本索引 + start_sample = int(start_time * self.audio_sr) + end_sample = int(end_time * self.audio_sr) + + # 安全截取 + if start_sample >= len(audio_input): + # 音频太短，用零填充或截断 + audio_segment = np.zeros(int(duration * self.audio_sr), dtype=np.float32) + else: + audio_segment = audio_input[start_sample:end_sample] + # 如果太短，补零 + target_len = int(duration * self.audio_sr) + if len(audio_segment) < target_len: + audio_segment = np.pad(audio_segment, (0, target_len - len(audio_segment)), mode='constant') + + # === 文本随机丢弃 === + if random.random() < self.text_drop_ratio: + text = '' + + return pixel_values, text, audio_segment, sample_rate + + def __len__(self): + return self.length + + def __getitem__(self, idx): + while True: + sample = {} + try: + pixel_values, text, audio, sample_rate = self.get_batch(idx) + sample["pixel_values"] = pixel_values + sample["text"] = text + sample["audio"] = torch.from_numpy(audio).float() # 转为 tensor + sample["sample_rate"] = sample_rate + sample["idx"] = idx + break + except Exception as e: + print(f"Error processing {idx}: {e}, retrying with random idx...") + idx = random.randint(0, self.length - 1) + + if self.enable_inpaint and not self.enable_bucket: + mask = get_random_mask(pixel_values.size(), image_start_only=True) + mask_pixel_values = pixel_values * (1 - mask) + torch.zeros_like(pixel_values) * mask + sample["mask_pixel_values"] = mask_pixel_values + sample["mask"] = mask + + clip_pixel_values = sample["pixel_values"][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + sample["clip_pixel_values"] = clip_pixel_values + + return sample + + +class VideoSpeechControlDataset(Dataset): + def __init__( + self, + ann_path, data_root=None, + video_sample_size=512, video_sample_stride=4, video_sample_n_frames=16, + enable_bucket=False, enable_inpaint=False, + audio_sr=16000, + text_drop_ratio=0.1, + enable_motion_info=False, + motion_frames=73, + ): + print(f"loading annotations from {ann_path} ...") + self.dataset = json.load(open(ann_path, 'r')) + self.length = len(self.dataset) + print(f"data scale: {self.length}") + + self.data_root = data_root + self.video_sample_stride = video_sample_stride + self.video_sample_n_frames = video_sample_n_frames + self.enable_bucket = enable_bucket + self.enable_inpaint = enable_inpaint + self.audio_sr = audio_sr + self.text_drop_ratio = text_drop_ratio + self.enable_motion_info = enable_motion_info + self.motion_frames = motion_frames + + video_sample_size = tuple(video_sample_size) if not isinstance(video_sample_size, int) else (video_sample_size, video_sample_size) + self.pixel_transforms = transforms.Compose( + [ + transforms.Resize(video_sample_size[0]), + transforms.CenterCrop(video_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ] + ) + + self.video_sample_size = video_sample_size + + def get_batch(self, idx): + video_dict = self.dataset[idx] + video_id, text = video_dict['file_path'], video_dict['text'] + audio_id = video_dict['audio_path'] + control_video_id = video_dict['control_file_path'] + + if self.data_root is None: + video_path = video_id + else: + video_path = os.path.join(self.data_root, video_id) + + if self.data_root is None: + audio_path = audio_id + else: + audio_path = os.path.join(self.data_root, audio_id) + + if self.data_root is None: + control_video_id = control_video_id + else: + control_video_id = os.path.join(self.data_root, control_video_id) + + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found for {video_path}") + + # Video information + with VideoReader_contextmanager(video_path, num_threads=2) as video_reader: + total_frames = len(video_reader) + fps = video_reader.get_avg_fps() + if fps <= 0: + raise ValueError(f"Video has negative fps: {video_path}") + local_video_sample_stride = self.video_sample_stride + new_fps = int(fps // local_video_sample_stride) + while new_fps > 30: + local_video_sample_stride = local_video_sample_stride + 1 + new_fps = int(fps // local_video_sample_stride) + + max_possible_frames = (total_frames - 1) // local_video_sample_stride + 1 + actual_n_frames = min(self.video_sample_n_frames, max_possible_frames) + if actual_n_frames <= 0: + raise ValueError(f"Video too short: {video_path}") + + max_start = total_frames - (actual_n_frames - 1) * local_video_sample_stride - 1 + start_frame = random.randint(0, max_start) if max_start > 0 else 0 + frame_indices = [start_frame + i * local_video_sample_stride for i in range(actual_n_frames)] + + try: + sample_args = (video_reader, frame_indices) + pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + _, height, width, channel = np.shape(pixel_values) + if self.enable_motion_info: + motion_pixel_values = np.ones([self.motion_frames, height, width, channel]) * 127.5 + if start_frame > 0: + motion_max_possible_frames = (start_frame - 1) // local_video_sample_stride + 1 + motion_frame_indices = [0 + i * local_video_sample_stride for i in range(motion_max_possible_frames)] + motion_frame_indices = motion_frame_indices[-self.motion_frames:] + + _motion_sample_args = (video_reader, motion_frame_indices) + _motion_pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=_motion_sample_args + ) + motion_pixel_values[-len(motion_frame_indices):] = _motion_pixel_values + + if not self.enable_bucket: + motion_pixel_values = torch.from_numpy(motion_pixel_values).permute(0, 3, 1, 2).contiguous() + motion_pixel_values = motion_pixel_values / 255. + motion_pixel_values = self.pixel_transforms(motion_pixel_values) + else: + motion_pixel_values = None + + if not self.enable_bucket: + pixel_values = torch.from_numpy(pixel_values).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + pixel_values = self.pixel_transforms(pixel_values) + + # Audio information + start_time = start_frame / fps + end_time = (start_frame + (actual_n_frames - 1) * local_video_sample_stride) / fps + duration = end_time - start_time + + audio_input, sample_rate = librosa.load(audio_path, sr=self.audio_sr) + start_sample = int(start_time * self.audio_sr) + end_sample = int(end_time * self.audio_sr) + + if start_sample >= len(audio_input): + raise ValueError(f"Audio file too short: {audio_path}") + else: + audio_segment = audio_input[start_sample:end_sample] + target_len = int(duration * self.audio_sr) + if len(audio_segment) < target_len: + raise ValueError(f"Audio file too short: {audio_path}") + + # Control information + with VideoReader_contextmanager(control_video_id, num_threads=2) as control_video_reader: + try: + sample_args = (control_video_reader, frame_indices) + control_pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + resized_frames = [] + for i in range(len(control_pixel_values)): + frame = control_pixel_values[i] + resized_frame = resize_frame(frame, max(self.video_sample_size)) + resized_frames.append(resized_frame) + control_pixel_values = np.array(control_pixel_values) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + if not self.enable_bucket: + control_pixel_values = torch.from_numpy(control_pixel_values).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + del control_video_reader + else: + control_pixel_values = control_pixel_values + + if not self.enable_bucket: + control_pixel_values = self.video_transforms(control_pixel_values) + + if random.random() < self.text_drop_ratio: + text = '' + + return pixel_values, motion_pixel_values, control_pixel_values, text, audio_segment, sample_rate, new_fps + + def __len__(self): + return self.length + + def __getitem__(self, idx): + while True: + sample = {} + try: + pixel_values, motion_pixel_values, control_pixel_values, text, audio, sample_rate, new_fps = self.get_batch(idx) + sample["pixel_values"] = pixel_values + sample["motion_pixel_values"] = motion_pixel_values + sample["control_pixel_values"] = control_pixel_values + sample["text"] = text + sample["audio"] = torch.from_numpy(audio).float() # 转为 tensor + sample["sample_rate"] = sample_rate + sample["fps"] = new_fps + sample["idx"] = idx + break + except Exception as e: + print(f"Error processing {idx}: {e}, retrying with random idx...") + idx = random.randint(0, self.length - 1) + + if self.enable_inpaint and not self.enable_bucket: + mask = get_random_mask(pixel_values.size(), image_start_only=True) + mask_pixel_values = pixel_values * (1 - mask) + torch.zeros_like(pixel_values) * mask + sample["mask_pixel_values"] = mask_pixel_values + sample["mask"] = mask + + clip_pixel_values = sample["pixel_values"][0].permute(1, 2, 0).contiguous() + clip_pixel_values = (clip_pixel_values * 0.5 + 0.5) * 255 + sample["clip_pixel_values"] = clip_pixel_values + + return sample + + +class VideoAnimateDataset(Dataset): + def __init__( + self, + ann_path, data_root=None, + video_sample_size=512, + video_sample_stride=4, + video_sample_n_frames=16, + video_repeat=0, + text_drop_ratio=0.1, + enable_bucket=False, + video_length_drop_start=0.1, + video_length_drop_end=0.9, + return_file_name=False, + ): + # Loading annotations from files + print(f"loading annotations from {ann_path} ...") + if ann_path.endswith('.csv'): + with open(ann_path, 'r') as csvfile: + dataset = list(csv.DictReader(csvfile)) + elif ann_path.endswith('.json'): + dataset = json.load(open(ann_path)) + + self.data_root = data_root + + # It's used to balance num of images and videos. + if video_repeat > 0: + self.dataset = [] + for data in dataset: + if data.get('type', 'image') != 'video': + self.dataset.append(data) + + for _ in range(video_repeat): + for data in dataset: + if data.get('type', 'image') == 'video': + self.dataset.append(data) + else: + self.dataset = dataset + del dataset + + self.length = len(self.dataset) + print(f"data scale: {self.length}") + # TODO: enable bucket training + self.enable_bucket = enable_bucket + self.text_drop_ratio = text_drop_ratio + + self.video_length_drop_start = video_length_drop_start + self.video_length_drop_end = video_length_drop_end + + # Video params + self.video_sample_stride = video_sample_stride + self.video_sample_n_frames = video_sample_n_frames + self.video_sample_size = tuple(video_sample_size) if not isinstance(video_sample_size, int) else (video_sample_size, video_sample_size) + self.video_transforms = transforms.Compose( + [ + transforms.Resize(min(self.video_sample_size)), + transforms.CenterCrop(self.video_sample_size), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], inplace=True), + ] + ) + + self.larger_side_of_image_and_video = min(self.video_sample_size) + + def get_batch(self, idx): + data_info = self.dataset[idx % len(self.dataset)] + video_id, text = data_info['file_path'], data_info['text'] + + if self.data_root is None: + video_dir = video_id + else: + video_dir = os.path.join(self.data_root, video_id) + + with VideoReader_contextmanager(video_dir, num_threads=2) as video_reader: + min_sample_n_frames = min( + self.video_sample_n_frames, + int(len(video_reader) * (self.video_length_drop_end - self.video_length_drop_start) // self.video_sample_stride) + ) + if min_sample_n_frames == 0: + raise ValueError(f"No Frames in video.") + + video_length = int(self.video_length_drop_end * len(video_reader)) + clip_length = min(video_length, (min_sample_n_frames - 1) * self.video_sample_stride + 1) + start_idx = random.randint(int(self.video_length_drop_start * video_length), video_length - clip_length) if video_length != clip_length else 0 + batch_index = np.linspace(start_idx, start_idx + clip_length - 1, min_sample_n_frames, dtype=int) + + try: + sample_args = (video_reader, batch_index) + pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + resized_frames = [] + for i in range(len(pixel_values)): + frame = pixel_values[i] + resized_frame = resize_frame(frame, self.larger_side_of_image_and_video) + resized_frames.append(resized_frame) + pixel_values = np.array(resized_frames) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + if not self.enable_bucket: + pixel_values = torch.from_numpy(pixel_values).permute(0, 3, 1, 2).contiguous() + pixel_values = pixel_values / 255. + del video_reader + else: + pixel_values = pixel_values + + if not self.enable_bucket: + pixel_values = self.video_transforms(pixel_values) + + # Random use no text generation + if random.random() < self.text_drop_ratio: + text = '' + + control_video_id = data_info['control_file_path'] + + if control_video_id is not None: + if self.data_root is None: + control_video_id = control_video_id + else: + control_video_id = os.path.join(self.data_root, control_video_id) + + if control_video_id is not None: + with VideoReader_contextmanager(control_video_id, num_threads=2) as control_video_reader: + try: + sample_args = (control_video_reader, batch_index) + control_pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + resized_frames = [] + for i in range(len(control_pixel_values)): + frame = control_pixel_values[i] + resized_frame = resize_frame(frame, self.larger_side_of_image_and_video) + resized_frames.append(resized_frame) + control_pixel_values = np.array(resized_frames) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + if not self.enable_bucket: + control_pixel_values = torch.from_numpy(control_pixel_values).permute(0, 3, 1, 2).contiguous() + control_pixel_values = control_pixel_values / 255. + del control_video_reader + else: + control_pixel_values = control_pixel_values + + if not self.enable_bucket: + control_pixel_values = self.video_transforms(control_pixel_values) + else: + if not self.enable_bucket: + control_pixel_values = torch.zeros_like(pixel_values) + else: + control_pixel_values = np.zeros_like(pixel_values) + + face_video_id = data_info['face_file_path'] + + if face_video_id is not None: + if self.data_root is None: + face_video_id = face_video_id + else: + face_video_id = os.path.join(self.data_root, face_video_id) + + if face_video_id is not None: + with VideoReader_contextmanager(face_video_id, num_threads=2) as face_video_reader: + try: + sample_args = (face_video_reader, batch_index) + face_pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + resized_frames = [] + for i in range(len(face_pixel_values)): + frame = face_pixel_values[i] + resized_frame = resize_frame(frame, self.larger_side_of_image_and_video) + resized_frames.append(resized_frame) + face_pixel_values = np.array(resized_frames) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + if not self.enable_bucket: + face_pixel_values = torch.from_numpy(face_pixel_values).permute(0, 3, 1, 2).contiguous() + face_pixel_values = face_pixel_values / 255. + del face_video_reader + else: + face_pixel_values = face_pixel_values + + if not self.enable_bucket: + face_pixel_values = self.video_transforms(face_pixel_values) + else: + if not self.enable_bucket: + face_pixel_values = torch.zeros_like(pixel_values) + else: + face_pixel_values = np.zeros_like(pixel_values) + + background_video_id = data_info.get('background_file_path', None) + + if background_video_id is not None: + if self.data_root is None: + background_video_id = background_video_id + else: + background_video_id = os.path.join(self.data_root, background_video_id) + + if background_video_id is not None: + with VideoReader_contextmanager(background_video_id, num_threads=2) as background_video_reader: + try: + sample_args = (background_video_reader, batch_index) + background_pixel_values = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + resized_frames = [] + for i in range(len(background_pixel_values)): + frame = background_pixel_values[i] + resized_frame = resize_frame(frame, self.larger_side_of_image_and_video) + resized_frames.append(resized_frame) + background_pixel_values = np.array(resized_frames) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + if not self.enable_bucket: + background_pixel_values = torch.from_numpy(background_pixel_values).permute(0, 3, 1, 2).contiguous() + background_pixel_values = background_pixel_values / 255. + del background_video_reader + else: + background_pixel_values = background_pixel_values + + if not self.enable_bucket: + background_pixel_values = self.video_transforms(background_pixel_values) + else: + if not self.enable_bucket: + background_pixel_values = torch.ones_like(pixel_values) * 127.5 + else: + background_pixel_values = np.ones_like(pixel_values) * 127.5 + + mask_video_id = data_info.get('mask_file_path', None) + + if mask_video_id is not None: + if self.data_root is None: + mask_video_id = mask_video_id + else: + mask_video_id = os.path.join(self.data_root, mask_video_id) + + if mask_video_id is not None: + with VideoReader_contextmanager(mask_video_id, num_threads=2) as mask_video_reader: + try: + sample_args = (mask_video_reader, batch_index) + mask = func_timeout( + VIDEO_READER_TIMEOUT, get_video_reader_batch, args=sample_args + ) + resized_frames = [] + for i in range(len(mask)): + frame = mask[i] + resized_frame = resize_frame(frame, self.larger_side_of_image_and_video) + resized_frames.append(resized_frame) + mask = np.array(resized_frames) + except FunctionTimedOut: + raise ValueError(f"Read {idx} timeout.") + except Exception as e: + raise ValueError(f"Failed to extract frames from video. Error is {e}.") + + if not self.enable_bucket: + mask = torch.from_numpy(mask).permute(0, 3, 1, 2).contiguous() + mask = mask / 255. + del mask_video_reader + else: + mask = mask + else: + if not self.enable_bucket: + mask = torch.ones_like(pixel_values) + else: + mask = np.ones_like(pixel_values) * 255 + mask = mask[:, :, :, :1] + + ref_pixel_values_path = data_info.get('ref_file_path', []) + if self.data_root is not None: + ref_pixel_values_path = os.path.join(self.data_root, ref_pixel_values_path) + ref_pixel_values = Image.open(ref_pixel_values_path).convert('RGB') + + if not self.enable_bucket: + raise ValueError("Not enable_bucket is not supported now. ") + else: + ref_pixel_values = np.array(ref_pixel_values) + + return pixel_values, control_pixel_values, face_pixel_values, background_pixel_values, mask, ref_pixel_values, text, "video" + + def __len__(self): + return self.length + + def __getitem__(self, idx): + data_info = self.dataset[idx % len(self.dataset)] + data_type = data_info.get('type', 'image') + while True: + sample = {} + try: + data_info_local = self.dataset[idx % len(self.dataset)] + data_type_local = data_info_local.get('type', 'image') + if data_type_local != data_type: + raise ValueError("data_type_local != data_type") + + pixel_values, control_pixel_values, face_pixel_values, background_pixel_values, mask, ref_pixel_values, name, data_type = \ + self.get_batch(idx) + + sample["pixel_values"] = pixel_values + sample["control_pixel_values"] = control_pixel_values + sample["face_pixel_values"] = face_pixel_values + sample["background_pixel_values"] = background_pixel_values + sample["mask"] = mask + sample["ref_pixel_values"] = ref_pixel_values + sample["clip_pixel_values"] = ref_pixel_values + sample["text"] = name + sample["data_type"] = data_type + sample["idx"] = idx + + if len(sample) > 0: + break + except Exception as e: + print(e, self.dataset[idx % len(self.dataset)]) + idx = random.randint(0, self.length-1) + + return sample + + +if __name__ == "__main__": + if 1: + dataset = VideoDataset( + json_path="./webvidval/results_2M_val.json", + sample_size=256, + sample_stride=4, sample_n_frames=16, + ) + + if 0: + dataset = WebVid10M( + csv_path="./webvid/results_2M_val.csv", + video_folder="./webvid/2M_val", + sample_size=256, + sample_stride=4, sample_n_frames=16, + is_image=False, + ) + + dataloader = torch.utils.data.DataLoader(dataset, batch_size=4, num_workers=0,) + for idx, batch in enumerate(dataloader): + print(batch["pixel_values"].shape, len(batch["text"])) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/data/utils.py b/VideoX-Fun/videox_fun/data/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..514a41bae567fbf2eb4d80f7163fefceaf9e8974 --- /dev/null +++ b/VideoX-Fun/videox_fun/data/utils.py @@ -0,0 +1,347 @@ +import csv +import gc +import io +import json +import math +import os +import random +from contextlib import contextmanager +from random import shuffle +from threading import Thread + +import albumentations +import cv2 +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms as transforms +from decord import VideoReader +from einops import rearrange +from func_timeout import FunctionTimedOut, func_timeout +from packaging import version as pver +from PIL import Image +from safetensors.torch import load_file +from torch.utils.data import BatchSampler, Sampler +from torch.utils.data.dataset import Dataset + +VIDEO_READER_TIMEOUT = 20 + +def get_random_mask(shape, image_start_only=False): + f, c, h, w = shape + mask = torch.zeros((f, 1, h, w), dtype=torch.uint8) + + if not image_start_only: + if f != 1: + mask_index = np.random.choice([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], p=[0.05, 0.2, 0.2, 0.2, 0.05, 0.05, 0.05, 0.1, 0.05, 0.05]) + else: + mask_index = np.random.choice([0, 1, 7, 8], p = [0.2, 0.7, 0.05, 0.05]) + if mask_index == 0: + center_x = torch.randint(0, w, (1,)).item() + center_y = torch.randint(0, h, (1,)).item() + block_size_x = torch.randint(w // 4, w // 4 * 3, (1,)).item() # 方块的宽度范围 + block_size_y = torch.randint(h // 4, h // 4 * 3, (1,)).item() # 方块的高度范围 + + start_x = max(center_x - block_size_x // 2, 0) + end_x = min(center_x + block_size_x // 2, w) + start_y = max(center_y - block_size_y // 2, 0) + end_y = min(center_y + block_size_y // 2, h) + mask[:, :, start_y:end_y, start_x:end_x] = 1 + elif mask_index == 1: + mask[:, :, :, :] = 1 + elif mask_index == 2: + mask_frame_index = np.random.randint(1, 5) + mask[mask_frame_index:, :, :, :] = 1 + elif mask_index == 3: + mask_frame_index = np.random.randint(1, 5) + mask[mask_frame_index:-mask_frame_index, :, :, :] = 1 + elif mask_index == 4: + center_x = torch.randint(0, w, (1,)).item() + center_y = torch.randint(0, h, (1,)).item() + block_size_x = torch.randint(w // 4, w // 4 * 3, (1,)).item() # 方块的宽度范围 + block_size_y = torch.randint(h // 4, h // 4 * 3, (1,)).item() # 方块的高度范围 + + start_x = max(center_x - block_size_x // 2, 0) + end_x = min(center_x + block_size_x // 2, w) + start_y = max(center_y - block_size_y // 2, 0) + end_y = min(center_y + block_size_y // 2, h) + + mask_frame_before = np.random.randint(0, f // 2) + mask_frame_after = np.random.randint(f // 2, f) + mask[mask_frame_before:mask_frame_after, :, start_y:end_y, start_x:end_x] = 1 + elif mask_index == 5: + mask = torch.randint(0, 2, (f, 1, h, w), dtype=torch.uint8) + elif mask_index == 6: + num_frames_to_mask = random.randint(1, max(f // 2, 1)) + frames_to_mask = random.sample(range(f), num_frames_to_mask) + + for i in frames_to_mask: + block_height = random.randint(1, h // 4) + block_width = random.randint(1, w // 4) + top_left_y = random.randint(0, h - block_height) + top_left_x = random.randint(0, w - block_width) + mask[i, 0, top_left_y:top_left_y + block_height, top_left_x:top_left_x + block_width] = 1 + elif mask_index == 7: + center_x = torch.randint(0, w, (1,)).item() + center_y = torch.randint(0, h, (1,)).item() + a = torch.randint(min(w, h) // 8, min(w, h) // 4, (1,)).item() # 长半轴 + b = torch.randint(min(h, w) // 8, min(h, w) // 4, (1,)).item() # 短半轴 + + for i in range(h): + for j in range(w): + if ((i - center_y) ** 2) / (b ** 2) + ((j - center_x) ** 2) / (a ** 2) < 1: + mask[:, :, i, j] = 1 + elif mask_index == 8: + center_x = torch.randint(0, w, (1,)).item() + center_y = torch.randint(0, h, (1,)).item() + radius = torch.randint(min(h, w) // 8, min(h, w) // 4, (1,)).item() + for i in range(h): + for j in range(w): + if (i - center_y) ** 2 + (j - center_x) ** 2 < radius ** 2: + mask[:, :, i, j] = 1 + elif mask_index == 9: + for idx in range(f): + if np.random.rand() > 0.5: + mask[idx, :, :, :] = 1 + else: + raise ValueError(f"The mask_index {mask_index} is not define") + else: + if f != 1: + mask[1:, :, :, :] = 1 + else: + mask[:, :, :, :] = 1 + return mask + +@contextmanager +def VideoReader_contextmanager(*args, **kwargs): + vr = VideoReader(*args, **kwargs) + try: + yield vr + finally: + del vr + gc.collect() + +def get_video_reader_batch(video_reader, batch_index): + frames = video_reader.get_batch(batch_index).asnumpy() + return frames + +def resize_frame(frame, target_short_side): + h, w, _ = frame.shape + if h < w: + if target_short_side > h: + return frame + new_h = target_short_side + new_w = int(target_short_side * w / h) + else: + if target_short_side > w: + return frame + new_w = target_short_side + new_h = int(target_short_side * h / w) + + resized_frame = cv2.resize(frame, (new_w, new_h)) + return resized_frame + +def padding_image(images, new_width, new_height): + new_image = Image.new('RGB', (new_width, new_height), (255, 255, 255)) + + aspect_ratio = images.width / images.height + if new_width / new_height > 1: + if aspect_ratio > new_width / new_height: + new_img_width = new_width + new_img_height = int(new_img_width / aspect_ratio) + else: + new_img_height = new_height + new_img_width = int(new_img_height * aspect_ratio) + else: + if aspect_ratio > new_width / new_height: + new_img_width = new_width + new_img_height = int(new_img_width / aspect_ratio) + else: + new_img_height = new_height + new_img_width = int(new_img_height * aspect_ratio) + + resized_img = images.resize((new_img_width, new_img_height)) + + paste_x = (new_width - new_img_width) // 2 + paste_y = (new_height - new_img_height) // 2 + + new_image.paste(resized_img, (paste_x, paste_y)) + + return new_image + +def resize_image_with_target_area(img: Image.Image, target_area: int = 1024 * 1024) -> Image.Image: + """ + 将 PIL 图像缩放到接近指定像素面积（target_area），保持原始宽高比， + 并确保新宽度和高度均为 32 的整数倍。 + + 参数: + img (PIL.Image.Image): 输入图像 + target_area (int): 目标像素总面积，例如 1024*1024 = 1048576 + + 返回: + PIL.Image.Image: Resize 后的图像 + """ + orig_w, orig_h = img.size + if orig_w == 0 or orig_h == 0: + raise ValueError("Input image has zero width or height.") + + ratio = orig_w / orig_h + ideal_width = math.sqrt(target_area * ratio) + ideal_height = ideal_width / ratio + + new_width = round(ideal_width / 32) * 32 + new_height = round(ideal_height / 32) * 32 + + new_width = max(32, new_width) + new_height = max(32, new_height) + + new_width = int(new_width) + new_height = int(new_height) + + resized_img = img.resize((new_width, new_height), Image.LANCZOS) + return resized_img + +class Camera(object): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + def __init__(self, entry): + fx, fy, cx, cy = entry[1:5] + self.fx = fx + self.fy = fy + self.cx = cx + self.cy = cy + w2c_mat = np.array(entry[7:]).reshape(3, 4) + w2c_mat_4x4 = np.eye(4) + w2c_mat_4x4[:3, :] = w2c_mat + self.w2c_mat = w2c_mat_4x4 + self.c2w_mat = np.linalg.inv(w2c_mat_4x4) + +def custom_meshgrid(*args): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + # ref: https://pytorch.org/docs/stable/generated/torch.meshgrid.html?highlight=meshgrid#torch.meshgrid + if pver.parse(torch.__version__) < pver.parse('1.10'): + return torch.meshgrid(*args) + else: + return torch.meshgrid(*args, indexing='ij') + +def get_relative_pose(cam_params): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + abs_w2cs = [cam_param.w2c_mat for cam_param in cam_params] + abs_c2ws = [cam_param.c2w_mat for cam_param in cam_params] + cam_to_origin = 0 + target_cam_c2w = np.array([ + [1, 0, 0, 0], + [0, 1, 0, -cam_to_origin], + [0, 0, 1, 0], + [0, 0, 0, 1] + ]) + abs2rel = target_cam_c2w @ abs_w2cs[0] + ret_poses = [target_cam_c2w, ] + [abs2rel @ abs_c2w for abs_c2w in abs_c2ws[1:]] + ret_poses = np.array(ret_poses, dtype=np.float32) + return ret_poses + +def ray_condition(K, c2w, H, W, device): + """Copied from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + # c2w: B, V, 4, 4 + # K: B, V, 4 + + B = K.shape[0] + + j, i = custom_meshgrid( + torch.linspace(0, H - 1, H, device=device, dtype=c2w.dtype), + torch.linspace(0, W - 1, W, device=device, dtype=c2w.dtype), + ) + i = i.reshape([1, 1, H * W]).expand([B, 1, H * W]) + 0.5 # [B, HxW] + j = j.reshape([1, 1, H * W]).expand([B, 1, H * W]) + 0.5 # [B, HxW] + + fx, fy, cx, cy = K.chunk(4, dim=-1) # B,V, 1 + + zs = torch.ones_like(i) # [B, HxW] + xs = (i - cx) / fx * zs + ys = (j - cy) / fy * zs + zs = zs.expand_as(ys) + + directions = torch.stack((xs, ys, zs), dim=-1) # B, V, HW, 3 + directions = directions / directions.norm(dim=-1, keepdim=True) # B, V, HW, 3 + + rays_d = directions @ c2w[..., :3, :3].transpose(-1, -2) # B, V, 3, HW + rays_o = c2w[..., :3, 3] # B, V, 3 + rays_o = rays_o[:, :, None].expand_as(rays_d) # B, V, 3, HW + # c2w @ dirctions + rays_dxo = torch.cross(rays_o, rays_d) + plucker = torch.cat([rays_dxo, rays_d], dim=-1) + plucker = plucker.reshape(B, c2w.shape[1], H, W, 6) # B, V, H, W, 6 + # plucker = plucker.permute(0, 1, 4, 2, 3) + return plucker + +def process_pose_file(pose_file_path, width=672, height=384, original_pose_width=1280, original_pose_height=720, device='cpu', return_poses=False): + """Modified from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + with open(pose_file_path, 'r') as f: + poses = f.readlines() + + poses = [pose.strip().split(' ') for pose in poses[1:]] + cam_params = [[float(x) for x in pose] for pose in poses] + if return_poses: + return cam_params + else: + cam_params = [Camera(cam_param) for cam_param in cam_params] + + sample_wh_ratio = width / height + pose_wh_ratio = original_pose_width / original_pose_height # Assuming placeholder ratios, change as needed + + if pose_wh_ratio > sample_wh_ratio: + resized_ori_w = height * pose_wh_ratio + for cam_param in cam_params: + cam_param.fx = resized_ori_w * cam_param.fx / width + else: + resized_ori_h = width / pose_wh_ratio + for cam_param in cam_params: + cam_param.fy = resized_ori_h * cam_param.fy / height + + intrinsic = np.asarray([[cam_param.fx * width, + cam_param.fy * height, + cam_param.cx * width, + cam_param.cy * height] + for cam_param in cam_params], dtype=np.float32) + + K = torch.as_tensor(intrinsic)[None] # [1, 1, 4] + c2ws = get_relative_pose(cam_params) # Assuming this function is defined elsewhere + c2ws = torch.as_tensor(c2ws)[None] # [1, n_frame, 4, 4] + plucker_embedding = ray_condition(K, c2ws, height, width, device=device)[0].permute(0, 3, 1, 2).contiguous() # V, 6, H, W + plucker_embedding = plucker_embedding[None] + plucker_embedding = rearrange(plucker_embedding, "b f c h w -> b f h w c")[0] + return plucker_embedding + +def process_pose_params(cam_params, width=672, height=384, original_pose_width=1280, original_pose_height=720, device='cpu'): + """Modified from https://github.com/hehao13/CameraCtrl/blob/main/inference.py + """ + cam_params = [Camera(cam_param) for cam_param in cam_params] + + sample_wh_ratio = width / height + pose_wh_ratio = original_pose_width / original_pose_height # Assuming placeholder ratios, change as needed + + if pose_wh_ratio > sample_wh_ratio: + resized_ori_w = height * pose_wh_ratio + for cam_param in cam_params: + cam_param.fx = resized_ori_w * cam_param.fx / width + else: + resized_ori_h = width / pose_wh_ratio + for cam_param in cam_params: + cam_param.fy = resized_ori_h * cam_param.fy / height + + intrinsic = np.asarray([[cam_param.fx * width, + cam_param.fy * height, + cam_param.cx * width, + cam_param.cy * height] + for cam_param in cam_params], dtype=np.float32) + + K = torch.as_tensor(intrinsic)[None] # [1, 1, 4] + c2ws = get_relative_pose(cam_params) # Assuming this function is defined elsewhere + c2ws = torch.as_tensor(c2ws)[None] # [1, n_frame, 4, 4] + plucker_embedding = ray_condition(K, c2ws, height, width, device=device)[0].permute(0, 3, 1, 2).contiguous() # V, 6, H, W + plucker_embedding = plucker_embedding[None] + plucker_embedding = rearrange(plucker_embedding, "b f c h w -> b f h w c")[0] + return plucker_embedding \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/attention_utils.py b/VideoX-Fun/videox_fun/models/attention_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ed6851739053a43bdec353d17c63642054042adf --- /dev/null +++ b/VideoX-Fun/videox_fun/models/attention_utils.py @@ -0,0 +1,211 @@ +import os + +import torch +import warnings + +try: + import flash_attn_interface + FLASH_ATTN_3_AVAILABLE = True +except ModuleNotFoundError: + FLASH_ATTN_3_AVAILABLE = False + +try: + import flash_attn + FLASH_ATTN_2_AVAILABLE = True +except ModuleNotFoundError: + FLASH_ATTN_2_AVAILABLE = False + +try: + major, minor = torch.cuda.get_device_capability(0) + if f"{major}.{minor}" == "8.0": + from sageattention_sm80 import sageattn + SAGE_ATTENTION_AVAILABLE = True + elif f"{major}.{minor}" == "8.6": + from sageattention_sm86 import sageattn + SAGE_ATTENTION_AVAILABLE = True + elif f"{major}.{minor}" == "8.9": + from sageattention_sm89 import sageattn + SAGE_ATTENTION_AVAILABLE = True + elif f"{major}.{minor}" == "9.0": + from sageattention_sm90 import sageattn + SAGE_ATTENTION_AVAILABLE = True + elif major>9: + from sageattention_sm120 import sageattn + SAGE_ATTENTION_AVAILABLE = True +except: + try: + from sageattention import sageattn + SAGE_ATTENTION_AVAILABLE = True + except: + sageattn = None + SAGE_ATTENTION_AVAILABLE = False + +def flash_attention( + q, + k, + v, + q_lens=None, + k_lens=None, + dropout_p=0., + softmax_scale=None, + q_scale=None, + causal=False, + window_size=(-1, -1), + deterministic=False, + dtype=torch.bfloat16, + version=None, +): + """ + q: [B, Lq, Nq, C1]. + k: [B, Lk, Nk, C1]. + v: [B, Lk, Nk, C2]. Nq must be divisible by Nk. + q_lens: [B]. + k_lens: [B]. + dropout_p: float. Dropout probability. + softmax_scale: float. The scaling of QK^T before applying softmax. + causal: bool. Whether to apply causal attention mask. + window_size: (left right). If not (-1, -1), apply sliding window local attention. + deterministic: bool. If True, slightly slower and uses more memory. + dtype: torch.dtype. Apply when dtype of q/k/v is not float16/bfloat16. + """ + half_dtypes = (torch.float16, torch.bfloat16) + assert dtype in half_dtypes + assert q.device.type == 'cuda' and q.size(-1) <= 256 + + # params + b, lq, lk, out_dtype = q.size(0), q.size(1), k.size(1), q.dtype + + def half(x): + return x if x.dtype in half_dtypes else x.to(dtype) + + # preprocess query + if q_lens is None: + q = half(q.flatten(0, 1)) + q_lens = torch.tensor( + [lq] * b, dtype=torch.int32).to( + device=q.device, non_blocking=True) + else: + q = half(torch.cat([u[:v] for u, v in zip(q, q_lens)])) + + # preprocess key, value + if k_lens is None: + k = half(k.flatten(0, 1)) + v = half(v.flatten(0, 1)) + k_lens = torch.tensor( + [lk] * b, dtype=torch.int32).to( + device=k.device, non_blocking=True) + else: + k = half(torch.cat([u[:v] for u, v in zip(k, k_lens)])) + v = half(torch.cat([u[:v] for u, v in zip(v, k_lens)])) + + q = q.to(v.dtype) + k = k.to(v.dtype) + + if q_scale is not None: + q = q * q_scale + + if version is not None and version == 3 and not FLASH_ATTN_3_AVAILABLE: + warnings.warn( + 'Flash attention 3 is not available, use flash attention 2 instead.' + ) + + # apply attention + if (version is None or version == 3) and FLASH_ATTN_3_AVAILABLE: + # Note: dropout_p, window_size are not supported in FA3 now. + x = flash_attn_interface.flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=torch.cat([q_lens.new_zeros([1]), q_lens]).cumsum( + 0, dtype=torch.int32).to(q.device, non_blocking=True), + cu_seqlens_k=torch.cat([k_lens.new_zeros([1]), k_lens]).cumsum( + 0, dtype=torch.int32).to(q.device, non_blocking=True), + seqused_q=None, + seqused_k=None, + max_seqlen_q=lq, + max_seqlen_k=lk, + softmax_scale=softmax_scale, + causal=causal, + deterministic=deterministic)[0].unflatten(0, (b, lq)) + else: + assert FLASH_ATTN_2_AVAILABLE + x = flash_attn.flash_attn_varlen_func( + q=q, + k=k, + v=v, + cu_seqlens_q=torch.cat([q_lens.new_zeros([1]), q_lens]).cumsum( + 0, dtype=torch.int32).to(q.device, non_blocking=True), + cu_seqlens_k=torch.cat([k_lens.new_zeros([1]), k_lens]).cumsum( + 0, dtype=torch.int32).to(q.device, non_blocking=True), + max_seqlen_q=lq, + max_seqlen_k=lk, + dropout_p=dropout_p, + softmax_scale=softmax_scale, + causal=causal, + window_size=window_size, + deterministic=deterministic).unflatten(0, (b, lq)) + + # output + return x.type(out_dtype) + + +def attention( + q, + k, + v, + q_lens=None, + k_lens=None, + dropout_p=0., + softmax_scale=None, + q_scale=None, + causal=False, + window_size=(-1, -1), + deterministic=False, + dtype=torch.bfloat16, + fa_version=None, + attention_type=None, + attn_mask=None, +): + attention_type = os.environ.get("VIDEOX_ATTENTION_TYPE", "FLASH_ATTENTION") if attention_type is None else attention_type + if torch.is_grad_enabled() and attention_type == "SAGE_ATTENTION": + attention_type = "FLASH_ATTENTION" + + if attention_type == "SAGE_ATTENTION" and SAGE_ATTENTION_AVAILABLE: + if q_lens is not None or k_lens is not None: + warnings.warn( + 'Padding mask is disabled when using scaled_dot_product_attention. It can have a significant impact on performance.' + ) + + out = sageattn( + q, k, v, attn_mask=attn_mask, tensor_layout="NHD", is_causal=causal, dropout_p=dropout_p) + + elif attention_type == "FLASH_ATTENTION" and (FLASH_ATTN_2_AVAILABLE or FLASH_ATTN_3_AVAILABLE): + return flash_attention( + q=q, + k=k, + v=v, + q_lens=q_lens, + k_lens=k_lens, + dropout_p=dropout_p, + softmax_scale=softmax_scale, + q_scale=q_scale, + causal=causal, + window_size=window_size, + deterministic=deterministic, + dtype=dtype, + version=fa_version, + ) + else: + if q_lens is not None or k_lens is not None: + warnings.warn( + 'Padding mask is disabled when using scaled_dot_product_attention. It can have a significant impact on performance.' + ) + q = q.transpose(1, 2) + k = k.transpose(1, 2) + v = v.transpose(1, 2) + + out = torch.nn.functional.scaled_dot_product_attention( + q, k, v, attn_mask=attn_mask, is_causal=causal, dropout_p=dropout_p) + + out = out.transpose(1, 2).contiguous() + return out diff --git a/VideoX-Fun/videox_fun/models/cache_utils.py b/VideoX-Fun/videox_fun/models/cache_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..3c2921fa1237dcf3be7005f7b6c8b7ac47aaf824 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/cache_utils.py @@ -0,0 +1,80 @@ +import numpy as np +import torch + +def get_teacache_coefficients(model_name): + if "wan2.1-t2v-1.3b" in model_name.lower() or "wan2.1-fun-1.3b" in model_name.lower() \ + or "wan2.1-fun-v1.1-1.3b" in model_name.lower() or "wan2.1-vace-1.3b" in model_name.lower(): + return [-5.21862437e+04, 9.23041404e+03, -5.28275948e+02, 1.36987616e+01, -4.99875664e-02] + elif "wan2.1-t2v-14b" in model_name.lower(): + return [-3.03318725e+05, 4.90537029e+04, -2.65530556e+03, 5.87365115e+01, -3.15583525e-01] + elif "wan2.1-i2v-14b-480p" in model_name.lower(): + return [2.57151496e+05, -3.54229917e+04, 1.40286849e+03, -1.35890334e+01, 1.32517977e-01] + elif "wan2.1-i2v-14b-720p" in model_name.lower() or "wan2.1-fun-14b" in model_name.lower() or "wan2.2-fun" in model_name.lower() \ + or "wan2.2-i2v-a14b" in model_name.lower() or "wan2.2-t2v-a14b" in model_name.lower() or "wan2.2-ti2v-5b" in model_name.lower() \ + or "wan2.2-s2v" in model_name.lower() or "wan2.1-vace-14b" in model_name.lower() or "wan2.2-vace-fun" in model_name.lower() \ + or "wan2.2-animate" in model_name.lower(): + return [8.10705460e+03, 2.13393892e+03, -3.72934672e+02, 1.66203073e+01, -4.17769401e-02] + elif "qwen-image" in model_name.lower(): + # Copied from https://github.com/chenpipi0807/ComfyUI-TeaCache/blob/main/nodes.py + return [-4.50000000e+02, 2.80000000e+02, -4.50000000e+01, 3.20000000e+00, -2.00000000e-02] + else: + print(f"The model {model_name} is not supported by TeaCache.") + return None + + +class TeaCache(): + """ + Timestep Embedding Aware Cache, a training-free caching approach that estimates and leverages + the fluctuating differences among model outputs across timesteps, thereby accelerating the inference. + Please refer to: + 1. https://github.com/ali-vilab/TeaCache. + 2. Liu, Feng, et al. "Timestep Embedding Tells: It's Time to Cache for Video Diffusion Model." arXiv preprint arXiv:2411.19108 (2024). + """ + def __init__( + self, + coefficients: list[float], + num_steps: int, + rel_l1_thresh: float = 0.0, + num_skip_start_steps: int = 0, + offload: bool = True, + ): + if num_steps < 1: + raise ValueError(f"`num_steps` must be greater than 0 but is {num_steps}.") + if rel_l1_thresh < 0: + raise ValueError(f"`rel_l1_thresh` must be greater than or equal to 0 but is {rel_l1_thresh}.") + if num_skip_start_steps < 0 or num_skip_start_steps > num_steps: + raise ValueError( + "`num_skip_start_steps` must be great than or equal to 0 and " + f"less than or equal to `num_steps={num_steps}` but is {num_skip_start_steps}." + ) + self.coefficients = coefficients + self.num_steps = num_steps + self.rel_l1_thresh = rel_l1_thresh + self.num_skip_start_steps = num_skip_start_steps + self.offload = offload + self.rescale_func = np.poly1d(self.coefficients) + + self.cnt = 0 + self.should_calc = True + self.accumulated_rel_l1_distance = 0 + self.previous_modulated_input = None + # Some pipelines concatenate the unconditional and text guide in forward. + self.previous_residual = None + # Some pipelines perform forward propagation separately on the unconditional and text guide. + self.previous_residual_cond = None + self.previous_residual_uncond = None + + @staticmethod + def compute_rel_l1_distance(prev: torch.Tensor, cur: torch.Tensor) -> torch.Tensor: + rel_l1_distance = (torch.abs(cur - prev).mean()) / torch.abs(prev).mean() + + return rel_l1_distance.cpu().item() + + def reset(self): + self.cnt = 0 + self.should_calc = True + self.accumulated_rel_l1_distance = 0 + self.previous_modulated_input = None + self.previous_residual = None + self.previous_residual_cond = None + self.previous_residual_uncond = None diff --git a/VideoX-Fun/videox_fun/models/cogvideox_transformer3d.py b/VideoX-Fun/videox_fun/models/cogvideox_transformer3d.py new file mode 100644 index 0000000000000000000000000000000000000000..0b15b93e41810473044a98f503d18d79997042f2 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/cogvideox_transformer3d.py @@ -0,0 +1,915 @@ +# Copyright 2024 The CogVideoX team, Tsinghua University & ZhipuAI and The HuggingFace Team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +import os +from typing import Any, Dict, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.attention import Attention, FeedForward +from diffusers.models.attention_processor import ( + AttentionProcessor, FusedCogVideoXAttnProcessor2_0) +from diffusers.models.embeddings import (CogVideoXPatchEmbed, + TimestepEmbedding, Timesteps, + get_3d_sincos_pos_embed) +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import AdaLayerNorm, CogVideoXLayerNormZero +from diffusers.utils import is_torch_version, logging +from diffusers.utils.torch_utils import maybe_allow_in_graph +from torch import nn + +from ..dist import (get_sequence_parallel_rank, + get_sequence_parallel_world_size, get_sp_group, + xFuserLongContextAttention) +from ..dist.cogvideox_xfuser import CogVideoXMultiGPUsAttnProcessor2_0 +from .attention_utils import attention + + +class CogVideoXAttnProcessor2_0: + r""" + Processor for implementing scaled dot-product attention for the CogVideoX model. It applies a rotary embedding on + query and key vectors, but does not include spatial normalization. + """ + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError("CogVideoXAttnProcessor requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.") + + def __call__( + self, + attn, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + attention_mask: torch.Tensor = None, + image_rotary_emb: torch.Tensor = None, + ) -> torch.Tensor: + text_seq_length = encoder_hidden_states.size(1) + + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + batch_size, sequence_length, _ = hidden_states.shape + + if attention_mask is not None: + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) + + query = attn.to_q(hidden_states) + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + if attn.norm_q is not None: + query = attn.norm_q(query) + if attn.norm_k is not None: + key = attn.norm_k(key) + + # Apply RoPE if needed + if image_rotary_emb is not None: + from diffusers.models.embeddings import apply_rotary_emb + + query[:, :, text_seq_length:] = apply_rotary_emb(query[:, :, text_seq_length:], image_rotary_emb) + if not attn.is_cross_attention: + key[:, :, text_seq_length:] = apply_rotary_emb(key[:, :, text_seq_length:], image_rotary_emb) + + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + + hidden_states = attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, causal=False + ) + hidden_states = hidden_states.reshape(batch_size, -1, attn.heads * head_dim) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + encoder_hidden_states, hidden_states = hidden_states.split( + [text_seq_length, hidden_states.size(1) - text_seq_length], dim=1 + ) + return hidden_states, encoder_hidden_states + + +class CogVideoXPatchEmbed(nn.Module): + def __init__( + self, + patch_size: int = 2, + patch_size_t: Optional[int] = None, + in_channels: int = 16, + embed_dim: int = 1920, + text_embed_dim: int = 4096, + bias: bool = True, + sample_width: int = 90, + sample_height: int = 60, + sample_frames: int = 49, + temporal_compression_ratio: int = 4, + max_text_seq_length: int = 226, + spatial_interpolation_scale: float = 1.875, + temporal_interpolation_scale: float = 1.0, + use_positional_embeddings: bool = True, + use_learned_positional_embeddings: bool = True, + ) -> None: + super().__init__() + + post_patch_height = sample_height // patch_size + post_patch_width = sample_width // patch_size + post_time_compression_frames = (sample_frames - 1) // temporal_compression_ratio + 1 + self.num_patches = post_patch_height * post_patch_width * post_time_compression_frames + self.post_patch_height = post_patch_height + self.post_patch_width = post_patch_width + self.post_time_compression_frames = post_time_compression_frames + self.patch_size = patch_size + self.patch_size_t = patch_size_t + self.embed_dim = embed_dim + self.sample_height = sample_height + self.sample_width = sample_width + self.sample_frames = sample_frames + self.temporal_compression_ratio = temporal_compression_ratio + self.max_text_seq_length = max_text_seq_length + self.spatial_interpolation_scale = spatial_interpolation_scale + self.temporal_interpolation_scale = temporal_interpolation_scale + self.use_positional_embeddings = use_positional_embeddings + self.use_learned_positional_embeddings = use_learned_positional_embeddings + + if patch_size_t is None: + # CogVideoX 1.0 checkpoints + self.proj = nn.Conv2d( + in_channels, embed_dim, kernel_size=(patch_size, patch_size), stride=patch_size, bias=bias + ) + else: + # CogVideoX 1.5 checkpoints + self.proj = nn.Linear(in_channels * patch_size * patch_size * patch_size_t, embed_dim) + + self.text_proj = nn.Linear(text_embed_dim, embed_dim) + + if use_positional_embeddings or use_learned_positional_embeddings: + persistent = use_learned_positional_embeddings + pos_embedding = self._get_positional_embeddings(sample_height, sample_width, sample_frames) + self.register_buffer("pos_embedding", pos_embedding, persistent=persistent) + + def _get_positional_embeddings(self, sample_height: int, sample_width: int, sample_frames: int) -> torch.Tensor: + post_patch_height = sample_height // self.patch_size + post_patch_width = sample_width // self.patch_size + post_time_compression_frames = (sample_frames - 1) // self.temporal_compression_ratio + 1 + num_patches = post_patch_height * post_patch_width * post_time_compression_frames + + pos_embedding = get_3d_sincos_pos_embed( + self.embed_dim, + (post_patch_width, post_patch_height), + post_time_compression_frames, + self.spatial_interpolation_scale, + self.temporal_interpolation_scale, + ) + pos_embedding = torch.from_numpy(pos_embedding).flatten(0, 1) + joint_pos_embedding = torch.zeros( + 1, self.max_text_seq_length + num_patches, self.embed_dim, requires_grad=False + ) + joint_pos_embedding.data[:, self.max_text_seq_length :].copy_(pos_embedding) + + return joint_pos_embedding + + def forward(self, text_embeds: torch.Tensor, image_embeds: torch.Tensor): + r""" + Args: + text_embeds (`torch.Tensor`): + Input text embeddings. Expected shape: (batch_size, seq_length, embedding_dim). + image_embeds (`torch.Tensor`): + Input image embeddings. Expected shape: (batch_size, num_frames, channels, height, width). + """ + text_embeds = self.text_proj(text_embeds) + + text_batch_size, text_seq_length, text_channels = text_embeds.shape + batch_size, num_frames, channels, height, width = image_embeds.shape + + if self.patch_size_t is None: + image_embeds = image_embeds.reshape(-1, channels, height, width) + image_embeds = self.proj(image_embeds) + image_embeds = image_embeds.view(batch_size, num_frames, *image_embeds.shape[1:]) + image_embeds = image_embeds.flatten(3).transpose(2, 3) # [batch, num_frames, height x width, channels] + image_embeds = image_embeds.flatten(1, 2) # [batch, num_frames x height x width, channels] + else: + p = self.patch_size + p_t = self.patch_size_t + + image_embeds = image_embeds.permute(0, 1, 3, 4, 2) + # b, f, h, w, c => b, f // 2, 2, h // 2, 2, w // 2, 2, c + image_embeds = image_embeds.reshape( + batch_size, num_frames // p_t, p_t, height // p, p, width // p, p, channels + ) + # b, f // 2, 2, h // 2, 2, w // 2, 2, c => b, f // 2, h // 2, w // 2, c, 2, 2, 2 + image_embeds = image_embeds.permute(0, 1, 3, 5, 7, 2, 4, 6).flatten(4, 7).flatten(1, 3) + image_embeds = self.proj(image_embeds) + + embeds = torch.cat( + [text_embeds, image_embeds], dim=1 + ).contiguous() # [batch, seq_length + num_frames x height x width, channels] + + if self.use_positional_embeddings or self.use_learned_positional_embeddings: + seq_length = height * width * num_frames // (self.patch_size**2) + # pos_embeds = self.pos_embedding[:, : text_seq_length + seq_length] + pos_embeds = self.pos_embedding + emb_size = embeds.size()[-1] + pos_embeds_without_text = pos_embeds[:, text_seq_length: ].view(1, self.post_time_compression_frames, self.post_patch_height, self.post_patch_width, emb_size) + pos_embeds_without_text = pos_embeds_without_text.permute([0, 4, 1, 2, 3]) + pos_embeds_without_text = F.interpolate(pos_embeds_without_text,size=[self.post_time_compression_frames, height // self.patch_size, width // self.patch_size], mode='trilinear', align_corners=False) + pos_embeds_without_text = pos_embeds_without_text.permute([0, 2, 3, 4, 1]).view(1, -1, emb_size) + pos_embeds = torch.cat([pos_embeds[:, :text_seq_length], pos_embeds_without_text], dim = 1) + pos_embeds = pos_embeds[:, : text_seq_length + seq_length] + embeds = embeds + pos_embeds + + return embeds + +@maybe_allow_in_graph +class CogVideoXBlock(nn.Module): + r""" + Transformer block used in [CogVideoX](https://github.com/THUDM/CogVideo) model. + + Parameters: + dim (`int`): + The number of channels in the input and output. + num_attention_heads (`int`): + The number of heads to use for multi-head attention. + attention_head_dim (`int`): + The number of channels in each head. + time_embed_dim (`int`): + The number of channels in timestep embedding. + dropout (`float`, defaults to `0.0`): + The dropout probability to use. + activation_fn (`str`, defaults to `"gelu-approximate"`): + Activation function to be used in feed-forward. + attention_bias (`bool`, defaults to `False`): + Whether or not to use bias in attention projection layers. + qk_norm (`bool`, defaults to `True`): + Whether or not to use normalization after query and key projections in Attention. + norm_elementwise_affine (`bool`, defaults to `True`): + Whether to use learnable elementwise affine parameters for normalization. + norm_eps (`float`, defaults to `1e-5`): + Epsilon value for normalization layers. + final_dropout (`bool` defaults to `False`): + Whether to apply a final dropout after the last feed-forward layer. + ff_inner_dim (`int`, *optional*, defaults to `None`): + Custom hidden dimension of Feed-forward layer. If not provided, `4 * dim` is used. + ff_bias (`bool`, defaults to `True`): + Whether or not to use bias in Feed-forward layer. + attention_out_bias (`bool`, defaults to `True`): + Whether or not to use bias in Attention output projection layer. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + time_embed_dim: int, + dropout: float = 0.0, + activation_fn: str = "gelu-approximate", + attention_bias: bool = False, + qk_norm: bool = True, + norm_elementwise_affine: bool = True, + norm_eps: float = 1e-5, + final_dropout: bool = True, + ff_inner_dim: Optional[int] = None, + ff_bias: bool = True, + attention_out_bias: bool = True, + ): + super().__init__() + + # 1. Self Attention + self.norm1 = CogVideoXLayerNormZero(time_embed_dim, dim, norm_elementwise_affine, norm_eps, bias=True) + + self.attn1 = Attention( + query_dim=dim, + dim_head=attention_head_dim, + heads=num_attention_heads, + qk_norm="layer_norm" if qk_norm else None, + eps=1e-6, + bias=attention_bias, + out_bias=attention_out_bias, + processor=CogVideoXAttnProcessor2_0(), + ) + + # 2. Feed Forward + self.norm2 = CogVideoXLayerNormZero(time_embed_dim, dim, norm_elementwise_affine, norm_eps, bias=True) + + self.ff = FeedForward( + dim, + dropout=dropout, + activation_fn=activation_fn, + final_dropout=final_dropout, + inner_dim=ff_inner_dim, + bias=ff_bias, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + ) -> torch.Tensor: + text_seq_length = encoder_hidden_states.size(1) + + # norm & modulate + norm_hidden_states, norm_encoder_hidden_states, gate_msa, enc_gate_msa = self.norm1( + hidden_states, encoder_hidden_states, temb + ) + + # attention + attn_hidden_states, attn_encoder_hidden_states = self.attn1( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + image_rotary_emb=image_rotary_emb, + ) + + hidden_states = hidden_states + gate_msa * attn_hidden_states + encoder_hidden_states = encoder_hidden_states + enc_gate_msa * attn_encoder_hidden_states + + # norm & modulate + norm_hidden_states, norm_encoder_hidden_states, gate_ff, enc_gate_ff = self.norm2( + hidden_states, encoder_hidden_states, temb + ) + + # feed-forward + norm_hidden_states = torch.cat([norm_encoder_hidden_states, norm_hidden_states], dim=1) + ff_output = self.ff(norm_hidden_states) + + hidden_states = hidden_states + gate_ff * ff_output[:, text_seq_length:] + encoder_hidden_states = encoder_hidden_states + enc_gate_ff * ff_output[:, :text_seq_length] + + return hidden_states, encoder_hidden_states + + +class CogVideoXTransformer3DModel(ModelMixin, ConfigMixin): + """ + A Transformer model for video-like data in [CogVideoX](https://github.com/THUDM/CogVideo). + + Parameters: + num_attention_heads (`int`, defaults to `30`): + The number of heads to use for multi-head attention. + attention_head_dim (`int`, defaults to `64`): + The number of channels in each head. + in_channels (`int`, defaults to `16`): + The number of channels in the input. + out_channels (`int`, *optional*, defaults to `16`): + The number of channels in the output. + flip_sin_to_cos (`bool`, defaults to `True`): + Whether to flip the sin to cos in the time embedding. + time_embed_dim (`int`, defaults to `512`): + Output dimension of timestep embeddings. + text_embed_dim (`int`, defaults to `4096`): + Input dimension of text embeddings from the text encoder. + num_layers (`int`, defaults to `30`): + The number of layers of Transformer blocks to use. + dropout (`float`, defaults to `0.0`): + The dropout probability to use. + attention_bias (`bool`, defaults to `True`): + Whether or not to use bias in the attention projection layers. + sample_width (`int`, defaults to `90`): + The width of the input latents. + sample_height (`int`, defaults to `60`): + The height of the input latents. + sample_frames (`int`, defaults to `49`): + The number of frames in the input latents. Note that this parameter was incorrectly initialized to 49 + instead of 13 because CogVideoX processed 13 latent frames at once in its default and recommended settings, + but cannot be changed to the correct value to ensure backwards compatibility. To create a transformer with + K latent frames, the correct value to pass here would be: ((K - 1) * temporal_compression_ratio + 1). + patch_size (`int`, defaults to `2`): + The size of the patches to use in the patch embedding layer. + temporal_compression_ratio (`int`, defaults to `4`): + The compression ratio across the temporal dimension. See documentation for `sample_frames`. + max_text_seq_length (`int`, defaults to `226`): + The maximum sequence length of the input text embeddings. + activation_fn (`str`, defaults to `"gelu-approximate"`): + Activation function to use in feed-forward. + timestep_activation_fn (`str`, defaults to `"silu"`): + Activation function to use when generating the timestep embeddings. + norm_elementwise_affine (`bool`, defaults to `True`): + Whether or not to use elementwise affine in normalization layers. + norm_eps (`float`, defaults to `1e-5`): + The epsilon value to use in normalization layers. + spatial_interpolation_scale (`float`, defaults to `1.875`): + Scaling factor to apply in 3D positional embeddings across spatial dimensions. + temporal_interpolation_scale (`float`, defaults to `1.0`): + Scaling factor to apply in 3D positional embeddings across temporal dimensions. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + num_attention_heads: int = 30, + attention_head_dim: int = 64, + in_channels: int = 16, + out_channels: Optional[int] = 16, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + time_embed_dim: int = 512, + text_embed_dim: int = 4096, + num_layers: int = 30, + dropout: float = 0.0, + attention_bias: bool = True, + sample_width: int = 90, + sample_height: int = 60, + sample_frames: int = 49, + patch_size: int = 2, + patch_size_t: Optional[int] = None, + temporal_compression_ratio: int = 4, + max_text_seq_length: int = 226, + activation_fn: str = "gelu-approximate", + timestep_activation_fn: str = "silu", + norm_elementwise_affine: bool = True, + norm_eps: float = 1e-5, + spatial_interpolation_scale: float = 1.875, + temporal_interpolation_scale: float = 1.0, + use_rotary_positional_embeddings: bool = False, + use_learned_positional_embeddings: bool = False, + patch_bias: bool = True, + add_noise_in_inpaint_model: bool = False, + ): + super().__init__() + inner_dim = num_attention_heads * attention_head_dim + self.patch_size_t = patch_size_t + if not use_rotary_positional_embeddings and use_learned_positional_embeddings: + raise ValueError( + "There are no CogVideoX checkpoints available with disable rotary embeddings and learned positional " + "embeddings. If you're using a custom model and/or believe this should be supported, please open an " + "issue at https://github.com/huggingface/diffusers/issues." + ) + + # 1. Patch embedding + self.patch_embed = CogVideoXPatchEmbed( + patch_size=patch_size, + patch_size_t=patch_size_t, + in_channels=in_channels, + embed_dim=inner_dim, + text_embed_dim=text_embed_dim, + bias=patch_bias, + sample_width=sample_width, + sample_height=sample_height, + sample_frames=sample_frames, + temporal_compression_ratio=temporal_compression_ratio, + max_text_seq_length=max_text_seq_length, + spatial_interpolation_scale=spatial_interpolation_scale, + temporal_interpolation_scale=temporal_interpolation_scale, + use_positional_embeddings=not use_rotary_positional_embeddings, + use_learned_positional_embeddings=use_learned_positional_embeddings, + ) + self.embedding_dropout = nn.Dropout(dropout) + + # 2. Time embeddings + self.time_proj = Timesteps(inner_dim, flip_sin_to_cos, freq_shift) + self.time_embedding = TimestepEmbedding(inner_dim, time_embed_dim, timestep_activation_fn) + + # 3. Define spatio-temporal transformers blocks + self.transformer_blocks = nn.ModuleList( + [ + CogVideoXBlock( + dim=inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + time_embed_dim=time_embed_dim, + dropout=dropout, + activation_fn=activation_fn, + attention_bias=attention_bias, + norm_elementwise_affine=norm_elementwise_affine, + norm_eps=norm_eps, + ) + for _ in range(num_layers) + ] + ) + self.norm_final = nn.LayerNorm(inner_dim, norm_eps, norm_elementwise_affine) + + # 4. Output blocks + self.norm_out = AdaLayerNorm( + embedding_dim=time_embed_dim, + output_dim=2 * inner_dim, + norm_elementwise_affine=norm_elementwise_affine, + norm_eps=norm_eps, + chunk_dim=1, + ) + + if patch_size_t is None: + # For CogVideox 1.0 + output_dim = patch_size * patch_size * out_channels + else: + # For CogVideoX 1.5 + output_dim = patch_size * patch_size * patch_size_t * out_channels + + self.proj_out = nn.Linear(inner_dim, output_dim) + + self.gradient_checkpointing = False + self.sp_world_size = 1 + self.sp_world_rank = 0 + + def _set_gradient_checkpointing(self, *args, **kwargs): + if "value" in kwargs: + self.gradient_checkpointing = kwargs["value"] + elif "enable" in kwargs: + self.gradient_checkpointing = kwargs["enable"] + else: + raise ValueError("Invalid set gradient checkpointing") + + def enable_multi_gpus_inference(self,): + self.sp_world_size = get_sequence_parallel_world_size() + self.sp_world_rank = get_sequence_parallel_rank() + self.set_attn_processor(CogVideoXMultiGPUsAttnProcessor2_0()) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.fuse_qkv_projections with FusedAttnProcessor2_0->FusedCogVideoXAttnProcessor2_0 + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, key, value) + are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + self.set_attn_processor(FusedCogVideoXAttnProcessor2_0()) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.unfuse_qkv_projections + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + + + This API is 🧪 experimental. + + + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + timestep: Union[int, float, torch.LongTensor], + timestep_cond: Optional[torch.Tensor] = None, + inpaint_latents: Optional[torch.Tensor] = None, + control_latents: Optional[torch.Tensor] = None, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + return_dict: bool = True, + ): + batch_size, num_frames, channels, height, width = hidden_states.shape + if num_frames == 1 and self.patch_size_t is not None: + hidden_states = torch.cat([hidden_states, torch.zeros_like(hidden_states)], dim=1) + if inpaint_latents is not None: + inpaint_latents = torch.concat([inpaint_latents, torch.zeros_like(inpaint_latents)], dim=1) + if control_latents is not None: + control_latents = torch.concat([control_latents, torch.zeros_like(control_latents)], dim=1) + local_num_frames = num_frames + 1 + else: + local_num_frames = num_frames + + # 1. Time embedding + timesteps = timestep + t_emb = self.time_proj(timesteps) + + # timesteps does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=hidden_states.dtype) + emb = self.time_embedding(t_emb, timestep_cond) + + # 2. Patch embedding + if inpaint_latents is not None: + hidden_states = torch.concat([hidden_states, inpaint_latents], 2) + if control_latents is not None: + hidden_states = torch.concat([hidden_states, control_latents], 2) + hidden_states = self.patch_embed(encoder_hidden_states, hidden_states) + hidden_states = self.embedding_dropout(hidden_states) + + text_seq_length = encoder_hidden_states.shape[1] + encoder_hidden_states = hidden_states[:, :text_seq_length] + hidden_states = hidden_states[:, text_seq_length:] + + # Context Parallel + if self.sp_world_size > 1: + hidden_states = torch.chunk(hidden_states, self.sp_world_size, dim=1)[self.sp_world_rank] + if image_rotary_emb is not None: + image_rotary_emb = ( + torch.chunk(image_rotary_emb[0], self.sp_world_size, dim=0)[self.sp_world_rank], + torch.chunk(image_rotary_emb[1], self.sp_world_size, dim=0)[self.sp_world_rank] + ) + + # 3. Transformer blocks + for i, block in enumerate(self.transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states, encoder_hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + emb, + image_rotary_emb, + **ckpt_kwargs, + ) + else: + hidden_states, encoder_hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=emb, + image_rotary_emb=image_rotary_emb, + ) + + if not self.config.use_rotary_positional_embeddings: + # CogVideoX-2B + hidden_states = self.norm_final(hidden_states) + else: + # CogVideoX-5B + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + hidden_states = self.norm_final(hidden_states) + hidden_states = hidden_states[:, text_seq_length:] + + # 4. Final block + hidden_states = self.norm_out(hidden_states, temb=emb) + hidden_states = self.proj_out(hidden_states) + + if self.sp_world_size > 1: + hidden_states = get_sp_group().all_gather(hidden_states, dim=1) + + # 5. Unpatchify + p = self.config.patch_size + p_t = self.config.patch_size_t + + if p_t is None: + output = hidden_states.reshape(batch_size, local_num_frames, height // p, width // p, -1, p, p) + output = output.permute(0, 1, 4, 2, 5, 3, 6).flatten(5, 6).flatten(3, 4) + else: + output = hidden_states.reshape( + batch_size, (local_num_frames + p_t - 1) // p_t, height // p, width // p, -1, p_t, p, p + ) + output = output.permute(0, 1, 5, 4, 2, 6, 3, 7).flatten(6, 7).flatten(4, 5).flatten(1, 2) + + if num_frames == 1: + output = output[:, :num_frames, :] + + if not return_dict: + return (output,) + return Transformer2DModelOutput(sample=output) + + @classmethod + def from_pretrained( + cls, pretrained_model_path, subfolder=None, transformer_additional_kwargs={}, + low_cpu_mem_usage=False, torch_dtype=torch.bfloat16 + ): + if subfolder is not None: + pretrained_model_path = os.path.join(pretrained_model_path, subfolder) + print(f"loaded 3D transformer's pretrained weights from {pretrained_model_path} ...") + + config_file = os.path.join(pretrained_model_path, 'config.json') + if not os.path.isfile(config_file): + raise RuntimeError(f"{config_file} does not exist") + with open(config_file, "r") as f: + config = json.load(f) + + from diffusers.utils import WEIGHTS_NAME + model_file = os.path.join(pretrained_model_path, WEIGHTS_NAME) + model_file_safetensors = model_file.replace(".bin", ".safetensors") + + if "dict_mapping" in transformer_additional_kwargs.keys(): + for key in transformer_additional_kwargs["dict_mapping"]: + transformer_additional_kwargs[transformer_additional_kwargs["dict_mapping"][key]] = config[key] + + if low_cpu_mem_usage: + try: + import re + + from diffusers import __version__ as diffusers_version + if diffusers_version >= "0.33.0": + from diffusers.models.model_loading_utils import \ + load_model_dict_into_meta + else: + from diffusers.models.modeling_utils import \ + load_model_dict_into_meta + from diffusers.utils import is_accelerate_available + if is_accelerate_available(): + import accelerate + + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + model = cls.from_config(config, **transformer_additional_kwargs) + + param_device = "cpu" + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + model._convert_deprecated_attention_blocks(state_dict) + + if diffusers_version >= "0.33.0": + # Diffusers has refactored `load_model_dict_into_meta` since version 0.33.0 in this commit: + # https://github.com/huggingface/diffusers/commit/f5929e03060d56063ff34b25a8308833bec7c785. + load_model_dict_into_meta( + model, + state_dict, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + else: + # move the params from meta device to cpu + missing_keys = set(model.state_dict().keys()) - set(state_dict.keys()) + if len(missing_keys) > 0: + raise ValueError( + f"Cannot load {cls} from {pretrained_model_path} because the following keys are" + f" missing: \n {', '.join(missing_keys)}. \n Please make sure to pass" + " `low_cpu_mem_usage=False` and `device_map=None` if you want to randomly initialize" + " those weights or else make sure your checkpoint file is correct." + ) + + unexpected_keys = load_model_dict_into_meta( + model, + state_dict, + device=param_device, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + + if cls._keys_to_ignore_on_load_unexpected is not None: + for pat in cls._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + print( + f"Some weights of the model checkpoint were not used when initializing {cls.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + + return model + except Exception as e: + print( + f"The low_cpu_mem_usage mode is not work because {e}. Use low_cpu_mem_usage=False instead." + ) + + model = cls.from_config(config, **transformer_additional_kwargs) + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + if model.state_dict()['patch_embed.proj.weight'].size() != state_dict['patch_embed.proj.weight'].size(): + new_shape = model.state_dict()['patch_embed.proj.weight'].size() + if len(new_shape) == 5: + state_dict['patch_embed.proj.weight'] = state_dict['patch_embed.proj.weight'].unsqueeze(2).expand(new_shape).clone() + state_dict['patch_embed.proj.weight'][:, :, :-1] = 0 + elif len(new_shape) == 2: + if model.state_dict()['patch_embed.proj.weight'].size()[1] > state_dict['patch_embed.proj.weight'].size()[1]: + model.state_dict()['patch_embed.proj.weight'][:, :state_dict['patch_embed.proj.weight'].size()[1]] = state_dict['patch_embed.proj.weight'] + model.state_dict()['patch_embed.proj.weight'][:, state_dict['patch_embed.proj.weight'].size()[1]:] = 0 + state_dict['patch_embed.proj.weight'] = model.state_dict()['patch_embed.proj.weight'] + else: + model.state_dict()['patch_embed.proj.weight'][:, :] = state_dict['patch_embed.proj.weight'][:, :model.state_dict()['patch_embed.proj.weight'].size()[1]] + state_dict['patch_embed.proj.weight'] = model.state_dict()['patch_embed.proj.weight'] + else: + if model.state_dict()['patch_embed.proj.weight'].size()[1] > state_dict['patch_embed.proj.weight'].size()[1]: + model.state_dict()['patch_embed.proj.weight'][:, :state_dict['patch_embed.proj.weight'].size()[1], :, :] = state_dict['patch_embed.proj.weight'] + model.state_dict()['patch_embed.proj.weight'][:, state_dict['patch_embed.proj.weight'].size()[1]:, :, :] = 0 + state_dict['patch_embed.proj.weight'] = model.state_dict()['patch_embed.proj.weight'] + else: + model.state_dict()['patch_embed.proj.weight'][:, :, :, :] = state_dict['patch_embed.proj.weight'][:, :model.state_dict()['patch_embed.proj.weight'].size()[1], :, :] + state_dict['patch_embed.proj.weight'] = model.state_dict()['patch_embed.proj.weight'] + + tmp_state_dict = {} + for key in state_dict: + if key in model.state_dict().keys() and model.state_dict()[key].size() == state_dict[key].size(): + tmp_state_dict[key] = state_dict[key] + else: + print(key, "Size don't match, skip") + + state_dict = tmp_state_dict + + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m) + + params = [p.numel() if "." in n else 0 for n, p in model.named_parameters()] + print(f"### All Parameters: {sum(params) / 1e6} M") + + params = [p.numel() if "attn1." in n else 0 for n, p in model.named_parameters()] + print(f"### attn1 Parameters: {sum(params) / 1e6} M") + + model = model.to(torch_dtype) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/cogvideox_vae.py b/VideoX-Fun/videox_fun/models/cogvideox_vae.py new file mode 100644 index 0000000000000000000000000000000000000000..56e0a3ea121e1d53d4fe6557c2dd4b0786634e0e --- /dev/null +++ b/VideoX-Fun/videox_fun/models/cogvideox_vae.py @@ -0,0 +1,1675 @@ +# Copyright 2024 The CogVideoX team, Tsinghua University & ZhipuAI and The HuggingFace Team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +import json +import os + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.utils import logging +from diffusers.utils.accelerate_utils import apply_forward_hook +from diffusers.models.activations import get_activation +from diffusers.models.downsampling import CogVideoXDownsample3D +from diffusers.models.modeling_outputs import AutoencoderKLOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.upsampling import CogVideoXUpsample3D +from diffusers.models.autoencoders.vae import DecoderOutput, DiagonalGaussianDistribution + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class CogVideoXSafeConv3d(nn.Conv3d): + r""" + A 3D convolution layer that splits the input tensor into smaller parts to avoid OOM in CogVideoX Model. + """ + + def forward(self, input: torch.Tensor) -> torch.Tensor: + memory_count = ( + (input.shape[0] * input.shape[1] * input.shape[2] * input.shape[3] * input.shape[4]) * 2 / 1024**3 + ) + + # Set to 2GB, suitable for CuDNN + if memory_count > 2: + kernel_size = self.kernel_size[0] + part_num = int(memory_count / 2) + 1 + input_chunks = torch.chunk(input, part_num, dim=2) + + if kernel_size > 1: + input_chunks = [input_chunks[0]] + [ + torch.cat((input_chunks[i - 1][:, :, -kernel_size + 1 :], input_chunks[i]), dim=2) + for i in range(1, len(input_chunks)) + ] + + output_chunks = [] + for input_chunk in input_chunks: + output_chunks.append(super().forward(input_chunk)) + output = torch.cat(output_chunks, dim=2) + return output + else: + return super().forward(input) + + +class CogVideoXCausalConv3d(nn.Module): + r"""A 3D causal convolution layer that pads the input tensor to ensure causality in CogVideoX Model. + + Args: + in_channels (`int`): Number of channels in the input tensor. + out_channels (`int`): Number of output channels produced by the convolution. + kernel_size (`int` or `Tuple[int, int, int]`): Kernel size of the convolutional kernel. + stride (`int`, defaults to `1`): Stride of the convolution. + dilation (`int`, defaults to `1`): Dilation rate of the convolution. + pad_mode (`str`, defaults to `"constant"`): Padding mode. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: Union[int, Tuple[int, int, int]], + stride: int = 1, + dilation: int = 1, + pad_mode: str = "constant", + ): + super().__init__() + + if isinstance(kernel_size, int): + kernel_size = (kernel_size,) * 3 + + time_kernel_size, height_kernel_size, width_kernel_size = kernel_size + + # TODO(aryan): configure calculation based on stride and dilation in the future. + # Since CogVideoX does not use it, it is currently tailored to "just work" with Mochi + time_pad = time_kernel_size - 1 + height_pad = (height_kernel_size - 1) // 2 + width_pad = (width_kernel_size - 1) // 2 + + self.pad_mode = pad_mode + self.height_pad = height_pad + self.width_pad = width_pad + self.time_pad = time_pad + self.time_causal_padding = (width_pad, width_pad, height_pad, height_pad, time_pad, 0) + + self.temporal_dim = 2 + self.time_kernel_size = time_kernel_size + + stride = stride if isinstance(stride, tuple) else (stride, 1, 1) + dilation = (dilation, 1, 1) + self.conv = CogVideoXSafeConv3d( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + dilation=dilation, + ) + + def fake_context_parallel_forward( + self, inputs: torch.Tensor, conv_cache: Optional[torch.Tensor] = None + ) -> torch.Tensor: + if self.pad_mode == "replicate": + inputs = F.pad(inputs, self.time_causal_padding, mode="replicate") + else: + kernel_size = self.time_kernel_size + if kernel_size > 1: + cached_inputs = [conv_cache] if conv_cache is not None else [inputs[:, :, :1]] * (kernel_size - 1) + inputs = torch.cat(cached_inputs + [inputs], dim=2) + return inputs + + def forward(self, inputs: torch.Tensor, conv_cache: Optional[torch.Tensor] = None) -> torch.Tensor: + inputs = self.fake_context_parallel_forward(inputs, conv_cache) + + if self.pad_mode == "replicate": + conv_cache = None + else: + padding_2d = (self.width_pad, self.width_pad, self.height_pad, self.height_pad) + conv_cache = inputs[:, :, -self.time_kernel_size + 1 :].clone() + inputs = F.pad(inputs, padding_2d, mode="constant", value=0) + + output = self.conv(inputs) + return output, conv_cache + + +class CogVideoXSpatialNorm3D(nn.Module): + r""" + Spatially conditioned normalization as defined in https://arxiv.org/abs/2209.09002. This implementation is specific + to 3D-video like data. + + CogVideoXSafeConv3d is used instead of nn.Conv3d to avoid OOM in CogVideoX Model. + + Args: + f_channels (`int`): + The number of channels for input to group normalization layer, and output of the spatial norm layer. + zq_channels (`int`): + The number of channels for the quantized vector as described in the paper. + groups (`int`): + Number of groups to separate the channels into for group normalization. + """ + + def __init__( + self, + f_channels: int, + zq_channels: int, + groups: int = 32, + ): + super().__init__() + self.norm_layer = nn.GroupNorm(num_channels=f_channels, num_groups=groups, eps=1e-6, affine=True) + self.conv_y = CogVideoXCausalConv3d(zq_channels, f_channels, kernel_size=1, stride=1) + self.conv_b = CogVideoXCausalConv3d(zq_channels, f_channels, kernel_size=1, stride=1) + + def forward( + self, f: torch.Tensor, zq: torch.Tensor, conv_cache: Optional[Dict[str, torch.Tensor]] = None + ) -> torch.Tensor: + new_conv_cache = {} + conv_cache = conv_cache or {} + + if f.shape[2] > 1 and f.shape[2] % 2 == 1: + f_first, f_rest = f[:, :, :1], f[:, :, 1:] + f_first_size, f_rest_size = f_first.shape[-3:], f_rest.shape[-3:] + z_first, z_rest = zq[:, :, :1], zq[:, :, 1:] + z_first = F.interpolate(z_first, size=f_first_size) + z_rest = F.interpolate(z_rest, size=f_rest_size) + zq = torch.cat([z_first, z_rest], dim=2) + else: + zq = F.interpolate(zq, size=f.shape[-3:]) + + conv_y, new_conv_cache["conv_y"] = self.conv_y(zq, conv_cache=conv_cache.get("conv_y")) + conv_b, new_conv_cache["conv_b"] = self.conv_b(zq, conv_cache=conv_cache.get("conv_b")) + + norm_f = self.norm_layer(f) + new_f = norm_f * conv_y + conv_b + return new_f, new_conv_cache + + +class CogVideoXUpsample3D(nn.Module): + r""" + A 3D Upsample layer using in CogVideoX by Tsinghua University & ZhipuAI # Todo: Wait for paper relase. + + Args: + in_channels (`int`): + Number of channels in the input image. + out_channels (`int`): + Number of channels produced by the convolution. + kernel_size (`int`, defaults to `3`): + Size of the convolving kernel. + stride (`int`, defaults to `1`): + Stride of the convolution. + padding (`int`, defaults to `1`): + Padding added to all four sides of the input. + compress_time (`bool`, defaults to `False`): + Whether or not to compress the time dimension. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: int = 3, + stride: int = 1, + padding: int = 1, + compress_time: bool = False, + ) -> None: + super().__init__() + + self.conv = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding) + self.compress_time = compress_time + + self.auto_split_process = True + self.first_frame_flag = False + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + if self.compress_time: + if self.auto_split_process: + if inputs.shape[2] > 1 and inputs.shape[2] % 2 == 1: + # split first frame + x_first, x_rest = inputs[:, :, 0], inputs[:, :, 1:] + + x_first = F.interpolate(x_first, scale_factor=2.0) + x_rest = F.interpolate(x_rest, scale_factor=2.0) + x_first = x_first[:, :, None, :, :] + inputs = torch.cat([x_first, x_rest], dim=2) + elif inputs.shape[2] > 1: + inputs = F.interpolate(inputs, scale_factor=2.0) + else: + inputs = inputs.squeeze(2) + inputs = F.interpolate(inputs, scale_factor=2.0) + inputs = inputs[:, :, None, :, :] + else: + if self.first_frame_flag: + inputs = inputs.squeeze(2) + inputs = F.interpolate(inputs, scale_factor=2.0) + inputs = inputs[:, :, None, :, :] + else: + inputs = F.interpolate(inputs, scale_factor=2.0) + else: + # only interpolate 2D + b, c, t, h, w = inputs.shape + inputs = inputs.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) + inputs = F.interpolate(inputs, scale_factor=2.0) + inputs = inputs.reshape(b, t, c, *inputs.shape[2:]).permute(0, 2, 1, 3, 4) + + b, c, t, h, w = inputs.shape + inputs = inputs.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) + inputs = self.conv(inputs) + inputs = inputs.reshape(b, t, *inputs.shape[1:]).permute(0, 2, 1, 3, 4) + + return inputs + + +class CogVideoXResnetBlock3D(nn.Module): + r""" + A 3D ResNet block used in the CogVideoX model. + + Args: + in_channels (`int`): + Number of input channels. + out_channels (`int`, *optional*): + Number of output channels. If None, defaults to `in_channels`. + dropout (`float`, defaults to `0.0`): + Dropout rate. + temb_channels (`int`, defaults to `512`): + Number of time embedding channels. + groups (`int`, defaults to `32`): + Number of groups to separate the channels into for group normalization. + eps (`float`, defaults to `1e-6`): + Epsilon value for normalization layers. + non_linearity (`str`, defaults to `"swish"`): + Activation function to use. + conv_shortcut (bool, defaults to `False`): + Whether or not to use a convolution shortcut. + spatial_norm_dim (`int`, *optional*): + The dimension to use for spatial norm if it is to be used instead of group norm. + pad_mode (str, defaults to `"first"`): + Padding mode. + """ + + def __init__( + self, + in_channels: int, + out_channels: Optional[int] = None, + dropout: float = 0.0, + temb_channels: int = 512, + groups: int = 32, + eps: float = 1e-6, + non_linearity: str = "swish", + conv_shortcut: bool = False, + spatial_norm_dim: Optional[int] = None, + pad_mode: str = "first", + ): + super().__init__() + + out_channels = out_channels or in_channels + + self.in_channels = in_channels + self.out_channels = out_channels + self.nonlinearity = get_activation(non_linearity) + self.use_conv_shortcut = conv_shortcut + self.spatial_norm_dim = spatial_norm_dim + + if spatial_norm_dim is None: + self.norm1 = nn.GroupNorm(num_channels=in_channels, num_groups=groups, eps=eps) + self.norm2 = nn.GroupNorm(num_channels=out_channels, num_groups=groups, eps=eps) + else: + self.norm1 = CogVideoXSpatialNorm3D( + f_channels=in_channels, + zq_channels=spatial_norm_dim, + groups=groups, + ) + self.norm2 = CogVideoXSpatialNorm3D( + f_channels=out_channels, + zq_channels=spatial_norm_dim, + groups=groups, + ) + + self.conv1 = CogVideoXCausalConv3d( + in_channels=in_channels, out_channels=out_channels, kernel_size=3, pad_mode=pad_mode + ) + + if temb_channels > 0: + self.temb_proj = nn.Linear(in_features=temb_channels, out_features=out_channels) + + self.dropout = nn.Dropout(dropout) + self.conv2 = CogVideoXCausalConv3d( + in_channels=out_channels, out_channels=out_channels, kernel_size=3, pad_mode=pad_mode + ) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + self.conv_shortcut = CogVideoXCausalConv3d( + in_channels=in_channels, out_channels=out_channels, kernel_size=3, pad_mode=pad_mode + ) + else: + self.conv_shortcut = CogVideoXSafeConv3d( + in_channels=in_channels, out_channels=out_channels, kernel_size=1, stride=1, padding=0 + ) + + def forward( + self, + inputs: torch.Tensor, + temb: Optional[torch.Tensor] = None, + zq: Optional[torch.Tensor] = None, + conv_cache: Optional[Dict[str, torch.Tensor]] = None, + ) -> torch.Tensor: + new_conv_cache = {} + conv_cache = conv_cache or {} + + hidden_states = inputs + + if zq is not None: + hidden_states, new_conv_cache["norm1"] = self.norm1(hidden_states, zq, conv_cache=conv_cache.get("norm1")) + else: + hidden_states = self.norm1(hidden_states) + + hidden_states = self.nonlinearity(hidden_states) + hidden_states, new_conv_cache["conv1"] = self.conv1(hidden_states, conv_cache=conv_cache.get("conv1")) + + if temb is not None: + hidden_states = hidden_states + self.temb_proj(self.nonlinearity(temb))[:, :, None, None, None] + + if zq is not None: + hidden_states, new_conv_cache["norm2"] = self.norm2(hidden_states, zq, conv_cache=conv_cache.get("norm2")) + else: + hidden_states = self.norm2(hidden_states) + + hidden_states = self.nonlinearity(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states, new_conv_cache["conv2"] = self.conv2(hidden_states, conv_cache=conv_cache.get("conv2")) + + if self.in_channels != self.out_channels: + if self.use_conv_shortcut: + inputs, new_conv_cache["conv_shortcut"] = self.conv_shortcut( + inputs, conv_cache=conv_cache.get("conv_shortcut") + ) + else: + inputs = self.conv_shortcut(inputs) + + hidden_states = hidden_states + inputs + return hidden_states, new_conv_cache + + +class CogVideoXDownBlock3D(nn.Module): + r""" + A downsampling block used in the CogVideoX model. + + Args: + in_channels (`int`): + Number of input channels. + out_channels (`int`, *optional*): + Number of output channels. If None, defaults to `in_channels`. + temb_channels (`int`, defaults to `512`): + Number of time embedding channels. + num_layers (`int`, defaults to `1`): + Number of resnet layers. + dropout (`float`, defaults to `0.0`): + Dropout rate. + resnet_eps (`float`, defaults to `1e-6`): + Epsilon value for normalization layers. + resnet_act_fn (`str`, defaults to `"swish"`): + Activation function to use. + resnet_groups (`int`, defaults to `32`): + Number of groups to separate the channels into for group normalization. + add_downsample (`bool`, defaults to `True`): + Whether or not to use a downsampling layer. If not used, output dimension would be same as input dimension. + compress_time (`bool`, defaults to `False`): + Whether or not to downsample across temporal dimension. + pad_mode (str, defaults to `"first"`): + Padding mode. + """ + + _supports_gradient_checkpointing = True + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + add_downsample: bool = True, + downsample_padding: int = 0, + compress_time: bool = False, + pad_mode: str = "first", + ): + super().__init__() + + resnets = [] + for i in range(num_layers): + in_channel = in_channels if i == 0 else out_channels + resnets.append( + CogVideoXResnetBlock3D( + in_channels=in_channel, + out_channels=out_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=resnet_groups, + eps=resnet_eps, + non_linearity=resnet_act_fn, + pad_mode=pad_mode, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.downsamplers = None + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + CogVideoXDownsample3D( + out_channels, out_channels, padding=downsample_padding, compress_time=compress_time + ) + ] + ) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.Tensor, + temb: Optional[torch.Tensor] = None, + zq: Optional[torch.Tensor] = None, + conv_cache: Optional[Dict[str, torch.Tensor]] = None, + ) -> torch.Tensor: + r"""Forward method of the `CogVideoXDownBlock3D` class.""" + + new_conv_cache = {} + conv_cache = conv_cache or {} + + for i, resnet in enumerate(self.resnets): + conv_cache_key = f"resnet_{i}" + + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def create_forward(*inputs): + return module(*inputs) + + return create_forward + + hidden_states, new_conv_cache[conv_cache_key] = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + zq, + conv_cache.get(conv_cache_key), + ) + else: + hidden_states, new_conv_cache[conv_cache_key] = resnet( + hidden_states, temb, zq, conv_cache=conv_cache.get(conv_cache_key) + ) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states, new_conv_cache + + +class CogVideoXMidBlock3D(nn.Module): + r""" + A middle block used in the CogVideoX model. + + Args: + in_channels (`int`): + Number of input channels. + temb_channels (`int`, defaults to `512`): + Number of time embedding channels. + dropout (`float`, defaults to `0.0`): + Dropout rate. + num_layers (`int`, defaults to `1`): + Number of resnet layers. + resnet_eps (`float`, defaults to `1e-6`): + Epsilon value for normalization layers. + resnet_act_fn (`str`, defaults to `"swish"`): + Activation function to use. + resnet_groups (`int`, defaults to `32`): + Number of groups to separate the channels into for group normalization. + spatial_norm_dim (`int`, *optional*): + The dimension to use for spatial norm if it is to be used instead of group norm. + pad_mode (str, defaults to `"first"`): + Padding mode. + """ + + _supports_gradient_checkpointing = True + + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + spatial_norm_dim: Optional[int] = None, + pad_mode: str = "first", + ): + super().__init__() + + resnets = [] + for _ in range(num_layers): + resnets.append( + CogVideoXResnetBlock3D( + in_channels=in_channels, + out_channels=in_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=resnet_groups, + eps=resnet_eps, + spatial_norm_dim=spatial_norm_dim, + non_linearity=resnet_act_fn, + pad_mode=pad_mode, + ) + ) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.Tensor, + temb: Optional[torch.Tensor] = None, + zq: Optional[torch.Tensor] = None, + conv_cache: Optional[Dict[str, torch.Tensor]] = None, + ) -> torch.Tensor: + r"""Forward method of the `CogVideoXMidBlock3D` class.""" + + new_conv_cache = {} + conv_cache = conv_cache or {} + + for i, resnet in enumerate(self.resnets): + conv_cache_key = f"resnet_{i}" + + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def create_forward(*inputs): + return module(*inputs) + + return create_forward + + hidden_states, new_conv_cache[conv_cache_key] = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, zq, conv_cache.get(conv_cache_key) + ) + else: + hidden_states, new_conv_cache[conv_cache_key] = resnet( + hidden_states, temb, zq, conv_cache=conv_cache.get(conv_cache_key) + ) + + return hidden_states, new_conv_cache + + +class CogVideoXUpBlock3D(nn.Module): + r""" + An upsampling block used in the CogVideoX model. + + Args: + in_channels (`int`): + Number of input channels. + out_channels (`int`, *optional*): + Number of output channels. If None, defaults to `in_channels`. + temb_channels (`int`, defaults to `512`): + Number of time embedding channels. + dropout (`float`, defaults to `0.0`): + Dropout rate. + num_layers (`int`, defaults to `1`): + Number of resnet layers. + resnet_eps (`float`, defaults to `1e-6`): + Epsilon value for normalization layers. + resnet_act_fn (`str`, defaults to `"swish"`): + Activation function to use. + resnet_groups (`int`, defaults to `32`): + Number of groups to separate the channels into for group normalization. + spatial_norm_dim (`int`, defaults to `16`): + The dimension to use for spatial norm if it is to be used instead of group norm. + add_upsample (`bool`, defaults to `True`): + Whether or not to use a upsampling layer. If not used, output dimension would be same as input dimension. + compress_time (`bool`, defaults to `False`): + Whether or not to downsample across temporal dimension. + pad_mode (str, defaults to `"first"`): + Padding mode. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + spatial_norm_dim: int = 16, + add_upsample: bool = True, + upsample_padding: int = 1, + compress_time: bool = False, + pad_mode: str = "first", + ): + super().__init__() + + resnets = [] + for i in range(num_layers): + in_channel = in_channels if i == 0 else out_channels + resnets.append( + CogVideoXResnetBlock3D( + in_channels=in_channel, + out_channels=out_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=resnet_groups, + eps=resnet_eps, + non_linearity=resnet_act_fn, + spatial_norm_dim=spatial_norm_dim, + pad_mode=pad_mode, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.upsamplers = None + + if add_upsample: + self.upsamplers = nn.ModuleList( + [ + CogVideoXUpsample3D( + out_channels, out_channels, padding=upsample_padding, compress_time=compress_time + ) + ] + ) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.Tensor, + temb: Optional[torch.Tensor] = None, + zq: Optional[torch.Tensor] = None, + conv_cache: Optional[Dict[str, torch.Tensor]] = None, + ) -> torch.Tensor: + r"""Forward method of the `CogVideoXUpBlock3D` class.""" + + new_conv_cache = {} + conv_cache = conv_cache or {} + + for i, resnet in enumerate(self.resnets): + conv_cache_key = f"resnet_{i}" + + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def create_forward(*inputs): + return module(*inputs) + + return create_forward + + hidden_states, new_conv_cache[conv_cache_key] = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + zq, + conv_cache.get(conv_cache_key), + ) + else: + hidden_states, new_conv_cache[conv_cache_key] = resnet( + hidden_states, temb, zq, conv_cache=conv_cache.get(conv_cache_key) + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states, new_conv_cache + + +class CogVideoXEncoder3D(nn.Module): + r""" + The `CogVideoXEncoder3D` layer of a variational autoencoder that encodes its input into a latent representation. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + down_block_types (`Tuple[str, ...]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + The types of down blocks to use. See `~diffusers.models.unet_2d_blocks.get_down_block` for available + options. + block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + The number of output channels for each block. + act_fn (`str`, *optional*, defaults to `"silu"`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for normalization. + """ + + _supports_gradient_checkpointing = True + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 16, + down_block_types: Tuple[str, ...] = ( + "CogVideoXDownBlock3D", + "CogVideoXDownBlock3D", + "CogVideoXDownBlock3D", + "CogVideoXDownBlock3D", + ), + block_out_channels: Tuple[int, ...] = (128, 256, 256, 512), + layers_per_block: int = 3, + act_fn: str = "silu", + norm_eps: float = 1e-6, + norm_num_groups: int = 32, + dropout: float = 0.0, + pad_mode: str = "first", + temporal_compression_ratio: float = 4, + ): + super().__init__() + + # log2 of temporal_compress_times + temporal_compress_level = int(np.log2(temporal_compression_ratio)) + + self.conv_in = CogVideoXCausalConv3d(in_channels, block_out_channels[0], kernel_size=3, pad_mode=pad_mode) + self.down_blocks = nn.ModuleList([]) + + # down blocks + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + compress_time = i < temporal_compress_level + + if down_block_type == "CogVideoXDownBlock3D": + down_block = CogVideoXDownBlock3D( + in_channels=input_channel, + out_channels=output_channel, + temb_channels=0, + dropout=dropout, + num_layers=layers_per_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + add_downsample=not is_final_block, + compress_time=compress_time, + ) + else: + raise ValueError("Invalid `down_block_type` encountered. Must be `CogVideoXDownBlock3D`") + + self.down_blocks.append(down_block) + + # mid block + self.mid_block = CogVideoXMidBlock3D( + in_channels=block_out_channels[-1], + temb_channels=0, + dropout=dropout, + num_layers=2, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + pad_mode=pad_mode, + ) + + self.norm_out = nn.GroupNorm(norm_num_groups, block_out_channels[-1], eps=1e-6) + self.conv_act = nn.SiLU() + self.conv_out = CogVideoXCausalConv3d( + block_out_channels[-1], 2 * out_channels, kernel_size=3, pad_mode=pad_mode + ) + + self.gradient_checkpointing = False + + def forward( + self, + sample: torch.Tensor, + temb: Optional[torch.Tensor] = None, + conv_cache: Optional[Dict[str, torch.Tensor]] = None, + ) -> torch.Tensor: + r"""The forward method of the `CogVideoXEncoder3D` class.""" + + new_conv_cache = {} + conv_cache = conv_cache or {} + + hidden_states, new_conv_cache["conv_in"] = self.conv_in(sample, conv_cache=conv_cache.get("conv_in")) + + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + # 1. Down + for i, down_block in enumerate(self.down_blocks): + conv_cache_key = f"down_block_{i}" + hidden_states, new_conv_cache[conv_cache_key] = torch.utils.checkpoint.checkpoint( + create_custom_forward(down_block), + hidden_states, + temb, + None, + conv_cache.get(conv_cache_key), + ) + + # 2. Mid + hidden_states, new_conv_cache["mid_block"] = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), + hidden_states, + temb, + None, + conv_cache.get("mid_block"), + ) + else: + # 1. Down + for i, down_block in enumerate(self.down_blocks): + conv_cache_key = f"down_block_{i}" + hidden_states, new_conv_cache[conv_cache_key] = down_block( + hidden_states, temb, None, conv_cache=conv_cache.get(conv_cache_key) + ) + + # 2. Mid + hidden_states, new_conv_cache["mid_block"] = self.mid_block( + hidden_states, temb, None, conv_cache=conv_cache.get("mid_block") + ) + + # 3. Post-process + hidden_states = self.norm_out(hidden_states) + hidden_states = self.conv_act(hidden_states) + + hidden_states, new_conv_cache["conv_out"] = self.conv_out(hidden_states, conv_cache=conv_cache.get("conv_out")) + + return hidden_states, new_conv_cache + + +class CogVideoXDecoder3D(nn.Module): + r""" + The `CogVideoXDecoder3D` layer of a variational autoencoder that decodes its latent representation into an output + sample. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + up_block_types (`Tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + The types of up blocks to use. See `~diffusers.models.unet_2d_blocks.get_up_block` for available options. + block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + The number of output channels for each block. + act_fn (`str`, *optional*, defaults to `"silu"`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for normalization. + """ + + _supports_gradient_checkpointing = True + + def __init__( + self, + in_channels: int = 16, + out_channels: int = 3, + up_block_types: Tuple[str, ...] = ( + "CogVideoXUpBlock3D", + "CogVideoXUpBlock3D", + "CogVideoXUpBlock3D", + "CogVideoXUpBlock3D", + ), + block_out_channels: Tuple[int, ...] = (128, 256, 256, 512), + layers_per_block: int = 3, + act_fn: str = "silu", + norm_eps: float = 1e-6, + norm_num_groups: int = 32, + dropout: float = 0.0, + pad_mode: str = "first", + temporal_compression_ratio: float = 4, + ): + super().__init__() + + reversed_block_out_channels = list(reversed(block_out_channels)) + + self.conv_in = CogVideoXCausalConv3d( + in_channels, reversed_block_out_channels[0], kernel_size=3, pad_mode=pad_mode + ) + + # mid block + self.mid_block = CogVideoXMidBlock3D( + in_channels=reversed_block_out_channels[0], + temb_channels=0, + num_layers=2, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + spatial_norm_dim=in_channels, + pad_mode=pad_mode, + ) + + # up blocks + self.up_blocks = nn.ModuleList([]) + + output_channel = reversed_block_out_channels[0] + temporal_compress_level = int(np.log2(temporal_compression_ratio)) + + for i, up_block_type in enumerate(up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + compress_time = i < temporal_compress_level + + if up_block_type == "CogVideoXUpBlock3D": + up_block = CogVideoXUpBlock3D( + in_channels=prev_output_channel, + out_channels=output_channel, + temb_channels=0, + dropout=dropout, + num_layers=layers_per_block + 1, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + spatial_norm_dim=in_channels, + add_upsample=not is_final_block, + compress_time=compress_time, + pad_mode=pad_mode, + ) + prev_output_channel = output_channel + else: + raise ValueError("Invalid `up_block_type` encountered. Must be `CogVideoXUpBlock3D`") + + self.up_blocks.append(up_block) + + self.norm_out = CogVideoXSpatialNorm3D(reversed_block_out_channels[-1], in_channels, groups=norm_num_groups) + self.conv_act = nn.SiLU() + self.conv_out = CogVideoXCausalConv3d( + reversed_block_out_channels[-1], out_channels, kernel_size=3, pad_mode=pad_mode + ) + + self.gradient_checkpointing = False + + def forward( + self, + sample: torch.Tensor, + temb: Optional[torch.Tensor] = None, + conv_cache: Optional[Dict[str, torch.Tensor]] = None, + ) -> torch.Tensor: + r"""The forward method of the `CogVideoXDecoder3D` class.""" + + new_conv_cache = {} + conv_cache = conv_cache or {} + + hidden_states, new_conv_cache["conv_in"] = self.conv_in(sample, conv_cache=conv_cache.get("conv_in")) + + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + # 1. Mid + hidden_states, new_conv_cache["mid_block"] = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), + hidden_states, + temb, + sample, + conv_cache.get("mid_block"), + ) + + # 2. Up + for i, up_block in enumerate(self.up_blocks): + conv_cache_key = f"up_block_{i}" + hidden_states, new_conv_cache[conv_cache_key] = torch.utils.checkpoint.checkpoint( + create_custom_forward(up_block), + hidden_states, + temb, + sample, + conv_cache.get(conv_cache_key), + ) + else: + # 1. Mid + hidden_states, new_conv_cache["mid_block"] = self.mid_block( + hidden_states, temb, sample, conv_cache=conv_cache.get("mid_block") + ) + + # 2. Up + for i, up_block in enumerate(self.up_blocks): + conv_cache_key = f"up_block_{i}" + hidden_states, new_conv_cache[conv_cache_key] = up_block( + hidden_states, temb, sample, conv_cache=conv_cache.get(conv_cache_key) + ) + + # 3. Post-process + hidden_states, new_conv_cache["norm_out"] = self.norm_out( + hidden_states, sample, conv_cache=conv_cache.get("norm_out") + ) + hidden_states = self.conv_act(hidden_states) + hidden_states, new_conv_cache["conv_out"] = self.conv_out(hidden_states, conv_cache=conv_cache.get("conv_out")) + + return hidden_states, new_conv_cache + + +class AutoencoderKLCogVideoX(ModelMixin, ConfigMixin, FromOriginalModelMixin): + r""" + A VAE model with KL loss for encoding images into latents and decoding latent representations into images. Used in + [CogVideoX](https://github.com/THUDM/CogVideo). + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + in_channels (int, *optional*, defaults to 3): Number of channels in the input image. + out_channels (int, *optional*, defaults to 3): Number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + Tuple of downsample block types. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + Tuple of upsample block types. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): + Tuple of block output channels. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + sample_size (`int`, *optional*, defaults to `32`): Sample input size. + scaling_factor (`float`, *optional*, defaults to `1.15258426`): + The component-wise standard deviation of the trained latent space computed using the first batch of the + training set. This is used to scale the latent space to have unit variance when training the diffusion + model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the + diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1 + / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image + Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752) paper. + force_upcast (`bool`, *optional*, default to `True`): + If enabled it will force the VAE to run in float32 for high image resolution pipelines, such as SD-XL. VAE + can be fine-tuned / trained to a lower range without loosing too much precision in which case + `force_upcast` can be set to `False` - see: https://huggingface.co/madebyollin/sdxl-vae-fp16-fix + """ + + _supports_gradient_checkpointing = True + _no_split_modules = ["CogVideoXResnetBlock3D"] + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str] = ( + "CogVideoXDownBlock3D", + "CogVideoXDownBlock3D", + "CogVideoXDownBlock3D", + "CogVideoXDownBlock3D", + ), + up_block_types: Tuple[str] = ( + "CogVideoXUpBlock3D", + "CogVideoXUpBlock3D", + "CogVideoXUpBlock3D", + "CogVideoXUpBlock3D", + ), + block_out_channels: Tuple[int] = (128, 256, 256, 512), + latent_channels: int = 16, + layers_per_block: int = 3, + act_fn: str = "silu", + norm_eps: float = 1e-6, + norm_num_groups: int = 32, + temporal_compression_ratio: float = 4, + sample_height: int = 480, + sample_width: int = 720, + scaling_factor: float = 1.15258426, + shift_factor: Optional[float] = None, + latents_mean: Optional[Tuple[float]] = None, + latents_std: Optional[Tuple[float]] = None, + force_upcast: float = True, + use_quant_conv: bool = False, + use_post_quant_conv: bool = False, + invert_scale_latents: bool = False, + ): + super().__init__() + + self.encoder = CogVideoXEncoder3D( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + norm_eps=norm_eps, + norm_num_groups=norm_num_groups, + temporal_compression_ratio=temporal_compression_ratio, + ) + self.decoder = CogVideoXDecoder3D( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + norm_eps=norm_eps, + norm_num_groups=norm_num_groups, + temporal_compression_ratio=temporal_compression_ratio, + ) + self.quant_conv = CogVideoXSafeConv3d(2 * out_channels, 2 * out_channels, 1) if use_quant_conv else None + self.post_quant_conv = CogVideoXSafeConv3d(out_channels, out_channels, 1) if use_post_quant_conv else None + + self.use_slicing = False + self.use_tiling = False + self.auto_split_process = False + + # Can be increased to decode more latent frames at once, but comes at a reasonable memory cost and it is not + # recommended because the temporal parts of the VAE, here, are tricky to understand. + # If you decode X latent frames together, the number of output frames is: + # (X + (2 conv cache) + (2 time upscale_1) + (4 time upscale_2) - (2 causal conv downscale)) => X + 6 frames + # + # Example with num_latent_frames_batch_size = 2: + # - 12 latent frames: (0, 1), (2, 3), (4, 5), (6, 7), (8, 9), (10, 11) are processed together + # => (12 // 2 frame slices) * ((2 num_latent_frames_batch_size) + (2 conv cache) + (2 time upscale_1) + (4 time upscale_2) - (2 causal conv downscale)) + # => 6 * 8 = 48 frames + # - 13 latent frames: (0, 1, 2) (special case), (3, 4), (5, 6), (7, 8), (9, 10), (11, 12) are processed together + # => (1 frame slice) * ((3 num_latent_frames_batch_size) + (2 conv cache) + (2 time upscale_1) + (4 time upscale_2) - (2 causal conv downscale)) + + # ((13 - 3) // 2) * ((2 num_latent_frames_batch_size) + (2 conv cache) + (2 time upscale_1) + (4 time upscale_2) - (2 causal conv downscale)) + # => 1 * 9 + 5 * 8 = 49 frames + # It has been implemented this way so as to not have "magic values" in the code base that would be hard to explain. Note that + # setting it to anything other than 2 would give poor results because the VAE hasn't been trained to be adaptive with different + # number of temporal frames. + self.num_latent_frames_batch_size = 2 + self.num_sample_frames_batch_size = 8 + + # We make the minimum height and width of sample for tiling half that of the generally supported + self.tile_sample_min_height = sample_height // 2 + self.tile_sample_min_width = sample_width // 2 + self.tile_latent_min_height = int( + self.tile_sample_min_height / (2 ** (len(self.config.block_out_channels) - 1)) + ) + self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** (len(self.config.block_out_channels) - 1))) + + # These are experimental overlap factors that were chosen based on experimentation and seem to work best for + # 720x480 (WxH) resolution. The above resolution is the strongly recommended generation resolution in CogVideoX + # and so the tiling implementation has only been tested on those specific resolutions. + self.tile_overlap_factor_height = 1 / 6 + self.tile_overlap_factor_width = 1 / 5 + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (CogVideoXEncoder3D, CogVideoXDecoder3D)): + module.gradient_checkpointing = value + + def enable_tiling( + self, + tile_sample_min_height: Optional[int] = None, + tile_sample_min_width: Optional[int] = None, + tile_overlap_factor_height: Optional[float] = None, + tile_overlap_factor_width: Optional[float] = None, + ) -> None: + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + + Args: + tile_sample_min_height (`int`, *optional*): + The minimum height required for a sample to be separated into tiles across the height dimension. + tile_sample_min_width (`int`, *optional*): + The minimum width required for a sample to be separated into tiles across the width dimension. + tile_overlap_factor_height (`int`, *optional*): + The minimum amount of overlap between two consecutive vertical tiles. This is to ensure that there are + no tiling artifacts produced across the height dimension. Must be between 0 and 1. Setting a higher + value might cause more tiles to be processed leading to slow down of the decoding process. + tile_overlap_factor_width (`int`, *optional*): + The minimum amount of overlap between two consecutive horizontal tiles. This is to ensure that there + are no tiling artifacts produced across the width dimension. Must be between 0 and 1. Setting a higher + value might cause more tiles to be processed leading to slow down of the decoding process. + """ + self.use_tiling = True + self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height + self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width + self.tile_latent_min_height = int( + self.tile_sample_min_height / (2 ** (len(self.config.block_out_channels) - 1)) + ) + self.tile_latent_min_width = int(self.tile_sample_min_width / (2 ** (len(self.config.block_out_channels) - 1))) + self.tile_overlap_factor_height = tile_overlap_factor_height or self.tile_overlap_factor_height + self.tile_overlap_factor_width = tile_overlap_factor_width or self.tile_overlap_factor_width + + def disable_tiling(self) -> None: + r""" + Disable tiled VAE decoding. If `enable_tiling` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_tiling = False + + def enable_slicing(self) -> None: + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.use_slicing = True + + def disable_slicing(self) -> None: + r""" + Disable sliced VAE decoding. If `enable_slicing` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_slicing = False + + def _set_first_frame(self): + for name, module in self.named_modules(): + if isinstance(module, CogVideoXUpsample3D): + module.auto_split_process = False + module.first_frame_flag = True + + def _set_rest_frame(self): + for name, module in self.named_modules(): + if isinstance(module, CogVideoXUpsample3D): + module.auto_split_process = False + module.first_frame_flag = False + + def enable_auto_split_process(self) -> None: + self.auto_split_process = True + for name, module in self.named_modules(): + if isinstance(module, CogVideoXUpsample3D): + module.auto_split_process = True + + def disable_auto_split_process(self) -> None: + self.auto_split_process = False + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, num_frames, height, width = x.shape + + if self.use_tiling and (width > self.tile_sample_min_width or height > self.tile_sample_min_height): + return self.tiled_encode(x) + + frame_batch_size = self.num_sample_frames_batch_size + # Note: We expect the number of frames to be either `1` or `frame_batch_size * k` or `frame_batch_size * k + 1` for some k. + # As the extra single frame is handled inside the loop, it is not required to round up here. + num_batches = max(num_frames // frame_batch_size, 1) + conv_cache = None + enc = [] + + for i in range(num_batches): + remaining_frames = num_frames % frame_batch_size + start_frame = frame_batch_size * i + (0 if i == 0 else remaining_frames) + end_frame = frame_batch_size * (i + 1) + remaining_frames + x_intermediate = x[:, :, start_frame:end_frame] + x_intermediate, conv_cache = self.encoder(x_intermediate, conv_cache=conv_cache) + if self.quant_conv is not None: + x_intermediate = self.quant_conv(x_intermediate) + enc.append(x_intermediate) + + enc = torch.cat(enc, dim=2) + return enc + + @apply_forward_hook + def encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + """ + Encode a batch of images into latents. + + Args: + x (`torch.Tensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + The latent representations of the encoded videos. If `return_dict` is True, a + [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. + """ + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [self._encode(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = self._encode(x) + + posterior = DiagonalGaussianDistribution(h) + + if not return_dict: + return (posterior,) + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + batch_size, num_channels, num_frames, height, width = z.shape + + if self.use_tiling and (width > self.tile_latent_min_width or height > self.tile_latent_min_height): + return self.tiled_decode(z, return_dict=return_dict) + + if self.auto_split_process: + frame_batch_size = self.num_latent_frames_batch_size + num_batches = max(num_frames // frame_batch_size, 1) + conv_cache = None + dec = [] + + for i in range(num_batches): + remaining_frames = num_frames % frame_batch_size + start_frame = frame_batch_size * i + (0 if i == 0 else remaining_frames) + end_frame = frame_batch_size * (i + 1) + remaining_frames + z_intermediate = z[:, :, start_frame:end_frame] + if self.post_quant_conv is not None: + z_intermediate = self.post_quant_conv(z_intermediate) + z_intermediate, conv_cache = self.decoder(z_intermediate, conv_cache=conv_cache) + dec.append(z_intermediate) + else: + conv_cache = None + start_frame = 0 + end_frame = 1 + dec = [] + + self._set_first_frame() + z_intermediate = z[:, :, start_frame:end_frame] + if self.post_quant_conv is not None: + z_intermediate = self.post_quant_conv(z_intermediate) + z_intermediate, conv_cache = self.decoder(z_intermediate, conv_cache=conv_cache) + dec.append(z_intermediate) + + self._set_rest_frame() + start_frame = end_frame + end_frame += self.num_latent_frames_batch_size + + while start_frame < num_frames: + z_intermediate = z[:, :, start_frame:end_frame] + if self.post_quant_conv is not None: + z_intermediate = self.post_quant_conv(z_intermediate) + z_intermediate, conv_cache = self.decoder(z_intermediate, conv_cache=conv_cache) + dec.append(z_intermediate) + start_frame = end_frame + end_frame += self.num_latent_frames_batch_size + + dec = torch.cat(dec, dim=2) + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + @apply_forward_hook + def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + """ + Decode a batch of images. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + if self.use_slicing and z.shape[0] > 1: + decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)] + decoded = torch.cat(decoded_slices) + else: + decoded = self._decode(z).sample + + if not return_dict: + return (decoded,) + return DecoderOutput(sample=decoded) + + def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[3], b.shape[3], blend_extent) + for y in range(blend_extent): + b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * ( + y / blend_extent + ) + return b + + def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[4], b.shape[4], blend_extent) + for x in range(blend_extent): + b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * ( + x / blend_extent + ) + return b + + def tiled_encode(self, x: torch.Tensor) -> torch.Tensor: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled encoding is + different from non-tiled encoding because each tile uses a different encoder. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. You may still see tile-sized changes in the + output, but they should be much less noticeable. + + Args: + x (`torch.Tensor`): Input batch of videos. + + Returns: + `torch.Tensor`: + The latent representation of the encoded videos. + """ + # For a rough memory estimate, take a look at the `tiled_decode` method. + batch_size, num_channels, num_frames, height, width = x.shape + + overlap_height = int(self.tile_sample_min_height * (1 - self.tile_overlap_factor_height)) + overlap_width = int(self.tile_sample_min_width * (1 - self.tile_overlap_factor_width)) + blend_extent_height = int(self.tile_latent_min_height * self.tile_overlap_factor_height) + blend_extent_width = int(self.tile_latent_min_width * self.tile_overlap_factor_width) + row_limit_height = self.tile_latent_min_height - blend_extent_height + row_limit_width = self.tile_latent_min_width - blend_extent_width + frame_batch_size = self.num_sample_frames_batch_size + + # Split x into overlapping tiles and encode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, height, overlap_height): + row = [] + for j in range(0, width, overlap_width): + # Note: We expect the number of frames to be either `1` or `frame_batch_size * k` or `frame_batch_size * k + 1` for some k. + # As the extra single frame is handled inside the loop, it is not required to round up here. + num_batches = max(num_frames // frame_batch_size, 1) + conv_cache = None + time = [] + + for k in range(num_batches): + remaining_frames = num_frames % frame_batch_size + start_frame = frame_batch_size * k + (0 if k == 0 else remaining_frames) + end_frame = frame_batch_size * (k + 1) + remaining_frames + tile = x[ + :, + :, + start_frame:end_frame, + i : i + self.tile_sample_min_height, + j : j + self.tile_sample_min_width, + ] + tile, conv_cache = self.encoder(tile, conv_cache=conv_cache) + if self.quant_conv is not None: + tile = self.quant_conv(tile) + time.append(tile) + + row.append(torch.cat(time, dim=2)) + rows.append(row) + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent_width) + result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width]) + result_rows.append(torch.cat(result_row, dim=4)) + + enc = torch.cat(result_rows, dim=3) + return enc + + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + r""" + Decode a batch of images using a tiled decoder. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + # Rough memory assessment: + # - In CogVideoX-2B, there are a total of 24 CausalConv3d layers. + # - The biggest intermediate dimensions are: [1, 128, 9, 480, 720]. + # - Assume fp16 (2 bytes per value). + # Memory required: 1 * 128 * 9 * 480 * 720 * 24 * 2 / 1024**3 = 17.8 GB + # + # Memory assessment when using tiling: + # - Assume everything as above but now HxW is 240x360 by tiling in half + # Memory required: 1 * 128 * 9 * 240 * 360 * 24 * 2 / 1024**3 = 4.5 GB + + batch_size, num_channels, num_frames, height, width = z.shape + + overlap_height = int(self.tile_latent_min_height * (1 - self.tile_overlap_factor_height)) + overlap_width = int(self.tile_latent_min_width * (1 - self.tile_overlap_factor_width)) + blend_extent_height = int(self.tile_sample_min_height * self.tile_overlap_factor_height) + blend_extent_width = int(self.tile_sample_min_width * self.tile_overlap_factor_width) + row_limit_height = self.tile_sample_min_height - blend_extent_height + row_limit_width = self.tile_sample_min_width - blend_extent_width + frame_batch_size = self.num_latent_frames_batch_size + + # Split z into overlapping tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, height, overlap_height): + row = [] + for j in range(0, width, overlap_width): + if self.auto_split_process: + num_batches = max(num_frames // frame_batch_size, 1) + conv_cache = None + time = [] + + for k in range(num_batches): + remaining_frames = num_frames % frame_batch_size + start_frame = frame_batch_size * k + (0 if k == 0 else remaining_frames) + end_frame = frame_batch_size * (k + 1) + remaining_frames + tile = z[ + :, + :, + start_frame:end_frame, + i : i + self.tile_latent_min_height, + j : j + self.tile_latent_min_width, + ] + if self.post_quant_conv is not None: + tile = self.post_quant_conv(tile) + tile, conv_cache = self.decoder(tile, conv_cache=conv_cache) + time.append(tile) + + row.append(torch.cat(time, dim=2)) + else: + conv_cache = None + start_frame = 0 + end_frame = 1 + dec = [] + + tile = z[ + :, + :, + start_frame:end_frame, + i : i + self.tile_latent_min_height, + j : j + self.tile_latent_min_width, + ] + + self._set_first_frame() + if self.post_quant_conv is not None: + tile = self.post_quant_conv(tile) + tile, conv_cache = self.decoder(tile, conv_cache=conv_cache) + dec.append(tile) + + self._set_rest_frame() + start_frame = end_frame + end_frame += self.num_latent_frames_batch_size + + while start_frame < num_frames: + tile = z[ + :, + :, + start_frame:end_frame, + i : i + self.tile_latent_min_height, + j : j + self.tile_latent_min_width, + ] + if self.post_quant_conv is not None: + tile = self.post_quant_conv(tile) + tile, conv_cache = self.decoder(tile, conv_cache=conv_cache) + dec.append(tile) + start_frame = end_frame + end_frame += self.num_latent_frames_batch_size + + row.append(torch.cat(dec, dim=2)) + rows.append(row) + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent_height) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent_width) + result_row.append(tile[:, :, :, :row_limit_height, :row_limit_width]) + result_rows.append(torch.cat(result_row, dim=4)) + + dec = torch.cat(result_rows, dim=3) + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + ) -> Union[torch.Tensor, torch.Tensor]: + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z) + if not return_dict: + return (dec,) + return dec + + @classmethod + def from_pretrained(cls, pretrained_model_path, subfolder=None, **vae_additional_kwargs): + if subfolder is not None: + pretrained_model_path = os.path.join(pretrained_model_path, subfolder) + + config_file = os.path.join(pretrained_model_path, 'config.json') + if not os.path.isfile(config_file): + raise RuntimeError(f"{config_file} does not exist") + with open(config_file, "r") as f: + config = json.load(f) + + model = cls.from_config(config, **vae_additional_kwargs) + from diffusers.utils import WEIGHTS_NAME + model_file = os.path.join(pretrained_model_path, WEIGHTS_NAME) + model_file_safetensors = model_file.replace(".bin", ".safetensors") + if os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + if not os.path.isfile(model_file): + raise RuntimeError(f"{model_file} does not exist") + state_dict = torch.load(model_file, map_location="cpu") + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m, u) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/fantasytalking_audio_encoder.py b/VideoX-Fun/videox_fun/models/fantasytalking_audio_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..7f2f7608d33f786edc99123191a05829f83063bf --- /dev/null +++ b/VideoX-Fun/videox_fun/models/fantasytalking_audio_encoder.py @@ -0,0 +1,52 @@ +# Modified from https://github.com/Wan-Video/Wan2.2/blob/main/wan/modules/s2v/audio_encoder.py +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +import math + +import librosa +import numpy as np +import torch +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.modeling_utils import ModelMixin +from transformers import Wav2Vec2Model, Wav2Vec2Processor + + +class FantasyTalkingAudioEncoder(ModelMixin, ConfigMixin, FromOriginalModelMixin): + def __init__(self, pretrained_model_path="facebook/wav2vec2-base-960h", device='cpu'): + super(FantasyTalkingAudioEncoder, self).__init__() + # load pretrained model + self.processor = Wav2Vec2Processor.from_pretrained(pretrained_model_path) + self.model = Wav2Vec2Model.from_pretrained(pretrained_model_path) + self.model = self.model.to(device) + + def extract_audio_feat(self, audio_path, num_frames = 81, fps = 16, sr = 16000): + audio_input, sample_rate = librosa.load(audio_path, sr=sr) + + start_time = 0 + end_time = num_frames / fps + + start_sample = int(start_time * sr) + end_sample = int(end_time * sr) + + try: + audio_segment = audio_input[start_sample:end_sample] + except: + audio_segment = audio_input + + input_values = self.processor( + audio_segment, sampling_rate=sample_rate, return_tensors="pt" + ).input_values.to(self.model.device, self.model.dtype) + + with torch.no_grad(): + fea = self.model(input_values).last_hidden_state + return fea + + def extract_audio_feat_without_file_load(self, audio_segment, sample_rate): + input_values = self.processor( + audio_segment, sampling_rate=sample_rate, return_tensors="pt" + ).input_values.to(self.model.device, self.model.dtype) + + with torch.no_grad(): + fea = self.model(input_values).last_hidden_state + return fea \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/fantasytalking_transformer3d.py b/VideoX-Fun/videox_fun/models/fantasytalking_transformer3d.py new file mode 100644 index 0000000000000000000000000000000000000000..6f7bb793182b79946b14f11da98fd3a230f01e38 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/fantasytalking_transformer3d.py @@ -0,0 +1,644 @@ +# Modified from https://github.com/Fantasy-AMAP/fantasy-talking/blob/main/diffsynth/models +# Copyright Alibaba Inc. All Rights Reserved. +import math +import os +from typing import Any, Dict + +import numpy as np +import torch +import torch.cuda.amp as amp +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import register_to_config +from diffusers.utils import is_torch_version + +from ..dist import sequence_parallel_all_gather, sequence_parallel_chunk +from ..utils import cfg_skip +from .attention_utils import attention +from .wan_transformer3d import (WanAttentionBlock, WanLayerNorm, WanRMSNorm, + WanSelfAttention, WanTransformer3DModel, + sinusoidal_embedding_1d) + + +class AudioProjModel(nn.Module): + def __init__(self, audio_in_dim=1024, cross_attention_dim=1024): + super().__init__() + self.cross_attention_dim = cross_attention_dim + self.proj = torch.nn.Linear(audio_in_dim, cross_attention_dim, bias=False) + self.norm = torch.nn.LayerNorm(cross_attention_dim) + + def forward(self, audio_embeds): + context_tokens = self.proj(audio_embeds) + context_tokens = self.norm(context_tokens) + return context_tokens # [B,L,C] + + +class AudioCrossAttentionProcessor(nn.Module): + def __init__(self, context_dim, hidden_dim): + super().__init__() + + self.context_dim = context_dim + self.hidden_dim = hidden_dim + + self.k_proj = nn.Linear(context_dim, hidden_dim, bias=False) + self.v_proj = nn.Linear(context_dim, hidden_dim, bias=False) + + nn.init.zeros_(self.k_proj.weight) + nn.init.zeros_(self.v_proj.weight) + + self.sp_world_size = 1 + self.sp_world_rank = 0 + self.all_gather = None + + def __call__( + self, + attn: nn.Module, + x: torch.Tensor, + context: torch.Tensor, + context_lens: torch.Tensor, + audio_proj: torch.Tensor, + audio_context_lens: torch.Tensor, + latents_num_frames: int = 21, + audio_scale: float = 1.0, + ) -> torch.Tensor: + """ + x: [B, L1, C]. + context: [B, L2, C]. + context_lens: [B]. + audio_proj: [B, 21, L3, C] + audio_context_lens: [B*21]. + """ + context_img = context[:, :257] + context = context[:, 257:] + b, n, d = x.size(0), attn.num_heads, attn.head_dim + + # Compute query, key, value + q = attn.norm_q(attn.q(x)).view(b, -1, n, d) + k = attn.norm_k(attn.k(context)).view(b, -1, n, d) + v = attn.v(context).view(b, -1, n, d) + k_img = attn.norm_k_img(attn.k_img(context_img)).view(b, -1, n, d) + v_img = attn.v_img(context_img).view(b, -1, n, d) + img_x = attention(q, k_img, v_img, k_lens=None) + # Compute attention + x = attention(q, k, v, k_lens=context_lens) + x = x.flatten(2) + img_x = img_x.flatten(2) + + if len(audio_proj.shape) == 4: + if self.sp_world_size > 1: + q = self.all_gather(q, dim=1) + + length = int(np.floor(q.size()[1] / latents_num_frames) * latents_num_frames) + origin_length = q.size()[1] + if origin_length > length: + q_pad = q[:, length:] + q = q[:, :length] + audio_q = q.view(b * latents_num_frames, -1, n, d) # [b, 21, l1, n, d] + ip_key = self.k_proj(audio_proj).view(b * latents_num_frames, -1, n, d) + ip_value = self.v_proj(audio_proj).view(b * latents_num_frames, -1, n, d) + audio_x = attention( + audio_q, ip_key, ip_value, k_lens=audio_context_lens, attention_type="NORMAL" + ) + audio_x = audio_x.view(b, q.size(1), n, d) + if self.sp_world_size > 1: + if origin_length > length: + audio_x = torch.cat([audio_x, q_pad], dim=1) + audio_x = torch.chunk(audio_x, self.sp_world_size, dim=1)[self.sp_world_rank] + audio_x = audio_x.flatten(2) + elif len(audio_proj.shape) == 3: + ip_key = self.k_proj(audio_proj).view(b, -1, n, d) + ip_value = self.v_proj(audio_proj).view(b, -1, n, d) + audio_x = attention(q, ip_key, ip_value, k_lens=audio_context_lens, attention_type="NORMAL") + audio_x = audio_x.flatten(2) + # Output + if isinstance(audio_scale, torch.Tensor): + audio_scale = audio_scale[:, None, None] + + x = x + img_x + audio_x * audio_scale + x = attn.o(x) + # print(audio_scale) + return x + + +class AudioCrossAttention(WanSelfAttention): + def __init__(self, dim, num_heads, window_size=(-1, -1), qk_norm=True, eps=1e-6): + super().__init__(dim, num_heads, window_size, qk_norm, eps) + + self.k_img = nn.Linear(dim, dim) + self.v_img = nn.Linear(dim, dim) + + self.norm_k_img = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + + self.processor = AudioCrossAttentionProcessor(2048, dim) + + def forward( + self, + x, + context, + context_lens, + audio_proj, + audio_context_lens, + latents_num_frames, + audio_scale: float = 1.0, + **kwargs, + ): + """ + x: [B, L1, C]. + context: [B, L2, C]. + context_lens: [B]. + """ + if audio_proj is None: + return self.processor(self, x, context, context_lens) + else: + return self.processor( + self, + x, + context, + context_lens, + audio_proj, + audio_context_lens, + latents_num_frames, + audio_scale, + ) + + +class AudioAttentionBlock(nn.Module): + def __init__( + self, + cross_attn_type, # Useless + dim, + ffn_dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=False, + eps=1e-6, + ): + super().__init__() + self.dim = dim + self.ffn_dim = ffn_dim + self.num_heads = num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.cross_attn_norm = cross_attn_norm + self.eps = eps + + # Layers + self.norm1 = WanLayerNorm(dim, eps) + self.self_attn = WanSelfAttention(dim, num_heads, window_size, qk_norm, eps) + self.norm3 = ( + WanLayerNorm(dim, eps, elementwise_affine=True) + if cross_attn_norm + else nn.Identity() + ) + self.cross_attn = AudioCrossAttention( + dim, num_heads, (-1, -1), qk_norm, eps + ) + self.norm2 = WanLayerNorm(dim, eps) + self.ffn = nn.Sequential( + nn.Linear(dim, ffn_dim), + nn.GELU(approximate="tanh"), + nn.Linear(ffn_dim, dim), + ) + + # Modulation + self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + + def forward( + self, + x, + e, + seq_lens, + grid_sizes, + freqs, + context, + context_lens, + audio_proj=None, + audio_context_lens=None, + audio_scale=1, + dtype=torch.bfloat16, + t=0, + ): + assert e.dtype == torch.float32 + with amp.autocast(dtype=torch.float32): + e = (self.modulation.to(dtype=e.dtype, device=e.device) + e).chunk(6, dim=1) + assert e[0].dtype == torch.float32 + + # self-attention + y = self.self_attn( + self.norm1(x).float() * (1 + e[1]) + e[0], seq_lens, grid_sizes, freqs, dtype, t=t + ) + with amp.autocast(dtype=torch.float32): + x = x + y * e[2] + + # Cross-attention & FFN function + def cross_attn_ffn(x, context, context_lens, e): + x = x + self.cross_attn( + self.norm3(x), context, context_lens, dtype=dtype, t=t, + audio_proj=audio_proj, audio_context_lens=audio_context_lens, audio_scale=audio_scale, + latents_num_frames=grid_sizes[0][0], + ) + y = self.ffn(self.norm2(x).float() * (1 + e[4]) + e[3]) + with amp.autocast(dtype=torch.float32): + x = x + y * e[5] + return x + + x = cross_attn_ffn(x, context, context_lens, e) + return x + + +class FantasyTalkingTransformer3DModel(WanTransformer3DModel): + @register_to_config + def __init__(self, + model_type='i2v', + patch_size=(1, 2, 2), + text_len=512, + in_dim=16, + dim=2048, + ffn_dim=8192, + freq_dim=256, + text_dim=4096, + out_dim=16, + num_heads=16, + num_layers=32, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=True, + eps=1e-6, + cross_attn_type=None, + audio_in_dim=768): + super().__init__(model_type, patch_size, text_len, in_dim, dim, ffn_dim, freq_dim, text_dim, out_dim, + num_heads, num_layers, window_size, qk_norm, cross_attn_norm, eps) + + if cross_attn_type is None: + cross_attn_type = 't2v_cross_attn' if model_type == 't2v' else 'i2v_cross_attn' + self.blocks = nn.ModuleList([ + AudioAttentionBlock(cross_attn_type, dim, ffn_dim, num_heads, + window_size, qk_norm, cross_attn_norm, eps) + for _ in range(num_layers) + ]) + for layer_idx, block in enumerate(self.blocks): + block.self_attn.layer_idx = layer_idx + block.self_attn.num_layers = self.num_layers + + self.proj_model = AudioProjModel(audio_in_dim, 2048) + + def split_audio_sequence(self, audio_proj_length, num_frames=81): + """ + Map the audio feature sequence to corresponding latent frame slices. + + Args: + audio_proj_length (int): The total length of the audio feature sequence + (e.g., 173 in audio_proj[1, 173, 768]). + num_frames (int): The number of video frames in the training data (default: 81). + + Returns: + list: A list of [start_idx, end_idx] pairs. Each pair represents the index range + (within the audio feature sequence) corresponding to a latent frame. + """ + # Average number of tokens per original video frame + tokens_per_frame = audio_proj_length / num_frames + + # Each latent frame covers 4 video frames, and we want the center + tokens_per_latent_frame = tokens_per_frame * 4 + half_tokens = int(tokens_per_latent_frame / 2) + + pos_indices = [] + for i in range(int((num_frames - 1) / 4) + 1): + if i == 0: + pos_indices.append(0) + else: + start_token = tokens_per_frame * ((i - 1) * 4 + 1) + end_token = tokens_per_frame * (i * 4 + 1) + center_token = int((start_token + end_token) / 2) - 1 + pos_indices.append(center_token) + + # Build index ranges centered around each position + pos_idx_ranges = [[idx - half_tokens, idx + half_tokens] for idx in pos_indices] + + # Adjust the first range to avoid negative start index + pos_idx_ranges[0] = [ + -(half_tokens * 2 - pos_idx_ranges[1][0]), + pos_idx_ranges[1][0], + ] + + return pos_idx_ranges + + def split_tensor_with_padding(self, input_tensor, pos_idx_ranges, expand_length=0): + """ + Split the input tensor into subsequences based on index ranges, and apply right-side zero-padding + if the range exceeds the input boundaries. + + Args: + input_tensor (Tensor): Input audio tensor of shape [1, L, 768]. + pos_idx_ranges (list): A list of index ranges, e.g. [[-7, 1], [1, 9], ..., [165, 173]]. + expand_length (int): Number of tokens to expand on both sides of each subsequence. + + Returns: + sub_sequences (Tensor): A tensor of shape [1, F, L, 768], where L is the length after padding. + Each element is a padded subsequence. + k_lens (Tensor): A tensor of shape [F], representing the actual (unpadded) length of each subsequence. + Useful for ignoring padding tokens in attention masks. + """ + pos_idx_ranges = [ + [idx[0] - expand_length, idx[1] + expand_length] for idx in pos_idx_ranges + ] + sub_sequences = [] + seq_len = input_tensor.size(1) # 173 + max_valid_idx = seq_len - 1 # 172 + k_lens_list = [] + for start, end in pos_idx_ranges: + # Calculate the fill amount + pad_front = max(-start, 0) + pad_back = max(end - max_valid_idx, 0) + + # Calculate the start and end indices of the valid part + valid_start = max(start, 0) + valid_end = min(end, max_valid_idx) + + # Extract the valid part + if valid_start <= valid_end: + valid_part = input_tensor[:, valid_start : valid_end + 1, :] + else: + valid_part = input_tensor.new_zeros((1, 0, input_tensor.size(2))) + + # In the sequence dimension (the 1st dimension) perform padding + padded_subseq = F.pad( + valid_part, + (0, 0, 0, pad_back + pad_front, 0, 0), + mode="constant", + value=0, + ) + k_lens_list.append(padded_subseq.size(-2) - pad_back - pad_front) + + sub_sequences.append(padded_subseq) + return torch.stack(sub_sequences, dim=1), torch.tensor( + k_lens_list, dtype=torch.long + ) + + def enable_multi_gpus_inference(self,): + super().enable_multi_gpus_inference() + for name, module in self.named_modules(): + if module.__class__.__name__ == 'AudioCrossAttentionProcessor': + module.sp_world_size = self.sp_world_size + module.sp_world_rank = self.sp_world_rank + module.all_gather = self.all_gather + + @cfg_skip() + def forward( + self, + x, + t, + context, + seq_len, + audio_wav2vec_fea=None, + clip_fea=None, + y=None, + audio_scale=1, + cond_flag=True + ): + r""" + Forward pass through the diffusion model + + Args: + x (List[Tensor]): + List of input video tensors, each with shape [C_in, F, H, W] + t (Tensor): + Diffusion timesteps tensor of shape [B] + context (List[Tensor]): + List of text embeddings each with shape [L, C] + seq_len (`int`): + Maximum sequence length for positional encoding + clip_fea (Tensor, *optional*): + CLIP image features for image-to-video mode + y (List[Tensor], *optional*): + Conditional video inputs for image-to-video mode, same shape as x + + Returns: + List[Tensor]: + List of denoised video tensors with original input shapes [C_out, F, H / 8, W / 8] + """ + # Wan2.2 don't need a clip. + # if self.model_type == 'i2v': + # assert clip_fea is not None and y is not None + # params + device = self.patch_embedding.weight.device + dtype = x.dtype + if self.freqs.device != device and torch.device(type="meta") != device: + self.freqs = self.freqs.to(device) + + if y is not None: + x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)] + + # embeddings + x = [self.patch_embedding(u.unsqueeze(0)) for u in x] + + grid_sizes = torch.stack( + [torch.tensor(u.shape[2:], dtype=torch.long) for u in x]) + + x = [u.flatten(2).transpose(1, 2) for u in x] + + seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long) + if self.sp_world_size > 1: + seq_len = int(math.ceil(seq_len / self.sp_world_size)) * self.sp_world_size + assert seq_lens.max() <= seq_len + x = torch.cat([ + torch.cat([u, u.new_zeros(1, seq_len - u.size(1), u.size(2))], + dim=1) for u in x + ]) + + # time embeddings + with amp.autocast(dtype=torch.float32): + if t.dim() != 1: + if t.size(1) < seq_len: + pad_size = seq_len - t.size(1) + last_elements = t[:, -1].unsqueeze(1) + padding = last_elements.repeat(1, pad_size) + t = torch.cat([t, padding], dim=1) + bt = t.size(0) + ft = t.flatten() + e = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, + ft).unflatten(0, (bt, seq_len)).float()) + e0 = self.time_projection(e).unflatten(2, (6, self.dim)) + else: + e = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, t).float()) + e0 = self.time_projection(e).unflatten(1, (6, self.dim)) + + # assert e.dtype == torch.float32 and e0.dtype == torch.float32 + # e0 = e0.to(dtype) + # e = e.to(dtype) + + # context + context_lens = None + context = self.text_embedding( + torch.stack([ + torch.cat( + [u, u.new_zeros(self.text_len - u.size(0), u.size(1))]) + for u in context + ])) + + if clip_fea is not None: + context_clip = self.img_emb(clip_fea) # bs x 257 x dim + context = torch.concat([context_clip, context], dim=1) + + num_frames = (grid_sizes[0][0] - 1) * 4 + 1 + audio_proj_fea = self.proj_model(audio_wav2vec_fea) + pos_idx_ranges = self.split_audio_sequence(audio_proj_fea.size(1), num_frames=num_frames) + audio_proj, audio_context_lens = self.split_tensor_with_padding( + audio_proj_fea, pos_idx_ranges, expand_length=4 + ) + + # Context Parallel + if self.sp_world_size > 1: + x = torch.chunk(x, self.sp_world_size, dim=1)[self.sp_world_rank] + if t.dim() != 1: + e0 = torch.chunk(e0, self.sp_world_size, dim=1)[self.sp_world_rank] + e = torch.chunk(e, self.sp_world_size, dim=1)[self.sp_world_rank] + + # TeaCache + if self.teacache is not None: + if cond_flag: + if t.dim() != 1: + modulated_inp = e0[:, -1, :] + else: + modulated_inp = e0 + skip_flag = self.teacache.cnt < self.teacache.num_skip_start_steps + if skip_flag: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + else: + if cond_flag: + rel_l1_distance = self.teacache.compute_rel_l1_distance(self.teacache.previous_modulated_input, modulated_inp) + self.teacache.accumulated_rel_l1_distance += self.teacache.rescale_func(rel_l1_distance) + if self.teacache.accumulated_rel_l1_distance < self.teacache.rel_l1_thresh: + self.should_calc = False + else: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + self.teacache.previous_modulated_input = modulated_inp + self.teacache.should_calc = self.should_calc + else: + self.should_calc = self.teacache.should_calc + + # TeaCache + if self.teacache is not None: + if not self.should_calc: + previous_residual = self.teacache.previous_residual_cond if cond_flag else self.teacache.previous_residual_uncond + x = x + previous_residual.to(x.device)[-x.size()[0]:,] + else: + ori_x = x.clone().cpu() if self.teacache.offload else x.clone() + + for block in self.blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + x, + e0, + seq_lens, + grid_sizes, + self.freqs, + context, + context_lens, + audio_proj, + audio_context_lens, + audio_scale, + dtype, + t, + **ckpt_kwargs, + ) + else: + # arguments + kwargs = dict( + e=e0, + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=self.freqs, + context=context, + context_lens=context_lens, + audio_proj=audio_proj, + audio_context_lens=audio_context_lens, + audio_scale=audio_scale, + dtype=dtype, + t=t + ) + x = block(x, **kwargs) + + if cond_flag: + self.teacache.previous_residual_cond = x.cpu() - ori_x if self.teacache.offload else x - ori_x + else: + self.teacache.previous_residual_uncond = x.cpu() - ori_x if self.teacache.offload else x - ori_x + else: + for block in self.blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + x, + e0, + seq_lens, + grid_sizes, + self.freqs, + context, + context_lens, + audio_proj, + audio_context_lens, + audio_scale, + dtype, + t, + **ckpt_kwargs, + ) + else: + # arguments + kwargs = dict( + e=e0, + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=self.freqs, + context=context, + context_lens=context_lens, + audio_proj=audio_proj, + audio_context_lens=audio_context_lens, + audio_scale=audio_scale, + dtype=dtype, + t=t + ) + x = block(x, **kwargs) + + # head + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.head), x, e, **ckpt_kwargs) + else: + x = self.head(x, e) + + if self.sp_world_size > 1: + x = self.all_gather(x, dim=1) + + # Unpatchify + x = self.unpatchify(x, grid_sizes) + x = torch.stack(x) + if self.teacache is not None and cond_flag: + self.teacache.cnt += 1 + if self.teacache.cnt == self.teacache.num_steps: + self.teacache.reset() + return x \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/flux2_image_processor.py b/VideoX-Fun/videox_fun/models/flux2_image_processor.py new file mode 100644 index 0000000000000000000000000000000000000000..d2edc9ec2cfcf3eb5bd137f7abba5ab5151189a9 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/flux2_image_processor.py @@ -0,0 +1,139 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/flux2/image_processor.py +# Copyright 2025 The Black Forest Labs Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Tuple + +import PIL.Image + +from diffusers.configuration_utils import register_to_config +from diffusers.image_processor import VaeImageProcessor + + +class Flux2ImageProcessor(VaeImageProcessor): + r""" + Image processor to preprocess the reference (character) image for the Flux2 model. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Whether to downscale the image's (height, width) dimensions to multiples of `vae_scale_factor`. Can accept + `height` and `width` arguments from [`image_processor.VaeImageProcessor.preprocess`] method. + vae_scale_factor (`int`, *optional*, defaults to `16`): + VAE (spatial) scale factor. If `do_resize` is `True`, the image is automatically resized to multiples of + this factor. + vae_latent_channels (`int`, *optional*, defaults to `32`): + VAE latent channels. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image to [-1,1]. + do_convert_rgb (`bool`, *optional*, defaults to be `True`): + Whether to convert the images to RGB format. + """ + + @register_to_config + def __init__( + self, + do_resize: bool = True, + vae_scale_factor: int = 16, + vae_latent_channels: int = 32, + do_normalize: bool = True, + do_convert_rgb: bool = True, + ): + super().__init__( + do_resize=do_resize, + vae_scale_factor=vae_scale_factor, + vae_latent_channels=vae_latent_channels, + do_normalize=do_normalize, + do_convert_rgb=do_convert_rgb, + ) + + @staticmethod + def check_image_input( + image: PIL.Image.Image, max_aspect_ratio: int = 8, min_side_length: int = 64, max_area: int = 1024 * 1024 + ) -> PIL.Image.Image: + """ + Check if image meets minimum size and aspect ratio requirements. + + Args: + image: PIL Image to validate + max_aspect_ratio: Maximum allowed aspect ratio (width/height or height/width) + min_side_length: Minimum pixels required for width and height + max_area: Maximum allowed area in pixels² + + Returns: + The input image if valid + + Raises: + ValueError: If image is too small or aspect ratio is too extreme + """ + if not isinstance(image, PIL.Image.Image): + raise ValueError(f"Image must be a PIL.Image.Image, got {type(image)}") + + width, height = image.size + + # Check minimum dimensions + if width < min_side_length or height < min_side_length: + raise ValueError( + f"Image too small: {width}×{height}. Both dimensions must be at least {min_side_length}px" + ) + + # Check aspect ratio + aspect_ratio = max(width / height, height / width) + if aspect_ratio > max_aspect_ratio: + raise ValueError( + f"Aspect ratio too extreme: {width}×{height} (ratio: {aspect_ratio:.1f}:1). " + f"Maximum allowed ratio is {max_aspect_ratio}:1" + ) + + return image + + @staticmethod + def _resize_to_target_area(image: PIL.Image.Image, target_area: int = 1024 * 1024) -> Tuple[int, int]: + image_width, image_height = image.size + + scale = math.sqrt(target_area / (image_width * image_height)) + width = int(image_width * scale) + height = int(image_height * scale) + + return image.resize((width, height), PIL.Image.Resampling.LANCZOS) + + def _resize_and_crop( + self, + image: PIL.Image.Image, + width: int, + height: int, + ) -> PIL.Image.Image: + r""" + center crop the image to the specified width and height. + + Args: + image (`PIL.Image.Image`): + The image to resize and crop. + width (`int`): + The width to resize the image to. + height (`int`): + The height to resize the image to. + + Returns: + `PIL.Image.Image`: + The resized and cropped image. + """ + image_width, image_height = image.size + + left = (image_width - width) // 2 + top = (image_height - height) // 2 + right = left + width + bottom = top + height + + return image.crop((left, top, right, bottom)) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/flux2_transformer2d.py b/VideoX-Fun/videox_fun/models/flux2_transformer2d.py new file mode 100644 index 0000000000000000000000000000000000000000..8b748c03d71f4ea9b8b42b8d1d064c0339f0fc37 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/flux2_transformer2d.py @@ -0,0 +1,1289 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_flux2.py +# Copyright 2025 Black Forest Labs, The HuggingFace Team and The InstantX Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import inspect +import json +import os +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin +from diffusers.models.attention_processor import Attention, AttentionProcessor +from diffusers.models.embeddings import (TimestepEmbedding, Timesteps, + apply_rotary_emb, + get_1d_rotary_pos_embed) +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import AdaLayerNormContinuous +from diffusers.utils import (USE_PEFT_BACKEND, is_torch_npu_available, + is_torch_version, logging, scale_lora_layers, + unscale_lora_layers) + +from ..dist import (Flux2MultiGPUsAttnProcessor2_0, get_sequence_parallel_rank, + get_sequence_parallel_world_size, get_sp_group) +from .attention_utils import attention + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def _get_projections(attn: "Flux2Attention", hidden_states, encoder_hidden_states=None): + query = attn.to_q(hidden_states) + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + + encoder_query = encoder_key = encoder_value = None + if encoder_hidden_states is not None and attn.added_kv_proj_dim is not None: + encoder_query = attn.add_q_proj(encoder_hidden_states) + encoder_key = attn.add_k_proj(encoder_hidden_states) + encoder_value = attn.add_v_proj(encoder_hidden_states) + + return query, key, value, encoder_query, encoder_key, encoder_value + + +def _get_qkv_projections(attn: "Flux2Attention", hidden_states, encoder_hidden_states=None): + return _get_projections(attn, hidden_states, encoder_hidden_states) + + +def apply_rotary_emb( + x: torch.Tensor, + freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], + use_real: bool = True, + use_real_unbind_dim: int = -1, + sequence_dim: int = 2, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings + to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are + reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting + tensors contain rotary embeddings and are returned as real tensors. + + Args: + x (`torch.Tensor`): + Query or key tensor to apply rotary embeddings. [B, H, S, D] xk (torch.Tensor): Key tensor to apply + freqs_cis (`Tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],) + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings. + """ + if use_real: + cos, sin = freqs_cis # [S, D] + if sequence_dim == 2: + cos = cos[None, None, :, :] + sin = sin[None, None, :, :] + elif sequence_dim == 1: + cos = cos[None, :, None, :] + sin = sin[None, :, None, :] + else: + raise ValueError(f"`sequence_dim={sequence_dim}` but should be 1 or 2.") + + cos, sin = cos.to(x.device), sin.to(x.device) + + if use_real_unbind_dim == -1: + # Used for flux, cogvideox, hunyuan-dit + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, H, S, D//2] + x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3) + elif use_real_unbind_dim == -2: + # Used for Stable Audio, OmniGen, CogView4 and Cosmos + x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) # [B, H, S, D//2] + x_rotated = torch.cat([-x_imag, x_real], dim=-1) + else: + raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") + + out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + + return out + else: + # used for lumina + x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2)) + freqs_cis = freqs_cis.unsqueeze(2) + x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3) + + return x_out.type_as(x) + + +class Flux2SwiGLU(nn.Module): + """ + Flux 2 uses a SwiGLU-style activation in the transformer feedforward sub-blocks, but with the linear projection + layer fused into the first linear layer of the FF sub-block. Thus, this module has no trainable parameters. + """ + + def __init__(self): + super().__init__() + self.gate_fn = nn.SiLU() + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x1, x2 = x.chunk(2, dim=-1) + x = self.gate_fn(x1) * x2 + return x + + +class Flux2FeedForward(nn.Module): + def __init__( + self, + dim: int, + dim_out: Optional[int] = None, + mult: float = 3.0, + inner_dim: Optional[int] = None, + bias: bool = False, + ): + super().__init__() + if inner_dim is None: + inner_dim = int(dim * mult) + dim_out = dim_out or dim + + # Flux2SwiGLU will reduce the dimension by half + self.linear_in = nn.Linear(dim, inner_dim * 2, bias=bias) + self.act_fn = Flux2SwiGLU() + self.linear_out = nn.Linear(inner_dim, dim_out, bias=bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.linear_in(x) + x = self.act_fn(x) + x = self.linear_out(x) + return x + + +class Flux2AttnProcessor: + _attention_backend = None + _parallel_config = None + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError(f"{self.__class__.__name__} requires PyTorch 2.0. Please upgrade your pytorch version.") + + def __call__( + self, + attn: Union["Flux2Attention", "Flux2ParallelSelfAttention"], + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + image_rotary_emb: Optional[torch.Tensor] = None, + text_seq_len: int = None, + ) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """ + Unified processor for both Flux2Attention and Flux2ParallelSelfAttention. + + Args: + attn: Attention module (either Flux2Attention or Flux2ParallelSelfAttention) + hidden_states: Input hidden states + encoder_hidden_states: Optional encoder hidden states (only for Flux2Attention) + attention_mask: Optional attention mask + image_rotary_emb: Optional rotary embeddings + + Returns: + For Flux2Attention with encoder_hidden_states: (hidden_states, encoder_hidden_states) + For Flux2Attention without encoder_hidden_states: hidden_states + For Flux2ParallelSelfAttention: hidden_states + """ + # Determine which type of attention we're processing + is_parallel_self_attn = hasattr(attn, 'to_qkv_mlp_proj') and attn.to_qkv_mlp_proj is not None + + if is_parallel_self_attn: + # ============================================ + # Parallel Self-Attention Path (with MLP) + # ============================================ + # Parallel in (QKV + MLP in) projection + hidden_states = attn.to_qkv_mlp_proj(hidden_states) + qkv, mlp_hidden_states = torch.split( + hidden_states, [3 * attn.inner_dim, attn.mlp_hidden_dim * attn.mlp_mult_factor], dim=-1 + ) + + # Handle the attention logic + query, key, value = qkv.chunk(3, dim=-1) + + else: + # ============================================ + # Standard Attention Path (possibly with encoder) + # ============================================ + query, key, value, encoder_query, encoder_key, encoder_value = _get_qkv_projections( + attn, hidden_states, encoder_hidden_states + ) + + # Common processing for query, key, value + query = query.unflatten(-1, (attn.heads, -1)) + key = key.unflatten(-1, (attn.heads, -1)) + value = value.unflatten(-1, (attn.heads, -1)) + + query = attn.norm_q(query) + key = attn.norm_k(key) + + # Handle encoder projections (only for standard attention) + if not is_parallel_self_attn and attn.added_kv_proj_dim is not None: + encoder_query = encoder_query.unflatten(-1, (attn.heads, -1)) + encoder_key = encoder_key.unflatten(-1, (attn.heads, -1)) + encoder_value = encoder_value.unflatten(-1, (attn.heads, -1)) + + encoder_query = attn.norm_added_q(encoder_query) + encoder_key = attn.norm_added_k(encoder_key) + + query = torch.cat([encoder_query, query], dim=1) + key = torch.cat([encoder_key, key], dim=1) + value = torch.cat([encoder_value, value], dim=1) + + # Apply rotary embeddings + if image_rotary_emb is not None: + query = apply_rotary_emb(query, image_rotary_emb, sequence_dim=1) + key = apply_rotary_emb(key, image_rotary_emb, sequence_dim=1) + + # Perform attention + hidden_states = attention( + query, key, value, attn_mask=attention_mask, + ) + hidden_states = hidden_states.flatten(2, 3) + hidden_states = hidden_states.to(query.dtype) + + if is_parallel_self_attn: + # ============================================ + # Parallel Self-Attention Output Path + # ============================================ + # Handle the feedforward (FF) logic + mlp_hidden_states = attn.mlp_act_fn(mlp_hidden_states) + + # Concatenate and parallel output projection + hidden_states = torch.cat([hidden_states, mlp_hidden_states], dim=-1) + hidden_states = attn.to_out(hidden_states) + + return hidden_states + + else: + # ============================================ + # Standard Attention Output Path + # ============================================ + # Split encoder and latent hidden states if encoder was used + if encoder_hidden_states is not None: + encoder_hidden_states, hidden_states = hidden_states.split_with_sizes( + [encoder_hidden_states.shape[1], hidden_states.shape[1] - encoder_hidden_states.shape[1]], dim=1 + ) + encoder_hidden_states = attn.to_add_out(encoder_hidden_states) + + # Project output + hidden_states = attn.to_out[0](hidden_states) + hidden_states = attn.to_out[1](hidden_states) + + if encoder_hidden_states is not None: + return hidden_states, encoder_hidden_states + else: + return hidden_states + + +class Flux2Attention(torch.nn.Module): + _default_processor_cls = Flux2AttnProcessor + _available_processors = [Flux2AttnProcessor] + + def __init__( + self, + query_dim: int, + heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + bias: bool = False, + added_kv_proj_dim: Optional[int] = None, + added_proj_bias: Optional[bool] = True, + out_bias: bool = True, + eps: float = 1e-5, + out_dim: int = None, + elementwise_affine: bool = True, + processor=None, + ): + super().__init__() + + self.head_dim = dim_head + self.inner_dim = out_dim if out_dim is not None else dim_head * heads + self.query_dim = query_dim + self.out_dim = out_dim if out_dim is not None else query_dim + self.heads = out_dim // dim_head if out_dim is not None else heads + + self.use_bias = bias + self.dropout = dropout + + self.added_kv_proj_dim = added_kv_proj_dim + self.added_proj_bias = added_proj_bias + + self.to_q = torch.nn.Linear(query_dim, self.inner_dim, bias=bias) + self.to_k = torch.nn.Linear(query_dim, self.inner_dim, bias=bias) + self.to_v = torch.nn.Linear(query_dim, self.inner_dim, bias=bias) + + # QK Norm + self.norm_q = torch.nn.RMSNorm(dim_head, eps=eps, elementwise_affine=elementwise_affine) + self.norm_k = torch.nn.RMSNorm(dim_head, eps=eps, elementwise_affine=elementwise_affine) + + self.to_out = torch.nn.ModuleList([]) + self.to_out.append(torch.nn.Linear(self.inner_dim, self.out_dim, bias=out_bias)) + self.to_out.append(torch.nn.Dropout(dropout)) + + if added_kv_proj_dim is not None: + self.norm_added_q = torch.nn.RMSNorm(dim_head, eps=eps) + self.norm_added_k = torch.nn.RMSNorm(dim_head, eps=eps) + self.add_q_proj = torch.nn.Linear(added_kv_proj_dim, self.inner_dim, bias=added_proj_bias) + self.add_k_proj = torch.nn.Linear(added_kv_proj_dim, self.inner_dim, bias=added_proj_bias) + self.add_v_proj = torch.nn.Linear(added_kv_proj_dim, self.inner_dim, bias=added_proj_bias) + self.to_add_out = torch.nn.Linear(self.inner_dim, query_dim, bias=out_bias) + + if processor is None: + processor = self._default_processor_cls() + self.set_processor(processor) + + def set_processor(self, processor: AttentionProcessor) -> None: + """ + Set the attention processor to use. + + Args: + processor (`AttnProcessor`): + The attention processor to use. + """ + # if current processor is in `self._modules` and if passed `processor` is not, we need to + # pop `processor` from `self._modules` + if ( + hasattr(self, "processor") + and isinstance(self.processor, torch.nn.Module) + and not isinstance(processor, torch.nn.Module) + ): + logger.info(f"You are removing possibly trained weights of {self.processor} with {processor}") + self._modules.pop("processor") + + self.processor = processor + + def get_processor(self, return_deprecated_lora: bool = False) -> "AttentionProcessor": + """ + Get the attention processor in use. + + Args: + return_deprecated_lora (`bool`, *optional*, defaults to `False`): + Set to `True` to return the deprecated LoRA attention processor. + + Returns: + "AttentionProcessor": The attention processor in use. + """ + if not return_deprecated_lora: + return self.processor + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + image_rotary_emb: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + attn_parameters = set(inspect.signature(self.processor.__call__).parameters.keys()) + unused_kwargs = [k for k, _ in kwargs.items() if k not in attn_parameters] + if len(unused_kwargs) > 0: + logger.warning( + f"joint_attention_kwargs {unused_kwargs} are not expected by {self.processor.__class__.__name__} and will be ignored." + ) + kwargs = {k: w for k, w in kwargs.items() if k in attn_parameters} + return self.processor(self, hidden_states, encoder_hidden_states, attention_mask, image_rotary_emb, **kwargs) + + +class Flux2ParallelSelfAttention(torch.nn.Module): + """ + Flux 2 parallel self-attention for the Flux 2 single-stream transformer blocks. + + This implements a parallel transformer block, where the attention QKV projections are fused to the feedforward (FF) + input projections, and the attention output projections are fused to the FF output projections. See the [ViT-22B + paper](https://arxiv.org/abs/2302.05442) for a visual depiction of this type of transformer block. + """ + + _default_processor_cls = Flux2AttnProcessor + _available_processors = [Flux2AttnProcessor] + # Does not support QKV fusion as the QKV projections are always fused + _supports_qkv_fusion = False + + def __init__( + self, + query_dim: int, + heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + bias: bool = False, + out_bias: bool = True, + eps: float = 1e-5, + out_dim: int = None, + elementwise_affine: bool = True, + mlp_ratio: float = 4.0, + mlp_mult_factor: int = 2, + processor=None, + ): + super().__init__() + + self.head_dim = dim_head + self.inner_dim = out_dim if out_dim is not None else dim_head * heads + self.query_dim = query_dim + self.out_dim = out_dim if out_dim is not None else query_dim + self.heads = out_dim // dim_head if out_dim is not None else heads + + self.use_bias = bias + self.dropout = dropout + + self.mlp_ratio = mlp_ratio + self.mlp_hidden_dim = int(query_dim * self.mlp_ratio) + self.mlp_mult_factor = mlp_mult_factor + + # Fused QKV projections + MLP input projection + self.to_qkv_mlp_proj = torch.nn.Linear( + self.query_dim, self.inner_dim * 3 + self.mlp_hidden_dim * self.mlp_mult_factor, bias=bias + ) + self.mlp_act_fn = Flux2SwiGLU() + + # QK Norm + self.norm_q = torch.nn.RMSNorm(dim_head, eps=eps, elementwise_affine=elementwise_affine) + self.norm_k = torch.nn.RMSNorm(dim_head, eps=eps, elementwise_affine=elementwise_affine) + + # Fused attention output projection + MLP output projection + self.to_out = torch.nn.Linear(self.inner_dim + self.mlp_hidden_dim, self.out_dim, bias=out_bias) + + if processor is None: + processor = self._default_processor_cls() + self.set_processor(processor) + + def set_processor(self, processor: AttentionProcessor) -> None: + """ + Set the attention processor to use. + + Args: + processor (`AttnProcessor`): + The attention processor to use. + """ + # if current processor is in `self._modules` and if passed `processor` is not, we need to + # pop `processor` from `self._modules` + if ( + hasattr(self, "processor") + and isinstance(self.processor, torch.nn.Module) + and not isinstance(processor, torch.nn.Module) + ): + logger.info(f"You are removing possibly trained weights of {self.processor} with {processor}") + self._modules.pop("processor") + + self.processor = processor + + def get_processor(self, return_deprecated_lora: bool = False) -> "AttentionProcessor": + """ + Get the attention processor in use. + + Args: + return_deprecated_lora (`bool`, *optional*, defaults to `False`): + Set to `True` to return the deprecated LoRA attention processor. + + Returns: + "AttentionProcessor": The attention processor in use. + """ + if not return_deprecated_lora: + return self.processor + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + image_rotary_emb: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + attn_parameters = set(inspect.signature(self.processor.__call__).parameters.keys()) + unused_kwargs = [k for k, _ in kwargs.items() if k not in attn_parameters] + if len(unused_kwargs) > 0: + logger.warning( + f"joint_attention_kwargs {unused_kwargs} are not expected by {self.processor.__class__.__name__} and will be ignored." + ) + kwargs = {k: w for k, w in kwargs.items() if k in attn_parameters} + return self.processor(self, hidden_states, encoder_hidden_states, attention_mask, image_rotary_emb, **kwargs) + + +class Flux2SingleTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float = 3.0, + eps: float = 1e-6, + bias: bool = False, + ): + super().__init__() + + self.norm = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + + # Note that the MLP in/out linear layers are fused with the attention QKV/out projections, respectively; this + # is often called a "parallel" transformer block. See the [ViT-22B paper](https://arxiv.org/abs/2302.05442) + # for a visual depiction of this type of transformer block. + self.attn = Flux2ParallelSelfAttention( + query_dim=dim, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=dim, + bias=bias, + out_bias=bias, + eps=eps, + mlp_ratio=mlp_ratio, + mlp_mult_factor=2, + processor=Flux2AttnProcessor(), + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor], + temb_mod_params: Tuple[torch.Tensor, torch.Tensor, torch.Tensor], + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # If encoder_hidden_states is None, hidden_states is assumed to have encoder_hidden_states already + # concatenated + if encoder_hidden_states is not None: + text_seq_len = encoder_hidden_states.shape[1] + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + mod_shift, mod_scale, mod_gate = temb_mod_params + + norm_hidden_states = self.norm(hidden_states) + norm_hidden_states = (1 + mod_scale) * norm_hidden_states + mod_shift + + joint_attention_kwargs = joint_attention_kwargs or {} + attn_output = self.attn( + hidden_states=norm_hidden_states, + image_rotary_emb=image_rotary_emb, + text_seq_len=text_seq_len, + **joint_attention_kwargs, + ) + + hidden_states = hidden_states + mod_gate * attn_output + if hidden_states.dtype == torch.float16: + hidden_states = hidden_states.clip(-65504, 65504) + + encoder_hidden_states, hidden_states = hidden_states[:, :text_seq_len], hidden_states[:, text_seq_len:] + return encoder_hidden_states, hidden_states + + +class Flux2TransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float = 3.0, + eps: float = 1e-6, + bias: bool = False, + ): + super().__init__() + self.mlp_hidden_dim = int(dim * mlp_ratio) + + self.norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.norm1_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + + self.attn = Flux2Attention( + query_dim=dim, + added_kv_proj_dim=dim, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=dim, + bias=bias, + added_proj_bias=bias, + out_bias=bias, + eps=eps, + processor=Flux2AttnProcessor(), + ) + + self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.ff = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias) + + self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.ff_context = Flux2FeedForward(dim=dim, dim_out=dim, mult=mlp_ratio, bias=bias) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb_mod_params_img: Tuple[Tuple[torch.Tensor, torch.Tensor, torch.Tensor], ...], + temb_mod_params_txt: Tuple[Tuple[torch.Tensor, torch.Tensor, torch.Tensor], ...], + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + joint_attention_kwargs = joint_attention_kwargs or {} + + # Modulation parameters shape: [1, 1, self.dim] + (shift_msa, scale_msa, gate_msa), (shift_mlp, scale_mlp, gate_mlp) = temb_mod_params_img + (c_shift_msa, c_scale_msa, c_gate_msa), (c_shift_mlp, c_scale_mlp, c_gate_mlp) = temb_mod_params_txt + + # Img stream + norm_hidden_states = self.norm1(hidden_states) + norm_hidden_states = (1 + scale_msa) * norm_hidden_states + shift_msa + + # Conditioning txt stream + norm_encoder_hidden_states = self.norm1_context(encoder_hidden_states) + norm_encoder_hidden_states = (1 + c_scale_msa) * norm_encoder_hidden_states + c_shift_msa + + # Attention on concatenated img + txt stream + attention_outputs = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + image_rotary_emb=image_rotary_emb, + **joint_attention_kwargs, + ) + + attn_output, context_attn_output = attention_outputs + + # Process attention outputs for the image stream (`hidden_states`). + attn_output = gate_msa * attn_output + hidden_states = hidden_states + attn_output + + norm_hidden_states = self.norm2(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp + + ff_output = self.ff(norm_hidden_states) + hidden_states = hidden_states + gate_mlp * ff_output + + # Process attention outputs for the text stream (`encoder_hidden_states`). + context_attn_output = c_gate_msa * context_attn_output + encoder_hidden_states = encoder_hidden_states + context_attn_output + + norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states) + norm_encoder_hidden_states = norm_encoder_hidden_states * (1 + c_scale_mlp) + c_shift_mlp + + context_ff_output = self.ff_context(norm_encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states + c_gate_mlp * context_ff_output + if encoder_hidden_states.dtype == torch.float16: + encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) + + return encoder_hidden_states, hidden_states + + +class Flux2PosEmbed(nn.Module): + # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 + def __init__(self, theta: int, axes_dim: List[int]): + super().__init__() + self.theta = theta + self.axes_dim = axes_dim + + def forward(self, ids: torch.Tensor) -> torch.Tensor: + # Expected ids shape: [S, len(self.axes_dim)] + cos_out = [] + sin_out = [] + pos = ids.float() + is_mps = ids.device.type == "mps" + is_npu = ids.device.type == "npu" + freqs_dtype = torch.float32 if (is_mps or is_npu) else torch.float64 + # Unlike Flux 1, loop over len(self.axes_dim) rather than ids.shape[-1] + for i in range(len(self.axes_dim)): + cos, sin = get_1d_rotary_pos_embed( + self.axes_dim[i], + pos[..., i], + theta=self.theta, + repeat_interleave_real=True, + use_real=True, + freqs_dtype=freqs_dtype, + ) + cos_out.append(cos) + sin_out.append(sin) + freqs_cos = torch.cat(cos_out, dim=-1).to(ids.device) + freqs_sin = torch.cat(sin_out, dim=-1).to(ids.device) + return freqs_cos, freqs_sin + + +class Flux2TimestepGuidanceEmbeddings(nn.Module): + def __init__(self, in_channels: int = 256, embedding_dim: int = 6144, bias: bool = False): + super().__init__() + + self.time_proj = Timesteps(num_channels=in_channels, flip_sin_to_cos=True, downscale_freq_shift=0) + self.timestep_embedder = TimestepEmbedding( + in_channels=in_channels, time_embed_dim=embedding_dim, sample_proj_bias=bias + ) + + self.guidance_embedder = TimestepEmbedding( + in_channels=in_channels, time_embed_dim=embedding_dim, sample_proj_bias=bias + ) + + def forward(self, timestep: torch.Tensor, guidance: torch.Tensor) -> torch.Tensor: + timesteps_proj = self.time_proj(timestep) + timesteps_emb = self.timestep_embedder(timesteps_proj.to(timestep.dtype)) # (N, D) + + guidance_proj = self.time_proj(guidance) + guidance_emb = self.guidance_embedder(guidance_proj.to(guidance.dtype)) # (N, D) + + time_guidance_emb = timesteps_emb + guidance_emb + + return time_guidance_emb + + +class Flux2Modulation(nn.Module): + def __init__(self, dim: int, mod_param_sets: int = 2, bias: bool = False): + super().__init__() + self.mod_param_sets = mod_param_sets + + self.linear = nn.Linear(dim, dim * 3 * self.mod_param_sets, bias=bias) + self.act_fn = nn.SiLU() + + def forward(self, temb: torch.Tensor) -> Tuple[Tuple[torch.Tensor, torch.Tensor, torch.Tensor], ...]: + mod = self.act_fn(temb) + mod = self.linear(mod) + + if mod.ndim == 2: + mod = mod.unsqueeze(1) + mod_params = torch.chunk(mod, 3 * self.mod_param_sets, dim=-1) + # Return tuple of 3-tuples of modulation params shift/scale/gate + return tuple(mod_params[3 * i : 3 * (i + 1)] for i in range(self.mod_param_sets)) + + +class Flux2Transformer2DModel( + ModelMixin, + ConfigMixin, + FromOriginalModelMixin, +): + """ + The Transformer model introduced in Flux 2. + + Reference: https://blackforestlabs.ai/announcing-black-forest-labs/ + + Args: + patch_size (`int`, defaults to `1`): + Patch size to turn the input data into small patches. + in_channels (`int`, defaults to `128`): + The number of channels in the input. + out_channels (`int`, *optional*, defaults to `None`): + The number of channels in the output. If not specified, it defaults to `in_channels`. + num_layers (`int`, defaults to `8`): + The number of layers of dual stream DiT blocks to use. + num_single_layers (`int`, defaults to `48`): + The number of layers of single stream DiT blocks to use. + attention_head_dim (`int`, defaults to `128`): + The number of dimensions to use for each attention head. + num_attention_heads (`int`, defaults to `48`): + The number of attention heads to use. + joint_attention_dim (`int`, defaults to `15360`): + The number of dimensions to use for the joint attention (embedding/channel dimension of + `encoder_hidden_states`). + pooled_projection_dim (`int`, defaults to `768`): + The number of dimensions to use for the pooled projection. + guidance_embeds (`bool`, defaults to `True`): + Whether to use guidance embeddings for guidance-distilled variant of the model. + axes_dims_rope (`Tuple[int]`, defaults to `(32, 32, 32, 32)`): + The dimensions to use for the rotary positional embeddings. + """ + + _supports_gradient_checkpointing = True + # _no_split_modules = ["Flux2TransformerBlock", "Flux2SingleTransformerBlock"] + # _skip_layerwise_casting_patterns = ["pos_embed", "norm"] + # _repeated_blocks = ["Flux2TransformerBlock", "Flux2SingleTransformerBlock"] + + @register_to_config + def __init__( + self, + patch_size: int = 1, + in_channels: int = 128, + out_channels: Optional[int] = None, + num_layers: int = 8, + num_single_layers: int = 48, + attention_head_dim: int = 128, + num_attention_heads: int = 48, + joint_attention_dim: int = 15360, + timestep_guidance_channels: int = 256, + mlp_ratio: float = 3.0, + axes_dims_rope: Tuple[int, ...] = (32, 32, 32, 32), + rope_theta: int = 2000, + eps: float = 1e-6, + ): + super().__init__() + self.out_channels = out_channels or in_channels + self.inner_dim = num_attention_heads * attention_head_dim + + # 1. Sinusoidal positional embedding for RoPE on image and text tokens + self.pos_embed = Flux2PosEmbed(theta=rope_theta, axes_dim=axes_dims_rope) + + # 2. Combined timestep + guidance embedding + self.time_guidance_embed = Flux2TimestepGuidanceEmbeddings( + in_channels=timestep_guidance_channels, embedding_dim=self.inner_dim, bias=False + ) + + # 3. Modulation (double stream and single stream blocks share modulation parameters, resp.) + # Two sets of shift/scale/gate modulation parameters for the double stream attn and FF sub-blocks + self.double_stream_modulation_img = Flux2Modulation(self.inner_dim, mod_param_sets=2, bias=False) + self.double_stream_modulation_txt = Flux2Modulation(self.inner_dim, mod_param_sets=2, bias=False) + # Only one set of modulation parameters as the attn and FF sub-blocks are run in parallel for single stream + self.single_stream_modulation = Flux2Modulation(self.inner_dim, mod_param_sets=1, bias=False) + + # 4. Input projections + self.x_embedder = nn.Linear(in_channels, self.inner_dim, bias=False) + self.context_embedder = nn.Linear(joint_attention_dim, self.inner_dim, bias=False) + + # 5. Double Stream Transformer Blocks + self.transformer_blocks = nn.ModuleList( + [ + Flux2TransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + mlp_ratio=mlp_ratio, + eps=eps, + bias=False, + ) + for _ in range(num_layers) + ] + ) + + # 6. Single Stream Transformer Blocks + self.single_transformer_blocks = nn.ModuleList( + [ + Flux2SingleTransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + mlp_ratio=mlp_ratio, + eps=eps, + bias=False, + ) + for _ in range(num_single_layers) + ] + ) + + # 7. Output layers + self.norm_out = AdaLayerNormContinuous( + self.inner_dim, self.inner_dim, elementwise_affine=False, eps=eps, bias=False + ) + self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=False) + + self.gradient_checkpointing = False + + self.sp_world_size = 1 + self.sp_world_rank = 0 + + def _set_gradient_checkpointing(self, *args, **kwargs): + if "value" in kwargs: + self.gradient_checkpointing = kwargs["value"] + elif "enable" in kwargs: + self.gradient_checkpointing = kwargs["enable"] + else: + raise ValueError("Invalid set gradient checkpointing") + + def enable_multi_gpus_inference(self,): + self.sp_world_size = get_sequence_parallel_world_size() + self.sp_world_rank = get_sequence_parallel_rank() + self.all_gather = get_sp_group().all_gather + self.set_attn_processor(Flux2MultiGPUsAttnProcessor2_0()) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + timestep: torch.LongTensor = None, + img_ids: torch.Tensor = None, + txt_ids: torch.Tensor = None, + guidance: torch.Tensor = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + return_dict: bool = True, + ) -> Union[torch.Tensor, Transformer2DModelOutput]: + """ + The [`FluxTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): + Input `hidden_states`. + encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + block_controlnet_hidden_states: (`list` of `torch.Tensor`): + A list of tensors that if specified are added to the residuals of transformer blocks. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + # 0. Handle input arguments + if joint_attention_kwargs is not None: + joint_attention_kwargs = joint_attention_kwargs.copy() + lora_scale = joint_attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + num_txt_tokens = encoder_hidden_states.shape[1] + + # 1. Calculate timestep embedding and modulation parameters + timestep = timestep.to(hidden_states.dtype) * 1000 + guidance = guidance.to(hidden_states.dtype) * 1000 + + temb = self.time_guidance_embed(timestep, guidance) + + double_stream_mod_img = self.double_stream_modulation_img(temb) + double_stream_mod_txt = self.double_stream_modulation_txt(temb) + single_stream_mod = self.single_stream_modulation(temb)[0] + + # 2. Input projection for image (hidden_states) and conditioning text (encoder_hidden_states) + hidden_states = self.x_embedder(hidden_states) + encoder_hidden_states = self.context_embedder(encoder_hidden_states) + + # 3. Calculate RoPE embeddings from image and text tokens + # NOTE: the below logic means that we can't support batched inference with images of different resolutions or + # text prompts of differents lengths. Is this a use case we want to support? + if img_ids.ndim == 3: + img_ids = img_ids[0] + if txt_ids.ndim == 3: + txt_ids = txt_ids[0] + + if is_torch_npu_available(): + freqs_cos_image, freqs_sin_image = self.pos_embed(img_ids.cpu()) + image_rotary_emb = (freqs_cos_image.npu(), freqs_sin_image.npu()) + freqs_cos_text, freqs_sin_text = self.pos_embed(txt_ids.cpu()) + text_rotary_emb = (freqs_cos_text.npu(), freqs_sin_text.npu()) + else: + image_rotary_emb = self.pos_embed(img_ids) + text_rotary_emb = self.pos_embed(txt_ids) + concat_rotary_emb = ( + torch.cat([text_rotary_emb[0], image_rotary_emb[0]], dim=0), + torch.cat([text_rotary_emb[1], image_rotary_emb[1]], dim=0), + ) + + # Context Parallel + if self.sp_world_size > 1: + hidden_states = torch.chunk(hidden_states, self.sp_world_size, dim=1)[self.sp_world_rank] + if concat_rotary_emb is not None: + txt_rotary_emb = ( + concat_rotary_emb[0][:encoder_hidden_states.shape[1]], + concat_rotary_emb[1][:encoder_hidden_states.shape[1]] + ) + concat_rotary_emb = ( + torch.chunk(concat_rotary_emb[0][encoder_hidden_states.shape[1]:], self.sp_world_size, dim=0)[self.sp_world_rank], + torch.chunk(concat_rotary_emb[1][encoder_hidden_states.shape[1]:], self.sp_world_size, dim=0)[self.sp_world_rank], + ) + concat_rotary_emb = [torch.cat([_txt_rotary_emb, _image_rotary_emb], dim=0) \ + for _txt_rotary_emb, _image_rotary_emb in zip(txt_rotary_emb, concat_rotary_emb)] + + # 4. Double Stream Transformer Blocks + for index_block, block in enumerate(self.transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + encoder_hidden_states, hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + double_stream_mod_img, + double_stream_mod_txt, + concat_rotary_emb, + joint_attention_kwargs, + **ckpt_kwargs, + ) + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb_mod_params_img=double_stream_mod_img, + temb_mod_params_txt=double_stream_mod_txt, + image_rotary_emb=concat_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # 5. Single Stream Transformer Blocks + for index_block, block in enumerate(self.single_transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + encoder_hidden_states, hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + single_stream_mod, + concat_rotary_emb, + joint_attention_kwargs, + **ckpt_kwargs, + ) + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb_mod_params=single_stream_mod, + image_rotary_emb=concat_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # 6. Output layers + hidden_states = self.norm_out(hidden_states, temb) + output = self.proj_out(hidden_states) + + if self.sp_world_size > 1: + output = self.all_gather(output, dim=1) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) + + @classmethod + def from_pretrained( + cls, pretrained_model_path, subfolder=None, transformer_additional_kwargs={}, + low_cpu_mem_usage=False, torch_dtype=torch.bfloat16 + ): + if subfolder is not None: + pretrained_model_path = os.path.join(pretrained_model_path, subfolder) + print(f"loaded 3D transformer's pretrained weights from {pretrained_model_path} ...") + + config_file = os.path.join(pretrained_model_path, 'config.json') + if not os.path.isfile(config_file): + raise RuntimeError(f"{config_file} does not exist") + with open(config_file, "r") as f: + config = json.load(f) + + from diffusers.utils import WEIGHTS_NAME + model_file = os.path.join(pretrained_model_path, WEIGHTS_NAME) + model_file_safetensors = model_file.replace(".bin", ".safetensors") + + if "dict_mapping" in transformer_additional_kwargs.keys(): + for key in transformer_additional_kwargs["dict_mapping"]: + transformer_additional_kwargs[transformer_additional_kwargs["dict_mapping"][key]] = config[key] + + if low_cpu_mem_usage: + try: + import re + + from diffusers import __version__ as diffusers_version + if diffusers_version >= "0.33.0": + from diffusers.models.model_loading_utils import \ + load_model_dict_into_meta + else: + from diffusers.models.modeling_utils import \ + load_model_dict_into_meta + from diffusers.utils import is_accelerate_available + if is_accelerate_available(): + import accelerate + + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + model = cls.from_config(config, **transformer_additional_kwargs) + + param_device = "cpu" + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + print(model_files_safetensors) + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + filtered_state_dict = {} + for key in state_dict: + if key in model.state_dict() and model.state_dict()[key].size() == state_dict[key].size(): + filtered_state_dict[key] = state_dict[key] + else: + print(f"Skipping key '{key}' due to size mismatch or absence in model.") + + model_keys = set(model.state_dict().keys()) + loaded_keys = set(filtered_state_dict.keys()) + missing_keys = model_keys - loaded_keys + + def initialize_missing_parameters(missing_keys, model_state_dict, torch_dtype=None): + initialized_dict = {} + + with torch.no_grad(): + for key in missing_keys: + param_shape = model_state_dict[key].shape + param_dtype = torch_dtype if torch_dtype is not None else model_state_dict[key].dtype + if "control" in key and key.replace("control_", "") in filtered_state_dict.keys(): + initialized_dict[key] = filtered_state_dict[key.replace("control_", "")].clone() + print(f"Initializing missing parameter '{key}' with model.state_dict().") + elif "after_proj" in key or "before_proj" in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + print(f"Initializing missing parameter '{key}' with zero.") + elif 'weight' in key: + if any(norm_type in key for norm_type in ['norm', 'ln_', 'layer_norm', 'group_norm', 'batch_norm']): + initialized_dict[key] = torch.ones(param_shape, dtype=param_dtype) + elif 'embedding' in key or 'embed' in key: + initialized_dict[key] = torch.randn(param_shape, dtype=param_dtype) * 0.02 + elif 'head' in key or 'output' in key or 'proj_out' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif len(param_shape) >= 2: + initialized_dict[key] = torch.empty(param_shape, dtype=param_dtype) + nn.init.xavier_uniform_(initialized_dict[key]) + else: + initialized_dict[key] = torch.randn(param_shape, dtype=param_dtype) * 0.02 + elif 'bias' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif 'running_mean' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif 'running_var' in key: + initialized_dict[key] = torch.ones(param_shape, dtype=param_dtype) + elif 'num_batches_tracked' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=torch.long) + else: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + + return initialized_dict + + if missing_keys: + print(f"Missing keys will be initialized: {sorted(missing_keys)}") + initialized_params = initialize_missing_parameters( + missing_keys, + model.state_dict(), + torch_dtype + ) + filtered_state_dict.update(initialized_params) + + if diffusers_version >= "0.33.0": + # Diffusers has refactored `load_model_dict_into_meta` since version 0.33.0 in this commit: + # https://github.com/huggingface/diffusers/commit/f5929e03060d56063ff34b25a8308833bec7c785. + load_model_dict_into_meta( + model, + filtered_state_dict, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + else: + model._convert_deprecated_attention_blocks(filtered_state_dict) + unexpected_keys = load_model_dict_into_meta( + model, + filtered_state_dict, + device=param_device, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + + if cls._keys_to_ignore_on_load_unexpected is not None: + for pat in cls._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + print( + f"Some weights of the model checkpoint were not used when initializing {cls.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + + params = [p.numel() if "." in n else 0 for n, p in model.named_parameters()] + print(f"### All Parameters: {sum(params) / 1e6} M") + + params = [p.numel() if "attn1." in n else 0 for n, p in model.named_parameters()] + print(f"### attn1 Parameters: {sum(params) / 1e6} M") + return model + except Exception as e: + print( + f"The low_cpu_mem_usage mode is not work because {e}. Use low_cpu_mem_usage=False instead." + ) + + model = cls.from_config(config, **transformer_additional_kwargs) + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + tmp_state_dict = {} + for key in state_dict: + if key in model.state_dict().keys() and model.state_dict()[key].size() == state_dict[key].size(): + tmp_state_dict[key] = state_dict[key] + else: + print(key, "Size don't match, skip") + + state_dict = tmp_state_dict + + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m) + + params = [p.numel() if "." in n else 0 for n, p in model.named_parameters()] + print(f"### All Parameters: {sum(params) / 1e6} M") + + params = [p.numel() if "attn1." in n else 0 for n, p in model.named_parameters()] + print(f"### attn1 Parameters: {sum(params) / 1e6} M") + + model = model.to(torch_dtype) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/flux2_transformer2d_control.py b/VideoX-Fun/videox_fun/models/flux2_transformer2d_control.py new file mode 100644 index 0000000000000000000000000000000000000000..2b356845671bb3018b6d352b31c1fc147a03ddd4 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/flux2_transformer2d_control.py @@ -0,0 +1,312 @@ +# Modified from https://github.com/ali-vilab/VACE/blob/main/vace/models/wan/wan_vace.py +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. + +import glob +import inspect +import json +import os +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin +from diffusers.models.attention_processor import Attention, AttentionProcessor +from diffusers.models.embeddings import (TimestepEmbedding, Timesteps, + apply_rotary_emb, + get_1d_rotary_pos_embed) +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import AdaLayerNormContinuous +from diffusers.utils import (USE_PEFT_BACKEND, is_torch_npu_available, + is_torch_version, logging, scale_lora_layers, + unscale_lora_layers) + +from .flux2_transformer2d import (Flux2SingleTransformerBlock, + Flux2Transformer2DModel, + Flux2TransformerBlock) + + +class Flux2ControlTransformerBlock(Flux2TransformerBlock): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float = 3.0, + eps: float = 1e-6, + bias: bool = False, + block_id=0 + ): + super().__init__(dim, num_attention_heads, attention_head_dim, mlp_ratio, eps, bias) + self.block_id = block_id + if block_id == 0: + self.before_proj = nn.Linear(dim, dim) + nn.init.zeros_(self.before_proj.weight) + nn.init.zeros_(self.before_proj.bias) + self.after_proj = nn.Linear(dim, dim) + nn.init.zeros_(self.after_proj.weight) + nn.init.zeros_(self.after_proj.bias) + + def forward(self, c, x, **kwargs): + if self.block_id == 0: + c = self.before_proj(c) + x + all_c = [] + else: + all_c = list(torch.unbind(c)) + c = all_c.pop(-1) + + encoder_hidden_states, c = super().forward(c, **kwargs) + c_skip = self.after_proj(c) + all_c += [c_skip, c] + c = torch.stack(all_c) + return encoder_hidden_states, c + + +class BaseFlux2TransformerBlock(Flux2TransformerBlock): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float = 3.0, + eps: float = 1e-6, + bias: bool = False, + block_id=0 + ): + super().__init__(dim, num_attention_heads, attention_head_dim, mlp_ratio, eps, bias) + self.block_id = block_id + + def forward(self, hidden_states, hints=None, context_scale=1.0, **kwargs): + encoder_hidden_states, hidden_states = super().forward(hidden_states, **kwargs) + if self.block_id is not None: + hidden_states = hidden_states + hints[self.block_id] * context_scale + return encoder_hidden_states, hidden_states + + +class Flux2ControlTransformer2DModel(Flux2Transformer2DModel): + @register_to_config + def __init__( + self, + control_layers=None, + control_in_dim=None, + patch_size: int = 1, + in_channels: int = 128, + out_channels: Optional[int] = None, + num_layers: int = 8, + num_single_layers: int = 48, + attention_head_dim: int = 128, + num_attention_heads: int = 48, + joint_attention_dim: int = 15360, + timestep_guidance_channels: int = 256, + mlp_ratio: float = 3.0, + axes_dims_rope: Tuple[int, ...] = (32, 32, 32, 32), + rope_theta: int = 2000, + eps: float = 1e-6, + ): + super().__init__( + patch_size, in_channels, out_channels, num_layers, num_single_layers, attention_head_dim, + num_attention_heads, joint_attention_dim, timestep_guidance_channels, mlp_ratio, axes_dims_rope, + rope_theta, eps + ) + + self.control_layers = [i for i in range(0, self.num_layers, 2)] if control_layers is None else control_layers + self.control_in_dim = self.in_dim if control_in_dim is None else control_in_dim + + assert 0 in self.control_layers + self.control_layers_mapping = {i: n for n, i in enumerate(self.control_layers)} + + # blocks + del self.transformer_blocks + self.transformer_blocks = nn.ModuleList( + [ + BaseFlux2TransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + mlp_ratio=mlp_ratio, + eps=eps, + block_id=self.control_layers_mapping[i] if i in self.control_layers else None + ) + for i in range(num_layers) + ] + ) + + # control blocks + self.control_transformer_blocks = nn.ModuleList( + [ + Flux2ControlTransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + mlp_ratio=mlp_ratio, + eps=eps, + block_id=i + ) + for i in self.control_layers + ] + ) + + # control patch embeddings + self.control_img_in = nn.Linear(self.control_in_dim, self.inner_dim) + + def forward_control( + self, + x, + control_context, + kwargs + ): + # embeddings + c = self.control_img_in(control_context) + # Context Parallel + if self.sp_world_size > 1: + c = torch.chunk(c, self.sp_world_size, dim=1)[self.sp_world_rank] + + # arguments + new_kwargs = dict(x=x) + new_kwargs.update(kwargs) + + for block in self.control_transformer_blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module, **static_kwargs): + def custom_forward(*inputs): + return module(*inputs, **static_kwargs) + return custom_forward + ckpt_kwargs = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + encoder_hidden_states, c = torch.utils.checkpoint.checkpoint( + create_custom_forward(block, **new_kwargs), + c, + **ckpt_kwargs, + ) + else: + encoder_hidden_states, c = block(c, **new_kwargs) + new_kwargs["encoder_hidden_states"] = encoder_hidden_states + + hints = torch.unbind(c)[:-1] + return hints + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + timestep: torch.LongTensor = None, + img_ids: torch.Tensor = None, + txt_ids: torch.Tensor = None, + guidance: torch.Tensor = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + control_context=None, + control_context_scale=1.0, + return_dict: bool = True, + ): + num_txt_tokens = encoder_hidden_states.shape[1] + + # 1. Calculate timestep embedding and modulation parameters + timestep = timestep.to(hidden_states.dtype) * 1000 + guidance = guidance.to(hidden_states.dtype) * 1000 + + temb = self.time_guidance_embed(timestep, guidance) + + double_stream_mod_img = self.double_stream_modulation_img(temb) + double_stream_mod_txt = self.double_stream_modulation_txt(temb) + single_stream_mod = self.single_stream_modulation(temb)[0] + + # 2. Input projection for image (hidden_states) and conditioning text (encoder_hidden_states) + hidden_states = self.x_embedder(hidden_states) + encoder_hidden_states = self.context_embedder(encoder_hidden_states) + + # 3. Calculate RoPE embeddings from image and text tokens + # NOTE: the below logic means that we can't support batched inference with images of different resolutions or + # text prompts of differents lengths. Is this a use case we want to support? + if img_ids.ndim == 3: + img_ids = img_ids[0] + if txt_ids.ndim == 3: + txt_ids = txt_ids[0] + + if is_torch_npu_available(): + freqs_cos_image, freqs_sin_image = self.pos_embed(img_ids.cpu()) + image_rotary_emb = (freqs_cos_image.npu(), freqs_sin_image.npu()) + freqs_cos_text, freqs_sin_text = self.pos_embed(txt_ids.cpu()) + text_rotary_emb = (freqs_cos_text.npu(), freqs_sin_text.npu()) + else: + image_rotary_emb = self.pos_embed(img_ids) + text_rotary_emb = self.pos_embed(txt_ids) + concat_rotary_emb = ( + torch.cat([text_rotary_emb[0], image_rotary_emb[0]], dim=0), + torch.cat([text_rotary_emb[1], image_rotary_emb[1]], dim=0), + ) + + # Arguments + kwargs = dict( + encoder_hidden_states=encoder_hidden_states, + temb_mod_params_img=double_stream_mod_img, + temb_mod_params_txt=double_stream_mod_txt, + image_rotary_emb=concat_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + hints = self.forward_control( + hidden_states, control_context, kwargs + ) + + for index_block, block in enumerate(self.transformer_blocks): + # Arguments + kwargs = dict( + encoder_hidden_states=encoder_hidden_states, + temb_mod_params_img=double_stream_mod_img, + temb_mod_params_txt=double_stream_mod_txt, + image_rotary_emb=concat_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + hints=hints, + context_scale=control_context_scale + ) + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module, **static_kwargs): + def custom_forward(*inputs): + return module(*inputs, **static_kwargs) + return custom_forward + + ckpt_kwargs = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + + encoder_hidden_states, hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block, **kwargs), + hidden_states, + **ckpt_kwargs, + ) + else: + encoder_hidden_states, hidden_states = block(hidden_states, **kwargs) + + for index_block, block in enumerate(self.single_transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + encoder_hidden_states, hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + single_stream_mod, + concat_rotary_emb, + joint_attention_kwargs, + **ckpt_kwargs, + ) + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb_mod_params=single_stream_mod, + image_rotary_emb=concat_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # 6. Output layers + hidden_states = self.norm_out(hidden_states, temb) + output = self.proj_out(hidden_states) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) diff --git a/VideoX-Fun/videox_fun/models/flux2_vae.py b/VideoX-Fun/videox_fun/models/flux2_vae.py new file mode 100644 index 0000000000000000000000000000000000000000..76326c52c9c210161f5d7076601aeb49f8779ee8 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/flux2_vae.py @@ -0,0 +1,543 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/autoencoders/autoencoder_kl_flux2.py +# Copyright 2025 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from typing import Dict, Optional, Tuple, Union + +import torch +import torch.nn as nn +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, CROSS_ATTENTION_PROCESSORS, Attention, + AttentionProcessor, AttnAddedKVProcessor, AttnProcessor, + FusedAttnProcessor2_0) +from diffusers.models.autoencoders.vae import (Decoder, + DecoderOutput, + DiagonalGaussianDistribution, + Encoder) +from diffusers.models.modeling_outputs import AutoencoderKLOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils import deprecate +from diffusers.utils.accelerate_utils import apply_forward_hook + + +class AutoencoderKLFlux2(ModelMixin, ConfigMixin, FromOriginalModelMixin): + r""" + A VAE model with KL loss for encoding images into latents and decoding latent representations into images. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + in_channels (int, *optional*, defaults to 3): Number of channels in the input image. + out_channels (int, *optional*, defaults to 3): Number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + Tuple of downsample block types. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + Tuple of upsample block types. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): + Tuple of block output channels. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + latent_channels (`int`, *optional*, defaults to 4): Number of channels in the latent space. + sample_size (`int`, *optional*, defaults to `32`): Sample input size. + force_upcast (`bool`, *optional*, default to `True`): + If enabled it will force the VAE to run in float32 for high image resolution pipelines, such as SD-XL. VAE + can be fine-tuned / trained to a lower range without losing too much precision in which case `force_upcast` + can be set to `False` - see: https://huggingface.co/madebyollin/sdxl-vae-fp16-fix + mid_block_add_attention (`bool`, *optional*, default to `True`): + If enabled, the mid_block of the Encoder and Decoder will have attention blocks. If set to false, the + mid_block will only have resnet blocks + """ + + _supports_gradient_checkpointing = True + _no_split_modules = ["BasicTransformerBlock", "ResnetBlock2D"] + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str, ...] = ( + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + ), + up_block_types: Tuple[str, ...] = ( + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + ), + block_out_channels: Tuple[int, ...] = ( + 128, + 256, + 512, + 512, + ), + layers_per_block: int = 2, + act_fn: str = "silu", + latent_channels: int = 32, + norm_num_groups: int = 32, + sample_size: int = 1024, # YiYi notes: not sure + force_upcast: bool = True, + use_quant_conv: bool = True, + use_post_quant_conv: bool = True, + mid_block_add_attention: bool = True, + batch_norm_eps: float = 1e-4, + batch_norm_momentum: float = 0.1, + patch_size: Tuple[int, int] = (2, 2), + ): + super().__init__() + + # pass init params to Encoder + self.encoder = Encoder( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + norm_num_groups=norm_num_groups, + double_z=True, + mid_block_add_attention=mid_block_add_attention, + ) + + # pass init params to Decoder + self.decoder = Decoder( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + norm_num_groups=norm_num_groups, + act_fn=act_fn, + mid_block_add_attention=mid_block_add_attention, + ) + + self.quant_conv = nn.Conv2d(2 * latent_channels, 2 * latent_channels, 1) if use_quant_conv else None + self.post_quant_conv = nn.Conv2d(latent_channels, latent_channels, 1) if use_post_quant_conv else None + + self.bn = nn.BatchNorm2d( + math.prod(patch_size) * latent_channels, + eps=batch_norm_eps, + momentum=batch_norm_momentum, + affine=False, + track_running_stats=True, + ) + + self.use_slicing = False + self.use_tiling = False + + # only relevant if vae tiling is enabled + self.tile_sample_min_size = self.config.sample_size + sample_size = ( + self.config.sample_size[0] + if isinstance(self.config.sample_size, (list, tuple)) + else self.config.sample_size + ) + self.tile_latent_min_size = int(sample_size / (2 ** (len(self.config.block_out_channels) - 1))) + self.tile_overlap_factor = 0.25 + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, height, width = x.shape + + if self.use_tiling and (width > self.tile_sample_min_size or height > self.tile_sample_min_size): + return self._tiled_encode(x) + + enc = self.encoder(x) + if self.quant_conv is not None: + enc = self.quant_conv(enc) + + return enc + + @apply_forward_hook + def encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + """ + Encode a batch of images into latents. + + Args: + x (`torch.Tensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + The latent representations of the encoded images. If `return_dict` is True, a + [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. + """ + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [self._encode(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = self._encode(x) + + posterior = DiagonalGaussianDistribution(h) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + if self.use_tiling and (z.shape[-1] > self.tile_latent_min_size or z.shape[-2] > self.tile_latent_min_size): + return self.tiled_decode(z, return_dict=return_dict) + + if self.post_quant_conv is not None: + z = self.post_quant_conv(z) + + dec = self.decoder(z) + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + @apply_forward_hook + def decode( + self, z: torch.FloatTensor, return_dict: bool = True, generator=None + ) -> Union[DecoderOutput, torch.FloatTensor]: + """ + Decode a batch of images. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + + """ + if self.use_slicing and z.shape[0] > 1: + decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)] + decoded = torch.cat(decoded_slices) + else: + decoded = self._decode(z).sample + + if not return_dict: + return (decoded,) + + return DecoderOutput(sample=decoded) + + def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[2], b.shape[2], blend_extent) + for y in range(blend_extent): + b[:, :, y, :] = a[:, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, y, :] * (y / blend_extent) + return b + + def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[3], b.shape[3], blend_extent) + for x in range(blend_extent): + b[:, :, :, x] = a[:, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, x] * (x / blend_extent) + return b + + def _tiled_encode(self, x: torch.Tensor) -> torch.Tensor: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled encoding is + different from non-tiled encoding because each tile uses a different encoder. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. You may still see tile-sized changes in the + output, but they should be much less noticeable. + + Args: + x (`torch.Tensor`): Input batch of images. + + Returns: + `torch.Tensor`: + The latent representation of the encoded videos. + """ + + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + # Split the image into 512x512 tiles and encode them separately. + rows = [] + for i in range(0, x.shape[2], overlap_size): + row = [] + for j in range(0, x.shape[3], overlap_size): + tile = x[:, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size] + tile = self.encoder(tile) + if self.config.use_quant_conv: + tile = self.quant_conv(tile) + row.append(tile) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + enc = torch.cat(result_rows, dim=2) + return enc + + def tiled_encode(self, x: torch.Tensor, return_dict: bool = True) -> AutoencoderKLOutput: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled encoding is + different from non-tiled encoding because each tile uses a different encoder. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. You may still see tile-sized changes in the + output, but they should be much less noticeable. + + Args: + x (`torch.Tensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + [`~models.autoencoder_kl.AutoencoderKLOutput`] or `tuple`: + If return_dict is True, a [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain + `tuple` is returned. + """ + deprecation_message = ( + "The tiled_encode implementation supporting the `return_dict` parameter is deprecated. In the future, the " + "implementation of this method will be replaced with that of `_tiled_encode` and you will no longer be able " + "to pass `return_dict`. You will also have to create a `DiagonalGaussianDistribution()` from the returned value." + ) + deprecate("tiled_encode", "1.0.0", deprecation_message, standard_warn=False) + + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + # Split the image into 512x512 tiles and encode them separately. + rows = [] + for i in range(0, x.shape[2], overlap_size): + row = [] + for j in range(0, x.shape[3], overlap_size): + tile = x[:, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size] + tile = self.encoder(tile) + if self.config.use_quant_conv: + tile = self.quant_conv(tile) + row.append(tile) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + moments = torch.cat(result_rows, dim=2) + posterior = DiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + r""" + Decode a batch of images using a tiled decoder. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor) + row_limit = self.tile_sample_min_size - blend_extent + + # Split z into overlapping 64x64 tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, z.shape[2], overlap_size): + row = [] + for j in range(0, z.shape[3], overlap_size): + tile = z[:, :, i : i + self.tile_latent_min_size, j : j + self.tile_latent_min_size] + if self.config.use_post_quant_conv: + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile) + row.append(decoded) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + dec = torch.cat(result_rows, dim=2) + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + ) -> Union[DecoderOutput, torch.Tensor]: + r""" + Args: + sample (`torch.Tensor`): Input sample. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample from the posterior. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z).sample + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.fuse_qkv_projections + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, key, value) + are fused. For cross-attention modules, key and value projection matrices are fused. + + > [!WARNING] > This API is 🧪 experimental. + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + self.set_attn_processor(FusedAttnProcessor2_0()) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.unfuse_qkv_projections + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + > [!WARNING] > This API is 🧪 experimental. + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/flux_transformer2d.py b/VideoX-Fun/videox_fun/models/flux_transformer2d.py new file mode 100644 index 0000000000000000000000000000000000000000..b77175b5d0403c62642ebfd9fd0b6588fc66bbe4 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/flux_transformer2d.py @@ -0,0 +1,832 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_flux.py +# Copyright 2025 Black Forest Labs, The HuggingFace Team and The InstantX Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.attention import FeedForward +from diffusers.models.attention_processor import AttentionProcessor +from diffusers.models.embeddings import ( + CombinedTimestepGuidanceTextProjEmbeddings, + CombinedTimestepTextProjEmbeddings, get_1d_rotary_pos_embed) +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import (AdaLayerNormContinuous, + AdaLayerNormZero, + AdaLayerNormZeroSingle) +from diffusers.utils import (USE_PEFT_BACKEND, is_torch_version, logging, + scale_lora_layers, unscale_lora_layers) +from diffusers.utils.torch_utils import maybe_allow_in_graph + +from ..dist import (FluxMultiGPUsAttnProcessor2_0, get_sequence_parallel_rank, + get_sequence_parallel_world_size, get_sp_group) +from .attention_utils import attention + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +def _get_projections(attn: "FluxAttention", hidden_states, encoder_hidden_states=None): + query = attn.to_q(hidden_states) + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + + encoder_query = encoder_key = encoder_value = None + if encoder_hidden_states is not None and attn.added_kv_proj_dim is not None: + encoder_query = attn.add_q_proj(encoder_hidden_states) + encoder_key = attn.add_k_proj(encoder_hidden_states) + encoder_value = attn.add_v_proj(encoder_hidden_states) + + return query, key, value, encoder_query, encoder_key, encoder_value + +def _get_qkv_projections(attn: "FluxAttention", hidden_states, encoder_hidden_states=None): + return _get_projections(attn, hidden_states, encoder_hidden_states) + +def apply_rotary_emb( + x: torch.Tensor, + freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], + use_real: bool = True, + use_real_unbind_dim: int = -1, + sequence_dim: int = 2, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings + to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are + reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting + tensors contain rotary embeddings and are returned as real tensors. + + Args: + x (`torch.Tensor`): + Query or key tensor to apply rotary embeddings. [B, H, S, D] xk (torch.Tensor): Key tensor to apply + freqs_cis (`Tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],) + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings. + """ + if use_real: + cos, sin = freqs_cis # [S, D] + if sequence_dim == 2: + cos = cos[None, None, :, :] + sin = sin[None, None, :, :] + elif sequence_dim == 1: + cos = cos[None, :, None, :] + sin = sin[None, :, None, :] + else: + raise ValueError(f"`sequence_dim={sequence_dim}` but should be 1 or 2.") + + cos, sin = cos.to(x.device), sin.to(x.device) + + if use_real_unbind_dim == -1: + # Used for flux, cogvideox, hunyuan-dit + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, H, S, D//2] + x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3) + elif use_real_unbind_dim == -2: + # Used for Stable Audio, OmniGen, CogView4 and Cosmos + x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) # [B, H, S, D//2] + x_rotated = torch.cat([-x_imag, x_real], dim=-1) + else: + raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") + + out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + + return out + else: + # used for lumina + x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2)) + freqs_cis = freqs_cis.unsqueeze(2) + x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3) + + return x_out.type_as(x) + + +class FluxAttnProcessor: + _attention_backend = None + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError(f"{self.__class__.__name__} requires PyTorch 2.0. Please upgrade your pytorch version.") + + def __call__( + self, + attn: "FluxAttention", + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + attention_mask: Optional[torch.Tensor] = None, + image_rotary_emb: Optional[torch.Tensor] = None, + text_seq_len: int = None, + ) -> torch.Tensor: + query, key, value, encoder_query, encoder_key, encoder_value = _get_qkv_projections( + attn, hidden_states, encoder_hidden_states + ) + + query = query.unflatten(-1, (attn.heads, -1)) + key = key.unflatten(-1, (attn.heads, -1)) + value = value.unflatten(-1, (attn.heads, -1)) + + query = attn.norm_q(query) + key = attn.norm_k(key) + + if attn.added_kv_proj_dim is not None: + encoder_query = encoder_query.unflatten(-1, (attn.heads, -1)) + encoder_key = encoder_key.unflatten(-1, (attn.heads, -1)) + encoder_value = encoder_value.unflatten(-1, (attn.heads, -1)) + + encoder_query = attn.norm_added_q(encoder_query) + encoder_key = attn.norm_added_k(encoder_key) + + query = torch.cat([encoder_query, query], dim=1) + key = torch.cat([encoder_key, key], dim=1) + value = torch.cat([encoder_value, value], dim=1) + + if image_rotary_emb is not None: + query = apply_rotary_emb(query, image_rotary_emb, sequence_dim=1) + key = apply_rotary_emb(key, image_rotary_emb, sequence_dim=1) + + hidden_states = attention( + query, key, value, attn_mask=attention_mask, + ) + hidden_states = hidden_states.flatten(2, 3) + hidden_states = hidden_states.to(query.dtype) + + if encoder_hidden_states is not None: + encoder_hidden_states, hidden_states = hidden_states.split_with_sizes( + [encoder_hidden_states.shape[1], hidden_states.shape[1] - encoder_hidden_states.shape[1]], dim=1 + ) + hidden_states = attn.to_out[0](hidden_states) + hidden_states = attn.to_out[1](hidden_states) + encoder_hidden_states = attn.to_add_out(encoder_hidden_states) + + return hidden_states, encoder_hidden_states + else: + return hidden_states + + +class FluxAttention(torch.nn.Module): + _default_processor_cls = FluxAttnProcessor + _available_processors = [ + FluxAttnProcessor, + ] + + def __init__( + self, + query_dim: int, + heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + bias: bool = False, + added_kv_proj_dim: Optional[int] = None, + added_proj_bias: Optional[bool] = True, + out_bias: bool = True, + eps: float = 1e-5, + out_dim: int = None, + context_pre_only: Optional[bool] = None, + pre_only: bool = False, + elementwise_affine: bool = True, + processor=None, + ): + super().__init__() + + self.head_dim = dim_head + self.inner_dim = out_dim if out_dim is not None else dim_head * heads + self.query_dim = query_dim + self.use_bias = bias + self.dropout = dropout + self.out_dim = out_dim if out_dim is not None else query_dim + self.context_pre_only = context_pre_only + self.pre_only = pre_only + self.heads = out_dim // dim_head if out_dim is not None else heads + self.added_kv_proj_dim = added_kv_proj_dim + self.added_proj_bias = added_proj_bias + + self.norm_q = torch.nn.RMSNorm(dim_head, eps=eps, elementwise_affine=elementwise_affine) + self.norm_k = torch.nn.RMSNorm(dim_head, eps=eps, elementwise_affine=elementwise_affine) + self.to_q = torch.nn.Linear(query_dim, self.inner_dim, bias=bias) + self.to_k = torch.nn.Linear(query_dim, self.inner_dim, bias=bias) + self.to_v = torch.nn.Linear(query_dim, self.inner_dim, bias=bias) + + if not self.pre_only: + self.to_out = torch.nn.ModuleList([]) + self.to_out.append(torch.nn.Linear(self.inner_dim, self.out_dim, bias=out_bias)) + self.to_out.append(torch.nn.Dropout(dropout)) + + if added_kv_proj_dim is not None: + self.norm_added_q = torch.nn.RMSNorm(dim_head, eps=eps) + self.norm_added_k = torch.nn.RMSNorm(dim_head, eps=eps) + self.add_q_proj = torch.nn.Linear(added_kv_proj_dim, self.inner_dim, bias=added_proj_bias) + self.add_k_proj = torch.nn.Linear(added_kv_proj_dim, self.inner_dim, bias=added_proj_bias) + self.add_v_proj = torch.nn.Linear(added_kv_proj_dim, self.inner_dim, bias=added_proj_bias) + self.to_add_out = torch.nn.Linear(self.inner_dim, query_dim, bias=out_bias) + + if processor is None: + self.processor = self._default_processor_cls() + else: + self.processor = processor + + def set_processor(self, processor: "AttnProcessor") -> None: + r""" + Set the attention processor to use. + + Args: + processor (`AttnProcessor`): + The attention processor to use. + """ + # if current processor is in `self._modules` and if passed `processor` is not, we need to + # pop `processor` from `self._modules` + if ( + hasattr(self, "processor") + and isinstance(self.processor, torch.nn.Module) + and not isinstance(processor, torch.nn.Module) + ): + logger.info(f"You are removing possibly trained weights of {self.processor} with {processor}") + self._modules.pop("processor") + + self.processor = processor + + def get_processor(self, return_deprecated_lora: bool = False) -> "AttentionProcessor": + r""" + Get the attention processor in use. + + Args: + return_deprecated_lora (`bool`, *optional*, defaults to `False`): + Set to `True` to return the deprecated LoRA attention processor. + + Returns: + "AttentionProcessor": The attention processor in use. + """ + if not return_deprecated_lora: + return self.processor + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + image_rotary_emb: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + attn_parameters = set(inspect.signature(self.processor.__call__).parameters.keys()) + quiet_attn_parameters = {"ip_adapter_masks", "ip_hidden_states"} + unused_kwargs = [k for k, _ in kwargs.items() if k not in attn_parameters and k not in quiet_attn_parameters] + if len(unused_kwargs) > 0: + logger.warning( + f"joint_attention_kwargs {unused_kwargs} are not expected by {self.processor.__class__.__name__} and will be ignored." + ) + kwargs = {k: w for k, w in kwargs.items() if k in attn_parameters} + return self.processor(self, hidden_states, encoder_hidden_states, attention_mask, image_rotary_emb, **kwargs) + + +@maybe_allow_in_graph +class FluxSingleTransformerBlock(nn.Module): + def __init__(self, dim: int, num_attention_heads: int, attention_head_dim: int, mlp_ratio: float = 4.0): + super().__init__() + self.mlp_hidden_dim = int(dim * mlp_ratio) + + self.norm = AdaLayerNormZeroSingle(dim) + self.proj_mlp = nn.Linear(dim, self.mlp_hidden_dim) + self.act_mlp = nn.GELU(approximate="tanh") + self.proj_out = nn.Linear(dim + self.mlp_hidden_dim, dim) + + self.attn = FluxAttention( + query_dim=dim, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=dim, + bias=True, + processor=FluxAttnProcessor(), + eps=1e-6, + pre_only=True, + ) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + text_seq_len = encoder_hidden_states.shape[1] + hidden_states = torch.cat([encoder_hidden_states, hidden_states], dim=1) + + residual = hidden_states + norm_hidden_states, gate = self.norm(hidden_states, emb=temb) + mlp_hidden_states = self.act_mlp(self.proj_mlp(norm_hidden_states)) + joint_attention_kwargs = joint_attention_kwargs or {} + attn_output = self.attn( + hidden_states=norm_hidden_states, + image_rotary_emb=image_rotary_emb, + text_seq_len=text_seq_len, + **joint_attention_kwargs, + ) + hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2) + gate = gate.unsqueeze(1) + hidden_states = gate * self.proj_out(hidden_states) + hidden_states = residual + hidden_states + if hidden_states.dtype == torch.float16: + hidden_states = hidden_states.clip(-65504, 65504) + + encoder_hidden_states, hidden_states = hidden_states[:, :text_seq_len], hidden_states[:, text_seq_len:] + return encoder_hidden_states, hidden_states + + +@maybe_allow_in_graph +class FluxTransformerBlock(nn.Module): + def __init__( + self, dim: int, num_attention_heads: int, attention_head_dim: int, qk_norm: str = "rms_norm", eps: float = 1e-6 + ): + super().__init__() + + self.norm1 = AdaLayerNormZero(dim) + self.norm1_context = AdaLayerNormZero(dim) + + self.attn = FluxAttention( + query_dim=dim, + added_kv_proj_dim=dim, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=dim, + context_pre_only=False, + bias=True, + processor=FluxAttnProcessor(), + eps=eps, + ) + + self.norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") + + self.norm2_context = nn.LayerNorm(dim, elementwise_affine=False, eps=1e-6) + self.ff_context = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb) + + norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context( + encoder_hidden_states, emb=temb + ) + joint_attention_kwargs = joint_attention_kwargs or {} + + # Attention. + attention_outputs = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + image_rotary_emb=image_rotary_emb, + **joint_attention_kwargs, + ) + + if len(attention_outputs) == 2: + attn_output, context_attn_output = attention_outputs + elif len(attention_outputs) == 3: + attn_output, context_attn_output, ip_attn_output = attention_outputs + + # Process attention outputs for the `hidden_states`. + attn_output = gate_msa.unsqueeze(1) * attn_output + hidden_states = hidden_states + attn_output + + norm_hidden_states = self.norm2(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + + ff_output = self.ff(norm_hidden_states) + ff_output = gate_mlp.unsqueeze(1) * ff_output + + hidden_states = hidden_states + ff_output + if len(attention_outputs) == 3: + hidden_states = hidden_states + ip_attn_output + + # Process attention outputs for the `encoder_hidden_states`. + context_attn_output = c_gate_msa.unsqueeze(1) * context_attn_output + encoder_hidden_states = encoder_hidden_states + context_attn_output + + norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states) + norm_encoder_hidden_states = norm_encoder_hidden_states * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None] + + context_ff_output = self.ff_context(norm_encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states + c_gate_mlp.unsqueeze(1) * context_ff_output + if encoder_hidden_states.dtype == torch.float16: + encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) + + return encoder_hidden_states, hidden_states + + +class FluxPosEmbed(nn.Module): + # modified from https://github.com/black-forest-labs/flux/blob/c00d7c60b085fce8058b9df845e036090873f2ce/src/flux/modules/layers.py#L11 + def __init__(self, theta: int, axes_dim: List[int]): + super().__init__() + self.theta = theta + self.axes_dim = axes_dim + + def forward(self, ids: torch.Tensor) -> torch.Tensor: + n_axes = ids.shape[-1] + cos_out = [] + sin_out = [] + pos = ids.float() + is_mps = ids.device.type == "mps" + is_npu = ids.device.type == "npu" + freqs_dtype = torch.float32 if (is_mps or is_npu) else torch.float64 + for i in range(n_axes): + cos, sin = get_1d_rotary_pos_embed( + self.axes_dim[i], + pos[:, i], + theta=self.theta, + repeat_interleave_real=True, + use_real=True, + freqs_dtype=freqs_dtype, + ) + cos_out.append(cos) + sin_out.append(sin) + freqs_cos = torch.cat(cos_out, dim=-1).to(ids.device) + freqs_sin = torch.cat(sin_out, dim=-1).to(ids.device) + return freqs_cos, freqs_sin + + +class FluxTransformer2DModel( + ModelMixin, + ConfigMixin, + PeftAdapterMixin, + FromOriginalModelMixin, +): + """ + The Transformer model introduced in Flux. + + Reference: https://blackforestlabs.ai/announcing-black-forest-labs/ + + Args: + patch_size (`int`, defaults to `1`): + Patch size to turn the input data into small patches. + in_channels (`int`, defaults to `64`): + The number of channels in the input. + out_channels (`int`, *optional*, defaults to `None`): + The number of channels in the output. If not specified, it defaults to `in_channels`. + num_layers (`int`, defaults to `19`): + The number of layers of dual stream DiT blocks to use. + num_single_layers (`int`, defaults to `38`): + The number of layers of single stream DiT blocks to use. + attention_head_dim (`int`, defaults to `128`): + The number of dimensions to use for each attention head. + num_attention_heads (`int`, defaults to `24`): + The number of attention heads to use. + joint_attention_dim (`int`, defaults to `4096`): + The number of dimensions to use for the joint attention (embedding/channel dimension of + `encoder_hidden_states`). + pooled_projection_dim (`int`, defaults to `768`): + The number of dimensions to use for the pooled projection. + guidance_embeds (`bool`, defaults to `False`): + Whether to use guidance embeddings for guidance-distilled variant of the model. + axes_dims_rope (`Tuple[int]`, defaults to `(16, 56, 56)`): + The dimensions to use for the rotary positional embeddings. + """ + + _supports_gradient_checkpointing = True + # _no_split_modules = ["FluxTransformerBlock", "FluxSingleTransformerBlock"] + # _skip_layerwise_casting_patterns = ["pos_embed", "norm"] + # _repeated_blocks = ["FluxTransformerBlock", "FluxSingleTransformerBlock"] + + @register_to_config + def __init__( + self, + patch_size: int = 1, + in_channels: int = 64, + out_channels: Optional[int] = None, + num_layers: int = 19, + num_single_layers: int = 38, + attention_head_dim: int = 128, + num_attention_heads: int = 24, + joint_attention_dim: int = 4096, + pooled_projection_dim: int = 768, + guidance_embeds: bool = False, + axes_dims_rope: Tuple[int, int, int] = (16, 56, 56), + ): + super().__init__() + self.out_channels = out_channels or in_channels + self.inner_dim = num_attention_heads * attention_head_dim + + self.pos_embed = FluxPosEmbed(theta=10000, axes_dim=axes_dims_rope) + + text_time_guidance_cls = ( + CombinedTimestepGuidanceTextProjEmbeddings if guidance_embeds else CombinedTimestepTextProjEmbeddings + ) + self.time_text_embed = text_time_guidance_cls( + embedding_dim=self.inner_dim, pooled_projection_dim=pooled_projection_dim + ) + + self.context_embedder = nn.Linear(joint_attention_dim, self.inner_dim) + self.x_embedder = nn.Linear(in_channels, self.inner_dim) + + self.transformer_blocks = nn.ModuleList( + [ + FluxTransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + ) + for _ in range(num_layers) + ] + ) + + self.single_transformer_blocks = nn.ModuleList( + [ + FluxSingleTransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + ) + for _ in range(num_single_layers) + ] + ) + + self.norm_out = AdaLayerNormContinuous(self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6) + self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True) + + self.gradient_checkpointing = False + + self.sp_world_size = 1 + self.sp_world_rank = 0 + + def _set_gradient_checkpointing(self, *args, **kwargs): + if "value" in kwargs: + self.gradient_checkpointing = kwargs["value"] + elif "enable" in kwargs: + self.gradient_checkpointing = kwargs["enable"] + else: + raise ValueError("Invalid set gradient checkpointing") + + def enable_multi_gpus_inference(self,): + self.sp_world_size = get_sequence_parallel_world_size() + self.sp_world_rank = get_sequence_parallel_rank() + self.all_gather = get_sp_group().all_gather + self.set_attn_processor(FluxMultiGPUsAttnProcessor2_0()) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + pooled_projections: torch.Tensor = None, + timestep: torch.LongTensor = None, + img_ids: torch.Tensor = None, + txt_ids: torch.Tensor = None, + guidance: torch.Tensor = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_block_samples=None, + controlnet_single_block_samples=None, + return_dict: bool = True, + controlnet_blocks_repeat: bool = False, + ) -> Union[torch.Tensor, Transformer2DModelOutput]: + """ + The [`FluxTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): + Input `hidden_states`. + encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + pooled_projections (`torch.Tensor` of shape `(batch_size, projection_dim)`): Embeddings projected + from the embeddings of input conditions. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + block_controlnet_hidden_states: (`list` of `torch.Tensor`): + A list of tensors that if specified are added to the residuals of transformer blocks. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + if joint_attention_kwargs is not None: + joint_attention_kwargs = joint_attention_kwargs.copy() + lora_scale = joint_attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + else: + if joint_attention_kwargs is not None and joint_attention_kwargs.get("scale", None) is not None: + logger.warning( + "Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective." + ) + + hidden_states = self.x_embedder(hidden_states) + + timestep = timestep.to(hidden_states.dtype) * 1000 + if guidance is not None: + guidance = guidance.to(hidden_states.dtype) * 1000 + + temb = ( + self.time_text_embed(timestep, pooled_projections) + if guidance is None + else self.time_text_embed(timestep, guidance, pooled_projections) + ) + encoder_hidden_states = self.context_embedder(encoder_hidden_states) + + if txt_ids.ndim == 3: + logger.warning( + "Passing `txt_ids` 3d torch.Tensor is deprecated." + "Please remove the batch dimension and pass it as a 2d torch Tensor" + ) + txt_ids = txt_ids[0] + if img_ids.ndim == 3: + logger.warning( + "Passing `img_ids` 3d torch.Tensor is deprecated." + "Please remove the batch dimension and pass it as a 2d torch Tensor" + ) + img_ids = img_ids[0] + + ids = torch.cat((txt_ids, img_ids), dim=0) + image_rotary_emb = self.pos_embed(ids) + + if joint_attention_kwargs is not None and "ip_adapter_image_embeds" in joint_attention_kwargs: + ip_adapter_image_embeds = joint_attention_kwargs.pop("ip_adapter_image_embeds") + ip_hidden_states = self.encoder_hid_proj(ip_adapter_image_embeds) + joint_attention_kwargs.update({"ip_hidden_states": ip_hidden_states}) + + # Context Parallel + if self.sp_world_size > 1: + hidden_states = torch.chunk(hidden_states, self.sp_world_size, dim=1)[self.sp_world_rank] + if image_rotary_emb is not None: + txt_rotary_emb = ( + image_rotary_emb[0][:encoder_hidden_states.shape[1]], + image_rotary_emb[1][:encoder_hidden_states.shape[1]] + ) + image_rotary_emb = ( + torch.chunk(image_rotary_emb[0][encoder_hidden_states.shape[1]:], self.sp_world_size, dim=0)[self.sp_world_rank], + torch.chunk(image_rotary_emb[1][encoder_hidden_states.shape[1]:], self.sp_world_size, dim=0)[self.sp_world_rank], + ) + image_rotary_emb = [torch.cat([_txt_rotary_emb, _image_rotary_emb], dim=0) \ + for _txt_rotary_emb, _image_rotary_emb in zip(txt_rotary_emb, image_rotary_emb)] + + for index_block, block in enumerate(self.transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + encoder_hidden_states, hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + temb, + image_rotary_emb, + joint_attention_kwargs, + **ckpt_kwargs, + ) + + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # controlnet residual + if controlnet_block_samples is not None: + interval_control = len(self.transformer_blocks) / len(controlnet_block_samples) + interval_control = int(np.ceil(interval_control)) + # For Xlabs ControlNet. + if controlnet_blocks_repeat: + hidden_states = ( + hidden_states + controlnet_block_samples[index_block % len(controlnet_block_samples)] + ) + else: + hidden_states = hidden_states + controlnet_block_samples[index_block // interval_control] + + for index_block, block in enumerate(self.single_transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + encoder_hidden_states, hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + temb, + image_rotary_emb, + joint_attention_kwargs, + **ckpt_kwargs, + ) + + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=joint_attention_kwargs, + ) + + # controlnet residual + if controlnet_single_block_samples is not None: + interval_control = len(self.single_transformer_blocks) / len(controlnet_single_block_samples) + interval_control = int(np.ceil(interval_control)) + hidden_states = hidden_states + controlnet_single_block_samples[index_block // interval_control] + + hidden_states = self.norm_out(hidden_states, temb) + output = self.proj_out(hidden_states) + + if self.sp_world_size > 1: + output = self.all_gather(output, dim=1) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/hunyuanvideo_transformer3d.py b/VideoX-Fun/videox_fun/models/hunyuanvideo_transformer3d.py new file mode 100644 index 0000000000000000000000000000000000000000..117f512f54de012b91e8ea16673966f23d04f550 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/hunyuanvideo_transformer3d.py @@ -0,0 +1,1478 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py +# Copyright 2025 The Hunyuan Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import json +import os +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin +from diffusers.models.attention import FeedForward +from diffusers.models.attention_processor import Attention, AttentionProcessor +from diffusers.models.embeddings import (CombinedTimestepTextProjEmbeddings, + PixArtAlphaTextProjection, + TimestepEmbedding, Timesteps, + get_1d_rotary_pos_embed) +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import (AdaLayerNormContinuous, + AdaLayerNormZero, + AdaLayerNormZeroSingle, + FP32LayerNorm) +from diffusers.utils import (USE_PEFT_BACKEND, is_torch_version, logging, + scale_lora_layers, unscale_lora_layers) + +from ..dist import (get_sequence_parallel_rank, + get_sequence_parallel_world_size, get_sp_group, + xFuserLongContextAttention) +from ..dist.hunyuanvideo_xfuser import HunyuanVideoMultiGPUsAttnProcessor2_0 +from .attention_utils import attention + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def apply_rotary_emb( + x: torch.Tensor, + freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], + use_real: bool = True, + use_real_unbind_dim: int = -1, + sequence_dim: int = 2, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings + to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are + reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting + tensors contain rotary embeddings and are returned as real tensors. + + Args: + x (`torch.Tensor`): + Query or key tensor to apply rotary embeddings. [B, H, S, D] xk (torch.Tensor): Key tensor to apply + freqs_cis (`Tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],) + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings. + """ + if use_real: + cos, sin = freqs_cis # [S, D] + if sequence_dim == 2: + cos = cos[None, None, :, :] + sin = sin[None, None, :, :] + elif sequence_dim == 1: + cos = cos[None, :, None, :] + sin = sin[None, :, None, :] + else: + raise ValueError(f"`sequence_dim={sequence_dim}` but should be 1 or 2.") + + cos, sin = cos.to(x.device), sin.to(x.device) + + if use_real_unbind_dim == -1: + # Used for flux, cogvideox, hunyuan-dit + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, H, S, D//2] + x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3) + elif use_real_unbind_dim == -2: + # Used for Stable Audio, OmniGen, CogView4 and Cosmos + x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) # [B, H, S, D//2] + x_rotated = torch.cat([-x_imag, x_real], dim=-1) + else: + raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") + + out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + + return out + else: + # used for lumina + x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2)) + freqs_cis = freqs_cis.unsqueeze(2) + x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3) + + return x_out.type_as(x) + +def extract_seqlens_from_mask(attn_mask): + if attn_mask is None: + return None + + if len(attn_mask.shape) == 4: + bs, _, _, seq_len = attn_mask.shape + + if attn_mask.dtype == torch.bool: + valid_mask = attn_mask.squeeze(1).squeeze(1) + else: + valid_mask = ~torch.isinf(attn_mask.squeeze(1).squeeze(1)) + elif len(attn_mask.shape) == 3: + raise ValueError( + "attn_mask should be 2D or 4D tensor, but got {}".format( + attn_mask.shape)) + + seqlens = valid_mask.sum(dim=1) + return seqlens + +class HunyuanVideoAttnProcessor2_0: + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError( + "HunyuanVideoAttnProcessor2_0 requires PyTorch 2.0. To use it, please upgrade PyTorch to 2.0." + ) + + def __call__( + self, + attn: Attention, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + image_rotary_emb: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if attn.add_q_proj is None and encoder_hidden_states is not None: + hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1) + + # 1. QKV projections + query = attn.to_q(hidden_states) + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + + query = query.unflatten(2, (attn.heads, -1)).transpose(1, 2) + key = key.unflatten(2, (attn.heads, -1)).transpose(1, 2) + value = value.unflatten(2, (attn.heads, -1)).transpose(1, 2) + + # 2. QK normalization + if attn.norm_q is not None: + query = attn.norm_q(query) + if attn.norm_k is not None: + key = attn.norm_k(key) + + # 3. Rotational positional embeddings applied to latent stream + if image_rotary_emb is not None: + if attn.add_q_proj is None and encoder_hidden_states is not None: + query = torch.cat( + [ + apply_rotary_emb(query[:, :, : -encoder_hidden_states.shape[1]], image_rotary_emb), + query[:, :, -encoder_hidden_states.shape[1] :], + ], + dim=2, + ) + key = torch.cat( + [ + apply_rotary_emb(key[:, :, : -encoder_hidden_states.shape[1]], image_rotary_emb), + key[:, :, -encoder_hidden_states.shape[1] :], + ], + dim=2, + ) + else: + query = apply_rotary_emb(query, image_rotary_emb) + key = apply_rotary_emb(key, image_rotary_emb) + + # 4. Encoder condition QKV projection and normalization + if attn.add_q_proj is not None and encoder_hidden_states is not None: + encoder_query = attn.add_q_proj(encoder_hidden_states) + encoder_key = attn.add_k_proj(encoder_hidden_states) + encoder_value = attn.add_v_proj(encoder_hidden_states) + + encoder_query = encoder_query.unflatten(2, (attn.heads, -1)).transpose(1, 2) + encoder_key = encoder_key.unflatten(2, (attn.heads, -1)).transpose(1, 2) + encoder_value = encoder_value.unflatten(2, (attn.heads, -1)).transpose(1, 2) + + if attn.norm_added_q is not None: + encoder_query = attn.norm_added_q(encoder_query) + if attn.norm_added_k is not None: + encoder_key = attn.norm_added_k(encoder_key) + + query = torch.cat([query, encoder_query], dim=2) + key = torch.cat([key, encoder_key], dim=2) + value = torch.cat([value, encoder_value], dim=2) + + # 5. Attention + query = query.transpose(1, 2) + key = key.transpose(1, 2) + value = value.transpose(1, 2) + + if attention_mask is not None: + q_lens = k_lens = extract_seqlens_from_mask(attention_mask) + + hidden_states = torch.zeros_like(query) + for i in range(len(q_lens)): + hidden_states[i][:q_lens[i]] = attention( + query[i][:q_lens[i]].unsqueeze(0), + key[i][:q_lens[i]].unsqueeze(0), + value[i][:q_lens[i]].unsqueeze(0), + attn_mask=None, + ) + else: + hidden_states = attention( + query, key, value, attn_mask=attention_mask, + ) + hidden_states = hidden_states.flatten(2, 3) + hidden_states = hidden_states.to(query.dtype) + + # 6. Output projection + if encoder_hidden_states is not None: + hidden_states, encoder_hidden_states = ( + hidden_states[:, : -encoder_hidden_states.shape[1]], + hidden_states[:, -encoder_hidden_states.shape[1] :], + ) + + if getattr(attn, "to_out", None) is not None: + hidden_states = attn.to_out[0](hidden_states) + hidden_states = attn.to_out[1](hidden_states) + + if getattr(attn, "to_add_out", None) is not None: + encoder_hidden_states = attn.to_add_out(encoder_hidden_states) + + return hidden_states, encoder_hidden_states + + +class HunyuanVideoPatchEmbed(nn.Module): + def __init__( + self, + patch_size: Union[int, Tuple[int, int, int]] = 16, + in_chans: int = 3, + embed_dim: int = 768, + ) -> None: + super().__init__() + + patch_size = (patch_size, patch_size, patch_size) if isinstance(patch_size, int) else patch_size + self.proj = nn.Conv3d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.proj(hidden_states) + hidden_states = hidden_states.flatten(2).transpose(1, 2) # BCFHW -> BNC + return hidden_states + + +class HunyuanVideoAdaNorm(nn.Module): + def __init__(self, in_features: int, out_features: Optional[int] = None) -> None: + super().__init__() + + out_features = out_features or 2 * in_features + self.linear = nn.Linear(in_features, out_features) + self.nonlinearity = nn.SiLU() + + def forward( + self, temb: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + temb = self.linear(self.nonlinearity(temb)) + gate_msa, gate_mlp = temb.chunk(2, dim=1) + gate_msa, gate_mlp = gate_msa.unsqueeze(1), gate_mlp.unsqueeze(1) + return gate_msa, gate_mlp + + +class HunyuanVideoTokenReplaceAdaLayerNormZero(nn.Module): + def __init__(self, embedding_dim: int, norm_type: str = "layer_norm", bias: bool = True): + super().__init__() + + self.silu = nn.SiLU() + self.linear = nn.Linear(embedding_dim, 6 * embedding_dim, bias=bias) + + if norm_type == "layer_norm": + self.norm = nn.LayerNorm(embedding_dim, elementwise_affine=False, eps=1e-6) + elif norm_type == "fp32_layer_norm": + self.norm = FP32LayerNorm(embedding_dim, elementwise_affine=False, bias=False) + else: + raise ValueError( + f"Unsupported `norm_type` ({norm_type}) provided. Supported ones are: 'layer_norm', 'fp32_layer_norm'." + ) + + def forward( + self, + hidden_states: torch.Tensor, + emb: torch.Tensor, + token_replace_emb: torch.Tensor, + first_frame_num_tokens: int, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + emb = self.linear(self.silu(emb)) + token_replace_emb = self.linear(self.silu(token_replace_emb)) + + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk(6, dim=1) + tr_shift_msa, tr_scale_msa, tr_gate_msa, tr_shift_mlp, tr_scale_mlp, tr_gate_mlp = token_replace_emb.chunk( + 6, dim=1 + ) + + norm_hidden_states = self.norm(hidden_states) + hidden_states_zero = ( + norm_hidden_states[:, :first_frame_num_tokens] * (1 + tr_scale_msa[:, None]) + tr_shift_msa[:, None] + ) + hidden_states_orig = ( + norm_hidden_states[:, first_frame_num_tokens:] * (1 + scale_msa[:, None]) + shift_msa[:, None] + ) + hidden_states = torch.cat([hidden_states_zero, hidden_states_orig], dim=1) + + return ( + hidden_states, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + tr_gate_msa, + tr_shift_mlp, + tr_scale_mlp, + tr_gate_mlp, + ) + + +class HunyuanVideoTokenReplaceAdaLayerNormZeroSingle(nn.Module): + def __init__(self, embedding_dim: int, norm_type: str = "layer_norm", bias: bool = True): + super().__init__() + + self.silu = nn.SiLU() + self.linear = nn.Linear(embedding_dim, 3 * embedding_dim, bias=bias) + + if norm_type == "layer_norm": + self.norm = nn.LayerNorm(embedding_dim, elementwise_affine=False, eps=1e-6) + else: + raise ValueError( + f"Unsupported `norm_type` ({norm_type}) provided. Supported ones are: 'layer_norm', 'fp32_layer_norm'." + ) + + def forward( + self, + hidden_states: torch.Tensor, + emb: torch.Tensor, + token_replace_emb: torch.Tensor, + first_frame_num_tokens: int, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + emb = self.linear(self.silu(emb)) + token_replace_emb = self.linear(self.silu(token_replace_emb)) + + shift_msa, scale_msa, gate_msa = emb.chunk(3, dim=1) + tr_shift_msa, tr_scale_msa, tr_gate_msa = token_replace_emb.chunk(3, dim=1) + + norm_hidden_states = self.norm(hidden_states) + hidden_states_zero = ( + norm_hidden_states[:, :first_frame_num_tokens] * (1 + tr_scale_msa[:, None]) + tr_shift_msa[:, None] + ) + hidden_states_orig = ( + norm_hidden_states[:, first_frame_num_tokens:] * (1 + scale_msa[:, None]) + shift_msa[:, None] + ) + hidden_states = torch.cat([hidden_states_zero, hidden_states_orig], dim=1) + + return hidden_states, gate_msa, tr_gate_msa + + +class HunyuanVideoConditionEmbedding(nn.Module): + def __init__( + self, + embedding_dim: int, + pooled_projection_dim: int, + guidance_embeds: bool, + image_condition_type: Optional[str] = None, + ): + super().__init__() + + self.image_condition_type = image_condition_type + + self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0) + self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim) + self.text_embedder = PixArtAlphaTextProjection(pooled_projection_dim, embedding_dim, act_fn="silu") + + self.guidance_embedder = None + if guidance_embeds: + self.guidance_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim) + + def forward( + self, timestep: torch.Tensor, pooled_projection: torch.Tensor, guidance: Optional[torch.Tensor] = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + timesteps_proj = self.time_proj(timestep) + timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=pooled_projection.dtype)) # (N, D) + pooled_projections = self.text_embedder(pooled_projection) + conditioning = timesteps_emb + pooled_projections + + token_replace_emb = None + if self.image_condition_type == "token_replace": + token_replace_timestep = torch.zeros_like(timestep) + token_replace_proj = self.time_proj(token_replace_timestep) + token_replace_emb = self.timestep_embedder(token_replace_proj.to(dtype=pooled_projection.dtype)) + token_replace_emb = token_replace_emb + pooled_projections + + if self.guidance_embedder is not None: + guidance_proj = self.time_proj(guidance) + guidance_emb = self.guidance_embedder(guidance_proj.to(dtype=pooled_projection.dtype)) + conditioning = conditioning + guidance_emb + + return conditioning, token_replace_emb + + +class HunyuanVideoIndividualTokenRefinerBlock(nn.Module): + def __init__( + self, + num_attention_heads: int, + attention_head_dim: int, + mlp_width_ratio: str = 4.0, + mlp_drop_rate: float = 0.0, + attention_bias: bool = True, + ) -> None: + super().__init__() + + hidden_size = num_attention_heads * attention_head_dim + + self.norm1 = nn.LayerNorm(hidden_size, elementwise_affine=True, eps=1e-6) + self.attn = Attention( + query_dim=hidden_size, + cross_attention_dim=None, + heads=num_attention_heads, + dim_head=attention_head_dim, + bias=attention_bias, + ) + self.attn.set_processor = None + + self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=True, eps=1e-6) + self.ff = FeedForward(hidden_size, mult=mlp_width_ratio, activation_fn="linear-silu", dropout=mlp_drop_rate) + + self.norm_out = HunyuanVideoAdaNorm(hidden_size, 2 * hidden_size) + + def forward( + self, + hidden_states: torch.Tensor, + temb: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + norm_hidden_states = self.norm1(hidden_states) + + attn_output = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=None, + attention_mask=attention_mask, + ) + + gate_msa, gate_mlp = self.norm_out(temb) + hidden_states = hidden_states + attn_output * gate_msa + + ff_output = self.ff(self.norm2(hidden_states)) + hidden_states = hidden_states + ff_output * gate_mlp + + return hidden_states + + +class HunyuanVideoIndividualTokenRefiner(nn.Module): + def __init__( + self, + num_attention_heads: int, + attention_head_dim: int, + num_layers: int, + mlp_width_ratio: float = 4.0, + mlp_drop_rate: float = 0.0, + attention_bias: bool = True, + ) -> None: + super().__init__() + + self.refiner_blocks = nn.ModuleList( + [ + HunyuanVideoIndividualTokenRefinerBlock( + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + mlp_width_ratio=mlp_width_ratio, + mlp_drop_rate=mlp_drop_rate, + attention_bias=attention_bias, + ) + for _ in range(num_layers) + ] + ) + + def forward( + self, + hidden_states: torch.Tensor, + temb: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + ) -> None: + self_attn_mask = None + if attention_mask is not None: + batch_size = attention_mask.shape[0] + seq_len = attention_mask.shape[1] + attention_mask = attention_mask.to(hidden_states.device).bool() + self_attn_mask_1 = attention_mask.view(batch_size, 1, 1, seq_len).repeat(1, 1, seq_len, 1) + self_attn_mask_2 = self_attn_mask_1.transpose(2, 3) + self_attn_mask = (self_attn_mask_1 & self_attn_mask_2).bool() + self_attn_mask[:, :, :, 0] = True + + for block in self.refiner_blocks: + hidden_states = block(hidden_states, temb, self_attn_mask) + + return hidden_states + + +class HunyuanVideoTokenRefiner(nn.Module): + def __init__( + self, + in_channels: int, + num_attention_heads: int, + attention_head_dim: int, + num_layers: int, + mlp_ratio: float = 4.0, + mlp_drop_rate: float = 0.0, + attention_bias: bool = True, + ) -> None: + super().__init__() + + hidden_size = num_attention_heads * attention_head_dim + + self.time_text_embed = CombinedTimestepTextProjEmbeddings( + embedding_dim=hidden_size, pooled_projection_dim=in_channels + ) + self.proj_in = nn.Linear(in_channels, hidden_size, bias=True) + self.token_refiner = HunyuanVideoIndividualTokenRefiner( + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + num_layers=num_layers, + mlp_width_ratio=mlp_ratio, + mlp_drop_rate=mlp_drop_rate, + attention_bias=attention_bias, + ) + + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.LongTensor, + attention_mask: Optional[torch.LongTensor] = None, + ) -> torch.Tensor: + if attention_mask is None: + pooled_projections = hidden_states.mean(dim=1) + else: + original_dtype = hidden_states.dtype + mask_float = attention_mask.float().unsqueeze(-1) + pooled_projections = (hidden_states * mask_float).sum(dim=1) / mask_float.sum(dim=1) + pooled_projections = pooled_projections.to(original_dtype) + + temb = self.time_text_embed(timestep, pooled_projections) + hidden_states = self.proj_in(hidden_states) + hidden_states = self.token_refiner(hidden_states, temb, attention_mask) + + return hidden_states + + +class HunyuanVideoRotaryPosEmbed(nn.Module): + def __init__(self, patch_size: int, patch_size_t: int, rope_dim: List[int], theta: float = 256.0) -> None: + super().__init__() + + self.patch_size = patch_size + self.patch_size_t = patch_size_t + self.rope_dim = rope_dim + self.theta = theta + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, num_frames, height, width = hidden_states.shape + rope_sizes = [num_frames // self.patch_size_t, height // self.patch_size, width // self.patch_size] + + axes_grids = [] + for i in range(3): + # Note: The following line diverges from original behaviour. We create the grid on the device, whereas + # original implementation creates it on CPU and then moves it to device. This results in numerical + # differences in layerwise debugging outputs, but visually it is the same. + grid = torch.arange(0, rope_sizes[i], device=hidden_states.device, dtype=torch.float32) + axes_grids.append(grid) + grid = torch.meshgrid(*axes_grids, indexing="ij") # [W, H, T] + grid = torch.stack(grid, dim=0) # [3, W, H, T] + + freqs = [] + for i in range(3): + freq = get_1d_rotary_pos_embed(self.rope_dim[i], grid[i].reshape(-1), self.theta, use_real=True) + freqs.append(freq) + + freqs_cos = torch.cat([f[0] for f in freqs], dim=1) # (W * H * T, D / 2) + freqs_sin = torch.cat([f[1] for f in freqs], dim=1) # (W * H * T, D / 2) + return freqs_cos, freqs_sin + + +class HunyuanVideoSingleTransformerBlock(nn.Module): + def __init__( + self, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float = 4.0, + qk_norm: str = "rms_norm", + ) -> None: + super().__init__() + + hidden_size = num_attention_heads * attention_head_dim + mlp_dim = int(hidden_size * mlp_ratio) + + self.attn = Attention( + query_dim=hidden_size, + cross_attention_dim=None, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=hidden_size, + bias=True, + processor=HunyuanVideoAttnProcessor2_0(), + qk_norm=qk_norm, + eps=1e-6, + pre_only=True, + ) + + self.norm = AdaLayerNormZeroSingle(hidden_size, norm_type="layer_norm") + self.proj_mlp = nn.Linear(hidden_size, mlp_dim) + self.act_mlp = nn.GELU(approximate="tanh") + self.proj_out = nn.Linear(hidden_size + mlp_dim, hidden_size) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + *args, + **kwargs, + ) -> Tuple[torch.Tensor, torch.Tensor]: + text_seq_length = encoder_hidden_states.shape[1] + hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1) + + residual = hidden_states + + # 1. Input normalization + norm_hidden_states, gate = self.norm(hidden_states, emb=temb) + mlp_hidden_states = self.act_mlp(self.proj_mlp(norm_hidden_states)) + + norm_hidden_states, norm_encoder_hidden_states = ( + norm_hidden_states[:, :-text_seq_length, :], + norm_hidden_states[:, -text_seq_length:, :], + ) + + # 2. Attention + attn_output, context_attn_output = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + attention_mask=attention_mask, + image_rotary_emb=image_rotary_emb, + ) + attn_output = torch.cat([attn_output, context_attn_output], dim=1) + + # 3. Modulation and residual connection + hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2) + hidden_states = gate.unsqueeze(1) * self.proj_out(hidden_states) + hidden_states = hidden_states + residual + + hidden_states, encoder_hidden_states = ( + hidden_states[:, :-text_seq_length, :], + hidden_states[:, -text_seq_length:, :], + ) + return hidden_states, encoder_hidden_states + + +class HunyuanVideoTransformerBlock(nn.Module): + def __init__( + self, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float, + qk_norm: str = "rms_norm", + ) -> None: + super().__init__() + + hidden_size = num_attention_heads * attention_head_dim + + self.norm1 = AdaLayerNormZero(hidden_size, norm_type="layer_norm") + self.norm1_context = AdaLayerNormZero(hidden_size, norm_type="layer_norm") + + self.attn = Attention( + query_dim=hidden_size, + cross_attention_dim=None, + added_kv_proj_dim=hidden_size, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=hidden_size, + context_pre_only=False, + bias=True, + processor=HunyuanVideoAttnProcessor2_0(), + qk_norm=qk_norm, + eps=1e-6, + ) + + self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.ff = FeedForward(hidden_size, mult=mlp_ratio, activation_fn="gelu-approximate") + + self.norm2_context = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.ff_context = FeedForward(hidden_size, mult=mlp_ratio, activation_fn="gelu-approximate") + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + *args, + **kwargs, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # 1. Input normalization + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1(hidden_states, emb=temb) + norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context( + encoder_hidden_states, emb=temb + ) + + # 2. Joint attention + attn_output, context_attn_output = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + attention_mask=attention_mask, + image_rotary_emb=freqs_cis, + ) + + # 3. Modulation and residual connection + hidden_states = hidden_states + attn_output * gate_msa.unsqueeze(1) + encoder_hidden_states = encoder_hidden_states + context_attn_output * c_gate_msa.unsqueeze(1) + + norm_hidden_states = self.norm2(hidden_states) + norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states) + + norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + norm_encoder_hidden_states = norm_encoder_hidden_states * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None] + + # 4. Feed-forward + ff_output = self.ff(norm_hidden_states) + context_ff_output = self.ff_context(norm_encoder_hidden_states) + + hidden_states = hidden_states + gate_mlp.unsqueeze(1) * ff_output + encoder_hidden_states = encoder_hidden_states + c_gate_mlp.unsqueeze(1) * context_ff_output + + return hidden_states, encoder_hidden_states + + +class HunyuanVideoTokenReplaceSingleTransformerBlock(nn.Module): + def __init__( + self, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float = 4.0, + qk_norm: str = "rms_norm", + ) -> None: + super().__init__() + + hidden_size = num_attention_heads * attention_head_dim + mlp_dim = int(hidden_size * mlp_ratio) + + self.attn = Attention( + query_dim=hidden_size, + cross_attention_dim=None, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=hidden_size, + bias=True, + processor=HunyuanVideoAttnProcessor2_0(), + qk_norm=qk_norm, + eps=1e-6, + pre_only=True, + ) + + self.norm = HunyuanVideoTokenReplaceAdaLayerNormZeroSingle(hidden_size, norm_type="layer_norm") + self.proj_mlp = nn.Linear(hidden_size, mlp_dim) + self.act_mlp = nn.GELU(approximate="tanh") + self.proj_out = nn.Linear(hidden_size + mlp_dim, hidden_size) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + token_replace_emb: torch.Tensor = None, + num_tokens: int = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + text_seq_length = encoder_hidden_states.shape[1] + hidden_states = torch.cat([hidden_states, encoder_hidden_states], dim=1) + + residual = hidden_states + + # 1. Input normalization + norm_hidden_states, gate, tr_gate = self.norm(hidden_states, temb, token_replace_emb, num_tokens) + mlp_hidden_states = self.act_mlp(self.proj_mlp(norm_hidden_states)) + + norm_hidden_states, norm_encoder_hidden_states = ( + norm_hidden_states[:, :-text_seq_length, :], + norm_hidden_states[:, -text_seq_length:, :], + ) + + # 2. Attention + attn_output, context_attn_output = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + attention_mask=attention_mask, + image_rotary_emb=image_rotary_emb, + ) + attn_output = torch.cat([attn_output, context_attn_output], dim=1) + + # 3. Modulation and residual connection + hidden_states = torch.cat([attn_output, mlp_hidden_states], dim=2) + + proj_output = self.proj_out(hidden_states) + hidden_states_zero = proj_output[:, :num_tokens] * tr_gate.unsqueeze(1) + hidden_states_orig = proj_output[:, num_tokens:] * gate.unsqueeze(1) + hidden_states = torch.cat([hidden_states_zero, hidden_states_orig], dim=1) + hidden_states = hidden_states + residual + + hidden_states, encoder_hidden_states = ( + hidden_states[:, :-text_seq_length, :], + hidden_states[:, -text_seq_length:, :], + ) + return hidden_states, encoder_hidden_states + + +class HunyuanVideoTokenReplaceTransformerBlock(nn.Module): + def __init__( + self, + num_attention_heads: int, + attention_head_dim: int, + mlp_ratio: float, + qk_norm: str = "rms_norm", + ) -> None: + super().__init__() + + hidden_size = num_attention_heads * attention_head_dim + + self.norm1 = HunyuanVideoTokenReplaceAdaLayerNormZero(hidden_size, norm_type="layer_norm") + self.norm1_context = AdaLayerNormZero(hidden_size, norm_type="layer_norm") + + self.attn = Attention( + query_dim=hidden_size, + cross_attention_dim=None, + added_kv_proj_dim=hidden_size, + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=hidden_size, + context_pre_only=False, + bias=True, + processor=HunyuanVideoAttnProcessor2_0(), + qk_norm=qk_norm, + eps=1e-6, + ) + + self.norm2 = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.ff = FeedForward(hidden_size, mult=mlp_ratio, activation_fn="gelu-approximate") + + self.norm2_context = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.ff_context = FeedForward(hidden_size, mult=mlp_ratio, activation_fn="gelu-approximate") + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + temb: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + freqs_cis: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + token_replace_emb: torch.Tensor = None, + num_tokens: int = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # 1. Input normalization + ( + norm_hidden_states, + gate_msa, + shift_mlp, + scale_mlp, + gate_mlp, + tr_gate_msa, + tr_shift_mlp, + tr_scale_mlp, + tr_gate_mlp, + ) = self.norm1(hidden_states, temb, token_replace_emb, num_tokens) + norm_encoder_hidden_states, c_gate_msa, c_shift_mlp, c_scale_mlp, c_gate_mlp = self.norm1_context( + encoder_hidden_states, emb=temb + ) + + # 2. Joint attention + attn_output, context_attn_output = self.attn( + hidden_states=norm_hidden_states, + encoder_hidden_states=norm_encoder_hidden_states, + attention_mask=attention_mask, + image_rotary_emb=freqs_cis, + ) + + # 3. Modulation and residual connection + hidden_states_zero = hidden_states[:, :num_tokens] + attn_output[:, :num_tokens] * tr_gate_msa.unsqueeze(1) + hidden_states_orig = hidden_states[:, num_tokens:] + attn_output[:, num_tokens:] * gate_msa.unsqueeze(1) + hidden_states = torch.cat([hidden_states_zero, hidden_states_orig], dim=1) + encoder_hidden_states = encoder_hidden_states + context_attn_output * c_gate_msa.unsqueeze(1) + + norm_hidden_states = self.norm2(hidden_states) + norm_encoder_hidden_states = self.norm2_context(encoder_hidden_states) + + hidden_states_zero = norm_hidden_states[:, :num_tokens] * (1 + tr_scale_mlp[:, None]) + tr_shift_mlp[:, None] + hidden_states_orig = norm_hidden_states[:, num_tokens:] * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + norm_hidden_states = torch.cat([hidden_states_zero, hidden_states_orig], dim=1) + norm_encoder_hidden_states = norm_encoder_hidden_states * (1 + c_scale_mlp[:, None]) + c_shift_mlp[:, None] + + # 4. Feed-forward + ff_output = self.ff(norm_hidden_states) + context_ff_output = self.ff_context(norm_encoder_hidden_states) + + hidden_states_zero = hidden_states[:, :num_tokens] + ff_output[:, :num_tokens] * tr_gate_mlp.unsqueeze(1) + hidden_states_orig = hidden_states[:, num_tokens:] + ff_output[:, num_tokens:] * gate_mlp.unsqueeze(1) + hidden_states = torch.cat([hidden_states_zero, hidden_states_orig], dim=1) + encoder_hidden_states = encoder_hidden_states + c_gate_mlp.unsqueeze(1) * context_ff_output + + return hidden_states, encoder_hidden_states + + +class HunyuanVideoTransformer3DModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): + r""" + A Transformer model for video-like data used in [HunyuanVideo](https://huggingface.co/tencent/HunyuanVideo). + + Args: + in_channels (`int`, defaults to `16`): + The number of channels in the input. + out_channels (`int`, defaults to `16`): + The number of channels in the output. + num_attention_heads (`int`, defaults to `24`): + The number of heads to use for multi-head attention. + attention_head_dim (`int`, defaults to `128`): + The number of channels in each head. + num_layers (`int`, defaults to `20`): + The number of layers of dual-stream blocks to use. + num_single_layers (`int`, defaults to `40`): + The number of layers of single-stream blocks to use. + num_refiner_layers (`int`, defaults to `2`): + The number of layers of refiner blocks to use. + mlp_ratio (`float`, defaults to `4.0`): + The ratio of the hidden layer size to the input size in the feedforward network. + patch_size (`int`, defaults to `2`): + The size of the spatial patches to use in the patch embedding layer. + patch_size_t (`int`, defaults to `1`): + The size of the tmeporal patches to use in the patch embedding layer. + qk_norm (`str`, defaults to `rms_norm`): + The normalization to use for the query and key projections in the attention layers. + guidance_embeds (`bool`, defaults to `True`): + Whether to use guidance embeddings in the model. + text_embed_dim (`int`, defaults to `4096`): + Input dimension of text embeddings from the text encoder. + pooled_projection_dim (`int`, defaults to `768`): + The dimension of the pooled projection of the text embeddings. + rope_theta (`float`, defaults to `256.0`): + The value of theta to use in the RoPE layer. + rope_axes_dim (`Tuple[int]`, defaults to `(16, 56, 56)`): + The dimensions of the axes to use in the RoPE layer. + image_condition_type (`str`, *optional*, defaults to `None`): + The type of image conditioning to use. If `None`, no image conditioning is used. If `latent_concat`, the + image is concatenated to the latent stream. If `token_replace`, the image is used to replace first-frame + tokens in the latent stream and apply conditioning. + """ + + _supports_gradient_checkpointing = True + _skip_layerwise_casting_patterns = ["x_embedder", "context_embedder", "norm"] + _no_split_modules = [ + "HunyuanVideoTransformerBlock", + "HunyuanVideoSingleTransformerBlock", + "HunyuanVideoPatchEmbed", + "HunyuanVideoTokenRefiner", + ] + _repeated_blocks = [ + "HunyuanVideoTransformerBlock", + "HunyuanVideoSingleTransformerBlock", + "HunyuanVideoPatchEmbed", + "HunyuanVideoTokenRefiner", + ] + + @register_to_config + def __init__( + self, + in_channels: int = 16, + out_channels: int = 16, + num_attention_heads: int = 24, + attention_head_dim: int = 128, + num_layers: int = 20, + num_single_layers: int = 40, + num_refiner_layers: int = 2, + mlp_ratio: float = 4.0, + patch_size: int = 2, + patch_size_t: int = 1, + qk_norm: str = "rms_norm", + guidance_embeds: bool = True, + text_embed_dim: int = 4096, + pooled_projection_dim: int = 768, + rope_theta: float = 256.0, + rope_axes_dim: Tuple[int, ...] = (16, 56, 56), + image_condition_type: Optional[str] = None, + ) -> None: + super().__init__() + + supported_image_condition_types = ["latent_concat", "token_replace"] + if image_condition_type is not None and image_condition_type not in supported_image_condition_types: + raise ValueError( + f"Invalid `image_condition_type` ({image_condition_type}). Supported ones are: {supported_image_condition_types}" + ) + + inner_dim = num_attention_heads * attention_head_dim + out_channels = out_channels or in_channels + + # 1. Latent and condition embedders + self.x_embedder = HunyuanVideoPatchEmbed((patch_size_t, patch_size, patch_size), in_channels, inner_dim) + self.context_embedder = HunyuanVideoTokenRefiner( + text_embed_dim, num_attention_heads, attention_head_dim, num_layers=num_refiner_layers + ) + + self.time_text_embed = HunyuanVideoConditionEmbedding( + inner_dim, pooled_projection_dim, guidance_embeds, image_condition_type + ) + + # 2. RoPE + self.rope = HunyuanVideoRotaryPosEmbed(patch_size, patch_size_t, rope_axes_dim, rope_theta) + + # 3. Dual stream transformer blocks + if image_condition_type == "token_replace": + self.transformer_blocks = nn.ModuleList( + [ + HunyuanVideoTokenReplaceTransformerBlock( + num_attention_heads, attention_head_dim, mlp_ratio=mlp_ratio, qk_norm=qk_norm + ) + for _ in range(num_layers) + ] + ) + else: + self.transformer_blocks = nn.ModuleList( + [ + HunyuanVideoTransformerBlock( + num_attention_heads, attention_head_dim, mlp_ratio=mlp_ratio, qk_norm=qk_norm + ) + for _ in range(num_layers) + ] + ) + + # 4. Single stream transformer blocks + if image_condition_type == "token_replace": + self.single_transformer_blocks = nn.ModuleList( + [ + HunyuanVideoTokenReplaceSingleTransformerBlock( + num_attention_heads, attention_head_dim, mlp_ratio=mlp_ratio, qk_norm=qk_norm + ) + for _ in range(num_single_layers) + ] + ) + else: + self.single_transformer_blocks = nn.ModuleList( + [ + HunyuanVideoSingleTransformerBlock( + num_attention_heads, attention_head_dim, mlp_ratio=mlp_ratio, qk_norm=qk_norm + ) + for _ in range(num_single_layers) + ] + ) + + # 5. Output projection + self.norm_out = AdaLayerNormContinuous(inner_dim, inner_dim, elementwise_affine=False, eps=1e-6) + self.proj_out = nn.Linear(inner_dim, patch_size_t * patch_size * patch_size * out_channels) + + + self.gradient_checkpointing = False + self.sp_world_size = 1 + self.sp_world_rank = 0 + + def _set_gradient_checkpointing(self, *args, **kwargs): + if "value" in kwargs: + self.gradient_checkpointing = kwargs["value"] + elif "enable" in kwargs: + self.gradient_checkpointing = kwargs["enable"] + else: + raise ValueError("Invalid set gradient checkpointing") + + def enable_multi_gpus_inference(self,): + self.sp_world_size = get_sequence_parallel_world_size() + self.sp_world_rank = get_sequence_parallel_rank() + self.set_attn_processor(HunyuanVideoMultiGPUsAttnProcessor2_0()) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor") and module.set_processor is not None: + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def forward( + self, + hidden_states: torch.Tensor, + timestep: torch.LongTensor, + encoder_hidden_states: torch.Tensor, + encoder_attention_mask: torch.Tensor, + pooled_projections: torch.Tensor, + guidance: torch.Tensor = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + return_dict: bool = True, + ) -> Union[Tuple[torch.Tensor], Transformer2DModelOutput]: + if attention_kwargs is not None: + attention_kwargs = attention_kwargs.copy() + lora_scale = attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + else: + if attention_kwargs is not None and attention_kwargs.get("scale", None) is not None: + logger.warning( + "Passing `scale` via `attention_kwargs` when not using the PEFT backend is ineffective." + ) + + batch_size, num_channels, num_frames, height, width = hidden_states.shape + p, p_t = self.config.patch_size, self.config.patch_size_t + post_patch_num_frames = num_frames // p_t + post_patch_height = height // p + post_patch_width = width // p + first_frame_num_tokens = 1 * post_patch_height * post_patch_width + + # 1. RoPE + image_rotary_emb = self.rope(hidden_states) + + # 2. Conditional embeddings + temb, token_replace_emb = self.time_text_embed(timestep, pooled_projections, guidance) + + hidden_states = self.x_embedder(hidden_states) + encoder_hidden_states = self.context_embedder(encoder_hidden_states, timestep, encoder_attention_mask) + + # 3. Attention mask preparation + latent_sequence_length = hidden_states.shape[1] + condition_sequence_length = encoder_hidden_states.shape[1] + sequence_length = latent_sequence_length + condition_sequence_length + attention_mask = torch.ones( + batch_size, sequence_length, device=hidden_states.device, dtype=torch.bool + ) # [B, N] + effective_condition_sequence_length = encoder_attention_mask.sum(dim=1, dtype=torch.int) # [B,] + effective_sequence_length = latent_sequence_length + effective_condition_sequence_length + indices = torch.arange(sequence_length, device=hidden_states.device).unsqueeze(0) # [1, N] + mask_indices = indices >= effective_sequence_length.unsqueeze(1) # [B, N] + attention_mask = attention_mask.masked_fill(mask_indices, False) + attention_mask = attention_mask.unsqueeze(1).unsqueeze(1) # [B, 1, 1, N] + + # Context Parallel + if self.sp_world_size > 1: + hidden_states = torch.chunk(hidden_states, self.sp_world_size, dim=1)[self.sp_world_rank] + if image_rotary_emb is not None: + image_rotary_emb = ( + torch.chunk(image_rotary_emb[0], self.sp_world_size, dim=0)[self.sp_world_rank], + torch.chunk(image_rotary_emb[1], self.sp_world_size, dim=0)[self.sp_world_rank] + ) + if self.sp_world_rank >=1: + first_frame_num_tokens = 0 + + # 4. Transformer blocks + if torch.is_grad_enabled() and self.gradient_checkpointing: + for block in self.transformer_blocks: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states, encoder_hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + temb, + attention_mask, + image_rotary_emb, + token_replace_emb, + first_frame_num_tokens, + **ckpt_kwargs, + ) + + for block in self.single_transformer_blocks: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states, encoder_hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + temb, + attention_mask, + image_rotary_emb, + token_replace_emb, + first_frame_num_tokens, + **ckpt_kwargs, + ) + + else: + for block in self.transformer_blocks: + hidden_states, encoder_hidden_states = block( + hidden_states, + encoder_hidden_states, + temb, + attention_mask, + image_rotary_emb, + token_replace_emb, + first_frame_num_tokens, + ) + + for block in self.single_transformer_blocks: + hidden_states, encoder_hidden_states = block( + hidden_states, + encoder_hidden_states, + temb, + attention_mask, + image_rotary_emb, + token_replace_emb, + first_frame_num_tokens, + ) + + # 5. Output projection + hidden_states = self.norm_out(hidden_states, temb) + hidden_states = self.proj_out(hidden_states) + + if self.sp_world_size > 1: + hidden_states = get_sp_group().all_gather(hidden_states, dim=1) + + hidden_states = hidden_states.reshape( + batch_size, post_patch_num_frames, post_patch_height, post_patch_width, -1, p_t, p, p + ) + hidden_states = hidden_states.permute(0, 4, 1, 5, 2, 6, 3, 7) + hidden_states = hidden_states.flatten(6, 7).flatten(4, 5).flatten(2, 3) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + if not return_dict: + return (hidden_states,) + + return Transformer2DModelOutput(sample=hidden_states) + + + @classmethod + def from_pretrained( + cls, pretrained_model_path, subfolder=None, transformer_additional_kwargs={}, + low_cpu_mem_usage=False, torch_dtype=torch.bfloat16 + ): + if subfolder is not None: + pretrained_model_path = os.path.join(pretrained_model_path, subfolder) + print(f"loaded 3D transformer's pretrained weights from {pretrained_model_path} ...") + + config_file = os.path.join(pretrained_model_path, 'config.json') + if not os.path.isfile(config_file): + raise RuntimeError(f"{config_file} does not exist") + with open(config_file, "r") as f: + config = json.load(f) + + from diffusers.utils import WEIGHTS_NAME + model_file = os.path.join(pretrained_model_path, WEIGHTS_NAME) + model_file_safetensors = model_file.replace(".bin", ".safetensors") + + if "dict_mapping" in transformer_additional_kwargs.keys(): + for key in transformer_additional_kwargs["dict_mapping"]: + transformer_additional_kwargs[transformer_additional_kwargs["dict_mapping"][key]] = config[key] + + if low_cpu_mem_usage: + try: + import re + + from diffusers import __version__ as diffusers_version + if diffusers_version >= "0.33.0": + from diffusers.models.model_loading_utils import \ + load_model_dict_into_meta + else: + from diffusers.models.modeling_utils import \ + load_model_dict_into_meta + from diffusers.utils import is_accelerate_available + if is_accelerate_available(): + import accelerate + + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + model = cls.from_config(config, **transformer_additional_kwargs) + + param_device = "cpu" + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + print(model_files_safetensors) + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + filtered_state_dict = {} + for key in state_dict: + if key in model.state_dict() and model.state_dict()[key].size() == state_dict[key].size(): + filtered_state_dict[key] = state_dict[key] + else: + print(f"Skipping key '{key}' due to size mismatch or absence in model.") + + model_keys = set(model.state_dict().keys()) + loaded_keys = set(filtered_state_dict.keys()) + missing_keys = model_keys - loaded_keys + + def initialize_missing_parameters(missing_keys, model_state_dict, torch_dtype=None): + initialized_dict = {} + + with torch.no_grad(): + for key in missing_keys: + param_shape = model_state_dict[key].shape + param_dtype = torch_dtype if torch_dtype is not None else model_state_dict[key].dtype + if 'weight' in key: + if any(norm_type in key for norm_type in ['norm', 'ln_', 'layer_norm', 'group_norm', 'batch_norm']): + initialized_dict[key] = torch.ones(param_shape, dtype=param_dtype) + elif 'embedding' in key or 'embed' in key: + initialized_dict[key] = torch.randn(param_shape, dtype=param_dtype) * 0.02 + elif 'head' in key or 'output' in key or 'proj_out' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif len(param_shape) >= 2: + initialized_dict[key] = torch.empty(param_shape, dtype=param_dtype) + nn.init.xavier_uniform_(initialized_dict[key]) + else: + initialized_dict[key] = torch.randn(param_shape, dtype=param_dtype) * 0.02 + elif 'bias' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif 'running_mean' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif 'running_var' in key: + initialized_dict[key] = torch.ones(param_shape, dtype=param_dtype) + elif 'num_batches_tracked' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=torch.long) + else: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + + return initialized_dict + + if missing_keys: + print(f"Missing keys will be initialized: {sorted(missing_keys)}") + initialized_params = initialize_missing_parameters( + missing_keys, + model.state_dict(), + torch_dtype + ) + filtered_state_dict.update(initialized_params) + + if diffusers_version >= "0.33.0": + # Diffusers has refactored `load_model_dict_into_meta` since version 0.33.0 in this commit: + # https://github.com/huggingface/diffusers/commit/f5929e03060d56063ff34b25a8308833bec7c785. + load_model_dict_into_meta( + model, + filtered_state_dict, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + else: + model._convert_deprecated_attention_blocks(filtered_state_dict) + unexpected_keys = load_model_dict_into_meta( + model, + filtered_state_dict, + device=param_device, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + + if cls._keys_to_ignore_on_load_unexpected is not None: + for pat in cls._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + print( + f"Some weights of the model checkpoint were not used when initializing {cls.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + + return model + except Exception as e: + print( + f"The low_cpu_mem_usage mode is not work because {e}. Use low_cpu_mem_usage=False instead." + ) + + model = cls.from_config(config, **transformer_additional_kwargs) + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + tmp_state_dict = {} + for key in state_dict: + if key in model.state_dict().keys() and model.state_dict()[key].size() == state_dict[key].size(): + tmp_state_dict[key] = state_dict[key] + else: + print(key, "Size don't match, skip") + + state_dict = tmp_state_dict + + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m) + + params = [p.numel() if "." in n else 0 for n, p in model.named_parameters()] + print(f"### All Parameters: {sum(params) / 1e6} M") + + params = [p.numel() if "attn1." in n else 0 for n, p in model.named_parameters()] + print(f"### attn1 Parameters: {sum(params) / 1e6} M") + + model = model.to(torch_dtype) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/hunyuanvideo_vae.py b/VideoX-Fun/videox_fun/models/hunyuanvideo_vae.py new file mode 100644 index 0000000000000000000000000000000000000000..66aa9687c36c2d449e56cd84fb2e0fdcaa5792f0 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/hunyuanvideo_vae.py @@ -0,0 +1,1082 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/autoencoders/autoencoder_kl_hunyuan_video.py +# Copyright 2025 The Hunyuan Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.activations import get_activation +from diffusers.models.attention import FeedForward +from diffusers.models.attention_processor import Attention +from diffusers.models.autoencoders.vae import (DecoderOutput, + DiagonalGaussianDistribution) +from diffusers.models.embeddings import TimestepEmbedding, Timesteps +from diffusers.models.modeling_outputs import (AutoencoderKLOutput, + Transformer2DModelOutput) +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import AdaLayerNormContinuous, RMSNorm +from diffusers.utils import (USE_PEFT_BACKEND, is_torch_version, logging, + scale_lora_layers, unscale_lora_layers) +from diffusers.utils.accelerate_utils import apply_forward_hook +from diffusers.utils.torch_utils import maybe_allow_in_graph + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def prepare_causal_attention_mask( + num_frames: int, height_width: int, dtype: torch.dtype, device: torch.device, batch_size: int = None +) -> torch.Tensor: + indices = torch.arange(1, num_frames + 1, dtype=torch.int32, device=device) + indices_blocks = indices.repeat_interleave(height_width) + x, y = torch.meshgrid(indices_blocks, indices_blocks, indexing="xy") + mask = torch.where(x <= y, 0, -float("inf")).to(dtype=dtype) + + if batch_size is not None: + mask = mask.unsqueeze(0).expand(batch_size, -1, -1) + return mask + + +class HunyuanVideoCausalConv3d(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: Union[int, Tuple[int, int, int]] = 3, + stride: Union[int, Tuple[int, int, int]] = 1, + padding: Union[int, Tuple[int, int, int]] = 0, + dilation: Union[int, Tuple[int, int, int]] = 1, + bias: bool = True, + pad_mode: str = "replicate", + ) -> None: + super().__init__() + + kernel_size = (kernel_size, kernel_size, kernel_size) if isinstance(kernel_size, int) else kernel_size + + self.pad_mode = pad_mode + self.time_causal_padding = ( + kernel_size[0] // 2, + kernel_size[0] // 2, + kernel_size[1] // 2, + kernel_size[1] // 2, + kernel_size[2] - 1, + 0, + ) + + self.conv = nn.Conv3d(in_channels, out_channels, kernel_size, stride, padding, dilation, bias=bias) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = F.pad(hidden_states, self.time_causal_padding, mode=self.pad_mode) + return self.conv(hidden_states) + + +class HunyuanVideoUpsampleCausal3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: Optional[int] = None, + kernel_size: int = 3, + stride: int = 1, + bias: bool = True, + upsample_factor: Tuple[float, float, float] = (2, 2, 2), + ) -> None: + super().__init__() + + out_channels = out_channels or in_channels + self.upsample_factor = upsample_factor + + self.conv = HunyuanVideoCausalConv3d(in_channels, out_channels, kernel_size, stride, bias=bias) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + num_frames = hidden_states.size(2) + + first_frame, other_frames = hidden_states.split((1, num_frames - 1), dim=2) + first_frame = F.interpolate( + first_frame.squeeze(2), scale_factor=self.upsample_factor[1:], mode="nearest" + ).unsqueeze(2) + + if num_frames > 1: + # See: https://github.com/pytorch/pytorch/issues/81665 + # Unless you have a version of pytorch where non-contiguous implementation of F.interpolate + # is fixed, this will raise either a runtime error, or fail silently with bad outputs. + # If you are encountering an error here, make sure to try running encoding/decoding with + # `vae.enable_tiling()` first. If that doesn't work, open an issue at: + # https://github.com/huggingface/diffusers/issues + other_frames = other_frames.contiguous() + other_frames = F.interpolate(other_frames, scale_factor=self.upsample_factor, mode="nearest") + hidden_states = torch.cat((first_frame, other_frames), dim=2) + else: + hidden_states = first_frame + + hidden_states = self.conv(hidden_states) + return hidden_states + + +class HunyuanVideoDownsampleCausal3D(nn.Module): + def __init__( + self, + channels: int, + out_channels: Optional[int] = None, + padding: int = 1, + kernel_size: int = 3, + bias: bool = True, + stride=2, + ) -> None: + super().__init__() + out_channels = out_channels or channels + + self.conv = HunyuanVideoCausalConv3d(channels, out_channels, kernel_size, stride, padding, bias=bias) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.conv(hidden_states) + return hidden_states + + +class HunyuanVideoResnetBlockCausal3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: Optional[int] = None, + dropout: float = 0.0, + groups: int = 32, + eps: float = 1e-6, + non_linearity: str = "swish", + ) -> None: + super().__init__() + out_channels = out_channels or in_channels + + self.nonlinearity = get_activation(non_linearity) + + self.norm1 = nn.GroupNorm(groups, in_channels, eps=eps, affine=True) + self.conv1 = HunyuanVideoCausalConv3d(in_channels, out_channels, 3, 1, 0) + + self.norm2 = nn.GroupNorm(groups, out_channels, eps=eps, affine=True) + self.dropout = nn.Dropout(dropout) + self.conv2 = HunyuanVideoCausalConv3d(out_channels, out_channels, 3, 1, 0) + + self.conv_shortcut = None + if in_channels != out_channels: + self.conv_shortcut = HunyuanVideoCausalConv3d(in_channels, out_channels, 1, 1, 0) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = hidden_states.contiguous() + residual = hidden_states + + hidden_states = self.norm1(hidden_states) + hidden_states = self.nonlinearity(hidden_states) + hidden_states = self.conv1(hidden_states) + + hidden_states = self.norm2(hidden_states) + hidden_states = self.nonlinearity(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.conv_shortcut is not None: + residual = self.conv_shortcut(residual) + + hidden_states = hidden_states + residual + return hidden_states + + +class HunyuanVideoMidBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + add_attention: bool = True, + attention_head_dim: int = 1, + ) -> None: + super().__init__() + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + self.add_attention = add_attention + + # There is always at least one resnet + resnets = [ + HunyuanVideoResnetBlockCausal3D( + in_channels=in_channels, + out_channels=in_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + non_linearity=resnet_act_fn, + ) + ] + attentions = [] + + for _ in range(num_layers): + if self.add_attention: + attentions.append( + Attention( + in_channels, + heads=in_channels // attention_head_dim, + dim_head=attention_head_dim, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + else: + attentions.append(None) + + resnets.append( + HunyuanVideoResnetBlockCausal3D( + in_channels=in_channels, + out_channels=in_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + non_linearity=resnet_act_fn, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func(self.resnets[0], hidden_states) + + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + batch_size, num_channels, num_frames, height, width = hidden_states.shape + hidden_states = hidden_states.permute(0, 2, 3, 4, 1).flatten(1, 3) + attention_mask = prepare_causal_attention_mask( + num_frames, height * width, hidden_states.dtype, hidden_states.device, batch_size=batch_size + ) + hidden_states = attn(hidden_states, attention_mask=attention_mask) + hidden_states = hidden_states.unflatten(1, (num_frames, height, width)).permute(0, 4, 1, 2, 3) + + hidden_states = self._gradient_checkpointing_func(resnet, hidden_states) + + else: + hidden_states = self.resnets[0](hidden_states) + + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + batch_size, num_channels, num_frames, height, width = hidden_states.shape + hidden_states = hidden_states.permute(0, 2, 3, 4, 1).flatten(1, 3) + attention_mask = prepare_causal_attention_mask( + num_frames, height * width, hidden_states.dtype, hidden_states.device, batch_size=batch_size + ) + hidden_states = attn(hidden_states, attention_mask=attention_mask) + hidden_states = hidden_states.unflatten(1, (num_frames, height, width)).permute(0, 4, 1, 2, 3) + + hidden_states = resnet(hidden_states) + + return hidden_states + + +class HunyuanVideoDownBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + add_downsample: bool = True, + downsample_stride: int = 2, + downsample_padding: int = 1, + ) -> None: + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + HunyuanVideoResnetBlockCausal3D( + in_channels=in_channels, + out_channels=out_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + non_linearity=resnet_act_fn, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + HunyuanVideoDownsampleCausal3D( + out_channels, + out_channels=out_channels, + padding=downsample_padding, + stride=downsample_stride, + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if torch.is_grad_enabled() and self.gradient_checkpointing: + for resnet in self.resnets: + hidden_states = self._gradient_checkpointing_func(resnet, hidden_states) + else: + for resnet in self.resnets: + hidden_states = resnet(hidden_states) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states + + +class HunyuanVideoUpBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + add_upsample: bool = True, + upsample_scale_factor: Tuple[int, int, int] = (2, 2, 2), + ) -> None: + super().__init__() + resnets = [] + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + HunyuanVideoResnetBlockCausal3D( + in_channels=input_channels, + out_channels=out_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + non_linearity=resnet_act_fn, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList( + [ + HunyuanVideoUpsampleCausal3D( + out_channels, + out_channels=out_channels, + upsample_factor=upsample_scale_factor, + ) + ] + ) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + if torch.is_grad_enabled() and self.gradient_checkpointing: + for resnet in self.resnets: + hidden_states = self._gradient_checkpointing_func(resnet, hidden_states) + + else: + for resnet in self.resnets: + hidden_states = resnet(hidden_states) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class HunyuanVideoEncoder3D(nn.Module): + r""" + Causal encoder for 3D video-like data introduced in [Hunyuan Video](https://huggingface.co/papers/2412.03603). + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str, ...] = ( + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + ), + block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + double_z: bool = True, + mid_block_add_attention=True, + temporal_compression_ratio: int = 4, + spatial_compression_ratio: int = 8, + ) -> None: + super().__init__() + + self.conv_in = HunyuanVideoCausalConv3d(in_channels, block_out_channels[0], kernel_size=3, stride=1) + self.mid_block = None + self.down_blocks = nn.ModuleList([]) + + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + if down_block_type != "HunyuanVideoDownBlock3D": + raise ValueError(f"Unsupported down_block_type: {down_block_type}") + + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + num_spatial_downsample_layers = int(np.log2(spatial_compression_ratio)) + num_time_downsample_layers = int(np.log2(temporal_compression_ratio)) + + if temporal_compression_ratio == 4: + add_spatial_downsample = bool(i < num_spatial_downsample_layers) + add_time_downsample = bool( + i >= (len(block_out_channels) - 1 - num_time_downsample_layers) and not is_final_block + ) + elif temporal_compression_ratio == 8: + add_spatial_downsample = bool(i < num_spatial_downsample_layers) + add_time_downsample = bool(i < num_time_downsample_layers) + else: + raise ValueError(f"Unsupported time_compression_ratio: {temporal_compression_ratio}") + + downsample_stride_HW = (2, 2) if add_spatial_downsample else (1, 1) + downsample_stride_T = (2,) if add_time_downsample else (1,) + downsample_stride = tuple(downsample_stride_T + downsample_stride_HW) + + down_block = HunyuanVideoDownBlock3D( + num_layers=layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + add_downsample=bool(add_spatial_downsample or add_time_downsample), + resnet_eps=1e-6, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + downsample_stride=downsample_stride, + downsample_padding=0, + ) + + self.down_blocks.append(down_block) + + self.mid_block = HunyuanVideoMidBlock3D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + add_attention=mid_block_add_attention, + ) + + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6) + self.conv_act = nn.SiLU() + + conv_out_channels = 2 * out_channels if double_z else out_channels + self.conv_out = HunyuanVideoCausalConv3d(block_out_channels[-1], conv_out_channels, kernel_size=3) + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.conv_in(hidden_states) + + if torch.is_grad_enabled() and self.gradient_checkpointing: + for down_block in self.down_blocks: + hidden_states = self._gradient_checkpointing_func(down_block, hidden_states) + + hidden_states = self._gradient_checkpointing_func(self.mid_block, hidden_states) + else: + for down_block in self.down_blocks: + hidden_states = down_block(hidden_states) + + hidden_states = self.mid_block(hidden_states) + + hidden_states = self.conv_norm_out(hidden_states) + hidden_states = self.conv_act(hidden_states) + hidden_states = self.conv_out(hidden_states) + + return hidden_states + + +class HunyuanVideoDecoder3D(nn.Module): + r""" + Causal decoder for 3D video-like data introduced in [Hunyuan Video](https://huggingface.co/papers/2412.03603). + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + up_block_types: Tuple[str, ...] = ( + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + ), + block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + mid_block_add_attention=True, + time_compression_ratio: int = 4, + spatial_compression_ratio: int = 8, + ): + super().__init__() + self.layers_per_block = layers_per_block + + self.conv_in = HunyuanVideoCausalConv3d(in_channels, block_out_channels[-1], kernel_size=3, stride=1) + self.up_blocks = nn.ModuleList([]) + + # mid + self.mid_block = HunyuanVideoMidBlock3D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + add_attention=mid_block_add_attention, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + if up_block_type != "HunyuanVideoUpBlock3D": + raise ValueError(f"Unsupported up_block_type: {up_block_type}") + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + num_spatial_upsample_layers = int(np.log2(spatial_compression_ratio)) + num_time_upsample_layers = int(np.log2(time_compression_ratio)) + + if time_compression_ratio == 4: + add_spatial_upsample = bool(i < num_spatial_upsample_layers) + add_time_upsample = bool( + i >= len(block_out_channels) - 1 - num_time_upsample_layers and not is_final_block + ) + else: + raise ValueError(f"Unsupported time_compression_ratio: {time_compression_ratio}") + + upsample_scale_factor_HW = (2, 2) if add_spatial_upsample else (1, 1) + upsample_scale_factor_T = (2,) if add_time_upsample else (1,) + upsample_scale_factor = tuple(upsample_scale_factor_T + upsample_scale_factor_HW) + + up_block = HunyuanVideoUpBlock3D( + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + add_upsample=bool(add_spatial_upsample or add_time_upsample), + upsample_scale_factor=upsample_scale_factor, + resnet_eps=1e-6, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + ) + + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6) + self.conv_act = nn.SiLU() + self.conv_out = HunyuanVideoCausalConv3d(block_out_channels[0], out_channels, kernel_size=3) + + self.gradient_checkpointing = False + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.conv_in(hidden_states) + + if torch.is_grad_enabled() and self.gradient_checkpointing: + hidden_states = self._gradient_checkpointing_func(self.mid_block, hidden_states) + + for up_block in self.up_blocks: + hidden_states = self._gradient_checkpointing_func(up_block, hidden_states) + else: + hidden_states = self.mid_block(hidden_states) + + for up_block in self.up_blocks: + hidden_states = up_block(hidden_states) + + # post-process + hidden_states = self.conv_norm_out(hidden_states) + hidden_states = self.conv_act(hidden_states) + hidden_states = self.conv_out(hidden_states) + + return hidden_states + + +class AutoencoderKLHunyuanVideo(ModelMixin, ConfigMixin, FromOriginalModelMixin): + r""" + A VAE model with KL loss for encoding videos into latents and decoding latent representations into videos. + Introduced in [HunyuanVideo](https://huggingface.co/papers/2412.03603). + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + latent_channels: int = 16, + down_block_types: Tuple[str, ...] = ( + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + "HunyuanVideoDownBlock3D", + ), + up_block_types: Tuple[str, ...] = ( + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + "HunyuanVideoUpBlock3D", + ), + block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), + layers_per_block: int = 2, + act_fn: str = "silu", + norm_num_groups: int = 32, + scaling_factor: float = 0.476986, + spatial_compression_ratio: int = 8, + temporal_compression_ratio: int = 4, + mid_block_add_attention: bool = True, + ) -> None: + super().__init__() + + self.time_compression_ratio = temporal_compression_ratio + + self.encoder = HunyuanVideoEncoder3D( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + norm_num_groups=norm_num_groups, + act_fn=act_fn, + double_z=True, + mid_block_add_attention=mid_block_add_attention, + temporal_compression_ratio=temporal_compression_ratio, + spatial_compression_ratio=spatial_compression_ratio, + ) + + self.decoder = HunyuanVideoDecoder3D( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + norm_num_groups=norm_num_groups, + act_fn=act_fn, + time_compression_ratio=temporal_compression_ratio, + spatial_compression_ratio=spatial_compression_ratio, + mid_block_add_attention=mid_block_add_attention, + ) + + self.quant_conv = nn.Conv3d(2 * latent_channels, 2 * latent_channels, kernel_size=1) + self.post_quant_conv = nn.Conv3d(latent_channels, latent_channels, kernel_size=1) + + self.spatial_compression_ratio = spatial_compression_ratio + self.temporal_compression_ratio = temporal_compression_ratio + + # When decoding a batch of video latents at a time, one can save memory by slicing across the batch dimension + # to perform decoding of a single video latent at a time. + self.use_slicing = False + + # When decoding spatially large video latents, the memory requirement is very high. By breaking the video latent + # frames spatially into smaller tiles and performing multiple forward passes for decoding, and then blending the + # intermediate tiles together, the memory requirement can be lowered. + self.use_tiling = True + + # When decoding temporally long video latents, the memory requirement is very high. By decoding latent frames + # at a fixed frame batch size (based on `self.tile_sample_min_num_frames`), the memory requirement can be lowered. + self.use_framewise_encoding = True + self.use_framewise_decoding = True + + # The minimal tile height and width for spatial tiling to be used + self.tile_sample_min_height = 256 + self.tile_sample_min_width = 256 + self.tile_sample_min_num_frames = 16 + + # The minimal distance between two spatial tiles + self.tile_sample_stride_height = 192 + self.tile_sample_stride_width = 192 + self.tile_sample_stride_num_frames = 12 + + def enable_tiling( + self, + tile_sample_min_height: Optional[int] = None, + tile_sample_min_width: Optional[int] = None, + tile_sample_min_num_frames: Optional[int] = None, + tile_sample_stride_height: Optional[float] = None, + tile_sample_stride_width: Optional[float] = None, + tile_sample_stride_num_frames: Optional[float] = None, + ) -> None: + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + + Args: + tile_sample_min_height (`int`, *optional*): + The minimum height required for a sample to be separated into tiles across the height dimension. + tile_sample_min_width (`int`, *optional*): + The minimum width required for a sample to be separated into tiles across the width dimension. + tile_sample_min_num_frames (`int`, *optional*): + The minimum number of frames required for a sample to be separated into tiles across the frame + dimension. + tile_sample_stride_height (`int`, *optional*): + The minimum amount of overlap between two consecutive vertical tiles. This is to ensure that there are + no tiling artifacts produced across the height dimension. + tile_sample_stride_width (`int`, *optional*): + The stride between two consecutive horizontal tiles. This is to ensure that there are no tiling + artifacts produced across the width dimension. + tile_sample_stride_num_frames (`int`, *optional*): + The stride between two consecutive frame tiles. This is to ensure that there are no tiling artifacts + produced across the frame dimension. + """ + self.use_tiling = True + self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height + self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width + self.tile_sample_min_num_frames = tile_sample_min_num_frames or self.tile_sample_min_num_frames + self.tile_sample_stride_height = tile_sample_stride_height or self.tile_sample_stride_height + self.tile_sample_stride_width = tile_sample_stride_width or self.tile_sample_stride_width + self.tile_sample_stride_num_frames = tile_sample_stride_num_frames or self.tile_sample_stride_num_frames + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + batch_size, num_channels, num_frames, height, width = x.shape + + if self.use_framewise_encoding and num_frames > self.tile_sample_min_num_frames: + return self._temporal_tiled_encode(x) + + if self.use_tiling and (width > self.tile_sample_min_width or height > self.tile_sample_min_height): + return self.tiled_encode(x) + + x = self.encoder(x) + enc = self.quant_conv(x) + return enc + + @apply_forward_hook + def encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + r""" + Encode a batch of images into latents. + + Args: + x (`torch.Tensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + The latent representations of the encoded videos. If `return_dict` is True, a + [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. + """ + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [self._encode(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = self._encode(x) + + posterior = DiagonalGaussianDistribution(h) + + if not return_dict: + return (posterior,) + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + batch_size, num_channels, num_frames, height, width = z.shape + tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio + tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio + tile_latent_min_num_frames = self.tile_sample_min_num_frames // self.temporal_compression_ratio + + if self.use_framewise_decoding and num_frames > tile_latent_min_num_frames: + return self._temporal_tiled_decode(z, return_dict=return_dict) + + if self.use_tiling and (width > tile_latent_min_width or height > tile_latent_min_height): + return self.tiled_decode(z, return_dict=return_dict) + + z = self.post_quant_conv(z) + dec = self.decoder(z) + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + @apply_forward_hook + def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + r""" + Decode a batch of images. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + if self.use_slicing and z.shape[0] > 1: + decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)] + decoded = torch.cat(decoded_slices) + else: + decoded = self._decode(z).sample + + if not return_dict: + return (decoded,) + + return DecoderOutput(sample=decoded) + + def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[-2], b.shape[-2], blend_extent) + for y in range(blend_extent): + b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * ( + y / blend_extent + ) + return b + + def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[-1], b.shape[-1], blend_extent) + for x in range(blend_extent): + b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * ( + x / blend_extent + ) + return b + + def blend_t(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[-3], b.shape[-3], blend_extent) + for x in range(blend_extent): + b[:, :, x, :, :] = a[:, :, -blend_extent + x, :, :] * (1 - x / blend_extent) + b[:, :, x, :, :] * ( + x / blend_extent + ) + return b + + def tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput: + r"""Encode a batch of images using a tiled encoder. + + Args: + x (`torch.Tensor`): Input batch of videos. + + Returns: + `torch.Tensor`: + The latent representation of the encoded videos. + """ + batch_size, num_channels, num_frames, height, width = x.shape + latent_height = height // self.spatial_compression_ratio + latent_width = width // self.spatial_compression_ratio + + tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio + tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio + tile_latent_stride_height = self.tile_sample_stride_height // self.spatial_compression_ratio + tile_latent_stride_width = self.tile_sample_stride_width // self.spatial_compression_ratio + + blend_height = tile_latent_min_height - tile_latent_stride_height + blend_width = tile_latent_min_width - tile_latent_stride_width + + # Split x into overlapping tiles and encode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, height, self.tile_sample_stride_height): + row = [] + for j in range(0, width, self.tile_sample_stride_width): + tile = x[:, :, :, i : i + self.tile_sample_min_height, j : j + self.tile_sample_min_width] + tile = self.encoder(tile) + tile = self.quant_conv(tile) + row.append(tile) + rows.append(row) + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_height) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_width) + result_row.append(tile[:, :, :, :tile_latent_stride_height, :tile_latent_stride_width]) + result_rows.append(torch.cat(result_row, dim=4)) + + enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width] + return enc + + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + r""" + Decode a batch of images using a tiled decoder. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + + batch_size, num_channels, num_frames, height, width = z.shape + sample_height = height * self.spatial_compression_ratio + sample_width = width * self.spatial_compression_ratio + + tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio + tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio + tile_latent_stride_height = self.tile_sample_stride_height // self.spatial_compression_ratio + tile_latent_stride_width = self.tile_sample_stride_width // self.spatial_compression_ratio + + blend_height = self.tile_sample_min_height - self.tile_sample_stride_height + blend_width = self.tile_sample_min_width - self.tile_sample_stride_width + + # Split z into overlapping tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, height, tile_latent_stride_height): + row = [] + for j in range(0, width, tile_latent_stride_width): + tile = z[:, :, :, i : i + tile_latent_min_height, j : j + tile_latent_min_width] + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile) + row.append(decoded) + rows.append(row) + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_height) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_width) + result_row.append(tile[:, :, :, : self.tile_sample_stride_height, : self.tile_sample_stride_width]) + result_rows.append(torch.cat(result_row, dim=-1)) + + dec = torch.cat(result_rows, dim=3)[:, :, :, :sample_height, :sample_width] + + if not return_dict: + return (dec,) + return DecoderOutput(sample=dec) + + def _temporal_tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput: + batch_size, num_channels, num_frames, height, width = x.shape + latent_num_frames = (num_frames - 1) // self.temporal_compression_ratio + 1 + + tile_latent_min_num_frames = self.tile_sample_min_num_frames // self.temporal_compression_ratio + tile_latent_stride_num_frames = self.tile_sample_stride_num_frames // self.temporal_compression_ratio + blend_num_frames = tile_latent_min_num_frames - tile_latent_stride_num_frames + + row = [] + for i in range(0, num_frames, self.tile_sample_stride_num_frames): + tile = x[:, :, i : i + self.tile_sample_min_num_frames + 1, :, :] + if self.use_tiling and (height > self.tile_sample_min_height or width > self.tile_sample_min_width): + tile = self.tiled_encode(tile) + else: + tile = self.encoder(tile) + tile = self.quant_conv(tile) + if i > 0: + tile = tile[:, :, 1:, :, :] + row.append(tile) + + result_row = [] + for i, tile in enumerate(row): + if i > 0: + tile = self.blend_t(row[i - 1], tile, blend_num_frames) + result_row.append(tile[:, :, :tile_latent_stride_num_frames, :, :]) + else: + result_row.append(tile[:, :, : tile_latent_stride_num_frames + 1, :, :]) + + enc = torch.cat(result_row, dim=2)[:, :, :latent_num_frames] + return enc + + def _temporal_tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + batch_size, num_channels, num_frames, height, width = z.shape + num_sample_frames = (num_frames - 1) * self.temporal_compression_ratio + 1 + + tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio + tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio + tile_latent_min_num_frames = self.tile_sample_min_num_frames // self.temporal_compression_ratio + tile_latent_stride_num_frames = self.tile_sample_stride_num_frames // self.temporal_compression_ratio + blend_num_frames = self.tile_sample_min_num_frames - self.tile_sample_stride_num_frames + + row = [] + for i in range(0, num_frames, tile_latent_stride_num_frames): + tile = z[:, :, i : i + tile_latent_min_num_frames + 1, :, :] + if self.use_tiling and (tile.shape[-1] > tile_latent_min_width or tile.shape[-2] > tile_latent_min_height): + decoded = self.tiled_decode(tile, return_dict=True).sample + else: + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile) + if i > 0: + decoded = decoded[:, :, 1:, :, :] + row.append(decoded) + + result_row = [] + for i, tile in enumerate(row): + if i > 0: + tile = self.blend_t(row[i - 1], tile, blend_num_frames) + result_row.append(tile[:, :, : self.tile_sample_stride_num_frames, :, :]) + else: + result_row.append(tile[:, :, : self.tile_sample_stride_num_frames + 1, :, :]) + + dec = torch.cat(result_row, dim=2)[:, :, :num_sample_frames] + + if not return_dict: + return (dec,) + return DecoderOutput(sample=dec) + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + ) -> Union[DecoderOutput, torch.Tensor]: + r""" + Args: + sample (`torch.Tensor`): Input sample. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample from the posterior. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z, return_dict=return_dict) + return dec \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/qwenimage_transformer2d.py b/VideoX-Fun/videox_fun/models/qwenimage_transformer2d.py new file mode 100644 index 0000000000000000000000000000000000000000..0f99b90029c081145dedae79624118e1ead2d948 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/qwenimage_transformer2d.py @@ -0,0 +1,1118 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_qwenimage.py +# Copyright 2025 Qwen-Image Team, The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import functools +import inspect +import glob +import json +import math +import os +import types +import warnings +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.cuda.amp as amp +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.attention import Attention, FeedForward +from diffusers.models.attention_processor import ( + Attention, AttentionProcessor, CogVideoXAttnProcessor2_0, + FusedCogVideoXAttnProcessor2_0) +from diffusers.models.embeddings import (CogVideoXPatchEmbed, + TimestepEmbedding, Timesteps, + get_3d_sincos_pos_embed) +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import (AdaLayerNorm, + AdaLayerNormContinuous, + CogVideoXLayerNormZero, RMSNorm) +from diffusers.utils import (USE_PEFT_BACKEND, is_torch_version, logging, + scale_lora_layers, unscale_lora_layers) +from diffusers.utils.torch_utils import maybe_allow_in_graph +from torch import nn + +from ..dist import (QwenImageMultiGPUsAttnProcessor2_0, + get_sequence_parallel_rank, + get_sequence_parallel_world_size, get_sp_group) +from .attention_utils import attention +from .cache_utils import TeaCache +from ..utils import cfg_skip + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def get_timestep_embedding( + timesteps: torch.Tensor, + embedding_dim: int, + flip_sin_to_cos: bool = False, + downscale_freq_shift: float = 1, + scale: float = 1, + max_period: int = 10000, +) -> torch.Tensor: + """ + This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings. + + Args + timesteps (torch.Tensor): + a 1-D Tensor of N indices, one per batch element. These may be fractional. + embedding_dim (int): + the dimension of the output. + flip_sin_to_cos (bool): + Whether the embedding order should be `cos, sin` (if True) or `sin, cos` (if False) + downscale_freq_shift (float): + Controls the delta between frequencies between dimensions + scale (float): + Scaling factor applied to the embeddings. + max_period (int): + Controls the maximum frequency of the embeddings + Returns + torch.Tensor: an [N x dim] Tensor of positional embeddings. + """ + assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array" + + half_dim = embedding_dim // 2 + exponent = -math.log(max_period) * torch.arange( + start=0, end=half_dim, dtype=torch.float32, device=timesteps.device + ) + exponent = exponent / (half_dim - downscale_freq_shift) + + emb = torch.exp(exponent).to(timesteps.dtype) + emb = timesteps[:, None].float() * emb[None, :] + + # scale embeddings + emb = scale * emb + + # concat sine and cosine embeddings + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1) + + # flip sine and cosine embeddings + if flip_sin_to_cos: + emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1) + + # zero pad + if embedding_dim % 2 == 1: + emb = torch.nn.functional.pad(emb, (0, 1, 0, 0)) + return emb + + +def apply_rotary_emb_qwen( + x: torch.Tensor, + freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor]], + use_real: bool = True, + use_real_unbind_dim: int = -1, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Apply rotary embeddings to input tensors using the given frequency tensor. This function applies rotary embeddings + to the given query or key 'x' tensors using the provided frequency tensor 'freqs_cis'. The input tensors are + reshaped as complex numbers, and the frequency tensor is reshaped for broadcasting compatibility. The resulting + tensors contain rotary embeddings and are returned as real tensors. + + Args: + x (`torch.Tensor`): + Query or key tensor to apply rotary embeddings. [B, S, H, D] xk (torch.Tensor): Key tensor to apply + freqs_cis (`Tuple[torch.Tensor]`): Precomputed frequency tensor for complex exponentials. ([S, D], [S, D],) + + Returns: + Tuple[torch.Tensor, torch.Tensor]: Tuple of modified query tensor and key tensor with rotary embeddings. + """ + if use_real: + cos, sin = freqs_cis # [S, D] + cos = cos[None, None] + sin = sin[None, None] + cos, sin = cos.to(x.device), sin.to(x.device) + + if use_real_unbind_dim == -1: + # Used for flux, cogvideox, hunyuan-dit + x_real, x_imag = x.reshape(*x.shape[:-1], -1, 2).unbind(-1) # [B, S, H, D//2] + x_rotated = torch.stack([-x_imag, x_real], dim=-1).flatten(3) + elif use_real_unbind_dim == -2: + # Used for Stable Audio, OmniGen, CogView4 and Cosmos + x_real, x_imag = x.reshape(*x.shape[:-1], 2, -1).unbind(-2) # [B, S, H, D//2] + x_rotated = torch.cat([-x_imag, x_real], dim=-1) + else: + raise ValueError(f"`use_real_unbind_dim={use_real_unbind_dim}` but should be -1 or -2.") + + out = (x.float() * cos + x_rotated.float() * sin).to(x.dtype) + + return out + else: + x_rotated = torch.view_as_complex(x.float().reshape(*x.shape[:-1], -1, 2)) + freqs_cis = freqs_cis.unsqueeze(1) + x_out = torch.view_as_real(x_rotated * freqs_cis).flatten(3) + + return x_out.type_as(x) + + +class QwenTimestepProjEmbeddings(nn.Module): + def __init__(self, embedding_dim): + super().__init__() + + self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000) + self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim) + + def forward(self, timestep, hidden_states): + timesteps_proj = self.time_proj(timestep) + timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_states.dtype)) # (N, D) + + conditioning = timesteps_emb + + return conditioning + + +class QwenEmbedRope(nn.Module): + def __init__(self, theta: int, axes_dim: List[int], scale_rope=False): + super().__init__() + self.theta = theta + self.axes_dim = axes_dim + pos_index = torch.arange(4096) + neg_index = torch.arange(4096).flip(0) * -1 - 1 + self.pos_freqs = torch.cat( + [ + self.rope_params(pos_index, self.axes_dim[0], self.theta), + self.rope_params(pos_index, self.axes_dim[1], self.theta), + self.rope_params(pos_index, self.axes_dim[2], self.theta), + ], + dim=1, + ) + self.neg_freqs = torch.cat( + [ + self.rope_params(neg_index, self.axes_dim[0], self.theta), + self.rope_params(neg_index, self.axes_dim[1], self.theta), + self.rope_params(neg_index, self.axes_dim[2], self.theta), + ], + dim=1, + ) + self.rope_cache = {} + + # DO NOT USING REGISTER BUFFER HERE, IT WILL CAUSE COMPLEX NUMBERS LOSE ITS IMAGINARY PART + self.scale_rope = scale_rope + + def rope_params(self, index, dim, theta=10000): + """ + Args: + index: [0, 1, 2, 3] 1D Tensor representing the position index of the token + """ + assert dim % 2 == 0 + freqs = torch.outer(index, 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float32).div(dim))) + freqs = torch.polar(torch.ones_like(freqs), freqs) + return freqs + + def forward(self, video_fhw, txt_seq_lens, device): + """ + Args: video_fhw: [frame, height, width] a list of 3 integers representing the shape of the video Args: + txt_length: [bs] a list of 1 integers representing the length of the text + """ + if self.pos_freqs.device != device: + self.pos_freqs = self.pos_freqs.to(device) + self.neg_freqs = self.neg_freqs.to(device) + + if isinstance(video_fhw, list): + video_fhw = video_fhw[0] + if not isinstance(video_fhw, list): + video_fhw = [video_fhw] + + vid_freqs = [] + max_vid_index = 0 + for idx, fhw in enumerate(video_fhw): + frame, height, width = fhw + rope_key = f"{idx}_{frame}_{height}_{width}" + if not torch.compiler.is_compiling(): + if rope_key not in self.rope_cache: + self.rope_cache[rope_key] = self._compute_video_freqs(frame, height, width, idx) + video_freq = self.rope_cache[rope_key] + else: + video_freq = self._compute_video_freqs(frame, height, width, idx) + video_freq = video_freq.to(device) + vid_freqs.append(video_freq) + + if self.scale_rope: + max_vid_index = max(height // 2, width // 2, max_vid_index) + else: + max_vid_index = max(height, width, max_vid_index) + + max_len = max(txt_seq_lens) + txt_freqs = self.pos_freqs[max_vid_index : max_vid_index + max_len, ...] + vid_freqs = torch.cat(vid_freqs, dim=0) + + return vid_freqs, txt_freqs + + @functools.lru_cache(maxsize=None) + def _compute_video_freqs(self, frame, height, width, idx=0): + seq_lens = frame * height * width + freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1) + freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1) + + freqs_frame = freqs_pos[0][idx : idx + frame].view(frame, 1, 1, -1).expand(frame, height, width, -1) + if self.scale_rope: + freqs_height = torch.cat([freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]], dim=0) + freqs_height = freqs_height.view(1, height, 1, -1).expand(frame, height, width, -1) + freqs_width = torch.cat([freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]], dim=0) + freqs_width = freqs_width.view(1, 1, width, -1).expand(frame, height, width, -1) + else: + freqs_height = freqs_pos[1][:height].view(1, height, 1, -1).expand(frame, height, width, -1) + freqs_width = freqs_pos[2][:width].view(1, 1, width, -1).expand(frame, height, width, -1) + + freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-1).reshape(seq_lens, -1) + return freqs.clone().contiguous() + + +class QwenDoubleStreamAttnProcessor2_0: + """ + Attention processor for Qwen double-stream architecture, matching DoubleStreamLayerMegatron logic. This processor + implements joint attention computation where text and image streams are processed together. + """ + + _attention_backend = None + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError( + "QwenDoubleStreamAttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0." + ) + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, # Image stream + encoder_hidden_states: torch.FloatTensor = None, # Text stream + encoder_hidden_states_mask: torch.FloatTensor = None, + attention_mask: Optional[torch.FloatTensor] = None, + image_rotary_emb: Optional[torch.Tensor] = None, + ) -> torch.FloatTensor: + if encoder_hidden_states is None: + raise ValueError("QwenDoubleStreamAttnProcessor2_0 requires encoder_hidden_states (text stream)") + + seq_txt = encoder_hidden_states.shape[1] + + # Compute QKV for image stream (sample projections) + img_query = attn.to_q(hidden_states) + img_key = attn.to_k(hidden_states) + img_value = attn.to_v(hidden_states) + + # Compute QKV for text stream (context projections) + txt_query = attn.add_q_proj(encoder_hidden_states) + txt_key = attn.add_k_proj(encoder_hidden_states) + txt_value = attn.add_v_proj(encoder_hidden_states) + + # Reshape for multi-head attention + img_query = img_query.unflatten(-1, (attn.heads, -1)) + img_key = img_key.unflatten(-1, (attn.heads, -1)) + img_value = img_value.unflatten(-1, (attn.heads, -1)) + + txt_query = txt_query.unflatten(-1, (attn.heads, -1)) + txt_key = txt_key.unflatten(-1, (attn.heads, -1)) + txt_value = txt_value.unflatten(-1, (attn.heads, -1)) + + # Apply QK normalization + if attn.norm_q is not None: + img_query = attn.norm_q(img_query) + if attn.norm_k is not None: + img_key = attn.norm_k(img_key) + if attn.norm_added_q is not None: + txt_query = attn.norm_added_q(txt_query) + if attn.norm_added_k is not None: + txt_key = attn.norm_added_k(txt_key) + + # Apply RoPE + if image_rotary_emb is not None: + img_freqs, txt_freqs = image_rotary_emb + img_query = apply_rotary_emb_qwen(img_query, img_freqs, use_real=False) + img_key = apply_rotary_emb_qwen(img_key, img_freqs, use_real=False) + txt_query = apply_rotary_emb_qwen(txt_query, txt_freqs, use_real=False) + txt_key = apply_rotary_emb_qwen(txt_key, txt_freqs, use_real=False) + + # Concatenate for joint attention + # Order: [text, image] + joint_query = torch.cat([txt_query, img_query], dim=1) + joint_key = torch.cat([txt_key, img_key], dim=1) + joint_value = torch.cat([txt_value, img_value], dim=1) + + joint_hidden_states = attention( + joint_query, joint_key, joint_value, attn_mask=attention_mask, dropout_p=0.0, causal=False + ) + + # Reshape back + joint_hidden_states = joint_hidden_states.flatten(2, 3) + joint_hidden_states = joint_hidden_states.to(joint_query.dtype) + + # Split attention outputs back + txt_attn_output = joint_hidden_states[:, :seq_txt, :] # Text part + img_attn_output = joint_hidden_states[:, seq_txt:, :] # Image part + + # Apply output projections + img_attn_output = attn.to_out[0](img_attn_output) + if len(attn.to_out) > 1: + img_attn_output = attn.to_out[1](img_attn_output) # dropout + + txt_attn_output = attn.to_add_out(txt_attn_output) + + return img_attn_output, txt_attn_output + + +@maybe_allow_in_graph +class QwenImageTransformerBlock(nn.Module): + def __init__( + self, dim: int, num_attention_heads: int, attention_head_dim: int, qk_norm: str = "rms_norm", eps: float = 1e-6 + ): + super().__init__() + + self.dim = dim + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + + # Image processing modules + self.img_mod = nn.Sequential( + nn.SiLU(), + nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2 + ) + self.img_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.attn = Attention( + query_dim=dim, + cross_attention_dim=None, # Enable cross attention for joint computation + added_kv_proj_dim=dim, # Enable added KV projections for text stream + dim_head=attention_head_dim, + heads=num_attention_heads, + out_dim=dim, + context_pre_only=False, + bias=True, + processor=QwenDoubleStreamAttnProcessor2_0(), + qk_norm=qk_norm, + eps=eps, + ) + self.img_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.img_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") + + # Text processing modules + self.txt_mod = nn.Sequential( + nn.SiLU(), + nn.Linear(dim, 6 * dim, bias=True), # For scale, shift, gate for norm1 and norm2 + ) + self.txt_norm1 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + # Text doesn't need separate attention - it's handled by img_attn joint computation + self.txt_norm2 = nn.LayerNorm(dim, elementwise_affine=False, eps=eps) + self.txt_mlp = FeedForward(dim=dim, dim_out=dim, activation_fn="gelu-approximate") + + def _modulate(self, x, mod_params): + """Apply modulation to input tensor""" + shift, scale, gate = mod_params.chunk(3, dim=-1) + return x * (1 + scale.unsqueeze(1)) + shift.unsqueeze(1), gate.unsqueeze(1) + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor, + encoder_hidden_states_mask: torch.Tensor, + temb: torch.Tensor, + image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + # Get modulation parameters for both streams + img_mod_params = self.img_mod(temb) # [B, 6*dim] + txt_mod_params = self.txt_mod(temb) # [B, 6*dim] + + # Split modulation parameters for norm1 and norm2 + img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1) # Each [B, 3*dim] + txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1) # Each [B, 3*dim] + + # Process image stream - norm1 + modulation + img_normed = self.img_norm1(hidden_states) + img_modulated, img_gate1 = self._modulate(img_normed, img_mod1) + + # Process text stream - norm1 + modulation + txt_normed = self.txt_norm1(encoder_hidden_states) + txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1) + + # Use QwenAttnProcessor2_0 for joint attention computation + # This directly implements the DoubleStreamLayerMegatron logic: + # 1. Computes QKV for both streams + # 2. Applies QK normalization and RoPE + # 3. Concatenates and runs joint attention + # 4. Splits results back to separate streams + joint_attention_kwargs = joint_attention_kwargs or {} + attn_output = self.attn( + hidden_states=img_modulated, # Image stream (will be processed as "sample") + encoder_hidden_states=txt_modulated, # Text stream (will be processed as "context") + encoder_hidden_states_mask=encoder_hidden_states_mask, + image_rotary_emb=image_rotary_emb, + **joint_attention_kwargs, + ) + + # QwenAttnProcessor2_0 returns (img_output, txt_output) when encoder_hidden_states is provided + img_attn_output, txt_attn_output = attn_output + + # Apply attention gates and add residual (like in Megatron) + hidden_states = hidden_states + img_gate1 * img_attn_output + encoder_hidden_states = encoder_hidden_states + txt_gate1 * txt_attn_output + + # Process image stream - norm2 + MLP + img_normed2 = self.img_norm2(hidden_states) + img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2) + img_mlp_output = self.img_mlp(img_modulated2) + hidden_states = hidden_states + img_gate2 * img_mlp_output + + # Process text stream - norm2 + MLP + txt_normed2 = self.txt_norm2(encoder_hidden_states) + txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2) + txt_mlp_output = self.txt_mlp(txt_modulated2) + encoder_hidden_states = encoder_hidden_states + txt_gate2 * txt_mlp_output + + # Clip to prevent overflow for fp16 + if encoder_hidden_states.dtype == torch.float16: + encoder_hidden_states = encoder_hidden_states.clip(-65504, 65504) + if hidden_states.dtype == torch.float16: + hidden_states = hidden_states.clip(-65504, 65504) + + return encoder_hidden_states, hidden_states + + +class QwenImageTransformer2DModel(ModelMixin, ConfigMixin, PeftAdapterMixin, FromOriginalModelMixin): + """ + The Transformer model introduced in Qwen. + + Args: + patch_size (`int`, defaults to `2`): + Patch size to turn the input data into small patches. + in_channels (`int`, defaults to `64`): + The number of channels in the input. + out_channels (`int`, *optional*, defaults to `None`): + The number of channels in the output. If not specified, it defaults to `in_channels`. + num_layers (`int`, defaults to `60`): + The number of layers of dual stream DiT blocks to use. + attention_head_dim (`int`, defaults to `128`): + The number of dimensions to use for each attention head. + num_attention_heads (`int`, defaults to `24`): + The number of attention heads to use. + joint_attention_dim (`int`, defaults to `3584`): + The number of dimensions to use for the joint attention (embedding/channel dimension of + `encoder_hidden_states`). + guidance_embeds (`bool`, defaults to `False`): + Whether to use guidance embeddings for guidance-distilled variant of the model. + axes_dims_rope (`Tuple[int]`, defaults to `(16, 56, 56)`): + The dimensions to use for the rotary positional embeddings. + """ + + # _supports_gradient_checkpointing = True + # _no_split_modules = ["QwenImageTransformerBlock"] + # _skip_layerwise_casting_patterns = ["pos_embed", "norm"] + # _repeated_blocks = ["QwenImageTransformerBlock"] + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + patch_size: int = 2, + in_channels: int = 64, + out_channels: Optional[int] = 16, + num_layers: int = 60, + attention_head_dim: int = 128, + num_attention_heads: int = 24, + joint_attention_dim: int = 3584, + guidance_embeds: bool = False, # TODO: this should probably be removed + axes_dims_rope: Tuple[int, int, int] = (16, 56, 56), + ): + super().__init__() + self.out_channels = out_channels or in_channels + self.inner_dim = num_attention_heads * attention_head_dim + + self.pos_embed = QwenEmbedRope(theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True) + + self.time_text_embed = QwenTimestepProjEmbeddings(embedding_dim=self.inner_dim) + + self.txt_norm = RMSNorm(joint_attention_dim, eps=1e-6) + + self.img_in = nn.Linear(in_channels, self.inner_dim) + self.txt_in = nn.Linear(joint_attention_dim, self.inner_dim) + + self.transformer_blocks = nn.ModuleList( + [ + QwenImageTransformerBlock( + dim=self.inner_dim, + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + ) + for _ in range(num_layers) + ] + ) + + self.norm_out = AdaLayerNormContinuous(self.inner_dim, self.inner_dim, elementwise_affine=False, eps=1e-6) + self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True) + + self.teacache = None + self.cfg_skip_ratio = None + self.current_steps = 0 + self.num_inference_steps = None + self.gradient_checkpointing = False + self.sp_world_size = 1 + self.sp_world_rank = 0 + + def _set_gradient_checkpointing(self, *args, **kwargs): + if "value" in kwargs: + self.gradient_checkpointing = kwargs["value"] + elif "enable" in kwargs: + self.gradient_checkpointing = kwargs["enable"] + else: + raise ValueError("Invalid set gradient checkpointing") + + def enable_multi_gpus_inference(self,): + self.sp_world_size = get_sequence_parallel_world_size() + self.sp_world_rank = get_sequence_parallel_rank() + self.all_gather = get_sp_group().all_gather + self.set_attn_processor(QwenImageMultiGPUsAttnProcessor2_0()) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def enable_cfg_skip(self, cfg_skip_ratio, num_steps): + if cfg_skip_ratio != 0: + self.cfg_skip_ratio = cfg_skip_ratio + self.current_steps = 0 + self.num_inference_steps = num_steps + else: + self.cfg_skip_ratio = None + self.current_steps = 0 + self.num_inference_steps = None + + def share_cfg_skip( + self, + transformer = None, + ): + self.cfg_skip_ratio = transformer.cfg_skip_ratio + self.current_steps = transformer.current_steps + self.num_inference_steps = transformer.num_inference_steps + + def disable_cfg_skip(self): + self.cfg_skip_ratio = None + self.current_steps = 0 + self.num_inference_steps = None + + def enable_teacache( + self, + coefficients, + num_steps: int, + rel_l1_thresh: float, + num_skip_start_steps: int = 0, + offload: bool = True, + ): + self.teacache = TeaCache( + coefficients, num_steps, rel_l1_thresh=rel_l1_thresh, num_skip_start_steps=num_skip_start_steps, offload=offload + ) + + def share_teacache( + self, + transformer = None, + ): + self.teacache = transformer.teacache + + def disable_teacache(self): + self.teacache = None + + @cfg_skip() + def forward_bs(self, x, *args, **kwargs): + func = self.forward + sig = inspect.signature(func) + + bs = len(x) + bs_half = int(bs // 2) + + if bs >= 2: + # cond + x_i = x[bs_half:] + args_i = [ + arg[bs_half:] if + isinstance(arg, + (torch.Tensor, list, tuple, np.ndarray)) and + len(arg) == bs else arg for arg in args + ] + kwargs_i = { + k: (v[bs_half:] if + isinstance(v, + (torch.Tensor, list, tuple, + np.ndarray)) and len(v) == bs else v + ) for k, v in kwargs.items() + } + if 'cond_flag' in sig.parameters: + kwargs_i["cond_flag"] = True + + cond_out = func(x_i, *args_i, **kwargs_i) + + # uncond + uncond_x_i = x[:bs_half] + uncond_args_i = [ + arg[:bs_half] if + isinstance(arg, + (torch.Tensor, list, tuple, np.ndarray)) and + len(arg) == bs else arg for arg in args + ] + uncond_kwargs_i = { + k: (v[:bs_half] if + isinstance(v, + (torch.Tensor, list, tuple, + np.ndarray)) and len(v) == bs else v + ) for k, v in kwargs.items() + } + if 'cond_flag' in sig.parameters: + uncond_kwargs_i["cond_flag"] = False + uncond_out = func(uncond_x_i, *uncond_args_i, + **uncond_kwargs_i) + + x = torch.cat([uncond_out, cond_out], dim=0) + else: + x = func(x, *args, **kwargs) + + return x + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: torch.Tensor = None, + encoder_hidden_states_mask: torch.Tensor = None, + timestep: torch.LongTensor = None, + img_shapes: Optional[List[Tuple[int, int, int]]] = None, + txt_seq_lens: Optional[List[int]] = None, + guidance: torch.Tensor = None, # TODO: this should probably be removed + attention_kwargs: Optional[Dict[str, Any]] = None, + cond_flag: bool = True, + return_dict: bool = True, + ) -> Union[torch.Tensor, Transformer2DModelOutput]: + """ + The [`QwenTransformer2DModel`] forward method. + + Args: + hidden_states (`torch.Tensor` of shape `(batch_size, image_sequence_length, in_channels)`): + Input `hidden_states`. + encoder_hidden_states (`torch.Tensor` of shape `(batch_size, text_sequence_length, joint_attention_dim)`): + Conditional embeddings (embeddings computed from the input conditions such as prompts) to use. + encoder_hidden_states_mask (`torch.Tensor` of shape `(batch_size, text_sequence_length)`): + Mask of the input conditions. + timestep ( `torch.LongTensor`): + Used to indicate denoising step. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.transformer_2d.Transformer2DModelOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + if attention_kwargs is not None: + attention_kwargs = attention_kwargs.copy() + lora_scale = attention_kwargs.pop("scale", 1.0) + else: + lora_scale = 1.0 + + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + else: + if attention_kwargs is not None and attention_kwargs.get("scale", None) is not None: + logger.warning( + "Passing `scale` via `joint_attention_kwargs` when not using the PEFT backend is ineffective." + ) + + if isinstance(encoder_hidden_states, list): + encoder_hidden_states = torch.stack(encoder_hidden_states) + encoder_hidden_states_mask = torch.stack(encoder_hidden_states_mask) + + hidden_states = self.img_in(hidden_states) + + timestep = timestep.to(hidden_states.dtype) + encoder_hidden_states = self.txt_norm(encoder_hidden_states) + encoder_hidden_states = self.txt_in(encoder_hidden_states) + + if guidance is not None: + guidance = guidance.to(hidden_states.dtype) * 1000 + + temb = ( + self.time_text_embed(timestep, hidden_states) + if guidance is None + else self.time_text_embed(timestep, guidance, hidden_states) + ) + + image_rotary_emb = self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device) + + # Context Parallel + if self.sp_world_size > 1: + hidden_states = torch.chunk(hidden_states, self.sp_world_size, dim=1)[self.sp_world_rank] + if image_rotary_emb is not None: + image_rotary_emb = ( + torch.chunk(image_rotary_emb[0], self.sp_world_size, dim=0)[self.sp_world_rank], + image_rotary_emb[1] + ) + + # TeaCache + if self.teacache is not None: + if cond_flag: + inp = hidden_states.clone() + temb_ = temb.clone() + encoder_hidden_states_ = encoder_hidden_states.clone() + + img_mod_params_ = self.transformer_blocks[0].img_mod(temb_) + img_mod1_, img_mod2_ = img_mod_params_.chunk(2, dim=-1) + img_normed_ = self.transformer_blocks[0].img_norm1(inp) + modulated_inp, img_gate1_ = self.transformer_blocks[0]._modulate(img_normed_, img_mod1_) + + skip_flag = self.teacache.cnt < self.teacache.num_skip_start_steps + if skip_flag: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + else: + if cond_flag: + rel_l1_distance = self.teacache.compute_rel_l1_distance(self.teacache.previous_modulated_input, modulated_inp) + self.teacache.accumulated_rel_l1_distance += self.teacache.rescale_func(rel_l1_distance) + if self.teacache.accumulated_rel_l1_distance < self.teacache.rel_l1_thresh: + self.should_calc = False + else: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + self.teacache.previous_modulated_input = modulated_inp + self.teacache.should_calc = self.should_calc + else: + self.should_calc = self.teacache.should_calc + + # TeaCache + if self.teacache is not None: + if not self.should_calc: + previous_residual = self.teacache.previous_residual_cond if cond_flag else self.teacache.previous_residual_uncond + hidden_states = hidden_states + previous_residual.to(hidden_states.device)[-hidden_states.size()[0]:,] + else: + ori_hidden_states = hidden_states.clone().cpu() if self.teacache.offload else hidden_states.clone() + + # 4. Transformer blocks + for i, block in enumerate(self.transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + encoder_hidden_states, hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + encoder_hidden_states_mask, + temb, + image_rotary_emb, + **ckpt_kwargs, + ) + + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + encoder_hidden_states_mask=encoder_hidden_states_mask, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=attention_kwargs, + ) + + if cond_flag: + self.teacache.previous_residual_cond = hidden_states.cpu() - ori_hidden_states if self.teacache.offload else hidden_states - ori_hidden_states + else: + self.teacache.previous_residual_uncond = hidden_states.cpu() - ori_hidden_states if self.teacache.offload else hidden_states - ori_hidden_states + del ori_hidden_states + else: + for index_block, block in enumerate(self.transformer_blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + encoder_hidden_states, hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + encoder_hidden_states, + encoder_hidden_states_mask, + temb, + image_rotary_emb, + **ckpt_kwargs, + ) + + else: + encoder_hidden_states, hidden_states = block( + hidden_states=hidden_states, + encoder_hidden_states=encoder_hidden_states, + encoder_hidden_states_mask=encoder_hidden_states_mask, + temb=temb, + image_rotary_emb=image_rotary_emb, + joint_attention_kwargs=attention_kwargs, + ) + + # Use only the image part (hidden_states) from the dual-stream blocks + hidden_states = self.norm_out(hidden_states, temb) + output = self.proj_out(hidden_states) + + if self.sp_world_size > 1: + output = self.all_gather(output, dim=1) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + if self.teacache is not None and cond_flag: + self.teacache.cnt += 1 + if self.teacache.cnt == self.teacache.num_steps: + self.teacache.reset() + return output + + @classmethod + def from_pretrained( + cls, pretrained_model_path, subfolder=None, transformer_additional_kwargs={}, + low_cpu_mem_usage=False, torch_dtype=torch.bfloat16 + ): + if subfolder is not None: + pretrained_model_path = os.path.join(pretrained_model_path, subfolder) + print(f"loaded 3D transformer's pretrained weights from {pretrained_model_path} ...") + + config_file = os.path.join(pretrained_model_path, 'config.json') + if not os.path.isfile(config_file): + raise RuntimeError(f"{config_file} does not exist") + with open(config_file, "r") as f: + config = json.load(f) + + from diffusers.utils import WEIGHTS_NAME + model_file = os.path.join(pretrained_model_path, WEIGHTS_NAME) + model_file_safetensors = model_file.replace(".bin", ".safetensors") + + if "dict_mapping" in transformer_additional_kwargs.keys(): + for key in transformer_additional_kwargs["dict_mapping"]: + transformer_additional_kwargs[transformer_additional_kwargs["dict_mapping"][key]] = config[key] + + if low_cpu_mem_usage: + try: + import re + + from diffusers import __version__ as diffusers_version + if diffusers_version >= "0.33.0": + from diffusers.models.model_loading_utils import \ + load_model_dict_into_meta + else: + from diffusers.models.modeling_utils import \ + load_model_dict_into_meta + from diffusers.utils import is_accelerate_available + if is_accelerate_available(): + import accelerate + + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + model = cls.from_config(config, **transformer_additional_kwargs) + + param_device = "cpu" + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + print(model_files_safetensors) + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + filtered_state_dict = {} + for key in state_dict: + if key in model.state_dict() and model.state_dict()[key].size() == state_dict[key].size(): + filtered_state_dict[key] = state_dict[key] + else: + print(f"Skipping key '{key}' due to size mismatch or absence in model.") + + model_keys = set(model.state_dict().keys()) + loaded_keys = set(filtered_state_dict.keys()) + missing_keys = model_keys - loaded_keys + + def initialize_missing_parameters(missing_keys, model_state_dict, torch_dtype=None): + initialized_dict = {} + + with torch.no_grad(): + for key in missing_keys: + param_shape = model_state_dict[key].shape + param_dtype = torch_dtype if torch_dtype is not None else model_state_dict[key].dtype + if 'weight' in key: + if any(norm_type in key for norm_type in ['norm', 'ln_', 'layer_norm', 'group_norm', 'batch_norm']): + initialized_dict[key] = torch.ones(param_shape, dtype=param_dtype) + elif 'embedding' in key or 'embed' in key: + initialized_dict[key] = torch.randn(param_shape, dtype=param_dtype) * 0.02 + elif 'head' in key or 'output' in key or 'proj_out' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif len(param_shape) >= 2: + initialized_dict[key] = torch.empty(param_shape, dtype=param_dtype) + nn.init.xavier_uniform_(initialized_dict[key]) + else: + initialized_dict[key] = torch.randn(param_shape, dtype=param_dtype) * 0.02 + elif 'bias' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif 'running_mean' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif 'running_var' in key: + initialized_dict[key] = torch.ones(param_shape, dtype=param_dtype) + elif 'num_batches_tracked' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=torch.long) + else: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + + return initialized_dict + + if missing_keys: + print(f"Missing keys will be initialized: {sorted(missing_keys)}") + initialized_params = initialize_missing_parameters( + missing_keys, + model.state_dict(), + torch_dtype + ) + filtered_state_dict.update(initialized_params) + + if diffusers_version >= "0.33.0": + # Diffusers has refactored `load_model_dict_into_meta` since version 0.33.0 in this commit: + # https://github.com/huggingface/diffusers/commit/f5929e03060d56063ff34b25a8308833bec7c785. + load_model_dict_into_meta( + model, + filtered_state_dict, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + else: + model._convert_deprecated_attention_blocks(filtered_state_dict) + unexpected_keys = load_model_dict_into_meta( + model, + filtered_state_dict, + device=param_device, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + + if cls._keys_to_ignore_on_load_unexpected is not None: + for pat in cls._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + print( + f"Some weights of the model checkpoint were not used when initializing {cls.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + + return model + except Exception as e: + print( + f"The low_cpu_mem_usage mode is not work because {e}. Use low_cpu_mem_usage=False instead." + ) + + model = cls.from_config(config, **transformer_additional_kwargs) + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + tmp_state_dict = {} + for key in state_dict: + if key in model.state_dict().keys() and model.state_dict()[key].size() == state_dict[key].size(): + tmp_state_dict[key] = state_dict[key] + else: + print(key, "Size don't match, skip") + + state_dict = tmp_state_dict + + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m) + + params = [p.numel() if "." in n else 0 for n, p in model.named_parameters()] + print(f"### All Parameters: {sum(params) / 1e6} M") + + params = [p.numel() if "attn1." in n else 0 for n, p in model.named_parameters()] + print(f"### attn1 Parameters: {sum(params) / 1e6} M") + + model = model.to(torch_dtype) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/qwenimage_vae.py b/VideoX-Fun/videox_fun/models/qwenimage_vae.py new file mode 100644 index 0000000000000000000000000000000000000000..72e34cc3054f7d4cf46165c0114aa4df369c6462 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/qwenimage_vae.py @@ -0,0 +1,1087 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/autoencoders/autoencoder_kl_qwenimage.py +# Copyright 2025 The Qwen-Image Team, Wan Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# We gratefully acknowledge the Wan Team for their outstanding contributions. +# QwenImageVAE is further fine-tuned from the Wan Video VAE to achieve improved performance. +# For more information about the Wan VAE, please refer to: +# - GitHub: https://github.com/Wan-Video/Wan2.1 +# - arXiv: https://arxiv.org/abs/2503.20314 + +import functools +import glob +import json +import math +import os +import types +import warnings +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.cuda.amp as amp +import torch.nn as nn +import torch.nn.functional as F +import torch.utils.checkpoint +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.activations import get_activation +from diffusers.models.attention import FeedForward +from diffusers.models.attention_processor import Attention +from diffusers.models.autoencoders.vae import (DecoderOutput, + DiagonalGaussianDistribution) +from diffusers.models.embeddings import TimestepEmbedding, Timesteps +from diffusers.models.modeling_outputs import (AutoencoderKLOutput, + Transformer2DModelOutput) +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import AdaLayerNormContinuous, RMSNorm +from diffusers.utils import (USE_PEFT_BACKEND, is_torch_version, logging, + scale_lora_layers, unscale_lora_layers) +from diffusers.utils.accelerate_utils import apply_forward_hook +from diffusers.utils.torch_utils import maybe_allow_in_graph +from torch import nn + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +CACHE_T = 2 + +class QwenImageCausalConv3d(nn.Conv3d): + r""" + A custom 3D causal convolution layer with feature caching support. + + This layer extends the standard Conv3D layer by ensuring causality in the time dimension and handling feature + caching for efficient inference. + + Args: + in_channels (int): Number of channels in the input image + out_channels (int): Number of channels produced by the convolution + kernel_size (int or tuple): Size of the convolving kernel + stride (int or tuple, optional): Stride of the convolution. Default: 1 + padding (int or tuple, optional): Zero-padding added to all three sides of the input. Default: 0 + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + kernel_size: Union[int, Tuple[int, int, int]], + stride: Union[int, Tuple[int, int, int]] = 1, + padding: Union[int, Tuple[int, int, int]] = 0, + ) -> None: + super().__init__( + in_channels=in_channels, + out_channels=out_channels, + kernel_size=kernel_size, + stride=stride, + padding=padding, + ) + + # Set up causal padding + self._padding = (self.padding[2], self.padding[2], self.padding[1], self.padding[1], 2 * self.padding[0], 0) + self.padding = (0, 0, 0) + + def forward(self, x, cache_x=None): + padding = list(self._padding) + if cache_x is not None and self._padding[4] > 0: + cache_x = cache_x.to(x.device) + x = torch.cat([cache_x, x], dim=2) + padding[4] -= cache_x.shape[2] + x = F.pad(x, padding) + return super().forward(x) + + +class QwenImageRMS_norm(nn.Module): + r""" + A custom RMS normalization layer. + + Args: + dim (int): The number of dimensions to normalize over. + channel_first (bool, optional): Whether the input tensor has channels as the first dimension. + Default is True. + images (bool, optional): Whether the input represents image data. Default is True. + bias (bool, optional): Whether to include a learnable bias term. Default is False. + """ + + def __init__(self, dim: int, channel_first: bool = True, images: bool = True, bias: bool = False) -> None: + super().__init__() + broadcastable_dims = (1, 1, 1) if not images else (1, 1) + shape = (dim, *broadcastable_dims) if channel_first else (dim,) + + self.channel_first = channel_first + self.scale = dim**0.5 + self.gamma = nn.Parameter(torch.ones(shape)) + self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.0 + + def forward(self, x): + return F.normalize(x, dim=(1 if self.channel_first else -1)) * self.scale * self.gamma + self.bias + + +class QwenImageUpsample(nn.Upsample): + r""" + Perform upsampling while ensuring the output tensor has the same data type as the input. + + Args: + x (torch.Tensor): Input tensor to be upsampled. + + Returns: + torch.Tensor: Upsampled tensor with the same data type as the input. + """ + + def forward(self, x): + return super().forward(x.float()).type_as(x) + + +class QwenImageResample(nn.Module): + r""" + A custom resampling module for 2D and 3D data. + + Args: + dim (int): The number of input/output channels. + mode (str): The resampling mode. Must be one of: + - 'none': No resampling (identity operation). + - 'upsample2d': 2D upsampling with nearest-exact interpolation and convolution. + - 'upsample3d': 3D upsampling with nearest-exact interpolation, convolution, and causal 3D convolution. + - 'downsample2d': 2D downsampling with zero-padding and convolution. + - 'downsample3d': 3D downsampling with zero-padding, convolution, and causal 3D convolution. + """ + + def __init__(self, dim: int, mode: str) -> None: + super().__init__() + self.dim = dim + self.mode = mode + + # layers + if mode == "upsample2d": + self.resample = nn.Sequential( + QwenImageUpsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim // 2, 3, padding=1), + ) + elif mode == "upsample3d": + self.resample = nn.Sequential( + QwenImageUpsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim // 2, 3, padding=1), + ) + self.time_conv = QwenImageCausalConv3d(dim, dim * 2, (3, 1, 1), padding=(1, 0, 0)) + + elif mode == "downsample2d": + self.resample = nn.Sequential(nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2))) + elif mode == "downsample3d": + self.resample = nn.Sequential(nn.ZeroPad2d((0, 1, 0, 1)), nn.Conv2d(dim, dim, 3, stride=(2, 2))) + self.time_conv = QwenImageCausalConv3d(dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0)) + + else: + self.resample = nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + b, c, t, h, w = x.size() + if self.mode == "upsample3d": + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = "Rep" + feat_idx[0] += 1 + else: + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None and feat_cache[idx] != "Rep": + # cache last frame of last two chunk + cache_x = torch.cat( + [feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], dim=2 + ) + if cache_x.shape[2] < 2 and feat_cache[idx] is not None and feat_cache[idx] == "Rep": + cache_x = torch.cat([torch.zeros_like(cache_x).to(cache_x.device), cache_x], dim=2) + if feat_cache[idx] == "Rep": + x = self.time_conv(x) + else: + x = self.time_conv(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + + x = x.reshape(b, 2, c, t, h, w) + x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), 3) + x = x.reshape(b, c, t * 2, h, w) + t = x.shape[2] + x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w) + x = self.resample(x) + x = x.view(b, t, x.size(1), x.size(2), x.size(3)).permute(0, 2, 1, 3, 4) + + if self.mode == "downsample3d": + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = x.clone() + feat_idx[0] += 1 + else: + cache_x = x[:, :, -1:, :, :].clone() + x = self.time_conv(torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2)) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + return x + + +class QwenImageResidualBlock(nn.Module): + r""" + A custom residual block module. + + Args: + in_dim (int): Number of input channels. + out_dim (int): Number of output channels. + dropout (float, optional): Dropout rate for the dropout layer. Default is 0.0. + non_linearity (str, optional): Type of non-linearity to use. Default is "silu". + """ + + def __init__( + self, + in_dim: int, + out_dim: int, + dropout: float = 0.0, + non_linearity: str = "silu", + ) -> None: + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + self.nonlinearity = get_activation(non_linearity) + + # layers + self.norm1 = QwenImageRMS_norm(in_dim, images=False) + self.conv1 = QwenImageCausalConv3d(in_dim, out_dim, 3, padding=1) + self.norm2 = QwenImageRMS_norm(out_dim, images=False) + self.dropout = nn.Dropout(dropout) + self.conv2 = QwenImageCausalConv3d(out_dim, out_dim, 3, padding=1) + self.conv_shortcut = QwenImageCausalConv3d(in_dim, out_dim, 1) if in_dim != out_dim else nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + # Apply shortcut connection + h = self.conv_shortcut(x) + + # First normalization and activation + x = self.norm1(x) + x = self.nonlinearity(x) + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat([feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], dim=2) + + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + # Second normalization and activation + x = self.norm2(x) + x = self.nonlinearity(x) + + # Dropout + x = self.dropout(x) + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat([feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], dim=2) + + x = self.conv2(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv2(x) + + # Add residual connection + return x + h + + +class QwenImageAttentionBlock(nn.Module): + r""" + Causal self-attention with a single head. + + Args: + dim (int): The number of channels in the input tensor. + """ + + def __init__(self, dim): + super().__init__() + self.dim = dim + + # layers + self.norm = QwenImageRMS_norm(dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + self.proj = nn.Conv2d(dim, dim, 1) + + def forward(self, x): + identity = x + batch_size, channels, time, height, width = x.size() + + x = x.permute(0, 2, 1, 3, 4).reshape(batch_size * time, channels, height, width) + x = self.norm(x) + + # compute query, key, value + qkv = self.to_qkv(x) + qkv = qkv.reshape(batch_size * time, 1, channels * 3, -1) + qkv = qkv.permute(0, 1, 3, 2).contiguous() + q, k, v = qkv.chunk(3, dim=-1) + + # apply attention + x = F.scaled_dot_product_attention(q, k, v) + + x = x.squeeze(1).permute(0, 2, 1).reshape(batch_size * time, channels, height, width) + + # output projection + x = self.proj(x) + + # Reshape back: [(b*t), c, h, w] -> [b, c, t, h, w] + x = x.view(batch_size, time, channels, height, width) + x = x.permute(0, 2, 1, 3, 4) + + return x + identity + + +class QwenImageMidBlock(nn.Module): + """ + Middle block for QwenImageVAE encoder and decoder. + + Args: + dim (int): Number of input/output channels. + dropout (float): Dropout rate. + non_linearity (str): Type of non-linearity to use. + """ + + def __init__(self, dim: int, dropout: float = 0.0, non_linearity: str = "silu", num_layers: int = 1): + super().__init__() + self.dim = dim + + # Create the components + resnets = [QwenImageResidualBlock(dim, dim, dropout, non_linearity)] + attentions = [] + for _ in range(num_layers): + attentions.append(QwenImageAttentionBlock(dim)) + resnets.append(QwenImageResidualBlock(dim, dim, dropout, non_linearity)) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward(self, x, feat_cache=None, feat_idx=[0]): + # First residual block + x = self.resnets[0](x, feat_cache, feat_idx) + + # Process through attention and residual blocks + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + x = attn(x) + + x = resnet(x, feat_cache, feat_idx) + + return x + + +class QwenImageEncoder3d(nn.Module): + r""" + A 3D encoder module. + + Args: + dim (int): The base number of channels in the first layer. + z_dim (int): The dimensionality of the latent space. + dim_mult (list of int): Multipliers for the number of channels in each block. + num_res_blocks (int): Number of residual blocks in each block. + attn_scales (list of float): Scales at which to apply attention mechanisms. + temperal_downsample (list of bool): Whether to downsample temporally in each block. + dropout (float): Dropout rate for the dropout layers. + non_linearity (str): Type of non-linearity to use. + """ + + def __init__( + self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[True, True, False], + dropout=0.0, + non_linearity: str = "silu", + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + self.nonlinearity = get_activation(non_linearity) + + # dimensions + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv_in = QwenImageCausalConv3d(3, dims[0], 3, padding=1) + + # downsample blocks + self.down_blocks = nn.ModuleList([]) + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + for _ in range(num_res_blocks): + self.down_blocks.append(QwenImageResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + self.down_blocks.append(QwenImageAttentionBlock(out_dim)) + in_dim = out_dim + + # downsample block + if i != len(dim_mult) - 1: + mode = "downsample3d" if temperal_downsample[i] else "downsample2d" + self.down_blocks.append(QwenImageResample(out_dim, mode=mode)) + scale /= 2.0 + + # middle blocks + self.mid_block = QwenImageMidBlock(out_dim, dropout, non_linearity, num_layers=1) + + # output blocks + self.norm_out = QwenImageRMS_norm(out_dim, images=False) + self.conv_out = QwenImageCausalConv3d(out_dim, z_dim, 3, padding=1) + + self.gradient_checkpointing = False + + def forward(self, x, feat_cache=None, feat_idx=[0]): + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], dim=2) + x = self.conv_in(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv_in(x) + + ## downsamples + for layer in self.down_blocks: + if feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## middle + x = self.mid_block(x, feat_cache, feat_idx) + + ## head + x = self.norm_out(x) + x = self.nonlinearity(x) + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], dim=2) + x = self.conv_out(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv_out(x) + return x + + +class QwenImageUpBlock(nn.Module): + """ + A block that handles upsampling for the QwenImageVAE decoder. + + Args: + in_dim (int): Input dimension + out_dim (int): Output dimension + num_res_blocks (int): Number of residual blocks + dropout (float): Dropout rate + upsample_mode (str, optional): Mode for upsampling ('upsample2d' or 'upsample3d') + non_linearity (str): Type of non-linearity to use + """ + + def __init__( + self, + in_dim: int, + out_dim: int, + num_res_blocks: int, + dropout: float = 0.0, + upsample_mode: Optional[str] = None, + non_linearity: str = "silu", + ): + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + # Create layers list + resnets = [] + # Add residual blocks and attention if needed + current_dim = in_dim + for _ in range(num_res_blocks + 1): + resnets.append(QwenImageResidualBlock(current_dim, out_dim, dropout, non_linearity)) + current_dim = out_dim + + self.resnets = nn.ModuleList(resnets) + + # Add upsampling layer if needed + self.upsamplers = None + if upsample_mode is not None: + self.upsamplers = nn.ModuleList([QwenImageResample(out_dim, mode=upsample_mode)]) + + self.gradient_checkpointing = False + + def forward(self, x, feat_cache=None, feat_idx=[0]): + """ + Forward pass through the upsampling block. + + Args: + x (torch.Tensor): Input tensor + feat_cache (list, optional): Feature cache for causal convolutions + feat_idx (list, optional): Feature index for cache management + + Returns: + torch.Tensor: Output tensor + """ + for resnet in self.resnets: + if feat_cache is not None: + x = resnet(x, feat_cache, feat_idx) + else: + x = resnet(x) + + if self.upsamplers is not None: + if feat_cache is not None: + x = self.upsamplers[0](x, feat_cache, feat_idx) + else: + x = self.upsamplers[0](x) + return x + + +class QwenImageDecoder3d(nn.Module): + r""" + A 3D decoder module. + + Args: + dim (int): The base number of channels in the first layer. + z_dim (int): The dimensionality of the latent space. + dim_mult (list of int): Multipliers for the number of channels in each block. + num_res_blocks (int): Number of residual blocks in each block. + attn_scales (list of float): Scales at which to apply attention mechanisms. + temperal_upsample (list of bool): Whether to upsample temporally in each block. + dropout (float): Dropout rate for the dropout layers. + non_linearity (str): Type of non-linearity to use. + """ + + def __init__( + self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_upsample=[False, True, True], + dropout=0.0, + non_linearity: str = "silu", + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_upsample = temperal_upsample + + self.nonlinearity = get_activation(non_linearity) + + # dimensions + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + scale = 1.0 / 2 ** (len(dim_mult) - 2) + + # init block + self.conv_in = QwenImageCausalConv3d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.mid_block = QwenImageMidBlock(dims[0], dropout, non_linearity, num_layers=1) + + # upsample blocks + self.up_blocks = nn.ModuleList([]) + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + if i > 0: + in_dim = in_dim // 2 + + # Determine if we need upsampling + upsample_mode = None + if i != len(dim_mult) - 1: + upsample_mode = "upsample3d" if temperal_upsample[i] else "upsample2d" + + # Create and add the upsampling block + up_block = QwenImageUpBlock( + in_dim=in_dim, + out_dim=out_dim, + num_res_blocks=num_res_blocks, + dropout=dropout, + upsample_mode=upsample_mode, + non_linearity=non_linearity, + ) + self.up_blocks.append(up_block) + + # Update scale for next iteration + if upsample_mode is not None: + scale *= 2.0 + + # output blocks + self.norm_out = QwenImageRMS_norm(out_dim, images=False) + self.conv_out = QwenImageCausalConv3d(out_dim, 3, 3, padding=1) + + self.gradient_checkpointing = False + + def forward(self, x, feat_cache=None, feat_idx=[0]): + ## conv1 + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], dim=2) + x = self.conv_in(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv_in(x) + + ## middle + x = self.mid_block(x, feat_cache, feat_idx) + + ## upsamples + for up_block in self.up_blocks: + x = up_block(x, feat_cache, feat_idx) + + ## head + x = self.norm_out(x) + x = self.nonlinearity(x) + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], dim=2) + x = self.conv_out(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv_out(x) + return x + + +class AutoencoderKLQwenImage(ModelMixin, ConfigMixin, FromOriginalModelMixin): + r""" + A VAE model with KL loss for encoding videos into latents and decoding latent representations into videos. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + """ + + _supports_gradient_checkpointing = False + + # fmt: off + @register_to_config + def __init__( + self, + base_dim: int = 96, + z_dim: int = 16, + dim_mult: Tuple[int] = [1, 2, 4, 4], + num_res_blocks: int = 2, + attn_scales: List[float] = [], + temperal_downsample: List[bool] = [False, True, True], + dropout: float = 0.0, + latents_mean: List[float] = [-0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508, 0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921], + latents_std: List[float] = [2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743, 3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160], + ) -> None: + # fmt: on + super().__init__() + + self.z_dim = z_dim + self.temperal_downsample = temperal_downsample + self.temperal_upsample = temperal_downsample[::-1] + + self.encoder = QwenImageEncoder3d( + base_dim, z_dim * 2, dim_mult, num_res_blocks, attn_scales, self.temperal_downsample, dropout + ) + self.quant_conv = QwenImageCausalConv3d(z_dim * 2, z_dim * 2, 1) + self.post_quant_conv = QwenImageCausalConv3d(z_dim, z_dim, 1) + + self.decoder = QwenImageDecoder3d( + base_dim, z_dim, dim_mult, num_res_blocks, attn_scales, self.temperal_upsample, dropout + ) + + self.spatial_compression_ratio = 2 ** len(self.temperal_downsample) + + # When decoding a batch of video latents at a time, one can save memory by slicing across the batch dimension + # to perform decoding of a single video latent at a time. + self.use_slicing = False + + # When decoding spatially large video latents, the memory requirement is very high. By breaking the video latent + # frames spatially into smaller tiles and performing multiple forward passes for decoding, and then blending the + # intermediate tiles together, the memory requirement can be lowered. + self.use_tiling = False + + # The minimal tile height and width for spatial tiling to be used + self.tile_sample_min_height = 256 + self.tile_sample_min_width = 256 + + # The minimal distance between two spatial tiles + self.tile_sample_stride_height = 192 + self.tile_sample_stride_width = 192 + + # Precompute and cache conv counts for encoder and decoder for clear_cache speedup + self._cached_conv_counts = { + "decoder": sum(isinstance(m, QwenImageCausalConv3d) for m in self.decoder.modules()) + if self.decoder is not None + else 0, + "encoder": sum(isinstance(m, QwenImageCausalConv3d) for m in self.encoder.modules()) + if self.encoder is not None + else 0, + } + + def enable_tiling( + self, + tile_sample_min_height: Optional[int] = None, + tile_sample_min_width: Optional[int] = None, + tile_sample_stride_height: Optional[float] = None, + tile_sample_stride_width: Optional[float] = None, + ) -> None: + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + + Args: + tile_sample_min_height (`int`, *optional*): + The minimum height required for a sample to be separated into tiles across the height dimension. + tile_sample_min_width (`int`, *optional*): + The minimum width required for a sample to be separated into tiles across the width dimension. + tile_sample_stride_height (`int`, *optional*): + The minimum amount of overlap between two consecutive vertical tiles. This is to ensure that there are + no tiling artifacts produced across the height dimension. + tile_sample_stride_width (`int`, *optional*): + The stride between two consecutive horizontal tiles. This is to ensure that there are no tiling + artifacts produced across the width dimension. + """ + self.use_tiling = True + self.tile_sample_min_height = tile_sample_min_height or self.tile_sample_min_height + self.tile_sample_min_width = tile_sample_min_width or self.tile_sample_min_width + self.tile_sample_stride_height = tile_sample_stride_height or self.tile_sample_stride_height + self.tile_sample_stride_width = tile_sample_stride_width or self.tile_sample_stride_width + + def disable_tiling(self) -> None: + r""" + Disable tiled VAE decoding. If `enable_tiling` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_tiling = False + + def enable_slicing(self) -> None: + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.use_slicing = True + + def disable_slicing(self) -> None: + r""" + Disable sliced VAE decoding. If `enable_slicing` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_slicing = False + + def clear_cache(self): + def _count_conv3d(model): + count = 0 + for m in model.modules(): + if isinstance(m, QwenImageCausalConv3d): + count += 1 + return count + + self._conv_num = _count_conv3d(self.decoder) + self._conv_idx = [0] + self._feat_map = [None] * self._conv_num + # cache encode + self._enc_conv_num = _count_conv3d(self.encoder) + self._enc_conv_idx = [0] + self._enc_feat_map = [None] * self._enc_conv_num + + def _encode(self, x: torch.Tensor): + _, _, num_frame, height, width = x.shape + + if self.use_tiling and (width > self.tile_sample_min_width or height > self.tile_sample_min_height): + return self.tiled_encode(x) + + self.clear_cache() + iter_ = 1 + (num_frame - 1) // 4 + for i in range(iter_): + self._enc_conv_idx = [0] + if i == 0: + out = self.encoder(x[:, :, :1, :, :], feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx) + else: + out_ = self.encoder( + x[:, :, 1 + 4 * (i - 1) : 1 + 4 * i, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx, + ) + out = torch.cat([out, out_], 2) + + enc = self.quant_conv(out) + self.clear_cache() + return enc + + @apply_forward_hook + def encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + r""" + Encode a batch of images into latents. + + Args: + x (`torch.Tensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + The latent representations of the encoded videos. If `return_dict` is True, a + [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. + """ + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [self._encode(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = self._encode(x) + posterior = DiagonalGaussianDistribution(h) + + if not return_dict: + return (posterior,) + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode(self, z: torch.Tensor, return_dict: bool = True): + _, _, num_frame, height, width = z.shape + tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio + tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio + + if self.use_tiling and (width > tile_latent_min_width or height > tile_latent_min_height): + return self.tiled_decode(z, return_dict=return_dict) + + self.clear_cache() + x = self.post_quant_conv(z) + for i in range(num_frame): + self._conv_idx = [0] + if i == 0: + out = self.decoder(x[:, :, i : i + 1, :, :], feat_cache=self._feat_map, feat_idx=self._conv_idx) + else: + out_ = self.decoder(x[:, :, i : i + 1, :, :], feat_cache=self._feat_map, feat_idx=self._conv_idx) + out = torch.cat([out, out_], 2) + + out = torch.clamp(out, min=-1.0, max=1.0) + self.clear_cache() + if not return_dict: + return (out,) + + return DecoderOutput(sample=out) + + @apply_forward_hook + def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + r""" + Decode a batch of images. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + if self.use_slicing and z.shape[0] > 1: + decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)] + decoded = torch.cat(decoded_slices) + else: + decoded = self._decode(z).sample + + if not return_dict: + return (decoded,) + return DecoderOutput(sample=decoded) + + def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[-2], b.shape[-2], blend_extent) + for y in range(blend_extent): + b[:, :, :, y, :] = a[:, :, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, :, y, :] * ( + y / blend_extent + ) + return b + + def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[-1], b.shape[-1], blend_extent) + for x in range(blend_extent): + b[:, :, :, :, x] = a[:, :, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, :, x] * ( + x / blend_extent + ) + return b + + def tiled_encode(self, x: torch.Tensor) -> AutoencoderKLOutput: + r"""Encode a batch of images using a tiled encoder. + + Args: + x (`torch.Tensor`): Input batch of videos. + + Returns: + `torch.Tensor`: + The latent representation of the encoded videos. + """ + _, _, num_frames, height, width = x.shape + latent_height = height // self.spatial_compression_ratio + latent_width = width // self.spatial_compression_ratio + + tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio + tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio + tile_latent_stride_height = self.tile_sample_stride_height // self.spatial_compression_ratio + tile_latent_stride_width = self.tile_sample_stride_width // self.spatial_compression_ratio + + blend_height = tile_latent_min_height - tile_latent_stride_height + blend_width = tile_latent_min_width - tile_latent_stride_width + + # Split x into overlapping tiles and encode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, height, self.tile_sample_stride_height): + row = [] + for j in range(0, width, self.tile_sample_stride_width): + self.clear_cache() + time = [] + frame_range = 1 + (num_frames - 1) // 4 + for k in range(frame_range): + self._enc_conv_idx = [0] + if k == 0: + tile = x[:, :, :1, i : i + self.tile_sample_min_height, j : j + self.tile_sample_min_width] + else: + tile = x[ + :, + :, + 1 + 4 * (k - 1) : 1 + 4 * k, + i : i + self.tile_sample_min_height, + j : j + self.tile_sample_min_width, + ] + tile = self.encoder(tile, feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx) + tile = self.quant_conv(tile) + time.append(tile) + row.append(torch.cat(time, dim=2)) + rows.append(row) + self.clear_cache() + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_height) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_width) + result_row.append(tile[:, :, :, :tile_latent_stride_height, :tile_latent_stride_width]) + result_rows.append(torch.cat(result_row, dim=-1)) + + enc = torch.cat(result_rows, dim=3)[:, :, :, :latent_height, :latent_width] + return enc + + def tiled_decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + r""" + Decode a batch of images using a tiled decoder. + + Args: + z (`torch.Tensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + _, _, num_frames, height, width = z.shape + sample_height = height * self.spatial_compression_ratio + sample_width = width * self.spatial_compression_ratio + + tile_latent_min_height = self.tile_sample_min_height // self.spatial_compression_ratio + tile_latent_min_width = self.tile_sample_min_width // self.spatial_compression_ratio + tile_latent_stride_height = self.tile_sample_stride_height // self.spatial_compression_ratio + tile_latent_stride_width = self.tile_sample_stride_width // self.spatial_compression_ratio + + blend_height = self.tile_sample_min_height - self.tile_sample_stride_height + blend_width = self.tile_sample_min_width - self.tile_sample_stride_width + + # Split z into overlapping tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, height, tile_latent_stride_height): + row = [] + for j in range(0, width, tile_latent_stride_width): + self.clear_cache() + time = [] + for k in range(num_frames): + self._conv_idx = [0] + tile = z[:, :, k : k + 1, i : i + tile_latent_min_height, j : j + tile_latent_min_width] + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile, feat_cache=self._feat_map, feat_idx=self._conv_idx) + time.append(decoded) + row.append(torch.cat(time, dim=2)) + rows.append(row) + self.clear_cache() + + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_height) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_width) + result_row.append(tile[:, :, :, : self.tile_sample_stride_height, : self.tile_sample_stride_width]) + result_rows.append(torch.cat(result_row, dim=-1)) + + dec = torch.cat(result_rows, dim=3)[:, :, :, :sample_height, :sample_width] + + if not return_dict: + return (dec,) + return DecoderOutput(sample=dec) + + def forward( + self, + sample: torch.Tensor, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + ) -> Union[DecoderOutput, torch.Tensor]: + """ + Args: + sample (`torch.Tensor`): Input sample. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z, return_dict=return_dict) + return dec \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_animate_adapter.py b/VideoX-Fun/videox_fun/models/wan_animate_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..1b3dd87b0c1a7e904d93fa3a76265653014e1dab --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_animate_adapter.py @@ -0,0 +1,397 @@ +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +import math +from typing import Optional, Tuple + +import torch +import torch.nn.functional as F +import numpy as np +from einops import rearrange +from torch import nn + +try: + from flash_attn import flash_attn_func, flash_attn_qkvpacked_func +except ImportError: + flash_attn_func = None + + +MEMORY_LAYOUT = { + "flash": ( + lambda x: x.view(x.shape[0] * x.shape[1], *x.shape[2:]), + lambda x: x, + ), + "torch": ( + lambda x: x.transpose(1, 2), + lambda x: x.transpose(1, 2), + ), + "vanilla": ( + lambda x: x.transpose(1, 2), + lambda x: x.transpose(1, 2), + ), +} + + +def attention( + q, + k, + v, + mode="flash", + drop_rate=0, + attn_mask=None, + causal=False, + max_seqlen_q=None, + batch_size=1, +): + """ + Perform QKV self attention. + + Args: + q (torch.Tensor): Query tensor with shape [b, s, a, d], where a is the number of heads. + k (torch.Tensor): Key tensor with shape [b, s1, a, d] + v (torch.Tensor): Value tensor with shape [b, s1, a, d] + mode (str): Attention mode. Choose from 'self_flash', 'cross_flash', 'torch', and 'vanilla'. + drop_rate (float): Dropout rate in attention map. (default: 0) + attn_mask (torch.Tensor): Attention mask with shape [b, s1] (cross_attn), or [b, a, s, s1] (torch or vanilla). + (default: None) + causal (bool): Whether to use causal attention. (default: False) + cu_seqlens_q (torch.Tensor): dtype torch.int32. The cumulative sequence lengths of the sequences in the batch, + used to index into q. + cu_seqlens_kv (torch.Tensor): dtype torch.int32. The cumulative sequence lengths of the sequences in the batch, + used to index into kv. + max_seqlen_q (int): The maximum sequence length in the batch of q. + max_seqlen_kv (int): The maximum sequence length in the batch of k and v. + + Returns: + torch.Tensor: Output tensor after self attention with shape [b, s, ad] + """ + pre_attn_layout, post_attn_layout = MEMORY_LAYOUT[mode] + + if mode == "torch": + if attn_mask is not None and attn_mask.dtype != torch.bool: + attn_mask = attn_mask.to(q.dtype) + x = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask, dropout_p=drop_rate, is_causal=causal) + + elif mode == "flash": + x = flash_attn_func( + q, + k, + v, + ) + x = x.view(batch_size, max_seqlen_q, x.shape[-2], x.shape[-1]) # reshape x to [b, s, a, d] + elif mode == "vanilla": + scale_factor = 1 / math.sqrt(q.size(-1)) + + b, a, s, _ = q.shape + s1 = k.size(2) + attn_bias = torch.zeros(b, a, s, s1, dtype=q.dtype, device=q.device) + if causal: + # Only applied to self attention + assert attn_mask is None, "Causal mask and attn_mask cannot be used together" + temp_mask = torch.ones(b, a, s, s, dtype=torch.bool, device=q.device).tril(diagonal=0) + attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf")) + attn_bias.to(q.dtype) + + if attn_mask is not None: + if attn_mask.dtype == torch.bool: + attn_bias.masked_fill_(attn_mask.logical_not(), float("-inf")) + else: + attn_bias += attn_mask + + attn = (q @ k.transpose(-2, -1)) * scale_factor + attn += attn_bias + attn = attn.softmax(dim=-1) + attn = torch.dropout(attn, p=drop_rate, train=True) + x = attn @ v + else: + raise NotImplementedError(f"Unsupported attention mode: {mode}") + + x = post_attn_layout(x) + b, s, a, d = x.shape + out = x.reshape(b, s, -1) + return out + + +class CausalConv1d(nn.Module): + + def __init__(self, chan_in, chan_out, kernel_size=3, stride=1, dilation=1, pad_mode="replicate", **kwargs): + super().__init__() + + self.pad_mode = pad_mode + padding = (kernel_size - 1, 0) # T + self.time_causal_padding = padding + + self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs) + + def forward(self, x): + x = F.pad(x, self.time_causal_padding, mode=self.pad_mode) + return self.conv(x) + + + +class FaceEncoder(nn.Module): + def __init__(self, in_dim: int, hidden_dim: int, num_heads=int, dtype=None, device=None): + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + + self.num_heads = num_heads + self.conv1_local = CausalConv1d(in_dim, 1024 * num_heads, 3, stride=1) + self.norm1 = nn.LayerNorm(hidden_dim // 8, elementwise_affine=False, eps=1e-6, **factory_kwargs) + self.act = nn.SiLU() + self.conv2 = CausalConv1d(1024, 1024, 3, stride=2) + self.conv3 = CausalConv1d(1024, 1024, 3, stride=2) + + self.out_proj = nn.Linear(1024, hidden_dim) + self.norm1 = nn.LayerNorm(1024, elementwise_affine=False, eps=1e-6, **factory_kwargs) + + self.norm2 = nn.LayerNorm(1024, elementwise_affine=False, eps=1e-6, **factory_kwargs) + + self.norm3 = nn.LayerNorm(1024, elementwise_affine=False, eps=1e-6, **factory_kwargs) + + self.padding_tokens = nn.Parameter(torch.zeros(1, 1, 1, hidden_dim)) + + def forward(self, x): + + x = rearrange(x, "b t c -> b c t") + b, c, t = x.shape + + x = self.conv1_local(x) + x = rearrange(x, "b (n c) t -> (b n) t c", n=self.num_heads) + + x = self.norm1(x) + x = self.act(x) + x = rearrange(x, "b t c -> b c t") + x = self.conv2(x) + x = rearrange(x, "b c t -> b t c") + x = self.norm2(x) + x = self.act(x) + x = rearrange(x, "b t c -> b c t") + x = self.conv3(x) + x = rearrange(x, "b c t -> b t c") + x = self.norm3(x) + x = self.act(x) + x = self.out_proj(x) + x = rearrange(x, "(b n) t c -> b t n c", b=b) + padding = self.padding_tokens.repeat(b, x.shape[1], 1, 1) + x = torch.cat([x, padding], dim=-2) + x_local = x.clone() + + return x_local + + + +class RMSNorm(nn.Module): + def __init__( + self, + dim: int, + elementwise_affine=True, + eps: float = 1e-6, + device=None, + dtype=None, + ): + """ + Initialize the RMSNorm normalization layer. + + Args: + dim (int): The dimension of the input tensor. + eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6. + + Attributes: + eps (float): A small value added to the denominator for numerical stability. + weight (nn.Parameter): Learnable scaling parameter. + + """ + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + self.eps = eps + if elementwise_affine: + self.weight = nn.Parameter(torch.ones(dim, **factory_kwargs)) + + def _norm(self, x): + """ + Apply the RMSNorm normalization to the input tensor. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The normalized tensor. + + """ + return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps) + + def forward(self, x): + """ + Forward pass through the RMSNorm layer. + + Args: + x (torch.Tensor): The input tensor. + + Returns: + torch.Tensor: The output tensor after applying RMSNorm. + + """ + output = self._norm(x.float()).type_as(x) + if hasattr(self, "weight"): + output = output * self.weight + return output + + +def get_norm_layer(norm_layer): + """ + Get the normalization layer. + + Args: + norm_layer (str): The type of normalization layer. + + Returns: + norm_layer (nn.Module): The normalization layer. + """ + if norm_layer == "layer": + return nn.LayerNorm + elif norm_layer == "rms": + return RMSNorm + else: + raise NotImplementedError(f"Norm layer {norm_layer} is not implemented") + + +class FaceAdapter(nn.Module): + def __init__( + self, + hidden_dim: int, + heads_num: int, + qk_norm: bool = True, + qk_norm_type: str = "rms", + num_adapter_layers: int = 1, + dtype=None, + device=None, + ): + + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + self.hidden_size = hidden_dim + self.heads_num = heads_num + self.fuser_blocks = nn.ModuleList( + [ + FaceBlock( + self.hidden_size, + self.heads_num, + qk_norm=qk_norm, + qk_norm_type=qk_norm_type, + **factory_kwargs, + ) + for _ in range(num_adapter_layers) + ] + ) + + def forward( + self, + x: torch.Tensor, + motion_embed: torch.Tensor, + idx: int, + freqs_cis_q: Tuple[torch.Tensor, torch.Tensor] = None, + freqs_cis_k: Tuple[torch.Tensor, torch.Tensor] = None, + ) -> torch.Tensor: + + return self.fuser_blocks[idx](x, motion_embed, freqs_cis_q, freqs_cis_k) + + + +class FaceBlock(nn.Module): + def __init__( + self, + hidden_size: int, + heads_num: int, + qk_norm: bool = True, + qk_norm_type: str = "rms", + qk_scale: float = None, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + ): + factory_kwargs = {"device": device, "dtype": dtype} + super().__init__() + + self.deterministic = False + self.hidden_size = hidden_size + self.heads_num = heads_num + head_dim = hidden_size // heads_num + self.scale = qk_scale or head_dim**-0.5 + + self.linear1_kv = nn.Linear(hidden_size, hidden_size * 2, **factory_kwargs) + self.linear1_q = nn.Linear(hidden_size, hidden_size, **factory_kwargs) + + self.linear2 = nn.Linear(hidden_size, hidden_size, **factory_kwargs) + + qk_norm_layer = get_norm_layer(qk_norm_type) + self.q_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) if qk_norm else nn.Identity() + ) + self.k_norm = ( + qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) if qk_norm else nn.Identity() + ) + + self.pre_norm_feat = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs) + + self.pre_norm_motion = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs) + + def forward( + self, + x: torch.Tensor, + motion_vec: torch.Tensor, + motion_mask: Optional[torch.Tensor] = None, + use_context_parallel=False, + all_gather=None, + sp_world_size=1, + sp_world_rank=0, + ) -> torch.Tensor: + dtype = x.dtype + B, T, N, C = motion_vec.shape + T_comp = T + + x_motion = self.pre_norm_motion(motion_vec) + x_feat = self.pre_norm_feat(x) + + kv = self.linear1_kv(x_motion) + q = self.linear1_q(x_feat) + + k, v = rearrange(kv, "B L N (K H D) -> K B L N H D", K=2, H=self.heads_num) + q = rearrange(q, "B S (H D) -> B S H D", H=self.heads_num) + + # Apply QK-Norm if needed. + q = self.q_norm(q).to(v) + k = self.k_norm(k).to(v) + + k = rearrange(k, "B L N H D -> (B L) N H D") + v = rearrange(v, "B L N H D -> (B L) N H D") + + if use_context_parallel: + q = all_gather(q, dim=1) + + length = int(np.floor(q.size()[1] / T_comp) * T_comp) + origin_length = q.size()[1] + if origin_length > length: + q_pad = q[:, length:] + q = q[:, :length] + + q = rearrange(q, "B (L S) H D -> (B L) S H D", L=T_comp) + q, k, v = q.to(dtype), k.to(dtype), v.to(dtype) + # Compute attention. + attn = attention( + q, + k, + v, + max_seqlen_q=q.shape[1], + batch_size=q.shape[0], + ) + + attn = rearrange(attn, "(B L) S C -> B (L S) C", L=T_comp) + if use_context_parallel: + q_pad = rearrange(q_pad, "B L H D -> B L (H D)") + if origin_length > length: + attn = torch.cat([attn, q_pad], dim=1) + attn = torch.chunk(attn, sp_world_size, dim=1)[sp_world_rank] + + output = self.linear2(attn) + + if motion_mask is not None: + output = output * rearrange(motion_mask, "B T H W -> B (T H W)").unsqueeze(-1) + + return output \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_animate_motion_encoder.py b/VideoX-Fun/videox_fun/models/wan_animate_motion_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..809fa123af55b28623c7d3fb4fe625cac46bcb1e --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_animate_motion_encoder.py @@ -0,0 +1,309 @@ +# Modified from ``https://github.com/wyhsirius/LIA`` +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +from torch.nn import functional as F + + +def custom_qr(input_tensor): + original_dtype = input_tensor.dtype + if original_dtype == torch.bfloat16: + q, r = torch.linalg.qr(input_tensor.to(torch.float32)) + return q.to(original_dtype), r.to(original_dtype) + return torch.linalg.qr(input_tensor) + +def fused_leaky_relu(input, bias, negative_slope=0.2, scale=2 ** 0.5): + return F.leaky_relu(input + bias, negative_slope) * scale + + +def upfirdn2d_native(input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, pad_y0, pad_y1): + _, minor, in_h, in_w = input.shape + kernel_h, kernel_w = kernel.shape + + out = input.view(-1, minor, in_h, 1, in_w, 1) + out = F.pad(out, [0, up_x - 1, 0, 0, 0, up_y - 1, 0, 0]) + out = out.view(-1, minor, in_h * up_y, in_w * up_x) + + out = F.pad(out, [max(pad_x0, 0), max(pad_x1, 0), max(pad_y0, 0), max(pad_y1, 0)]) + out = out[:, :, max(-pad_y0, 0): out.shape[2] - max(-pad_y1, 0), + max(-pad_x0, 0): out.shape[3] - max(-pad_x1, 0), ] + + out = out.reshape([-1, 1, in_h * up_y + pad_y0 + pad_y1, in_w * up_x + pad_x0 + pad_x1]) + w = torch.flip(kernel, [0, 1]).view(1, 1, kernel_h, kernel_w) + out = F.conv2d(out, w) + out = out.reshape(-1, minor, in_h * up_y + pad_y0 + pad_y1 - kernel_h + 1, + in_w * up_x + pad_x0 + pad_x1 - kernel_w + 1, ) + return out[:, :, ::down_y, ::down_x] + + +def upfirdn2d(input, kernel, up=1, down=1, pad=(0, 0)): + return upfirdn2d_native(input, kernel, up, up, down, down, pad[0], pad[1], pad[0], pad[1]) + + +def make_kernel(k): + k = torch.tensor(k, dtype=torch.float32) + if k.ndim == 1: + k = k[None, :] * k[:, None] + k /= k.sum() + return k + + +class FusedLeakyReLU(nn.Module): + def __init__(self, channel, negative_slope=0.2, scale=2 ** 0.5): + super().__init__() + self.bias = nn.Parameter(torch.zeros(1, channel, 1, 1)) + self.negative_slope = negative_slope + self.scale = scale + + def forward(self, input): + out = fused_leaky_relu(input, self.bias, self.negative_slope, self.scale) + return out + + +class Blur(nn.Module): + def __init__(self, kernel, pad, upsample_factor=1): + super().__init__() + + kernel = make_kernel(kernel) + + if upsample_factor > 1: + kernel = kernel * (upsample_factor ** 2) + + self.register_buffer('kernel', kernel) + + self.pad = pad + + def forward(self, input): + return upfirdn2d(input, self.kernel, pad=self.pad) + + +class ScaledLeakyReLU(nn.Module): + def __init__(self, negative_slope=0.2): + super().__init__() + + self.negative_slope = negative_slope + + def forward(self, input): + return F.leaky_relu(input, negative_slope=self.negative_slope) + + +class EqualConv2d(nn.Module): + def __init__(self, in_channel, out_channel, kernel_size, stride=1, padding=0, bias=True): + super().__init__() + + self.weight = nn.Parameter(torch.randn(out_channel, in_channel, kernel_size, kernel_size)) + self.scale = 1 / math.sqrt(in_channel * kernel_size ** 2) + + self.stride = stride + self.padding = padding + + if bias: + self.bias = nn.Parameter(torch.zeros(out_channel)) + else: + self.bias = None + + def forward(self, input): + + return F.conv2d(input, self.weight * self.scale, bias=self.bias, stride=self.stride, padding=self.padding) + + def __repr__(self): + return ( + f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},' + f' {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})' + ) + + +class EqualLinear(nn.Module): + def __init__(self, in_dim, out_dim, bias=True, bias_init=0, lr_mul=1, activation=None): + super().__init__() + + self.weight = nn.Parameter(torch.randn(out_dim, in_dim).div_(lr_mul)) + + if bias: + self.bias = nn.Parameter(torch.zeros(out_dim).fill_(bias_init)) + else: + self.bias = None + + self.activation = activation + + self.scale = (1 / math.sqrt(in_dim)) * lr_mul + self.lr_mul = lr_mul + + def forward(self, input): + + if self.activation: + out = F.linear(input, self.weight * self.scale) + out = fused_leaky_relu(out, self.bias * self.lr_mul) + else: + out = F.linear(input, self.weight * self.scale, bias=self.bias * self.lr_mul) + + return out + + def __repr__(self): + return (f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]})') + + +class ConvLayer(nn.Sequential): + def __init__( + self, + in_channel, + out_channel, + kernel_size, + downsample=False, + blur_kernel=[1, 3, 3, 1], + bias=True, + activate=True, + ): + layers = [] + + if downsample: + factor = 2 + p = (len(blur_kernel) - factor) + (kernel_size - 1) + pad0 = (p + 1) // 2 + pad1 = p // 2 + + layers.append(Blur(blur_kernel, pad=(pad0, pad1))) + + stride = 2 + self.padding = 0 + + else: + stride = 1 + self.padding = kernel_size // 2 + + layers.append(EqualConv2d(in_channel, out_channel, kernel_size, padding=self.padding, stride=stride, + bias=bias and not activate)) + + if activate: + if bias: + layers.append(FusedLeakyReLU(out_channel)) + else: + layers.append(ScaledLeakyReLU(0.2)) + + super().__init__(*layers) + + +class ResBlock(nn.Module): + def __init__(self, in_channel, out_channel, blur_kernel=[1, 3, 3, 1]): + super().__init__() + + self.conv1 = ConvLayer(in_channel, in_channel, 3) + self.conv2 = ConvLayer(in_channel, out_channel, 3, downsample=True) + + self.skip = ConvLayer(in_channel, out_channel, 1, downsample=True, activate=False, bias=False) + + def forward(self, input): + out = self.conv1(input) + out = self.conv2(out) + + skip = self.skip(input) + out = (out + skip) / math.sqrt(2) + + return out + + +class EncoderApp(nn.Module): + def __init__(self, size, w_dim=512): + super(EncoderApp, self).__init__() + + channels = { + 4: 512, + 8: 512, + 16: 512, + 32: 512, + 64: 256, + 128: 128, + 256: 64, + 512: 32, + 1024: 16 + } + + self.w_dim = w_dim + log_size = int(math.log(size, 2)) + + self.convs = nn.ModuleList() + self.convs.append(ConvLayer(3, channels[size], 1)) + + in_channel = channels[size] + for i in range(log_size, 2, -1): + out_channel = channels[2 ** (i - 1)] + self.convs.append(ResBlock(in_channel, out_channel)) + in_channel = out_channel + + self.convs.append(EqualConv2d(in_channel, self.w_dim, 4, padding=0, bias=False)) + + def forward(self, x): + + res = [] + h = x + for conv in self.convs: + h = conv(h) + res.append(h) + + return res[-1].squeeze(-1).squeeze(-1), res[::-1][2:] + + +class Encoder(nn.Module): + def __init__(self, size, dim=512, dim_motion=20): + super(Encoder, self).__init__() + + # appearance netmork + self.net_app = EncoderApp(size, dim) + + # motion network + fc = [EqualLinear(dim, dim)] + for i in range(3): + fc.append(EqualLinear(dim, dim)) + + fc.append(EqualLinear(dim, dim_motion)) + self.fc = nn.Sequential(*fc) + + def enc_app(self, x): + h_source = self.net_app(x) + return h_source + + def enc_motion(self, x): + h, _ = self.net_app(x) + h_motion = self.fc(h) + return h_motion + + +class Direction(nn.Module): + def __init__(self, motion_dim): + super(Direction, self).__init__() + self.weight = nn.Parameter(torch.randn(512, motion_dim)) + + def forward(self, input): + + weight = self.weight + 1e-8 + Q, R = custom_qr(weight) + if input is None: + return Q + else: + input_diag = torch.diag_embed(input) # alpha, diagonal matrix + out = torch.matmul(input_diag, Q.T) + out = torch.sum(out, dim=1) + return out + + +class Synthesis(nn.Module): + def __init__(self, motion_dim): + super(Synthesis, self).__init__() + self.direction = Direction(motion_dim) + + +class Generator(nn.Module): + def __init__(self, size, style_dim=512, motion_dim=20): + super().__init__() + + self.enc = Encoder(size, style_dim, motion_dim) + self.dec = Synthesis(motion_dim) + + def get_motion(self, img): + #motion_feat = self.enc.enc_motion(img) + motion_feat = torch.utils.checkpoint.checkpoint((self.enc.enc_motion), img, use_reentrant=True) + with torch.cuda.amp.autocast(dtype=torch.float32): + motion = self.dec.direction(motion_feat) + return motion \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_audio_encoder.py b/VideoX-Fun/videox_fun/models/wan_audio_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..5dc652c8c5396f6265dc1fe89d8644d21bcb8965 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_audio_encoder.py @@ -0,0 +1,213 @@ +# Modified from https://github.com/Wan-Video/Wan2.2/blob/main/wan/modules/s2v/audio_encoder.py +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +import math + +import librosa +import numpy as np +import torch +import torch.nn.functional as F +from transformers import Wav2Vec2ForCTC, Wav2Vec2Processor +from diffusers.configuration_utils import ConfigMixin +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.modeling_utils import ModelMixin + + +def get_sample_indices(original_fps, + total_frames, + target_fps, + num_sample, + fixed_start=None): + required_duration = num_sample / target_fps + required_origin_frames = int(np.ceil(required_duration * original_fps)) + if required_duration > total_frames / original_fps: + raise ValueError("required_duration must be less than video length") + + if not fixed_start is None and fixed_start >= 0: + start_frame = fixed_start + else: + max_start = total_frames - required_origin_frames + if max_start < 0: + raise ValueError("video length is too short") + start_frame = np.random.randint(0, max_start + 1) + start_time = start_frame / original_fps + + end_time = start_time + required_duration + time_points = np.linspace(start_time, end_time, num_sample, endpoint=False) + + frame_indices = np.round(np.array(time_points) * original_fps).astype(int) + frame_indices = np.clip(frame_indices, 0, total_frames - 1) + return frame_indices + + +def linear_interpolation(features, input_fps, output_fps, output_len=None): + """ + features: shape=[1, T, 512] + input_fps: fps for audio, f_a + output_fps: fps for video, f_m + output_len: video length + """ + features = features.transpose(1, 2) # [1, 512, T] + seq_len = features.shape[2] / float(input_fps) # T/f_a + if output_len is None: + output_len = int(seq_len * output_fps) # f_m*T/f_a + output_features = F.interpolate( + features, size=output_len, align_corners=True, + mode='linear') # [1, 512, output_len] + return output_features.transpose(1, 2) # [1, output_len, 512] + + +class WanAudioEncoder(ModelMixin, ConfigMixin, FromOriginalModelMixin): + + def __init__(self, pretrained_model_path="facebook/wav2vec2-base-960h", device='cpu'): + super(WanAudioEncoder, self).__init__() + # load pretrained model + self.processor = Wav2Vec2Processor.from_pretrained(pretrained_model_path) + self.model = Wav2Vec2ForCTC.from_pretrained(pretrained_model_path) + + self.model = self.model.to(device) + + self.video_rate = 30 + + def extract_audio_feat(self, + audio_path, + return_all_layers=False, + dtype=torch.float32): + audio_input, sample_rate = librosa.load(audio_path, sr=16000) + + input_values = self.processor( + audio_input, sampling_rate=sample_rate, return_tensors="pt" + ).input_values + + # INFERENCE + + # retrieve logits & take argmax + res = self.model( + input_values.to(self.model.device), output_hidden_states=True) + if return_all_layers: + feat = torch.cat(res.hidden_states) + else: + feat = res.hidden_states[-1] + feat = linear_interpolation( + feat, input_fps=50, output_fps=self.video_rate) + + z = feat.to(dtype) # Encoding for the motion + return z + + def extract_audio_feat_without_file_load(self, audio_input, sample_rate, return_all_layers=False, dtype=torch.float32): + input_values = self.processor( + audio_input, sampling_rate=sample_rate, return_tensors="pt" + ).input_values + + # INFERENCE + # retrieve logits & take argmax + res = self.model( + input_values.to(self.model.device), output_hidden_states=True) + if return_all_layers: + feat = torch.cat(res.hidden_states) + else: + feat = res.hidden_states[-1] + feat = linear_interpolation( + feat, input_fps=50, output_fps=self.video_rate) + + z = feat.to(dtype) # Encoding for the motion + return z + + def get_audio_embed_bucket(self, + audio_embed, + stride=2, + batch_frames=12, + m=2): + num_layers, audio_frame_num, audio_dim = audio_embed.shape + + if num_layers > 1: + return_all_layers = True + else: + return_all_layers = False + + min_batch_num = int(audio_frame_num / (batch_frames * stride)) + 1 + + bucket_num = min_batch_num * batch_frames + batch_idx = [stride * i for i in range(bucket_num)] + batch_audio_eb = [] + for bi in batch_idx: + if bi < audio_frame_num: + audio_sample_stride = 2 + chosen_idx = list( + range(bi - m * audio_sample_stride, + bi + (m + 1) * audio_sample_stride, + audio_sample_stride)) + chosen_idx = [0 if c < 0 else c for c in chosen_idx] + chosen_idx = [ + audio_frame_num - 1 if c >= audio_frame_num else c + for c in chosen_idx + ] + + if return_all_layers: + frame_audio_embed = audio_embed[:, chosen_idx].flatten( + start_dim=-2, end_dim=-1) + else: + frame_audio_embed = audio_embed[0][chosen_idx].flatten() + else: + frame_audio_embed = \ + torch.zeros([audio_dim * (2 * m + 1)], device=audio_embed.device) if not return_all_layers \ + else torch.zeros([num_layers, audio_dim * (2 * m + 1)], device=audio_embed.device) + batch_audio_eb.append(frame_audio_embed) + batch_audio_eb = torch.cat([c.unsqueeze(0) for c in batch_audio_eb], + dim=0) + + return batch_audio_eb, min_batch_num + + def get_audio_embed_bucket_fps(self, + audio_embed, + fps=16, + batch_frames=81, + m=0): + num_layers, audio_frame_num, audio_dim = audio_embed.shape + + if num_layers > 1: + return_all_layers = True + else: + return_all_layers = False + + scale = self.video_rate / fps + + min_batch_num = int(audio_frame_num / (batch_frames * scale)) + 1 + + bucket_num = min_batch_num * batch_frames + padd_audio_num = math.ceil(min_batch_num * batch_frames / fps * + self.video_rate) - audio_frame_num + batch_idx = get_sample_indices( + original_fps=self.video_rate, + total_frames=audio_frame_num + padd_audio_num, + target_fps=fps, + num_sample=bucket_num, + fixed_start=0) + batch_audio_eb = [] + audio_sample_stride = int(self.video_rate / fps) + for bi in batch_idx: + if bi < audio_frame_num: + + chosen_idx = list( + range(bi - m * audio_sample_stride, + bi + (m + 1) * audio_sample_stride, + audio_sample_stride)) + chosen_idx = [0 if c < 0 else c for c in chosen_idx] + chosen_idx = [ + audio_frame_num - 1 if c >= audio_frame_num else c + for c in chosen_idx + ] + + if return_all_layers: + frame_audio_embed = audio_embed[:, chosen_idx].flatten( + start_dim=-2, end_dim=-1) + else: + frame_audio_embed = audio_embed[0][chosen_idx].flatten() + else: + frame_audio_embed = \ + torch.zeros([audio_dim * (2 * m + 1)], device=audio_embed.device) if not return_all_layers \ + else torch.zeros([num_layers, audio_dim * (2 * m + 1)], device=audio_embed.device) + batch_audio_eb.append(frame_audio_embed) + batch_audio_eb = torch.cat([c.unsqueeze(0) for c in batch_audio_eb], + dim=0) + + return batch_audio_eb, min_batch_num \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_audio_injector.py b/VideoX-Fun/videox_fun/models/wan_audio_injector.py new file mode 100644 index 0000000000000000000000000000000000000000..35568c8c188e79f716563d246dc23e74cd53b867 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_audio_injector.py @@ -0,0 +1,1093 @@ +# Modified from https://github.com/Wan-Video/Wan2.2/blob/main/wan/modules/s2v/motioner.py +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +import importlib.metadata +import math +from typing import Any, Dict, List, Literal, Optional, Tuple, Union + +import numpy as np +import torch +import torch.cuda.amp as amp +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin +from diffusers.models import ModelMixin +from diffusers.models.attention import AdaLayerNorm +from diffusers.utils import BaseOutput, is_torch_version, logging +from einops import rearrange, repeat + +from .attention_utils import attention +from .wan_transformer3d import WanAttentionBlock, WanCrossAttention + + +def rope_precompute(x, grid_sizes, freqs, start=None): + b, s, n, c = x.size(0), x.size(1), x.size(2), x.size(3) // 2 + + # split freqs + if type(freqs) is list: + trainable_freqs = freqs[1] + freqs = freqs[0] + freqs = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1) + + # loop over samples + output = torch.view_as_complex(x.detach().reshape(b, s, n, -1, + 2).to(torch.float64)) + seq_bucket = [0] + if not type(grid_sizes) is list: + grid_sizes = [grid_sizes] + for g in grid_sizes: + if not type(g) is list: + g = [torch.zeros_like(g), g] + batch_size = g[0].shape[0] + for i in range(batch_size): + if start is None: + f_o, h_o, w_o = g[0][i] + else: + f_o, h_o, w_o = start[i] + + f, h, w = g[1][i] + t_f, t_h, t_w = g[2][i] + seq_f, seq_h, seq_w = f - f_o, h - h_o, w - w_o + seq_len = int(seq_f * seq_h * seq_w) + if seq_len > 0: + if t_f > 0: + factor_f, factor_h, factor_w = (t_f / seq_f).item(), ( + t_h / seq_h).item(), (t_w / seq_w).item() + # Generate a list of seq_f integers starting from f_o and ending at math.ceil(factor_f * seq_f.item() + f_o.item()) + if f_o >= 0: + f_sam = np.linspace(f_o.item(), (t_f + f_o).item() - 1, + seq_f).astype(int).tolist() + else: + f_sam = np.linspace(-f_o.item(), + (-t_f - f_o).item() + 1, + seq_f).astype(int).tolist() + h_sam = np.linspace(h_o.item(), (t_h + h_o).item() - 1, + seq_h).astype(int).tolist() + w_sam = np.linspace(w_o.item(), (t_w + w_o).item() - 1, + seq_w).astype(int).tolist() + + assert f_o * f >= 0 and h_o * h >= 0 and w_o * w >= 0 + freqs_0 = freqs[0][f_sam] if f_o >= 0 else freqs[0][ + f_sam].conj() + freqs_0 = freqs_0.view(seq_f, 1, 1, -1) + + freqs_i = torch.cat([ + freqs_0.expand(seq_f, seq_h, seq_w, -1), + freqs[1][h_sam].view(1, seq_h, 1, -1).expand( + seq_f, seq_h, seq_w, -1), + freqs[2][w_sam].view(1, 1, seq_w, -1).expand( + seq_f, seq_h, seq_w, -1), + ], + dim=-1).reshape(seq_len, 1, -1) + elif t_f < 0: + freqs_i = trainable_freqs.unsqueeze(1) + # apply rotary embedding + output[i, seq_bucket[-1]:seq_bucket[-1] + seq_len] = freqs_i + seq_bucket.append(seq_bucket[-1] + seq_len) + return output + + +def sinusoidal_embedding_1d(dim, position): + # preprocess + assert dim % 2 == 0 + half = dim // 2 + position = position.type(torch.float64) + + # calculation + sinusoid = torch.outer( + position, torch.pow(10000, -torch.arange(half).to(position).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + return x + + +@amp.autocast(enabled=False) +def rope_params(max_seq_len, dim, theta=10000): + assert dim % 2 == 0 + freqs = torch.outer( + torch.arange(max_seq_len), + 1.0 / torch.pow(theta, + torch.arange(0, dim, 2).to(torch.float64).div(dim))) + freqs = torch.polar(torch.ones_like(freqs), freqs) + return freqs + + +@amp.autocast(enabled=False) +def rope_apply(x, grid_sizes, freqs, start=None): + n, c = x.size(2), x.size(3) // 2 + + # split freqs + if type(freqs) is list: + trainable_freqs = freqs[1] + freqs = freqs[0] + freqs = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1) + + # loop over samples + output = [] + output = x.clone() + seq_bucket = [0] + if not type(grid_sizes) is list: + grid_sizes = [grid_sizes] + for g in grid_sizes: + if not type(g) is list: + g = [torch.zeros_like(g), g] + batch_size = g[0].shape[0] + for i in range(batch_size): + if start is None: + f_o, h_o, w_o = g[0][i] + else: + f_o, h_o, w_o = start[i] + + f, h, w = g[1][i] + t_f, t_h, t_w = g[2][i] + seq_f, seq_h, seq_w = f - f_o, h - h_o, w - w_o + seq_len = int(seq_f * seq_h * seq_w) + if seq_len > 0: + if t_f > 0: + factor_f, factor_h, factor_w = (t_f / seq_f).item(), ( + t_h / seq_h).item(), (t_w / seq_w).item() + + if f_o >= 0: + f_sam = np.linspace(f_o.item(), (t_f + f_o).item() - 1, + seq_f).astype(int).tolist() + else: + f_sam = np.linspace(-f_o.item(), + (-t_f - f_o).item() + 1, + seq_f).astype(int).tolist() + h_sam = np.linspace(h_o.item(), (t_h + h_o).item() - 1, + seq_h).astype(int).tolist() + w_sam = np.linspace(w_o.item(), (t_w + w_o).item() - 1, + seq_w).astype(int).tolist() + + assert f_o * f >= 0 and h_o * h >= 0 and w_o * w >= 0 + freqs_0 = freqs[0][f_sam] if f_o >= 0 else freqs[0][ + f_sam].conj() + freqs_0 = freqs_0.view(seq_f, 1, 1, -1) + + freqs_i = torch.cat([ + freqs_0.expand(seq_f, seq_h, seq_w, -1), + freqs[1][h_sam].view(1, seq_h, 1, -1).expand( + seq_f, seq_h, seq_w, -1), + freqs[2][w_sam].view(1, 1, seq_w, -1).expand( + seq_f, seq_h, seq_w, -1), + ], + dim=-1).reshape(seq_len, 1, -1) + elif t_f < 0: + freqs_i = trainable_freqs.unsqueeze(1) + # apply rotary embedding + # precompute multipliers + x_i = torch.view_as_complex( + x[i, seq_bucket[-1]:seq_bucket[-1] + seq_len].to( + torch.float64).reshape(seq_len, n, -1, 2)) + x_i = torch.view_as_real(x_i * freqs_i).flatten(2) + output[i, seq_bucket[-1]:seq_bucket[-1] + seq_len] = x_i + seq_bucket.append(seq_bucket[-1] + seq_len) + return output.float() + + + +class CausalConv1d(nn.Module): + + def __init__(self, + chan_in, + chan_out, + kernel_size=3, + stride=1, + dilation=1, + pad_mode='replicate', + **kwargs): + super().__init__() + + self.pad_mode = pad_mode + padding = (kernel_size - 1, 0) # T + self.time_causal_padding = padding + + self.conv = nn.Conv1d( + chan_in, + chan_out, + kernel_size, + stride=stride, + dilation=dilation, + **kwargs) + + def forward(self, x): + x = F.pad(x, self.time_causal_padding, mode=self.pad_mode) + return self.conv(x) + + +class MotionEncoder_tc(nn.Module): + + def __init__(self, + in_dim: int, + hidden_dim: int, + num_heads=int, + need_global=True, + dtype=None, + device=None): + factory_kwargs = {"dtype": dtype, "device": device} + super().__init__() + + self.num_heads = num_heads + self.need_global = need_global + self.conv1_local = CausalConv1d( + in_dim, hidden_dim // 4 * num_heads, 3, stride=1) + if need_global: + self.conv1_global = CausalConv1d( + in_dim, hidden_dim // 4, 3, stride=1) + self.norm1 = nn.LayerNorm( + hidden_dim // 4, + elementwise_affine=False, + eps=1e-6, + **factory_kwargs) + self.act = nn.SiLU() + self.conv2 = CausalConv1d(hidden_dim // 4, hidden_dim // 2, 3, stride=2) + self.conv3 = CausalConv1d(hidden_dim // 2, hidden_dim, 3, stride=2) + + if need_global: + self.final_linear = nn.Linear(hidden_dim, hidden_dim, + **factory_kwargs) + + self.norm1 = nn.LayerNorm( + hidden_dim // 4, + elementwise_affine=False, + eps=1e-6, + **factory_kwargs) + + self.norm2 = nn.LayerNorm( + hidden_dim // 2, + elementwise_affine=False, + eps=1e-6, + **factory_kwargs) + + self.norm3 = nn.LayerNorm( + hidden_dim, elementwise_affine=False, eps=1e-6, **factory_kwargs) + + self.padding_tokens = nn.Parameter(torch.zeros(1, 1, 1, hidden_dim)) + + def forward(self, x): + x = rearrange(x, 'b t c -> b c t') + x_ori = x.clone() + b, c, t = x.shape + x = self.conv1_local(x) + x = rearrange(x, 'b (n c) t -> (b n) t c', n=self.num_heads) + x = self.norm1(x) + x = self.act(x) + x = rearrange(x, 'b t c -> b c t') + x = self.conv2(x) + x = rearrange(x, 'b c t -> b t c') + x = self.norm2(x) + x = self.act(x) + x = rearrange(x, 'b t c -> b c t') + x = self.conv3(x) + x = rearrange(x, 'b c t -> b t c') + x = self.norm3(x) + x = self.act(x) + x = rearrange(x, '(b n) t c -> b t n c', b=b) + padding = self.padding_tokens.repeat(b, x.shape[1], 1, 1) + x = torch.cat([x, padding], dim=-2) + x_local = x.clone() + + if not self.need_global: + return x_local + + x = self.conv1_global(x_ori) + x = rearrange(x, 'b c t -> b t c') + x = self.norm1(x) + x = self.act(x) + x = rearrange(x, 'b t c -> b c t') + x = self.conv2(x) + x = rearrange(x, 'b c t -> b t c') + x = self.norm2(x) + x = self.act(x) + x = rearrange(x, 'b t c -> b c t') + x = self.conv3(x) + x = rearrange(x, 'b c t -> b t c') + x = self.norm3(x) + x = self.act(x) + x = self.final_linear(x) + x = rearrange(x, '(b n) t c -> b t n c', b=b) + + return x, x_local + + +class CausalAudioEncoder(nn.Module): + + def __init__(self, + dim=5120, + num_layers=25, + out_dim=2048, + video_rate=8, + num_token=4, + need_global=False): + super().__init__() + self.encoder = MotionEncoder_tc( + in_dim=dim, + hidden_dim=out_dim, + num_heads=num_token, + need_global=need_global) + weight = torch.ones((1, num_layers, 1, 1)) * 0.01 + + self.weights = torch.nn.Parameter(weight) + self.act = torch.nn.SiLU() + + def forward(self, features): + with amp.autocast(dtype=torch.float32): + # features B * num_layers * dim * video_length + weights = self.act(self.weights) + weights_sum = weights.sum(dim=1, keepdims=True) + weighted_feat = ((features * weights) / weights_sum).sum( + dim=1) # b dim f + weighted_feat = weighted_feat.permute(0, 2, 1) # b f dim + res = self.encoder(weighted_feat) # b f n dim + + return res # b f n dim + + +class AudioCrossAttention(WanCrossAttention): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def forward(self, x, context, context_lens, dtype=torch.bfloat16, t=0): + r""" + Args: + x(Tensor): Shape [B, L1, C] + context(Tensor): Shape [B, L2, C] + context_lens(Tensor): Shape [B] + """ + b, n, d = x.size(0), self.num_heads, self.head_dim + # compute query, key, value + q = self.norm_q(self.q(x.to(dtype))).view(b, -1, n, d) + k = self.norm_k(self.k(context.to(dtype))).view(b, -1, n, d) + v = self.v(context.to(dtype)).view(b, -1, n, d) + # compute attention + x = attention(q.to(dtype), k.to(dtype), v.to(dtype), k_lens=context_lens, attention_type="FLASH_ATTENTION") + # output + x = x.flatten(2) + x = self.o(x.to(dtype)) + return x + + +class AudioInjector_WAN(nn.Module): + + def __init__(self, + all_modules, + all_modules_names, + dim=2048, + num_heads=32, + inject_layer=[0, 27], + root_net=None, + enable_adain=False, + adain_dim=2048, + need_adain_ont=False): + super().__init__() + num_injector_layers = len(inject_layer) + self.injected_block_id = {} + audio_injector_id = 0 + for mod_name, mod in zip(all_modules_names, all_modules): + if isinstance(mod, WanAttentionBlock): + for inject_id in inject_layer: + if f'transformer_blocks.{inject_id}' in mod_name: + self.injected_block_id[inject_id] = audio_injector_id + audio_injector_id += 1 + + self.injector = nn.ModuleList([ + AudioCrossAttention( + dim=dim, + num_heads=num_heads, + qk_norm=True, + ) for _ in range(audio_injector_id) + ]) + self.injector_pre_norm_feat = nn.ModuleList([ + nn.LayerNorm( + dim, + elementwise_affine=False, + eps=1e-6, + ) for _ in range(audio_injector_id) + ]) + self.injector_pre_norm_vec = nn.ModuleList([ + nn.LayerNorm( + dim, + elementwise_affine=False, + eps=1e-6, + ) for _ in range(audio_injector_id) + ]) + if enable_adain: + self.injector_adain_layers = nn.ModuleList([ + AdaLayerNorm( + output_dim=dim * 2, embedding_dim=adain_dim, chunk_dim=1) + for _ in range(audio_injector_id) + ]) + if need_adain_ont: + self.injector_adain_output_layers = nn.ModuleList( + [nn.Linear(dim, dim) for _ in range(audio_injector_id)]) + + +class RMSNorm(nn.Module): + + def __init__(self, dim, eps=1e-5): + super().__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def forward(self, x): + return self._norm(x.float()).type_as(x) * self.weight + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps) + + +class LayerNorm(nn.LayerNorm): + + def __init__(self, dim, eps=1e-6, elementwise_affine=False): + super().__init__(dim, elementwise_affine=elementwise_affine, eps=eps) + + def forward(self, x): + return super().forward(x.float()).type_as(x) + + +class SelfAttention(nn.Module): + + def __init__(self, + dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + eps=1e-6): + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.eps = eps + + # layers + self.q = nn.Linear(dim, dim) + self.k = nn.Linear(dim, dim) + self.v = nn.Linear(dim, dim) + self.o = nn.Linear(dim, dim) + self.norm_q = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + self.norm_k = RMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + + def forward(self, x, seq_lens, grid_sizes, freqs): + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + + # query, key, value function + def qkv_fn(x): + q = self.norm_q(self.q(x)).view(b, s, n, d) + k = self.norm_k(self.k(x)).view(b, s, n, d) + v = self.v(x).view(b, s, n, d) + return q, k, v + + q, k, v = qkv_fn(x) + + x = attention( + q=rope_apply(q, grid_sizes, freqs), + k=rope_apply(k, grid_sizes, freqs), + v=v, + k_lens=seq_lens, + window_size=self.window_size) + + # output + x = x.flatten(2) + x = self.o(x) + return x + + +class SwinSelfAttention(SelfAttention): + + def forward(self, x, seq_lens, grid_sizes, freqs): + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + assert b == 1, 'Only support batch_size 1' + + # query, key, value function + def qkv_fn(x): + q = self.norm_q(self.q(x)).view(b, s, n, d) + k = self.norm_k(self.k(x)).view(b, s, n, d) + v = self.v(x).view(b, s, n, d) + return q, k, v + + q, k, v = qkv_fn(x) + + q = rope_apply(q, grid_sizes, freqs) + k = rope_apply(k, grid_sizes, freqs) + T, H, W = grid_sizes[0].tolist() + + q = rearrange(q, 'b (t h w) n d -> (b t) (h w) n d', t=T, h=H, w=W) + k = rearrange(k, 'b (t h w) n d -> (b t) (h w) n d', t=T, h=H, w=W) + v = rearrange(v, 'b (t h w) n d -> (b t) (h w) n d', t=T, h=H, w=W) + + ref_q = q[-1:] + q = q[:-1] + + ref_k = repeat( + k[-1:], "1 s n d -> t s n d", t=k.shape[0] - 1) # t hw n d + k = k[:-1] + k = torch.cat([k[:1], k, k[-1:]]) + k = torch.cat([k[1:-1], k[2:], k[:-2], ref_k], dim=1) # (bt) (3hw) n d + + ref_v = repeat(v[-1:], "1 s n d -> t s n d", t=v.shape[0] - 1) + v = v[:-1] + v = torch.cat([v[:1], v, v[-1:]]) + v = torch.cat([v[1:-1], v[2:], v[:-2], ref_v], dim=1) + + # q: b (t h w) n d + # k: b (t h w) n d + out = attention( + q=q, + k=k, + v=v, + # k_lens=torch.tensor([k.shape[1]] * k.shape[0], device=x.device, dtype=torch.long), + window_size=self.window_size) + out = torch.cat([out, ref_v[:1]], axis=0) + out = rearrange(out, '(b t) (h w) n d -> b (t h w) n d', t=T, h=H, w=W) + x = out + + # output + x = x.flatten(2) + x = self.o(x) + return x + + +#Fix the reference frame RoPE to 1,H,W. +#Set the current frame RoPE to 1. +#Set the previous frame RoPE to 0. +class CasualSelfAttention(SelfAttention): + + def forward(self, x, seq_lens, grid_sizes, freqs): + shifting = 3 + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + assert b == 1, 'Only support batch_size 1' + + # query, key, value function + def qkv_fn(x): + q = self.norm_q(self.q(x)).view(b, s, n, d) + k = self.norm_k(self.k(x)).view(b, s, n, d) + v = self.v(x).view(b, s, n, d) + return q, k, v + + q, k, v = qkv_fn(x) + + T, H, W = grid_sizes[0].tolist() + + q = rearrange(q, 'b (t h w) n d -> (b t) (h w) n d', t=T, h=H, w=W) + k = rearrange(k, 'b (t h w) n d -> (b t) (h w) n d', t=T, h=H, w=W) + v = rearrange(v, 'b (t h w) n d -> (b t) (h w) n d', t=T, h=H, w=W) + + ref_q = q[-1:] + q = q[:-1] + + grid_sizes = torch.tensor([[1, H, W]] * q.shape[0], dtype=torch.long) + start = [[shifting, 0, 0]] * q.shape[0] + q = rope_apply(q, grid_sizes, freqs, start=start) + + ref_k = k[-1:] + grid_sizes = torch.tensor([[1, H, W]], dtype=torch.long) + # start = [[shifting, H, W]] + + start = [[shifting + 10, 0, 0]] + ref_k = rope_apply(ref_k, grid_sizes, freqs, start) + ref_k = repeat( + ref_k, "1 s n d -> t s n d", t=k.shape[0] - 1) # t hw n d + + k = k[:-1] + k = torch.cat([*([k[:1]] * shifting), k]) + cat_k = [] + for i in range(shifting): + cat_k.append(k[i:i - shifting]) + cat_k.append(k[shifting:]) + k = torch.cat(cat_k, dim=1) # (bt) (3hw) n d + + grid_sizes = torch.tensor( + [[shifting + 1, H, W]] * q.shape[0], dtype=torch.long) + k = rope_apply(k, grid_sizes, freqs) + k = torch.cat([k, ref_k], dim=1) + + ref_v = repeat(v[-1:], "1 s n d -> t s n d", t=q.shape[0]) # t hw n d + v = v[:-1] + v = torch.cat([*([v[:1]] * shifting), v]) + cat_v = [] + for i in range(shifting): + cat_v.append(v[i:i - shifting]) + cat_v.append(v[shifting:]) + v = torch.cat(cat_v, dim=1) # (bt) (3hw) n d + v = torch.cat([v, ref_v], dim=1) + + # q: b (t h w) n d + # k: b (t h w) n d + outs = [] + for i in range(q.shape[0]): + out = attention( + q=q[i:i + 1], + k=k[i:i + 1], + v=v[i:i + 1], + window_size=self.window_size) + outs.append(out) + out = torch.cat(outs, dim=0) + out = torch.cat([out, ref_v[:1]], axis=0) + out = rearrange(out, '(b t) (h w) n d -> b (t h w) n d', t=T, h=H, w=W) + x = out + + # output + x = x.flatten(2) + x = self.o(x) + return x + + +class MotionerAttentionBlock(nn.Module): + + def __init__(self, + dim, + ffn_dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=False, + eps=1e-6, + self_attn_block="SelfAttention"): + super().__init__() + self.dim = dim + self.ffn_dim = ffn_dim + self.num_heads = num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.cross_attn_norm = cross_attn_norm + self.eps = eps + + # layers + self.norm1 = LayerNorm(dim, eps) + if self_attn_block == "SelfAttention": + self.self_attn = SelfAttention(dim, num_heads, window_size, qk_norm, + eps) + elif self_attn_block == "SwinSelfAttention": + self.self_attn = SwinSelfAttention(dim, num_heads, window_size, + qk_norm, eps) + elif self_attn_block == "CasualSelfAttention": + self.self_attn = CasualSelfAttention(dim, num_heads, window_size, + qk_norm, eps) + + self.norm2 = LayerNorm(dim, eps) + self.ffn = nn.Sequential( + nn.Linear(dim, ffn_dim), nn.GELU(approximate='tanh'), + nn.Linear(ffn_dim, dim)) + + def forward( + self, + x, + seq_lens, + grid_sizes, + freqs, + ): + # self-attention + y = self.self_attn(self.norm1(x).float(), seq_lens, grid_sizes, freqs) + x = x + y + y = self.ffn(self.norm2(x).float()) + x = x + y + return x + + +class Head(nn.Module): + + def __init__(self, dim, out_dim, patch_size, eps=1e-6): + super().__init__() + self.dim = dim + self.out_dim = out_dim + self.patch_size = patch_size + self.eps = eps + + # layers + out_dim = math.prod(patch_size) * out_dim + self.norm = LayerNorm(dim, eps) + self.head = nn.Linear(dim, out_dim) + + def forward(self, x): + x = self.head(self.norm(x)) + return x + + +class MotionerTransformers(nn.Module, PeftAdapterMixin): + + def __init__( + self, + patch_size=(1, 2, 2), + in_dim=16, + dim=2048, + ffn_dim=8192, + freq_dim=256, + out_dim=16, + num_heads=16, + num_layers=32, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=False, + eps=1e-6, + self_attn_block="SelfAttention", + motion_token_num=1024, + enable_tsm=False, + motion_stride=4, + expand_ratio=2, + trainable_token_pos_emb=False, + ): + super().__init__() + self.patch_size = patch_size + self.in_dim = in_dim + self.dim = dim + self.ffn_dim = ffn_dim + self.freq_dim = freq_dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.window_size = window_size + self.qk_norm = qk_norm + self.cross_attn_norm = cross_attn_norm + self.eps = eps + + self.enable_tsm = enable_tsm + self.motion_stride = motion_stride + self.expand_ratio = expand_ratio + self.sample_c = self.patch_size[0] + + # embeddings + self.patch_embedding = nn.Conv3d( + in_dim, dim, kernel_size=patch_size, stride=patch_size) + + # blocks + self.blocks = nn.ModuleList([ + MotionerAttentionBlock( + dim, + ffn_dim, + num_heads, + window_size, + qk_norm, + cross_attn_norm, + eps, + self_attn_block=self_attn_block) for _ in range(num_layers) + ]) + + # buffers (don't use register_buffer otherwise dtype will be changed in to()) + assert (dim % num_heads) == 0 and (dim // num_heads) % 2 == 0 + d = dim // num_heads + self.freqs = torch.cat([ + rope_params(1024, d - 4 * (d // 6)), + rope_params(1024, 2 * (d // 6)), + rope_params(1024, 2 * (d // 6)) + ], + dim=1) + + self.gradient_checkpointing = False + + self.motion_side_len = int(math.sqrt(motion_token_num)) + assert self.motion_side_len**2 == motion_token_num + self.token = nn.Parameter( + torch.zeros(1, motion_token_num, dim).contiguous()) + + self.trainable_token_pos_emb = trainable_token_pos_emb + if trainable_token_pos_emb: + x = torch.zeros([1, motion_token_num, num_heads, d]) + x[..., ::2] = 1 + + gride_sizes = [[ + torch.tensor([0, 0, 0]).unsqueeze(0).repeat(1, 1), + torch.tensor([1, self.motion_side_len, + self.motion_side_len]).unsqueeze(0).repeat(1, 1), + torch.tensor([1, self.motion_side_len, + self.motion_side_len]).unsqueeze(0).repeat(1, 1), + ]] + token_freqs = rope_apply(x, gride_sizes, self.freqs) + token_freqs = token_freqs[0, :, 0].reshape(motion_token_num, -1, 2) + token_freqs = token_freqs * 0.01 + self.token_freqs = torch.nn.Parameter(token_freqs) + + def after_patch_embedding(self, x): + return x + + def forward( + self, + x, + ): + """ + x: A list of videos each with shape [C, T, H, W]. + t: [B]. + context: A list of text embeddings each with shape [L, C]. + """ + # params + motion_frames = x[0].shape[1] + device = self.patch_embedding.weight.device + freqs = self.freqs + if freqs.device != device: + freqs = freqs.to(device) + + if self.trainable_token_pos_emb: + with amp.autocast(dtype=torch.float64): + token_freqs = self.token_freqs.to(torch.float64) + token_freqs = token_freqs / token_freqs.norm( + dim=-1, keepdim=True) + freqs = [freqs, torch.view_as_complex(token_freqs)] + + if self.enable_tsm: + sample_idx = [ + sample_indices( + u.shape[1], + stride=self.motion_stride, + expand_ratio=self.expand_ratio, + c=self.sample_c) for u in x + ] + x = [ + torch.flip(torch.flip(u, [1])[:, idx], [1]) + for idx, u in zip(sample_idx, x) + ] + + # embeddings + x = [self.patch_embedding(u.unsqueeze(0)) for u in x] + x = self.after_patch_embedding(x) + + seq_f, seq_h, seq_w = x[0].shape[-3:] + batch_size = len(x) + if not self.enable_tsm: + grid_sizes = torch.stack( + [torch.tensor(u.shape[2:], dtype=torch.long) for u in x]) + grid_sizes = [[ + torch.zeros_like(grid_sizes), grid_sizes, grid_sizes + ]] + seq_f = 0 + else: + grid_sizes = [] + for idx in sample_idx[0][::-1][::self.sample_c]: + tsm_frame_grid_sizes = [[ + torch.tensor([idx, 0, + 0]).unsqueeze(0).repeat(batch_size, 1), + torch.tensor([idx + 1, seq_h, + seq_w]).unsqueeze(0).repeat(batch_size, 1), + torch.tensor([1, seq_h, + seq_w]).unsqueeze(0).repeat(batch_size, 1), + ]] + grid_sizes += tsm_frame_grid_sizes + seq_f = sample_idx[0][-1] + 1 + + x = [u.flatten(2).transpose(1, 2) for u in x] + seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long) + x = torch.cat([u for u in x]) + + batch_size = len(x) + + token_grid_sizes = [[ + torch.tensor([seq_f, 0, 0]).unsqueeze(0).repeat(batch_size, 1), + torch.tensor( + [seq_f + 1, self.motion_side_len, + self.motion_side_len]).unsqueeze(0).repeat(batch_size, 1), + torch.tensor( + [1 if not self.trainable_token_pos_emb else -1, seq_h, + seq_w]).unsqueeze(0).repeat(batch_size, 1), + ] # 第三行代表rope emb的想要覆盖到的范围 + ] + + grid_sizes = grid_sizes + token_grid_sizes + token_unpatch_grid_sizes = torch.stack([ + torch.tensor([1, 32, 32], dtype=torch.long) + for b in range(batch_size) + ]) + token_len = self.token.shape[1] + token = self.token.clone().repeat(x.shape[0], 1, 1).contiguous() + seq_lens = seq_lens + torch.tensor([t.size(0) for t in token], + dtype=torch.long) + x = torch.cat([x, token], dim=1) + # arguments + kwargs = dict( + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=freqs, + ) + + # grad ckpt args + def create_custom_forward(module, return_dict=None): + + def custom_forward(*inputs, **kwargs): + if return_dict is not None: + return module(*inputs, **kwargs, return_dict=return_dict) + else: + return module(*inputs, **kwargs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = ({ + "use_reentrant": False + } if is_torch_version(">=", "1.11.0") else {}) + + for idx, block in enumerate(self.blocks): + if self.training and self.gradient_checkpointing: + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + x, + **kwargs, + **ckpt_kwargs, + ) + else: + x = block(x, **kwargs) + # head + out = x[:, -token_len:] + return out + + def unpatchify(self, x, grid_sizes): + c = self.out_dim + out = [] + for u, v in zip(x, grid_sizes.tolist()): + u = u[:math.prod(v)].view(*v, *self.patch_size, c) + u = torch.einsum('fhwpqrc->cfphqwr', u) + u = u.reshape(c, *[i * j for i, j in zip(v, self.patch_size)]) + out.append(u) + return out + + def init_weights(self): + # basic init + for m in self.modules(): + if isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.zeros_(m.bias) + + # init embeddings + nn.init.xavier_uniform_(self.patch_embedding.weight.flatten(1)) + + +class FramePackMotioner(nn.Module): + + def __init__( + self, + inner_dim=1024, + num_heads=16, # Used to indicate the number of heads in the backbone network; unrelated to this module's design + zip_frame_buckets=[ + 1, 2, 16 + ], # Three numbers representing the number of frames sampled for patch operations from the nearest to the farthest frames + drop_mode="drop", # If not "drop", it will use "padd", meaning padding instead of deletion + *args, + **kwargs): + super().__init__(*args, **kwargs) + self.proj = nn.Conv3d( + 16, inner_dim, kernel_size=(1, 2, 2), stride=(1, 2, 2)) + self.proj_2x = nn.Conv3d( + 16, inner_dim, kernel_size=(2, 4, 4), stride=(2, 4, 4)) + self.proj_4x = nn.Conv3d( + 16, inner_dim, kernel_size=(4, 8, 8), stride=(4, 8, 8)) + self.zip_frame_buckets = torch.tensor( + zip_frame_buckets, dtype=torch.long) + + self.inner_dim = inner_dim + self.num_heads = num_heads + + assert (inner_dim % + num_heads) == 0 and (inner_dim // num_heads) % 2 == 0 + d = inner_dim // num_heads + self.freqs = torch.cat([ + rope_params(1024, d - 4 * (d // 6)), + rope_params(1024, 2 * (d // 6)), + rope_params(1024, 2 * (d // 6)) + ], + dim=1) + self.drop_mode = drop_mode + + def forward(self, motion_latents, add_last_motion=2): + motion_frames = motion_latents[0].shape[1] + mot = [] + mot_remb = [] + for m in motion_latents: + lat_height, lat_width = m.shape[2], m.shape[3] + padd_lat = torch.zeros(16, self.zip_frame_buckets.sum(), lat_height, + lat_width).to( + device=m.device, dtype=m.dtype) + overlap_frame = min(padd_lat.shape[1], m.shape[1]) + if overlap_frame > 0: + padd_lat[:, -overlap_frame:] = m[:, -overlap_frame:] + + if add_last_motion < 2 and self.drop_mode != "drop": + zero_end_frame = self.zip_frame_buckets[:self.zip_frame_buckets. + __len__() - + add_last_motion - + 1].sum() + padd_lat[:, -zero_end_frame:] = 0 + + padd_lat = padd_lat.unsqueeze(0) + clean_latents_4x, clean_latents_2x, clean_latents_post = padd_lat[:, :, -self.zip_frame_buckets.sum( + ):, :, :].split( + list(self.zip_frame_buckets)[::-1], dim=2) # 16, 2 ,1 + + # patchfy + clean_latents_post = self.proj(clean_latents_post).flatten( + 2).transpose(1, 2) + clean_latents_2x = self.proj_2x(clean_latents_2x).flatten( + 2).transpose(1, 2) + clean_latents_4x = self.proj_4x(clean_latents_4x).flatten( + 2).transpose(1, 2) + + if add_last_motion < 2 and self.drop_mode == "drop": + clean_latents_post = clean_latents_post[:, : + 0] if add_last_motion < 2 else clean_latents_post + clean_latents_2x = clean_latents_2x[:, : + 0] if add_last_motion < 1 else clean_latents_2x + + motion_lat = torch.cat( + [clean_latents_post, clean_latents_2x, clean_latents_4x], dim=1) + + # rope + start_time_id = -(self.zip_frame_buckets[:1].sum()) + end_time_id = start_time_id + self.zip_frame_buckets[0] + grid_sizes = [] if add_last_motion < 2 and self.drop_mode == "drop" else \ + [ + [torch.tensor([start_time_id, 0, 0]).unsqueeze(0).repeat(1, 1), + torch.tensor([end_time_id, lat_height // 2, lat_width // 2]).unsqueeze(0).repeat(1, 1), + torch.tensor([self.zip_frame_buckets[0], lat_height // 2, lat_width // 2]).unsqueeze(0).repeat(1, 1), ] + ] + + start_time_id = -(self.zip_frame_buckets[:2].sum()) + end_time_id = start_time_id + self.zip_frame_buckets[1] // 2 + grid_sizes_2x = [] if add_last_motion < 1 and self.drop_mode == "drop" else \ + [ + [torch.tensor([start_time_id, 0, 0]).unsqueeze(0).repeat(1, 1), + torch.tensor([end_time_id, lat_height // 4, lat_width // 4]).unsqueeze(0).repeat(1, 1), + torch.tensor([self.zip_frame_buckets[1], lat_height // 2, lat_width // 2]).unsqueeze(0).repeat(1, 1), ] + ] + + start_time_id = -(self.zip_frame_buckets[:3].sum()) + end_time_id = start_time_id + self.zip_frame_buckets[2] // 4 + grid_sizes_4x = [[ + torch.tensor([start_time_id, 0, 0]).unsqueeze(0).repeat(1, 1), + torch.tensor([end_time_id, lat_height // 8, + lat_width // 8]).unsqueeze(0).repeat(1, 1), + torch.tensor([ + self.zip_frame_buckets[2], lat_height // 2, lat_width // 2 + ]).unsqueeze(0).repeat(1, 1), + ]] + + grid_sizes = grid_sizes + grid_sizes_2x + grid_sizes_4x + + motion_rope_emb = rope_precompute( + motion_lat.detach().view(1, motion_lat.shape[1], self.num_heads, + self.inner_dim // self.num_heads), + grid_sizes, + self.freqs, + start=None) + + mot.append(motion_lat) + mot_remb.append(motion_rope_emb) + return mot, mot_remb + + +def sample_indices(N, stride, expand_ratio, c): + indices = [] + current_start = 0 + + while current_start < N: + bucket_width = int(stride * (expand_ratio**(len(indices) / stride))) + + interval = int(bucket_width / stride * c) + current_end = min(N, current_start + bucket_width) + bucket_samples = [] + for i in range(current_end - 1, current_start - 1, -interval): + for near in range(c): + bucket_samples.append(i - near) + + indices += bucket_samples[::-1] + current_start += bucket_width + + return indices \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_camera_adapter.py b/VideoX-Fun/videox_fun/models/wan_camera_adapter.py new file mode 100644 index 0000000000000000000000000000000000000000..956ce8002d6ef115a9f475714e4b298a2a5ee9b8 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_camera_adapter.py @@ -0,0 +1,64 @@ +import torch +import torch.nn as nn + + +class SimpleAdapter(nn.Module): + def __init__(self, in_dim, out_dim, kernel_size, stride, downscale_factor=8, num_residual_blocks=1): + super(SimpleAdapter, self).__init__() + + # Pixel Unshuffle: reduce spatial dimensions by a factor of 8 + self.pixel_unshuffle = nn.PixelUnshuffle(downscale_factor=downscale_factor) + + # Convolution: reduce spatial dimensions by a factor + # of 2 (without overlap) + self.conv = nn.Conv2d(in_dim * downscale_factor * downscale_factor, out_dim, kernel_size=kernel_size, stride=stride, padding=0) + + # Residual blocks for feature extraction + self.residual_blocks = nn.Sequential( + *[ResidualBlock(out_dim) for _ in range(num_residual_blocks)] + ) + + def forward(self, x): + # Reshape to merge the frame dimension into batch + bs, c, f, h, w = x.size() + x = x.permute(0, 2, 1, 3, 4).contiguous().view(bs * f, c, h, w) + + # Pixel Unshuffle operation + x_unshuffled = self.pixel_unshuffle(x) + + # Convolution operation + x_conv = self.conv(x_unshuffled) + + # Feature extraction with residual blocks + out = self.residual_blocks(x_conv) + + # Reshape to restore original bf dimension + out = out.view(bs, f, out.size(1), out.size(2), out.size(3)) + + # Permute dimensions to reorder (if needed), e.g., swap channels and feature frames + out = out.permute(0, 2, 1, 3, 4) + + return out + + +class ResidualBlock(nn.Module): + def __init__(self, dim): + super(ResidualBlock, self).__init__() + self.conv1 = nn.Conv2d(dim, dim, kernel_size=3, padding=1) + self.relu = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(dim, dim, kernel_size=3, padding=1) + + def forward(self, x): + residual = x + out = self.relu(self.conv1(x)) + out = self.conv2(out) + out += residual + return out + +# Example usage +# in_dim = 3 +# out_dim = 64 +# adapter = SimpleAdapterWithReshape(in_dim, out_dim) +# x = torch.randn(1, in_dim, 4, 64, 64) # e.g., batch size = 1, channels = 3, frames/features = 4 +# output = adapter(x) +# print(output.shape) # Should reflect transformed dimensions diff --git a/VideoX-Fun/videox_fun/models/wan_image_encoder.py b/VideoX-Fun/videox_fun/models/wan_image_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..dc95899e4abac2ea0c14dfe143c0b15b0b119d4d --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_image_encoder.py @@ -0,0 +1,553 @@ +# Modified from ``https://github.com/openai/CLIP'' and ``https://github.com/mlfoundations/open_clip'' +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torchvision.transforms as T + +from .attention_utils import attention, flash_attention +from .wan_xlm_roberta import XLMRoberta +from diffusers.configuration_utils import ConfigMixin +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.modeling_utils import ModelMixin + + +__all__ = [ + 'XLMRobertaCLIP', + 'clip_xlm_roberta_vit_h_14', + 'CLIPModel', +] + + +def pos_interpolate(pos, seq_len): + if pos.size(1) == seq_len: + return pos + else: + src_grid = int(math.sqrt(pos.size(1))) + tar_grid = int(math.sqrt(seq_len)) + n = pos.size(1) - src_grid * src_grid + return torch.cat([ + pos[:, :n], + F.interpolate( + pos[:, n:].float().reshape(1, src_grid, src_grid, -1).permute( + 0, 3, 1, 2), + size=(tar_grid, tar_grid), + mode='bicubic', + align_corners=False).flatten(2).transpose(1, 2) + ], + dim=1) + + +class QuickGELU(nn.Module): + + def forward(self, x): + return x * torch.sigmoid(1.702 * x) + + +class LayerNorm(nn.LayerNorm): + + def forward(self, x): + return super().forward(x.float()).type_as(x) + + +class SelfAttention(nn.Module): + + def __init__(self, + dim, + num_heads, + causal=False, + attn_dropout=0.0, + proj_dropout=0.0): + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.causal = causal + self.attn_dropout = attn_dropout + self.proj_dropout = proj_dropout + + # layers + self.to_qkv = nn.Linear(dim, dim * 3) + self.proj = nn.Linear(dim, dim) + + def forward(self, x): + """ + x: [B, L, C]. + """ + b, s, c, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + q, k, v = self.to_qkv(x).view(b, s, 3, n, d).unbind(2) + + # compute attention + p = self.attn_dropout if self.training else 0.0 + x = attention(q, k, v, dropout_p=p, causal=self.causal, attention_type="none") + x = x.reshape(b, s, c) + + # output + x = self.proj(x) + x = F.dropout(x, self.proj_dropout, self.training) + return x + + +class SwiGLU(nn.Module): + + def __init__(self, dim, mid_dim): + super().__init__() + self.dim = dim + self.mid_dim = mid_dim + + # layers + self.fc1 = nn.Linear(dim, mid_dim) + self.fc2 = nn.Linear(dim, mid_dim) + self.fc3 = nn.Linear(mid_dim, dim) + + def forward(self, x): + x = F.silu(self.fc1(x)) * self.fc2(x) + x = self.fc3(x) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, + dim, + mlp_ratio, + num_heads, + post_norm=False, + causal=False, + activation='quick_gelu', + attn_dropout=0.0, + proj_dropout=0.0, + norm_eps=1e-5): + assert activation in ['quick_gelu', 'gelu', 'swi_glu'] + super().__init__() + self.dim = dim + self.mlp_ratio = mlp_ratio + self.num_heads = num_heads + self.post_norm = post_norm + self.causal = causal + self.norm_eps = norm_eps + + # layers + self.norm1 = LayerNorm(dim, eps=norm_eps) + self.attn = SelfAttention(dim, num_heads, causal, attn_dropout, + proj_dropout) + self.norm2 = LayerNorm(dim, eps=norm_eps) + if activation == 'swi_glu': + self.mlp = SwiGLU(dim, int(dim * mlp_ratio)) + else: + self.mlp = nn.Sequential( + nn.Linear(dim, int(dim * mlp_ratio)), + QuickGELU() if activation == 'quick_gelu' else nn.GELU(), + nn.Linear(int(dim * mlp_ratio), dim), nn.Dropout(proj_dropout)) + + def forward(self, x): + if self.post_norm: + x = x + self.norm1(self.attn(x)) + x = x + self.norm2(self.mlp(x)) + else: + x = x + self.attn(self.norm1(x)) + x = x + self.mlp(self.norm2(x)) + return x + + +class AttentionPool(nn.Module): + + def __init__(self, + dim, + mlp_ratio, + num_heads, + activation='gelu', + proj_dropout=0.0, + norm_eps=1e-5): + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.mlp_ratio = mlp_ratio + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.proj_dropout = proj_dropout + self.norm_eps = norm_eps + + # layers + gain = 1.0 / math.sqrt(dim) + self.cls_embedding = nn.Parameter(gain * torch.randn(1, 1, dim)) + self.to_q = nn.Linear(dim, dim) + self.to_kv = nn.Linear(dim, dim * 2) + self.proj = nn.Linear(dim, dim) + self.norm = LayerNorm(dim, eps=norm_eps) + self.mlp = nn.Sequential( + nn.Linear(dim, int(dim * mlp_ratio)), + QuickGELU() if activation == 'quick_gelu' else nn.GELU(), + nn.Linear(int(dim * mlp_ratio), dim), nn.Dropout(proj_dropout)) + + def forward(self, x): + """ + x: [B, L, C]. + """ + b, s, c, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + q = self.to_q(self.cls_embedding).view(1, 1, n, d).expand(b, -1, -1, -1) + k, v = self.to_kv(x).view(b, s, 2, n, d).unbind(2) + + # compute attention + x = flash_attention(q, k, v, version=2) + x = x.reshape(b, 1, c) + + # output + x = self.proj(x) + x = F.dropout(x, self.proj_dropout, self.training) + + # mlp + x = x + self.mlp(self.norm(x)) + return x[:, 0] + + +class VisionTransformer(nn.Module): + + def __init__(self, + image_size=224, + patch_size=16, + dim=768, + mlp_ratio=4, + out_dim=512, + num_heads=12, + num_layers=12, + pool_type='token', + pre_norm=True, + post_norm=False, + activation='quick_gelu', + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0, + norm_eps=1e-5): + if image_size % patch_size != 0: + print( + '[WARNING] image_size is not divisible by patch_size', + flush=True) + assert pool_type in ('token', 'token_fc', 'attn_pool') + out_dim = out_dim or dim + super().__init__() + self.image_size = image_size + self.patch_size = patch_size + self.num_patches = (image_size // patch_size)**2 + self.dim = dim + self.mlp_ratio = mlp_ratio + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.pool_type = pool_type + self.post_norm = post_norm + self.norm_eps = norm_eps + + # embeddings + gain = 1.0 / math.sqrt(dim) + self.patch_embedding = nn.Conv2d( + 3, + dim, + kernel_size=patch_size, + stride=patch_size, + bias=not pre_norm) + if pool_type in ('token', 'token_fc'): + self.cls_embedding = nn.Parameter(gain * torch.randn(1, 1, dim)) + self.pos_embedding = nn.Parameter(gain * torch.randn( + 1, self.num_patches + + (1 if pool_type in ('token', 'token_fc') else 0), dim)) + self.dropout = nn.Dropout(embedding_dropout) + + # transformer + self.pre_norm = LayerNorm(dim, eps=norm_eps) if pre_norm else None + self.transformer = nn.Sequential(*[ + AttentionBlock(dim, mlp_ratio, num_heads, post_norm, False, + activation, attn_dropout, proj_dropout, norm_eps) + for _ in range(num_layers) + ]) + self.post_norm = LayerNorm(dim, eps=norm_eps) + + # head + if pool_type == 'token': + self.head = nn.Parameter(gain * torch.randn(dim, out_dim)) + elif pool_type == 'token_fc': + self.head = nn.Linear(dim, out_dim) + elif pool_type == 'attn_pool': + self.head = AttentionPool(dim, mlp_ratio, num_heads, activation, + proj_dropout, norm_eps) + + def forward(self, x, interpolation=False, use_31_block=False): + b = x.size(0) + + # embeddings + x = self.patch_embedding(x).flatten(2).permute(0, 2, 1) + if self.pool_type in ('token', 'token_fc'): + x = torch.cat([self.cls_embedding.expand(b, -1, -1), x], dim=1) + if interpolation: + e = pos_interpolate(self.pos_embedding, x.size(1)) + else: + e = self.pos_embedding + x = self.dropout(x + e) + if self.pre_norm is not None: + x = self.pre_norm(x) + + # transformer + if use_31_block: + x = self.transformer[:-1](x) + return x + else: + x = self.transformer(x) + return x + + +class XLMRobertaWithHead(XLMRoberta): + + def __init__(self, **kwargs): + self.out_dim = kwargs.pop('out_dim') + super().__init__(**kwargs) + + # head + mid_dim = (self.dim + self.out_dim) // 2 + self.head = nn.Sequential( + nn.Linear(self.dim, mid_dim, bias=False), nn.GELU(), + nn.Linear(mid_dim, self.out_dim, bias=False)) + + def forward(self, ids): + # xlm-roberta + x = super().forward(ids) + + # average pooling + mask = ids.ne(self.pad_id).unsqueeze(-1).to(x) + x = (x * mask).sum(dim=1) / mask.sum(dim=1) + + # head + x = self.head(x) + return x + + +class XLMRobertaCLIP(nn.Module): + + def __init__(self, + embed_dim=1024, + image_size=224, + patch_size=14, + vision_dim=1280, + vision_mlp_ratio=4, + vision_heads=16, + vision_layers=32, + vision_pool='token', + vision_pre_norm=True, + vision_post_norm=False, + activation='gelu', + vocab_size=250002, + max_text_len=514, + type_size=1, + pad_id=1, + text_dim=1024, + text_heads=16, + text_layers=24, + text_post_norm=True, + text_dropout=0.1, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0, + norm_eps=1e-5): + super().__init__() + self.embed_dim = embed_dim + self.image_size = image_size + self.patch_size = patch_size + self.vision_dim = vision_dim + self.vision_mlp_ratio = vision_mlp_ratio + self.vision_heads = vision_heads + self.vision_layers = vision_layers + self.vision_pre_norm = vision_pre_norm + self.vision_post_norm = vision_post_norm + self.activation = activation + self.vocab_size = vocab_size + self.max_text_len = max_text_len + self.type_size = type_size + self.pad_id = pad_id + self.text_dim = text_dim + self.text_heads = text_heads + self.text_layers = text_layers + self.text_post_norm = text_post_norm + self.norm_eps = norm_eps + + # models + self.visual = VisionTransformer( + image_size=image_size, + patch_size=patch_size, + dim=vision_dim, + mlp_ratio=vision_mlp_ratio, + out_dim=embed_dim, + num_heads=vision_heads, + num_layers=vision_layers, + pool_type=vision_pool, + pre_norm=vision_pre_norm, + post_norm=vision_post_norm, + activation=activation, + attn_dropout=attn_dropout, + proj_dropout=proj_dropout, + embedding_dropout=embedding_dropout, + norm_eps=norm_eps) + self.textual = XLMRobertaWithHead( + vocab_size=vocab_size, + max_seq_len=max_text_len, + type_size=type_size, + pad_id=pad_id, + dim=text_dim, + out_dim=embed_dim, + num_heads=text_heads, + num_layers=text_layers, + post_norm=text_post_norm, + dropout=text_dropout) + self.log_scale = nn.Parameter(math.log(1 / 0.07) * torch.ones([])) + + def forward(self, imgs, txt_ids): + """ + imgs: [B, 3, H, W] of torch.float32. + - mean: [0.48145466, 0.4578275, 0.40821073] + - std: [0.26862954, 0.26130258, 0.27577711] + txt_ids: [B, L] of torch.long. + Encoded by data.CLIPTokenizer. + """ + xi = self.visual(imgs) + xt = self.textual(txt_ids) + return xi, xt + + def param_groups(self): + groups = [{ + 'params': [ + p for n, p in self.named_parameters() + if 'norm' in n or n.endswith('bias') + ], + 'weight_decay': 0.0 + }, { + 'params': [ + p for n, p in self.named_parameters() + if not ('norm' in n or n.endswith('bias')) + ] + }] + return groups + + +def _clip(pretrained=False, + pretrained_name=None, + model_cls=XLMRobertaCLIP, + return_transforms=False, + return_tokenizer=False, + tokenizer_padding='eos', + dtype=torch.float32, + device='cpu', + **kwargs): + # init a model on device + with torch.device(device): + model = model_cls(**kwargs) + + # set device + model = model.to(dtype=dtype, device=device) + output = (model,) + + # init transforms + if return_transforms: + # mean and std + if 'siglip' in pretrained_name.lower(): + mean, std = [0.5, 0.5, 0.5], [0.5, 0.5, 0.5] + else: + mean = [0.48145466, 0.4578275, 0.40821073] + std = [0.26862954, 0.26130258, 0.27577711] + + # transforms + transforms = T.Compose([ + T.Resize((model.image_size, model.image_size), + interpolation=T.InterpolationMode.BICUBIC), + T.ToTensor(), + T.Normalize(mean=mean, std=std) + ]) + output += (transforms,) + return output[0] if len(output) == 1 else output + + +def clip_xlm_roberta_vit_h_14( + pretrained=False, + pretrained_name='open-clip-xlm-roberta-large-vit-huge-14', + **kwargs): + cfg = dict( + embed_dim=1024, + image_size=224, + patch_size=14, + vision_dim=1280, + vision_mlp_ratio=4, + vision_heads=16, + vision_layers=32, + vision_pool='token', + activation='gelu', + vocab_size=250002, + max_text_len=514, + type_size=1, + pad_id=1, + text_dim=1024, + text_heads=16, + text_layers=24, + text_post_norm=True, + text_dropout=0.1, + attn_dropout=0.0, + proj_dropout=0.0, + embedding_dropout=0.0) + cfg.update(**kwargs) + return _clip(pretrained, pretrained_name, XLMRobertaCLIP, **cfg) + + +class CLIPModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): + + def __init__(self): + super(CLIPModel, self).__init__() + # init model + self.model, self.transforms = clip_xlm_roberta_vit_h_14( + pretrained=False, + return_transforms=True, + return_tokenizer=False) + + def forward(self, videos): + # preprocess + size = (self.model.image_size,) * 2 + videos = torch.cat([ + F.interpolate( + u.transpose(0, 1), + size=size, + mode='bicubic', + align_corners=False) for u in videos + ]) + videos = self.transforms.transforms[-1](videos.mul_(0.5).add_(0.5)) + + # forward + with torch.cuda.amp.autocast(dtype=self.dtype): + out = self.model.visual(videos, use_31_block=True) + return out + + @classmethod + def from_pretrained(cls, pretrained_model_path, transformer_additional_kwargs={}): + def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + + model = cls(**filter_kwargs(cls, transformer_additional_kwargs)) + if pretrained_model_path.endswith(".safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(pretrained_model_path) + else: + state_dict = torch.load(pretrained_model_path, map_location="cpu") + tmp_state_dict = {} + for key in state_dict: + tmp_state_dict["model." + key] = state_dict[key] + state_dict = tmp_state_dict + m, u = model.load_state_dict(state_dict) + + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m, u) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_text_encoder.py b/VideoX-Fun/videox_fun/models/wan_text_encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..0076d78dec9a3b8ca83c6ab4639e35e4fc86c222 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_text_encoder.py @@ -0,0 +1,395 @@ +# Modified from https://github.com/Wan-Video/Wan2.1/blob/main/wan/modules/t5.py +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +import math +from typing import Optional + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.modeling_utils import ModelMixin + + +def fp16_clamp(x): + if x.dtype == torch.float16 and torch.isinf(x).any(): + clamp = torch.finfo(x.dtype).max - 1000 + x = torch.clamp(x, min=-clamp, max=clamp) + return x + + +def init_weights(m): + if isinstance(m, T5LayerNorm): + nn.init.ones_(m.weight) + elif isinstance(m, T5FeedForward): + nn.init.normal_(m.gate[0].weight, std=m.dim**-0.5) + nn.init.normal_(m.fc1.weight, std=m.dim**-0.5) + nn.init.normal_(m.fc2.weight, std=m.dim_ffn**-0.5) + elif isinstance(m, T5Attention): + nn.init.normal_(m.q.weight, std=(m.dim * m.dim_attn)**-0.5) + nn.init.normal_(m.k.weight, std=m.dim**-0.5) + nn.init.normal_(m.v.weight, std=m.dim**-0.5) + nn.init.normal_(m.o.weight, std=(m.num_heads * m.dim_attn)**-0.5) + elif isinstance(m, T5RelativeEmbedding): + nn.init.normal_( + m.embedding.weight, std=(2 * m.num_buckets * m.num_heads)**-0.5) + + +class GELU(nn.Module): + def forward(self, x): + return 0.5 * x * (1.0 + torch.tanh( + math.sqrt(2.0 / math.pi) * (x + 0.044715 * torch.pow(x, 3.0)))) + + +class T5LayerNorm(nn.Module): + def __init__(self, dim, eps=1e-6): + super(T5LayerNorm, self).__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def forward(self, x): + x = x * torch.rsqrt(x.float().pow(2).mean(dim=-1, keepdim=True) + + self.eps) + if self.weight.dtype in [torch.float16, torch.bfloat16]: + x = x.type_as(self.weight) + return self.weight * x + + +class T5Attention(nn.Module): + def __init__(self, dim, dim_attn, num_heads, dropout=0.1): + assert dim_attn % num_heads == 0 + super(T5Attention, self).__init__() + self.dim = dim + self.dim_attn = dim_attn + self.num_heads = num_heads + self.head_dim = dim_attn // num_heads + + # layers + self.q = nn.Linear(dim, dim_attn, bias=False) + self.k = nn.Linear(dim, dim_attn, bias=False) + self.v = nn.Linear(dim, dim_attn, bias=False) + self.o = nn.Linear(dim_attn, dim, bias=False) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, context=None, mask=None, pos_bias=None): + """ + x: [B, L1, C]. + context: [B, L2, C] or None. + mask: [B, L2] or [B, L1, L2] or None. + """ + # check inputs + context = x if context is None else context + b, n, c = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.q(x).view(b, -1, n, c) + k = self.k(context).view(b, -1, n, c) + v = self.v(context).view(b, -1, n, c) + + # attention bias + attn_bias = x.new_zeros(b, n, q.size(1), k.size(1)) + if pos_bias is not None: + attn_bias += pos_bias + if mask is not None: + assert mask.ndim in [2, 3] + mask = mask.view(b, 1, 1, + -1) if mask.ndim == 2 else mask.unsqueeze(1) + attn_bias.masked_fill_(mask == 0, torch.finfo(x.dtype).min) + + # compute attention (T5 does not use scaling) + attn = torch.einsum('binc,bjnc->bnij', q, k) + attn_bias + attn = F.softmax(attn.float(), dim=-1).type_as(attn) + x = torch.einsum('bnij,bjnc->binc', attn, v) + + # output + x = x.reshape(b, -1, n * c) + x = self.o(x) + x = self.dropout(x) + return x + + +class T5FeedForward(nn.Module): + + def __init__(self, dim, dim_ffn, dropout=0.1): + super(T5FeedForward, self).__init__() + self.dim = dim + self.dim_ffn = dim_ffn + + # layers + self.gate = nn.Sequential(nn.Linear(dim, dim_ffn, bias=False), GELU()) + self.fc1 = nn.Linear(dim, dim_ffn, bias=False) + self.fc2 = nn.Linear(dim_ffn, dim, bias=False) + self.dropout = nn.Dropout(dropout) + + def forward(self, x): + x = self.fc1(x) * self.gate(x) + x = self.dropout(x) + x = self.fc2(x) + x = self.dropout(x) + return x + + +class T5SelfAttention(nn.Module): + def __init__(self, + dim, + dim_attn, + dim_ffn, + num_heads, + num_buckets, + shared_pos=True, + dropout=0.1): + super(T5SelfAttention, self).__init__() + self.dim = dim + self.dim_attn = dim_attn + self.dim_ffn = dim_ffn + self.num_heads = num_heads + self.num_buckets = num_buckets + self.shared_pos = shared_pos + + # layers + self.norm1 = T5LayerNorm(dim) + self.attn = T5Attention(dim, dim_attn, num_heads, dropout) + self.norm2 = T5LayerNorm(dim) + self.ffn = T5FeedForward(dim, dim_ffn, dropout) + self.pos_embedding = None if shared_pos else T5RelativeEmbedding( + num_buckets, num_heads, bidirectional=True) + + def forward(self, x, mask=None, pos_bias=None): + e = pos_bias if self.shared_pos else self.pos_embedding( + x.size(1), x.size(1)) + x = fp16_clamp(x + self.attn(self.norm1(x), mask=mask, pos_bias=e)) + x = fp16_clamp(x + self.ffn(self.norm2(x))) + return x + + +class T5CrossAttention(nn.Module): + def __init__(self, + dim, + dim_attn, + dim_ffn, + num_heads, + num_buckets, + shared_pos=True, + dropout=0.1): + super(T5CrossAttention, self).__init__() + self.dim = dim + self.dim_attn = dim_attn + self.dim_ffn = dim_ffn + self.num_heads = num_heads + self.num_buckets = num_buckets + self.shared_pos = shared_pos + + # layers + self.norm1 = T5LayerNorm(dim) + self.self_attn = T5Attention(dim, dim_attn, num_heads, dropout) + self.norm2 = T5LayerNorm(dim) + self.cross_attn = T5Attention(dim, dim_attn, num_heads, dropout) + self.norm3 = T5LayerNorm(dim) + self.ffn = T5FeedForward(dim, dim_ffn, dropout) + self.pos_embedding = None if shared_pos else T5RelativeEmbedding( + num_buckets, num_heads, bidirectional=False) + + def forward(self, + x, + mask=None, + encoder_states=None, + encoder_mask=None, + pos_bias=None): + e = pos_bias if self.shared_pos else self.pos_embedding( + x.size(1), x.size(1)) + x = fp16_clamp(x + self.self_attn(self.norm1(x), mask=mask, pos_bias=e)) + x = fp16_clamp(x + self.cross_attn( + self.norm2(x), context=encoder_states, mask=encoder_mask)) + x = fp16_clamp(x + self.ffn(self.norm3(x))) + return x + + +class T5RelativeEmbedding(nn.Module): + def __init__(self, num_buckets, num_heads, bidirectional, max_dist=128): + super(T5RelativeEmbedding, self).__init__() + self.num_buckets = num_buckets + self.num_heads = num_heads + self.bidirectional = bidirectional + self.max_dist = max_dist + + # layers + self.embedding = nn.Embedding(num_buckets, num_heads) + + def forward(self, lq, lk): + device = self.embedding.weight.device + # rel_pos = torch.arange(lk).unsqueeze(0).to(device) - \ + # torch.arange(lq).unsqueeze(1).to(device) + if torch.device(type="meta") != device: + rel_pos = torch.arange(lk, device=device).unsqueeze(0) - \ + torch.arange(lq, device=device).unsqueeze(1) + else: + rel_pos = torch.arange(lk).unsqueeze(0) - \ + torch.arange(lq).unsqueeze(1) + rel_pos = self._relative_position_bucket(rel_pos) + rel_pos_embeds = self.embedding(rel_pos) + rel_pos_embeds = rel_pos_embeds.permute(2, 0, 1).unsqueeze( + 0) # [1, N, Lq, Lk] + return rel_pos_embeds.contiguous() + + def _relative_position_bucket(self, rel_pos): + # preprocess + if self.bidirectional: + num_buckets = self.num_buckets // 2 + rel_buckets = (rel_pos > 0).long() * num_buckets + rel_pos = torch.abs(rel_pos) + else: + num_buckets = self.num_buckets + rel_buckets = 0 + rel_pos = -torch.min(rel_pos, torch.zeros_like(rel_pos)) + + # embeddings for small and large positions + max_exact = num_buckets // 2 + rel_pos_large = max_exact + (torch.log(rel_pos.float() / max_exact) / + math.log(self.max_dist / max_exact) * + (num_buckets - max_exact)).long() + rel_pos_large = torch.min( + rel_pos_large, torch.full_like(rel_pos_large, num_buckets - 1)) + rel_buckets += torch.where(rel_pos < max_exact, rel_pos, rel_pos_large) + return rel_buckets + +class WanT5EncoderModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): + def __init__(self, + vocab, + dim, + dim_attn, + dim_ffn, + num_heads, + num_layers, + num_buckets, + shared_pos=True, + dropout=0.1): + super(WanT5EncoderModel, self).__init__() + self.dim = dim + self.dim_attn = dim_attn + self.dim_ffn = dim_ffn + self.num_heads = num_heads + self.num_layers = num_layers + self.num_buckets = num_buckets + self.shared_pos = shared_pos + + # layers + self.token_embedding = vocab if isinstance(vocab, nn.Embedding) \ + else nn.Embedding(vocab, dim) + self.pos_embedding = T5RelativeEmbedding( + num_buckets, num_heads, bidirectional=True) if shared_pos else None + self.dropout = nn.Dropout(dropout) + self.blocks = nn.ModuleList([ + T5SelfAttention(dim, dim_attn, dim_ffn, num_heads, num_buckets, + shared_pos, dropout) for _ in range(num_layers) + ]) + self.norm = T5LayerNorm(dim) + + # initialize weights + self.apply(init_weights) + + def forward( + self, + input_ids: Optional[torch.LongTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + ): + x = self.token_embedding(input_ids) + x = self.dropout(x) + e = self.pos_embedding(x.size(1), + x.size(1)) if self.shared_pos else None + for block in self.blocks: + x = block(x, attention_mask, pos_bias=e) + x = self.norm(x) + x = self.dropout(x) + return (x, ) + + @classmethod + def from_pretrained(cls, pretrained_model_path, additional_kwargs={}, low_cpu_mem_usage=False, torch_dtype=torch.bfloat16): + def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + + if low_cpu_mem_usage: + try: + import re + + from diffusers import __version__ as diffusers_version + if diffusers_version >= "0.33.0": + from diffusers.models.model_loading_utils import \ + load_model_dict_into_meta + else: + from diffusers.models.modeling_utils import \ + load_model_dict_into_meta + from diffusers.utils import is_accelerate_available + if is_accelerate_available(): + import accelerate + + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + model = cls(**filter_kwargs(cls, additional_kwargs)) + + param_device = "cpu" + if pretrained_model_path.endswith(".safetensors"): + from safetensors.torch import load_file + state_dict = load_file(pretrained_model_path) + else: + state_dict = torch.load(pretrained_model_path, map_location="cpu") + + if diffusers_version >= "0.33.0": + # Diffusers has refactored `load_model_dict_into_meta` since version 0.33.0 in this commit: + # https://github.com/huggingface/diffusers/commit/f5929e03060d56063ff34b25a8308833bec7c785. + load_model_dict_into_meta( + model, + state_dict, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + else: + # move the params from meta device to cpu + missing_keys = set(model.state_dict().keys()) - set(state_dict.keys()) + if len(missing_keys) > 0: + raise ValueError( + f"Cannot load {cls} from {pretrained_model_path} because the following keys are" + f" missing: \n {', '.join(missing_keys)}. \n Please make sure to pass" + " `low_cpu_mem_usage=False` and `device_map=None` if you want to randomly initialize" + " those weights or else make sure your checkpoint file is correct." + ) + + unexpected_keys = load_model_dict_into_meta( + model, + state_dict, + device=param_device, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + + if cls._keys_to_ignore_on_load_unexpected is not None: + for pat in cls._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + print( + f"Some weights of the model checkpoint were not used when initializing {cls.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + + return model + except Exception as e: + print( + f"The low_cpu_mem_usage mode is not work because {e}. Use low_cpu_mem_usage=False instead." + ) + + model = cls(**filter_kwargs(cls, additional_kwargs)) + if pretrained_model_path.endswith(".safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(pretrained_model_path) + else: + state_dict = torch.load(pretrained_model_path, map_location="cpu") + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m, u) + + model = model.to(torch_dtype) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_transformer3d.py b/VideoX-Fun/videox_fun/models/wan_transformer3d.py new file mode 100644 index 0000000000000000000000000000000000000000..36d906ba4b7e78b026c49702ef0f9f8792179981 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_transformer3d.py @@ -0,0 +1,1394 @@ +# Modified from https://github.com/Wan-Video/Wan2.1/blob/main/wan/modules/model.py +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. + +import glob +import json +import math +import os +import types +import warnings +from typing import Any, Dict, Optional, Union + +import numpy as np +import torch +import torch.cuda.amp as amp +import torch.nn as nn +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils import is_torch_version, logging +from torch import nn + +from ..dist import (get_sequence_parallel_rank, + get_sequence_parallel_world_size, get_sp_group, + usp_attn_forward, xFuserLongContextAttention) +from ..utils import cfg_skip +from .attention_utils import attention +from .cache_utils import TeaCache +from .wan_camera_adapter import SimpleAdapter + + +def sinusoidal_embedding_1d(dim, position): + # preprocess + assert dim % 2 == 0 + half = dim // 2 + position = position.type(torch.float64) + + # calculation + sinusoid = torch.outer( + position, torch.pow(10000, -torch.arange(half).to(position).div(half))) + x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1) + return x + + +@amp.autocast(enabled=False) +def rope_params(max_seq_len, dim, theta=10000): + assert dim % 2 == 0 + freqs = torch.outer( + torch.arange(max_seq_len), + 1.0 / torch.pow(theta, + torch.arange(0, dim, 2).to(torch.float64).div(dim))) + freqs = torch.polar(torch.ones_like(freqs), freqs) + return freqs + + +# modified from https://github.com/thu-ml/RIFLEx/blob/main/riflex_utils.py +@amp.autocast(enabled=False) +def get_1d_rotary_pos_embed_riflex( + pos: Union[np.ndarray, int], + dim: int, + theta: float = 10000.0, + use_real=False, + k: Optional[int] = None, + L_test: Optional[int] = None, + L_test_scale: Optional[int] = None, +): + """ + RIFLEx: Precompute the frequency tensor for complex exponentials (cis) with given dimensions. + + This function calculates a frequency tensor with complex exponentials using the given dimension 'dim' and the end + index 'end'. The 'theta' parameter scales the frequencies. The returned tensor contains complex values in complex64 + data type. + + Args: + dim (`int`): Dimension of the frequency tensor. + pos (`np.ndarray` or `int`): Position indices for the frequency tensor. [S] or scalar + theta (`float`, *optional*, defaults to 10000.0): + Scaling factor for frequency computation. Defaults to 10000.0. + use_real (`bool`, *optional*): + If True, return real part and imaginary part separately. Otherwise, return complex numbers. + k (`int`, *optional*, defaults to None): the index for the intrinsic frequency in RoPE + L_test (`int`, *optional*, defaults to None): the number of frames for inference + Returns: + `torch.Tensor`: Precomputed frequency tensor with complex exponentials. [S, D/2] + """ + assert dim % 2 == 0 + + if isinstance(pos, int): + pos = torch.arange(pos) + if isinstance(pos, np.ndarray): + pos = torch.from_numpy(pos) # type: ignore # [S] + + freqs = 1.0 / torch.pow(theta, + torch.arange(0, dim, 2).to(torch.float64).div(dim)) + + # === Riflex modification start === + # Reduce the intrinsic frequency to stay within a single period after extrapolation (see Eq. (8)). + # Empirical observations show that a few videos may exhibit repetition in the tail frames. + # To be conservative, we multiply by 0.9 to keep the extrapolated length below 90% of a single period. + if k is not None: + freqs[k-1] = 0.9 * 2 * torch.pi / L_test + # === Riflex modification end === + if L_test_scale is not None: + freqs[k-1] = freqs[k-1] / L_test_scale + + freqs = torch.outer(pos, freqs) # type: ignore # [S, D/2] + if use_real: + freqs_cos = freqs.cos().repeat_interleave(2, dim=1).float() # [S, D] + freqs_sin = freqs.sin().repeat_interleave(2, dim=1).float() # [S, D] + return freqs_cos, freqs_sin + else: + # lumina + freqs_cis = torch.polar(torch.ones_like(freqs), freqs) # complex64 # [S, D/2] + return freqs_cis + + +# Similar to diffusers.pipelines.hunyuandit.pipeline_hunyuandit.get_resize_crop_region_for_grid +def get_resize_crop_region_for_grid(src, tgt_width, tgt_height): + tw = tgt_width + th = tgt_height + h, w = src + r = h / w + if r > (th / tw): + resize_height = th + resize_width = int(round(th / h * w)) + else: + resize_width = tw + resize_height = int(round(tw / w * h)) + + crop_top = int(round((th - resize_height) / 2.0)) + crop_left = int(round((tw - resize_width) / 2.0)) + + return (crop_top, crop_left), (crop_top + resize_height, crop_left + resize_width) + + +@amp.autocast(enabled=False) +@torch.compiler.disable() +def rope_apply(x, grid_sizes, freqs): + n, c = x.size(2), x.size(3) // 2 + + # split freqs + freqs = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1) + + # loop over samples + output = [] + for i, (f, h, w) in enumerate(grid_sizes.tolist()): + seq_len = f * h * w + + # precompute multipliers + x_i = torch.view_as_complex(x[i, :seq_len].to(torch.float32).reshape( + seq_len, n, -1, 2)) + freqs_i = torch.cat([ + freqs[0][:f].view(f, 1, 1, -1).expand(f, h, w, -1), + freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1), + freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1) + ], + dim=-1).reshape(seq_len, 1, -1) + + # apply rotary embedding + x_i = torch.view_as_real(x_i * freqs_i).flatten(2) + x_i = torch.cat([x_i, x[i, seq_len:]]) + + # append to collection + output.append(x_i) + return torch.stack(output).to(x.dtype) + + +def rope_apply_qk(q, k, grid_sizes, freqs): + q = rope_apply(q, grid_sizes, freqs) + k = rope_apply(k, grid_sizes, freqs) + return q, k + + +class WanRMSNorm(nn.Module): + + def __init__(self, dim, eps=1e-5): + super().__init__() + self.dim = dim + self.eps = eps + self.weight = nn.Parameter(torch.ones(dim)) + + def forward(self, x): + r""" + Args: + x(Tensor): Shape [B, L, C] + """ + return self._norm(x) * self.weight + + def _norm(self, x): + return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps).to(x.dtype) + + +class WanLayerNorm(nn.LayerNorm): + + def __init__(self, dim, eps=1e-6, elementwise_affine=False): + super().__init__(dim, elementwise_affine=elementwise_affine, eps=eps) + + def forward(self, x): + r""" + Args: + x(Tensor): Shape [B, L, C] + """ + return super().forward(x) + + +class WanSelfAttention(nn.Module): + + def __init__(self, + dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + eps=1e-6): + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.eps = eps + + # layers + self.q = nn.Linear(dim, dim) + self.k = nn.Linear(dim, dim) + self.v = nn.Linear(dim, dim) + self.o = nn.Linear(dim, dim) + self.norm_q = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + self.norm_k = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + + def forward(self, x, seq_lens, grid_sizes, freqs, dtype=torch.bfloat16, t=0): + r""" + Args: + x(Tensor): Shape [B, L, num_heads, C / num_heads] + seq_lens(Tensor): Shape [B] + grid_sizes(Tensor): Shape [B, 3], the second dimension contains (F, H, W) + freqs(Tensor): Rope freqs, shape [1024, C / num_heads / 2] + """ + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + + # query, key, value function + def qkv_fn(x): + q = self.norm_q(self.q(x.to(dtype))).view(b, s, n, d) + k = self.norm_k(self.k(x.to(dtype))).view(b, s, n, d) + v = self.v(x.to(dtype)).view(b, s, n, d) + return q, k, v + + q, k, v = qkv_fn(x) + + q, k = rope_apply_qk(q, k, grid_sizes, freqs) + + x = attention( + q.to(dtype), + k.to(dtype), + v=v.to(dtype), + k_lens=seq_lens, + window_size=self.window_size) + x = x.to(dtype) + + # output + x = x.flatten(2) + x = self.o(x) + return x + + +class WanT2VCrossAttention(WanSelfAttention): + + def forward(self, x, context, context_lens, dtype=torch.bfloat16, t=0): + r""" + Args: + x(Tensor): Shape [B, L1, C] + context(Tensor): Shape [B, L2, C] + context_lens(Tensor): Shape [B] + """ + b, n, d = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.norm_q(self.q(x.to(dtype))).view(b, -1, n, d) + k = self.norm_k(self.k(context.to(dtype))).view(b, -1, n, d) + v = self.v(context.to(dtype)).view(b, -1, n, d) + + # compute attention + x = attention( + q.to(dtype), + k.to(dtype), + v.to(dtype), + k_lens=context_lens + ) + x = x.to(dtype) + + # output + x = x.flatten(2) + x = self.o(x) + return x + + +class WanI2VCrossAttention(WanSelfAttention): + + def __init__(self, + dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + eps=1e-6): + super().__init__(dim, num_heads, window_size, qk_norm, eps) + + self.k_img = nn.Linear(dim, dim) + self.v_img = nn.Linear(dim, dim) + # self.alpha = nn.Parameter(torch.zeros((1, ))) + self.norm_k_img = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity() + + def forward(self, x, context, context_lens, dtype=torch.bfloat16, t=0): + r""" + Args: + x(Tensor): Shape [B, L1, C] + context(Tensor): Shape [B, L2, C] + context_lens(Tensor): Shape [B] + """ + context_img = context[:, :257] + context = context[:, 257:] + b, n, d = x.size(0), self.num_heads, self.head_dim + + # compute query, key, value + q = self.norm_q(self.q(x.to(dtype))).view(b, -1, n, d) + k = self.norm_k(self.k(context.to(dtype))).view(b, -1, n, d) + v = self.v(context.to(dtype)).view(b, -1, n, d) + k_img = self.norm_k_img(self.k_img(context_img.to(dtype))).view(b, -1, n, d) + v_img = self.v_img(context_img.to(dtype)).view(b, -1, n, d) + + img_x = attention( + q.to(dtype), + k_img.to(dtype), + v_img.to(dtype), + k_lens=None + ) + img_x = img_x.to(dtype) + # compute attention + x = attention( + q.to(dtype), + k.to(dtype), + v.to(dtype), + k_lens=context_lens + ) + x = x.to(dtype) + + # output + x = x.flatten(2) + img_x = img_x.flatten(2) + x = x + img_x + x = self.o(x) + return x + + +class WanCrossAttention(WanSelfAttention): + def forward(self, x, context, context_lens, dtype=torch.bfloat16, t=0): + r""" + Args: + x(Tensor): Shape [B, L1, C] + context(Tensor): Shape [B, L2, C] + context_lens(Tensor): Shape [B] + """ + b, n, d = x.size(0), self.num_heads, self.head_dim + # compute query, key, value + q = self.norm_q(self.q(x.to(dtype))).view(b, -1, n, d) + k = self.norm_k(self.k(context.to(dtype))).view(b, -1, n, d) + v = self.v(context.to(dtype)).view(b, -1, n, d) + # compute attention + x = attention(q.to(dtype), k.to(dtype), v.to(dtype), k_lens=context_lens) + # output + x = x.flatten(2) + x = self.o(x.to(dtype)) + return x + + +WAN_CROSSATTENTION_CLASSES = { + 't2v_cross_attn': WanT2VCrossAttention, + 'i2v_cross_attn': WanI2VCrossAttention, + 'cross_attn': WanCrossAttention, +} + + +class WanAttentionBlock(nn.Module): + + def __init__(self, + cross_attn_type, + dim, + ffn_dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=False, + eps=1e-6): + super().__init__() + self.dim = dim + self.ffn_dim = ffn_dim + self.num_heads = num_heads + self.window_size = window_size + self.qk_norm = qk_norm + self.cross_attn_norm = cross_attn_norm + self.eps = eps + + # layers + self.norm1 = WanLayerNorm(dim, eps) + self.self_attn = WanSelfAttention(dim, num_heads, window_size, qk_norm, + eps) + self.norm3 = WanLayerNorm( + dim, eps, + elementwise_affine=True) if cross_attn_norm else nn.Identity() + self.cross_attn = WAN_CROSSATTENTION_CLASSES[cross_attn_type](dim, + num_heads, + (-1, -1), + qk_norm, + eps) + self.norm2 = WanLayerNorm(dim, eps) + self.ffn = nn.Sequential( + nn.Linear(dim, ffn_dim), nn.GELU(approximate='tanh'), + nn.Linear(ffn_dim, dim)) + + # modulation + self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + + def forward( + self, + x, + e, + seq_lens, + grid_sizes, + freqs, + context, + context_lens, + dtype=torch.bfloat16, + t=0, + ): + r""" + Args: + x(Tensor): Shape [B, L, C] + e(Tensor): Shape [B, 6, C] + seq_lens(Tensor): Shape [B], length of each sequence in batch + grid_sizes(Tensor): Shape [B, 3], the second dimension contains (F, H, W) + freqs(Tensor): Rope freqs, shape [1024, C / num_heads / 2] + """ + if e.dim() > 3: + e = (self.modulation.unsqueeze(0) + e).chunk(6, dim=2) + e = [e.squeeze(2) for e in e] + else: + e = (self.modulation + e).chunk(6, dim=1) + + # self-attention + temp_x = self.norm1(x) * (1 + e[1]) + e[0] + temp_x = temp_x.to(dtype) + + y = self.self_attn(temp_x, seq_lens, grid_sizes, freqs, dtype, t=t) + x = x + y * e[2] + + # cross-attention & ffn function + def cross_attn_ffn(x, context, context_lens, e): + # cross-attention + x = x + self.cross_attn(self.norm3(x), context, context_lens, dtype, t=t) + + # ffn function + temp_x = self.norm2(x) * (1 + e[4]) + e[3] + temp_x = temp_x.to(dtype) + + y = self.ffn(temp_x) + x = x + y * e[5] + return x + + x = cross_attn_ffn(x, context, context_lens, e) + return x + + +class Head(nn.Module): + + def __init__(self, dim, out_dim, patch_size, eps=1e-6): + super().__init__() + self.dim = dim + self.out_dim = out_dim + self.patch_size = patch_size + self.eps = eps + + # layers + out_dim = math.prod(patch_size) * out_dim + self.norm = WanLayerNorm(dim, eps) + self.head = nn.Linear(dim, out_dim) + + # modulation + self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5) + + def forward(self, x, e): + r""" + Args: + x(Tensor): Shape [B, L1, C] + e(Tensor): Shape [B, C] + """ + if e.dim() > 2: + e = (self.modulation.unsqueeze(0) + e.unsqueeze(2)).chunk(2, dim=2) + e = [e.squeeze(2) for e in e] + else: + e = (self.modulation + e.unsqueeze(1)).chunk(2, dim=1) + + x = (self.head(self.norm(x) * (1 + e[1]) + e[0])) + return x + + +class MLPProj(torch.nn.Module): + + def __init__(self, in_dim, out_dim): + super().__init__() + + self.proj = torch.nn.Sequential( + torch.nn.LayerNorm(in_dim), torch.nn.Linear(in_dim, in_dim), + torch.nn.GELU(), torch.nn.Linear(in_dim, out_dim), + torch.nn.LayerNorm(out_dim)) + + def forward(self, image_embeds): + clip_extra_context_tokens = self.proj(image_embeds) + return clip_extra_context_tokens + + + +class WanTransformer3DModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): + r""" + Wan diffusion backbone supporting both text-to-video and image-to-video. + """ + + # ignore_for_config = [ + # 'patch_size', 'cross_attn_norm', 'qk_norm', 'text_dim', 'window_size' + # ] + # _no_split_modules = ['WanAttentionBlock'] + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + model_type='t2v', + patch_size=(1, 2, 2), + text_len=512, + in_dim=16, + dim=2048, + ffn_dim=8192, + freq_dim=256, + text_dim=4096, + out_dim=16, + num_heads=16, + num_layers=32, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=True, + eps=1e-6, + in_channels=16, + hidden_size=2048, + add_control_adapter=False, + in_dim_control_adapter=24, + downscale_factor_control_adapter=8, + add_ref_conv=False, + in_dim_ref_conv=16, + cross_attn_type=None, + ): + r""" + Initialize the diffusion model backbone. + + Args: + model_type (`str`, *optional*, defaults to 't2v'): + Model variant - 't2v' (text-to-video) or 'i2v' (image-to-video) + patch_size (`tuple`, *optional*, defaults to (1, 2, 2)): + 3D patch dimensions for video embedding (t_patch, h_patch, w_patch) + text_len (`int`, *optional*, defaults to 512): + Fixed length for text embeddings + in_dim (`int`, *optional*, defaults to 16): + Input video channels (C_in) + dim (`int`, *optional*, defaults to 2048): + Hidden dimension of the transformer + ffn_dim (`int`, *optional*, defaults to 8192): + Intermediate dimension in feed-forward network + freq_dim (`int`, *optional*, defaults to 256): + Dimension for sinusoidal time embeddings + text_dim (`int`, *optional*, defaults to 4096): + Input dimension for text embeddings + out_dim (`int`, *optional*, defaults to 16): + Output video channels (C_out) + num_heads (`int`, *optional*, defaults to 16): + Number of attention heads + num_layers (`int`, *optional*, defaults to 32): + Number of transformer blocks + window_size (`tuple`, *optional*, defaults to (-1, -1)): + Window size for local attention (-1 indicates global attention) + qk_norm (`bool`, *optional*, defaults to True): + Enable query/key normalization + cross_attn_norm (`bool`, *optional*, defaults to False): + Enable cross-attention normalization + eps (`float`, *optional*, defaults to 1e-6): + Epsilon value for normalization layers + """ + + super().__init__() + + # assert model_type in ['t2v', 'i2v', 'ti2v'] + self.model_type = model_type + + self.patch_size = patch_size + self.text_len = text_len + self.in_dim = in_dim + self.dim = dim + self.ffn_dim = ffn_dim + self.freq_dim = freq_dim + self.text_dim = text_dim + self.out_dim = out_dim + self.num_heads = num_heads + self.num_layers = num_layers + self.window_size = window_size + self.qk_norm = qk_norm + self.cross_attn_norm = cross_attn_norm + self.eps = eps + + # embeddings + self.patch_embedding = nn.Conv3d( + in_dim, dim, kernel_size=patch_size, stride=patch_size) + self.text_embedding = nn.Sequential( + nn.Linear(text_dim, dim), nn.GELU(approximate='tanh'), + nn.Linear(dim, dim)) + + self.time_embedding = nn.Sequential( + nn.Linear(freq_dim, dim), nn.SiLU(), nn.Linear(dim, dim)) + self.time_projection = nn.Sequential(nn.SiLU(), nn.Linear(dim, dim * 6)) + + # blocks + if cross_attn_type is None: + cross_attn_type = 't2v_cross_attn' if model_type == 't2v' else 'i2v_cross_attn' + self.blocks = nn.ModuleList([ + WanAttentionBlock(cross_attn_type, dim, ffn_dim, num_heads, + window_size, qk_norm, cross_attn_norm, eps) + for _ in range(num_layers) + ]) + for layer_idx, block in enumerate(self.blocks): + block.self_attn.layer_idx = layer_idx + block.self_attn.num_layers = self.num_layers + + # head + self.head = Head(dim, out_dim, patch_size, eps) + + # buffers (don't use register_buffer otherwise dtype will be changed in to()) + assert (dim % num_heads) == 0 and (dim // num_heads) % 2 == 0 + d = dim // num_heads + self.d = d + self.dim = dim + self.freqs = torch.cat( + [ + rope_params(1024, d - 4 * (d // 6)), + rope_params(1024, 2 * (d // 6)), + rope_params(1024, 2 * (d // 6)) + ], + dim=1 + ) + + if model_type == 'i2v': + self.img_emb = MLPProj(1280, dim) + + if add_control_adapter: + self.control_adapter = SimpleAdapter(in_dim_control_adapter, dim, kernel_size=patch_size[1:], stride=patch_size[1:], downscale_factor=downscale_factor_control_adapter) + else: + self.control_adapter = None + + if add_ref_conv: + self.ref_conv = nn.Conv2d(in_dim_ref_conv, dim, kernel_size=patch_size[1:], stride=patch_size[1:]) + else: + self.ref_conv = None + + self.teacache = None + self.cfg_skip_ratio = None + self.current_steps = 0 + self.num_inference_steps = None + self.gradient_checkpointing = False + self.all_gather = None + self.sp_world_size = 1 + self.sp_world_rank = 0 + self.init_weights() + + def _set_gradient_checkpointing(self, *args, **kwargs): + if "value" in kwargs: + self.gradient_checkpointing = kwargs["value"] + if hasattr(self, "motioner") and hasattr(self.motioner, "gradient_checkpointing"): + self.motioner.gradient_checkpointing = kwargs["value"] + elif "enable" in kwargs: + self.gradient_checkpointing = kwargs["enable"] + if hasattr(self, "motioner") and hasattr(self.motioner, "gradient_checkpointing"): + self.motioner.gradient_checkpointing = kwargs["enable"] + else: + raise ValueError("Invalid set gradient checkpointing") + + def enable_teacache( + self, + coefficients, + num_steps: int, + rel_l1_thresh: float, + num_skip_start_steps: int = 0, + offload: bool = True, + ): + self.teacache = TeaCache( + coefficients, num_steps, rel_l1_thresh=rel_l1_thresh, num_skip_start_steps=num_skip_start_steps, offload=offload + ) + + def share_teacache( + self, + transformer = None, + ): + self.teacache = transformer.teacache + + def disable_teacache(self): + self.teacache = None + + def enable_cfg_skip(self, cfg_skip_ratio, num_steps): + if cfg_skip_ratio != 0: + self.cfg_skip_ratio = cfg_skip_ratio + self.current_steps = 0 + self.num_inference_steps = num_steps + else: + self.cfg_skip_ratio = None + self.current_steps = 0 + self.num_inference_steps = None + + def share_cfg_skip( + self, + transformer = None, + ): + self.cfg_skip_ratio = transformer.cfg_skip_ratio + self.current_steps = transformer.current_steps + self.num_inference_steps = transformer.num_inference_steps + + def disable_cfg_skip(self): + self.cfg_skip_ratio = None + self.current_steps = 0 + self.num_inference_steps = None + + def enable_riflex( + self, + k = 6, + L_test = 66, + L_test_scale = 4.886, + ): + device = self.freqs.device + self.freqs = torch.cat( + [ + get_1d_rotary_pos_embed_riflex(1024, self.d - 4 * (self.d // 6), use_real=False, k=k, L_test=L_test, L_test_scale=L_test_scale), + rope_params(1024, 2 * (self.d // 6)), + rope_params(1024, 2 * (self.d // 6)) + ], + dim=1 + ).to(device) + + def disable_riflex(self): + device = self.freqs.device + self.freqs = torch.cat( + [ + rope_params(1024, self.d - 4 * (self.d // 6)), + rope_params(1024, 2 * (self.d // 6)), + rope_params(1024, 2 * (self.d // 6)) + ], + dim=1 + ).to(device) + + def enable_multi_gpus_inference(self,): + self.sp_world_size = get_sequence_parallel_world_size() + self.sp_world_rank = get_sequence_parallel_rank() + self.all_gather = get_sp_group().all_gather + + # For normal model. + for block in self.blocks: + block.self_attn.forward = types.MethodType( + usp_attn_forward, block.self_attn) + + # For vace model. + if hasattr(self, 'vace_blocks'): + for block in self.vace_blocks: + block.self_attn.forward = types.MethodType( + usp_attn_forward, block.self_attn) + + @cfg_skip() + def forward( + self, + x, + t, + context, + seq_len, + clip_fea=None, + y=None, + y_camera=None, + full_ref=None, + subject_ref=None, + cond_flag=True, + ): + r""" + Forward pass through the diffusion model + + Args: + x (List[Tensor]): + List of input video tensors, each with shape [C_in, F, H, W] + t (Tensor): + Diffusion timesteps tensor of shape [B] + context (List[Tensor]): + List of text embeddings each with shape [L, C] + seq_len (`int`): + Maximum sequence length for positional encoding + clip_fea (Tensor, *optional*): + CLIP image features for image-to-video mode + y (List[Tensor], *optional*): + Conditional video inputs for image-to-video mode, same shape as x + cond_flag (`bool`, *optional*, defaults to True): + Flag to indicate whether to forward the condition input + + Returns: + List[Tensor]: + List of denoised video tensors with original input shapes [C_out, F, H / 8, W / 8] + """ + # Wan2.2 don't need a clip. + # if self.model_type == 'i2v': + # assert clip_fea is not None and y is not None + # params + device = self.patch_embedding.weight.device + dtype = x.dtype + if self.freqs.device != device and torch.device(type="meta") != device: + self.freqs = self.freqs.to(device) + + if y is not None: + x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)] + + # embeddings + x = [self.patch_embedding(u.unsqueeze(0)) for u in x] + # add control adapter + if self.control_adapter is not None and y_camera is not None: + y_camera = self.control_adapter(y_camera) + x = [u + v for u, v in zip(x, y_camera)] + + grid_sizes = torch.stack( + [torch.tensor(u.shape[2:], dtype=torch.long) for u in x]) + + x = [u.flatten(2).transpose(1, 2) for u in x] + if self.ref_conv is not None and full_ref is not None: + full_ref = self.ref_conv(full_ref).flatten(2).transpose(1, 2) + grid_sizes = torch.stack([torch.tensor([u[0] + 1, u[1], u[2]]) for u in grid_sizes]).to(grid_sizes.device) + seq_len += full_ref.size(1) + x = [torch.concat([_full_ref.unsqueeze(0), u], dim=1) for _full_ref, u in zip(full_ref, x)] + if t.dim() != 1 and t.size(1) < seq_len: + pad_size = seq_len - t.size(1) + last_elements = t[:, -1].unsqueeze(1) + padding = last_elements.repeat(1, pad_size) + t = torch.cat([padding, t], dim=1) + + if subject_ref is not None: + subject_ref_frames = subject_ref.size(2) + subject_ref = self.patch_embedding(subject_ref).flatten(2).transpose(1, 2) + grid_sizes = torch.stack([torch.tensor([u[0] + subject_ref_frames, u[1], u[2]]) for u in grid_sizes]).to(grid_sizes.device) + seq_len += subject_ref.size(1) + x = [torch.concat([u, _subject_ref.unsqueeze(0)], dim=1) for _subject_ref, u in zip(subject_ref, x)] + if t.dim() != 1 and t.size(1) < seq_len: + pad_size = seq_len - t.size(1) + last_elements = t[:, -1].unsqueeze(1) + padding = last_elements.repeat(1, pad_size) + t = torch.cat([t, padding], dim=1) + + seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long) + if self.sp_world_size > 1: + seq_len = int(math.ceil(seq_len / self.sp_world_size)) * self.sp_world_size + assert seq_lens.max() <= seq_len + x = torch.cat([ + torch.cat([u, u.new_zeros(1, seq_len - u.size(1), u.size(2))], + dim=1) for u in x + ]) + + # time embeddings + with amp.autocast(dtype=torch.float32): + if t.dim() != 1: + if t.size(1) < seq_len: + pad_size = seq_len - t.size(1) + last_elements = t[:, -1].unsqueeze(1) + padding = last_elements.repeat(1, pad_size) + t = torch.cat([t, padding], dim=1) + bt = t.size(0) + ft = t.flatten() + e = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, + ft).unflatten(0, (bt, seq_len)).float()) + e0 = self.time_projection(e).unflatten(2, (6, self.dim)) + else: + e = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, t).float()) + e0 = self.time_projection(e).unflatten(1, (6, self.dim)) + + # assert e.dtype == torch.float32 and e0.dtype == torch.float32 + # e0 = e0.to(dtype) + # e = e.to(dtype) + + # context + context_lens = None + context = self.text_embedding( + torch.stack([ + torch.cat( + [u, u.new_zeros(self.text_len - u.size(0), u.size(1))]) + for u in context + ])) + + if clip_fea is not None: + context_clip = self.img_emb(clip_fea) # bs x 257 x dim + context = torch.concat([context_clip, context], dim=1) + + # Context Parallel + if self.sp_world_size > 1: + x = torch.chunk(x, self.sp_world_size, dim=1)[self.sp_world_rank] + if t.dim() != 1: + e0 = torch.chunk(e0, self.sp_world_size, dim=1)[self.sp_world_rank] + e = torch.chunk(e, self.sp_world_size, dim=1)[self.sp_world_rank] + + # TeaCache + if self.teacache is not None: + if cond_flag: + if t.dim() != 1: + modulated_inp = e0[:, -1, :] + else: + modulated_inp = e0 + skip_flag = self.teacache.cnt < self.teacache.num_skip_start_steps + if skip_flag: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + else: + if cond_flag: + rel_l1_distance = self.teacache.compute_rel_l1_distance(self.teacache.previous_modulated_input, modulated_inp) + self.teacache.accumulated_rel_l1_distance += self.teacache.rescale_func(rel_l1_distance) + if self.teacache.accumulated_rel_l1_distance < self.teacache.rel_l1_thresh: + self.should_calc = False + else: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + self.teacache.previous_modulated_input = modulated_inp + self.teacache.should_calc = self.should_calc + else: + self.should_calc = self.teacache.should_calc + + # TeaCache + if self.teacache is not None: + if not self.should_calc: + previous_residual = self.teacache.previous_residual_cond if cond_flag else self.teacache.previous_residual_uncond + x = x + previous_residual.to(x.device)[-x.size()[0]:,] + else: + ori_x = x.clone().cpu() if self.teacache.offload else x.clone() + + for block in self.blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + x, + e0, + seq_lens, + grid_sizes, + self.freqs, + context, + context_lens, + dtype, + t, + **ckpt_kwargs, + ) + else: + # arguments + kwargs = dict( + e=e0, + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=self.freqs, + context=context, + context_lens=context_lens, + dtype=dtype, + t=t + ) + x = block(x, **kwargs) + + if cond_flag: + self.teacache.previous_residual_cond = x.cpu() - ori_x if self.teacache.offload else x - ori_x + else: + self.teacache.previous_residual_uncond = x.cpu() - ori_x if self.teacache.offload else x - ori_x + else: + for block in self.blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + x, + e0, + seq_lens, + grid_sizes, + self.freqs, + context, + context_lens, + dtype, + t, + **ckpt_kwargs, + ) + else: + # arguments + kwargs = dict( + e=e0, + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=self.freqs, + context=context, + context_lens=context_lens, + dtype=dtype, + t=t + ) + x = block(x, **kwargs) + + # head + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.head), x, e, **ckpt_kwargs) + else: + x = self.head(x, e) + + if self.sp_world_size > 1: + x = self.all_gather(x, dim=1) + + if self.ref_conv is not None and full_ref is not None: + full_ref_length = full_ref.size(1) + x = x[:, full_ref_length:] + grid_sizes = torch.stack([torch.tensor([u[0] - 1, u[1], u[2]]) for u in grid_sizes]).to(grid_sizes.device) + + if subject_ref is not None: + subject_ref_length = subject_ref.size(1) + x = x[:, :-subject_ref_length] + grid_sizes = torch.stack([torch.tensor([u[0] - subject_ref_frames, u[1], u[2]]) for u in grid_sizes]).to(grid_sizes.device) + + # unpatchify + x = self.unpatchify(x, grid_sizes) + x = torch.stack(x) + if self.teacache is not None and cond_flag: + self.teacache.cnt += 1 + if self.teacache.cnt == self.teacache.num_steps: + self.teacache.reset() + return x + + + def unpatchify(self, x, grid_sizes): + r""" + Reconstruct video tensors from patch embeddings. + + Args: + x (List[Tensor]): + List of patchified features, each with shape [L, C_out * prod(patch_size)] + grid_sizes (Tensor): + Original spatial-temporal grid dimensions before patching, + shape [B, 3] (3 dimensions correspond to F_patches, H_patches, W_patches) + + Returns: + List[Tensor]: + Reconstructed video tensors with shape [C_out, F, H / 8, W / 8] + """ + + c = self.out_dim + out = [] + for u, v in zip(x, grid_sizes.tolist()): + u = u[:math.prod(v)].view(*v, *self.patch_size, c) + u = torch.einsum('fhwpqrc->cfphqwr', u) + u = u.reshape(c, *[i * j for i, j in zip(v, self.patch_size)]) + out.append(u) + return out + + def init_weights(self): + r""" + Initialize model parameters using Xavier initialization. + """ + + # basic init + for m in self.modules(): + if isinstance(m, nn.Linear): + nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.zeros_(m.bias) + + # init embeddings + nn.init.xavier_uniform_(self.patch_embedding.weight.flatten(1)) + for m in self.text_embedding.modules(): + if isinstance(m, nn.Linear): + nn.init.normal_(m.weight, std=.02) + for m in self.time_embedding.modules(): + if isinstance(m, nn.Linear): + nn.init.normal_(m.weight, std=.02) + + # init output layer + nn.init.zeros_(self.head.head.weight) + + @classmethod + def from_pretrained( + cls, pretrained_model_path, subfolder=None, transformer_additional_kwargs={}, + low_cpu_mem_usage=False, torch_dtype=torch.bfloat16 + ): + if subfolder is not None: + pretrained_model_path = os.path.join(pretrained_model_path, subfolder) + print(f"loaded 3D transformer's pretrained weights from {pretrained_model_path} ...") + + config_file = os.path.join(pretrained_model_path, 'config.json') + if not os.path.isfile(config_file): + raise RuntimeError(f"{config_file} does not exist") + with open(config_file, "r") as f: + config = json.load(f) + + from diffusers.utils import WEIGHTS_NAME + model_file = os.path.join(pretrained_model_path, WEIGHTS_NAME) + model_file_safetensors = model_file.replace(".bin", ".safetensors") + + if "dict_mapping" in transformer_additional_kwargs.keys(): + for key in transformer_additional_kwargs["dict_mapping"]: + transformer_additional_kwargs[transformer_additional_kwargs["dict_mapping"][key]] = config[key] + + if low_cpu_mem_usage: + try: + import re + + from diffusers import __version__ as diffusers_version + if diffusers_version >= "0.33.0": + from diffusers.models.model_loading_utils import \ + load_model_dict_into_meta + else: + from diffusers.models.modeling_utils import \ + load_model_dict_into_meta + from diffusers.utils import is_accelerate_available + if is_accelerate_available(): + import accelerate + + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + model = cls.from_config(config, **transformer_additional_kwargs) + + param_device = "cpu" + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + print(model_files_safetensors) + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + if model.state_dict()['patch_embedding.weight'].size() != state_dict['patch_embedding.weight'].size(): + model.state_dict()['patch_embedding.weight'][:, :state_dict['patch_embedding.weight'].size()[1], :, :] = state_dict['patch_embedding.weight'][:, :model.state_dict()['patch_embedding.weight'].size()[1], :, :] + model.state_dict()['patch_embedding.weight'][:, state_dict['patch_embedding.weight'].size()[1]:, :, :] = 0 + state_dict['patch_embedding.weight'] = model.state_dict()['patch_embedding.weight'] + + filtered_state_dict = {} + for key in state_dict: + if key in model.state_dict() and model.state_dict()[key].size() == state_dict[key].size(): + filtered_state_dict[key] = state_dict[key] + else: + print(f"Skipping key '{key}' due to size mismatch or absence in model.") + + model_keys = set(model.state_dict().keys()) + loaded_keys = set(filtered_state_dict.keys()) + missing_keys = model_keys - loaded_keys + + def initialize_missing_parameters(missing_keys, model_state_dict, torch_dtype=None): + initialized_dict = {} + + with torch.no_grad(): + for key in missing_keys: + param_shape = model_state_dict[key].shape + param_dtype = torch_dtype if torch_dtype is not None else model_state_dict[key].dtype + if 'weight' in key: + if any(norm_type in key for norm_type in ['norm', 'ln_', 'layer_norm', 'group_norm', 'batch_norm']): + initialized_dict[key] = torch.ones(param_shape, dtype=param_dtype) + elif 'embedding' in key or 'embed' in key: + initialized_dict[key] = torch.randn(param_shape, dtype=param_dtype) * 0.02 + elif 'head' in key or 'output' in key or 'proj_out' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif len(param_shape) >= 2: + initialized_dict[key] = torch.empty(param_shape, dtype=param_dtype) + nn.init.xavier_uniform_(initialized_dict[key]) + else: + initialized_dict[key] = torch.randn(param_shape, dtype=param_dtype) * 0.02 + elif 'bias' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif 'running_mean' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif 'running_var' in key: + initialized_dict[key] = torch.ones(param_shape, dtype=param_dtype) + elif 'num_batches_tracked' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=torch.long) + else: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + + return initialized_dict + + if missing_keys: + print(f"Missing keys will be initialized: {sorted(missing_keys)}") + initialized_params = initialize_missing_parameters( + missing_keys, + model.state_dict(), + torch_dtype + ) + filtered_state_dict.update(initialized_params) + + if diffusers_version >= "0.33.0": + # Diffusers has refactored `load_model_dict_into_meta` since version 0.33.0 in this commit: + # https://github.com/huggingface/diffusers/commit/f5929e03060d56063ff34b25a8308833bec7c785. + load_model_dict_into_meta( + model, + filtered_state_dict, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + else: + model._convert_deprecated_attention_blocks(filtered_state_dict) + unexpected_keys = load_model_dict_into_meta( + model, + filtered_state_dict, + device=param_device, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + + if cls._keys_to_ignore_on_load_unexpected is not None: + for pat in cls._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + print( + f"Some weights of the model checkpoint were not used when initializing {cls.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + + return model + except Exception as e: + print( + f"The low_cpu_mem_usage mode is not work because {e}. Use low_cpu_mem_usage=False instead." + ) + + model = cls.from_config(config, **transformer_additional_kwargs) + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + if model.state_dict()['patch_embedding.weight'].size() != state_dict['patch_embedding.weight'].size(): + model.state_dict()['patch_embedding.weight'][:, :state_dict['patch_embedding.weight'].size()[1], :, :] = state_dict['patch_embedding.weight'][:, :model.state_dict()['patch_embedding.weight'].size()[1], :, :] + model.state_dict()['patch_embedding.weight'][:, state_dict['patch_embedding.weight'].size()[1]:, :, :] = 0 + state_dict['patch_embedding.weight'] = model.state_dict()['patch_embedding.weight'] + + tmp_state_dict = {} + for key in state_dict: + if key in model.state_dict().keys() and model.state_dict()[key].size() == state_dict[key].size(): + tmp_state_dict[key] = state_dict[key] + else: + print(key, "Size don't match, skip") + + state_dict = tmp_state_dict + + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m) + + params = [p.numel() if "." in n else 0 for n, p in model.named_parameters()] + print(f"### All Parameters: {sum(params) / 1e6} M") + + params = [p.numel() if "attn1." in n else 0 for n, p in model.named_parameters()] + print(f"### attn1 Parameters: {sum(params) / 1e6} M") + + model = model.to(torch_dtype) + return model + + +class Wan2_2Transformer3DModel(WanTransformer3DModel): + r""" + Wan diffusion backbone supporting both text-to-video and image-to-video. + """ + + # ignore_for_config = [ + # 'patch_size', 'cross_attn_norm', 'qk_norm', 'text_dim', 'window_size' + # ] + # _no_split_modules = ['WanAttentionBlock'] + _supports_gradient_checkpointing = True + + def __init__( + self, + model_type='t2v', + patch_size=(1, 2, 2), + text_len=512, + in_dim=16, + dim=2048, + ffn_dim=8192, + freq_dim=256, + text_dim=4096, + out_dim=16, + num_heads=16, + num_layers=32, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=True, + eps=1e-6, + in_channels=16, + hidden_size=2048, + add_control_adapter=False, + in_dim_control_adapter=24, + downscale_factor_control_adapter=8, + add_ref_conv=False, + in_dim_ref_conv=16, + ): + r""" + Initialize the diffusion model backbone. + Args: + model_type (`str`, *optional*, defaults to 't2v'): + Model variant - 't2v' (text-to-video) or 'i2v' (image-to-video) + patch_size (`tuple`, *optional*, defaults to (1, 2, 2)): + 3D patch dimensions for video embedding (t_patch, h_patch, w_patch) + text_len (`int`, *optional*, defaults to 512): + Fixed length for text embeddings + in_dim (`int`, *optional*, defaults to 16): + Input video channels (C_in) + dim (`int`, *optional*, defaults to 2048): + Hidden dimension of the transformer + ffn_dim (`int`, *optional*, defaults to 8192): + Intermediate dimension in feed-forward network + freq_dim (`int`, *optional*, defaults to 256): + Dimension for sinusoidal time embeddings + text_dim (`int`, *optional*, defaults to 4096): + Input dimension for text embeddings + out_dim (`int`, *optional*, defaults to 16): + Output video channels (C_out) + num_heads (`int`, *optional*, defaults to 16): + Number of attention heads + num_layers (`int`, *optional*, defaults to 32): + Number of transformer blocks + window_size (`tuple`, *optional*, defaults to (-1, -1)): + Window size for local attention (-1 indicates global attention) + qk_norm (`bool`, *optional*, defaults to True): + Enable query/key normalization + cross_attn_norm (`bool`, *optional*, defaults to False): + Enable cross-attention normalization + eps (`float`, *optional*, defaults to 1e-6): + Epsilon value for normalization layers + """ + super().__init__( + model_type=model_type, + patch_size=patch_size, + text_len=text_len, + in_dim=in_dim, + dim=dim, + ffn_dim=ffn_dim, + freq_dim=freq_dim, + text_dim=text_dim, + out_dim=out_dim, + num_heads=num_heads, + num_layers=num_layers, + window_size=window_size, + qk_norm=qk_norm, + cross_attn_norm=cross_attn_norm, + eps=eps, + in_channels=in_channels, + hidden_size=hidden_size, + add_control_adapter=add_control_adapter, + in_dim_control_adapter=in_dim_control_adapter, + downscale_factor_control_adapter=downscale_factor_control_adapter, + add_ref_conv=add_ref_conv, + in_dim_ref_conv=in_dim_ref_conv, + cross_attn_type="cross_attn" + ) + + if hasattr(self, "img_emb"): + del self.img_emb diff --git a/VideoX-Fun/videox_fun/models/wan_transformer3d_animate.py b/VideoX-Fun/videox_fun/models/wan_transformer3d_animate.py new file mode 100644 index 0000000000000000000000000000000000000000..227a3bd5bf11444ee5cf6c584f0be37f0b8f3e31 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_transformer3d_animate.py @@ -0,0 +1,302 @@ +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +import math +import types +from copy import deepcopy +from typing import List + +import numpy as np +import torch +import torch.cuda.amp as amp +import torch.nn as nn +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import PeftAdapterMixin +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils import is_torch_version, logging +from einops import rearrange + +from .attention_utils import attention +from .wan_animate_adapter import FaceAdapter, FaceEncoder +from .wan_animate_motion_encoder import Generator +from .wan_transformer3d import (Head, MLPProj, WanAttentionBlock, WanLayerNorm, + WanRMSNorm, WanSelfAttention, + WanTransformer3DModel, rope_apply, + sinusoidal_embedding_1d) +from ..utils import cfg_skip + + +class Wan2_2Transformer3DModel_Animate(WanTransformer3DModel): + # _no_split_modules = ['WanAnimateAttentionBlock'] + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + patch_size=(1, 2, 2), + text_len=512, + in_dim=36, + dim=5120, + ffn_dim=13824, + freq_dim=256, + text_dim=4096, + out_dim=16, + num_heads=40, + num_layers=40, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=True, + eps=1e-6, + motion_encoder_dim=512, + use_img_emb=True + ): + model_type = "i2v" # TODO: Hard code for both preview and official versions. + super().__init__(model_type, patch_size, text_len, in_dim, dim, ffn_dim, freq_dim, text_dim, out_dim, + num_heads, num_layers, window_size, qk_norm, cross_attn_norm, eps) + + self.motion_encoder_dim = motion_encoder_dim + self.use_img_emb = use_img_emb + + self.pose_patch_embedding = nn.Conv3d( + 16, dim, kernel_size=patch_size, stride=patch_size + ) + + # initialize weights + self.init_weights() + + self.motion_encoder = Generator(size=512, style_dim=512, motion_dim=20) + self.face_adapter = FaceAdapter( + heads_num=self.num_heads, + hidden_dim=self.dim, + num_adapter_layers=self.num_layers // 5, + ) + + self.face_encoder = FaceEncoder( + in_dim=motion_encoder_dim, + hidden_dim=self.dim, + num_heads=4, + ) + + def after_patch_embedding(self, x: List[torch.Tensor], pose_latents, face_pixel_values): + pose_latents = [self.pose_patch_embedding(u.unsqueeze(0)) for u in pose_latents] + for x_, pose_latents_ in zip(x, pose_latents): + x_[:, :, 1:] += pose_latents_ + + b,c,T,h,w = face_pixel_values.shape + face_pixel_values = rearrange(face_pixel_values, "b c t h w -> (b t) c h w") + + encode_bs = 8 + face_pixel_values_tmp = [] + for i in range(math.ceil(face_pixel_values.shape[0]/encode_bs)): + face_pixel_values_tmp.append(self.motion_encoder.get_motion(face_pixel_values[i*encode_bs:(i+1)*encode_bs])) + + motion_vec = torch.cat(face_pixel_values_tmp) + + motion_vec = rearrange(motion_vec, "(b t) c -> b t c", t=T) + motion_vec = self.face_encoder(motion_vec) + + B, L, H, C = motion_vec.shape + pad_face = torch.zeros(B, 1, H, C).type_as(motion_vec) + motion_vec = torch.cat([pad_face, motion_vec], dim=1) + return x, motion_vec + + def after_transformer_block(self, block_idx, x, motion_vec, motion_masks=None): + if block_idx % 5 == 0: + use_context_parallel = self.sp_world_size > 1 + adapter_args = [x, motion_vec, motion_masks, use_context_parallel, self.all_gather, self.sp_world_size, self.sp_world_rank] + residual_out = self.face_adapter.fuser_blocks[block_idx // 5](*adapter_args) + x = residual_out + x + return x + + @cfg_skip() + def forward( + self, + x, + t, + clip_fea, + context, + seq_len, + y=None, + pose_latents=None, + face_pixel_values=None, + cond_flag=True + ): + # params + device = self.patch_embedding.weight.device + dtype = x.dtype + if self.freqs.device != device and torch.device(type="meta") != device: + self.freqs = self.freqs.to(device) + + if y is not None: + x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)] + + # embeddings + x = [self.patch_embedding(u.unsqueeze(0)) for u in x] + x, motion_vec = self.after_patch_embedding(x, pose_latents, face_pixel_values) + + grid_sizes = torch.stack( + [torch.tensor(u.shape[2:], dtype=torch.long) for u in x]) + x = [u.flatten(2).transpose(1, 2) for u in x] + seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long) + if self.sp_world_size > 1: + seq_len = int(math.ceil(seq_len / self.sp_world_size)) * self.sp_world_size + assert seq_lens.max() <= seq_len + x = torch.cat([ + torch.cat([u, u.new_zeros(1, seq_len - u.size(1), u.size(2))], + dim=1) for u in x + ]) + + # time embeddings + with amp.autocast(dtype=torch.float32): + e = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, t).float() + ) + e0 = self.time_projection(e).unflatten(1, (6, self.dim)) + assert e.dtype == torch.float32 and e0.dtype == torch.float32 + + # context + context_lens = None + context = self.text_embedding( + torch.stack([ + torch.cat( + [u, u.new_zeros(self.text_len - u.size(0), u.size(1))]) + for u in context + ])) + + if self.use_img_emb: + context_clip = self.img_emb(clip_fea) # bs x 257 x dim + context = torch.concat([context_clip, context], dim=1) + + # Context Parallel + if self.sp_world_size > 1: + x = torch.chunk(x, self.sp_world_size, dim=1)[self.sp_world_rank] + if t.dim() != 1: + e0 = torch.chunk(e0, self.sp_world_size, dim=1)[self.sp_world_rank] + e = torch.chunk(e, self.sp_world_size, dim=1)[self.sp_world_rank] + + # TeaCache + if self.teacache is not None: + if cond_flag: + if t.dim() != 1: + modulated_inp = e0[0][:, -1, :] + else: + modulated_inp = e0[0] + skip_flag = self.teacache.cnt < self.teacache.num_skip_start_steps + if skip_flag: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + else: + if cond_flag: + rel_l1_distance = self.teacache.compute_rel_l1_distance(self.teacache.previous_modulated_input, modulated_inp) + self.teacache.accumulated_rel_l1_distance += self.teacache.rescale_func(rel_l1_distance) + if self.teacache.accumulated_rel_l1_distance < self.teacache.rel_l1_thresh: + self.should_calc = False + else: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + self.teacache.previous_modulated_input = modulated_inp + self.teacache.should_calc = self.should_calc + else: + self.should_calc = self.teacache.should_calc + + # TeaCache + if self.teacache is not None: + if not self.should_calc: + previous_residual = self.teacache.previous_residual_cond if cond_flag else self.teacache.previous_residual_uncond + x = x + previous_residual.to(x.device)[-x.size()[0]:,] + else: + ori_x = x.clone().cpu() if self.teacache.offload else x.clone() + for idx, block in enumerate(self.blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + x, + e0, + seq_lens, + grid_sizes, + self.freqs, + context, + context_lens, + dtype, + t, + **ckpt_kwargs, + ) + x, motion_vec = x.to(dtype), motion_vec.to(dtype) + x = self.after_transformer_block(idx, x, motion_vec) + else: + # arguments + kwargs = dict( + e=e0, + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=self.freqs, + context=context, + context_lens=context_lens, + dtype=dtype, + t=t + ) + x = block(x, **kwargs) + x, motion_vec = x.to(dtype), motion_vec.to(dtype) + x = self.after_transformer_block(idx, x, motion_vec) + + if cond_flag: + self.teacache.previous_residual_cond = x.cpu() - ori_x if self.teacache.offload else x - ori_x + else: + self.teacache.previous_residual_uncond = x.cpu() - ori_x if self.teacache.offload else x - ori_x + else: + for idx, block in enumerate(self.blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + x, + e0, + seq_lens, + grid_sizes, + self.freqs, + context, + context_lens, + dtype, + t, + **ckpt_kwargs, + ) + x, motion_vec = x.to(dtype), motion_vec.to(dtype) + x = self.after_transformer_block(idx, x, motion_vec) + else: + # arguments + kwargs = dict( + e=e0, + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=self.freqs, + context=context, + context_lens=context_lens, + dtype=dtype, + t=t + ) + x = block(x, **kwargs) + x, motion_vec = x.to(dtype), motion_vec.to(dtype) + x = self.after_transformer_block(idx, x, motion_vec) + + # head + x = self.head(x, e) + + # Context Parallel + if self.sp_world_size > 1: + x = self.all_gather(x.contiguous(), dim=1) + + # unpatchify + x = self.unpatchify(x, grid_sizes) + x = torch.stack(x) + return x \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_transformer3d_s2v.py b/VideoX-Fun/videox_fun/models/wan_transformer3d_s2v.py new file mode 100644 index 0000000000000000000000000000000000000000..e55f86a33f9788bf0e1cb75f33d508be5c645d66 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_transformer3d_s2v.py @@ -0,0 +1,932 @@ +# Modified from https://github.com/Wan-Video/Wan2.2/blob/main/wan/modules/s2v/model_s2v.py +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. + +import math +import types +from copy import deepcopy +from typing import Any, Dict + +import torch +import torch.cuda.amp as amp +import torch.nn as nn +from diffusers.configuration_utils import register_to_config +from diffusers.utils import is_torch_version +from einops import rearrange + +from ..dist import (get_sequence_parallel_rank, + get_sequence_parallel_world_size, get_sp_group, + usp_attn_s2v_forward) +from .attention_utils import attention +from .wan_audio_injector import (AudioInjector_WAN, CausalAudioEncoder, + FramePackMotioner, MotionerTransformers, + rope_precompute) +from .wan_transformer3d import (Wan2_2Transformer3DModel, WanAttentionBlock, + WanLayerNorm, WanSelfAttention, + sinusoidal_embedding_1d) +from ..utils import cfg_skip + + +def zero_module(module): + """ + Zero out the parameters of a module and return it. + """ + for p in module.parameters(): + p.detach().zero_() + return module + + +def torch_dfs(model: nn.Module, parent_name='root'): + module_names, modules = [], [] + current_name = parent_name if parent_name else 'root' + module_names.append(current_name) + modules.append(model) + + for name, child in model.named_children(): + if parent_name: + child_name = f'{parent_name}.{name}' + else: + child_name = name + child_modules, child_names = torch_dfs(child, child_name) + module_names += child_names + modules += child_modules + return modules, module_names + + +@amp.autocast(enabled=False) +@torch.compiler.disable() +def s2v_rope_apply(x, grid_sizes, freqs, start=None): + n, c = x.size(2), x.size(3) // 2 + # loop over samples + output = [] + for i, _ in enumerate(x): + s = x.size(1) + x_i = torch.view_as_complex(x[i, :s].to(torch.float64).reshape(s, n, -1, 2)) + freqs_i = freqs[i, :s] + # apply rotary embedding + x_i = torch.view_as_real(x_i * freqs_i).flatten(2) + x_i = torch.cat([x_i, x[i, s:]]) + # append to collection + output.append(x_i) + return torch.stack(output).float() + + +def s2v_rope_apply_qk(q, k, grid_sizes, freqs): + q = s2v_rope_apply(q, grid_sizes, freqs) + k = s2v_rope_apply(k, grid_sizes, freqs) + return q, k + + +class WanS2VSelfAttention(WanSelfAttention): + + def forward(self, x, seq_lens, grid_sizes, freqs, dtype=torch.bfloat16, t=0): + """ + Args: + x(Tensor): Shape [B, L, num_heads, C / num_heads] + seq_lens(Tensor): Shape [B] + grid_sizes(Tensor): Shape [B, 3], the second dimension contains (F, H, W) + freqs(Tensor): Rope freqs, shape [1024, C / num_heads / 2] + """ + b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim + + # query, key, value function + def qkv_fn(x): + q = self.norm_q(self.q(x)).view(b, s, n, d) + k = self.norm_k(self.k(x)).view(b, s, n, d) + v = self.v(x).view(b, s, n, d) + return q, k, v + + q, k, v = qkv_fn(x) + + q, k = s2v_rope_apply_qk(q, k, grid_sizes, freqs) + + x = attention( + q.to(dtype), + k.to(dtype), + v=v.to(dtype), + k_lens=seq_lens, + window_size=self.window_size) + x = x.to(dtype) + + # output + x = x.flatten(2) + x = self.o(x) + return x + + +class WanS2VAttentionBlock(WanAttentionBlock): + + def __init__(self, + cross_attn_type, + dim, + ffn_dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=False, + eps=1e-6): + super().__init__( + cross_attn_type, dim, ffn_dim, num_heads, window_size, qk_norm, cross_attn_norm, eps + ) + self.self_attn = WanS2VSelfAttention(dim, num_heads, window_size,qk_norm, eps) + + def forward(self, x, e, seq_lens, grid_sizes, freqs, context, context_lens, dtype=torch.bfloat16, t=0): + # e + seg_idx = e[1].item() + seg_idx = min(max(0, seg_idx), x.size(1)) + seg_idx = [0, seg_idx, x.size(1)] + e = e[0] + modulation = self.modulation.unsqueeze(2) + e = (modulation + e).chunk(6, dim=1) + e = [element.squeeze(1) for element in e] + + # norm + norm_x = self.norm1(x).float() + parts = [] + for i in range(2): + parts.append(norm_x[:, seg_idx[i]:seg_idx[i + 1]] * + (1 + e[1][:, i:i + 1]) + e[0][:, i:i + 1]) + norm_x = torch.cat(parts, dim=1) + # self-attention + y = self.self_attn(norm_x, seq_lens, grid_sizes, freqs) + with amp.autocast(dtype=torch.float32): + z = [] + for i in range(2): + z.append(y[:, seg_idx[i]:seg_idx[i + 1]] * e[2][:, i:i + 1]) + y = torch.cat(z, dim=1) + x = x + y + + # cross-attention & ffn function + def cross_attn_ffn(x, context, context_lens, e): + x = x + self.cross_attn(self.norm3(x), context, context_lens) + norm2_x = self.norm2(x).float() + parts = [] + for i in range(2): + parts.append(norm2_x[:, seg_idx[i]:seg_idx[i + 1]] * + (1 + e[4][:, i:i + 1]) + e[3][:, i:i + 1]) + norm2_x = torch.cat(parts, dim=1) + y = self.ffn(norm2_x) + with amp.autocast(dtype=torch.float32): + z = [] + for i in range(2): + z.append(y[:, seg_idx[i]:seg_idx[i + 1]] * e[5][:, i:i + 1]) + y = torch.cat(z, dim=1) + x = x + y + return x + + x = cross_attn_ffn(x, context, context_lens, e) + return x + + +class Wan2_2Transformer3DModel_S2V(Wan2_2Transformer3DModel): + # ignore_for_config = [ + # 'args', 'kwargs', 'patch_size', 'cross_attn_norm', 'qk_norm', + # 'text_dim', 'window_size' + # ] + # _no_split_modules = ['WanS2VAttentionBlock'] + + @register_to_config + def __init__( + self, + cond_dim=0, + audio_dim=5120, + num_audio_token=4, + enable_adain=False, + adain_mode="attn_norm", + audio_inject_layers=[0, 4, 8, 12, 16, 20, 24, 27], + zero_init=False, + zero_timestep=False, + enable_motioner=True, + add_last_motion=True, + enable_tsm=False, + trainable_token_pos_emb=False, + motion_token_num=1024, + enable_framepack=False, # Mutually exclusive with enable_motioner + framepack_drop_mode="drop", + model_type='s2v', + patch_size=(1, 2, 2), + text_len=512, + in_dim=16, + dim=2048, + ffn_dim=8192, + freq_dim=256, + text_dim=4096, + out_dim=16, + num_heads=16, + num_layers=32, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=True, + eps=1e-6, + in_channels=16, + hidden_size=2048, + *args, + **kwargs + ): + super().__init__( + model_type=model_type, + patch_size=patch_size, + text_len=text_len, + in_dim=in_dim, + dim=dim, + ffn_dim=ffn_dim, + freq_dim=freq_dim, + text_dim=text_dim, + out_dim=out_dim, + num_heads=num_heads, + num_layers=num_layers, + window_size=window_size, + qk_norm=qk_norm, + cross_attn_norm=cross_attn_norm, + eps=eps, + in_channels=in_channels, + hidden_size=hidden_size + ) + + assert model_type == 's2v' + self.enbale_adain = enable_adain + # Whether to assign 0 value timestep to ref/motion + self.adain_mode = adain_mode + self.zero_timestep = zero_timestep + self.enable_motioner = enable_motioner + self.add_last_motion = add_last_motion + self.enable_framepack = enable_framepack + + # Replace blocks + self.blocks = nn.ModuleList([ + WanS2VAttentionBlock("cross_attn", dim, ffn_dim, num_heads, window_size, qk_norm, + cross_attn_norm, eps) + for _ in range(num_layers) + ]) + + # init audio injector + all_modules, all_modules_names = torch_dfs(self.blocks, parent_name="root.transformer_blocks") + if cond_dim > 0: + self.cond_encoder = nn.Conv3d( + cond_dim, + self.dim, + kernel_size=self.patch_size, + stride=self.patch_size) + self.trainable_cond_mask = nn.Embedding(3, self.dim) + self.casual_audio_encoder = CausalAudioEncoder( + dim=audio_dim, + out_dim=self.dim, + num_token=num_audio_token, + need_global=enable_adain) + self.audio_injector = AudioInjector_WAN( + all_modules, + all_modules_names, + dim=self.dim, + num_heads=self.num_heads, + inject_layer=audio_inject_layers, + root_net=self, + enable_adain=enable_adain, + adain_dim=self.dim, + need_adain_ont=adain_mode != "attn_norm", + ) + + if zero_init: + self.zero_init_weights() + + # init motioner + if enable_motioner and enable_framepack: + raise ValueError( + "enable_motioner and enable_framepack are mutually exclusive, please set one of them to False" + ) + if enable_motioner: + motioner_dim = 2048 + self.motioner = MotionerTransformers( + patch_size=(2, 4, 4), + dim=motioner_dim, + ffn_dim=motioner_dim, + freq_dim=256, + out_dim=16, + num_heads=16, + num_layers=13, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=False, + eps=1e-6, + motion_token_num=motion_token_num, + enable_tsm=enable_tsm, + motion_stride=4, + expand_ratio=2, + trainable_token_pos_emb=trainable_token_pos_emb, + ) + self.zip_motion_out = torch.nn.Sequential( + WanLayerNorm(motioner_dim), + zero_module(nn.Linear(motioner_dim, self.dim))) + + self.trainable_token_pos_emb = trainable_token_pos_emb + if trainable_token_pos_emb: + d = self.dim // self.num_heads + x = torch.zeros([1, motion_token_num, self.num_heads, d]) + x[..., ::2] = 1 + + gride_sizes = [[ + torch.tensor([0, 0, 0]).unsqueeze(0).repeat(1, 1), + torch.tensor([ + 1, self.motioner.motion_side_len, + self.motioner.motion_side_len + ]).unsqueeze(0).repeat(1, 1), + torch.tensor([ + 1, self.motioner.motion_side_len, + self.motioner.motion_side_len + ]).unsqueeze(0).repeat(1, 1), + ]] + token_freqs = s2v_rope_apply(x, gride_sizes, self.freqs) + token_freqs = token_freqs[0, :, + 0].reshape(motion_token_num, -1, 2) + token_freqs = token_freqs * 0.01 + self.token_freqs = torch.nn.Parameter(token_freqs) + + if enable_framepack: + self.frame_packer = FramePackMotioner( + inner_dim=self.dim, + num_heads=self.num_heads, + zip_frame_buckets=[1, 2, 16], + drop_mode=framepack_drop_mode) + + def enable_multi_gpus_inference(self,): + self.sp_world_size = get_sequence_parallel_world_size() + self.sp_world_rank = get_sequence_parallel_rank() + self.all_gather = get_sp_group().all_gather + for block in self.blocks: + block.self_attn.forward = types.MethodType( + usp_attn_s2v_forward, block.self_attn) + + def process_motion(self, motion_latents, drop_motion_frames=False): + if drop_motion_frames or motion_latents[0].shape[1] == 0: + return [], [] + self.lat_motion_frames = motion_latents[0].shape[1] + mot = [self.patch_embedding(m.unsqueeze(0)) for m in motion_latents] + batch_size = len(mot) + + mot_remb = [] + flattern_mot = [] + for bs in range(batch_size): + height, width = mot[bs].shape[3], mot[bs].shape[4] + flat_mot = mot[bs].flatten(2).transpose(1, 2).contiguous() + motion_grid_sizes = [[ + torch.tensor([-self.lat_motion_frames, 0, + 0]).unsqueeze(0).repeat(1, 1), + torch.tensor([0, height, width]).unsqueeze(0).repeat(1, 1), + torch.tensor([self.lat_motion_frames, height, + width]).unsqueeze(0).repeat(1, 1) + ]] + motion_rope_emb = rope_precompute( + flat_mot.detach().view(1, flat_mot.shape[1], self.num_heads, + self.dim // self.num_heads), + motion_grid_sizes, + self.freqs, + start=None) + mot_remb.append(motion_rope_emb) + flattern_mot.append(flat_mot) + return flattern_mot, mot_remb + + def process_motion_frame_pack(self, + motion_latents, + drop_motion_frames=False, + add_last_motion=2): + flattern_mot, mot_remb = self.frame_packer(motion_latents, + add_last_motion) + if drop_motion_frames: + return [m[:, :0] for m in flattern_mot + ], [m[:, :0] for m in mot_remb] + else: + return flattern_mot, mot_remb + + def process_motion_transformer_motioner(self, + motion_latents, + drop_motion_frames=False, + add_last_motion=True): + batch_size, height, width = len( + motion_latents), motion_latents[0].shape[2] // self.patch_size[ + 1], motion_latents[0].shape[3] // self.patch_size[2] + + freqs = self.freqs + device = self.patch_embedding.weight.device + if freqs.device != device: + freqs = freqs.to(device) + if self.trainable_token_pos_emb: + with amp.autocast(dtype=torch.float64): + token_freqs = self.token_freqs.to(torch.float64) + token_freqs = token_freqs / token_freqs.norm( + dim=-1, keepdim=True) + freqs = [freqs, torch.view_as_complex(token_freqs)] + + if not drop_motion_frames and add_last_motion: + last_motion_latent = [u[:, -1:] for u in motion_latents] + last_mot = [ + self.patch_embedding(m.unsqueeze(0)) for m in last_motion_latent + ] + last_mot = [m.flatten(2).transpose(1, 2) for m in last_mot] + last_mot = torch.cat(last_mot) + gride_sizes = [[ + torch.tensor([-1, 0, 0]).unsqueeze(0).repeat(batch_size, 1), + torch.tensor([0, height, + width]).unsqueeze(0).repeat(batch_size, 1), + torch.tensor([1, height, + width]).unsqueeze(0).repeat(batch_size, 1) + ]] + else: + last_mot = torch.zeros([batch_size, 0, self.dim], + device=motion_latents[0].device, + dtype=motion_latents[0].dtype) + gride_sizes = [] + + zip_motion = self.motioner(motion_latents) + zip_motion = self.zip_motion_out(zip_motion) + if drop_motion_frames: + zip_motion = zip_motion * 0.0 + zip_motion_grid_sizes = [[ + torch.tensor([-1, 0, 0]).unsqueeze(0).repeat(batch_size, 1), + torch.tensor([ + 0, self.motioner.motion_side_len, self.motioner.motion_side_len + ]).unsqueeze(0).repeat(batch_size, 1), + torch.tensor( + [1 if not self.trainable_token_pos_emb else -1, height, + width]).unsqueeze(0).repeat(batch_size, 1), + ]] + + mot = torch.cat([last_mot, zip_motion], dim=1) + gride_sizes = gride_sizes + zip_motion_grid_sizes + + motion_rope_emb = rope_precompute( + mot.detach().view(batch_size, mot.shape[1], self.num_heads, + self.dim // self.num_heads), + gride_sizes, + freqs, + start=None) + return [m.unsqueeze(0) for m in mot + ], [r.unsqueeze(0) for r in motion_rope_emb] + + def inject_motion(self, + x, + seq_lens, + rope_embs, + mask_input, + motion_latents, + drop_motion_frames=False, + add_last_motion=True): + # Inject the motion frames token to the hidden states + if self.enable_motioner: + mot, mot_remb = self.process_motion_transformer_motioner( + motion_latents, + drop_motion_frames=drop_motion_frames, + add_last_motion=add_last_motion) + elif self.enable_framepack: + mot, mot_remb = self.process_motion_frame_pack( + motion_latents, + drop_motion_frames=drop_motion_frames, + add_last_motion=add_last_motion) + else: + mot, mot_remb = self.process_motion( + motion_latents, drop_motion_frames=drop_motion_frames) + + if len(mot) > 0: + x = [torch.cat([u, m], dim=1) for u, m in zip(x, mot)] + seq_lens = seq_lens + torch.tensor([r.size(1) for r in mot], + dtype=torch.long) + rope_embs = [ + torch.cat([u, m], dim=1) for u, m in zip(rope_embs, mot_remb) + ] + mask_input = [ + torch.cat([ + m, 2 * torch.ones([1, u.shape[1] - m.shape[1]], + device=m.device, + dtype=m.dtype) + ], + dim=1) for m, u in zip(mask_input, x) + ] + return x, seq_lens, rope_embs, mask_input + + def after_transformer_block(self, block_idx, hidden_states): + if block_idx in self.audio_injector.injected_block_id.keys(): + audio_attn_id = self.audio_injector.injected_block_id[block_idx] + audio_emb = self.merged_audio_emb # b f n c + num_frames = audio_emb.shape[1] + + if self.sp_world_size > 1: + hidden_states = self.all_gather(hidden_states, dim=1) + + input_hidden_states = hidden_states[:, :self.original_seq_len].clone() + input_hidden_states = rearrange( + input_hidden_states, "b (t n) c -> (b t) n c", t=num_frames) + + if self.enbale_adain and self.adain_mode == "attn_norm": + audio_emb_global = self.audio_emb_global + audio_emb_global = rearrange(audio_emb_global, + "b t n c -> (b t) n c") + adain_hidden_states = self.audio_injector.injector_adain_layers[audio_attn_id]( + input_hidden_states, temb=audio_emb_global[:, 0] + ) + attn_hidden_states = adain_hidden_states + else: + attn_hidden_states = self.audio_injector.injector_pre_norm_feat[audio_attn_id]( + input_hidden_states + ) + audio_emb = rearrange(audio_emb, "b t n c -> (b t) n c", t=num_frames) + attn_audio_emb = audio_emb + context_lens = torch.ones( + attn_hidden_states.shape[0], dtype=torch.long, device=attn_hidden_states.device + ) * attn_audio_emb.shape[1] + + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + residual_out = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.audio_injector.injector[audio_attn_id]), + attn_hidden_states, + attn_audio_emb, + context_lens, + **ckpt_kwargs + ) + else: + residual_out = self.audio_injector.injector[audio_attn_id]( + x=attn_hidden_states, + context=attn_audio_emb, + context_lens=context_lens) + residual_out = rearrange(residual_out, "(b t) n c -> b (t n) c", t=num_frames) + hidden_states[:, :self.original_seq_len] = hidden_states[:, :self.original_seq_len] + residual_out + + if self.sp_world_size > 1: + hidden_states = torch.chunk( + hidden_states, self.sp_world_size, dim=1)[self.sp_world_rank] + + return hidden_states + + @cfg_skip() + def forward( + self, + x, + t, + context, + seq_len, + ref_latents, + motion_latents, + cond_states, + audio_input=None, + motion_frames=[17, 5], + add_last_motion=2, + drop_motion_frames=False, + cond_flag=True, + *extra_args, + **extra_kwargs + ): + """ + x: A list of videos each with shape [C, T, H, W]. + t: [B]. + context: A list of text embeddings each with shape [L, C]. + seq_len: A list of video token lens, no need for this model. + ref_latents A list of reference image for each video with shape [C, 1, H, W]. + motion_latents A list of motion frames for each video with shape [C, T_m, H, W]. + cond_states A list of condition frames (i.e. pose) each with shape [C, T, H, W]. + audio_input The input audio embedding [B, num_wav2vec_layer, C_a, T_a]. + motion_frames The number of motion frames and motion latents frames encoded by vae, i.e. [17, 5] + add_last_motion For the motioner, if add_last_motion > 0, it means that the most recent frame (i.e., the last frame) will be added. + For frame packing, the behavior depends on the value of add_last_motion: + add_last_motion = 0: Only the farthest part of the latent (i.e., clean_latents_4x) is included. + add_last_motion = 1: Both clean_latents_2x and clean_latents_4x are included. + add_last_motion = 2: All motion-related latents are used. + drop_motion_frames Bool, whether drop the motion frames info + """ + device = self.patch_embedding.weight.device + dtype = x.dtype + if self.freqs.device != device and torch.device(type="meta") != device: + self.freqs = self.freqs.to(device) + add_last_motion = self.add_last_motion * add_last_motion + + # Embeddings + x = [self.patch_embedding(u.unsqueeze(0)) for u in x] + + if isinstance(motion_frames[0], list): + motion_frames_0 = motion_frames[0][0] + motion_frames_1 = motion_frames[0][1] + else: + motion_frames_0 = motion_frames[0] + motion_frames_1 = motion_frames[1] + # Audio process + audio_input = torch.cat([audio_input[..., 0:1].repeat(1, 1, 1, motion_frames_0), audio_input], dim=-1) + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + audio_emb_res = torch.utils.checkpoint.checkpoint(create_custom_forward(self.casual_audio_encoder), audio_input, **ckpt_kwargs) + else: + audio_emb_res = self.casual_audio_encoder(audio_input) + if self.enbale_adain: + audio_emb_global, audio_emb = audio_emb_res + self.audio_emb_global = audio_emb_global[:, motion_frames_1:].clone() + else: + audio_emb = audio_emb_res + self.merged_audio_emb = audio_emb[:, motion_frames_1:, :] + + # Cond states + cond = [self.cond_encoder(c.unsqueeze(0)) for c in cond_states] + x = [x_ + pose for x_, pose in zip(x, cond)] + + grid_sizes = torch.stack( + [torch.tensor(u.shape[2:], dtype=torch.long) for u in x]) + x = [u.flatten(2).transpose(1, 2) for u in x] + seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long) + + original_grid_sizes = deepcopy(grid_sizes) + grid_sizes = [[torch.zeros_like(grid_sizes), grid_sizes, grid_sizes]] + + # Ref latents + ref = [self.patch_embedding(r.unsqueeze(0)) for r in ref_latents] + batch_size = len(ref) + height, width = ref[0].shape[3], ref[0].shape[4] + ref = [r.flatten(2).transpose(1, 2) for r in ref] # r: 1 c f h w + x = [torch.cat([u, r], dim=1) for u, r in zip(x, ref)] + + self.original_seq_len = seq_lens[0] + seq_lens = seq_lens + torch.tensor([r.size(1) for r in ref], dtype=torch.long) + ref_grid_sizes = [ + [ + torch.tensor([30, 0, 0]).unsqueeze(0).repeat(batch_size, 1), # the start index + torch.tensor([31, height,width]).unsqueeze(0).repeat(batch_size, 1), # the end index + torch.tensor([1, height, width]).unsqueeze(0).repeat(batch_size, 1), + ] # the range + ] + grid_sizes = grid_sizes + ref_grid_sizes + + # Compute the rope embeddings for the input + x = torch.cat(x) + b, s, n, d = x.size(0), x.size(1), self.num_heads, self.dim // self.num_heads + self.pre_compute_freqs = rope_precompute( + x.detach().view(b, s, n, d), grid_sizes, self.freqs, start=None) + x = [u.unsqueeze(0) for u in x] + self.pre_compute_freqs = [u.unsqueeze(0) for u in self.pre_compute_freqs] + + # Inject Motion latents. + # Initialize masks to indicate noisy latent, ref latent, and motion latent. + # However, at this point, only the first two (noisy and ref latents) are marked; + # the marking of motion latent will be implemented inside `inject_motion`. + mask_input = [ + torch.zeros([1, u.shape[1]], dtype=torch.long, device=x[0].device) + for u in x + ] + for i in range(len(mask_input)): + mask_input[i][:, self.original_seq_len:] = 1 + + self.lat_motion_frames = motion_latents[0].shape[1] + x, seq_lens, self.pre_compute_freqs, mask_input = self.inject_motion( + x, + seq_lens, + self.pre_compute_freqs, + mask_input, + motion_latents, + drop_motion_frames=drop_motion_frames, + add_last_motion=add_last_motion) + x = torch.cat(x, dim=0) + self.pre_compute_freqs = torch.cat(self.pre_compute_freqs, dim=0) + mask_input = torch.cat(mask_input, dim=0) + + # Apply trainable_cond_mask + x = x + self.trainable_cond_mask(mask_input).to(x.dtype) + + seq_len = seq_lens.max() + if self.sp_world_size > 1: + seq_len = int(math.ceil(seq_len / self.sp_world_size)) * self.sp_world_size + assert seq_lens.max() <= seq_len + x = torch.cat([ + torch.cat([u.unsqueeze(0), u.new_zeros(1, seq_len - u.size(0), u.size(1))], + dim=1) for u in x + ]) + + # Time embeddings + if self.zero_timestep: + t = torch.cat([t, torch.zeros([1], dtype=t.dtype, device=t.device)]) + with amp.autocast(dtype=torch.float32): + e = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, t).float()) + e0 = self.time_projection(e).unflatten(1, (6, self.dim)) + assert e.dtype == torch.float32 and e0.dtype == torch.float32 + + if self.zero_timestep: + e = e[:-1] + zero_e0 = e0[-1:] + e0 = e0[:-1] + token_len = x.shape[1] + + e0 = torch.cat( + [ + e0.unsqueeze(2), + zero_e0.unsqueeze(2).repeat(e0.size(0), 1, 1, 1) + ], + dim=2 + ) + e0 = [e0, self.original_seq_len] + else: + e0 = e0.unsqueeze(2).repeat(1, 1, 2, 1) + e0 = [e0, 0] + + # context + context_lens = None + context = self.text_embedding( + torch.stack([ + torch.cat( + [u, u.new_zeros(self.text_len - u.size(0), u.size(1))]) + for u in context + ])) + + if self.sp_world_size > 1: + # Sharded tensors for long context attn + x = torch.chunk(x, self.sp_world_size, dim=1) + sq_size = [u.shape[1] for u in x] + sq_start_size = sum(sq_size[:self.sp_world_rank]) + x = x[self.sp_world_rank] + # Confirm the application range of the time embedding in e0[0] for each sequence: + # - For tokens before seg_id: apply e0[0][:, :, 0] + # - For tokens after seg_id: apply e0[0][:, :, 1] + sp_size = x.shape[1] + seg_idx = e0[1] - sq_start_size + e0[1] = seg_idx + + self.pre_compute_freqs = torch.chunk(self.pre_compute_freqs, self.sp_world_size, dim=1) + self.pre_compute_freqs = self.pre_compute_freqs[self.sp_world_rank] + + # TeaCache + if self.teacache is not None: + if cond_flag: + if t.dim() != 1: + modulated_inp = e0[0][:, -1, :] + else: + modulated_inp = e0[0] + skip_flag = self.teacache.cnt < self.teacache.num_skip_start_steps + if skip_flag: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + else: + if cond_flag: + rel_l1_distance = self.teacache.compute_rel_l1_distance(self.teacache.previous_modulated_input, modulated_inp) + self.teacache.accumulated_rel_l1_distance += self.teacache.rescale_func(rel_l1_distance) + if self.teacache.accumulated_rel_l1_distance < self.teacache.rel_l1_thresh: + self.should_calc = False + else: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + self.teacache.previous_modulated_input = modulated_inp + self.teacache.should_calc = self.should_calc + else: + self.should_calc = self.teacache.should_calc + + # TeaCache + if self.teacache is not None: + if not self.should_calc: + previous_residual = self.teacache.previous_residual_cond if cond_flag else self.teacache.previous_residual_uncond + x = x + previous_residual.to(x.device)[-x.size()[0]:,] + else: + ori_x = x.clone().cpu() if self.teacache.offload else x.clone() + + for idx, block in enumerate(self.blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + x, + e0, + seq_lens, + grid_sizes, + self.pre_compute_freqs, + context, + context_lens, + dtype, + t, + **ckpt_kwargs, + ) + x = self.after_transformer_block(idx, x) + else: + # arguments + kwargs = dict( + e=e0, + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=self.pre_compute_freqs, + context=context, + context_lens=context_lens, + dtype=dtype, + t=t + ) + x = block(x, **kwargs) + x = self.after_transformer_block(idx, x) + + if cond_flag: + self.teacache.previous_residual_cond = x.cpu() - ori_x if self.teacache.offload else x - ori_x + else: + self.teacache.previous_residual_uncond = x.cpu() - ori_x if self.teacache.offload else x - ori_x + else: + for idx, block in enumerate(self.blocks): + if torch.is_grad_enabled() and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + x, + e0, + seq_lens, + grid_sizes, + self.pre_compute_freqs, + context, + context_lens, + dtype, + t, + **ckpt_kwargs, + ) + x = self.after_transformer_block(idx, x) + else: + # arguments + kwargs = dict( + e=e0, + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=self.pre_compute_freqs, + context=context, + context_lens=context_lens, + dtype=dtype, + t=t + ) + x = block(x, **kwargs) + x = self.after_transformer_block(idx, x) + + # Context Parallel + if self.sp_world_size > 1: + x = self.all_gather(x.contiguous(), dim=1) + + # Unpatchify + x = x[:, :self.original_seq_len] + # head + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.head), x, e, **ckpt_kwargs) + else: + x = self.head(x, e) + x = self.unpatchify(x, original_grid_sizes) + x = torch.stack(x) + if self.teacache is not None and cond_flag: + self.teacache.cnt += 1 + if self.teacache.cnt == self.teacache.num_steps: + self.teacache.reset() + return x + + def unpatchify(self, x, grid_sizes): + """ + Reconstruct video tensors from patch embeddings. + + Args: + x (List[Tensor]): + List of patchified features, each with shape [L, C_out * prod(patch_size)] + grid_sizes (Tensor): + Original spatial-temporal grid dimensions before patching, + shape [B, 3] (3 dimensions correspond to F_patches, H_patches, W_patches) + + Returns: + List[Tensor]: + Reconstructed video tensors with shape [C_out, F, H / 8, W / 8] + """ + + c = self.out_dim + out = [] + for u, v in zip(x, grid_sizes.tolist()): + u = u[:math.prod(v)].view(*v, *self.patch_size, c) + u = torch.einsum('fhwpqrc->cfphqwr', u) + u = u.reshape(c, *[i * j for i, j in zip(v, self.patch_size)]) + out.append(u) + return out + + def zero_init_weights(self): + with torch.no_grad(): + self.trainable_cond_mask = zero_module(self.trainable_cond_mask) + if hasattr(self, "cond_encoder"): + self.cond_encoder = zero_module(self.cond_encoder) + + for i in range(self.audio_injector.injector.__len__()): + self.audio_injector.injector[i].o = zero_module( + self.audio_injector.injector[i].o) + if self.enbale_adain: + self.audio_injector.injector_adain_layers[i].linear = \ + zero_module(self.audio_injector.injector_adain_layers[i].linear) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_transformer3d_vace.py b/VideoX-Fun/videox_fun/models/wan_transformer3d_vace.py new file mode 100644 index 0000000000000000000000000000000000000000..afd89d1a63c46f215c69d0825be21123c3d6d4a4 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_transformer3d_vace.py @@ -0,0 +1,394 @@ +# Modified from https://github.com/ali-vilab/VACE/blob/main/vace/models/wan/wan_vace.py +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. +from typing import Any, Dict + +import os +import math +import torch +import torch.cuda.amp as amp +import torch.nn as nn +from diffusers.configuration_utils import register_to_config +from diffusers.utils import is_torch_version + +from .wan_transformer3d import (WanAttentionBlock, WanTransformer3DModel, + sinusoidal_embedding_1d) +from ..utils import cfg_skip + + +VIDEOX_OFFLOAD_VACE_LATENTS = os.environ.get("VIDEOX_OFFLOAD_VACE_LATENTS", False) + +class VaceWanAttentionBlock(WanAttentionBlock): + def __init__( + self, + cross_attn_type, + dim, + ffn_dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=False, + eps=1e-6, + block_id=0 + ): + super().__init__(cross_attn_type, dim, ffn_dim, num_heads, window_size, qk_norm, cross_attn_norm, eps) + self.block_id = block_id + if block_id == 0: + self.before_proj = nn.Linear(self.dim, self.dim) + nn.init.zeros_(self.before_proj.weight) + nn.init.zeros_(self.before_proj.bias) + self.after_proj = nn.Linear(self.dim, self.dim) + nn.init.zeros_(self.after_proj.weight) + nn.init.zeros_(self.after_proj.bias) + + def forward(self, c, x, **kwargs): + if self.block_id == 0: + c = self.before_proj(c) + x + all_c = [] + else: + all_c = list(torch.unbind(c)) + c = all_c.pop(-1) + + if VIDEOX_OFFLOAD_VACE_LATENTS: + c = c.to(x.device) + + c = super().forward(c, **kwargs) + c_skip = self.after_proj(c) + + if VIDEOX_OFFLOAD_VACE_LATENTS: + c_skip = c_skip.to("cpu") + c = c.to("cpu") + + all_c += [c_skip, c] + c = torch.stack(all_c) + return c + + +class BaseWanAttentionBlock(WanAttentionBlock): + def __init__( + self, + cross_attn_type, + dim, + ffn_dim, + num_heads, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=False, + eps=1e-6, + block_id=None + ): + super().__init__(cross_attn_type, dim, ffn_dim, num_heads, window_size, qk_norm, cross_attn_norm, eps) + self.block_id = block_id + + def forward(self, x, hints, context_scale=1.0, **kwargs): + x = super().forward(x, **kwargs) + if self.block_id is not None: + if VIDEOX_OFFLOAD_VACE_LATENTS: + x = x + hints[self.block_id].to(x.device) * context_scale + else: + x = x + hints[self.block_id] * context_scale + return x + + +class VaceWanTransformer3DModel(WanTransformer3DModel): + @register_to_config + def __init__(self, + vace_layers=None, + vace_in_dim=None, + model_type='t2v', + patch_size=(1, 2, 2), + text_len=512, + in_dim=16, + dim=2048, + ffn_dim=8192, + freq_dim=256, + text_dim=4096, + out_dim=16, + num_heads=16, + num_layers=32, + window_size=(-1, -1), + qk_norm=True, + cross_attn_norm=True, + eps=1e-6): + model_type = "t2v" # TODO: Hard code for both preview and official versions. + super().__init__(model_type, patch_size, text_len, in_dim, dim, ffn_dim, freq_dim, text_dim, out_dim, + num_heads, num_layers, window_size, qk_norm, cross_attn_norm, eps) + + self.vace_layers = [i for i in range(0, self.num_layers, 2)] if vace_layers is None else vace_layers + self.vace_in_dim = self.in_dim if vace_in_dim is None else vace_in_dim + + assert 0 in self.vace_layers + self.vace_layers_mapping = {i: n for n, i in enumerate(self.vace_layers)} + + # blocks + self.blocks = nn.ModuleList([ + BaseWanAttentionBlock('t2v_cross_attn', self.dim, self.ffn_dim, self.num_heads, self.window_size, self.qk_norm, + self.cross_attn_norm, self.eps, + block_id=self.vace_layers_mapping[i] if i in self.vace_layers else None) + for i in range(self.num_layers) + ]) + + # vace blocks + self.vace_blocks = nn.ModuleList([ + VaceWanAttentionBlock('t2v_cross_attn', self.dim, self.ffn_dim, self.num_heads, self.window_size, self.qk_norm, + self.cross_attn_norm, self.eps, block_id=i) + for i in self.vace_layers + ]) + + # vace patch embeddings + self.vace_patch_embedding = nn.Conv3d( + self.vace_in_dim, self.dim, kernel_size=self.patch_size, stride=self.patch_size + ) + + def forward_vace( + self, + x, + vace_context, + seq_len, + kwargs + ): + # embeddings + c = [self.vace_patch_embedding(u.unsqueeze(0)) for u in vace_context] + c = [u.flatten(2).transpose(1, 2) for u in c] + c = torch.cat([ + torch.cat([u, u.new_zeros(1, seq_len - u.size(1), u.size(2))], + dim=1) for u in c + ]) + # Context Parallel + if self.sp_world_size > 1: + c = torch.chunk(c, self.sp_world_size, dim=1)[self.sp_world_rank] + + # arguments + new_kwargs = dict(x=x) + new_kwargs.update(kwargs) + + for block in self.vace_blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module, **static_kwargs): + def custom_forward(*inputs): + return module(*inputs, **static_kwargs) + return custom_forward + ckpt_kwargs = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + c = torch.utils.checkpoint.checkpoint( + create_custom_forward(block, **new_kwargs), + c, + **ckpt_kwargs, + ) + else: + c = block(c, **new_kwargs) + hints = torch.unbind(c)[:-1] + return hints + + @cfg_skip() + def forward( + self, + x, + t, + vace_context, + context, + seq_len, + vace_context_scale=1.0, + clip_fea=None, + y=None, + cond_flag=True + ): + r""" + Forward pass through the diffusion model + + Args: + x (List[Tensor]): + List of input video tensors, each with shape [C_in, F, H, W] + t (Tensor): + Diffusion timesteps tensor of shape [B] + context (List[Tensor]): + List of text embeddings each with shape [L, C] + seq_len (`int`): + Maximum sequence length for positional encoding + clip_fea (Tensor, *optional*): + CLIP image features for image-to-video mode + y (List[Tensor], *optional*): + Conditional video inputs for image-to-video mode, same shape as x + + Returns: + List[Tensor]: + List of denoised video tensors with original input shapes [C_out, F, H / 8, W / 8] + """ + # if self.model_type == 'i2v': + # assert clip_fea is not None and y is not None + # params + device = self.patch_embedding.weight.device + dtype = x.dtype + if self.freqs.device != device and torch.device(type="meta") != device: + self.freqs = self.freqs.to(device) + + # if y is not None: + # x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)] + + # embeddings + x = [self.patch_embedding(u.unsqueeze(0)) for u in x] + grid_sizes = torch.stack( + [torch.tensor(u.shape[2:], dtype=torch.long) for u in x]) + x = [u.flatten(2).transpose(1, 2) for u in x] + seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long) + if self.sp_world_size > 1: + seq_len = int(math.ceil(seq_len / self.sp_world_size)) * self.sp_world_size + assert seq_lens.max() <= seq_len + x = torch.cat([ + torch.cat([u, u.new_zeros(1, seq_len - u.size(1), u.size(2))], + dim=1) for u in x + ]) + + # time embeddings + with amp.autocast(dtype=torch.float32): + e = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, t).float()) + e0 = self.time_projection(e).unflatten(1, (6, self.dim)) + assert e.dtype == torch.float32 and e0.dtype == torch.float32 + + # context + context_lens = None + context = self.text_embedding( + torch.stack([ + torch.cat( + [u, u.new_zeros(self.text_len - u.size(0), u.size(1))]) + for u in context + ])) + + # Context Parallel + if self.sp_world_size > 1: + x = torch.chunk(x, self.sp_world_size, dim=1)[self.sp_world_rank] + + # arguments + kwargs = dict( + e=e0, + seq_lens=seq_lens, + grid_sizes=grid_sizes, + freqs=self.freqs, + context=context, + context_lens=context_lens, + dtype=dtype, + t=t) + hints = self.forward_vace(x, vace_context, seq_len, kwargs) + + kwargs['hints'] = hints + kwargs['context_scale'] = vace_context_scale + + # TeaCache + if self.teacache is not None: + if cond_flag: + if t.dim() != 1: + modulated_inp = e0[:, -1, :] + else: + modulated_inp = e0 + skip_flag = self.teacache.cnt < self.teacache.num_skip_start_steps + if skip_flag: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + else: + if cond_flag: + rel_l1_distance = self.teacache.compute_rel_l1_distance(self.teacache.previous_modulated_input, modulated_inp) + self.teacache.accumulated_rel_l1_distance += self.teacache.rescale_func(rel_l1_distance) + if self.teacache.accumulated_rel_l1_distance < self.teacache.rel_l1_thresh: + self.should_calc = False + else: + self.should_calc = True + self.teacache.accumulated_rel_l1_distance = 0 + self.teacache.previous_modulated_input = modulated_inp + self.teacache.should_calc = self.should_calc + else: + self.should_calc = self.teacache.should_calc + + # TeaCache + if self.teacache is not None: + if not self.should_calc: + previous_residual = self.teacache.previous_residual_cond if cond_flag else self.teacache.previous_residual_uncond + x = x + previous_residual.to(x.device)[-x.size()[0]:,] + else: + ori_x = x.clone().cpu() if self.teacache.offload else x.clone() + + for block in self.blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module, **static_kwargs): + def custom_forward(*inputs): + return module(*inputs, **static_kwargs) + return custom_forward + extra_kwargs = { + 'e': e0, + 'seq_lens': seq_lens, + 'grid_sizes': grid_sizes, + 'freqs': self.freqs, + 'context': context, + 'context_lens': context_lens, + 'dtype': dtype, + 't': t, + } + + ckpt_kwargs = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block, **extra_kwargs), + x, + hints, + vace_context_scale, + **ckpt_kwargs, + ) + else: + x = block(x, **kwargs) + + if cond_flag: + self.teacache.previous_residual_cond = x.cpu() - ori_x if self.teacache.offload else x - ori_x + else: + self.teacache.previous_residual_uncond = x.cpu() - ori_x if self.teacache.offload else x - ori_x + else: + for block in self.blocks: + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module, **static_kwargs): + def custom_forward(*inputs): + return module(*inputs, **static_kwargs) + return custom_forward + extra_kwargs = { + 'e': e0, + 'seq_lens': seq_lens, + 'grid_sizes': grid_sizes, + 'freqs': self.freqs, + 'context': context, + 'context_lens': context_lens, + 'dtype': dtype, + 't': t, + } + + ckpt_kwargs = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block, **extra_kwargs), + x, + hints, + vace_context_scale, + **ckpt_kwargs, + ) + else: + x = block(x, **kwargs) + + # head + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.head), x, e, **ckpt_kwargs) + else: + x = self.head(x, e) + + if self.sp_world_size > 1: + x = self.all_gather(x, dim=1) + + # unpatchify + x = self.unpatchify(x, grid_sizes) + x = torch.stack(x) + if self.teacache is not None and cond_flag: + self.teacache.cnt += 1 + if self.teacache.cnt == self.teacache.num_steps: + self.teacache.reset() + return x \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_vae.py b/VideoX-Fun/videox_fun/models/wan_vae.py new file mode 100644 index 0000000000000000000000000000000000000000..7815e7b8582c102c80df4c54ccd064ff44fc88c5 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_vae.py @@ -0,0 +1,860 @@ +# Modified from https://github.com/Wan-Video/Wan2.1/blob/main/wan/modules/vae.py +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +from typing import Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.autoencoders.vae import (DecoderOutput, + DiagonalGaussianDistribution) +from diffusers.models.modeling_outputs import AutoencoderKLOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils.accelerate_utils import apply_forward_hook +from einops import rearrange + + +CACHE_T = 2 + + +class CausalConv3d(nn.Conv3d): + """ + Causal 3d convolusion. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._padding = (self.padding[2], self.padding[2], self.padding[1], + self.padding[1], 2 * self.padding[0], 0) + self.padding = (0, 0, 0) + + def forward(self, x, cache_x=None): + padding = list(self._padding) + if cache_x is not None and self._padding[4] > 0: + cache_x = cache_x.to(x.device) + x = torch.cat([cache_x, x], dim=2) + padding[4] -= cache_x.shape[2] + x = F.pad(x, padding) + + return super().forward(x) + + +class RMS_norm(nn.Module): + + def __init__(self, dim, channel_first=True, images=True, bias=False): + super().__init__() + broadcastable_dims = (1, 1, 1) if not images else (1, 1) + shape = (dim, *broadcastable_dims) if channel_first else (dim,) + + self.channel_first = channel_first + self.scale = dim**0.5 + self.gamma = nn.Parameter(torch.ones(shape)) + self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0. + + def forward(self, x): + return F.normalize( + x, dim=(1 if self.channel_first else + -1)) * self.scale * self.gamma + self.bias + + +class Upsample(nn.Upsample): + + def forward(self, x): + """ + Fix bfloat16 support for nearest neighbor interpolation. + """ + return super().forward(x.float()).type_as(x) + + +class Resample(nn.Module): + + def __init__(self, dim, mode): + assert mode in ('none', 'upsample2d', 'upsample3d', 'downsample2d', + 'downsample3d') + super().__init__() + self.dim = dim + self.mode = mode + + # layers + if mode == 'upsample2d': + self.resample = nn.Sequential( + Upsample(scale_factor=(2., 2.), mode='nearest-exact'), + nn.Conv2d(dim, dim // 2, 3, padding=1)) + elif mode == 'upsample3d': + self.resample = nn.Sequential( + Upsample(scale_factor=(2., 2.), mode='nearest-exact'), + nn.Conv2d(dim, dim // 2, 3, padding=1)) + self.time_conv = CausalConv3d( + dim, dim * 2, (3, 1, 1), padding=(1, 0, 0)) + + elif mode == 'downsample2d': + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), + nn.Conv2d(dim, dim, 3, stride=(2, 2))) + elif mode == 'downsample3d': + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), + nn.Conv2d(dim, dim, 3, stride=(2, 2))) + self.time_conv = CausalConv3d( + dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0)) + + else: + self.resample = nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + b, c, t, h, w = x.size() + if self.mode == 'upsample3d': + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = 'Rep' + feat_idx[0] += 1 + else: + + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[ + idx] is not None and feat_cache[idx] != 'Rep': + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + if cache_x.shape[2] < 2 and feat_cache[ + idx] is not None and feat_cache[idx] == 'Rep': + cache_x = torch.cat([ + torch.zeros_like(cache_x).to(cache_x.device), + cache_x + ], + dim=2) + if feat_cache[idx] == 'Rep': + x = self.time_conv(x) + else: + x = self.time_conv(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + + x = x.reshape(b, 2, c, t, h, w) + x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), + 3) + x = x.reshape(b, c, t * 2, h, w) + t = x.shape[2] + x = rearrange(x, 'b c t h w -> (b t) c h w') + x = self.resample(x) + x = rearrange(x, '(b t) c h w -> b c t h w', t=t) + + if self.mode == 'downsample3d': + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = x.clone() + feat_idx[0] += 1 + else: + + cache_x = x[:, :, -1:, :, :].clone() + # if cache_x.shape[2] < 2 and feat_cache[idx] is not None and feat_cache[idx]!='Rep': + # # cache last frame of last two chunk + # cache_x = torch.cat([feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x], dim=2) + + x = self.time_conv( + torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2)) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + return x + + def init_weight(self, conv): + conv_weight = conv.weight + nn.init.zeros_(conv_weight) + c1, c2, t, h, w = conv_weight.size() + one_matrix = torch.eye(c1, c2) + init_matrix = one_matrix + nn.init.zeros_(conv_weight) + #conv_weight.data[:,:,-1,1,1] = init_matrix * 0.5 + conv_weight.data[:, :, 1, 0, 0] = init_matrix #* 0.5 + conv.weight.data.copy_(conv_weight) + nn.init.zeros_(conv.bias.data) + + def init_weight2(self, conv): + conv_weight = conv.weight.data + nn.init.zeros_(conv_weight) + c1, c2, t, h, w = conv_weight.size() + init_matrix = torch.eye(c1 // 2, c2) + #init_matrix = repeat(init_matrix, 'o ... -> (o 2) ...').permute(1,0,2).contiguous().reshape(c1,c2) + conv_weight[:c1 // 2, :, -1, 0, 0] = init_matrix + conv_weight[c1 // 2:, :, -1, 0, 0] = init_matrix + conv.weight.data.copy_(conv_weight) + nn.init.zeros_(conv.bias.data) + + +class ResidualBlock(nn.Module): + + def __init__(self, in_dim, out_dim, dropout=0.0): + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + # layers + self.residual = nn.Sequential( + RMS_norm(in_dim, images=False), nn.SiLU(), + CausalConv3d(in_dim, out_dim, 3, padding=1), + RMS_norm(out_dim, images=False), nn.SiLU(), nn.Dropout(dropout), + CausalConv3d(out_dim, out_dim, 3, padding=1)) + self.shortcut = CausalConv3d(in_dim, out_dim, 1) \ + if in_dim != out_dim else nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + h = self.shortcut(x) + for layer in self.residual: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x + h + + +class AttentionBlock(nn.Module): + """ + Causal self-attention with a single head. + """ + + def __init__(self, dim): + super().__init__() + self.dim = dim + + # layers + self.norm = RMS_norm(dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x): + identity = x + b, c, t, h, w = x.size() + x = rearrange(x, 'b c t h w -> (b t) c h w') + x = self.norm(x) + # compute query, key, value + q, k, v = self.to_qkv(x).reshape(b * t, 1, c * 3, + -1).permute(0, 1, 3, + 2).contiguous().chunk( + 3, dim=-1) + + # apply attention + x = F.scaled_dot_product_attention( + q, + k, + v, + ) + x = x.squeeze(1).permute(0, 2, 1).reshape(b * t, c, h, w) + + # output + x = self.proj(x) + x = rearrange(x, '(b t) c h w-> b c t h w', t=t) + return x + identity + + +class Encoder3d(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[True, True, False], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + + # dimensions + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv1 = CausalConv3d(3, dims[0], 3, padding=1) + + # downsample blocks + downsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + for _ in range(num_res_blocks): + downsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + downsamples.append(AttentionBlock(out_dim)) + in_dim = out_dim + + # downsample block + if i != len(dim_mult) - 1: + mode = 'downsample3d' if temperal_downsample[ + i] else 'downsample2d' + downsamples.append(Resample(out_dim, mode=mode)) + scale /= 2.0 + self.downsamples = nn.Sequential(*downsamples) + + # middle blocks + self.middle = nn.Sequential( + ResidualBlock(out_dim, out_dim, dropout), AttentionBlock(out_dim), + ResidualBlock(out_dim, out_dim, dropout)) + + # output blocks + self.head = nn.Sequential( + RMS_norm(out_dim, images=False), nn.SiLU(), + CausalConv3d(out_dim, z_dim, 3, padding=1)) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + ## downsamples + for layer in self.downsamples: + if feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## middle + for layer in self.middle: + if isinstance(layer, ResidualBlock) and feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## head + for layer in self.head: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x + + +class Decoder3d(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_upsample=[False, True, True], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_upsample = temperal_upsample + + # dimensions + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + scale = 1.0 / 2**(len(dim_mult) - 2) + + # init block + self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.middle = nn.Sequential( + ResidualBlock(dims[0], dims[0], dropout), AttentionBlock(dims[0]), + ResidualBlock(dims[0], dims[0], dropout)) + + # upsample blocks + upsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + # residual (+attention) blocks + if i == 1 or i == 2 or i == 3: + in_dim = in_dim // 2 + for _ in range(num_res_blocks + 1): + upsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + if scale in attn_scales: + upsamples.append(AttentionBlock(out_dim)) + in_dim = out_dim + + # upsample block + if i != len(dim_mult) - 1: + mode = 'upsample3d' if temperal_upsample[i] else 'upsample2d' + upsamples.append(Resample(out_dim, mode=mode)) + scale *= 2.0 + self.upsamples = nn.Sequential(*upsamples) + + # output blocks + self.head = nn.Sequential( + RMS_norm(out_dim, images=False), nn.SiLU(), + CausalConv3d(out_dim, 3, 3, padding=1)) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + ## conv1 + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + ## middle + for layer in self.middle: + if isinstance(layer, ResidualBlock) and feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## upsamples + for layer in self.upsamples: + if feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## head + for layer in self.head: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat([ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), cache_x + ], + dim=2) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x + + +def count_conv3d(model): + count = 0 + for m in model.modules(): + if isinstance(m, CausalConv3d): + count += 1 + return count + + +class AutoencoderKLWan_(nn.Module): + + def __init__(self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[True, True, False], + dropout=0.0): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + self.temperal_upsample = temperal_downsample[::-1] + + # modules + self.encoder = Encoder3d(dim, z_dim * 2, dim_mult, num_res_blocks, + attn_scales, self.temperal_downsample, dropout) + self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1) + self.conv2 = CausalConv3d(z_dim, z_dim, 1) + self.decoder = Decoder3d(dim, z_dim, dim_mult, num_res_blocks, + attn_scales, self.temperal_upsample, dropout) + + def forward(self, x): + mu, log_var = self.encode(x) + z = self.reparameterize(mu, log_var) + x_recon = self.decode(z) + return x_recon, mu, log_var + + def encode(self, x, scale=None): + self.clear_cache() + ## cache + t = x.shape[2] + iter_ = 1 + (t - 1) // 4 + if scale != None: + scale = [item.to(x.device, x.dtype) for item in scale] + ## 对encode输入的x，按时间拆分为1、4、4、4.... + for i in range(iter_): + self._enc_conv_idx = [0] + if i == 0: + out = self.encoder( + x[:, :, :1, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx) + else: + out_ = self.encoder( + x[:, :, 1 + 4 * (i - 1):1 + 4 * i, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx) + out = torch.cat([out, out_], 2) + mu, log_var = self.conv1(out).chunk(2, dim=1) + if scale != None: + if isinstance(scale[0], torch.Tensor): + mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view( + 1, self.z_dim, 1, 1, 1) + else: + mu = (mu - scale[0]) * scale[1] + x = torch.cat([mu, log_var], dim = 1) + self.clear_cache() + return x + + def decode(self, z, scale=None): + self.clear_cache() + # z: [b,c,t,h,w] + if scale != None: + scale = [item.to(z.device, z.dtype) for item in scale] + if isinstance(scale[0], torch.Tensor): + z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view( + 1, self.z_dim, 1, 1, 1) + else: + z = z / scale[1] + scale[0] + iter_ = z.shape[2] + x = self.conv2(z) + for i in range(iter_): + self._conv_idx = [0] + if i == 0: + out = self.decoder( + x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx) + else: + out_ = self.decoder( + x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx) + out = torch.cat([out, out_], 2) + self.clear_cache() + return out + + def reparameterize(self, mu, log_var): + std = torch.exp(0.5 * log_var) + eps = torch.randn_like(std) + return eps * std + mu + + def sample(self, imgs, deterministic=False): + mu, log_var = self.encode(imgs) + if deterministic: + return mu + std = torch.exp(0.5 * log_var.clamp(-30.0, 20.0)) + return mu + std * torch.randn_like(std) + + def clear_cache(self): + self._conv_num = count_conv3d(self.decoder) + self._conv_idx = [0] + self._feat_map = [None] * self._conv_num + #cache encode + self._enc_conv_num = count_conv3d(self.encoder) + self._enc_conv_idx = [0] + self._enc_feat_map = [None] * self._enc_conv_num + + +def _video_vae(z_dim=None, **kwargs): + """ + Autoencoder3d adapted from Stable Diffusion 1.x, 2.x and XL. + """ + # params + cfg = dict( + dim=96, + z_dim=z_dim, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[False, True, True], + dropout=0.0) + cfg.update(**kwargs) + + # init model + model = AutoencoderKLWan_(**cfg) + + return model + + +class AutoencoderKLWan(ModelMixin, ConfigMixin, FromOriginalModelMixin): + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + latent_channels=16, + temporal_compression_ratio=4, + spatial_compression_ratio=8 + ): + super().__init__() + mean = [ + -0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508, + 0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921 + ] + std = [ + 2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743, + 3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160 + ] + self.mean = torch.tensor(mean, dtype=torch.float32) + self.std = torch.tensor(std, dtype=torch.float32) + self.scale = [self.mean, 1.0 / self.std] + + # init model + self.model = _video_vae( + z_dim=latent_channels, + ) + + self.gradient_checkpointing = False + + def _set_gradient_checkpointing(self, *args, **kwargs): + if "value" in kwargs: + self.gradient_checkpointing = kwargs["value"] + elif "enable" in kwargs: + self.gradient_checkpointing = kwargs["enable"] + else: + raise ValueError("Invalid set gradient checkpointing") + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + x = [ + self.model.encode(u.unsqueeze(0), self.scale).squeeze(0) + for u in x + ] + x = torch.stack(x) + return x + + @apply_forward_hook + def encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + h = self._encode(x) + + posterior = DiagonalGaussianDistribution(h) + + if not return_dict: + return (posterior,) + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode(self, zs): + dec = [ + self.model.decode(u.unsqueeze(0), self.scale).clamp_(-1, 1).squeeze(0) + for u in zs + ] + dec = torch.stack(dec) + + return DecoderOutput(sample=dec) + + @apply_forward_hook + def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + decoded = self._decode(z).sample + + if not return_dict: + return (decoded,) + return DecoderOutput(sample=decoded) + + @classmethod + def from_pretrained(cls, pretrained_model_path, additional_kwargs={}): + def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + + model = cls(**filter_kwargs(cls, additional_kwargs)) + if pretrained_model_path.endswith(".safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(pretrained_model_path) + else: + state_dict = torch.load(pretrained_model_path, map_location="cpu") + tmp_state_dict = {} + for key in state_dict: + tmp_state_dict["model." + key] = state_dict[key] + state_dict = tmp_state_dict + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m, u) + return model + + +class AutoencoderKLWanCompileQwenImage(ModelMixin, ConfigMixin, FromOriginalModelMixin): + @register_to_config + def __init__( + self, + attn_scales = [], + base_dim = 96, + dim_mult = [ + 1, + 2, + 4, + 4 + ], + dropout = 0.0, + latents_mean = [ + -0.7571, + -0.7089, + -0.9113, + 0.1075, + -0.1745, + 0.9653, + -0.1517, + 1.5508, + 0.4134, + -0.0715, + 0.5517, + -0.3632, + -0.1922, + -0.9497, + 0.2503, + -0.2921 + ], + latents_std = [ + 2.8184, + 1.4541, + 2.3275, + 2.6558, + 1.2196, + 1.7708, + 2.6052, + 2.0743, + 3.2687, + 2.1526, + 2.8652, + 1.5579, + 1.6382, + 1.1253, + 2.8251, + 1.916 + ], + num_res_blocks = 2, + temperal_downsample = [ + False, + True, + True + ], + z_dim = 16 + ): + super().__init__() + cfg = dict( + dim=base_dim, + z_dim=z_dim, + dim_mult=dim_mult, + num_res_blocks=num_res_blocks, + attn_scales=attn_scales, + temperal_downsample=temperal_downsample, + dropout=dropout) + + # init model + self.model = AutoencoderKLWan_(**cfg) + + self.dim = base_dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + self.temperal_upsample = temperal_downsample[::-1] + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + x = [ + self.model.encode(u.unsqueeze(0)).squeeze(0) + for u in x + ] + x = torch.stack(x) + return x + + @apply_forward_hook + def encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + h = self._encode(x) + + posterior = DiagonalGaussianDistribution(h) + + if not return_dict: + return (posterior,) + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode(self, zs): + dec = [ + self.model.decode(u.unsqueeze(0)).clamp_(-1, 1).squeeze(0) + for u in zs + ] + dec = torch.stack(dec) + + return DecoderOutput(sample=dec) + + @apply_forward_hook + def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + decoded = self._decode(z).sample + + if not return_dict: + return (decoded,) + return DecoderOutput(sample=decoded) + + @classmethod + def from_pretrained(cls, pretrained_model_path, additional_kwargs={}): + def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + + model = cls(**filter_kwargs(cls, additional_kwargs)) + if pretrained_model_path.endswith(".safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(pretrained_model_path) + else: + state_dict = torch.load(pretrained_model_path, map_location="cpu") + tmp_state_dict = {} + for key in state_dict: + tmp_state_dict["model." + key] = state_dict[key] + state_dict = tmp_state_dict + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m, u) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_vae3_8.py b/VideoX-Fun/videox_fun/models/wan_vae3_8.py new file mode 100644 index 0000000000000000000000000000000000000000..e7898d07f87c0f6a49ce5c4fa9a25b036c6b5926 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_vae3_8.py @@ -0,0 +1,1091 @@ +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +import logging +from typing import Tuple, Union + +import torch +import torch.cuda.amp as amp +import torch.nn as nn +import torch.nn.functional as F +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders.single_file_model import FromOriginalModelMixin +from diffusers.models.autoencoders.vae import (DecoderOutput, + DiagonalGaussianDistribution) +from diffusers.models.modeling_outputs import AutoencoderKLOutput +from diffusers.models.modeling_utils import ModelMixin +from diffusers.utils.accelerate_utils import apply_forward_hook +from einops import rearrange + + +CACHE_T = 2 + + +class CausalConv3d(nn.Conv3d): + """ + Causal 3d convolusion. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._padding = ( + self.padding[2], + self.padding[2], + self.padding[1], + self.padding[1], + 2 * self.padding[0], + 0, + ) + self.padding = (0, 0, 0) + + def forward(self, x, cache_x=None): + padding = list(self._padding) + if cache_x is not None and self._padding[4] > 0: + cache_x = cache_x.to(x.device) + x = torch.cat([cache_x, x], dim=2) + padding[4] -= cache_x.shape[2] + x = F.pad(x, padding) + + return super().forward(x) + + +class RMS_norm(nn.Module): + + def __init__(self, dim, channel_first=True, images=True, bias=False): + super().__init__() + broadcastable_dims = (1, 1, 1) if not images else (1, 1) + shape = (dim, *broadcastable_dims) if channel_first else (dim,) + + self.channel_first = channel_first + self.scale = dim**0.5 + self.gamma = nn.Parameter(torch.ones(shape)) + self.bias = nn.Parameter(torch.zeros(shape)) if bias else 0.0 + + def forward(self, x): + return (F.normalize(x, dim=(1 if self.channel_first else -1)) * + self.scale * self.gamma + self.bias) + + +class Upsample(nn.Upsample): + + def forward(self, x): + """ + Fix bfloat16 support for nearest neighbor interpolation. + """ + return super().forward(x.float()).type_as(x) + + +class Resample(nn.Module): + + def __init__(self, dim, mode): + assert mode in ( + "none", + "upsample2d", + "upsample3d", + "downsample2d", + "downsample3d", + ) + super().__init__() + self.dim = dim + self.mode = mode + + # layers + if mode == "upsample2d": + self.resample = nn.Sequential( + Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim, 3, padding=1), + ) + elif mode == "upsample3d": + self.resample = nn.Sequential( + Upsample(scale_factor=(2.0, 2.0), mode="nearest-exact"), + nn.Conv2d(dim, dim, 3, padding=1), + # nn.Conv2d(dim, dim//2, 3, padding=1) + ) + self.time_conv = CausalConv3d( + dim, dim * 2, (3, 1, 1), padding=(1, 0, 0)) + elif mode == "downsample2d": + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), + nn.Conv2d(dim, dim, 3, stride=(2, 2))) + elif mode == "downsample3d": + self.resample = nn.Sequential( + nn.ZeroPad2d((0, 1, 0, 1)), + nn.Conv2d(dim, dim, 3, stride=(2, 2))) + self.time_conv = CausalConv3d( + dim, dim, (3, 1, 1), stride=(2, 1, 1), padding=(0, 0, 0)) + else: + self.resample = nn.Identity() + + def forward(self, x, feat_cache=None, feat_idx=[0]): + b, c, t, h, w = x.size() + if self.mode == "upsample3d": + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = "Rep" + feat_idx[0] += 1 + else: + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if (cache_x.shape[2] < 2 and feat_cache[idx] is not None and + feat_cache[idx] != "Rep"): + # cache last frame of last two chunk + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), + cache_x, + ], + dim=2, + ) + if (cache_x.shape[2] < 2 and feat_cache[idx] is not None and + feat_cache[idx] == "Rep"): + cache_x = torch.cat( + [ + torch.zeros_like(cache_x).to(cache_x.device), + cache_x + ], + dim=2, + ) + if feat_cache[idx] == "Rep": + x = self.time_conv(x) + else: + x = self.time_conv(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + x = x.reshape(b, 2, c, t, h, w) + x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), + 3) + x = x.reshape(b, c, t * 2, h, w) + t = x.shape[2] + x = rearrange(x, "b c t h w -> (b t) c h w") + x = self.resample(x) + x = rearrange(x, "(b t) c h w -> b c t h w", t=t) + + if self.mode == "downsample3d": + if feat_cache is not None: + idx = feat_idx[0] + if feat_cache[idx] is None: + feat_cache[idx] = x.clone() + feat_idx[0] += 1 + else: + cache_x = x[:, :, -1:, :, :].clone() + x = self.time_conv( + torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2)) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + return x + + def init_weight(self, conv): + conv_weight = conv.weight.detach().clone() + nn.init.zeros_(conv_weight) + c1, c2, t, h, w = conv_weight.size() + one_matrix = torch.eye(c1, c2) + init_matrix = one_matrix + nn.init.zeros_(conv_weight) + conv_weight.data[:, :, 1, 0, 0] = init_matrix # * 0.5 + conv.weight = nn.Parameter(conv_weight) + nn.init.zeros_(conv.bias.data) + + def init_weight2(self, conv): + conv_weight = conv.weight.data.detach().clone() + nn.init.zeros_(conv_weight) + c1, c2, t, h, w = conv_weight.size() + init_matrix = torch.eye(c1 // 2, c2) + conv_weight[:c1 // 2, :, -1, 0, 0] = init_matrix + conv_weight[c1 // 2:, :, -1, 0, 0] = init_matrix + conv.weight = nn.Parameter(conv_weight) + nn.init.zeros_(conv.bias.data) + + +class ResidualBlock(nn.Module): + + def __init__(self, in_dim, out_dim, dropout=0.0): + super().__init__() + self.in_dim = in_dim + self.out_dim = out_dim + + # layers + self.residual = nn.Sequential( + RMS_norm(in_dim, images=False), + nn.SiLU(), + CausalConv3d(in_dim, out_dim, 3, padding=1), + RMS_norm(out_dim, images=False), + nn.SiLU(), + nn.Dropout(dropout), + CausalConv3d(out_dim, out_dim, 3, padding=1), + ) + self.shortcut = ( + CausalConv3d(in_dim, out_dim, 1) + if in_dim != out_dim else nn.Identity()) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + h = self.shortcut(x) + for layer in self.residual: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + # cache last frame of last two chunk + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), + cache_x, + ], + dim=2, + ) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x + h + + +class AttentionBlock(nn.Module): + """ + Causal self-attention with a single head. + """ + + def __init__(self, dim): + super().__init__() + self.dim = dim + + # layers + self.norm = RMS_norm(dim) + self.to_qkv = nn.Conv2d(dim, dim * 3, 1) + self.proj = nn.Conv2d(dim, dim, 1) + + # zero out the last layer params + nn.init.zeros_(self.proj.weight) + + def forward(self, x): + identity = x + b, c, t, h, w = x.size() + x = rearrange(x, "b c t h w -> (b t) c h w") + x = self.norm(x) + # compute query, key, value + q, k, v = ( + self.to_qkv(x).reshape(b * t, 1, c * 3, + -1).permute(0, 1, 3, + 2).contiguous().chunk(3, dim=-1)) + + # apply attention + x = F.scaled_dot_product_attention( + q, + k, + v, + ) + x = x.squeeze(1).permute(0, 2, 1).reshape(b * t, c, h, w) + + # output + x = self.proj(x) + x = rearrange(x, "(b t) c h w-> b c t h w", t=t) + return x + identity + + +def patchify(x, patch_size): + if patch_size == 1: + return x + if x.dim() == 4: + x = rearrange( + x, "b c (h q) (w r) -> b (c r q) h w", q=patch_size, r=patch_size) + elif x.dim() == 5: + x = rearrange( + x, + "b c f (h q) (w r) -> b (c r q) f h w", + q=patch_size, + r=patch_size, + ) + else: + raise ValueError(f"Invalid input shape: {x.shape}") + + return x + + +def unpatchify(x, patch_size): + if patch_size == 1: + return x + + if x.dim() == 4: + x = rearrange( + x, "b (c r q) h w -> b c (h q) (w r)", q=patch_size, r=patch_size) + elif x.dim() == 5: + x = rearrange( + x, + "b (c r q) f h w -> b c f (h q) (w r)", + q=patch_size, + r=patch_size, + ) + return x + + +class AvgDown3D(nn.Module): + + def __init__( + self, + in_channels, + out_channels, + factor_t, + factor_s=1, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + + assert in_channels * self.factor % out_channels == 0 + self.group_size = in_channels * self.factor // out_channels + + def forward(self, x: torch.Tensor) -> torch.Tensor: + pad_t = (self.factor_t - x.shape[2] % self.factor_t) % self.factor_t + pad = (0, 0, 0, 0, pad_t, 0) + x = F.pad(x, pad) + B, C, T, H, W = x.shape + x = x.view( + B, + C, + T // self.factor_t, + self.factor_t, + H // self.factor_s, + self.factor_s, + W // self.factor_s, + self.factor_s, + ) + x = x.permute(0, 1, 3, 5, 7, 2, 4, 6).contiguous() + x = x.view( + B, + C * self.factor, + T // self.factor_t, + H // self.factor_s, + W // self.factor_s, + ) + x = x.view( + B, + self.out_channels, + self.group_size, + T // self.factor_t, + H // self.factor_s, + W // self.factor_s, + ) + x = x.mean(dim=2) + return x + + +class DupUp3D(nn.Module): + + def __init__( + self, + in_channels: int, + out_channels: int, + factor_t, + factor_s=1, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + + self.factor_t = factor_t + self.factor_s = factor_s + self.factor = self.factor_t * self.factor_s * self.factor_s + + assert out_channels * self.factor % in_channels == 0 + self.repeats = out_channels * self.factor // in_channels + + def forward(self, x: torch.Tensor, first_chunk=False) -> torch.Tensor: + x = x.repeat_interleave(self.repeats, dim=1) + x = x.view( + x.size(0), + self.out_channels, + self.factor_t, + self.factor_s, + self.factor_s, + x.size(2), + x.size(3), + x.size(4), + ) + x = x.permute(0, 1, 5, 2, 6, 3, 7, 4).contiguous() + x = x.view( + x.size(0), + self.out_channels, + x.size(2) * self.factor_t, + x.size(4) * self.factor_s, + x.size(6) * self.factor_s, + ) + if first_chunk: + x = x[:, :, self.factor_t - 1:, :, :] + return x + + +class Down_ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + out_dim, + dropout, + mult, + temperal_downsample=False, + down_flag=False): + super().__init__() + + # Shortcut path with downsample + self.avg_shortcut = AvgDown3D( + in_dim, + out_dim, + factor_t=2 if temperal_downsample else 1, + factor_s=2 if down_flag else 1, + ) + + # Main path with residual blocks and downsample + downsamples = [] + for _ in range(mult): + downsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + in_dim = out_dim + + # Add the final downsample block + if down_flag: + mode = "downsample3d" if temperal_downsample else "downsample2d" + downsamples.append(Resample(out_dim, mode=mode)) + + self.downsamples = nn.Sequential(*downsamples) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + x_copy = x.clone() + for module in self.downsamples: + x = module(x, feat_cache, feat_idx) + + return x + self.avg_shortcut(x_copy) + + +class Up_ResidualBlock(nn.Module): + + def __init__(self, + in_dim, + out_dim, + dropout, + mult, + temperal_upsample=False, + up_flag=False): + super().__init__() + # Shortcut path with upsample + if up_flag: + self.avg_shortcut = DupUp3D( + in_dim, + out_dim, + factor_t=2 if temperal_upsample else 1, + factor_s=2 if up_flag else 1, + ) + else: + self.avg_shortcut = None + + # Main path with residual blocks and upsample + upsamples = [] + for _ in range(mult): + upsamples.append(ResidualBlock(in_dim, out_dim, dropout)) + in_dim = out_dim + + # Add the final upsample block + if up_flag: + mode = "upsample3d" if temperal_upsample else "upsample2d" + upsamples.append(Resample(out_dim, mode=mode)) + + self.upsamples = nn.Sequential(*upsamples) + + def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False): + x_main = x.clone() + for module in self.upsamples: + x_main = module(x_main, feat_cache, feat_idx) + if self.avg_shortcut is not None: + x_shortcut = self.avg_shortcut(x, first_chunk) + return x_main + x_shortcut + else: + return x_main + + +class Encoder3d(nn.Module): + + def __init__( + self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[True, True, False], + dropout=0.0, + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + + # dimensions + dims = [dim * u for u in [1] + dim_mult] + scale = 1.0 + + # init block + self.conv1 = CausalConv3d(12, dims[0], 3, padding=1) + + # downsample blocks + downsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + t_down_flag = ( + temperal_downsample[i] + if i < len(temperal_downsample) else False) + downsamples.append( + Down_ResidualBlock( + in_dim=in_dim, + out_dim=out_dim, + dropout=dropout, + mult=num_res_blocks, + temperal_downsample=t_down_flag, + down_flag=i != len(dim_mult) - 1, + )) + scale /= 2.0 + self.downsamples = nn.Sequential(*downsamples) + + # middle blocks + self.middle = nn.Sequential( + ResidualBlock(out_dim, out_dim, dropout), + AttentionBlock(out_dim), + ResidualBlock(out_dim, out_dim, dropout), + ) + + # # output blocks + self.head = nn.Sequential( + RMS_norm(out_dim, images=False), + nn.SiLU(), + CausalConv3d(out_dim, z_dim, 3, padding=1), + ) + + def forward(self, x, feat_cache=None, feat_idx=[0]): + + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + ## downsamples + for layer in self.downsamples: + if feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## middle + for layer in self.middle: + if isinstance(layer, ResidualBlock) and feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## head + for layer in self.head: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), + cache_x, + ], + dim=2, + ) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + + return x + + +class Decoder3d(nn.Module): + + def __init__( + self, + dim=128, + z_dim=4, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_upsample=[False, True, True], + dropout=0.0, + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_upsample = temperal_upsample + + # dimensions + dims = [dim * u for u in [dim_mult[-1]] + dim_mult[::-1]] + scale = 1.0 / 2**(len(dim_mult) - 2) + # init block + self.conv1 = CausalConv3d(z_dim, dims[0], 3, padding=1) + + # middle blocks + self.middle = nn.Sequential( + ResidualBlock(dims[0], dims[0], dropout), + AttentionBlock(dims[0]), + ResidualBlock(dims[0], dims[0], dropout), + ) + + # upsample blocks + upsamples = [] + for i, (in_dim, out_dim) in enumerate(zip(dims[:-1], dims[1:])): + t_up_flag = temperal_upsample[i] if i < len( + temperal_upsample) else False + upsamples.append( + Up_ResidualBlock( + in_dim=in_dim, + out_dim=out_dim, + dropout=dropout, + mult=num_res_blocks + 1, + temperal_upsample=t_up_flag, + up_flag=i != len(dim_mult) - 1, + )) + self.upsamples = nn.Sequential(*upsamples) + + # output blocks + self.head = nn.Sequential( + RMS_norm(out_dim, images=False), + nn.SiLU(), + CausalConv3d(out_dim, 12, 3, padding=1), + ) + + def forward(self, x, feat_cache=None, feat_idx=[0], first_chunk=False): + if feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), + cache_x, + ], + dim=2, + ) + x = self.conv1(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = self.conv1(x) + + for layer in self.middle: + if isinstance(layer, ResidualBlock) and feat_cache is not None: + x = layer(x, feat_cache, feat_idx) + else: + x = layer(x) + + ## upsamples + for layer in self.upsamples: + if feat_cache is not None: + x = layer(x, feat_cache, feat_idx, first_chunk) + else: + x = layer(x) + + ## head + for layer in self.head: + if isinstance(layer, CausalConv3d) and feat_cache is not None: + idx = feat_idx[0] + cache_x = x[:, :, -CACHE_T:, :, :].clone() + if cache_x.shape[2] < 2 and feat_cache[idx] is not None: + cache_x = torch.cat( + [ + feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to( + cache_x.device), + cache_x, + ], + dim=2, + ) + x = layer(x, feat_cache[idx]) + feat_cache[idx] = cache_x + feat_idx[0] += 1 + else: + x = layer(x) + return x + + +def count_conv3d(model): + count = 0 + for m in model.modules(): + if isinstance(m, CausalConv3d): + count += 1 + return count + + +class AutoencoderKLWan2_2_(nn.Module): + + def __init__( + self, + dim=160, + dec_dim=256, + z_dim=16, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[True, True, False], + dropout=0.0, + ): + super().__init__() + self.dim = dim + self.z_dim = z_dim + self.dim_mult = dim_mult + self.num_res_blocks = num_res_blocks + self.attn_scales = attn_scales + self.temperal_downsample = temperal_downsample + self.temperal_upsample = temperal_downsample[::-1] + + # modules + self.encoder = Encoder3d( + dim, + z_dim * 2, + dim_mult, + num_res_blocks, + attn_scales, + self.temperal_downsample, + dropout, + ) + self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1) + self.conv2 = CausalConv3d(z_dim, z_dim, 1) + self.decoder = Decoder3d( + dec_dim, + z_dim, + dim_mult, + num_res_blocks, + attn_scales, + self.temperal_upsample, + dropout, + ) + + def forward(self, x, scale=[0, 1]): + mu = self.encode(x, scale) + x_recon = self.decode(mu, scale) + return x_recon, mu + + def encode(self, x, scale): + self.clear_cache() + # z: [b,c,t,h,w] + scale = [item.to(x.device, x.dtype) for item in scale] + x = patchify(x, patch_size=2) + t = x.shape[2] + iter_ = 1 + (t - 1) // 4 + for i in range(iter_): + self._enc_conv_idx = [0] + if i == 0: + out = self.encoder( + x[:, :, :1, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx, + ) + else: + out_ = self.encoder( + x[:, :, 1 + 4 * (i - 1):1 + 4 * i, :, :], + feat_cache=self._enc_feat_map, + feat_idx=self._enc_conv_idx, + ) + out = torch.cat([out, out_], 2) + mu, log_var = self.conv1(out).chunk(2, dim=1) + if isinstance(scale[0], torch.Tensor): + mu = (mu - scale[0].view(1, self.z_dim, 1, 1, 1)) * scale[1].view( + 1, self.z_dim, 1, 1, 1) + else: + mu = (mu - scale[0]) * scale[1] + x = torch.cat([mu, log_var], dim = 1) + self.clear_cache() + return x + + def decode(self, z, scale): + self.clear_cache() + # z: [b,c,t,h,w] + scale = [item.to(z.device, z.dtype) for item in scale] + if isinstance(scale[0], torch.Tensor): + z = z / scale[1].view(1, self.z_dim, 1, 1, 1) + scale[0].view( + 1, self.z_dim, 1, 1, 1) + else: + z = z / scale[1] + scale[0] + iter_ = z.shape[2] + x = self.conv2(z) + for i in range(iter_): + self._conv_idx = [0] + if i == 0: + out = self.decoder( + x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx, + first_chunk=True, + ) + else: + out_ = self.decoder( + x[:, :, i:i + 1, :, :], + feat_cache=self._feat_map, + feat_idx=self._conv_idx, + ) + out = torch.cat([out, out_], 2) + out = unpatchify(out, patch_size=2) + self.clear_cache() + return out + + def reparameterize(self, mu, log_var): + std = torch.exp(0.5 * log_var) + eps = torch.randn_like(std) + return eps * std + mu + + def sample(self, imgs, deterministic=False): + mu, log_var = self.encode(imgs) + if deterministic: + return mu + std = torch.exp(0.5 * log_var.clamp(-30.0, 20.0)) + return mu + std * torch.randn_like(std) + + def clear_cache(self): + self._conv_num = count_conv3d(self.decoder) + self._conv_idx = [0] + self._feat_map = [None] * self._conv_num + # cache encode + self._enc_conv_num = count_conv3d(self.encoder) + self._enc_conv_idx = [0] + self._enc_feat_map = [None] * self._enc_conv_num + + +def _video_vae(pretrained_path=None, z_dim=16, dim=160, device="cpu", **kwargs): + # params + cfg = dict( + dim=dim, + z_dim=z_dim, + dim_mult=[1, 2, 4, 4], + num_res_blocks=2, + attn_scales=[], + temperal_downsample=[True, True, True], + dropout=0.0, + ) + cfg.update(**kwargs) + + # init model + model = AutoencoderKLWan2_2_(**cfg) + + return model + + +class AutoencoderKLWan3_8(ModelMixin, ConfigMixin, FromOriginalModelMixin): + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + latent_channels=48, + c_dim=160, + vae_pth=None, + dim_mult=[1, 2, 4, 4], + temperal_downsample=[False, True, True], + temporal_compression_ratio=4, + spatial_compression_ratio=8 + ): + super().__init__() + mean = torch.tensor( + [ + -0.2289, + -0.0052, + -0.1323, + -0.2339, + -0.2799, + 0.0174, + 0.1838, + 0.1557, + -0.1382, + 0.0542, + 0.2813, + 0.0891, + 0.1570, + -0.0098, + 0.0375, + -0.1825, + -0.2246, + -0.1207, + -0.0698, + 0.5109, + 0.2665, + -0.2108, + -0.2158, + 0.2502, + -0.2055, + -0.0322, + 0.1109, + 0.1567, + -0.0729, + 0.0899, + -0.2799, + -0.1230, + -0.0313, + -0.1649, + 0.0117, + 0.0723, + -0.2839, + -0.2083, + -0.0520, + 0.3748, + 0.0152, + 0.1957, + 0.1433, + -0.2944, + 0.3573, + -0.0548, + -0.1681, + -0.0667, + ], dtype=torch.float32 + ) + std = torch.tensor( + [ + 0.4765, + 1.0364, + 0.4514, + 1.1677, + 0.5313, + 0.4990, + 0.4818, + 0.5013, + 0.8158, + 1.0344, + 0.5894, + 1.0901, + 0.6885, + 0.6165, + 0.8454, + 0.4978, + 0.5759, + 0.3523, + 0.7135, + 0.6804, + 0.5833, + 1.4146, + 0.8986, + 0.5659, + 0.7069, + 0.5338, + 0.4889, + 0.4917, + 0.4069, + 0.4999, + 0.6866, + 0.4093, + 0.5709, + 0.6065, + 0.6415, + 0.4944, + 0.5726, + 1.2042, + 0.5458, + 1.6887, + 0.3971, + 1.0600, + 0.3943, + 0.5537, + 0.5444, + 0.4089, + 0.7468, + 0.7744, + ], dtype=torch.float32 + ) + self.scale = [mean, 1.0 / std] + + # init model + self.model = _video_vae( + pretrained_path=vae_pth, + z_dim=latent_channels, + dim=c_dim, + dim_mult=dim_mult, + temperal_downsample=temperal_downsample, + ).eval().requires_grad_(False) + + self.gradient_checkpointing = False + + def _set_gradient_checkpointing(self, *args, **kwargs): + if "value" in kwargs: + self.gradient_checkpointing = kwargs["value"] + elif "enable" in kwargs: + self.gradient_checkpointing = kwargs["enable"] + else: + raise ValueError("Invalid set gradient checkpointing") + + def _encode(self, x: torch.Tensor) -> torch.Tensor: + x = [ + self.model.encode(u.unsqueeze(0), self.scale).squeeze(0) + for u in x + ] + x = torch.stack(x) + return x + + @apply_forward_hook + def encode( + self, x: torch.Tensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + h = self._encode(x) + + posterior = DiagonalGaussianDistribution(h) + + if not return_dict: + return (posterior,) + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode(self, zs): + dec = [ + self.model.decode(u.unsqueeze(0), self.scale).clamp_(-1, 1).squeeze(0) + for u in zs + ] + dec = torch.stack(dec) + + return DecoderOutput(sample=dec) + + @apply_forward_hook + def decode(self, z: torch.Tensor, return_dict: bool = True) -> Union[DecoderOutput, torch.Tensor]: + decoded = self._decode(z).sample + + if not return_dict: + return (decoded,) + return DecoderOutput(sample=decoded) + + @classmethod + def from_pretrained(cls, pretrained_model_path, additional_kwargs={}): + def filter_kwargs(cls, kwargs): + import inspect + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + + model = cls(**filter_kwargs(cls, additional_kwargs)) + if pretrained_model_path.endswith(".safetensors"): + from safetensors.torch import load_file, safe_open + state_dict = load_file(pretrained_model_path) + else: + state_dict = torch.load(pretrained_model_path, map_location="cpu") + tmp_state_dict = {} + for key in state_dict: + tmp_state_dict["model." + key] = state_dict[key] + state_dict = tmp_state_dict + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m, u) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/wan_xlm_roberta.py b/VideoX-Fun/videox_fun/models/wan_xlm_roberta.py new file mode 100644 index 0000000000000000000000000000000000000000..755baf394431bee95e1eac835b5dafe6ed37c5b9 --- /dev/null +++ b/VideoX-Fun/videox_fun/models/wan_xlm_roberta.py @@ -0,0 +1,170 @@ +# Modified from transformers.models.xlm_roberta.modeling_xlm_roberta +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. +import torch +import torch.nn as nn +import torch.nn.functional as F + +__all__ = ['XLMRoberta', 'xlm_roberta_large'] + + +class SelfAttention(nn.Module): + + def __init__(self, dim, num_heads, dropout=0.1, eps=1e-5): + assert dim % num_heads == 0 + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.head_dim = dim // num_heads + self.eps = eps + + # layers + self.q = nn.Linear(dim, dim) + self.k = nn.Linear(dim, dim) + self.v = nn.Linear(dim, dim) + self.o = nn.Linear(dim, dim) + self.dropout = nn.Dropout(dropout) + + def forward(self, x, mask): + """ + x: [B, L, C]. + """ + b, s, c, n, d = *x.size(), self.num_heads, self.head_dim + + # compute query, key, value + q = self.q(x).reshape(b, s, n, d).permute(0, 2, 1, 3) + k = self.k(x).reshape(b, s, n, d).permute(0, 2, 1, 3) + v = self.v(x).reshape(b, s, n, d).permute(0, 2, 1, 3) + + # compute attention + p = self.dropout.p if self.training else 0.0 + x = F.scaled_dot_product_attention(q, k, v, mask, p) + x = x.permute(0, 2, 1, 3).reshape(b, s, c) + + # output + x = self.o(x) + x = self.dropout(x) + return x + + +class AttentionBlock(nn.Module): + + def __init__(self, dim, num_heads, post_norm, dropout=0.1, eps=1e-5): + super().__init__() + self.dim = dim + self.num_heads = num_heads + self.post_norm = post_norm + self.eps = eps + + # layers + self.attn = SelfAttention(dim, num_heads, dropout, eps) + self.norm1 = nn.LayerNorm(dim, eps=eps) + self.ffn = nn.Sequential( + nn.Linear(dim, dim * 4), nn.GELU(), nn.Linear(dim * 4, dim), + nn.Dropout(dropout)) + self.norm2 = nn.LayerNorm(dim, eps=eps) + + def forward(self, x, mask): + if self.post_norm: + x = self.norm1(x + self.attn(x, mask)) + x = self.norm2(x + self.ffn(x)) + else: + x = x + self.attn(self.norm1(x), mask) + x = x + self.ffn(self.norm2(x)) + return x + + +class XLMRoberta(nn.Module): + """ + XLMRobertaModel with no pooler and no LM head. + """ + + def __init__(self, + vocab_size=250002, + max_seq_len=514, + type_size=1, + pad_id=1, + dim=1024, + num_heads=16, + num_layers=24, + post_norm=True, + dropout=0.1, + eps=1e-5): + super().__init__() + self.vocab_size = vocab_size + self.max_seq_len = max_seq_len + self.type_size = type_size + self.pad_id = pad_id + self.dim = dim + self.num_heads = num_heads + self.num_layers = num_layers + self.post_norm = post_norm + self.eps = eps + + # embeddings + self.token_embedding = nn.Embedding(vocab_size, dim, padding_idx=pad_id) + self.type_embedding = nn.Embedding(type_size, dim) + self.pos_embedding = nn.Embedding(max_seq_len, dim, padding_idx=pad_id) + self.dropout = nn.Dropout(dropout) + + # blocks + self.blocks = nn.ModuleList([ + AttentionBlock(dim, num_heads, post_norm, dropout, eps) + for _ in range(num_layers) + ]) + + # norm layer + self.norm = nn.LayerNorm(dim, eps=eps) + + def forward(self, ids): + """ + ids: [B, L] of torch.LongTensor. + """ + b, s = ids.shape + mask = ids.ne(self.pad_id).long() + + # embeddings + x = self.token_embedding(ids) + \ + self.type_embedding(torch.zeros_like(ids)) + \ + self.pos_embedding(self.pad_id + torch.cumsum(mask, dim=1) * mask) + if self.post_norm: + x = self.norm(x) + x = self.dropout(x) + + # blocks + mask = torch.where( + mask.view(b, 1, 1, s).gt(0), 0.0, + torch.finfo(x.dtype).min) + for block in self.blocks: + x = block(x, mask) + + # output + if not self.post_norm: + x = self.norm(x) + return x + + +def xlm_roberta_large(pretrained=False, + return_tokenizer=False, + device='cpu', + **kwargs): + """ + XLMRobertaLarge adapted from Huggingface. + """ + # params + cfg = dict( + vocab_size=250002, + max_seq_len=514, + type_size=1, + pad_id=1, + dim=1024, + num_heads=16, + num_layers=24, + post_norm=True, + dropout=0.1, + eps=1e-5) + cfg.update(**kwargs) + + # init a model on device + with torch.device(device): + model = XLMRoberta(**cfg) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/z_image_transformer2d.py b/VideoX-Fun/videox_fun/models/z_image_transformer2d.py new file mode 100644 index 0000000000000000000000000000000000000000..5e2221099368592dba81f37baccb01978e66df8c --- /dev/null +++ b/VideoX-Fun/videox_fun/models/z_image_transformer2d.py @@ -0,0 +1,1188 @@ +# Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import glob +import inspect +import json +import os +import math +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.utils.rnn import pad_sequence + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin +from diffusers.models.attention_processor import Attention +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm +from diffusers.utils.torch_utils import maybe_allow_in_graph +from diffusers.models.attention_processor import Attention, AttentionProcessor +from diffusers.utils import (USE_PEFT_BACKEND, is_torch_version, logging, + scale_lora_layers, unscale_lora_layers) + +from .attention_utils import attention +from ..dist import (ZMultiGPUsSingleStreamAttnProcessor, get_sequence_parallel_rank, + get_sequence_parallel_world_size, get_sp_group) +import torch.nn.functional as F + + +ADALN_EMBED_DIM = 256 +SEQ_MULTI_OF = 32 + + +def pad_stack(seq_list: List[torch.Tensor], max_len: int, pad_value: float = 0.0) -> torch.Tensor: + """Pad a list of variable-length tensors to the same length and stack. + + seq_list: list of tensors shaped [seq_len, ...] + returns: tensor shaped [batch, max_len, ...] + """ + if len(seq_list) == 0: + return torch.empty(0) + padded = [] + for t in seq_list: + pad_len = max_len - t.shape[0] + if pad_len > 0: + # Pad on sequence dimension (second from the end for 2D inputs) + pad_shape = (0, 0) * (t.dim() - 1) + (0, pad_len) + t = F.pad(t, pad=pad_shape, value=pad_value) + padded.append(t) + return torch.stack(padded, dim=0) + + +class TimestepEmbedder(nn.Module): + def __init__(self, out_size, mid_size=None, frequency_embedding_size=256): + super().__init__() + if mid_size is None: + mid_size = out_size + self.mlp = nn.Sequential( + nn.Linear( + frequency_embedding_size, + mid_size, + bias=True, + ), + nn.SiLU(), + nn.Linear( + mid_size, + out_size, + bias=True, + ), + ) + + self.frequency_embedding_size = frequency_embedding_size + + @staticmethod + def timestep_embedding(t, dim, max_period=10000): + with torch.amp.autocast("cuda", enabled=False): + half = dim // 2 + freqs = torch.exp( + -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32, device=t.device) / half + ) + args = t[:, None].float() * freqs[None] + embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1) + if dim % 2: + embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1) + return embedding + + def forward(self, t): + t_freq = self.timestep_embedding(t, self.frequency_embedding_size) + weight_dtype = self.mlp[0].weight.dtype + if weight_dtype.is_floating_point: + t_freq = t_freq.to(weight_dtype) + t_emb = self.mlp(t_freq) + return t_emb + + +class ZSingleStreamAttnProcessor: + """ + Processor for Z-Image single stream attention that adapts the existing Attention class to match the behavior of the + original Z-ImageAttention module. + """ + + _attention_backend = None + _parallel_config = None + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError( + "ZSingleStreamAttnProcessor requires PyTorch 2.0. To use it, please upgrade PyTorch to version 2.0 or higher." + ) + + def __call__( + self, + attn: Attention, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + freqs_cis: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + query = attn.to_q(hidden_states) + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + + query = query.unflatten(-1, (attn.heads, -1)) + key = key.unflatten(-1, (attn.heads, -1)) + value = value.unflatten(-1, (attn.heads, -1)) + + # Apply Norms + if attn.norm_q is not None: + query = attn.norm_q(query) + if attn.norm_k is not None: + key = attn.norm_k(key) + + # Apply RoPE + def apply_rotary_emb(x_in: torch.Tensor, freqs_cis: torch.Tensor) -> torch.Tensor: + with torch.amp.autocast("cuda", enabled=False): + # Use real-valued RoPE (cos/sin) while keeping shape aligned with heads and pairs. + # x_in: [B, Seq, Heads, HeadDim] + x = x_in.float().reshape(*x_in.shape[:-1], -1, 2) # -> [..., HeadDim/2, 2] + freqs = freqs_cis.reshape(*freqs_cis.shape[:-1], -1, 2) # [B, Seq, HeadDim/2, 2] + cos = freqs[..., 0].unsqueeze(2) # [B, Seq, 1, HeadDim/2] + sin = freqs[..., 1].unsqueeze(2) + x_even = x[..., 0] + x_odd = x[..., 1] + rot_even = x_even * cos - x_odd * sin + rot_odd = x_odd * cos + x_even * sin + x_out = torch.stack((rot_even, rot_odd), dim=-1) + return x_out.reshape_as(x_in).type_as(x_in) + + if freqs_cis is not None: + query = apply_rotary_emb(query, freqs_cis) + key = apply_rotary_emb(key, freqs_cis) + + # Cast to correct dtype + dtype = query.dtype + query, key = query.to(dtype), key.to(dtype) + + # From [batch, seq_len] to [batch, 1, 1, seq_len] -> broadcast to [batch, heads, seq_len, seq_len] + if attention_mask is not None and attention_mask.ndim == 2: + attention_mask = attention_mask[:, None, None, :] + + # Compute joint attention + hidden_states = attention( + query, + key, + value, + attn_mask=attention_mask + ) + + # Reshape back + hidden_states = hidden_states.flatten(2, 3) + hidden_states = hidden_states.to(dtype) + + output = attn.to_out[0](hidden_states) + if len(attn.to_out) > 1: # dropout + output = attn.to_out[1](output) + + return output + + +class FeedForward(nn.Module): + def __init__(self, dim: int, hidden_dim: int): + super().__init__() + self.w1 = nn.Linear(dim, hidden_dim, bias=False) + self.w2 = nn.Linear(hidden_dim, dim, bias=False) + self.w3 = nn.Linear(dim, hidden_dim, bias=False) + + def _forward_silu_gating(self, x1, x3): + return F.silu(x1) * x3 + + def forward(self, x): + return self.w2(self._forward_silu_gating(self.w1(x), self.w3(x))) + + +@maybe_allow_in_graph +class ZImageTransformerBlock(nn.Module): + def __init__( + self, + layer_id: int, + dim: int, + n_heads: int, + n_kv_heads: int, + norm_eps: float, + qk_norm: bool, + modulation=True, + ): + super().__init__() + self.dim = dim + self.head_dim = dim // n_heads + + # Refactored to use diffusers Attention with custom processor + # Original Z-Image params: dim, n_heads, n_kv_heads, qk_norm + self.attention = Attention( + query_dim=dim, + cross_attention_dim=None, + dim_head=dim // n_heads, + heads=n_heads, + qk_norm="rms_norm" if qk_norm else None, + eps=1e-5, + bias=False, + out_bias=False, + processor=ZSingleStreamAttnProcessor(), + ) + + self.feed_forward = FeedForward(dim=dim, hidden_dim=int(dim / 3 * 8)) + self.layer_id = layer_id + + self.attention_norm1 = RMSNorm(dim, eps=norm_eps) + self.ffn_norm1 = RMSNorm(dim, eps=norm_eps) + + self.attention_norm2 = RMSNorm(dim, eps=norm_eps) + self.ffn_norm2 = RMSNorm(dim, eps=norm_eps) + + self.modulation = modulation + if modulation: + self.adaLN_modulation = nn.Sequential( + nn.Linear(min(dim, ADALN_EMBED_DIM), 4 * dim, bias=True), + ) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def forward( + self, + x: torch.Tensor, + attn_mask: torch.Tensor, + freqs_cis: torch.Tensor, + adaln_input: Optional[torch.Tensor] = None, + ): + if self.modulation: + assert adaln_input is not None + scale_msa, gate_msa, scale_mlp, gate_mlp = self.adaLN_modulation(adaln_input).unsqueeze(1).chunk(4, dim=2) + gate_msa, gate_mlp = gate_msa.tanh(), gate_mlp.tanh() + scale_msa, scale_mlp = 1.0 + scale_msa, 1.0 + scale_mlp + + # Attention block + attn_out = self.attention( + self.attention_norm1(x) * scale_msa, + attention_mask=attn_mask, + freqs_cis=freqs_cis, + ) + x = x + gate_msa * self.attention_norm2(attn_out) + + # FFN block + x = x + gate_mlp * self.ffn_norm2( + self.feed_forward( + self.ffn_norm1(x) * scale_mlp, + ) + ) + else: + # Attention block + attn_out = self.attention( + self.attention_norm1(x), + attention_mask=attn_mask, + freqs_cis=freqs_cis, + ) + x = x + self.attention_norm2(attn_out) + + # FFN block + x = x + self.ffn_norm2( + self.feed_forward( + self.ffn_norm1(x), + ) + ) + + return x + + +class FinalLayer(nn.Module): + def __init__(self, hidden_size, out_channels): + super().__init__() + self.norm_final = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6) + self.linear = nn.Linear(hidden_size, out_channels, bias=True) + + self.adaLN_modulation = nn.Sequential( + nn.SiLU(), + nn.Linear(min(hidden_size, ADALN_EMBED_DIM), hidden_size, bias=True), + ) + + def forward(self, x, c): + scale = 1.0 + self.adaLN_modulation(c) + x = self.norm_final(x) * scale.unsqueeze(1) + x = self.linear(x) + return x + + +class RopeEmbedder: + def __init__( + self, + theta: float = 256.0, + axes_dims: List[int] = (16, 56, 56), + axes_lens: List[int] = (64, 128, 128), + ): + self.theta = theta + self.axes_dims = axes_dims + self.axes_lens = axes_lens + assert len(axes_dims) == len(axes_lens), "axes_dims and axes_lens must have the same length" + self.freqs_cis = None + + @staticmethod + def precompute_freqs_cis(dim: List[int], end: List[int], theta: float = 256.0): + with torch.device("cpu"): + freqs_cis = [] + for i, (d, e) in enumerate(zip(dim, end)): + freqs = 1.0 / (theta ** (torch.arange(0, d, 2, dtype=torch.float64, device="cpu") / d)) + timestep = torch.arange(e, device=freqs.device, dtype=torch.float64) + freqs = torch.outer(timestep, freqs).float() + cos = torch.cos(freqs) + sin = torch.sin(freqs) + # Interleave cos/sin so each rotary pair stays contiguous for reshape(..., 2). + freqs_cis_i = torch.stack((cos, sin), dim=-1).reshape(freqs.shape[0], -1) + freqs_cis.append(freqs_cis_i) + + return freqs_cis + + def __call__(self, ids: torch.Tensor): + assert ids.ndim == 2 + assert ids.shape[-1] == len(self.axes_dims) + device = ids.device + + if self.freqs_cis is None: + self.freqs_cis = self.precompute_freqs_cis(self.axes_dims, self.axes_lens, theta=self.theta) + self.freqs_cis = [freqs_cis.to(device) for freqs_cis in self.freqs_cis] + else: + # Ensure freqs_cis are on the same device as ids + if self.freqs_cis[0].device != device: + self.freqs_cis = [freqs_cis.to(device) for freqs_cis in self.freqs_cis] + + result = [] + for i in range(len(self.axes_dims)): + index = ids[:, i] + result.append(self.freqs_cis[i][index]) + return torch.cat(result, dim=-1) + + +class ZImageTransformer2DModel(ModelMixin, ConfigMixin, FromOriginalModelMixin): + _supports_gradient_checkpointing = True + # _no_split_modules = ["ZImageTransformerBlock"] + # _skip_layerwise_casting_patterns = ["t_embedder", "cap_embedder"] # precision sensitive layers + + @register_to_config + def __init__( + self, + all_patch_size=(2,), + all_f_patch_size=(1,), + in_channels=16, + dim=3840, + n_layers=30, + n_refiner_layers=2, + n_heads=30, + n_kv_heads=30, + norm_eps=1e-5, + qk_norm=True, + cap_feat_dim=2560, + rope_theta=256.0, + t_scale=1000.0, + axes_dims=[32, 48, 48], + axes_lens=[1024, 512, 512], + ) -> None: + super().__init__() + self.in_channels = in_channels + self.out_channels = in_channels + self.all_patch_size = all_patch_size + self.all_f_patch_size = all_f_patch_size + self.dim = dim + self.n_heads = n_heads + + self.rope_theta = rope_theta + self.t_scale = t_scale + self.gradient_checkpointing = False + + assert len(all_patch_size) == len(all_f_patch_size) + + all_x_embedder = {} + all_final_layer = {} + for patch_idx, (patch_size, f_patch_size) in enumerate(zip(all_patch_size, all_f_patch_size)): + x_embedder = nn.Linear(f_patch_size * patch_size * patch_size * in_channels, dim, bias=True) + all_x_embedder[f"{patch_size}-{f_patch_size}"] = x_embedder + + final_layer = FinalLayer(dim, patch_size * patch_size * f_patch_size * self.out_channels) + all_final_layer[f"{patch_size}-{f_patch_size}"] = final_layer + + self.all_x_embedder = nn.ModuleDict(all_x_embedder) + self.all_final_layer = nn.ModuleDict(all_final_layer) + self.noise_refiner = nn.ModuleList( + [ + ZImageTransformerBlock( + 1000 + layer_id, + dim, + n_heads, + n_kv_heads, + norm_eps, + qk_norm, + modulation=True, + ) + for layer_id in range(n_refiner_layers) + ] + ) + self.context_refiner = nn.ModuleList( + [ + ZImageTransformerBlock( + layer_id, + dim, + n_heads, + n_kv_heads, + norm_eps, + qk_norm, + modulation=False, + ) + for layer_id in range(n_refiner_layers) + ] + ) + self.t_embedder = TimestepEmbedder(min(dim, ADALN_EMBED_DIM), mid_size=1024) + self.cap_embedder = nn.Sequential( + RMSNorm(cap_feat_dim, eps=norm_eps), + nn.Linear(cap_feat_dim, dim, bias=True), + ) + + self.x_pad_token = nn.Parameter(torch.empty((1, dim))) + self.cap_pad_token = nn.Parameter(torch.empty((1, dim))) + + self.layers = nn.ModuleList( + [ + ZImageTransformerBlock(layer_id, dim, n_heads, n_kv_heads, norm_eps, qk_norm) + for layer_id in range(n_layers) + ] + ) + head_dim = dim // n_heads + assert head_dim == sum(axes_dims) + self.axes_dims = axes_dims + self.axes_lens = axes_lens + + self.rope_embedder = RopeEmbedder(theta=rope_theta, axes_dims=axes_dims, axes_lens=axes_lens) + + self.sp_world_size = 1 + self.sp_world_rank = 0 + + def _set_gradient_checkpointing(self, *args, **kwargs): + if "value" in kwargs: + self.gradient_checkpointing = kwargs["value"] + elif "enable" in kwargs: + self.gradient_checkpointing = kwargs["enable"] + else: + raise ValueError("Invalid set gradient checkpointing") + + def enable_multi_gpus_inference(self,): + self.sp_world_size = get_sequence_parallel_world_size() + self.sp_world_rank = get_sequence_parallel_rank() + self.all_gather = get_sp_group().all_gather + for layer in self.layers: + layer.set_attn_processor(ZMultiGPUsSingleStreamAttnProcessor()) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor() + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def unpatchify(self, x: List[torch.Tensor], size: List[Tuple], patch_size, f_patch_size) -> List[torch.Tensor]: + pH = pW = patch_size + pF = f_patch_size + bsz = len(x) + assert len(size) == bsz + for i in range(bsz): + F, H, W = size[i] + ori_len = (F // pF) * (H // pH) * (W // pW) + # "f h w pf ph pw c -> c (f pf) (h ph) (w pw)" + x[i] = ( + x[i][:ori_len] + .view(F // pF, H // pH, W // pW, pF, pH, pW, self.out_channels) + .permute(6, 0, 3, 1, 4, 2, 5) + .reshape(self.out_channels, F, H, W) + ) + return x + + @staticmethod + def create_coordinate_grid(size, start=None, device=None): + if start is None: + start = (0 for _ in size) + + axes = [torch.arange(x0, x0 + span, dtype=torch.int32, device=device) for x0, span in zip(start, size)] + grids = torch.meshgrid(axes, indexing="ij") + return torch.stack(grids, dim=-1) + + def patchify( + self, + all_image: List[torch.Tensor], + patch_size: int, + f_patch_size: int, + cap_padding_len: int, + ): + pH = pW = patch_size + pF = f_patch_size + device = all_image[0].device + + all_image_out = [] + all_image_size = [] + all_image_pos_ids = [] + all_image_pad_mask = [] + + for i, image in enumerate(all_image): + ### Process Image + C, F, H, W = image.size() + all_image_size.append((F, H, W)) + F_tokens, H_tokens, W_tokens = F // pF, H // pH, W // pW + + image = image.view(C, F_tokens, pF, H_tokens, pH, W_tokens, pW) + # "c f pf h ph w pw -> (f h w) (pf ph pw c)" + image = image.permute(1, 3, 5, 2, 4, 6, 0).reshape(F_tokens * H_tokens * W_tokens, pF * pH * pW * C) + + image_ori_len = len(image) + image_padding_len = (-image_ori_len) % SEQ_MULTI_OF + + image_ori_pos_ids = self.create_coordinate_grid( + size=(F_tokens, H_tokens, W_tokens), + start=(cap_padding_len + 1, 0, 0), + device=device, + ).flatten(0, 2) + + if image_padding_len > 0: + image_padding_pos_ids = ( + self.create_coordinate_grid( + size=(1, 1, 1), + start=(0, 0, 0), + device=device, + ) + .flatten(0, 2) + .repeat(image_padding_len, 1) + ) + image_padded_pos_ids = torch.cat([image_ori_pos_ids, image_padding_pos_ids], dim=0) + all_image_pad_mask.append( + torch.cat( + [ + torch.zeros((image_ori_len,), dtype=torch.bool, device=device), + torch.ones((image_padding_len,), dtype=torch.bool, device=device), + ], + dim=0, + ) + ) + image_padded_feat = torch.cat([image, image[-1:].repeat(image_padding_len, 1)], dim=0) + else: + image_padded_pos_ids = image_ori_pos_ids + all_image_pad_mask.append(torch.zeros((image_ori_len,), dtype=torch.bool, device=device)) + image_padded_feat = image + + all_image_pos_ids.append(image_padded_pos_ids) + all_image_out.append(image_padded_feat) + + return ( + all_image_out, + all_image_size, + all_image_pos_ids, + all_image_pad_mask, + ) + + def patchify_and_embed( + self, + all_image: List[torch.Tensor], + all_cap_feats: List[torch.Tensor], + patch_size: int, + f_patch_size: int, + ): + pH = pW = patch_size + pF = f_patch_size + device = all_image[0].device + + all_image_out = [] + all_image_size = [] + all_image_pos_ids = [] + all_image_pad_mask = [] + all_cap_pos_ids = [] + all_cap_pad_mask = [] + all_cap_feats_out = [] + + for i, (image, cap_feat) in enumerate(zip(all_image, all_cap_feats)): + ### Process Caption + cap_ori_len = len(cap_feat) + cap_padding_len = (-cap_ori_len) % SEQ_MULTI_OF + # padded position ids + cap_padded_pos_ids = self.create_coordinate_grid( + size=(cap_ori_len + cap_padding_len, 1, 1), + start=(1, 0, 0), + device=device, + ).flatten(0, 2) + all_cap_pos_ids.append(cap_padded_pos_ids) + # pad mask + if cap_padding_len > 0: + all_cap_pad_mask.append( + torch.cat( + [ + torch.zeros((cap_ori_len,), dtype=torch.bool, device=device), + torch.ones((cap_padding_len,), dtype=torch.bool, device=device), + ], + dim=0, + ) + ) + cap_padded_feat = torch.cat( + [cap_feat, cap_feat[-1:].repeat(cap_padding_len, 1)], + dim=0, + ) + else: + all_cap_pad_mask.append(torch.zeros((cap_ori_len,), dtype=torch.bool, device=device)) + cap_padded_feat = cap_feat + + all_cap_feats_out.append(cap_padded_feat) + + ### Process Image + C, F, H, W = image.size() + all_image_size.append((F, H, W)) + F_tokens, H_tokens, W_tokens = F // pF, H // pH, W // pW + + image = image.view(C, F_tokens, pF, H_tokens, pH, W_tokens, pW) + # "c f pf h ph w pw -> (f h w) (pf ph pw c)" + image = image.permute(1, 3, 5, 2, 4, 6, 0).reshape(F_tokens * H_tokens * W_tokens, pF * pH * pW * C) + + image_ori_len = len(image) + image_padding_len = (-image_ori_len) % SEQ_MULTI_OF + + image_ori_pos_ids = self.create_coordinate_grid( + size=(F_tokens, H_tokens, W_tokens), + start=(cap_ori_len + cap_padding_len + 1, 0, 0), + device=device, + ).flatten(0, 2) + if image_padding_len > 0: + image_padding_pos_ids = ( + self.create_coordinate_grid( + size=(1, 1, 1), + start=(0, 0, 0), + device=device, + ) + .flatten(0, 2) + .repeat(image_padding_len, 1) + ) + image_padded_pos_ids = torch.cat([image_ori_pos_ids, image_padding_pos_ids], dim=0) + all_image_pad_mask.append( + torch.cat( + [ + torch.zeros((image_ori_len,), dtype=torch.bool, device=device), + torch.ones((image_padding_len,), dtype=torch.bool, device=device), + ], + dim=0, + ) + ) + image_padded_feat = torch.cat([image, image[-1:].repeat(image_padding_len, 1)], dim=0) + else: + image_padded_pos_ids = image_ori_pos_ids + all_image_pad_mask.append(torch.zeros((image_ori_len,), dtype=torch.bool, device=device)) + image_padded_feat = image + + all_image_pos_ids.append(image_padded_pos_ids) + all_image_out.append(image_padded_feat) + + return ( + all_image_out, + all_cap_feats_out, + all_image_size, + all_image_pos_ids, + all_cap_pos_ids, + all_image_pad_mask, + all_cap_pad_mask, + ) + + def forward( + self, + x: List[torch.Tensor], + t, + cap_feats: List[torch.Tensor], + patch_size=2, + f_patch_size=1, + ): + assert patch_size in self.all_patch_size + assert f_patch_size in self.all_f_patch_size + + bsz = len(x) + device = x[0].device + t = t * self.t_scale + t = self.t_embedder(t) + + ( + x, + cap_feats, + x_size, + x_pos_ids, + cap_pos_ids, + x_inner_pad_mask, + cap_inner_pad_mask, + ) = self.patchify_and_embed(x, cap_feats, patch_size, f_patch_size) + + # x embed & refine + x_item_seqlens = [len(_) for _ in x] + assert all(_ % SEQ_MULTI_OF == 0 for _ in x_item_seqlens) + x_max_item_seqlen = max(x_item_seqlens) + + x = torch.cat(x, dim=0) + x = self.all_x_embedder[f"{patch_size}-{f_patch_size}"](x) + + # Match t_embedder output dtype to x for layerwise casting compatibility + adaln_input = t.type_as(x) + ################################################## + # x[torch.cat(x_inner_pad_mask)] = self.x_pad_token # TODO: 可能会产生 ScatterND 算子, 会导致 ort 报错 + # 替换原来的 x[torch.cat(x_inner_pad_mask)] = self.x_pad_token + mask = torch.cat(x_inner_pad_mask) + # 使用加权混合避免 ScatterND/Tile/Where，兼容 ONNX + pad_token = self.x_pad_token + if pad_token.dim() == 1: + pad_token = pad_token.unsqueeze(0) # [1, hidden] + pad_token = pad_token.to(dtype=x.dtype, device=x.device) + mask_f = mask.unsqueeze(1).to(dtype=x.dtype) # [seq, 1] + x = x + mask_f * (pad_token.expand_as(x) - x) + ################################################## + + x = list(x.split(x_item_seqlens, dim=0)) + x_freqs_cis = list(self.rope_embedder(torch.cat(x_pos_ids, dim=0)).split(x_item_seqlens, dim=0)) + + x = pad_stack(x, x_max_item_seqlen, pad_value=0.0) + x_freqs_cis = pad_stack(x_freqs_cis, x_max_item_seqlen, pad_value=0.0) + x_attn_mask = torch.zeros((bsz, x_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(x_item_seqlens): + x_attn_mask[i, :seq_len] = 1 + + if torch.is_grad_enabled() and self.gradient_checkpointing: + for layer in self.noise_refiner: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer), + x, x_attn_mask, x_freqs_cis, adaln_input, + **ckpt_kwargs, + ) + else: + for layer in self.noise_refiner: + x = layer(x, x_attn_mask, x_freqs_cis, adaln_input) + + # cap embed & refine + cap_item_seqlens = [len(_) for _ in cap_feats] + assert all(_ % SEQ_MULTI_OF == 0 for _ in cap_item_seqlens) + cap_max_item_seqlen = max(cap_item_seqlens) + + cap_feats = torch.cat(cap_feats, dim=0) + cap_feats = self.cap_embedder(cap_feats) + cap_mask = torch.cat(cap_inner_pad_mask) + cap_pad = self.cap_pad_token + if cap_pad.dim() == 1: + cap_pad = cap_pad.unsqueeze(0) + cap_pad = cap_pad.to(dtype=cap_feats.dtype, device=cap_feats.device) + cap_mask_f = cap_mask.unsqueeze(1).to(dtype=cap_feats.dtype) + cap_feats = cap_feats + cap_mask_f * (cap_pad.expand_as(cap_feats) - cap_feats) + cap_feats = list(cap_feats.split(cap_item_seqlens, dim=0)) + cap_freqs_cis = list(self.rope_embedder(torch.cat(cap_pos_ids, dim=0)).split(cap_item_seqlens, dim=0)) + + cap_feats = pad_stack(cap_feats, cap_max_item_seqlen, pad_value=0.0) + cap_freqs_cis = pad_stack(cap_freqs_cis, cap_max_item_seqlen, pad_value=0.0) + cap_attn_mask = torch.zeros((bsz, cap_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(cap_item_seqlens): + cap_attn_mask[i, :seq_len] = 1 + + if torch.is_grad_enabled() and self.gradient_checkpointing: + for layer in self.context_refiner: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + cap_feats = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer), + cap_feats, + cap_attn_mask, + cap_freqs_cis, + **ckpt_kwargs, + ) + else: + for layer in self.context_refiner: + cap_feats = layer(cap_feats, cap_attn_mask, cap_freqs_cis) + + # Context Parallel + if self.sp_world_size > 1: + x = torch.chunk(x, self.sp_world_size, dim=1)[self.sp_world_rank] + + x_item_seqlens = [len(_) for _ in x] + assert all(_ % SEQ_MULTI_OF == 0 for _ in x_item_seqlens) + x_max_item_seqlen = max(x_item_seqlens) + x_attn_mask = torch.zeros((bsz, x_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(x_item_seqlens): + x_attn_mask[i, :seq_len] = 1 + + if x_freqs_cis is not None: + x_freqs_cis = torch.chunk(x_freqs_cis, self.sp_world_size, dim=1)[self.sp_world_rank] + + # unified + unified = [] + unified_freqs_cis = [] + for i in range(bsz): + x_len = x_item_seqlens[i] + cap_len = cap_item_seqlens[i] + unified.append(torch.cat([x[i][:x_len], cap_feats[i][:cap_len]])) + unified_freqs_cis.append(torch.cat([x_freqs_cis[i][:x_len], cap_freqs_cis[i][:cap_len]])) + unified_item_seqlens = [a + b for a, b in zip(cap_item_seqlens, x_item_seqlens)] + assert unified_item_seqlens == [len(_) for _ in unified] + unified_max_item_seqlen = max(unified_item_seqlens) + unified = pad_stack(unified, unified_max_item_seqlen, pad_value=0.0) + unified_freqs_cis = pad_stack(unified_freqs_cis, unified_max_item_seqlen, pad_value=0.0) + unified_attn_mask = torch.zeros((bsz, unified_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(unified_item_seqlens): + unified_attn_mask[i, :seq_len] = 1 + + if torch.is_grad_enabled() and self.gradient_checkpointing: + for layer in self.layers: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + unified = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer), + unified, + unified_attn_mask, + unified_freqs_cis, + adaln_input, + **ckpt_kwargs, + ) + else: + for layer in self.layers: + unified = layer(unified, unified_attn_mask, unified_freqs_cis, adaln_input) + + if self.sp_world_size > 1: + unified_out = [] + for i in range(bsz): + x_len = x_item_seqlens[i] + unified_out.append(unified[i, :x_len]) + unified = torch.stack(unified_out) + unified = self.all_gather(unified, dim=1) + + unified = self.all_final_layer[f"{patch_size}-{f_patch_size}"](unified, adaln_input) + unified = list(unified.unbind(dim=0)) + x = self.unpatchify(unified, x_size, patch_size, f_patch_size) + + x = torch.stack(x) + return x, {} + + + @classmethod + def from_pretrained( + cls, pretrained_model_path, subfolder=None, transformer_additional_kwargs={}, + low_cpu_mem_usage=False, torch_dtype=torch.bfloat16 + ): + if subfolder is not None: + pretrained_model_path = os.path.join(pretrained_model_path, subfolder) + print(f"loaded 3D transformer's pretrained weights from {pretrained_model_path} ...") + + config_file = os.path.join(pretrained_model_path, 'config.json') + if not os.path.isfile(config_file): + raise RuntimeError(f"{config_file} does not exist") + with open(config_file, "r") as f: + config = json.load(f) + + from diffusers.utils import WEIGHTS_NAME + model_file = os.path.join(pretrained_model_path, WEIGHTS_NAME) + model_file_safetensors = model_file.replace(".bin", ".safetensors") + + if "dict_mapping" in transformer_additional_kwargs.keys(): + for key in transformer_additional_kwargs["dict_mapping"]: + transformer_additional_kwargs[transformer_additional_kwargs["dict_mapping"][key]] = config[key] + + if low_cpu_mem_usage: + try: + import re + + from diffusers import __version__ as diffusers_version + if diffusers_version >= "0.33.0": + from diffusers.models.model_loading_utils import \ + load_model_dict_into_meta + else: + from diffusers.models.modeling_utils import \ + load_model_dict_into_meta + from diffusers.utils import is_accelerate_available + if is_accelerate_available(): + import accelerate + + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + model = cls.from_config(config, **transformer_additional_kwargs) + + param_device = "cpu" + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + print(model_files_safetensors) + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + filtered_state_dict = {} + for key in state_dict: + if key in model.state_dict() and model.state_dict()[key].size() == state_dict[key].size(): + filtered_state_dict[key] = state_dict[key] + else: + print(f"Skipping key '{key}' due to size mismatch or absence in model.") + + model_keys = set(model.state_dict().keys()) + loaded_keys = set(filtered_state_dict.keys()) + missing_keys = model_keys - loaded_keys + + def initialize_missing_parameters(missing_keys, model_state_dict, torch_dtype=None): + initialized_dict = {} + + with torch.no_grad(): + for key in missing_keys: + param_shape = model_state_dict[key].shape + param_dtype = torch_dtype if torch_dtype is not None else model_state_dict[key].dtype + if "control" in key and key.replace("control_", "") in filtered_state_dict.keys(): + initialized_dict[key] = filtered_state_dict[key.replace("control_", "")].clone() + print(f"Initializing missing parameter '{key}' with model.state_dict().") + elif "after_proj" in key or "before_proj" in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + print(f"Initializing missing parameter '{key}' with zero.") + elif 'weight' in key: + if any(norm_type in key for norm_type in ['norm', 'ln_', 'layer_norm', 'group_norm', 'batch_norm']): + initialized_dict[key] = torch.ones(param_shape, dtype=param_dtype) + elif 'embedding' in key or 'embed' in key: + initialized_dict[key] = torch.randn(param_shape, dtype=param_dtype) * 0.02 + elif 'head' in key or 'output' in key or 'proj_out' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif len(param_shape) >= 2: + initialized_dict[key] = torch.empty(param_shape, dtype=param_dtype) + nn.init.xavier_uniform_(initialized_dict[key]) + else: + initialized_dict[key] = torch.randn(param_shape, dtype=param_dtype) * 0.02 + elif 'bias' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif 'running_mean' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + elif 'running_var' in key: + initialized_dict[key] = torch.ones(param_shape, dtype=param_dtype) + elif 'num_batches_tracked' in key: + initialized_dict[key] = torch.zeros(param_shape, dtype=torch.long) + else: + initialized_dict[key] = torch.zeros(param_shape, dtype=param_dtype) + + return initialized_dict + + if missing_keys: + print(f"Missing keys will be initialized: {sorted(missing_keys)}") + initialized_params = initialize_missing_parameters( + missing_keys, + model.state_dict(), + torch_dtype + ) + filtered_state_dict.update(initialized_params) + + if diffusers_version >= "0.33.0": + # Diffusers has refactored `load_model_dict_into_meta` since version 0.33.0 in this commit: + # https://github.com/huggingface/diffusers/commit/f5929e03060d56063ff34b25a8308833bec7c785. + load_model_dict_into_meta( + model, + filtered_state_dict, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + else: + model._convert_deprecated_attention_blocks(filtered_state_dict) + unexpected_keys = load_model_dict_into_meta( + model, + filtered_state_dict, + device=param_device, + dtype=torch_dtype, + model_name_or_path=pretrained_model_path, + ) + + if cls._keys_to_ignore_on_load_unexpected is not None: + for pat in cls._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + print( + f"Some weights of the model checkpoint were not used when initializing {cls.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + + params = [p.numel() if "." in n else 0 for n, p in model.named_parameters()] + print(f"### All Parameters: {sum(params) / 1e6} M") + + params = [p.numel() if "attn1." in n else 0 for n, p in model.named_parameters()] + print(f"### attn1 Parameters: {sum(params) / 1e6} M") + return model + except Exception as e: + print( + f"The low_cpu_mem_usage mode is not work because {e}. Use low_cpu_mem_usage=False instead." + ) + + model = cls.from_config(config, **transformer_additional_kwargs) + if os.path.exists(model_file): + state_dict = torch.load(model_file, map_location="cpu") + elif os.path.exists(model_file_safetensors): + from safetensors.torch import load_file, safe_open + state_dict = load_file(model_file_safetensors) + else: + from safetensors.torch import load_file, safe_open + model_files_safetensors = glob.glob(os.path.join(pretrained_model_path, "*.safetensors")) + state_dict = {} + for _model_file_safetensors in model_files_safetensors: + _state_dict = load_file(_model_file_safetensors) + for key in _state_dict: + state_dict[key] = _state_dict[key] + + tmp_state_dict = {} + for key in state_dict: + if key in model.state_dict().keys() and model.state_dict()[key].size() == state_dict[key].size(): + tmp_state_dict[key] = state_dict[key] + else: + print(key, "Size don't match, skip") + + for key in model.state_dict(): + if "control" in key and key.replace("control_", "") in state_dict.keys() and model.state_dict()[key].size() == state_dict[key.replace("control_", "")].size(): + tmp_state_dict[key] = state_dict[key.replace("control_", "")].clone() + print(f"Initializing missing parameter '{key}' with model.state_dict().") + + state_dict = tmp_state_dict + + m, u = model.load_state_dict(state_dict, strict=False) + print(f"### missing keys: {len(m)}; \n### unexpected keys: {len(u)};") + print(m) + + params = [p.numel() if "." in n else 0 for n, p in model.named_parameters()] + print(f"### All Parameters: {sum(params) / 1e6} M") + + params = [p.numel() if "attn1." in n else 0 for n, p in model.named_parameters()] + print(f"### attn1 Parameters: {sum(params) / 1e6} M") + + model = model.to(torch_dtype) + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/models/z_image_transformer2d_control.py b/VideoX-Fun/videox_fun/models/z_image_transformer2d_control.py new file mode 100644 index 0000000000000000000000000000000000000000..d578ef7e898c7d2a198183292a3d000820f0f5fa --- /dev/null +++ b/VideoX-Fun/videox_fun/models/z_image_transformer2d_control.py @@ -0,0 +1,551 @@ +# Modified from https://github.com/ali-vilab/VACE/blob/main/vace/models/wan/wan_vace.py +# -*- coding: utf-8 -*- +# Copyright (c) Alibaba, Inc. and its affiliates. + +from typing import Any, Dict, List, Optional, Tuple + +import torch +import torch.nn as nn +from diffusers.configuration_utils import register_to_config +from diffusers.models.modeling_outputs import Transformer2DModelOutput +from diffusers.utils import (USE_PEFT_BACKEND, is_torch_version, + scale_lora_layers, unscale_lora_layers) +import glob +import inspect +import json +import os +import math +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.nn.functional as F + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.loaders import FromOriginalModelMixin, PeftAdapterMixin +from diffusers.models.attention_processor import Attention +from diffusers.models.modeling_utils import ModelMixin +from diffusers.models.normalization import RMSNorm +from diffusers.utils.torch_utils import maybe_allow_in_graph +from diffusers.models.attention_processor import Attention, AttentionProcessor +from diffusers.utils import (USE_PEFT_BACKEND, is_torch_version, logging, + scale_lora_layers, unscale_lora_layers) +from .z_image_transformer2d import (ZImageTransformer2DModel, FinalLayer, + ZImageTransformerBlock, pad_stack) + + +ADALN_EMBED_DIM = 256 +SEQ_MULTI_OF = 32 + + +class ZImageControlTransformerBlock(ZImageTransformerBlock): + def __init__( + self, + layer_id: int, + dim: int, + n_heads: int, + n_kv_heads: int, + norm_eps: float, + qk_norm: bool, + modulation=True, + block_id=0 + ): + super().__init__(layer_id, dim, n_heads, n_kv_heads, norm_eps, qk_norm, modulation) + self.block_id = block_id + if block_id == 0: + self.before_proj = nn.Linear(self.dim, self.dim) + nn.init.zeros_(self.before_proj.weight) + nn.init.zeros_(self.before_proj.bias) + self.after_proj = nn.Linear(self.dim, self.dim) + nn.init.zeros_(self.after_proj.weight) + nn.init.zeros_(self.after_proj.bias) + + def forward(self, c, x, **kwargs): + if self.block_id == 0: + c = self.before_proj(c) + x + all_c = [] + else: + all_c = list(torch.unbind(c)) + c = all_c.pop(-1) + + c = super().forward(c, **kwargs) + c_skip = self.after_proj(c) + all_c += [c_skip, c] + c = torch.stack(all_c) + return c + + +class BaseZImageTransformerBlock(ZImageTransformerBlock): + def __init__( + self, + layer_id: int, + dim: int, + n_heads: int, + n_kv_heads: int, + norm_eps: float, + qk_norm: bool, + modulation=True, + block_id=0 + ): + super().__init__(layer_id, dim, n_heads, n_kv_heads, norm_eps, qk_norm, modulation) + self.block_id = block_id + + def forward(self, hidden_states, hints=None, context_scale=1.0, **kwargs): + hidden_states = super().forward(hidden_states, **kwargs) + if self.block_id is not None: + hidden_states = hidden_states + hints[self.block_id] * context_scale + return hidden_states + +class ZImageControlTransformer2DModel(ZImageTransformer2DModel): + @register_to_config + def __init__( + self, + control_layers_places=None, + control_in_dim=None, + all_patch_size=(2,), + all_f_patch_size=(1,), + in_channels=16, + dim=3840, + n_layers=30, + n_refiner_layers=2, + n_heads=30, + n_kv_heads=30, + norm_eps=1e-5, + qk_norm=True, + cap_feat_dim=2560, + rope_theta=256.0, + t_scale=1000.0, + axes_dims=[32, 48, 48], + axes_lens=[1024, 512, 512], + ): + super().__init__( + all_patch_size=all_patch_size, + all_f_patch_size=all_f_patch_size, + in_channels=in_channels, + dim=dim, + n_layers=n_layers, + n_refiner_layers=n_refiner_layers, + n_heads=n_heads, + n_kv_heads=n_kv_heads, + norm_eps=norm_eps, + qk_norm=qk_norm, + cap_feat_dim=cap_feat_dim, + rope_theta=rope_theta, + t_scale=t_scale, + axes_dims=axes_dims, + axes_lens=axes_lens, + ) + + self.control_layers_places = [i for i in range(0, self.num_layers, 2)] if control_layers_places is None else control_layers_places + self.control_in_dim = self.in_dim if control_in_dim is None else control_in_dim + + assert 0 in self.control_layers_places + self.control_layers_mapping = {i: n for n, i in enumerate(self.control_layers_places)} + + # blocks + del self.layers + self.layers = nn.ModuleList( + [ + BaseZImageTransformerBlock( + i, + dim, + n_heads, + n_kv_heads, + norm_eps, + qk_norm, + block_id=self.control_layers_mapping[i] if i in self.control_layers_places else None + ) + for i in range(n_layers) + ] + ) + + # control blocks + self.control_layers = nn.ModuleList( + [ + ZImageControlTransformerBlock( + i, + dim, + n_heads, + n_kv_heads, + norm_eps, + qk_norm, + block_id=i + ) + for i in self.control_layers_places + ] + ) + + # control patch embeddings + all_x_embedder = {} + for patch_idx, (patch_size, f_patch_size) in enumerate(zip(all_patch_size, all_f_patch_size)): + x_embedder = nn.Linear(f_patch_size * patch_size * patch_size * self.control_in_dim, dim, bias=True) + all_x_embedder[f"{patch_size}-{f_patch_size}"] = x_embedder + + self.control_all_x_embedder = nn.ModuleDict(all_x_embedder) + self.control_noise_refiner = nn.ModuleList( + [ + ZImageTransformerBlock( + 1000 + layer_id, + dim, + n_heads, + n_kv_heads, + norm_eps, + qk_norm, + modulation=True, + ) + for layer_id in range(n_refiner_layers) + ] + ) + + def forward_control( + self, + x, + cap_feats, + control_context, + kwargs, + t=None, + patch_size=2, + f_patch_size=1, + ): + # embeddings + bsz = len(control_context) + device = control_context[0].device + ( + control_context, + x_size, + x_pos_ids, + x_inner_pad_mask, + ) = self.patchify(control_context, patch_size, f_patch_size, cap_feats[0].size(0)) + + # control_context embed & refine + x_item_seqlens = [len(_) for _ in control_context] + assert all(_ % SEQ_MULTI_OF == 0 for _ in x_item_seqlens) + x_max_item_seqlen = max(x_item_seqlens) + + control_context = torch.cat(control_context, dim=0) + control_context = self.control_all_x_embedder[f"{patch_size}-{f_patch_size}"](control_context) + + # Match t_embedder output dtype to control_context for layerwise casting compatibility + adaln_input = t.type_as(control_context) + ################################################## + # 替换原来的 control_context[torch.cat(x_inner_pad_mask)] = self.x_pad_token + mask = torch.cat(x_inner_pad_mask) + if torch.onnx.is_in_onnx_export(): + # ONNX 导出模式：使用 torch.where 避免 ScatterND + # 1. 确保 x_pad_token 形状正确 + if self.x_pad_token.dim() == 1: + x_pad_token_2d = self.x_pad_token.unsqueeze(0) # [1, 3840] + else: + x_pad_token_2d = self.x_pad_token + + # 2. 扩展 mask 到正确的形状 + mask_2d = mask.unsqueeze(1) # [6720, 1] + + # 3. 扩展 x_pad_token 到与 control_context 相同的形状 + x_pad_expanded = x_pad_token_2d.expand_as(control_context) # [6720, 3840] + + # 4. 使用 torch.where + control_context = torch.where(mask_2d, x_pad_expanded, control_context) + else: + # 原始 PyTorch 代码 + control_context[mask] = self.x_pad_token + ################################################## + control_context = list(control_context.split(x_item_seqlens, dim=0)) + x_freqs_cis = list(self.rope_embedder(torch.cat(x_pos_ids, dim=0)).split(x_item_seqlens, dim=0)) + + control_context = pad_stack(control_context, x_max_item_seqlen, pad_value=0.0) + x_freqs_cis = pad_stack(x_freqs_cis, x_max_item_seqlen, pad_value=0.0) + x_attn_mask = torch.zeros((bsz, x_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(x_item_seqlens): + x_attn_mask[i, :seq_len] = 1 + + if torch.is_grad_enabled() and self.gradient_checkpointing: + for layer in self.control_noise_refiner: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + control_context = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer), + control_context, x_attn_mask, x_freqs_cis, adaln_input, + **ckpt_kwargs, + ) + else: + for layer in self.control_noise_refiner: + control_context = layer(control_context, x_attn_mask, x_freqs_cis, adaln_input) + + # Context Parallel + if self.sp_world_size > 1: + control_context = torch.chunk(control_context, self.sp_world_size, dim=1)[self.sp_world_rank] + + # unified + cap_item_seqlens = [len(_) for _ in cap_feats] + control_context_unified = [] + for i in range(bsz): + x_len = x_item_seqlens[i] + cap_len = cap_item_seqlens[i] + control_context_unified.append(torch.cat([control_context[i][:x_len], cap_feats[i][:cap_len]])) + control_context_unified = pad_stack(control_context_unified, control_context_unified[0].shape[0], pad_value=0.0) + c = control_context_unified + + # arguments + new_kwargs = dict(x=x) + new_kwargs.update(kwargs) + + for layer in self.control_layers: + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module, **static_kwargs): + def custom_forward(*inputs): + return module(*inputs, **static_kwargs) + return custom_forward + ckpt_kwargs = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + c = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer, **new_kwargs), + c, + **ckpt_kwargs, + ) + else: + c = layer(c, **new_kwargs) + + hints = torch.unbind(c)[:-1] + return hints + + + def forward( + self, + x: List[torch.Tensor], + t, + cap_feats: List[torch.Tensor], + patch_size=2, + f_patch_size=1, + control_context=None, + control_context_scale=1.0, + ): + # assert patch_size in self.all_patch_size + # assert f_patch_size in self.all_f_patch_size + + bsz = len(x) + device = x[0].device + t = t * self.t_scale + t = self.t_embedder(t) + + ( + x, + cap_feats, + x_size, + x_pos_ids, + cap_pos_ids, + x_inner_pad_mask, + cap_inner_pad_mask, + ) = self.patchify_and_embed(x, cap_feats, patch_size, f_patch_size) + + # x embed & refine + x_item_seqlens = [len(_) for _ in x] + assert all(_ % SEQ_MULTI_OF == 0 for _ in x_item_seqlens) + x_max_item_seqlen = max(x_item_seqlens) + + x = torch.cat(x, dim=0) + x = self.all_x_embedder[f"{patch_size}-{f_patch_size}"](x) + + # Match t_embedder output dtype to x for layerwise casting compatibility + adaln_input = t.type_as(x) + ################################################## + # x[torch.cat(x_inner_pad_mask)] = self.x_pad_token # TODO: 可能会产生 ScatterND 算子, 会导致 ort 报错 + # 替换原来的 x[torch.cat(x_inner_pad_mask)] = self.x_pad_token + mask = torch.cat(x_inner_pad_mask) + # import pdb; pdb.set_trace() + if torch.onnx.is_in_onnx_export(): + # ONNX 导出模式：使用 torch.where 避免 ScatterND + # 1. 确保 x_pad_token 形状正确 + if self.x_pad_token.dim() == 1: + x_pad_token_2d = self.x_pad_token.unsqueeze(0) # [1, 3840] + else: + x_pad_token_2d = self.x_pad_token + + # 2. 扩展 mask 到正确的形状 + mask_2d = mask.unsqueeze(1) # [6720, 1] + + # 3. 扩展 x_pad_token 到与 x 相同的形状 + x_pad_expanded = x_pad_token_2d.expand_as(x) # [6720, 3840] + + # 4. 使用 torch.where + x = torch.where(mask_2d, x_pad_expanded, x) + else: + # 原始 PyTorch 代码 + x[mask] = self.x_pad_token + ################################################## + x = list(x.split(x_item_seqlens, dim=0)) + x_freqs_cis = list(self.rope_embedder(torch.cat(x_pos_ids, dim=0)).split(x_item_seqlens, dim=0)) + + x = pad_stack(x, x_max_item_seqlen, pad_value=0.0) + x_freqs_cis = pad_stack(x_freqs_cis, x_max_item_seqlen, pad_value=0.0) + x_attn_mask = torch.zeros((bsz, x_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(x_item_seqlens): + x_attn_mask[i, :seq_len] = 1 + + if torch.is_grad_enabled() and self.gradient_checkpointing: + for layer in self.noise_refiner: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer), + x, x_attn_mask, x_freqs_cis, adaln_input, + **ckpt_kwargs, + ) + else: + for layer in self.noise_refiner: + x = layer(x, x_attn_mask, x_freqs_cis, adaln_input) + + # cap embed & refine + cap_item_seqlens = [len(_) for _ in cap_feats] + assert all(_ % SEQ_MULTI_OF == 0 for _ in cap_item_seqlens) + cap_max_item_seqlen = max(cap_item_seqlens) + + cap_feats = torch.cat(cap_feats, dim=0) + cap_feats = self.cap_embedder(cap_feats) + + ################################################## + # 替换原来的 cap_feats[torch.cat(cap_inner_pad_mask)] = self.cap_pad_token + mask = torch.cat(cap_inner_pad_mask) + # import pdb; pdb.set_trace() + if torch.onnx.is_in_onnx_export(): + # ONNX 导出模式：使用 torch.where 避免 ScatterND + # 1. 确保 cap_pad_token 形状正确 + if self.cap_pad_token.dim() == 1: + cap_pad_token_2d = self.cap_pad_token.unsqueeze(0) # [1, 3840] + else: + cap_pad_token_2d = self.cap_pad_token + + # 2. 扩展 mask 到正确的形状 + mask_2d = mask.unsqueeze(1) # [6720, 1] + + # 3. 扩展 cap_pad_token 到与 cap_feats 相同的形状 + cap_pad_expanded = cap_pad_token_2d.expand_as(cap_feats) # [6720, 3840] + + # 4. 使用 torch.where + cap_feats = torch.where(mask_2d, cap_pad_expanded, cap_feats) + else: + # 原始 PyTorch 代码 + cap_feats[mask] = self.cap_pad_token + ################################################## + + cap_feats = list(cap_feats.split(cap_item_seqlens, dim=0)) + cap_freqs_cis = list(self.rope_embedder(torch.cat(cap_pos_ids, dim=0)).split(cap_item_seqlens, dim=0)) + + cap_feats = pad_stack(cap_feats, cap_max_item_seqlen, pad_value=0.0) + cap_freqs_cis = pad_stack(cap_freqs_cis, cap_max_item_seqlen, pad_value=0.0) + cap_attn_mask = torch.zeros((bsz, cap_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(cap_item_seqlens): + cap_attn_mask[i, :seq_len] = 1 + + if torch.is_grad_enabled() and self.gradient_checkpointing: + for layer in self.context_refiner: + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + cap_feats = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer), + cap_feats, + cap_attn_mask, + cap_freqs_cis, + **ckpt_kwargs, + ) + else: + for layer in self.context_refiner: + cap_feats = layer(cap_feats, cap_attn_mask, cap_freqs_cis) + + # Context Parallel + if self.sp_world_size > 1: + x = torch.chunk(x, self.sp_world_size, dim=1)[self.sp_world_rank] + + x_item_seqlens = [len(_) for _ in x] + assert all(_ % SEQ_MULTI_OF == 0 for _ in x_item_seqlens) + x_max_item_seqlen = max(x_item_seqlens) + x_attn_mask = torch.zeros((bsz, x_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(x_item_seqlens): + x_attn_mask[i, :seq_len] = 1 + + if x_freqs_cis is not None: + x_freqs_cis = torch.chunk(x_freqs_cis, self.sp_world_size, dim=1)[self.sp_world_rank] + + # unified + unified = [] + unified_freqs_cis = [] + for i in range(bsz): + x_len = x_item_seqlens[i] + cap_len = cap_item_seqlens[i] + unified.append(torch.cat([x[i][:x_len], cap_feats[i][:cap_len]])) + unified_freqs_cis.append(torch.cat([x_freqs_cis[i][:x_len], cap_freqs_cis[i][:cap_len]])) + unified_item_seqlens = [a + b for a, b in zip(cap_item_seqlens, x_item_seqlens)] + assert unified_item_seqlens == [len(_) for _ in unified] + unified_max_item_seqlen = max(unified_item_seqlens) + + unified = pad_stack(unified, unified_max_item_seqlen, pad_value=0.0) + unified_freqs_cis = pad_stack(unified_freqs_cis, unified_max_item_seqlen, pad_value=0.0) + unified_attn_mask = torch.zeros((bsz, unified_max_item_seqlen), dtype=torch.bool, device=device) + for i, seq_len in enumerate(unified_item_seqlens): + unified_attn_mask[i, :seq_len] = 1 + + # Arguments + kwargs = dict( + attn_mask=unified_attn_mask, + freqs_cis=unified_freqs_cis, + adaln_input=adaln_input, + ) + # import pdb; pdb.set_trace() + hints = self.forward_control( + unified, cap_feats, control_context, kwargs, t=t, patch_size=patch_size, f_patch_size=f_patch_size, + ) + + for layer in self.layers: + # Arguments + kwargs = dict( + attn_mask=unified_attn_mask, + freqs_cis=unified_freqs_cis, + adaln_input=adaln_input, + hints=hints, + context_scale=control_context_scale + ) + if torch.is_grad_enabled() and self.gradient_checkpointing: + def create_custom_forward(module, **static_kwargs): + def custom_forward(*inputs): + return module(*inputs, **static_kwargs) + return custom_forward + + ckpt_kwargs = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + + unified = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer, **kwargs), + unified, + **ckpt_kwargs, + ) + else: + unified = layer(unified, **kwargs) + + if self.sp_world_size > 1: + unified_out = [] + for i in range(bsz): + x_len = x_item_seqlens[i] + unified_out.append(unified[i, :x_len]) + unified = torch.stack(unified_out) + unified = self.all_gather(unified, dim=1) + + unified = self.all_final_layer[f"{patch_size}-{f_patch_size}"](unified, adaln_input) + unified = list(unified.unbind(dim=0)) + x = self.unpatchify(unified, x_size, patch_size, f_patch_size) + + x = torch.stack(x) + return x, {} \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_cogvideox_fun.py b/VideoX-Fun/videox_fun/pipeline/pipeline_cogvideox_fun.py new file mode 100644 index 0000000000000000000000000000000000000000..68568a6069ddc30ad4b11bb833b6307b5de3ceb6 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_cogvideox_fun.py @@ -0,0 +1,862 @@ +# Copyright 2024 The CogVideoX team, Tsinghua University & ZhipuAI and The HuggingFace Team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import CogVideoXDDIMScheduler, CogVideoXDPMScheduler +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor + +from ..models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.models.embeddings.get_3d_rotary_pos_embed +def get_3d_rotary_pos_embed( + embed_dim, + crops_coords, + grid_size, + temporal_size, + theta: int = 10000, + use_real: bool = True, + grid_type: str = "linspace", + max_size: Optional[Tuple[int, int]] = None, +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """ + RoPE for video tokens with 3D structure. + + Args: + embed_dim: (`int`): + The embedding dimension size, corresponding to hidden_size_head. + crops_coords (`Tuple[int]`): + The top-left and bottom-right coordinates of the crop. + grid_size (`Tuple[int]`): + The grid size of the spatial positional embedding (height, width). + temporal_size (`int`): + The size of the temporal dimension. + theta (`float`): + Scaling factor for frequency computation. + grid_type (`str`): + Whether to use "linspace" or "slice" to compute grids. + + Returns: + `torch.Tensor`: positional embedding with shape `(temporal_size * grid_size[0] * grid_size[1], embed_dim/2)`. + """ + if use_real is not True: + raise ValueError(" `use_real = False` is not currently supported for get_3d_rotary_pos_embed") + + if grid_type == "linspace": + start, stop = crops_coords + grid_size_h, grid_size_w = grid_size + grid_h = np.linspace(start[0], stop[0], grid_size_h, endpoint=False, dtype=np.float32) + grid_w = np.linspace(start[1], stop[1], grid_size_w, endpoint=False, dtype=np.float32) + grid_t = np.arange(temporal_size, dtype=np.float32) + grid_t = np.linspace(0, temporal_size, temporal_size, endpoint=False, dtype=np.float32) + elif grid_type == "slice": + max_h, max_w = max_size + grid_size_h, grid_size_w = grid_size + grid_h = np.arange(max_h, dtype=np.float32) + grid_w = np.arange(max_w, dtype=np.float32) + grid_t = np.arange(temporal_size, dtype=np.float32) + else: + raise ValueError("Invalid value passed for `grid_type`.") + + # Compute dimensions for each axis + dim_t = embed_dim // 4 + dim_h = embed_dim // 8 * 3 + dim_w = embed_dim // 8 * 3 + + # Temporal frequencies + freqs_t = get_1d_rotary_pos_embed(dim_t, grid_t, use_real=True) + # Spatial frequencies for height and width + freqs_h = get_1d_rotary_pos_embed(dim_h, grid_h, use_real=True) + freqs_w = get_1d_rotary_pos_embed(dim_w, grid_w, use_real=True) + + # BroadCast and concatenate temporal and spaial frequencie (height and width) into a 3d tensor + def combine_time_height_width(freqs_t, freqs_h, freqs_w): + freqs_t = freqs_t[:, None, None, :].expand( + -1, grid_size_h, grid_size_w, -1 + ) # temporal_size, grid_size_h, grid_size_w, dim_t + freqs_h = freqs_h[None, :, None, :].expand( + temporal_size, -1, grid_size_w, -1 + ) # temporal_size, grid_size_h, grid_size_2, dim_h + freqs_w = freqs_w[None, None, :, :].expand( + temporal_size, grid_size_h, -1, -1 + ) # temporal_size, grid_size_h, grid_size_2, dim_w + + freqs = torch.cat( + [freqs_t, freqs_h, freqs_w], dim=-1 + ) # temporal_size, grid_size_h, grid_size_w, (dim_t + dim_h + dim_w) + freqs = freqs.view( + temporal_size * grid_size_h * grid_size_w, -1 + ) # (temporal_size * grid_size_h * grid_size_w), (dim_t + dim_h + dim_w) + return freqs + + t_cos, t_sin = freqs_t # both t_cos and t_sin has shape: temporal_size, dim_t + h_cos, h_sin = freqs_h # both h_cos and h_sin has shape: grid_size_h, dim_h + w_cos, w_sin = freqs_w # both w_cos and w_sin has shape: grid_size_w, dim_w + + if grid_type == "slice": + t_cos, t_sin = t_cos[:temporal_size], t_sin[:temporal_size] + h_cos, h_sin = h_cos[:grid_size_h], h_sin[:grid_size_h] + w_cos, w_sin = w_cos[:grid_size_w], w_sin[:grid_size_w] + + cos = combine_time_height_width(t_cos, h_cos, w_cos) + sin = combine_time_height_width(t_sin, h_sin, w_sin) + return cos, sin + + +# Similar to diffusers.pipelines.hunyuandit.pipeline_hunyuandit.get_resize_crop_region_for_grid +def get_resize_crop_region_for_grid(src, tgt_width, tgt_height): + tw = tgt_width + th = tgt_height + h, w = src + r = h / w + if r > (th / tw): + resize_height = th + resize_width = int(round(th / h * w)) + else: + resize_width = tw + resize_height = int(round(tw / w * h)) + + crop_top = int(round((th - resize_height) / 2.0)) + crop_left = int(round((tw - resize_width) / 2.0)) + + return (crop_top, crop_left), (crop_top + resize_height, crop_left + resize_width) + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class CogVideoXFunPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class CogVideoXFunPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using CogVideoX_Fun. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations. + text_encoder ([`T5EncoderModel`]): + Frozen text-encoder. CogVideoX uses + [T5](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5EncoderModel); specifically the + [t5-v1_1-xxl](https://huggingface.co/PixArt-alpha/PixArt-alpha/tree/main/t5-v1_1-xxl) variant. + tokenizer (`T5Tokenizer`): + Tokenizer of class + [T5Tokenizer](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5Tokenizer). + transformer ([`CogVideoXTransformer3DModel`]): + A text conditioned `CogVideoXTransformer3DModel` to denoise the encoded video latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `transformer` to denoise the encoded video latents. + """ + + _optional_components = [] + model_cpu_offload_seq = "text_encoder->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + vae: AutoencoderKLCogVideoX, + transformer: CogVideoXTransformer3DModel, + scheduler: Union[CogVideoXDDIMScheduler, CogVideoXDPMScheduler], + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, scheduler=scheduler + ) + self.vae_scale_factor_spatial = ( + 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8 + ) + self.vae_scale_factor_temporal = ( + self.vae.config.temporal_compression_ratio if hasattr(self, "vae") and self.vae is not None else 4 + ) + + self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 226, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder(text_input_ids.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return prompt_embeds + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 226, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + (num_frames - 1) // self.vae_scale_factor_temporal + 1, + num_channels_latents, + height // self.vae_scale_factor_spatial, + width // self.vae_scale_factor_spatial, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + latents = latents.permute(0, 2, 1, 3, 4) # [batch_size, num_channels, num_frames, height, width] + latents = 1 / self.vae.config.scaling_factor * latents + + frames = self.vae.decode(latents).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def fuse_qkv_projections(self) -> None: + r"""Enables fused QKV projections.""" + self.fusing_transformer = True + self.transformer.fuse_qkv_projections() + + def unfuse_qkv_projections(self) -> None: + r"""Disable QKV projection fusion if enabled.""" + if not self.fusing_transformer: + logger.warning("The Transformer was not initially fused for QKV projections. Doing nothing.") + else: + self.transformer.unfuse_qkv_projections() + self.fusing_transformer = False + + def _prepare_rotary_positional_embeddings( + self, + height: int, + width: int, + num_frames: int, + device: torch.device, + ) -> Tuple[torch.Tensor, torch.Tensor]: + grid_height = height // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) + grid_width = width // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) + + p = self.transformer.config.patch_size + p_t = self.transformer.config.patch_size_t + + base_size_width = self.transformer.config.sample_width // p + base_size_height = self.transformer.config.sample_height // p + + if p_t is None: + # CogVideoX 1.0 + grid_crops_coords = get_resize_crop_region_for_grid( + (grid_height, grid_width), base_size_width, base_size_height + ) + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=self.transformer.config.attention_head_dim, + crops_coords=grid_crops_coords, + grid_size=(grid_height, grid_width), + temporal_size=num_frames, + ) + else: + # CogVideoX 1.5 + base_num_frames = (num_frames + p_t - 1) // p_t + + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=self.transformer.config.attention_head_dim, + crops_coords=None, + grid_size=(grid_height, grid_width), + temporal_size=base_num_frames, + grid_type="slice", + max_size=(base_size_height, base_size_width), + ) + + freqs_cos = freqs_cos.to(device=device) + freqs_sin = freqs_sin.to(device=device) + return freqs_cos, freqs_sin + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + use_dynamic_cfg: bool = False, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 226, + ) -> Union[CogVideoXFunPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + num_frames (`int`, defaults to `48`): + Number of frames to generate. Must be divisible by self.vae_scale_factor_temporal. Generated video will + contain 1 extra frame because CogVideoX is conditioned with (num_seconds * fps + 1) frames where + num_seconds is 6 and fps is 4. However, since videos can be saved at any fps, the only condition that + needs to be satisfied is that of divisibility mentioned above. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of videos to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int`, defaults to `226`): + Maximum sequence length in encoded prompt. Must be consistent with + `self.transformer.config.max_text_seq_length` otherwise may lead to poor results. + + Examples: + + Returns: + [`~pipelines.cogvideo.pipeline_cogvideox.CogVideoXFunPipelineOutput`] or `tuple`: + [`~pipelines.cogvideo.pipeline_cogvideox.CogVideoXFunPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + + height = height or self.transformer.config.sample_height * self.vae_scale_factor_spatial + width = width or self.transformer.config.sample_width * self.vae_scale_factor_spatial + num_frames = num_frames or self.transformer.config.sample_frames + + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + + # 5. Prepare latents + latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 + + # For CogVideoX 1.5, the latent frames should be padded to make it divisible by patch_size_t + patch_size_t = self.transformer.config.patch_size_t + additional_frames = 0 + if num_frames != 1 and patch_size_t is not None and latent_frames % patch_size_t != 0: + additional_frames = patch_size_t - latent_frames % patch_size_t + num_frames += additional_frames * self.vae_scale_factor_temporal + + latent_channels = self.transformer.config.in_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Create rotary embeds if required + image_rotary_emb = ( + self._prepare_rotary_positional_embeddings(height, width, latents.size(1), device) + if self.transformer.config.use_rotary_positional_embeddings + else None + ) + + # 8. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + # for DPM-solver++ + old_pred_original_sample = None + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + # predict noise model_output + noise_pred = self.transformer( + hidden_states=latent_model_input, + encoder_hidden_states=prompt_embeds, + timestep=timestep, + image_rotary_emb=image_rotary_emb, + return_dict=False, + )[0] + noise_pred = noise_pred.float() + + # perform guidance + if use_dynamic_cfg: + self._guidance_scale = 1 + guidance_scale * ( + (1 - math.cos(math.pi * ((num_inference_steps - t.item()) / num_inference_steps) ** 5.0)) / 2 + ) + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + if not isinstance(self.scheduler, CogVideoXDPMScheduler): + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + else: + latents, old_pred_original_sample = self.scheduler.step( + noise_pred, + old_pred_original_sample, + t, + timesteps[i - 1] if i > 0 else None, + latents, + **extra_step_kwargs, + return_dict=False, + ) + latents = latents.to(prompt_embeds.dtype) + + # call the callback, if provided + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return CogVideoXFunPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_cogvideox_fun_control.py b/VideoX-Fun/videox_fun/pipeline/pipeline_cogvideox_fun_control.py new file mode 100644 index 0000000000000000000000000000000000000000..e91df20dab9553896394a42af3e698dfea33f333 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_cogvideox_fun_control.py @@ -0,0 +1,956 @@ +# Copyright 2024 The CogVideoX team, Tsinghua University & ZhipuAI and The HuggingFace Team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import (get_1d_rotary_pos_embed, + get_3d_rotary_pos_embed) +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import CogVideoXDDIMScheduler, CogVideoXDPMScheduler +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange + +from ..models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.models.embeddings.get_3d_rotary_pos_embed +def get_3d_rotary_pos_embed( + embed_dim, + crops_coords, + grid_size, + temporal_size, + theta: int = 10000, + use_real: bool = True, + grid_type: str = "linspace", + max_size: Optional[Tuple[int, int]] = None, +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """ + RoPE for video tokens with 3D structure. + + Args: + embed_dim: (`int`): + The embedding dimension size, corresponding to hidden_size_head. + crops_coords (`Tuple[int]`): + The top-left and bottom-right coordinates of the crop. + grid_size (`Tuple[int]`): + The grid size of the spatial positional embedding (height, width). + temporal_size (`int`): + The size of the temporal dimension. + theta (`float`): + Scaling factor for frequency computation. + grid_type (`str`): + Whether to use "linspace" or "slice" to compute grids. + + Returns: + `torch.Tensor`: positional embedding with shape `(temporal_size * grid_size[0] * grid_size[1], embed_dim/2)`. + """ + if use_real is not True: + raise ValueError(" `use_real = False` is not currently supported for get_3d_rotary_pos_embed") + + if grid_type == "linspace": + start, stop = crops_coords + grid_size_h, grid_size_w = grid_size + grid_h = np.linspace(start[0], stop[0], grid_size_h, endpoint=False, dtype=np.float32) + grid_w = np.linspace(start[1], stop[1], grid_size_w, endpoint=False, dtype=np.float32) + grid_t = np.arange(temporal_size, dtype=np.float32) + grid_t = np.linspace(0, temporal_size, temporal_size, endpoint=False, dtype=np.float32) + elif grid_type == "slice": + max_h, max_w = max_size + grid_size_h, grid_size_w = grid_size + grid_h = np.arange(max_h, dtype=np.float32) + grid_w = np.arange(max_w, dtype=np.float32) + grid_t = np.arange(temporal_size, dtype=np.float32) + else: + raise ValueError("Invalid value passed for `grid_type`.") + + # Compute dimensions for each axis + dim_t = embed_dim // 4 + dim_h = embed_dim // 8 * 3 + dim_w = embed_dim // 8 * 3 + + # Temporal frequencies + freqs_t = get_1d_rotary_pos_embed(dim_t, grid_t, use_real=True) + # Spatial frequencies for height and width + freqs_h = get_1d_rotary_pos_embed(dim_h, grid_h, use_real=True) + freqs_w = get_1d_rotary_pos_embed(dim_w, grid_w, use_real=True) + + # BroadCast and concatenate temporal and spaial frequencie (height and width) into a 3d tensor + def combine_time_height_width(freqs_t, freqs_h, freqs_w): + freqs_t = freqs_t[:, None, None, :].expand( + -1, grid_size_h, grid_size_w, -1 + ) # temporal_size, grid_size_h, grid_size_w, dim_t + freqs_h = freqs_h[None, :, None, :].expand( + temporal_size, -1, grid_size_w, -1 + ) # temporal_size, grid_size_h, grid_size_2, dim_h + freqs_w = freqs_w[None, None, :, :].expand( + temporal_size, grid_size_h, -1, -1 + ) # temporal_size, grid_size_h, grid_size_2, dim_w + + freqs = torch.cat( + [freqs_t, freqs_h, freqs_w], dim=-1 + ) # temporal_size, grid_size_h, grid_size_w, (dim_t + dim_h + dim_w) + freqs = freqs.view( + temporal_size * grid_size_h * grid_size_w, -1 + ) # (temporal_size * grid_size_h * grid_size_w), (dim_t + dim_h + dim_w) + return freqs + + t_cos, t_sin = freqs_t # both t_cos and t_sin has shape: temporal_size, dim_t + h_cos, h_sin = freqs_h # both h_cos and h_sin has shape: grid_size_h, dim_h + w_cos, w_sin = freqs_w # both w_cos and w_sin has shape: grid_size_w, dim_w + + if grid_type == "slice": + t_cos, t_sin = t_cos[:temporal_size], t_sin[:temporal_size] + h_cos, h_sin = h_cos[:grid_size_h], h_sin[:grid_size_h] + w_cos, w_sin = w_cos[:grid_size_w], w_sin[:grid_size_w] + + cos = combine_time_height_width(t_cos, h_cos, w_cos) + sin = combine_time_height_width(t_sin, h_sin, w_sin) + return cos, sin + + +# Similar to diffusers.pipelines.hunyuandit.pipeline_hunyuandit.get_resize_crop_region_for_grid +def get_resize_crop_region_for_grid(src, tgt_width, tgt_height): + tw = tgt_width + th = tgt_height + h, w = src + r = h / w + if r > (th / tw): + resize_height = th + resize_width = int(round(th / h * w)) + else: + resize_width = tw + resize_height = int(round(tw / w * h)) + + crop_top = int(round((th - resize_height) / 2.0)) + crop_left = int(round((tw - resize_width) / 2.0)) + + return (crop_top, crop_left), (crop_top + resize_height, crop_left + resize_width) + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class CogVideoXFunPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class CogVideoXFunControlPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using CogVideoX. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations. + text_encoder ([`T5EncoderModel`]): + Frozen text-encoder. CogVideoX_Fun uses + [T5](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5EncoderModel); specifically the + [t5-v1_1-xxl](https://huggingface.co/PixArt-alpha/PixArt-alpha/tree/main/t5-v1_1-xxl) variant. + tokenizer (`T5Tokenizer`): + Tokenizer of class + [T5Tokenizer](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5Tokenizer). + transformer ([`CogVideoXTransformer3DModel`]): + A text conditioned `CogVideoXTransformer3DModel` to denoise the encoded video latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `transformer` to denoise the encoded video latents. + """ + + _optional_components = [] + model_cpu_offload_seq = "text_encoder->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + vae: AutoencoderKLCogVideoX, + transformer: CogVideoXTransformer3DModel, + scheduler: Union[CogVideoXDDIMScheduler, CogVideoXDPMScheduler], + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, scheduler=scheduler + ) + self.vae_scale_factor_spatial = ( + 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8 + ) + self.vae_scale_factor_temporal = ( + self.vae.config.temporal_compression_ratio if hasattr(self, "vae") and self.vae is not None else 4 + ) + + self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) + + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 226, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder(text_input_ids.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return prompt_embeds + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 226, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + shape = ( + batch_size, + (num_frames - 1) // self.vae_scale_factor_temporal + 1, + num_channels_latents, + height // self.vae_scale_factor_spatial, + width // self.vae_scale_factor_spatial, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_control_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if mask is not None: + mask = mask.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask = [] + for i in range(0, mask.shape[0], bs): + mask_bs = mask[i : i + bs] + mask_bs = self.vae.encode(mask_bs)[0] + mask_bs = mask_bs.mode() + new_mask.append(mask_bs) + mask = torch.cat(new_mask, dim = 0) + mask = mask * self.vae.config.scaling_factor + + if masked_image is not None: + masked_image = masked_image.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask_pixel_values = [] + for i in range(0, masked_image.shape[0], bs): + mask_pixel_values_bs = masked_image[i : i + bs] + mask_pixel_values_bs = self.vae.encode(mask_pixel_values_bs)[0] + mask_pixel_values_bs = mask_pixel_values_bs.mode() + new_mask_pixel_values.append(mask_pixel_values_bs) + masked_image_latents = torch.cat(new_mask_pixel_values, dim = 0) + masked_image_latents = masked_image_latents * self.vae.config.scaling_factor + else: + masked_image_latents = None + + return mask, masked_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + latents = latents.permute(0, 2, 1, 3, 4) # [batch_size, num_channels, num_frames, height, width] + latents = 1 / self.vae.config.scaling_factor * latents + + frames = self.vae.decode(latents).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def fuse_qkv_projections(self) -> None: + r"""Enables fused QKV projections.""" + self.fusing_transformer = True + self.transformer.fuse_qkv_projections() + + def unfuse_qkv_projections(self) -> None: + r"""Disable QKV projection fusion if enabled.""" + if not self.fusing_transformer: + logger.warning("The Transformer was not initially fused for QKV projections. Doing nothing.") + else: + self.transformer.unfuse_qkv_projections() + self.fusing_transformer = False + + def _prepare_rotary_positional_embeddings( + self, + height: int, + width: int, + num_frames: int, + device: torch.device, + ) -> Tuple[torch.Tensor, torch.Tensor]: + grid_height = height // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) + grid_width = width // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) + + p = self.transformer.config.patch_size + p_t = self.transformer.config.patch_size_t + + base_size_width = self.transformer.config.sample_width // p + base_size_height = self.transformer.config.sample_height // p + + if p_t is None: + # CogVideoX 1.0 + grid_crops_coords = get_resize_crop_region_for_grid( + (grid_height, grid_width), base_size_width, base_size_height + ) + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=self.transformer.config.attention_head_dim, + crops_coords=grid_crops_coords, + grid_size=(grid_height, grid_width), + temporal_size=num_frames, + ) + else: + # CogVideoX 1.5 + base_num_frames = (num_frames + p_t - 1) // p_t + + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=self.transformer.config.attention_head_dim, + crops_coords=None, + grid_size=(grid_height, grid_width), + temporal_size=base_num_frames, + grid_type="slice", + max_size=(base_size_height, base_size_width), + ) + + freqs_cos = freqs_cos.to(device=device) + freqs_sin = freqs_sin.to(device=device) + return freqs_cos, freqs_sin + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + + return timesteps, num_inference_steps - t_start + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + video: Union[torch.FloatTensor] = None, + control_video: Union[torch.FloatTensor] = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + use_dynamic_cfg: bool = False, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 226, + comfyui_progressbar: bool = False, + ) -> Union[CogVideoXFunPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + num_frames (`int`, defaults to `48`): + Number of frames to generate. Must be divisible by self.vae_scale_factor_temporal. Generated video will + contain 1 extra frame because CogVideoX_Fun is conditioned with (num_seconds * fps + 1) frames where + num_seconds is 6 and fps is 4. However, since videos can be saved at any fps, the only condition that + needs to be satisfied is that of divisibility mentioned above. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of videos to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int`, defaults to `226`): + Maximum sequence length in encoded prompt. Must be consistent with + `self.transformer.config.max_text_seq_length` otherwise may lead to poor results. + + Examples: + + Returns: + [`~pipelines.cogvideo.pipeline_cogvideox.CogVideoXFunPipelineOutput`] or `tuple`: + [`~pipelines.cogvideo.pipeline_cogvideox.CogVideoXFunPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + + height = height or self.transformer.config.sample_height * self.vae_scale_factor_spatial + width = width or self.transformer.config.sample_width * self.vae_scale_factor_spatial + num_frames = num_frames or self.transformer.config.sample_frames + + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + if control_video is not None: + video_length = control_video.shape[2] + control_video = self.image_processor.preprocess(rearrange(control_video, "b c f h w -> (b f) c h w"), height=height, width=width) + control_video = control_video.to(dtype=torch.float32) + control_video = rearrange(control_video, "(b f) c h w -> b c f h w", f=video_length) + else: + control_video = None + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (num_frames - 1) // self.vae_scale_factor_temporal + 1 + # For CogVideoX 1.5, the latent frames should be clipped to make it divisible by patch_size_t + patch_size_t = self.transformer.config.patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + num_frames -= additional_frames * self.vae_scale_factor_temporal + if num_frames <= 0: + num_frames = 1 + if video_length > num_frames: + logger.warning("The length of condition video is not right, the latent frames should be clipped to make it divisible by patch_size_t. ") + video_length = num_frames + control_video = control_video[:, :, :video_length] + + # 5. Prepare latents. + latent_channels = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + if comfyui_progressbar: + pbar.update(1) + + control_video_latents = self.prepare_control_latents( + None, + control_video, + batch_size, + height, + width, + prompt_embeds.dtype, + device, + generator, + do_classifier_free_guidance + )[1] + control_video_latents_input = ( + torch.cat([control_video_latents] * 2) if do_classifier_free_guidance else control_video_latents + ) + control_latents = rearrange(control_video_latents_input, "b c f h w -> b f c h w") + + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Create rotary embeds if required + image_rotary_emb = ( + self._prepare_rotary_positional_embeddings(height, width, latents.size(1), device) + if self.transformer.config.use_rotary_positional_embeddings + else None + ) + + # 8. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + # for DPM-solver++ + old_pred_original_sample = None + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + # predict noise model_output + noise_pred = self.transformer( + hidden_states=latent_model_input, + encoder_hidden_states=prompt_embeds, + timestep=timestep, + image_rotary_emb=image_rotary_emb, + return_dict=False, + control_latents=control_latents, + )[0] + noise_pred = noise_pred.float() + + # perform guidance + if use_dynamic_cfg: + self._guidance_scale = 1 + guidance_scale * ( + (1 - math.cos(math.pi * ((num_inference_steps - t.item()) / num_inference_steps) ** 5.0)) / 2 + ) + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + if not isinstance(self.scheduler, CogVideoXDPMScheduler): + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + else: + latents, old_pred_original_sample = self.scheduler.step( + noise_pred, + old_pred_original_sample, + t, + timesteps[i - 1] if i > 0 else None, + latents, + **extra_step_kwargs, + return_dict=False, + ) + latents = latents.to(prompt_embeds.dtype) + + # call the callback, if provided + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return CogVideoXFunPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_cogvideox_fun_inpaint.py b/VideoX-Fun/videox_fun/pipeline/pipeline_cogvideox_fun_inpaint.py new file mode 100644 index 0000000000000000000000000000000000000000..7044d9d034079b18d862c9d80df631204e30569a --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_cogvideox_fun_inpaint.py @@ -0,0 +1,1136 @@ +# Copyright 2024 The CogVideoX team, Tsinghua University & ZhipuAI and The HuggingFace Team. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import CogVideoXDDIMScheduler, CogVideoXDPMScheduler +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange + +from ..models import (AutoencoderKLCogVideoX, + CogVideoXTransformer3DModel, T5EncoderModel, + T5Tokenizer) + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + +# Copied from diffusers.models.embeddings.get_3d_rotary_pos_embed +def get_3d_rotary_pos_embed( + embed_dim, + crops_coords, + grid_size, + temporal_size, + theta: int = 10000, + use_real: bool = True, + grid_type: str = "linspace", + max_size: Optional[Tuple[int, int]] = None, +) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]: + """ + RoPE for video tokens with 3D structure. + + Args: + embed_dim: (`int`): + The embedding dimension size, corresponding to hidden_size_head. + crops_coords (`Tuple[int]`): + The top-left and bottom-right coordinates of the crop. + grid_size (`Tuple[int]`): + The grid size of the spatial positional embedding (height, width). + temporal_size (`int`): + The size of the temporal dimension. + theta (`float`): + Scaling factor for frequency computation. + grid_type (`str`): + Whether to use "linspace" or "slice" to compute grids. + + Returns: + `torch.Tensor`: positional embedding with shape `(temporal_size * grid_size[0] * grid_size[1], embed_dim/2)`. + """ + if use_real is not True: + raise ValueError(" `use_real = False` is not currently supported for get_3d_rotary_pos_embed") + + if grid_type == "linspace": + start, stop = crops_coords + grid_size_h, grid_size_w = grid_size + grid_h = np.linspace(start[0], stop[0], grid_size_h, endpoint=False, dtype=np.float32) + grid_w = np.linspace(start[1], stop[1], grid_size_w, endpoint=False, dtype=np.float32) + grid_t = np.arange(temporal_size, dtype=np.float32) + grid_t = np.linspace(0, temporal_size, temporal_size, endpoint=False, dtype=np.float32) + elif grid_type == "slice": + max_h, max_w = max_size + grid_size_h, grid_size_w = grid_size + grid_h = np.arange(max_h, dtype=np.float32) + grid_w = np.arange(max_w, dtype=np.float32) + grid_t = np.arange(temporal_size, dtype=np.float32) + else: + raise ValueError("Invalid value passed for `grid_type`.") + + # Compute dimensions for each axis + dim_t = embed_dim // 4 + dim_h = embed_dim // 8 * 3 + dim_w = embed_dim // 8 * 3 + + # Temporal frequencies + freqs_t = get_1d_rotary_pos_embed(dim_t, grid_t, use_real=True) + # Spatial frequencies for height and width + freqs_h = get_1d_rotary_pos_embed(dim_h, grid_h, use_real=True) + freqs_w = get_1d_rotary_pos_embed(dim_w, grid_w, use_real=True) + + # BroadCast and concatenate temporal and spaial frequencie (height and width) into a 3d tensor + def combine_time_height_width(freqs_t, freqs_h, freqs_w): + freqs_t = freqs_t[:, None, None, :].expand( + -1, grid_size_h, grid_size_w, -1 + ) # temporal_size, grid_size_h, grid_size_w, dim_t + freqs_h = freqs_h[None, :, None, :].expand( + temporal_size, -1, grid_size_w, -1 + ) # temporal_size, grid_size_h, grid_size_2, dim_h + freqs_w = freqs_w[None, None, :, :].expand( + temporal_size, grid_size_h, -1, -1 + ) # temporal_size, grid_size_h, grid_size_2, dim_w + + freqs = torch.cat( + [freqs_t, freqs_h, freqs_w], dim=-1 + ) # temporal_size, grid_size_h, grid_size_w, (dim_t + dim_h + dim_w) + freqs = freqs.view( + temporal_size * grid_size_h * grid_size_w, -1 + ) # (temporal_size * grid_size_h * grid_size_w), (dim_t + dim_h + dim_w) + return freqs + + t_cos, t_sin = freqs_t # both t_cos and t_sin has shape: temporal_size, dim_t + h_cos, h_sin = freqs_h # both h_cos and h_sin has shape: grid_size_h, dim_h + w_cos, w_sin = freqs_w # both w_cos and w_sin has shape: grid_size_w, dim_w + + if grid_type == "slice": + t_cos, t_sin = t_cos[:temporal_size], t_sin[:temporal_size] + h_cos, h_sin = h_cos[:grid_size_h], h_sin[:grid_size_h] + w_cos, w_sin = w_cos[:grid_size_w], w_sin[:grid_size_w] + + cos = combine_time_height_width(t_cos, h_cos, w_cos) + sin = combine_time_height_width(t_sin, h_sin, w_sin) + return cos, sin + + +# Similar to diffusers.pipelines.hunyuandit.pipeline_hunyuandit.get_resize_crop_region_for_grid +def get_resize_crop_region_for_grid(src, tgt_width, tgt_height): + tw = tgt_width + th = tgt_height + h, w = src + r = h / w + if r > (th / tw): + resize_height = th + resize_width = int(round(th / h * w)) + else: + resize_width = tw + resize_height = int(round(tw / w * h)) + + crop_top = int(round((th - resize_height) / 2.0)) + crop_left = int(round((tw - resize_width) / 2.0)) + + return (crop_top, crop_left), (crop_top + resize_height, crop_left + resize_width) + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +def add_noise_to_reference_video(image, ratio=None): + if ratio is None: + sigma = torch.normal(mean=-3.0, std=0.5, size=(image.shape[0],)).to(image.device) + sigma = torch.exp(sigma).to(image.dtype) + else: + sigma = torch.ones((image.shape[0],)).to(image.device, image.dtype) * ratio + + image_noise = torch.randn_like(image) * sigma[:, None, None, None, None] + image_noise = torch.where(image==-1, torch.zeros_like(image), image_noise) + image = image + image_noise + return image + + +@dataclass +class CogVideoXFunPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class CogVideoXFunInpaintPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using CogVideoX. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations. + text_encoder ([`T5EncoderModel`]): + Frozen text-encoder. CogVideoX_Fun uses + [T5](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5EncoderModel); specifically the + [t5-v1_1-xxl](https://huggingface.co/PixArt-alpha/PixArt-alpha/tree/main/t5-v1_1-xxl) variant. + tokenizer (`T5Tokenizer`): + Tokenizer of class + [T5Tokenizer](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5Tokenizer). + transformer ([`CogVideoXTransformer3DModel`]): + A text conditioned `CogVideoXTransformer3DModel` to denoise the encoded video latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `transformer` to denoise the encoded video latents. + """ + + _optional_components = [] + model_cpu_offload_seq = "text_encoder->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + vae: AutoencoderKLCogVideoX, + transformer: CogVideoXTransformer3DModel, + scheduler: Union[CogVideoXDDIMScheduler, CogVideoXDPMScheduler], + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, scheduler=scheduler + ) + self.vae_scale_factor_spatial = ( + 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8 + ) + self.vae_scale_factor_temporal = ( + self.vae.config.temporal_compression_ratio if hasattr(self, "vae") and self.vae is not None else 4 + ) + + self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) + + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 226, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder(text_input_ids.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return prompt_embeds + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 226, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + video_length, + dtype, + device, + generator, + latents=None, + video=None, + timestep=None, + is_strength_max=True, + return_noise=False, + return_video_latents=False, + ): + shape = ( + batch_size, + (video_length - 1) // self.vae_scale_factor_temporal + 1, + num_channels_latents, + height // self.vae_scale_factor_spatial, + width // self.vae_scale_factor_spatial, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if return_video_latents or (latents is None and not is_strength_max): + video = video.to(device=device, dtype=self.vae.dtype) + + bs = 1 + new_video = [] + for i in range(0, video.shape[0], bs): + video_bs = video[i : i + bs] + video_bs = self.vae.encode(video_bs)[0] + video_bs = video_bs.sample() + new_video.append(video_bs) + video = torch.cat(new_video, dim = 0) + video = video * self.vae.config.scaling_factor + + video_latents = video.repeat(batch_size // video.shape[0], 1, 1, 1, 1) + video_latents = video_latents.to(device=device, dtype=dtype) + video_latents = rearrange(video_latents, "b c f h w -> b f c h w") + + if latents is None: + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + # if strength is 1. then initialise the latents to noise, else initial to image + noise + latents = noise if is_strength_max else self.scheduler.add_noise(video_latents, noise, timestep) + # if pure noise then scale the initial latents by the Scheduler's init sigma + latents = latents * self.scheduler.init_noise_sigma if is_strength_max else latents + else: + noise = latents.to(device) + latents = noise * self.scheduler.init_noise_sigma + + # scale the initial noise by the standard deviation required by the scheduler + outputs = (latents,) + + if return_noise: + outputs += (noise,) + + if return_video_latents: + outputs += (video_latents,) + + return outputs + + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance, noise_aug_strength + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if mask is not None: + mask = mask.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask = [] + for i in range(0, mask.shape[0], bs): + mask_bs = mask[i : i + bs] + mask_bs = self.vae.encode(mask_bs)[0] + mask_bs = mask_bs.mode() + new_mask.append(mask_bs) + mask = torch.cat(new_mask, dim = 0) + mask = mask * self.vae.config.scaling_factor + + if masked_image is not None: + if self.transformer.config.add_noise_in_inpaint_model: + masked_image = add_noise_to_reference_video(masked_image, ratio=noise_aug_strength) + masked_image = masked_image.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask_pixel_values = [] + for i in range(0, masked_image.shape[0], bs): + mask_pixel_values_bs = masked_image[i : i + bs] + mask_pixel_values_bs = self.vae.encode(mask_pixel_values_bs)[0] + mask_pixel_values_bs = mask_pixel_values_bs.mode() + new_mask_pixel_values.append(mask_pixel_values_bs) + masked_image_latents = torch.cat(new_mask_pixel_values, dim = 0) + masked_image_latents = masked_image_latents * self.vae.config.scaling_factor + else: + masked_image_latents = None + + return mask, masked_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + latents = latents.permute(0, 2, 1, 3, 4) # [batch_size, num_channels, num_frames, height, width] + latents = 1 / self.vae.config.scaling_factor * latents + + frames = self.vae.decode(latents).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def fuse_qkv_projections(self) -> None: + r"""Enables fused QKV projections.""" + self.fusing_transformer = True + self.transformer.fuse_qkv_projections() + + def unfuse_qkv_projections(self) -> None: + r"""Disable QKV projection fusion if enabled.""" + if not self.fusing_transformer: + logger.warning("The Transformer was not initially fused for QKV projections. Doing nothing.") + else: + self.transformer.unfuse_qkv_projections() + self.fusing_transformer = False + + def _prepare_rotary_positional_embeddings( + self, + height: int, + width: int, + num_frames: int, + device: torch.device, + ) -> Tuple[torch.Tensor, torch.Tensor]: + grid_height = height // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) + grid_width = width // (self.vae_scale_factor_spatial * self.transformer.config.patch_size) + + p = self.transformer.config.patch_size + p_t = self.transformer.config.patch_size_t + + base_size_width = self.transformer.config.sample_width // p + base_size_height = self.transformer.config.sample_height // p + + if p_t is None: + # CogVideoX 1.0 + grid_crops_coords = get_resize_crop_region_for_grid( + (grid_height, grid_width), base_size_width, base_size_height + ) + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=self.transformer.config.attention_head_dim, + crops_coords=grid_crops_coords, + grid_size=(grid_height, grid_width), + temporal_size=num_frames, + ) + else: + # CogVideoX 1.5 + base_num_frames = (num_frames + p_t - 1) // p_t + + freqs_cos, freqs_sin = get_3d_rotary_pos_embed( + embed_dim=self.transformer.config.attention_head_dim, + crops_coords=None, + grid_size=(grid_height, grid_width), + temporal_size=base_num_frames, + grid_type="slice", + max_size=(base_size_height, base_size_width), + ) + + freqs_cos = freqs_cos.to(device=device) + freqs_sin = freqs_sin.to(device=device) + return freqs_cos, freqs_sin + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + + return timesteps, num_inference_steps - t_start + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + video: Union[torch.FloatTensor] = None, + mask_video: Union[torch.FloatTensor] = None, + masked_video_latents: Union[torch.FloatTensor] = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + use_dynamic_cfg: bool = False, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 226, + strength: float = 1, + noise_aug_strength: float = 0.0563, + comfyui_progressbar: bool = False, + ) -> Union[CogVideoXFunPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + num_frames (`int`, defaults to `48`): + Number of frames to generate. Must be divisible by self.vae_scale_factor_temporal. Generated video will + contain 1 extra frame because CogVideoX_Fun is conditioned with (num_seconds * fps + 1) frames where + num_seconds is 6 and fps is 4. However, since videos can be saved at any fps, the only condition that + needs to be satisfied is that of divisibility mentioned above. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of videos to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int`, defaults to `226`): + Maximum sequence length in encoded prompt. Must be consistent with + `self.transformer.config.max_text_seq_length` otherwise may lead to poor results. + + Examples: + + Returns: + [`~pipelines.cogvideo.pipeline_cogvideox.CogVideoXFunPipelineOutput`] or `tuple`: + [`~pipelines.cogvideo.pipeline_cogvideox.CogVideoXFunPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + + height = height or self.transformer.config.sample_height * self.vae_scale_factor_spatial + width = width or self.transformer.config.sample_width * self.vae_scale_factor_spatial + num_frames = num_frames or self.transformer.config.sample_frames + + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + + # 4. set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps( + num_inference_steps=num_inference_steps, strength=strength, device=device + ) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + # at which timestep to set the initial noise (n.b. 50% if strength is 0.5) + latent_timestep = timesteps[:1].repeat(batch_size * num_videos_per_prompt) + # create a boolean to check if the strength is set to 1. if so then initialise the latents with pure noise + is_strength_max = strength == 1.0 + + # 5. Prepare latents. + if video is not None: + video_length = video.shape[2] + init_video = self.image_processor.preprocess(rearrange(video, "b c f h w -> (b f) c h w"), height=height, width=width) + init_video = init_video.to(dtype=torch.float32) + init_video = rearrange(init_video, "(b f) c h w -> b c f h w", f=video_length) + else: + init_video = None + + # Magvae needs the number of frames to be 4n + 1. + local_latent_length = (num_frames - 1) // self.vae_scale_factor_temporal + 1 + # For CogVideoX 1.5, the latent frames should be clipped to make it divisible by patch_size_t + patch_size_t = self.transformer.config.patch_size_t + additional_frames = 0 + if patch_size_t is not None and local_latent_length % patch_size_t != 0: + additional_frames = local_latent_length % patch_size_t + num_frames -= additional_frames * self.vae_scale_factor_temporal + if num_frames <= 0: + num_frames = 1 + if video_length > num_frames: + logger.warning("The length of condition video is not right, the latent frames should be clipped to make it divisible by patch_size_t. ") + video_length = num_frames + video = video[:, :, :video_length] + init_video = init_video[:, :, :video_length] + mask_video = mask_video[:, :, :video_length] + + num_channels_latents = self.vae.config.latent_channels + num_channels_transformer = self.transformer.config.in_channels + return_image_latents = num_channels_transformer == num_channels_latents + + latents_outputs = self.prepare_latents( + batch_size * num_videos_per_prompt, + num_channels_latents, + height, + width, + video_length, + prompt_embeds.dtype, + device, + generator, + latents, + video=init_video, + timestep=latent_timestep, + is_strength_max=is_strength_max, + return_noise=True, + return_video_latents=return_image_latents, + ) + if return_image_latents: + latents, noise, image_latents = latents_outputs + else: + latents, noise = latents_outputs + if comfyui_progressbar: + pbar.update(1) + + if mask_video is not None: + if (mask_video == 255).all(): + mask_latents = torch.zeros_like(latents)[:, :, :1].to(latents.device, latents.dtype) + masked_video_latents = torch.zeros_like(latents).to(latents.device, latents.dtype) + + mask_input = torch.cat([mask_latents] * 2) if do_classifier_free_guidance else mask_latents + masked_video_latents_input = ( + torch.cat([masked_video_latents] * 2) if do_classifier_free_guidance else masked_video_latents + ) + inpaint_latents = torch.cat([mask_input, masked_video_latents_input], dim=2).to(latents.dtype) + else: + # Prepare mask latent variables + video_length = video.shape[2] + mask_condition = self.mask_processor.preprocess(rearrange(mask_video, "b c f h w -> (b f) c h w"), height=height, width=width) + mask_condition = mask_condition.to(dtype=torch.float32) + mask_condition = rearrange(mask_condition, "(b f) c h w -> b c f h w", f=video_length) + + if num_channels_transformer != num_channels_latents: + mask_condition_tile = torch.tile(mask_condition, [1, 3, 1, 1, 1]) + if masked_video_latents is None: + masked_video = init_video * (mask_condition_tile < 0.5) + torch.ones_like(init_video) * (mask_condition_tile > 0.5) * -1 + else: + masked_video = masked_video_latents + + _, masked_video_latents = self.prepare_mask_latents( + None, + masked_video, + batch_size, + height, + width, + prompt_embeds.dtype, + device, + generator, + do_classifier_free_guidance, + noise_aug_strength=noise_aug_strength, + ) + mask_latents = resize_mask(1 - mask_condition, masked_video_latents) + mask_latents = mask_latents.to(masked_video_latents.device) * self.vae.config.scaling_factor + + mask = torch.tile(mask_condition, [1, num_channels_latents, 1, 1, 1]) + mask = F.interpolate(mask, size=latents.size()[-3:], mode='trilinear', align_corners=True).to(latents.device, latents.dtype) + + mask_input = torch.cat([mask_latents] * 2) if do_classifier_free_guidance else mask_latents + masked_video_latents_input = ( + torch.cat([masked_video_latents] * 2) if do_classifier_free_guidance else masked_video_latents + ) + + mask = rearrange(mask, "b c f h w -> b f c h w") + mask_input = rearrange(mask_input, "b c f h w -> b f c h w") + masked_video_latents_input = rearrange(masked_video_latents_input, "b c f h w -> b f c h w") + + inpaint_latents = torch.cat([mask_input, masked_video_latents_input], dim=2).to(latents.dtype) + else: + mask = torch.tile(mask_condition, [1, num_channels_latents, 1, 1, 1]) + mask = F.interpolate(mask, size=latents.size()[-3:], mode='trilinear', align_corners=True).to(latents.device, latents.dtype) + mask = rearrange(mask, "b c f h w -> b f c h w") + + inpaint_latents = None + else: + if num_channels_transformer != num_channels_latents: + mask = torch.zeros_like(latents).to(latents.device, latents.dtype) + masked_video_latents = torch.zeros_like(latents).to(latents.device, latents.dtype) + + mask_input = torch.cat([mask] * 2) if do_classifier_free_guidance else mask + masked_video_latents_input = ( + torch.cat([masked_video_latents] * 2) if do_classifier_free_guidance else masked_video_latents + ) + inpaint_latents = torch.cat([mask_input, masked_video_latents_input], dim=1).to(latents.dtype) + else: + mask = torch.zeros_like(init_video[:, :1]) + mask = torch.tile(mask, [1, num_channels_latents, 1, 1, 1]) + mask = F.interpolate(mask, size=latents.size()[-3:], mode='trilinear', align_corners=True).to(latents.device, latents.dtype) + mask = rearrange(mask, "b c f h w -> b f c h w") + + inpaint_latents = None + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Create rotary embeds if required + image_rotary_emb = ( + self._prepare_rotary_positional_embeddings(height, width, latents.size(1), device) + if self.transformer.config.use_rotary_positional_embeddings + else None + ) + + # 8. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + # for DPM-solver++ + old_pred_original_sample = None + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + # predict noise model_output + noise_pred = self.transformer( + hidden_states=latent_model_input, + encoder_hidden_states=prompt_embeds, + timestep=timestep, + image_rotary_emb=image_rotary_emb, + return_dict=False, + inpaint_latents=inpaint_latents, + )[0] + noise_pred = noise_pred.float() + + # perform guidance + if use_dynamic_cfg: + self._guidance_scale = 1 + guidance_scale * ( + (1 - math.cos(math.pi * ((num_inference_steps - t.item()) / num_inference_steps) ** 5.0)) / 2 + ) + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + if not isinstance(self.scheduler, CogVideoXDPMScheduler): + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + else: + latents, old_pred_original_sample = self.scheduler.step( + noise_pred, + old_pred_original_sample, + t, + timesteps[i - 1] if i > 0 else None, + latents, + **extra_step_kwargs, + return_dict=False, + ) + latents = latents.to(prompt_embeds.dtype) + + # call the callback, if provided + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return CogVideoXFunPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_fantasy_talking.py b/VideoX-Fun/videox_fun/pipeline/pipeline_fantasy_talking.py new file mode 100644 index 0000000000000000000000000000000000000000..114bd2280f5d8b6b7feecf66ddb5c1ba00c31631 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_fantasy_talking.py @@ -0,0 +1,754 @@ +import inspect +import math +import copy +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange +from PIL import Image +from torchvision import transforms +from transformers import T5Tokenizer + +from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel_S2V, WanAudioEncoder, + WanT5EncoderModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class FantasyTalkingPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = ["transformer_2", "audio_encoder"] + model_cpu_offload_seq = "text_encoder->transformer_2->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + audio_encoder: WanAudioEncoder, + vae: AutoencoderKLWan, + transformer: Wan2_2Transformer3DModel_S2V, + clip_image_encoder: CLIPModel, + transformer_2: Wan2_2Transformer3DModel_S2V = None, + scheduler: FlowMatchEulerDiscreteScheduler = None, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, + transformer_2=transformer_2, scheduler=scheduler, clip_image_encoder=clip_image_encoder, audio_encoder=audio_encoder + ) + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None, num_length_latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1 if num_length_latents is None else num_length_latents, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance, noise_aug_strength + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if mask is not None: + mask = mask.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask = [] + for i in range(0, mask.shape[0], bs): + mask_bs = mask[i : i + bs] + mask_bs = self.vae.encode(mask_bs)[0] + mask_bs = mask_bs.mode() + new_mask.append(mask_bs) + mask = torch.cat(new_mask, dim = 0) + # mask = mask * self.vae.config.scaling_factor + + if masked_image is not None: + masked_image = masked_image.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask_pixel_values = [] + for i in range(0, masked_image.shape[0], bs): + mask_pixel_values_bs = masked_image[i : i + bs] + mask_pixel_values_bs = self.vae.encode(mask_pixel_values_bs)[0] + mask_pixel_values_bs = mask_pixel_values_bs.mode() + new_mask_pixel_values.append(mask_pixel_values_bs) + masked_image_latents = torch.cat(new_mask_pixel_values, dim = 0) + # masked_image_latents = masked_image_latents * self.vae.config.scaling_factor + else: + masked_image_latents = None + + return mask, masked_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + video: Union[torch.FloatTensor] = None, + mask_video: Union[torch.FloatTensor] = None, + audio_path = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + clip_image: Image = None, + max_sequence_length: int = 512, + comfyui_progressbar: bool = False, + shift: int = 5, + fps: int = 16, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + # 5. Prepare latents. + if video is not None: + video_length = video.shape[2] + init_video = self.image_processor.preprocess(rearrange(video, "b c f h w -> (b f) c h w"), height=height, width=width) + init_video = init_video.to(dtype=torch.float32) + init_video = rearrange(init_video, "(b f) c h w -> b c f h w", f=video_length) + else: + init_video = None + + latent_channels = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + ) + if comfyui_progressbar: + pbar.update(1) + + # Prepare mask latent variables + if init_video is not None: + if (mask_video == 255).all(): + mask_latents = torch.tile( + torch.zeros_like(latents)[:, :1].to(device, weight_dtype), [1, 4, 1, 1, 1] + ) + masked_video_latents = torch.zeros_like(latents).to(device, weight_dtype) + else: + bs, _, video_length, height, width = video.size() + mask_condition = self.mask_processor.preprocess(rearrange(mask_video, "b c f h w -> (b f) c h w"), height=height, width=width) + mask_condition = mask_condition.to(dtype=torch.float32) + mask_condition = rearrange(mask_condition, "(b f) c h w -> b c f h w", f=video_length) + + masked_video = init_video * (torch.tile(mask_condition, [1, 3, 1, 1, 1]) < 0.5) + _, masked_video_latents = self.prepare_mask_latents( + None, + masked_video, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance, + noise_aug_strength=None, + ) + + mask_condition = torch.concat( + [ + torch.repeat_interleave(mask_condition[:, :, 0:1], repeats=4, dim=2), + mask_condition[:, :, 1:] + ], dim=2 + ) + mask_condition = mask_condition.view(bs, mask_condition.shape[2] // 4, 4, height, width) + mask_condition = mask_condition.transpose(1, 2) + mask_latents = resize_mask(1 - mask_condition, masked_video_latents, True).to(device, weight_dtype) + + # Prepare clip latent variables + if clip_image is not None: + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(device, weight_dtype) + clip_context = self.clip_image_encoder([clip_image[:, None, :, :]]) + else: + clip_image = Image.new("RGB", (512, 512), color=(0, 0, 0)) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(device, weight_dtype) + clip_context = self.clip_image_encoder([clip_image[:, None, :, :]]) + clip_context = torch.zeros_like(clip_context) + + # Extract audio emb + audio_wav2vec_fea = self.audio_encoder.extract_audio_feat(audio_path, num_frames=num_frames, fps=fps) + + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, (num_frames - 1) // self.vae.temporal_compression_ratio + 1, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + if init_video is not None: + mask_input = torch.cat([mask_latents] * 2) if do_classifier_free_guidance else mask_latents + masked_video_latents_input = ( + torch.cat([masked_video_latents] * 2) if do_classifier_free_guidance else masked_video_latents + ) + y = torch.cat([mask_input, masked_video_latents_input], dim=1).to(device, weight_dtype) + + clip_context_input = ( + torch.cat([clip_context] * 2) if do_classifier_free_guidance else clip_context + ) + + audio_wav2vec_fea_input = ( + torch.cat([audio_wav2vec_fea] * 2) if do_classifier_free_guidance else audio_wav2vec_fea + ) + + audio_scale = torch.tensor( + [0.75, 1] + ).to(latent_model_input.device, latent_model_input.dtype) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = self.transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + y=y, + audio_wav2vec_fea=audio_wav2vec_fea_input, + audio_scale=audio_scale, + clip_fea=clip_context_input, + ) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_flux.py b/VideoX-Fun/videox_fun/pipeline/pipeline_flux.py new file mode 100644 index 0000000000000000000000000000000000000000..15cea7226fc319609dcb246b2a905c46f28e9e44 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_flux.py @@ -0,0 +1,978 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/flux/pipeline_flux.py +# Copyright 2025 Black Forest Labs and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from diffusers.image_processor import VaeImageProcessor, PipelineImageInput +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import (BaseOutput, is_torch_xla_available, logging, + replace_example_docstring) +from diffusers.utils.torch_utils import randn_tensor + +from ..models import (CLIPImageProcessor, CLIPTextModel, + CLIPTokenizer, CLIPVisionModelWithProjection, + FluxTransformer2DModel, T5EncoderModel, AutoencoderKL, + T5TokenizerFast) + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import FluxPipeline + + >>> pipe = FluxPipeline.from_pretrained("black-forest-labs/FLUX.1-schnell", torch_dtype=torch.bfloat16) + >>> pipe.to("cuda") + >>> prompt = "A cat holding a sign that says hello world" + >>> # Depending on the variant being used, the pipeline call will slightly vary. + >>> # Refer to the pipeline documentation for more details. + >>> image = pipe(prompt, num_inference_steps=4, guidance_scale=0.0).images[0] + >>> image.save("flux.png") + ``` +""" + + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class FluxPipelineOutput(BaseOutput): + """ + Output class for Flux image generation pipelines. + + Args: + images (`List[PIL.Image.Image]` or `torch.Tensor` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array or torch tensor of shape `(batch_size, + height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion + pipeline. Torch tensors can represent either the denoised images or the intermediate latents ready to be + passed to the decoder. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +@dataclass +class FluxPriorReduxPipelineOutput(BaseOutput): + """ + Output class for Flux Prior Redux pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + """ + + prompt_embeds: torch.Tensor + pooled_prompt_embeds: torch.Tensor + + +class FluxPipeline( + DiffusionPipeline, +): + r""" + The Flux pipeline for text-to-image generation. + + Reference: https://blackforestlabs.ai/announcing-black-forest-labs/ + + Args: + transformer ([`FluxTransformer2DModel`]): + Conditional Transformer (MMDiT) architecture to denoise the encoded image latents. + scheduler ([`FlowMatchEulerDiscreteScheduler`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([`T5EncoderModel`]): + [T5](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5EncoderModel), specifically + the [google/t5-v1_1-xxl](https://huggingface.co/google/t5-v1_1-xxl) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`T5TokenizerFast`): + Second Tokenizer of class + [T5TokenizerFast](https://huggingface.co/docs/transformers/en/model_doc/t5#transformers.T5TokenizerFast). + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->transformer->vae" + _optional_components = ["image_encoder", "feature_extractor"] + _callback_tensor_inputs = ["latents", "prompt_embeds"] + + def __init__( + self, + scheduler: FlowMatchEulerDiscreteScheduler, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + text_encoder_2: T5EncoderModel, + tokenizer_2: T5TokenizerFast, + transformer: FluxTransformer2DModel, + image_encoder: CLIPVisionModelWithProjection = None, + feature_extractor: CLIPImageProcessor = None, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + transformer=transformer, + scheduler=scheduler, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 + # Flux latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible + # by the patch size. So the vae scale factor is multiplied by the patch size to account for this + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.tokenizer_max_length = ( + self.tokenizer.model_max_length if hasattr(self, "tokenizer") and self.tokenizer is not None else 77 + ) + self.default_sample_size = 128 + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_images_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer_2( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_length=False, + return_overflowing_tokens=False, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer_2(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer_2.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False)[0] + + dtype = self.text_encoder_2.dtype + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + _, seq_len, _ = prompt_embeds.shape + + # duplicate text embeddings and attention mask for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + return prompt_embeds + + def _get_clip_prompt_embeds( + self, + prompt: Union[str, List[str]], + num_images_per_prompt: int = 1, + device: Optional[torch.device] = None, + ): + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer_max_length, + truncation=True, + return_overflowing_tokens=False, + return_length=False, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer_max_length} tokens: {removed_text}" + ) + prompt_embeds = self.text_encoder(text_input_ids.to(device), output_hidden_states=False) + + # Use pooled output of CLIPTextModel + prompt_embeds = prompt_embeds.pooler_output + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, -1) + + return prompt_embeds + + def encode_prompt( + self, + prompt: Union[str, List[str]], + prompt_2: Optional[Union[str, List[str]]] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + max_sequence_length: int = 512, + lora_scale: Optional[float] = None, + ): + r""" + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in all text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # We only use the pooled prompt output from the CLIPTextModel + pooled_prompt_embeds = self._get_clip_prompt_embeds( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + ) + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt_2, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + ) + + dtype = self.text_encoder.dtype if self.text_encoder is not None else self.transformer.dtype + text_ids = torch.zeros(prompt_embeds.shape[1], 3).to(device=device, dtype=dtype) + + return prompt_embeds, pooled_prompt_embeds, text_ids + + def encode_image(self, image, device, num_images_per_prompt): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + return image_embeds + + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt + ): + image_embeds = [] + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != self.transformer.encoder_hid_proj.num_ip_adapters: + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {self.transformer.encoder_hid_proj.num_ip_adapters} IP Adapters." + ) + + for single_ip_adapter_image in ip_adapter_image: + single_image_embeds = self.encode_image(single_ip_adapter_image, device, 1) + image_embeds.append(single_image_embeds[None, :]) + else: + if not isinstance(ip_adapter_image_embeds, list): + ip_adapter_image_embeds = [ip_adapter_image_embeds] + + if len(ip_adapter_image_embeds) != self.transformer.encoder_hid_proj.num_ip_adapters: + raise ValueError( + f"`ip_adapter_image_embeds` must have same length as the number of IP Adapters. Got {len(ip_adapter_image_embeds)} image embeds and {self.transformer.encoder_hid_proj.num_ip_adapters} IP Adapters." + ) + + for single_image_embeds in ip_adapter_image_embeds: + image_embeds.append(single_image_embeds) + + ip_adapter_image_embeds = [] + for single_image_embeds in image_embeds: + single_image_embeds = torch.cat([single_image_embeds] * num_images_per_prompt, dim=0) + single_image_embeds = single_image_embeds.to(device=device) + ip_adapter_image_embeds.append(single_image_embeds) + + return ip_adapter_image_embeds + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + max_sequence_length=None, + ): + if height % (self.vae_scale_factor * 2) != 0 or width % (self.vae_scale_factor * 2) != 0: + logger.warning( + f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} but are {height} and {width}. Dimensions will be resized accordingly" + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + if max_sequence_length is not None and max_sequence_length > 512: + raise ValueError(f"`max_sequence_length` cannot be greater than 512 but is {max_sequence_length}") + + @staticmethod + def _prepare_latent_image_ids(batch_size, height, width, device, dtype): + latent_image_ids = torch.zeros(height, width, 3) + latent_image_ids[..., 1] = latent_image_ids[..., 1] + torch.arange(height)[:, None] + latent_image_ids[..., 2] = latent_image_ids[..., 2] + torch.arange(width)[None, :] + + latent_image_id_height, latent_image_id_width, latent_image_id_channels = latent_image_ids.shape + + latent_image_ids = latent_image_ids.reshape( + latent_image_id_height * latent_image_id_width, latent_image_id_channels + ) + + return latent_image_ids.to(device=device, dtype=dtype) + + @staticmethod + def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + + return latents + + @staticmethod + def _unpack_latents(latents, height, width, vae_scale_factor): + batch_size, num_patches, channels = latents.shape + + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + + latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2) + latents = latents.permute(0, 3, 1, 4, 2, 5) + + latents = latents.reshape(batch_size, channels // (2 * 2), height, width) + + return latents + + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + ): + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (self.vae_scale_factor * 2)) + width = 2 * (int(width) // (self.vae_scale_factor * 2)) + + shape = (batch_size, num_channels_latents, height, width) + + if latents is not None: + latent_image_ids = self._prepare_latent_image_ids(batch_size, height // 2, width // 2, device, dtype) + return latents.to(device=device, dtype=dtype), latent_image_ids + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width) + + latent_image_ids = self._prepare_latent_image_ids(batch_size, height // 2, width // 2, device, dtype) + + return latents, latent_image_ids + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def joint_attention_kwargs(self): + return self._joint_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def current_timestep(self): + return self._current_timestep + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt: Union[str, List[str]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + true_cfg_scale: float = 1.0, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 28, + sigmas: Optional[List[float]] = None, + guidance_scale: float = 3.5, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_ip_adapter_image: Optional[PipelineImageInput] = None, + negative_ip_adapter_image_embeds: Optional[List[torch.Tensor]] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + will be used instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is + not greater than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. + true_cfg_scale (`float`, *optional*, defaults to 1.0): + True classifier-free guidance (guidance scale) is enabled when `true_cfg_scale` > 1 and + `negative_prompt` is provided. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + guidance_scale (`float`, *optional*, defaults to 3.5): + Embedded guiddance scale is enabled by setting `guidance_scale` > 1. Higher `guidance_scale` encourages + a model to generate images more aligned with `prompt` at the expense of lower image quality. + + Guidance-distilled models approximates true classifer-free guidance for `guidance_scale` > 1. Refer to + the [paper](https://huggingface.co/papers/2210.03142) to learn more. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will be generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of + IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not + provided, embeddings are computed from the `ip_adapter_image` input argument. + negative_ip_adapter_image: + (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + negative_ip_adapter_image_embeds (`List[torch.Tensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of + IP-adapters. Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. If not + provided, embeddings are computed from the `ip_adapter_image` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.flux.FluxPipelineOutput`] instead of a plain tuple. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`. + + Examples: + + Returns: + [`~pipelines.flux.FluxPipelineOutput`] or `tuple`: [`~pipelines.flux.FluxPipelineOutput`] if `return_dict` + is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the generated + images. + """ + + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + max_sequence_length=max_sequence_length, + ) + + self._guidance_scale = guidance_scale + self._joint_attention_kwargs = joint_attention_kwargs + self._current_timestep = None + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + lora_scale = ( + self.joint_attention_kwargs.get("scale", None) if self.joint_attention_kwargs is not None else None + ) + has_neg_prompt = negative_prompt is not None or ( + negative_prompt_embeds is not None and negative_pooled_prompt_embeds is not None + ) + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + ( + prompt_embeds, + pooled_prompt_embeds, + text_ids, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + lora_scale=lora_scale, + ) + if do_true_cfg: + ( + negative_prompt_embeds, + negative_pooled_prompt_embeds, + negative_text_ids, + ) = self.encode_prompt( + prompt=negative_prompt, + prompt_2=negative_prompt_2, + prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=negative_pooled_prompt_embeds, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + lora_scale=lora_scale, + ) + + # 4. Prepare latent variables + num_channels_latents = self.transformer.config.in_channels // 4 + latents, latent_image_ids = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 5. Prepare timesteps + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas + if hasattr(self.scheduler.config, "use_flow_sigmas") and self.scheduler.config.use_flow_sigmas: + sigmas = None + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + self.scheduler.config.get("base_image_seq_len", 256), + self.scheduler.config.get("max_image_seq_len", 4096), + self.scheduler.config.get("base_shift", 0.5), + self.scheduler.config.get("max_shift", 1.15), + ) + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + sigmas=sigmas, + mu=mu, + ) + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self._num_timesteps = len(timesteps) + + # handle guidance + if self.transformer.config.guidance_embeds: + guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32) + guidance = guidance.expand(latents.shape[0]) + else: + guidance = None + + if (ip_adapter_image is not None or ip_adapter_image_embeds is not None) and ( + negative_ip_adapter_image is None and negative_ip_adapter_image_embeds is None + ): + negative_ip_adapter_image = np.zeros((width, height, 3), dtype=np.uint8) + negative_ip_adapter_image = [negative_ip_adapter_image] * self.transformer.encoder_hid_proj.num_ip_adapters + + elif (ip_adapter_image is None and ip_adapter_image_embeds is None) and ( + negative_ip_adapter_image is not None or negative_ip_adapter_image_embeds is not None + ): + ip_adapter_image = np.zeros((width, height, 3), dtype=np.uint8) + ip_adapter_image = [ip_adapter_image] * self.transformer.encoder_hid_proj.num_ip_adapters + + if self.joint_attention_kwargs is None: + self._joint_attention_kwargs = {} + + image_embeds = None + negative_image_embeds = None + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + ) + if negative_ip_adapter_image is not None or negative_ip_adapter_image_embeds is not None: + negative_image_embeds = self.prepare_ip_adapter_image_embeds( + negative_ip_adapter_image, + negative_ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + ) + + # 6. Denoising loop + # We set the index here to remove DtoH sync, helpful especially during compilation. + # Check out more details here: https://github.com/huggingface/diffusers/pull/11696 + self.scheduler.set_begin_index(0) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + self._current_timestep = t + if image_embeds is not None: + self._joint_attention_kwargs["ip_adapter_image_embeds"] = image_embeds + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latents.shape[0]).to(latents.dtype) + + with torch.cuda.amp.autocast(dtype=latents.dtype), torch.cuda.device(device=latents.device): + noise_pred = self.transformer( + hidden_states=latents, + timestep=timestep / 1000, + guidance=guidance, + pooled_projections=pooled_prompt_embeds, + encoder_hidden_states=prompt_embeds, + txt_ids=text_ids, + img_ids=latent_image_ids, + joint_attention_kwargs=self.joint_attention_kwargs, + return_dict=False, + )[0] + + if do_true_cfg: + with torch.cuda.amp.autocast(dtype=latents.dtype), torch.cuda.device(device=latents.device): + neg_noise_pred = self.transformer( + hidden_states=latents, + timestep=timestep / 1000, + guidance=guidance, + pooled_projections=negative_pooled_prompt_embeds, + encoder_hidden_states=negative_prompt_embeds, + txt_ids=negative_text_ids, + img_ids=latent_image_ids, + joint_attention_kwargs=self.joint_attention_kwargs, + return_dict=False, + )[0] + noise_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + + # compute the previous noisy sample x_t -> x_t-1 + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + if latents.dtype != latents_dtype: + if torch.backends.mps.is_available(): + # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 + latents = latents.to(latents_dtype) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if XLA_AVAILABLE: + xm.mark_step() + + self._current_timestep = None + + if output_type == "latent": + image = latents + else: + latents = self._unpack_latents(latents, height, width, self.vae_scale_factor) + latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor + image = self.vae.decode(latents, return_dict=False)[0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return FluxPipelineOutput(images=image) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_flux2.py b/VideoX-Fun/videox_fun/pipeline/pipeline_flux2.py new file mode 100644 index 0000000000000000000000000000000000000000..26a8741cf93a9977aee49a2887aa2d60683acf8a --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_flux2.py @@ -0,0 +1,900 @@ +# https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/flux2/pipeline_flux2.py +# Copyright 2025 Black Forest Labs and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from diffusers.utils import (BaseOutput, is_torch_xla_available, logging, + replace_example_docstring) +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL +import torch +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import (is_torch_xla_available, logging, + replace_example_docstring) +from diffusers.utils.torch_utils import randn_tensor + +from ..models import (AutoencoderKLFlux2, Flux2ImageProcessor, + Flux2Transformer2DModel, Mistral3ForConditionalGeneration, AutoProcessor) + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import Flux2Pipeline + + >>> pipe = Flux2Pipeline.from_pretrained("black-forest-labs/FLUX.2-dev", torch_dtype=torch.bfloat16) + >>> pipe.to("cuda") + >>> prompt = "A cat holding a sign that says hello world" + >>> # Depending on the variant being used, the pipeline call will slightly vary. + >>> # Refer to the pipeline documentation for more details. + >>> image = pipe(prompt, num_inference_steps=50, guidance_scale=2.5).images[0] + >>> image.save("flux.png") + ``` +""" + + +def format_text_input(prompts: List[str], system_message: str = None): + # Remove [IMG] tokens from prompts to avoid Pixtral validation issues + # when truncation is enabled. The processor counts [IMG] tokens and fails + # if the count changes after truncation. + cleaned_txt = [prompt.replace("[IMG]", "") for prompt in prompts] + + return [ + [ + { + "role": "system", + "content": [{"type": "text", "text": system_message}], + }, + {"role": "user", "content": [{"type": "text", "text": prompt}]}, + ] + for prompt in cleaned_txt + ] + + +def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float: + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return float(mu) + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +@dataclass +class Flux2PipelineOutput(BaseOutput): + """ + Output class for Flux2 image generation pipelines. + + Args: + images (`List[PIL.Image.Image]` or `torch.Tensor` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array or torch tensor of shape `(batch_size, + height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion + pipeline. Torch tensors can represent either the denoised images or the intermediate latents ready to be + passed to the decoder. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +class Flux2Pipeline(DiffusionPipeline): + r""" + The Flux2 pipeline for text-to-image generation. + + Reference: [https://bfl.ai/blog/flux-2](https://bfl.ai/blog/flux-2) + + Args: + transformer ([`Flux2Transformer2DModel`]): + Conditional Transformer (MMDiT) architecture to denoise the encoded image latents. + scheduler ([`FlowMatchEulerDiscreteScheduler`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + vae ([`AutoencoderKLFlux2`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`Mistral3ForConditionalGeneration`]): + [Mistral3ForConditionalGeneration](https://huggingface.co/docs/transformers/en/model_doc/mistral3#transformers.Mistral3ForConditionalGeneration) + tokenizer (`AutoProcessor`): + Tokenizer of class + [PixtralProcessor](https://huggingface.co/docs/transformers/en/model_doc/pixtral#transformers.PixtralProcessor). + """ + + model_cpu_offload_seq = "text_encoder->transformer->vae" + _callback_tensor_inputs = ["latents", "prompt_embeds"] + + def __init__( + self, + scheduler: FlowMatchEulerDiscreteScheduler, + vae: AutoencoderKLFlux2, + text_encoder: Mistral3ForConditionalGeneration, + tokenizer: AutoProcessor, + transformer: Flux2Transformer2DModel, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + transformer=transformer, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 + # Flux latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible + # by the patch size. So the vae scale factor is multiplied by the patch size to account for this + self.image_processor = Flux2ImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.tokenizer_max_length = 512 + self.default_sample_size = 128 + + # fmt: off + self.system_message = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation." + # fmt: on + + @staticmethod + def _get_mistral_3_small_prompt_embeds( + text_encoder: Mistral3ForConditionalGeneration, + tokenizer: AutoProcessor, + prompt: Union[str, List[str]], + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + max_sequence_length: int = 512, + # fmt: off + system_message: str = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation.", + # fmt: on + hidden_states_layers: List[int] = (10, 20, 30), + ): + dtype = text_encoder.dtype if dtype is None else dtype + device = text_encoder.device if device is None else device + + prompt = [prompt] if isinstance(prompt, str) else prompt + + # Format input messages + messages_batch = format_text_input(prompts=prompt, system_message=system_message) + + # Process all messages at once + inputs = tokenizer.apply_chat_template( + messages_batch, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=max_sequence_length, + ) + + # Move to device + input_ids = inputs["input_ids"].to(device) + attention_mask = inputs["attention_mask"].to(device) + + # Forward pass through the model + output = text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + use_cache=False, + ) + + # Only use outputs from intermediate layers and stack them + out = torch.stack([output.hidden_states[k] for k in hidden_states_layers], dim=1) + out = out.to(dtype=dtype, device=device) + + batch_size, num_channels, seq_len, hidden_dim = out.shape + prompt_embeds = out.permute(0, 2, 1, 3).reshape(batch_size, seq_len, num_channels * hidden_dim) + + return prompt_embeds + + @staticmethod + def _prepare_text_ids( + x: torch.Tensor, # (B, L, D) or (L, D) + t_coord: Optional[torch.Tensor] = None, + ): + B, L, _ = x.shape + out_ids = [] + + for i in range(B): + t = torch.arange(1) if t_coord is None else t_coord[i] + h = torch.arange(1) + w = torch.arange(1) + l = torch.arange(L) + + coords = torch.cartesian_prod(t, h, w, l) + out_ids.append(coords) + + return torch.stack(out_ids) + + def encode_prompt( + self, + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + text_encoder_out_layers: Tuple[int] = (10, 20, 30), + ): + device = device or self._execution_device + + if prompt is None: + prompt = "" + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt_embeds is None: + prompt_embeds = self._get_mistral_3_small_prompt_embeds( + text_encoder=self.text_encoder, + tokenizer=self.tokenizer, + prompt=prompt, + device=device, + max_sequence_length=max_sequence_length, + system_message=self.system_message, + hidden_states_layers=text_encoder_out_layers, + ) + + batch_size, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + text_ids = self._prepare_text_ids(prompt_embeds) + text_ids = text_ids.to(device) + return prompt_embeds, text_ids + + @staticmethod + def _prepare_latent_ids( + latents: torch.Tensor, # (B, C, H, W) + ): + r""" + Generates 4D position coordinates (T, H, W, L) for latent tensors. + + Args: + latents (torch.Tensor): + Latent tensor of shape (B, C, H, W) + + Returns: + torch.Tensor: + Position IDs tensor of shape (B, H*W, 4) All batches share the same coordinate structure: T=0, + H=[0..H-1], W=[0..W-1], L=0 + """ + + batch_size, _, height, width = latents.shape + + t = torch.arange(1) # [0] - time dimension + h = torch.arange(height) + w = torch.arange(width) + l = torch.arange(1) # [0] - layer dimension + + # Create position IDs: (H*W, 4) + latent_ids = torch.cartesian_prod(t, h, w, l) + + # Expand to batch: (B, H*W, 4) + latent_ids = latent_ids.unsqueeze(0).expand(batch_size, -1, -1) + + return latent_ids + + @staticmethod + def _prepare_image_ids( + image_latents: List[torch.Tensor], # [(1, C, H, W), (1, C, H, W), ...] + scale: int = 10, + ): + r""" + Generates 4D time-space coordinates (T, H, W, L) for a sequence of image latents. + + This function creates a unique coordinate for every pixel/patch across all input latent with different + dimensions. + + Args: + image_latents (List[torch.Tensor]): + A list of image latent feature tensors, typically of shape (C, H, W). + scale (int, optional): + A factor used to define the time separation (T-coordinate) between latents. T-coordinate for the i-th + latent is: 'scale + scale * i'. Defaults to 10. + + Returns: + torch.Tensor: + The combined coordinate tensor. Shape: (1, N_total, 4) Where N_total is the sum of (H * W) for all + input latents. + + Coordinate Components (Dimension 4): + - T (Time): The unique index indicating which latent image the coordinate belongs to. + - H (Height): The row index within that latent image. + - W (Width): The column index within that latent image. + - L (Seq. Length): A sequence length dimension, which is always fixed at 0 (size 1) + """ + + if not isinstance(image_latents, list): + raise ValueError(f"Expected `image_latents` to be a list, got {type(image_latents)}.") + + # create time offset for each reference image + t_coords = [scale + scale * t for t in torch.arange(0, len(image_latents))] + t_coords = [t.view(-1) for t in t_coords] + + image_latent_ids = [] + for x, t in zip(image_latents, t_coords): + x = x.squeeze(0) + _, height, width = x.shape + + x_ids = torch.cartesian_prod(t, torch.arange(height), torch.arange(width), torch.arange(1)) + image_latent_ids.append(x_ids) + + image_latent_ids = torch.cat(image_latent_ids, dim=0) + image_latent_ids = image_latent_ids.unsqueeze(0) + + return image_latent_ids + + @staticmethod + def _patchify_latents(latents): + batch_size, num_channels_latents, height, width = latents.shape + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 1, 3, 5, 2, 4) + latents = latents.reshape(batch_size, num_channels_latents * 4, height // 2, width // 2) + return latents + + @staticmethod + def _unpatchify_latents(latents): + batch_size, num_channels_latents, height, width = latents.shape + latents = latents.reshape(batch_size, num_channels_latents // (2 * 2), 2, 2, height, width) + latents = latents.permute(0, 1, 4, 2, 5, 3) + latents = latents.reshape(batch_size, num_channels_latents // (2 * 2), height * 2, width * 2) + return latents + + @staticmethod + def _pack_latents(latents): + """ + pack latents: (batch_size, num_channels, height, width) -> (batch_size, height * width, num_channels) + """ + + batch_size, num_channels, height, width = latents.shape + latents = latents.reshape(batch_size, num_channels, height * width).permute(0, 2, 1) + + return latents + + @staticmethod + def _unpack_latents_with_ids(x: torch.Tensor, x_ids: torch.Tensor) -> list[torch.Tensor]: + """ + using position ids to scatter tokens into place + """ + x_list = [] + for data, pos in zip(x, x_ids): + _, ch = data.shape # noqa: F841 + h_ids = pos[:, 1].to(torch.int64) + w_ids = pos[:, 2].to(torch.int64) + + h = torch.max(h_ids) + 1 + w = torch.max(w_ids) + 1 + + flat_ids = h_ids * w + w_ids + + out = torch.zeros((h * w, ch), device=data.device, dtype=data.dtype) + out.scatter_(0, flat_ids.unsqueeze(1).expand(-1, ch), data) + + # reshape from (H * W, C) to (H, W, C) and permute to (C, H, W) + + out = out.view(h, w, ch).permute(2, 0, 1) + x_list.append(out) + + return torch.stack(x_list, dim=0) + + def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator): + if image.ndim != 4: + raise ValueError(f"Expected image dims 4, got {image.ndim}.") + + image_latents = retrieve_latents(self.vae.encode(image), generator=generator, sample_mode="argmax") + image_latents = self._patchify_latents(image_latents) + + latents_bn_mean = self.vae.bn.running_mean.view(1, -1, 1, 1).to(image_latents.device, image_latents.dtype) + latents_bn_std = torch.sqrt(self.vae.bn.running_var.view(1, -1, 1, 1) + self.vae.config.batch_norm_eps) + image_latents = (image_latents - latents_bn_mean) / latents_bn_std + + return image_latents + + def prepare_latents( + self, + batch_size, + num_latents_channels, + height, + width, + dtype, + device, + generator: torch.Generator, + latents: Optional[torch.Tensor] = None, + ): + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (self.vae_scale_factor * 2)) + width = 2 * (int(width) // (self.vae_scale_factor * 2)) + + shape = (batch_size, num_latents_channels * 4, height // 2, width // 2) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device=device, dtype=dtype) + + latent_ids = self._prepare_latent_ids(latents) + latent_ids = latent_ids.to(device) + + latents = self._pack_latents(latents) # [B, C, H, W] -> [B, H*W, C] + return latents, latent_ids + + def prepare_image_latents( + self, + images: List[torch.Tensor], + batch_size, + generator: torch.Generator, + device, + dtype, + ): + image_latents = [] + for image in images: + image = image.to(device=device, dtype=dtype) + imagge_latent = self._encode_vae_image(image=image, generator=generator) + image_latents.append(imagge_latent) # (1, 128, 32, 32) + + image_latent_ids = self._prepare_image_ids(image_latents) + + # Pack each latent and concatenate + packed_latents = [] + for latent in image_latents: + # latent: (1, 128, 32, 32) + packed = self._pack_latents(latent) # (1, 1024, 128) + packed = packed.squeeze(0) # (1024, 128) - remove batch dim + packed_latents.append(packed) + + # Concatenate all reference tokens along sequence dimension + image_latents = torch.cat(packed_latents, dim=0) # (N*1024, 128) + image_latents = image_latents.unsqueeze(0) # (1, N*1024, 128) + + image_latents = image_latents.repeat(batch_size, 1, 1) + image_latent_ids = image_latent_ids.repeat(batch_size, 1, 1) + image_latent_ids = image_latent_ids.to(device) + + return image_latents, image_latent_ids + + def check_inputs( + self, + prompt, + height, + width, + prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if ( + height is not None + and height % (self.vae_scale_factor * 2) != 0 + or width is not None + and width % (self.vae_scale_factor * 2) != 0 + ): + logger.warning( + f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} but are {height} and {width}. Dimensions will be resized accordingly" + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def joint_attention_kwargs(self): + return self._joint_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def current_timestep(self): + return self._current_timestep + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image: Optional[Union[List[PIL.Image.Image], PIL.Image.Image]] = None, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + sigmas: Optional[List[float]] = None, + guidance_scale: Optional[float] = 4.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + text_encoder_out_layers: Tuple[int] = (10, 20, 30), + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + guidance_scale (`float`, *optional*, defaults to 1.0): + Guidance scale as defined in [Classifier-Free Diffusion + Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. + of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting + `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to + the text `prompt`, usually at the expense of lower image quality. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.Tensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will be generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.qwenimage.QwenImagePipelineOutput`] instead of a plain tuple. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`. + text_encoder_out_layers (`Tuple[int]`): + Layer indices to use in the `text_encoder` to derive the final prompt embeddings. + + Examples: + + Returns: + [`~pipelines.flux2.Flux2PipelineOutput`] or `tuple`: [`~pipelines.flux2.Flux2PipelineOutput`] if + `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the + generated images. + """ + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt=prompt, + height=height, + width=width, + prompt_embeds=prompt_embeds, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._current_timestep = None + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. prepare text embeddings + prompt_embeds, text_ids = self.encode_prompt( + prompt=prompt, + prompt_embeds=prompt_embeds, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + text_encoder_out_layers=text_encoder_out_layers, + ) + + # 4. process images + if image is not None and not isinstance(image, list): + image = [image] + + condition_images = None + if image is not None: + for img in image: + self.image_processor.check_image_input(img) + + condition_images = [] + for img in image: + image_width, image_height = img.size + if image_width * image_height > 1024 * 1024: + img = self.image_processor._resize_to_target_area(img, 1024 * 1024) + image_width, image_height = img.size + + multiple_of = self.vae_scale_factor * 2 + image_width = (image_width // multiple_of) * multiple_of + image_height = (image_height // multiple_of) * multiple_of + img = self.image_processor.preprocess(img, height=image_height, width=image_width, resize_mode="crop") + condition_images.append(img) + height = height or image_height + width = width or image_width + + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + # 5. prepare latent variables + num_channels_latents = self.transformer.config.in_channels // 4 + latents, latent_ids = self.prepare_latents( + batch_size=batch_size * num_images_per_prompt, + num_latents_channels=num_channels_latents, + height=height, + width=width, + dtype=prompt_embeds.dtype, + device=device, + generator=generator, + latents=latents, + ) + + image_latents = None + image_latent_ids = None + if condition_images is not None: + image_latents, image_latent_ids = self.prepare_image_latents( + images=condition_images, + batch_size=batch_size * num_images_per_prompt, + generator=generator, + device=device, + dtype=self.vae.dtype, + ) + + # 6. Prepare timesteps + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas + if hasattr(self.scheduler.config, "use_flow_sigmas") and self.scheduler.config.use_flow_sigmas: + sigmas = None + image_seq_len = latents.shape[1] + mu = compute_empirical_mu(image_seq_len=image_seq_len, num_steps=num_inference_steps) + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + sigmas=sigmas, + mu=mu, + ) + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self._num_timesteps = len(timesteps) + + # handle guidance + guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32) + guidance = guidance.expand(latents.shape[0]) + + # 7. Denoising loop + # We set the index here to remove DtoH sync, helpful especially during compilation. + # Check out more details here: https://github.com/huggingface/diffusers/pull/11696 + self.scheduler.set_begin_index(0) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + self._current_timestep = t + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latents.shape[0]).to(latents.dtype) + + latent_model_input = latents.to(self.transformer.dtype) + latent_image_ids = latent_ids + + if image_latents is not None: + latent_model_input = torch.cat([latents, image_latents], dim=1).to(self.transformer.dtype) + latent_image_ids = torch.cat([latent_ids, image_latent_ids], dim=1) + + noise_pred = self.transformer( + hidden_states=latent_model_input, # (B, image_seq_len, C) + timestep=timestep / 1000, + guidance=guidance, + encoder_hidden_states=prompt_embeds, + txt_ids=text_ids, # B, text_seq_len, 4 + img_ids=latent_image_ids, # B, image_seq_len, 4 + joint_attention_kwargs=self._attention_kwargs, + return_dict=False, + )[0] + + noise_pred = noise_pred[:, : latents.size(1) :] + + # compute the previous noisy sample x_t -> x_t-1 + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + if latents.dtype != latents_dtype: + if torch.backends.mps.is_available(): + # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 + latents = latents.to(latents_dtype) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if XLA_AVAILABLE: + xm.mark_step() + + self._current_timestep = None + + if output_type == "latent": + image = latents + else: + torch.save({"pred": latents}, "pred_d.pt") + latents = self._unpack_latents_with_ids(latents, latent_ids) + + latents_bn_mean = self.vae.bn.running_mean.view(1, -1, 1, 1).to(latents.device, latents.dtype) + latents_bn_std = torch.sqrt(self.vae.bn.running_var.view(1, -1, 1, 1) + self.vae.config.batch_norm_eps).to( + latents.device, latents.dtype + ) + latents = latents * latents_bn_std + latents_bn_mean + latents = self._unpatchify_latents(latents) + + image = self.vae.decode(latents, return_dict=False)[0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return Flux2PipelineOutput(images=image) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_flux2_control.py b/VideoX-Fun/videox_fun/pipeline/pipeline_flux2_control.py new file mode 100644 index 0000000000000000000000000000000000000000..f2c5aee517406464be1f6442921b02cd92140eb2 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_flux2_control.py @@ -0,0 +1,973 @@ +# https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/flux2/pipeline_flux2.py +# Copyright 2025 Black Forest Labs and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from diffusers.utils import (BaseOutput, is_torch_xla_available, logging, + replace_example_docstring) +from diffusers.image_processor import VaeImageProcessor +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL +import torch +import torch.nn.functional as F +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import (is_torch_xla_available, logging, + replace_example_docstring) +from diffusers.utils.torch_utils import randn_tensor + +from ..models import (AutoencoderKLFlux2, Flux2ImageProcessor, + Flux2ControlTransformer2DModel, Mistral3ForConditionalGeneration, AutoProcessor) + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import Flux2Pipeline + + >>> pipe = Flux2Pipeline.from_pretrained("black-forest-labs/FLUX.2-dev", torch_dtype=torch.bfloat16) + >>> pipe.to("cuda") + >>> prompt = "A cat holding a sign that says hello world" + >>> # Depending on the variant being used, the pipeline call will slightly vary. + >>> # Refer to the pipeline documentation for more details. + >>> image = pipe(prompt, num_inference_steps=50, guidance_scale=2.5).images[0] + >>> image.save("flux.png") + ``` +""" + + +def format_text_input(prompts: List[str], system_message: str = None): + # Remove [IMG] tokens from prompts to avoid Pixtral validation issues + # when truncation is enabled. The processor counts [IMG] tokens and fails + # if the count changes after truncation. + cleaned_txt = [prompt.replace("[IMG]", "") for prompt in prompts] + + return [ + [ + { + "role": "system", + "content": [{"type": "text", "text": system_message}], + }, + {"role": "user", "content": [{"type": "text", "text": prompt}]}, + ] + for prompt in cleaned_txt + ] + + +def compute_empirical_mu(image_seq_len: int, num_steps: int) -> float: + a1, b1 = 8.73809524e-05, 1.89833333 + a2, b2 = 0.00016927, 0.45666666 + + if image_seq_len > 4300: + mu = a2 * image_seq_len + b2 + return float(mu) + + m_200 = a2 * image_seq_len + b2 + m_10 = a1 * image_seq_len + b1 + + a = (m_200 - m_10) / 190.0 + b = m_200 - 200.0 * a + mu = a * num_steps + b + + return float(mu) + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +@dataclass +class Flux2PipelineOutput(BaseOutput): + """ + Output class for Flux2 image generation pipelines. + + Args: + images (`List[PIL.Image.Image]` or `torch.Tensor` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array or torch tensor of shape `(batch_size, + height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion + pipeline. Torch tensors can represent either the denoised images or the intermediate latents ready to be + passed to the decoder. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +class Flux2ControlPipeline(DiffusionPipeline): + r""" + The Flux2 pipeline for text-to-image generation. + + Reference: [https://bfl.ai/blog/flux-2](https://bfl.ai/blog/flux-2) + + Args: + transformer ([`Flux2ControlTransformer2DModel`]): + Conditional Transformer (MMDiT) architecture to denoise the encoded image latents. + scheduler ([`FlowMatchEulerDiscreteScheduler`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + vae ([`AutoencoderKLFlux2`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`Mistral3ForConditionalGeneration`]): + [Mistral3ForConditionalGeneration](https://huggingface.co/docs/transformers/en/model_doc/mistral3#transformers.Mistral3ForConditionalGeneration) + tokenizer (`AutoProcessor`): + Tokenizer of class + [PixtralProcessor](https://huggingface.co/docs/transformers/en/model_doc/pixtral#transformers.PixtralProcessor). + """ + + model_cpu_offload_seq = "text_encoder->transformer->vae" + _callback_tensor_inputs = ["latents", "prompt_embeds"] + + def __init__( + self, + scheduler: FlowMatchEulerDiscreteScheduler, + vae: AutoencoderKLFlux2, + text_encoder: Mistral3ForConditionalGeneration, + tokenizer: AutoProcessor, + transformer: Flux2ControlTransformer2DModel, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + transformer=transformer, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) if getattr(self, "vae", None) else 8 + # Flux latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible + # by the patch size. So the vae scale factor is multiplied by the patch size to account for this + self.image_processor = Flux2ImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.diffusers_image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + self.tokenizer_max_length = 512 + self.default_sample_size = 128 + + # fmt: off + self.system_message = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation." + # fmt: on + + @staticmethod + def _get_mistral_3_small_prompt_embeds( + text_encoder: Mistral3ForConditionalGeneration, + tokenizer: AutoProcessor, + prompt: Union[str, List[str]], + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + max_sequence_length: int = 512, + # fmt: off + system_message: str = "You are an AI that reasons about image descriptions. You give structured responses focusing on object relationships, object attribution and actions without speculation.", + # fmt: on + hidden_states_layers: List[int] = (10, 20, 30), + ): + dtype = text_encoder.dtype if dtype is None else dtype + device = text_encoder.device if device is None else device + + prompt = [prompt] if isinstance(prompt, str) else prompt + + # Format input messages + messages_batch = format_text_input(prompts=prompt, system_message=system_message) + + # Process all messages at once + inputs = tokenizer.apply_chat_template( + messages_batch, + add_generation_prompt=False, + tokenize=True, + return_dict=True, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=max_sequence_length, + ) + + # Move to device + input_ids = inputs["input_ids"].to(device) + attention_mask = inputs["attention_mask"].to(device) + + # Forward pass through the model + output = text_encoder( + input_ids=input_ids, + attention_mask=attention_mask, + output_hidden_states=True, + use_cache=False, + ) + + # Only use outputs from intermediate layers and stack them + out = torch.stack([output.hidden_states[k] for k in hidden_states_layers], dim=1) + out = out.to(dtype=dtype, device=device) + + batch_size, num_channels, seq_len, hidden_dim = out.shape + prompt_embeds = out.permute(0, 2, 1, 3).reshape(batch_size, seq_len, num_channels * hidden_dim) + + return prompt_embeds + + @staticmethod + def _prepare_text_ids( + x: torch.Tensor, # (B, L, D) or (L, D) + t_coord: Optional[torch.Tensor] = None, + ): + B, L, _ = x.shape + out_ids = [] + + for i in range(B): + t = torch.arange(1) if t_coord is None else t_coord[i] + h = torch.arange(1) + w = torch.arange(1) + l = torch.arange(L) + + coords = torch.cartesian_prod(t, h, w, l) + out_ids.append(coords) + + return torch.stack(out_ids) + + def encode_prompt( + self, + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + text_encoder_out_layers: Tuple[int] = (10, 20, 30), + ): + device = device or self._execution_device + + if prompt is None: + prompt = "" + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt_embeds is None: + prompt_embeds = self._get_mistral_3_small_prompt_embeds( + text_encoder=self.text_encoder, + tokenizer=self.tokenizer, + prompt=prompt, + device=device, + max_sequence_length=max_sequence_length, + system_message=self.system_message, + hidden_states_layers=text_encoder_out_layers, + ) + + batch_size, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + text_ids = self._prepare_text_ids(prompt_embeds) + text_ids = text_ids.to(device) + return prompt_embeds, text_ids + + @staticmethod + def _prepare_latent_ids( + latents: torch.Tensor, # (B, C, H, W) + ): + r""" + Generates 4D position coordinates (T, H, W, L) for latent tensors. + + Args: + latents (torch.Tensor): + Latent tensor of shape (B, C, H, W) + + Returns: + torch.Tensor: + Position IDs tensor of shape (B, H*W, 4) All batches share the same coordinate structure: T=0, + H=[0..H-1], W=[0..W-1], L=0 + """ + + batch_size, _, height, width = latents.shape + + t = torch.arange(1) # [0] - time dimension + h = torch.arange(height) + w = torch.arange(width) + l = torch.arange(1) # [0] - layer dimension + + # Create position IDs: (H*W, 4) + latent_ids = torch.cartesian_prod(t, h, w, l) + + # Expand to batch: (B, H*W, 4) + latent_ids = latent_ids.unsqueeze(0).expand(batch_size, -1, -1) + + return latent_ids + + @staticmethod + def _prepare_image_ids( + image_latents: List[torch.Tensor], # [(1, C, H, W), (1, C, H, W), ...] + scale: int = 10, + ): + r""" + Generates 4D time-space coordinates (T, H, W, L) for a sequence of image latents. + + This function creates a unique coordinate for every pixel/patch across all input latent with different + dimensions. + + Args: + image_latents (List[torch.Tensor]): + A list of image latent feature tensors, typically of shape (C, H, W). + scale (int, optional): + A factor used to define the time separation (T-coordinate) between latents. T-coordinate for the i-th + latent is: 'scale + scale * i'. Defaults to 10. + + Returns: + torch.Tensor: + The combined coordinate tensor. Shape: (1, N_total, 4) Where N_total is the sum of (H * W) for all + input latents. + + Coordinate Components (Dimension 4): + - T (Time): The unique index indicating which latent image the coordinate belongs to. + - H (Height): The row index within that latent image. + - W (Width): The column index within that latent image. + - L (Seq. Length): A sequence length dimension, which is always fixed at 0 (size 1) + """ + + if not isinstance(image_latents, list): + raise ValueError(f"Expected `image_latents` to be a list, got {type(image_latents)}.") + + # create time offset for each reference image + t_coords = [scale + scale * t for t in torch.arange(0, len(image_latents))] + t_coords = [t.view(-1) for t in t_coords] + + image_latent_ids = [] + for x, t in zip(image_latents, t_coords): + x = x.squeeze(0) + _, height, width = x.shape + + x_ids = torch.cartesian_prod(t, torch.arange(height), torch.arange(width), torch.arange(1)) + image_latent_ids.append(x_ids) + + image_latent_ids = torch.cat(image_latent_ids, dim=0) + image_latent_ids = image_latent_ids.unsqueeze(0) + + return image_latent_ids + + @staticmethod + def _patchify_latents(latents): + batch_size, num_channels_latents, height, width = latents.shape + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 1, 3, 5, 2, 4) + latents = latents.reshape(batch_size, num_channels_latents * 4, height // 2, width // 2) + return latents + + @staticmethod + def _unpatchify_latents(latents): + batch_size, num_channels_latents, height, width = latents.shape + latents = latents.reshape(batch_size, num_channels_latents // (2 * 2), 2, 2, height, width) + latents = latents.permute(0, 1, 4, 2, 5, 3) + latents = latents.reshape(batch_size, num_channels_latents // (2 * 2), height * 2, width * 2) + return latents + + @staticmethod + def _pack_latents(latents): + """ + pack latents: (batch_size, num_channels, height, width) -> (batch_size, height * width, num_channels) + """ + + batch_size, num_channels, height, width = latents.shape + latents = latents.reshape(batch_size, num_channels, height * width).permute(0, 2, 1) + + return latents + + @staticmethod + def _unpack_latents_with_ids(x: torch.Tensor, x_ids: torch.Tensor) -> list[torch.Tensor]: + """ + using position ids to scatter tokens into place + """ + x_list = [] + for data, pos in zip(x, x_ids): + _, ch = data.shape # noqa: F841 + h_ids = pos[:, 1].to(torch.int64) + w_ids = pos[:, 2].to(torch.int64) + + h = torch.max(h_ids) + 1 + w = torch.max(w_ids) + 1 + + flat_ids = h_ids * w + w_ids + + out = torch.zeros((h * w, ch), device=data.device, dtype=data.dtype) + out.scatter_(0, flat_ids.unsqueeze(1).expand(-1, ch), data) + + # reshape from (H * W, C) to (H, W, C) and permute to (C, H, W) + + out = out.view(h, w, ch).permute(2, 0, 1) + x_list.append(out) + + return torch.stack(x_list, dim=0) + + def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator): + if image.ndim != 4: + raise ValueError(f"Expected image dims 4, got {image.ndim}.") + + image_latents = retrieve_latents(self.vae.encode(image), generator=generator, sample_mode="argmax") + image_latents = self._patchify_latents(image_latents) + + latents_bn_mean = self.vae.bn.running_mean.view(1, -1, 1, 1).to(image_latents.device, image_latents.dtype) + latents_bn_std = torch.sqrt(self.vae.bn.running_var.view(1, -1, 1, 1) + self.vae.config.batch_norm_eps) + image_latents = (image_latents - latents_bn_mean) / latents_bn_std + + return image_latents + + def prepare_latents( + self, + batch_size, + num_latents_channels, + height, + width, + dtype, + device, + generator: torch.Generator, + latents: Optional[torch.Tensor] = None, + ): + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (self.vae_scale_factor * 2)) + width = 2 * (int(width) // (self.vae_scale_factor * 2)) + + shape = (batch_size, num_latents_channels * 4, height // 2, width // 2) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device=device, dtype=dtype) + + latent_ids = self._prepare_latent_ids(latents) + latent_ids = latent_ids.to(device) + + latents = self._pack_latents(latents) # [B, C, H, W] -> [B, H*W, C] + return latents, latent_ids + + def prepare_image_latents( + self, + images: List[torch.Tensor], + batch_size, + generator: torch.Generator, + device, + dtype, + ): + image_latents = [] + for image in images: + image = image.to(device=device, dtype=dtype) + imagge_latent = self._encode_vae_image(image=image, generator=generator) + image_latents.append(imagge_latent) # (1, 128, 32, 32) + + image_latent_ids = self._prepare_image_ids(image_latents) + + # Pack each latent and concatenate + packed_latents = [] + for latent in image_latents: + # latent: (1, 128, 32, 32) + packed = self._pack_latents(latent) # (1, 1024, 128) + packed = packed.squeeze(0) # (1024, 128) - remove batch dim + packed_latents.append(packed) + + # Concatenate all reference tokens along sequence dimension + image_latents = torch.cat(packed_latents, dim=0) # (N*1024, 128) + image_latents = image_latents.unsqueeze(0) # (1, N*1024, 128) + + image_latents = image_latents.repeat(batch_size, 1, 1) + image_latent_ids = image_latent_ids.repeat(batch_size, 1, 1) + image_latent_ids = image_latent_ids.to(device) + + return image_latents, image_latent_ids + + def check_inputs( + self, + prompt, + height, + width, + prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if ( + height is not None + and height % (self.vae_scale_factor * 2) != 0 + or width is not None + and width % (self.vae_scale_factor * 2) != 0 + ): + logger.warning( + f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} but are {height} and {width}. Dimensions will be resized accordingly" + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def joint_attention_kwargs(self): + return self._joint_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def current_timestep(self): + return self._current_timestep + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + + image: Optional[Union[List[PIL.Image.Image], PIL.Image.Image]] = None, + inpaint_image: Optional[Union[List[PIL.Image.Image], PIL.Image.Image]] = None, + mask_image: Union[torch.FloatTensor] = None, + control_image: Union[torch.FloatTensor] = None, + control_context_scale: float = 1.0, + + num_inference_steps: int = 50, + sigmas: Optional[List[float]] = None, + guidance_scale: Optional[float] = 4.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + text_encoder_out_layers: Tuple[int] = (10, 20, 30), + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + guidance_scale (`float`, *optional*, defaults to 1.0): + Guidance scale as defined in [Classifier-Free Diffusion + Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. + of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting + `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to + the text `prompt`, usually at the expense of lower image quality. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.Tensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will be generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.qwenimage.QwenImagePipelineOutput`] instead of a plain tuple. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`. + text_encoder_out_layers (`Tuple[int]`): + Layer indices to use in the `text_encoder` to derive the final prompt embeddings. + + Examples: + + Returns: + [`~pipelines.flux2.Flux2PipelineOutput`] or `tuple`: [`~pipelines.flux2.Flux2PipelineOutput`] if + `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the + generated images. + """ + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt=prompt, + height=height, + width=width, + prompt_embeds=prompt_embeds, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._current_timestep = None + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + latents_bn_mean = self.vae.bn.running_mean.view(1, -1, 1, 1).to(device, weight_dtype) + latents_bn_std = torch.sqrt(self.vae.bn.running_var.view(1, -1, 1, 1) + self.vae.config.batch_norm_eps).to( + device, weight_dtype + ) + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + num_channels_latents = self.transformer.config.in_channels // 4 + + # Prepare mask latent variables + if mask_image is not None: + mask_condition = self.mask_processor.preprocess(mask_image, height=height, width=width) + mask_condition = torch.tile(mask_condition, [1, 3, 1, 1]).to(dtype=weight_dtype, device=device) + + if inpaint_image is not None: + init_image = self.diffusers_image_processor.preprocess(inpaint_image, height=height, width=width) + init_image = init_image.to(dtype=weight_dtype, device=device) * (mask_condition < 0.5) + inpaint_latent = self.vae.encode(init_image)[0].mode() + else: + inpaint_latent = torch.zeros((batch_size, num_channels_latents * 4, height // 2 // self.vae_scale_factor, width // 2 // self.vae_scale_factor)).to(device, weight_dtype) + + if control_image is not None: + control_image = self.diffusers_image_processor.preprocess(control_image, height=height, width=width) + control_image = control_image.to(dtype=weight_dtype, device=device) + control_latents = self.vae.encode(control_image)[0].mode() + else: + control_latents = torch.zeros_like(inpaint_latent) + + mask_condition = F.interpolate(1 - mask_condition[:, :1], size=control_latents.size()[-2:], mode='nearest').to(device, weight_dtype) + mask_condition = self._patchify_latents(mask_condition) + mask_condition = self._pack_latents(mask_condition) + + if inpaint_image is not None: + inpaint_latent = self._patchify_latents(inpaint_latent) + inpaint_latent = (inpaint_latent - latents_bn_mean) / latents_bn_std + inpaint_latent = self._pack_latents(inpaint_latent) + else: + inpaint_latent = self._patchify_latents(inpaint_latent) + inpaint_latent = self._pack_latents(inpaint_latent) + + if control_image is not None: + control_latents = self._patchify_latents(control_latents) + control_latents = (control_latents - latents_bn_mean) / latents_bn_std + control_latents = self._pack_latents(control_latents) + else: + control_latents = self._patchify_latents(control_latents) + control_latents = self._pack_latents(control_latents) + control_context = torch.concat([control_latents, mask_condition, inpaint_latent], dim=2) + + # 3. prepare text embeddings + prompt_embeds, text_ids = self.encode_prompt( + prompt=prompt, + prompt_embeds=prompt_embeds, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + text_encoder_out_layers=text_encoder_out_layers, + ) + + # 4. process images + if image is not None and not isinstance(image, list): + image = [image] + + condition_images = None + if image is not None: + for img in image: + self.image_processor.check_image_input(img) + + condition_images = [] + for img in image: + image_width, image_height = img.size + if image_width * image_height > 1024 * 1024: + img = self.image_processor._resize_to_target_area(img, 1024 * 1024) + image_width, image_height = img.size + + multiple_of = self.vae_scale_factor * 2 + image_width = (image_width // multiple_of) * multiple_of + image_height = (image_height // multiple_of) * multiple_of + img = self.image_processor.preprocess(img, height=image_height, width=image_width, resize_mode="crop") + condition_images.append(img) + height = height or image_height + width = width or image_width + + # 5. prepare latent variables + latents, latent_ids = self.prepare_latents( + batch_size=batch_size * num_images_per_prompt, + num_latents_channels=num_channels_latents, + height=height, + width=width, + dtype=prompt_embeds.dtype, + device=device, + generator=generator, + latents=latents, + ) + + image_latents = None + image_latent_ids = None + if condition_images is not None: + image_latents, image_latent_ids = self.prepare_image_latents( + images=condition_images, + batch_size=batch_size * num_images_per_prompt, + generator=generator, + device=device, + dtype=self.vae.dtype, + ) + + # 6. Prepare timesteps + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas + if hasattr(self.scheduler.config, "use_flow_sigmas") and self.scheduler.config.use_flow_sigmas: + sigmas = None + image_seq_len = latents.shape[1] + mu = compute_empirical_mu(image_seq_len=image_seq_len, num_steps=num_inference_steps) + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + sigmas=sigmas, + mu=mu, + ) + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self._num_timesteps = len(timesteps) + + # handle guidance + guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32) + guidance = guidance.expand(latents.shape[0]) + + # 7. Denoising loop + # We set the index here to remove DtoH sync, helpful especially during compilation. + # Check out more details here: https://github.com/huggingface/diffusers/pull/11696 + self.scheduler.set_begin_index(0) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + self._current_timestep = t + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latents.shape[0]).to(latents.dtype) + + latent_model_input = latents.to(self.transformer.dtype) + control_context_input = control_context.to(self.transformer.dtype) + latent_image_ids = latent_ids + + if image_latents is not None: + latent_model_input = torch.cat([latents, image_latents], dim=1).to(self.transformer.dtype) + latent_image_ids = torch.cat([latent_ids, image_latent_ids], dim=1) + + local_bs, local_length, local_c = control_context.size() + control_context_input = torch.cat( + [ + control_context, + torch.zeros( + [ + local_bs, + image_latents.size()[1], + local_c + ] + ).to(control_context.device, control_context.dtype)], + dim=1 + ).to(self.transformer.dtype) + + noise_pred = self.transformer( + hidden_states=latent_model_input, # (B, image_seq_len, C) + timestep=timestep / 1000, + guidance=guidance, + encoder_hidden_states=prompt_embeds, + txt_ids=text_ids, # B, text_seq_len, 4 + img_ids=latent_image_ids, # B, image_seq_len, 4 + joint_attention_kwargs=self._attention_kwargs, + control_context=control_context_input, + control_context_scale=control_context_scale, + return_dict=False, + )[0] + + noise_pred = noise_pred[:, : latents.size(1) :] + + # compute the previous noisy sample x_t -> x_t-1 + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + if latents.dtype != latents_dtype: + if torch.backends.mps.is_available(): + # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 + latents = latents.to(latents_dtype) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if XLA_AVAILABLE: + xm.mark_step() + + self._current_timestep = None + + if output_type == "latent": + image = latents + else: + latents = self._unpack_latents_with_ids(latents, latent_ids) + + latents_bn_mean = self.vae.bn.running_mean.view(1, -1, 1, 1).to(latents.device, latents.dtype) + latents_bn_std = torch.sqrt(self.vae.bn.running_var.view(1, -1, 1, 1) + self.vae.config.batch_norm_eps).to( + latents.device, latents.dtype + ) + latents = latents * latents_bn_std + latents_bn_mean + latents = self._unpatchify_latents(latents) + + image = self.vae.decode(latents, return_dict=False)[0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return Flux2PipelineOutput(images=image) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_hunyuanvideo.py b/VideoX-Fun/videox_fun/pipeline/pipeline_hunyuanvideo.py new file mode 100644 index 0000000000000000000000000000000000000000..9afe5c7939b822d4b40f5f8402b2fad08e424f2f --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_hunyuanvideo.py @@ -0,0 +1,805 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video_image2video.py +# Copyright 2025 The HunyuanVideo Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.loaders import HunyuanVideoLoraLoaderMixin +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import (BaseOutput, deprecate, is_torch_xla_available, + logging, replace_example_docstring) +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor + +from ..models import (AutoencoderKLHunyuanVideo, CLIPImageProcessor, + CLIPTextModel, CLIPTokenizer, + HunyuanVideoTransformer3DModel, LlamaModel, + LlamaTokenizerFast, LlavaForConditionalGeneration) + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + >>> import torch + >>> from diffusers import HunyuanVideoPipeline, HunyuanVideoTransformer3DModel + >>> from diffusers.utils import export_to_video + + >>> model_id = "hunyuanvideo-community/HunyuanVideo" + >>> transformer = HunyuanVideoTransformer3DModel.from_pretrained( + ... model_id, subfolder="transformer", torch_dtype=torch.bfloat16 + ... ) + >>> pipe = HunyuanVideoPipeline.from_pretrained(model_id, transformer=transformer, torch_dtype=torch.float16) + >>> pipe.vae.enable_tiling() + >>> pipe.to("cuda") + + >>> output = pipe( + ... prompt="A cat walks on the grass, realistic", + ... height=320, + ... width=512, + ... num_frames=61, + ... num_inference_steps=30, + ... ).frames[0] + >>> export_to_video(output, "output.mp4", fps=15) + ``` +""" + + +DEFAULT_PROMPT_TEMPLATE = { + "template": ( + "<|start_header_id|>system<|end_header_id|>\n\nDescribe the video by detailing the following aspects: " + "1. The main content and theme of the video." + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects." + "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects." + "4. background environment, light, style and atmosphere." + "5. camera angles, movements, and transitions used in the video:<|eot_id|>" + "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + ), + "crop_start": 95, +} + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class HunyuanVideoPipelineOutput(BaseOutput): + r""" + Output class for video pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class HunyuanVideoPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using HunyuanVideo. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + text_encoder ([`LlamaModel`]): + [Llava Llama3-8B](https://huggingface.co/xtuner/llava-llama-3-8b-v1_1-transformers). + tokenizer (`LlamaTokenizer`): + Tokenizer from [Llava Llama3-8B](https://huggingface.co/xtuner/llava-llama-3-8b-v1_1-transformers). + transformer ([`HunyuanVideoTransformer3DModel`]): + Conditional Transformer to denoise the encoded image latents. + scheduler ([`FlowMatchEulerDiscreteScheduler`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + vae ([`AutoencoderKLHunyuanVideo`]): + Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations. + text_encoder_2 ([`CLIPTextModel`]): + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer_2 (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer). + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->transformer->vae" + _callback_tensor_inputs = ["latents", "prompt_embeds"] + + def __init__( + self, + text_encoder: LlamaModel, + tokenizer: LlamaTokenizerFast, + transformer: HunyuanVideoTransformer3DModel, + vae: AutoencoderKLHunyuanVideo, + scheduler: FlowMatchEulerDiscreteScheduler, + text_encoder_2: CLIPTextModel, + tokenizer_2: CLIPTokenizer, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + transformer=transformer, + scheduler=scheduler, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + ) + + self.vae_scale_factor_temporal = self.vae.temporal_compression_ratio if getattr(self, "vae", None) else 4 + self.vae_scale_factor_spatial = self.vae.spatial_compression_ratio if getattr(self, "vae", None) else 8 + self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) + + def _get_llama_prompt_embeds( + self, + prompt: Union[str, List[str]], + prompt_template: Dict[str, Any], + num_videos_per_prompt: int = 1, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + max_sequence_length: int = 256, + num_hidden_layers_to_skip: int = 2, + ) -> Tuple[torch.Tensor, torch.Tensor]: + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + prompt = [prompt_template["template"].format(p) for p in prompt] + + crop_start = prompt_template.get("crop_start", None) + if crop_start is None: + prompt_template_input = self.tokenizer( + prompt_template["template"], + padding="max_length", + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=False, + ) + crop_start = prompt_template_input["input_ids"].shape[-1] + # Remove <|eot_id|> token and placeholder {} + crop_start -= 2 + + max_sequence_length += crop_start + text_inputs = self.tokenizer( + prompt, + max_length=max_sequence_length, + padding="max_length", + truncation=True, + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=True, + ) + text_input_ids = text_inputs.input_ids.to(device=device) + prompt_attention_mask = text_inputs.attention_mask.to(device=device) + + prompt_embeds = self.text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_attention_mask, + output_hidden_states=True, + ).hidden_states[-(num_hidden_layers_to_skip + 1)] + prompt_embeds = prompt_embeds.to(dtype=dtype) + + if crop_start is not None and crop_start > 0: + prompt_embeds = prompt_embeds[:, crop_start:] + prompt_attention_mask = prompt_attention_mask[:, crop_start:] + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + prompt_attention_mask = prompt_attention_mask.repeat(1, num_videos_per_prompt) + prompt_attention_mask = prompt_attention_mask.view(batch_size * num_videos_per_prompt, seq_len) + + return prompt_embeds, prompt_attention_mask + + def _get_clip_prompt_embeds( + self, + prompt: Union[str, List[str]], + num_videos_per_prompt: int = 1, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + max_sequence_length: int = 77, + ) -> torch.Tensor: + device = device or self._execution_device + dtype = dtype or self.text_encoder_2.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer_2( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer_2(prompt, padding="longest", return_tensors="pt").input_ids + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer_2.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {max_sequence_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False).pooler_output + + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, -1) + + return prompt_embeds + + def encode_prompt( + self, + prompt: Union[str, List[str]], + prompt_2: Union[str, List[str]] = None, + prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + pooled_prompt_embeds: Optional[torch.Tensor] = None, + prompt_attention_mask: Optional[torch.Tensor] = None, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + max_sequence_length: int = 256, + ): + if prompt_embeds is None: + prompt_embeds, prompt_attention_mask = self._get_llama_prompt_embeds( + prompt, + prompt_template, + num_videos_per_prompt, + device=device, + dtype=dtype, + max_sequence_length=max_sequence_length, + ) + + if pooled_prompt_embeds is None: + if prompt_2 is None: + prompt_2 = prompt + pooled_prompt_embeds = self._get_clip_prompt_embeds( + prompt, + num_videos_per_prompt, + device=device, + dtype=dtype, + max_sequence_length=77, + ) + + return prompt_embeds, pooled_prompt_embeds, prompt_attention_mask + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + prompt_template=None, + ): + if height % 16 != 0 or width % 16 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if prompt_template is not None: + if not isinstance(prompt_template, dict): + raise ValueError(f"`prompt_template` has to be of type `dict` but is {type(prompt_template)}") + if "template" not in prompt_template: + raise ValueError( + f"`prompt_template` has to contain a key `template` but only found {prompt_template.keys()}" + ) + + def prepare_latents( + self, + batch_size: int, + num_channels_latents: int = 32, + height: int = 720, + width: int = 1280, + num_frames: int = 129, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if latents is not None: + return latents.to(device=device, dtype=dtype) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae_scale_factor_temporal + 1, + int(height) // self.vae_scale_factor_spatial, + int(width) // self.vae_scale_factor_spatial, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + return latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + latents = 1 / self.vae.config.scaling_factor * latents + + frames = self.vae.decode(latents).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + depr_message = f"Calling `enable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_slicing()`." + deprecate( + "enable_vae_slicing", + "0.40.0", + depr_message, + ) + self.vae.enable_slicing() + + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + depr_message = f"Calling `disable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_slicing()`." + deprecate( + "disable_vae_slicing", + "0.40.0", + depr_message, + ) + self.vae.disable_slicing() + + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + depr_message = f"Calling `enable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_tiling()`." + deprecate( + "enable_vae_tiling", + "0.40.0", + depr_message, + ) + self.vae.enable_tiling() + + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + depr_message = f"Calling `disable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_tiling()`." + deprecate( + "disable_vae_tiling", + "0.40.0", + depr_message, + ) + self.vae.disable_tiling() + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def current_timestep(self): + return self._current_timestep + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Union[str, List[str]] = None, + negative_prompt: Union[str, List[str]] = None, + negative_prompt_2: Union[str, List[str]] = None, + height: int = 720, + width: int = 1280, + num_frames: int = 129, + num_inference_steps: int = 50, + sigmas: List[float] = None, + true_cfg_scale: float = 1.0, + guidance_scale: float = 6.0, + num_videos_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + pooled_prompt_embeds: Optional[torch.Tensor] = None, + prompt_attention_mask: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_attention_mask: Optional[torch.Tensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + max_sequence_length: int = 256, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + will be used instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is + not greater than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. + height (`int`, defaults to `720`): + The height in pixels of the generated image. + width (`int`, defaults to `1280`): + The width in pixels of the generated image. + num_frames (`int`, defaults to `129`): + The number of frames in the generated video. + num_inference_steps (`int`, defaults to `50`): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + true_cfg_scale (`float`, *optional*, defaults to 1.0): + True classifier-free guidance (guidance scale) is enabled when `true_cfg_scale` > 1 and + `negative_prompt` is provided. + guidance_scale (`float`, defaults to `6.0`): + Embedded guiddance scale is enabled by setting `guidance_scale` > 1. Higher `guidance_scale` encourages + a model to generate images more aligned with `prompt` at the expense of lower image quality. + + Guidance-distilled models approximates true classifer-free guidance for `guidance_scale` > 1. Refer to + the [paper](https://huggingface.co/papers/2210.03142) to learn more. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.Tensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`HunyuanVideoPipelineOutput`] instead of a plain tuple. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): + A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of + each denoising step during the inference. with the following arguments: `callback_on_step_end(self: + DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a + list of all tensors as specified by `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~HunyuanVideoPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`HunyuanVideoPipelineOutput`] is returned, otherwise a `tuple` is returned + where the first element is a list with the generated images and the second element is a list of `bool`s + indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content. + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + prompt_embeds, + callback_on_step_end_tensor_inputs, + prompt_template, + ) + + has_neg_prompt = negative_prompt is not None or ( + negative_prompt_embeds is not None and negative_pooled_prompt_embeds is not None + ) + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._current_timestep = None + self._interrupt = False + + device = self._execution_device + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # 3. Encode input prompt + transformer_dtype = self.transformer.dtype + prompt_embeds, pooled_prompt_embeds, prompt_attention_mask = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + prompt_template=prompt_template, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + device=device, + max_sequence_length=max_sequence_length, + ) + prompt_embeds = prompt_embeds.to(transformer_dtype) + prompt_attention_mask = prompt_attention_mask.to(transformer_dtype) + pooled_prompt_embeds = pooled_prompt_embeds.to(transformer_dtype) + + if do_true_cfg: + negative_prompt_embeds, negative_pooled_prompt_embeds, negative_prompt_attention_mask = self.encode_prompt( + prompt=negative_prompt, + prompt_2=negative_prompt_2, + prompt_template=prompt_template, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=negative_pooled_prompt_embeds, + prompt_attention_mask=negative_prompt_attention_mask, + device=device, + max_sequence_length=max_sequence_length, + ) + negative_prompt_embeds = negative_prompt_embeds.to(transformer_dtype) + negative_prompt_attention_mask = negative_prompt_attention_mask.to(transformer_dtype) + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.to(transformer_dtype) + + # 4. Prepare timesteps + sigmas = np.linspace(1.0, 0.0, num_inference_steps + 1)[:-1] if sigmas is None else sigmas + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, sigmas=sigmas) + + # 5. Prepare latent variables + num_channels_latents = self.transformer.config.in_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + num_channels_latents, + height, + width, + num_frames, + torch.float32, + device, + generator, + latents, + ) + + # 6. Prepare guidance condition + guidance = torch.tensor([guidance_scale] * latents.shape[0], dtype=transformer_dtype, device=device) * 1000.0 + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + self._current_timestep = t + latent_model_input = latents.to(transformer_dtype) + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latents.shape[0]).to(latents.dtype) + + noise_pred = self.transformer( + hidden_states=latent_model_input, + timestep=timestep, + encoder_hidden_states=prompt_embeds, + encoder_attention_mask=prompt_attention_mask, + pooled_projections=pooled_prompt_embeds, + guidance=guidance, + attention_kwargs=attention_kwargs, + return_dict=False, + )[0] + + if do_true_cfg: + neg_noise_pred = self.transformer( + hidden_states=latent_model_input, + timestep=timestep, + encoder_hidden_states=negative_prompt_embeds, + encoder_attention_mask=negative_prompt_attention_mask, + pooled_projections=negative_pooled_prompt_embeds, + guidance=guidance, + attention_kwargs=attention_kwargs, + return_dict=False, + )[0] + noise_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if XLA_AVAILABLE: + xm.mark_step() + + self._current_timestep = None + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return HunyuanVideoPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_hunyuanvideo_i2v.py b/VideoX-Fun/videox_fun/pipeline/pipeline_hunyuanvideo_i2v.py new file mode 100644 index 0000000000000000000000000000000000000000..e2628ef48c7ba244e8c47e341db960c72454e844 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_hunyuanvideo_i2v.py @@ -0,0 +1,972 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/hunyuan_video/pipeline_hunyuan_video.py +# Copyright 2025 The HunyuanVideo Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL +import torch +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.loaders import HunyuanVideoLoraLoaderMixin +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import (BaseOutput, deprecate, is_torch_xla_available, + logging, replace_example_docstring) +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor + +from ..models import (AutoencoderKLHunyuanVideo, CLIPImageProcessor, + CLIPTextModel, CLIPTokenizer, + HunyuanVideoTransformer3DModel, LlamaModel, + LlamaTokenizerFast, LlavaForConditionalGeneration) + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + ``` +""" + + +DEFAULT_PROMPT_TEMPLATE = { + "template": ( + "<|start_header_id|>system<|end_header_id|>\n\n\nDescribe the video by detailing the following aspects according to the reference image: " + "1. The main content and theme of the video." + "2. The color, shape, size, texture, quantity, text, and spatial relationships of the objects." + "3. Actions, events, behaviors temporal relationships, physical movement changes of the objects." + "4. background environment, light, style and atmosphere." + "5. camera angles, movements, and transitions used in the video:<|eot_id|>\n\n" + "<|start_header_id|>user<|end_header_id|>\n\n{}<|eot_id|>" + "<|start_header_id|>assistant<|end_header_id|>\n\n" + ), + "crop_start": 103, + "image_emb_start": 5, + "image_emb_end": 581, + "image_emb_len": 576, + "double_return_token_id": 271, +} + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def _expand_input_ids_with_image_tokens( + text_input_ids, + prompt_attention_mask, + max_sequence_length, + image_token_index, + image_emb_len, + image_emb_start, + image_emb_end, + pad_token_id, +): + special_image_token_mask = text_input_ids == image_token_index + num_special_image_tokens = torch.sum(special_image_token_mask, dim=-1) + batch_indices, non_image_indices = torch.where(text_input_ids != image_token_index) + + max_expanded_length = max_sequence_length + (num_special_image_tokens.max() * (image_emb_len - 1)) + new_token_positions = torch.cumsum((special_image_token_mask * (image_emb_len - 1) + 1), -1) - 1 + text_to_overwrite = new_token_positions[batch_indices, non_image_indices] + + expanded_input_ids = torch.full( + (text_input_ids.shape[0], max_expanded_length), + pad_token_id, + dtype=text_input_ids.dtype, + device=text_input_ids.device, + ) + expanded_input_ids[batch_indices, text_to_overwrite] = text_input_ids[batch_indices, non_image_indices] + expanded_input_ids[batch_indices, image_emb_start:image_emb_end] = image_token_index + + expanded_attention_mask = torch.zeros( + (text_input_ids.shape[0], max_expanded_length), + dtype=prompt_attention_mask.dtype, + device=prompt_attention_mask.device, + ) + attn_batch_indices, attention_indices = torch.where(expanded_input_ids != pad_token_id) + expanded_attention_mask[attn_batch_indices, attention_indices] = 1.0 + expanded_attention_mask = expanded_attention_mask.to(prompt_attention_mask.dtype) + position_ids = (expanded_attention_mask.cumsum(-1) - 1).masked_fill_((expanded_attention_mask == 0), 1) + + return { + "input_ids": expanded_input_ids, + "attention_mask": expanded_attention_mask, + "position_ids": position_ids, + } + + +@dataclass +class HunyuanVideoPipelineOutput(BaseOutput): + r""" + Output class for video pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + +class HunyuanVideoI2VPipeline(DiffusionPipeline): + r""" + Pipeline for image-to-video generation using HunyuanVideo. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + text_encoder ([`LlamaModel`]): + [Llava Llama3-8B](https://huggingface.co/xtuner/llava-llama-3-8b-v1_1-transformers). + tokenizer (`LlamaTokenizer`): + Tokenizer from [Llava Llama3-8B](https://huggingface.co/xtuner/llava-llama-3-8b-v1_1-transformers). + transformer ([`HunyuanVideoTransformer3DModel`]): + Conditional Transformer to denoise the encoded image latents. + scheduler ([`FlowMatchEulerDiscreteScheduler`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + vae ([`AutoencoderKLHunyuanVideo`]): + Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations. + text_encoder_2 ([`CLIPTextModel`]): + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer_2 (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer). + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->transformer->vae" + _callback_tensor_inputs = ["latents", "prompt_embeds"] + + def __init__( + self, + text_encoder: LlavaForConditionalGeneration, + tokenizer: LlamaTokenizerFast, + transformer: HunyuanVideoTransformer3DModel, + vae: AutoencoderKLHunyuanVideo, + scheduler: FlowMatchEulerDiscreteScheduler, + text_encoder_2: CLIPTextModel, + tokenizer_2: CLIPTokenizer, + image_processor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + transformer=transformer, + scheduler=scheduler, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + image_processor=image_processor, + ) + + self.vae_scaling_factor = self.vae.config.scaling_factor if getattr(self, "vae", None) else 0.476986 + self.vae_scale_factor_temporal = self.vae.temporal_compression_ratio if getattr(self, "vae", None) else 4 + self.vae_scale_factor_spatial = self.vae.spatial_compression_ratio if getattr(self, "vae", None) else 8 + self.video_processor = VideoProcessor(vae_scale_factor=self.vae_scale_factor_spatial) + + def _get_llama_prompt_embeds( + self, + image: torch.Tensor, + prompt: Union[str, List[str]], + prompt_template: Dict[str, Any], + num_videos_per_prompt: int = 1, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + max_sequence_length: int = 256, + num_hidden_layers_to_skip: int = 2, + image_embed_interleave: int = 2, + ) -> Tuple[torch.Tensor, torch.Tensor]: + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + prompt = [prompt_template["template"].format(p) for p in prompt] + + crop_start = prompt_template.get("crop_start", None) + + image_emb_len = prompt_template.get("image_emb_len", 576) + image_emb_start = prompt_template.get("image_emb_start", 5) + image_emb_end = prompt_template.get("image_emb_end", 581) + double_return_token_id = prompt_template.get("double_return_token_id", 271) + + if crop_start is None: + prompt_template_input = self.tokenizer( + prompt_template["template"], + padding="max_length", + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=False, + ) + crop_start = prompt_template_input["input_ids"].shape[-1] + # Remove <|start_header_id|>, <|end_header_id|>, assistant, <|eot_id|>, and placeholder {} + crop_start -= 5 + + max_sequence_length += crop_start + text_inputs = self.tokenizer( + prompt, + max_length=max_sequence_length, + padding="max_length", + truncation=True, + return_tensors="pt", + return_length=False, + return_overflowing_tokens=False, + return_attention_mask=True, + ) + text_input_ids = text_inputs.input_ids.to(device=device) + prompt_attention_mask = text_inputs.attention_mask.to(device=device) + + image_embeds = self.image_processor(image, return_tensors="pt").pixel_values.to(device) + + image_token_index = self.text_encoder.config.image_token_index + pad_token_id = self.text_encoder.config.pad_token_id + expanded_inputs = _expand_input_ids_with_image_tokens( + text_input_ids, + prompt_attention_mask, + max_sequence_length, + image_token_index, + image_emb_len, + image_emb_start, + image_emb_end, + pad_token_id, + ) + prompt_embeds = self.text_encoder( + **expanded_inputs, + pixel_values=image_embeds, + output_hidden_states=True, + ).hidden_states[-(num_hidden_layers_to_skip + 1)] + prompt_embeds = prompt_embeds.to(dtype=dtype) + + if crop_start is not None and crop_start > 0: + text_crop_start = crop_start - 1 + image_emb_len + batch_indices, last_double_return_token_indices = torch.where(text_input_ids == double_return_token_id) + + if last_double_return_token_indices.shape[0] == 3: + # in case the prompt is too long + last_double_return_token_indices = torch.cat( + (last_double_return_token_indices, torch.tensor([text_input_ids.shape[-1]])) + ) + batch_indices = torch.cat((batch_indices, torch.tensor([0]))) + + last_double_return_token_indices = last_double_return_token_indices.reshape(text_input_ids.shape[0], -1)[ + :, -1 + ] + batch_indices = batch_indices.reshape(text_input_ids.shape[0], -1)[:, -1] + assistant_crop_start = last_double_return_token_indices - 1 + image_emb_len - 4 + assistant_crop_end = last_double_return_token_indices - 1 + image_emb_len + attention_mask_assistant_crop_start = last_double_return_token_indices - 4 + attention_mask_assistant_crop_end = last_double_return_token_indices + + prompt_embed_list = [] + prompt_attention_mask_list = [] + image_embed_list = [] + image_attention_mask_list = [] + + for i in range(text_input_ids.shape[0]): + prompt_embed_list.append( + torch.cat( + [ + prompt_embeds[i, text_crop_start : assistant_crop_start[i].item()], + prompt_embeds[i, assistant_crop_end[i].item() :], + ] + ) + ) + prompt_attention_mask_list.append( + torch.cat( + [ + prompt_attention_mask[i, crop_start : attention_mask_assistant_crop_start[i].item()], + prompt_attention_mask[i, attention_mask_assistant_crop_end[i].item() :], + ] + ) + ) + image_embed_list.append(prompt_embeds[i, image_emb_start:image_emb_end]) + image_attention_mask_list.append( + torch.ones(image_embed_list[-1].shape[0]).to(prompt_embeds.device).to(prompt_attention_mask.dtype) + ) + + prompt_embed_list = torch.stack(prompt_embed_list) + prompt_attention_mask_list = torch.stack(prompt_attention_mask_list) + image_embed_list = torch.stack(image_embed_list) + image_attention_mask_list = torch.stack(image_attention_mask_list) + + if 0 < image_embed_interleave < 6: + image_embed_list = image_embed_list[:, ::image_embed_interleave, :] + image_attention_mask_list = image_attention_mask_list[:, ::image_embed_interleave] + + assert ( + prompt_embed_list.shape[0] == prompt_attention_mask_list.shape[0] + and image_embed_list.shape[0] == image_attention_mask_list.shape[0] + ) + + prompt_embeds = torch.cat([image_embed_list, prompt_embed_list], dim=1) + prompt_attention_mask = torch.cat([image_attention_mask_list, prompt_attention_mask_list], dim=1) + + return prompt_embeds, prompt_attention_mask + + def _get_clip_prompt_embeds( + self, + prompt: Union[str, List[str]], + num_videos_per_prompt: int = 1, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + max_sequence_length: int = 77, + ) -> torch.Tensor: + device = device or self._execution_device + dtype = dtype or self.text_encoder_2.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + + text_inputs = self.tokenizer_2( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer_2(prompt, padding="longest", return_tensors="pt").input_ids + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer_2.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {max_sequence_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder_2(text_input_ids.to(device), output_hidden_states=False).pooler_output + return prompt_embeds + + def encode_prompt( + self, + image: torch.Tensor, + prompt: Union[str, List[str]], + prompt_2: Union[str, List[str]] = None, + prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + pooled_prompt_embeds: Optional[torch.Tensor] = None, + prompt_attention_mask: Optional[torch.Tensor] = None, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + max_sequence_length: int = 256, + image_embed_interleave: int = 2, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if prompt_embeds is None: + prompt_embeds, prompt_attention_mask = self._get_llama_prompt_embeds( + image, + prompt, + prompt_template, + num_videos_per_prompt, + device=device, + dtype=dtype, + max_sequence_length=max_sequence_length, + image_embed_interleave=image_embed_interleave, + ) + + if pooled_prompt_embeds is None: + if prompt_2 is None: + prompt_2 = prompt + pooled_prompt_embeds = self._get_clip_prompt_embeds( + prompt, + num_videos_per_prompt, + device=device, + dtype=dtype, + max_sequence_length=77, + ) + + return prompt_embeds, pooled_prompt_embeds, prompt_attention_mask + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + prompt_template=None, + true_cfg_scale=1.0, + guidance_scale=1.0, + ): + if height % 16 != 0 or width % 16 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 16 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if prompt_template is not None: + if not isinstance(prompt_template, dict): + raise ValueError(f"`prompt_template` has to be of type `dict` but is {type(prompt_template)}") + if "template" not in prompt_template: + raise ValueError( + f"`prompt_template` has to contain a key `template` but only found {prompt_template.keys()}" + ) + + if true_cfg_scale > 1.0 and guidance_scale > 1.0: + logger.warning( + "Both `true_cfg_scale` and `guidance_scale` are greater than 1.0. This will result in both " + "classifier-free guidance and embedded-guidance to be applied. This is not recommended " + "as it may lead to higher memory usage, slower inference and potentially worse results." + ) + + def prepare_latents( + self, + image: torch.Tensor, + batch_size: int, + num_channels_latents: int = 32, + height: int = 720, + width: int = 1280, + num_frames: int = 129, + dtype: Optional[torch.dtype] = None, + device: Optional[torch.device] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + num_latent_frames = (num_frames - 1) // self.vae_scale_factor_temporal + 1 + latent_height, latent_width = height // self.vae_scale_factor_spatial, width // self.vae_scale_factor_spatial + shape = (batch_size, num_channels_latents, num_latent_frames, latent_height, latent_width) + + image = image.unsqueeze(2) # [B, C, 1, H, W] + if isinstance(generator, list): + image_latents = [ + retrieve_latents(self.vae.encode(image[i].unsqueeze(0)), generator[i], "argmax") + for i in range(batch_size) + ] + else: + image_latents = [retrieve_latents(self.vae.encode(img.unsqueeze(0)), generator, "argmax") for img in image] + + image_latents = torch.cat(image_latents, dim=0).to(dtype) * self.vae_scaling_factor + image_latents = image_latents.repeat(1, 1, num_latent_frames, 1, 1) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device=device, dtype=dtype) + + t = torch.tensor([0.999]).to(device=device) + latents = latents * t + image_latents * (1 - t) + + image_latents = image_latents[:, :, :1] + return latents, image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + latents = 1 / self.vae.config.scaling_factor * latents + + frames = self.vae.decode(latents).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + depr_message = f"Calling `enable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_slicing()`." + deprecate( + "enable_vae_slicing", + "0.40.0", + depr_message, + ) + self.vae.enable_slicing() + + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + depr_message = f"Calling `disable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_slicing()`." + deprecate( + "disable_vae_slicing", + "0.40.0", + depr_message, + ) + self.vae.disable_slicing() + + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + depr_message = f"Calling `enable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_tiling()`." + deprecate( + "enable_vae_tiling", + "0.40.0", + depr_message, + ) + self.vae.enable_tiling() + + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + depr_message = f"Calling `disable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_tiling()`." + deprecate( + "disable_vae_tiling", + "0.40.0", + depr_message, + ) + self.vae.disable_tiling() + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def current_timestep(self): + return self._current_timestep + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Union[str, List[str]] = None, + negative_prompt: Union[str, List[str]] = None, + negative_prompt_2: Union[str, List[str]] = None, + height: int = 720, + width: int = 1280, + num_frames: int = 129, + num_inference_steps: int = 50, + sigmas: List[float] = None, + true_cfg_scale: float = 1.0, + guidance_scale: float = 6.0, + num_videos_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + pooled_prompt_embeds: Optional[torch.Tensor] = None, + prompt_attention_mask: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_pooled_prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_attention_mask: Optional[torch.Tensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + prompt_template: Dict[str, Any] = DEFAULT_PROMPT_TEMPLATE, + image: PIL.Image.Image = None, + max_sequence_length: int = 256, + image_embed_interleave: Optional[int] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + will be used instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is + not greater than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in all the text-encoders. + height (`int`, defaults to `720`): + The height in pixels of the generated image. + width (`int`, defaults to `1280`): + The width in pixels of the generated image. + num_frames (`int`, defaults to `129`): + The number of frames in the generated video. + num_inference_steps (`int`, defaults to `50`): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + true_cfg_scale (`float`, *optional*, defaults to 1.0): + When > 1.0 and a provided `negative_prompt`, enables true classifier-free guidance. + guidance_scale (`float`, defaults to `1.0`): + Guidance scale as defined in [Classifier-Free Diffusion + Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. + of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting + `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to + the text `prompt`, usually at the expense of lower image quality. Note that the only available + HunyuanVideo model is CFG-distilled, which means that traditional guidance between unconditional and + conditional latent is not applied. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.Tensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`HunyuanVideoPipelineOutput`] instead of a plain tuple. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): + A function or a subclass of `PipelineCallback` or `MultiPipelineCallbacks` that is called at the end of + each denoising step during the inference. with the following arguments: `callback_on_step_end(self: + DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a + list of all tensors as specified by `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~HunyuanVideoPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`HunyuanVideoPipelineOutput`] is returned, otherwise a `tuple` is returned + where the first element is a list with the generated images and the second element is a list of `bool`s + indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content. + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + prompt_embeds, + callback_on_step_end_tensor_inputs, + prompt_template, + true_cfg_scale, + guidance_scale, + ) + + has_neg_prompt = negative_prompt is not None or ( + negative_prompt_embeds is not None and negative_pooled_prompt_embeds is not None + ) + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + image_embed_interleave = ( + image_embed_interleave + if image_embed_interleave is not None + else 4 + ) + + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._current_timestep = None + self._interrupt = False + + device = self._execution_device + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # 3. Prepare latent variables + vae_dtype = self.vae.dtype + image_tensor = self.video_processor.preprocess(image, height, width).to(device, vae_dtype) + + num_channels_latents = self.transformer.config.in_channels + + latents, image_latents = self.prepare_latents( + image_tensor, + batch_size * num_videos_per_prompt, + num_channels_latents, + height, + width, + num_frames, + torch.float32, + device, + generator, + latents, + ) + + # 4. Encode input prompt + transformer_dtype = self.transformer.dtype + prompt_embeds, pooled_prompt_embeds, prompt_attention_mask = self.encode_prompt( + image=image, + prompt=prompt, + prompt_2=prompt_2, + prompt_template=prompt_template, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + device=device, + max_sequence_length=max_sequence_length, + image_embed_interleave=image_embed_interleave, + ) + prompt_embeds = prompt_embeds.to(transformer_dtype) + prompt_attention_mask = prompt_attention_mask.to(transformer_dtype) + pooled_prompt_embeds = pooled_prompt_embeds.to(transformer_dtype) + + if do_true_cfg: + black_image = PIL.Image.new("RGB", (width, height), 0) + negative_prompt_embeds, negative_pooled_prompt_embeds, negative_prompt_attention_mask = self.encode_prompt( + image=black_image, + prompt=negative_prompt, + prompt_2=negative_prompt_2, + prompt_template=prompt_template, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=negative_pooled_prompt_embeds, + prompt_attention_mask=negative_prompt_attention_mask, + device=device, + max_sequence_length=max_sequence_length, + ) + negative_prompt_embeds = negative_prompt_embeds.to(transformer_dtype) + negative_prompt_attention_mask = negative_prompt_attention_mask.to(transformer_dtype) + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.to(transformer_dtype) + + # 5. Prepare timesteps + sigmas = np.linspace(1.0, 0.0, num_inference_steps + 1)[:-1] if sigmas is None else sigmas + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, sigmas=sigmas) + + # 6. Prepare guidance condition + guidance = None + if self.transformer.config.guidance_embeds: + guidance = ( + torch.tensor([guidance_scale] * latents.shape[0], dtype=transformer_dtype, device=device) * 1000.0 + ) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + self._current_timestep = t + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latents.shape[0]).to(latents.dtype) + + latent_model_input = torch.cat([image_latents, latents[:, :, 1:]], dim=2).to(transformer_dtype) + + noise_pred = self.transformer( + hidden_states=latent_model_input, + timestep=timestep, + encoder_hidden_states=prompt_embeds, + encoder_attention_mask=prompt_attention_mask, + pooled_projections=pooled_prompt_embeds, + guidance=guidance, + attention_kwargs=attention_kwargs, + return_dict=False, + )[0] + + if do_true_cfg: + neg_noise_pred = self.transformer( + hidden_states=latent_model_input, + timestep=timestep, + encoder_hidden_states=negative_prompt_embeds, + encoder_attention_mask=negative_prompt_attention_mask, + pooled_projections=negative_pooled_prompt_embeds, + guidance=guidance, + attention_kwargs=attention_kwargs, + return_dict=False, + )[0] + noise_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + + # compute the previous noisy sample x_t -> x_t-1 + latents = latents = self.scheduler.step( + noise_pred[:, :, 1:], t, latents[:, :, 1:], return_dict=False + )[0] + latents = torch.cat([image_latents, latents], dim=2) + latents = latents.to(self.vae.dtype) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if XLA_AVAILABLE: + xm.mark_step() + + self._current_timestep = None + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return HunyuanVideoPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_qwenimage.py b/VideoX-Fun/videox_fun/pipeline/pipeline_qwenimage.py new file mode 100644 index 0000000000000000000000000000000000000000..e61ac0e700bb2ee5757eb5118f0d29325cca0fdf --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_qwenimage.py @@ -0,0 +1,767 @@ +# Modified from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/qwenimage/pipeline_qwenimage.py +# Copyright 2025 Qwen-Image Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from diffusers.image_processor import VaeImageProcessor +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import (BaseOutput, is_torch_xla_available, logging, + replace_example_docstring) +from diffusers.utils.torch_utils import randn_tensor + +from ..models import (AutoencoderKLQwenImage, + Qwen2_5_VLForConditionalGeneration, + Qwen2Tokenizer, QwenImageTransformer2DModel) + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import QwenImagePipeline + + >>> pipe = QwenImagePipeline.from_pretrained("Qwen/Qwen-Image", torch_dtype=torch.bfloat16) + >>> pipe.to("cuda") + >>> prompt = "A cat holding a sign that says hello world" + >>> # Depending on the variant being used, the pipeline call will slightly vary. + >>> # Refer to the pipeline documentation for more details. + >>> image = pipe(prompt, num_inference_steps=50).images[0] + >>> image.save("qwenimage.png") + ``` +""" + + +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class QwenImagePipelineOutput(BaseOutput): + """ + Output class for Stable Diffusion pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +class QwenImagePipeline(DiffusionPipeline): + r""" + The QwenImage pipeline for text-to-image generation. + + Args: + transformer ([`QwenImageTransformer2DModel`]): + Conditional Transformer (MMDiT) architecture to denoise the encoded image latents. + scheduler ([`FlowMatchEulerDiscreteScheduler`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`Qwen2.5-VL-7B-Instruct`]): + [Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct), specifically the + [Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) variant. + tokenizer (`QwenTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer). + """ + + model_cpu_offload_seq = "text_encoder->transformer->vae" + _callback_tensor_inputs = ["latents", "prompt_embeds"] + + def __init__( + self, + scheduler: FlowMatchEulerDiscreteScheduler, + vae: AutoencoderKLQwenImage, + text_encoder: Qwen2_5_VLForConditionalGeneration, + tokenizer: Qwen2Tokenizer, + transformer: QwenImageTransformer2DModel, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + transformer=transformer, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 + # QwenImage latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible + # by the patch size. So the vae scale factor is multiplied by the patch size to account for this + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.tokenizer_max_length = 1024 + self.prompt_template_encode = "<|im_start|>system\nDescribe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" + self.prompt_template_encode_start_idx = 34 + self.default_sample_size = 128 + + def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + + def _get_qwen_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + + template = self.prompt_template_encode + drop_idx = self.prompt_template_encode_start_idx + txt = [template.format(e) for e in prompt] + txt_tokens = self.tokenizer( + txt, max_length=self.tokenizer_max_length + drop_idx, padding=True, truncation=True, return_tensors="pt" + ).to(device) + encoder_hidden_states = self.text_encoder( + input_ids=txt_tokens.input_ids, + attention_mask=txt_tokens.attention_mask, + output_hidden_states=True, + ) + hidden_states = encoder_hidden_states.hidden_states[-1] + split_hidden_states = self._extract_masked_hidden(hidden_states, txt_tokens.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + return prompt_embeds, encoder_attention_mask + + def encode_prompt( + self, + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + prompt_embeds_mask: Optional[torch.Tensor] = None, + max_sequence_length: int = 1024, + ): + r""" + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) if prompt_embeds is None else prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds, prompt_embeds_mask = self._get_qwen_prompt_embeds(prompt, device) + + prompt_embeds = prompt_embeds[:, :max_sequence_length] + prompt_embeds_mask = prompt_embeds_mask[:, :max_sequence_length] + + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1) + prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len) + + return prompt_embeds, prompt_embeds_mask + + def check_inputs( + self, + prompt, + height, + width, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + prompt_embeds_mask=None, + negative_prompt_embeds_mask=None, + callback_on_step_end_tensor_inputs=None, + max_sequence_length=None, + ): + if height % (self.vae_scale_factor * 2) != 0 or width % (self.vae_scale_factor * 2) != 0: + logger.warning( + f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} but are {height} and {width}. Dimensions will be resized accordingly" + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and prompt_embeds_mask is None: + raise ValueError( + "If `prompt_embeds` are provided, `prompt_embeds_mask` also have to be passed. Make sure to generate `prompt_embeds_mask` from the same text encoder that was used to generate `prompt_embeds`." + ) + if negative_prompt_embeds is not None and negative_prompt_embeds_mask is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_prompt_embeds_mask` also have to be passed. Make sure to generate `negative_prompt_embeds_mask` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + if max_sequence_length is not None and max_sequence_length > 1024: + raise ValueError(f"`max_sequence_length` cannot be greater than 1024 but is {max_sequence_length}") + + @staticmethod + def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + + return latents + + @staticmethod + def _unpack_latents(latents, height, width, vae_scale_factor): + batch_size, num_patches, channels = latents.shape + + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + + latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2) + latents = latents.permute(0, 3, 1, 4, 2, 5) + + latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width) + + return latents + + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + ): + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (self.vae_scale_factor * 2)) + width = 2 * (int(width) // (self.vae_scale_factor * 2)) + + shape = (batch_size, 1, num_channels_latents, height, width) + + if latents is not None: + return latents.to(device=device, dtype=dtype) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width) + + return latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def current_timestep(self): + return self._current_timestep + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + negative_prompt: Union[str, List[str]] = None, + true_cfg_scale: float = 4.0, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + sigmas: Optional[List[float]] = None, + guidance_scale: float = 1.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + prompt_embeds_mask: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds_mask: Optional[torch.Tensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + comfyui_progressbar: bool = False, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is + not greater than `1`). + true_cfg_scale (`float`, *optional*, defaults to 1.0): + When > 1.0 and a provided `negative_prompt`, enables true classifier-free guidance. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + guidance_scale (`float`, *optional*, defaults to 3.5): + Guidance scale as defined in [Classifier-Free Diffusion + Guidance](https://huggingface.co/papers/2207.12598). `guidance_scale` is defined as `w` of equation 2. + of [Imagen Paper](https://huggingface.co/papers/2205.11487). Guidance scale is enabled by setting + `guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked to + the text `prompt`, usually at the expense of lower image quality. + + This parameter in the pipeline is there to support future guidance-distilled models when they come up. + Note that passing `guidance_scale` to the pipeline is ineffective. To enable classifier-free guidance, + please pass `true_cfg_scale` and `negative_prompt` (even an empty negative prompt like " ") should + enable classifier-free guidance computations. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.Tensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will be generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.qwenimage.QwenImagePipelineOutput`] instead of a plain tuple. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`. + + Examples: + + Returns: + [`~pipelines.qwenimage.QwenImagePipelineOutput`] or `tuple`: + [`~pipelines.qwenimage.QwenImagePipelineOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is a list with the generated images. + """ + + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + prompt_embeds_mask=prompt_embeds_mask, + negative_prompt_embeds_mask=negative_prompt_embeds_mask, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + max_sequence_length=max_sequence_length, + ) + + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._current_timestep = None + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + has_neg_prompt = negative_prompt is not None or ( + negative_prompt_embeds is not None and negative_prompt_embeds_mask is not None + ) + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + prompt_embeds, prompt_embeds_mask = self.encode_prompt( + prompt=prompt, + prompt_embeds=prompt_embeds, + prompt_embeds_mask=prompt_embeds_mask, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + if do_true_cfg: + negative_prompt_embeds, negative_prompt_embeds_mask = self.encode_prompt( + prompt=negative_prompt, + prompt_embeds=negative_prompt_embeds, + prompt_embeds_mask=negative_prompt_embeds_mask, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + + # 4. Prepare latent variables + num_channels_latents = self.transformer.config.in_channels // 4 + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + img_shapes = [[(1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2)]] * batch_size + if comfyui_progressbar: + pbar.update(1) + + # 5. Prepare timesteps + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + self.scheduler.config.get("base_image_seq_len", 256), + self.scheduler.config.get("max_image_seq_len", 4096), + self.scheduler.config.get("base_shift", 0.5), + self.scheduler.config.get("max_shift", 1.15), + ) + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + sigmas=sigmas, + mu=mu, + ) + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self._num_timesteps = len(timesteps) + + # handle guidance + if self.transformer.config.guidance_embeds: + guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32) + guidance = guidance.expand(latents.shape[0]) + else: + guidance = None + + if self.attention_kwargs is None: + self._attention_kwargs = {} + + txt_seq_lens = prompt_embeds_mask.sum(dim=1).tolist() if prompt_embeds_mask is not None else None + negative_txt_seq_lens = ( + negative_prompt_embeds_mask.sum(dim=1).tolist() if negative_prompt_embeds_mask is not None else None + ) + if comfyui_progressbar: + pbar.update(1) + + # 6. Denoising loop + self.scheduler.set_begin_index(0) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + if self.interrupt: + continue + + if do_true_cfg: + latent_model_input = torch.cat([latents] * 2) + prompt_embeds_mask_input = [_negative_prompt_embeds_mask for _negative_prompt_embeds_mask in negative_prompt_embeds_mask] + [_prompt_embeds_mask for _prompt_embeds_mask in prompt_embeds_mask] + prompt_embeds_input = [_negative_prompt_embeds for _negative_prompt_embeds in negative_prompt_embeds] + [_prompt_embeds for _prompt_embeds in prompt_embeds] + img_shapes_input = img_shapes * 2 + txt_seq_lens_input = negative_txt_seq_lens + txt_seq_lens + else: + latent_model_input = latents + prompt_embeds_mask_input = prompt_embeds_mask + prompt_embeds_input = prompt_embeds + img_shapes_input = img_shapes + txt_seq_lens_input = txt_seq_lens + + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # handle guidance + if self.transformer.config.guidance_embeds: + guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32) + guidance = guidance.expand(latent_model_input.shape[0]) + else: + guidance = None + + self._current_timestep = t + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]).to(latent_model_input.dtype) + + with torch.cuda.amp.autocast(dtype=latents.dtype), torch.cuda.device(device=latents.device): + noise_pred = self.transformer.forward_bs( + x=latent_model_input, + timestep=timestep / 1000, + guidance=guidance, + encoder_hidden_states_mask=prompt_embeds_mask_input, + encoder_hidden_states=prompt_embeds_input, + img_shapes=img_shapes_input, + txt_seq_lens=txt_seq_lens_input, + attention_kwargs=self.attention_kwargs, + return_dict=False, + ) + + if do_true_cfg: + neg_noise_pred, noise_pred = noise_pred.chunk(2) + comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) + noise_pred = comb_pred * (cond_norm / noise_norm) + + # compute the previous noisy sample x_t -> x_t-1 + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + if latents.dtype != latents_dtype: + if torch.backends.mps.is_available(): + # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 + latents = latents.to(latents_dtype) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if XLA_AVAILABLE: + xm.mark_step() + + if comfyui_progressbar: + pbar.update(1) + + self._current_timestep = None + if output_type == "latent": + image = latents + else: + latents = self._unpack_latents(latents, height, width, self.vae_scale_factor) + latents = latents.to(self.vae.dtype) + latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, self.vae.config.z_dim, 1, 1, 1) + .to(latents.device, latents.dtype) + ) + latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to( + latents.device, latents.dtype + ) + latents = latents / latents_std + latents_mean + image = self.vae.decode(latents, return_dict=False)[0][:, :, 0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return QwenImagePipelineOutput(images=image) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_qwenimage_edit.py b/VideoX-Fun/videox_fun/pipeline/pipeline_qwenimage_edit.py new file mode 100644 index 0000000000000000000000000000000000000000..fc0a3d3efed6241a78b5b4fc6c62602a1ee1ed6d --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_qwenimage_edit.py @@ -0,0 +1,952 @@ +# Modified from https://github.com/naykun/diffusers/blob/main/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit.py +# Copyright 2025 Qwen-Image Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import math +import PIL.Image +import torch +from diffusers.image_processor import VaeImageProcessor +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import (BaseOutput, is_torch_xla_available, logging, + replace_example_docstring) +from diffusers.utils.torch_utils import randn_tensor + +from ..models import (AutoencoderKLQwenImage, + Qwen2_5_VLForConditionalGeneration, Qwen2VLProcessor, + Qwen2Tokenizer, QwenImageTransformer2DModel) + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from PIL import Image + >>> from diffusers import QwenImageEditPipeline + + >>> pipe = QwenImageEditPipeline.from_pretrained("Qwen/Qwen-Image-Edit", torch_dtype=torch.bfloat16) + >>> pipe.to("cuda") + >>> prompt = "Change the cat to a dog" + >>> image = Image.open("cat.png") + >>> # Depending on the variant being used, the pipeline call will slightly vary. + >>> # Refer to the pipeline documentation for more details. + >>> image = pipe(image, prompt, num_inference_steps=50).images[0] + >>> image.save("qwenimageedit.png") + ``` +""" +PREFERRED_QWENIMAGE_RESOLUTIONS = [ + (672, 1568), + (688, 1504), + (720, 1456), + (752, 1392), + (800, 1328), + (832, 1248), + (880, 1184), + (944, 1104), + (1024, 1024), + (1104, 944), + (1184, 880), + (1248, 832), + (1328, 800), + (1392, 752), + (1456, 720), + (1504, 688), + (1568, 672), +] + +# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.calculate_shift +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +def calculate_dimensions(target_area, ratio): + width = math.sqrt(target_area * ratio) + height = width / ratio + + width = round(width / 32) * 32 + height = round(height / 32) * 32 + + return width, height + + +@dataclass +class QwenImagePipelineOutput(BaseOutput): + """ + Output class for Stable Diffusion pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +class QwenImageEditPipeline(DiffusionPipeline): + r""" + The QwenImage pipeline for text-to-image generation. + + Args: + transformer ([`QwenImageTransformer2DModel`]): + Conditional Transformer (MMDiT) architecture to denoise the encoded image latents. + scheduler ([`FlowMatchEulerDiscreteScheduler`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`Qwen2.5-VL-7B-Instruct`]): + [Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct), specifically the + [Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) variant. + tokenizer (`QwenTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer). + """ + + model_cpu_offload_seq = "text_encoder->transformer->vae" + _callback_tensor_inputs = ["latents", "prompt_embeds"] + + def __init__( + self, + scheduler: FlowMatchEulerDiscreteScheduler, + vae: AutoencoderKLQwenImage, + text_encoder: Qwen2_5_VLForConditionalGeneration, + tokenizer: Qwen2Tokenizer, + processor: Qwen2VLProcessor, + transformer: QwenImageTransformer2DModel, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + processor=processor, + transformer=transformer, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 + self.latent_channels = self.vae.config.z_dim if getattr(self, "vae", None) else 16 + # QwenImage latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible + # by the patch size. So the vae scale factor is multiplied by the patch size to account for this + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.tokenizer_max_length = 1024 + + self.prompt_template_encode = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>{}<|im_end|>\n<|im_start|>assistant\n" + self.prompt_template_encode_start_idx = 64 + self.default_sample_size = 128 + + # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._extract_masked_hidden + def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + + def _get_qwen_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + image: Optional[torch.Tensor] = None, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + + template = self.prompt_template_encode + drop_idx = self.prompt_template_encode_start_idx + txt = [template.format(e) for e in prompt] + + model_inputs = self.processor( + text=txt, + images=image, + padding=True, + return_tensors="pt", + ).to(device) + + outputs = self.text_encoder( + input_ids=model_inputs.input_ids, + attention_mask=model_inputs.attention_mask, + pixel_values=model_inputs.pixel_values, + image_grid_thw=model_inputs.image_grid_thw, + output_hidden_states=True, + ) + + hidden_states = outputs.hidden_states[-1] + split_hidden_states = self._extract_masked_hidden(hidden_states, model_inputs.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + return prompt_embeds, encoder_attention_mask + + def encode_prompt( + self, + prompt: Union[str, List[str]], + image: Optional[torch.Tensor] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + prompt_embeds_mask: Optional[torch.Tensor] = None, + max_sequence_length: int = 1024, + ): + r""" + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + image (`torch.Tensor`, *optional*): + image to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) if prompt_embeds is None else prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds, prompt_embeds_mask = self._get_qwen_prompt_embeds(prompt, image, device) + + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1) + prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len) + + return prompt_embeds, prompt_embeds_mask + + def check_inputs( + self, + prompt, + height, + width, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + prompt_embeds_mask=None, + negative_prompt_embeds_mask=None, + callback_on_step_end_tensor_inputs=None, + max_sequence_length=None, + ): + if height % (self.vae_scale_factor * 2) != 0 or width % (self.vae_scale_factor * 2) != 0: + logger.warning( + f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} but are {height} and {width}. Dimensions will be resized accordingly" + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and prompt_embeds_mask is None: + raise ValueError( + "If `prompt_embeds` are provided, `prompt_embeds_mask` also have to be passed. Make sure to generate `prompt_embeds_mask` from the same text encoder that was used to generate `prompt_embeds`." + ) + if negative_prompt_embeds is not None and negative_prompt_embeds_mask is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_prompt_embeds_mask` also have to be passed. Make sure to generate `negative_prompt_embeds_mask` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + if max_sequence_length is not None and max_sequence_length > 1024: + raise ValueError(f"`max_sequence_length` cannot be greater than 1024 but is {max_sequence_length}") + + @staticmethod + # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._pack_latents + def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + + return latents + + @staticmethod + # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._unpack_latents + def _unpack_latents(latents, height, width, vae_scale_factor): + batch_size, num_patches, channels = latents.shape + + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + + latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2) + latents = latents.permute(0, 3, 1, 4, 2, 5) + + latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width) + + return latents + + def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator): + if isinstance(generator, list): + image_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i], sample_mode="argmax") + for i in range(image.shape[0]) + ] + image_latents = torch.cat(image_latents, dim=0) + else: + image_latents = retrieve_latents(self.vae.encode(image), generator=generator, sample_mode="argmax") + latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, self.latent_channels, 1, 1, 1) + .to(image_latents.device, image_latents.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std) + .view(1, self.latent_channels, 1, 1, 1) + .to(image_latents.device, image_latents.dtype) + ) + image_latents = (image_latents - latents_mean) / latents_std + + return image_latents + + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + depr_message = f"Calling `enable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_slicing()`." + deprecate( + "enable_vae_slicing", + "0.40.0", + depr_message, + ) + self.vae.enable_slicing() + + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + depr_message = f"Calling `disable_vae_slicing()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_slicing()`." + deprecate( + "disable_vae_slicing", + "0.40.0", + depr_message, + ) + self.vae.disable_slicing() + + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + depr_message = f"Calling `enable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.enable_tiling()`." + deprecate( + "enable_vae_tiling", + "0.40.0", + depr_message, + ) + self.vae.enable_tiling() + + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + depr_message = f"Calling `disable_vae_tiling()` on a `{self.__class__.__name__}` is deprecated and this method will be removed in a future version. Please use `pipe.vae.disable_tiling()`." + deprecate( + "disable_vae_tiling", + "0.40.0", + depr_message, + ) + self.vae.disable_tiling() + + def prepare_latents( + self, + image, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + ): + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (self.vae_scale_factor * 2)) + width = 2 * (int(width) // (self.vae_scale_factor * 2)) + + shape = (batch_size, 1, num_channels_latents, height, width) + + image_latents = None + if image is not None: + image = image.to(device=device, dtype=dtype) + if image.shape[1] != self.latent_channels: + image_latents = self._encode_vae_image(image=image, generator=generator) + else: + image_latents = image + if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0: + # expand init_latents for batch_size + additional_image_per_prompt = batch_size // image_latents.shape[0] + image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0) + elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts." + ) + else: + image_latents = torch.cat([image_latents], dim=0) + + image_latent_height, image_latent_width = image_latents.shape[3:] + image_latents = self._pack_latents( + image_latents, batch_size, num_channels_latents, image_latent_height, image_latent_width + ) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width) + else: + latents = latents.to(device=device, dtype=dtype) + + return latents, image_latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def current_timestep(self): + return self._current_timestep + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image = None, + prompt: Union[str, List[str]] = None, + negative_prompt: Union[str, List[str]] = None, + true_cfg_scale: float = 4.0, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + sigmas: Optional[List[float]] = None, + guidance_scale: Optional[float] = None, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + prompt_embeds_mask: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds_mask: Optional[torch.Tensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + comfyui_progressbar: bool = False, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is + not greater than `1`). + true_cfg_scale (`float`, *optional*, defaults to 1.0): + true_cfg_scale (`float`, *optional*, defaults to 1.0): Guidance scale as defined in [Classifier-Free + Diffusion Guidance](https://huggingface.co/papers/2207.12598). `true_cfg_scale` is defined as `w` of + equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Classifier-free guidance is + enabled by setting `true_cfg_scale > 1` and a provided `negative_prompt`. Higher guidance scale + encourages to generate images that are closely linked to the text `prompt`, usually at the expense of + lower image quality. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + guidance_scale (`float`, *optional*, defaults to None): + A guidance scale value for guidance distilled models. Unlike the traditional classifier-free guidance + where the guidance scale is applied during inference through noise prediction rescaling, guidance + distilled models take the guidance scale directly as an input parameter during forward pass. Guidance + scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images + that are closely linked to the text `prompt`, usually at the expense of lower image quality. This + parameter in the pipeline is there to support future guidance-distilled models when they come up. It is + ignored when not using guidance distilled models. To enable traditional classifier-free guidance, + please pass `true_cfg_scale > 1.0` and `negative_prompt` (even an empty negative prompt like " " should + enable classifier-free guidance computations). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.Tensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will be generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.qwenimage.QwenImagePipelineOutput`] instead of a plain tuple. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`. + + Examples: + + Returns: + [`~pipelines.qwenimage.QwenImagePipelineOutput`] or `tuple`: + [`~pipelines.qwenimage.QwenImagePipelineOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is a list with the generated images. + """ + image_size = image[0].size if isinstance(image, list) else image.size + calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] / image_size[1]) + height = height or calculated_height + width = width or calculated_width + + multiple_of = self.vae_scale_factor * 2 + width = width // multiple_of * multiple_of + height = height // multiple_of * multiple_of + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + prompt_embeds_mask=prompt_embeds_mask, + negative_prompt_embeds_mask=negative_prompt_embeds_mask, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + max_sequence_length=max_sequence_length, + ) + + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._current_timestep = None + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + # 3. Preprocess image + if image is not None and not (isinstance(image, torch.Tensor) and image.size(1) == self.latent_channels): + image = self.image_processor.resize(image, calculated_height, calculated_width) + prompt_image = image + image = self.image_processor.preprocess(image, calculated_height, calculated_width) + image = image.unsqueeze(2) + + has_neg_prompt = negative_prompt is not None or ( + negative_prompt_embeds is not None and negative_prompt_embeds_mask is not None + ) + + if true_cfg_scale > 1 and not has_neg_prompt: + logger.warning( + f"true_cfg_scale is passed as {true_cfg_scale}, but classifier-free guidance is not enabled since no negative_prompt is provided." + ) + elif true_cfg_scale <= 1 and has_neg_prompt: + logger.warning( + " negative_prompt is passed but classifier-free guidance is not enabled since true_cfg_scale <= 1" + ) + + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + prompt_embeds, prompt_embeds_mask = self.encode_prompt( + image=prompt_image, + prompt=prompt, + prompt_embeds=prompt_embeds, + prompt_embeds_mask=prompt_embeds_mask, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + if do_true_cfg: + negative_prompt_embeds, negative_prompt_embeds_mask = self.encode_prompt( + image=prompt_image, + prompt=negative_prompt, + prompt_embeds=negative_prompt_embeds, + prompt_embeds_mask=negative_prompt_embeds_mask, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + if comfyui_progressbar: + pbar.update(1) + + # 4. Prepare latent variables + num_channels_latents = self.transformer.config.in_channels // 4 + latents, image_latents = self.prepare_latents( + image, + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + img_shapes = [ + [ + (1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2), + (1, calculated_height // self.vae_scale_factor // 2, calculated_width // self.vae_scale_factor // 2), + ] + ] * batch_size + + # 5. Prepare timesteps + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + self.scheduler.config.get("base_image_seq_len", 256), + self.scheduler.config.get("max_image_seq_len", 4096), + self.scheduler.config.get("base_shift", 0.5), + self.scheduler.config.get("max_shift", 1.15), + ) + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + sigmas=sigmas, + mu=mu, + ) + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self._num_timesteps = len(timesteps) + + # handle guidance + if self.transformer.config.guidance_embeds and guidance_scale is None: + raise ValueError("guidance_scale is required for guidance-distilled model.") + elif self.transformer.config.guidance_embeds: + guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32) + guidance = guidance.expand(latents.shape[0]) + elif not self.transformer.config.guidance_embeds and guidance_scale is not None: + logger.warning( + f"guidance_scale is passed as {guidance_scale}, but ignored since the model is not guidance-distilled." + ) + guidance = None + elif not self.transformer.config.guidance_embeds and guidance_scale is None: + guidance = None + + if self.attention_kwargs is None: + self._attention_kwargs = {} + + txt_seq_lens = prompt_embeds_mask.sum(dim=1).tolist() if prompt_embeds_mask is not None else None + negative_txt_seq_lens = ( + negative_prompt_embeds_mask.sum(dim=1).tolist() if negative_prompt_embeds_mask is not None else None + ) + if comfyui_progressbar: + pbar.update(1) + + # 6. Denoising loop + self.scheduler.set_begin_index(0) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + if self.interrupt: + continue + + if image_latents is not None: + latents_and_image_latents = torch.cat([latents, image_latents], dim=1) + else: + latents_and_image_latents = latents + + if do_true_cfg: + latent_model_input = torch.cat([latents_and_image_latents] * 2) + prompt_embeds_mask_input = [_negative_prompt_embeds_mask for _negative_prompt_embeds_mask in negative_prompt_embeds_mask] + [_prompt_embeds_mask for _prompt_embeds_mask in prompt_embeds_mask] + prompt_embeds_input = [_negative_prompt_embeds for _negative_prompt_embeds in negative_prompt_embeds] + [_prompt_embeds for _prompt_embeds in prompt_embeds] + img_shapes_input = img_shapes * 2 + txt_seq_lens_input = negative_txt_seq_lens + txt_seq_lens + else: + latent_model_input = latents_and_image_latents + prompt_embeds_mask_input = prompt_embeds_mask + prompt_embeds_input = prompt_embeds + img_shapes_input = img_shapes + txt_seq_lens_input = txt_seq_lens + + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # handle guidance + if self.transformer.config.guidance_embeds: + guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32) + guidance = guidance.expand(latent_model_input.shape[0]) + else: + guidance = None + + self._current_timestep = t + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]).to(latent_model_input.dtype) + + with torch.cuda.amp.autocast(dtype=latents.dtype), torch.cuda.device(device=latents.device): + noise_pred = self.transformer.forward_bs( + x=latent_model_input, + timestep=timestep / 1000, + guidance=guidance, + encoder_hidden_states_mask=prompt_embeds_mask_input, + encoder_hidden_states=prompt_embeds_input, + img_shapes=img_shapes_input, + txt_seq_lens=txt_seq_lens_input, + attention_kwargs=self.attention_kwargs, + return_dict=False, + ) + noise_pred = noise_pred[:, : latents.size(1)] + + if do_true_cfg: + neg_noise_pred, noise_pred = noise_pred.chunk(2) + comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) + noise_pred = comb_pred * (cond_norm / noise_norm) + + # compute the previous noisy sample x_t -> x_t-1 + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + if latents.dtype != latents_dtype: + if torch.backends.mps.is_available(): + # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 + latents = latents.to(latents_dtype) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if XLA_AVAILABLE: + xm.mark_step() + + if comfyui_progressbar: + pbar.update(1) + + self._current_timestep = None + if output_type == "latent": + image = latents + else: + latents = self._unpack_latents(latents, height, width, self.vae_scale_factor) + latents = latents.to(self.vae.dtype) + latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, self.vae.config.z_dim, 1, 1, 1) + .to(latents.device, latents.dtype) + ) + latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to( + latents.device, latents.dtype + ) + latents = latents / latents_std + latents_mean + image = self.vae.decode(latents, return_dict=False)[0][:, :, 0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return QwenImagePipelineOutput(images=image) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_qwenimage_edit_plus.py b/VideoX-Fun/videox_fun/pipeline/pipeline_qwenimage_edit_plus.py new file mode 100644 index 0000000000000000000000000000000000000000..885550cc2d3529360e09b89c13b96511bbe1c9f1 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_qwenimage_edit_plus.py @@ -0,0 +1,937 @@ +# Modified from https://github.com/naykun/diffusers/blob/main/src/diffusers/pipelines/qwenimage/pipeline_qwenimage_edit.py +# Copyright 2025 Qwen-Image Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import math +import PIL.Image +import torch +from diffusers.image_processor import VaeImageProcessor +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import (BaseOutput, is_torch_xla_available, logging, + replace_example_docstring) +from diffusers.utils.torch_utils import randn_tensor + +from ..models import (AutoencoderKLQwenImage, + Qwen2_5_VLForConditionalGeneration, Qwen2VLProcessor, + Qwen2Tokenizer, QwenImageTransformer2DModel) + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from PIL import Image + >>> from diffusers import QwenImageEditPipeline + + >>> pipe = QwenImageEditPipeline.from_pretrained("Qwen/Qwen-Image-Edit", torch_dtype=torch.bfloat16) + >>> pipe.to("cuda") + >>> prompt = "Change the cat to a dog" + >>> image = Image.open("cat.png") + >>> # Depending on the variant being used, the pipeline call will slightly vary. + >>> # Refer to the pipeline documentation for more details. + >>> image = pipe(image, prompt, num_inference_steps=50).images[0] + >>> image.save("qwenimageedit.png") + ``` +""" + +CONDITION_IMAGE_SIZE = 384 * 384 +VAE_IMAGE_SIZE = 1024 * 1024 + +PREFERRED_QWENIMAGE_RESOLUTIONS = [ + (672, 1568), + (688, 1504), + (720, 1456), + (752, 1392), + (800, 1328), + (832, 1248), + (880, 1184), + (944, 1104), + (1024, 1024), + (1104, 944), + (1184, 880), + (1248, 832), + (1328, 800), + (1392, 752), + (1456, 720), + (1504, 688), + (1568, 672), +] + +# Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.calculate_shift +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +def calculate_dimensions(target_area, ratio): + width = math.sqrt(target_area * ratio) + height = width / ratio + + width = round(width / 32) * 32 + height = round(height / 32) * 32 + + return width, height + + +@dataclass +class QwenImagePipelineOutput(BaseOutput): + """ + Output class for Stable Diffusion pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +class QwenImageEditPlusPipeline(DiffusionPipeline): + r""" + The QwenImage pipeline for text-to-image generation. + + Args: + transformer ([`QwenImageTransformer2DModel`]): + Conditional Transformer (MMDiT) architecture to denoise the encoded image latents. + scheduler ([`FlowMatchEulerDiscreteScheduler`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`Qwen2.5-VL-7B-Instruct`]): + [Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct), specifically the + [Qwen2.5-VL-7B-Instruct](https://huggingface.co/Qwen/Qwen2.5-VL-7B-Instruct) variant. + tokenizer (`QwenTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/en/model_doc/clip#transformers.CLIPTokenizer). + """ + + model_cpu_offload_seq = "text_encoder->transformer->vae" + _callback_tensor_inputs = ["latents", "prompt_embeds"] + + def __init__( + self, + scheduler: FlowMatchEulerDiscreteScheduler, + vae: AutoencoderKLQwenImage, + text_encoder: Qwen2_5_VLForConditionalGeneration, + tokenizer: Qwen2Tokenizer, + processor: Qwen2VLProcessor, + transformer: QwenImageTransformer2DModel, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + processor=processor, + transformer=transformer, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** len(self.vae.temperal_downsample) if getattr(self, "vae", None) else 8 + self.latent_channels = self.vae.config.z_dim if getattr(self, "vae", None) else 16 + # QwenImage latents are turned into 2x2 patches and packed. This means the latent width and height has to be divisible + # by the patch size. So the vae scale factor is multiplied by the patch size to account for this + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.tokenizer_max_length = 1024 + + self.prompt_template_encode = "<|im_start|>system\nDescribe the key features of the input image (color, shape, size, texture, objects, background), then explain how the user's text instruction should alter or modify the image. Generate a new image that meets the user's requirements while maintaining consistency with the original input where appropriate.<|im_end|>\n<|im_start|>user\n{}<|im_end|>\n<|im_start|>assistant\n" + self.prompt_template_encode_start_idx = 64 + self.default_sample_size = 128 + + # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._extract_masked_hidden + def _extract_masked_hidden(self, hidden_states: torch.Tensor, mask: torch.Tensor): + bool_mask = mask.bool() + valid_lengths = bool_mask.sum(dim=1) + selected = hidden_states[bool_mask] + split_result = torch.split(selected, valid_lengths.tolist(), dim=0) + + return split_result + + def _get_qwen_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + image: Optional[torch.Tensor] = None, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + img_prompt_template = "Picture {}: <|vision_start|><|image_pad|><|vision_end|>" + if isinstance(image, list): + base_img_prompt = "" + for i, img in enumerate(image): + base_img_prompt += img_prompt_template.format(i + 1) + elif image is not None: + base_img_prompt = img_prompt_template.format(1) + else: + base_img_prompt = "" + + template = self.prompt_template_encode + + drop_idx = self.prompt_template_encode_start_idx + txt = [template.format(base_img_prompt + e) for e in prompt] + + model_inputs = self.processor( + text=txt, + images=image, + padding=True, + return_tensors="pt", + ).to(device) + + outputs = self.text_encoder( + input_ids=model_inputs.input_ids, + attention_mask=model_inputs.attention_mask, + pixel_values=model_inputs.pixel_values, + image_grid_thw=model_inputs.image_grid_thw, + output_hidden_states=True, + ) + + hidden_states = outputs.hidden_states[-1] + split_hidden_states = self._extract_masked_hidden(hidden_states, model_inputs.attention_mask) + split_hidden_states = [e[drop_idx:] for e in split_hidden_states] + attn_mask_list = [torch.ones(e.size(0), dtype=torch.long, device=e.device) for e in split_hidden_states] + max_seq_len = max([e.size(0) for e in split_hidden_states]) + prompt_embeds = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0), u.size(1))]) for u in split_hidden_states] + ) + encoder_attention_mask = torch.stack( + [torch.cat([u, u.new_zeros(max_seq_len - u.size(0))]) for u in attn_mask_list] + ) + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + return prompt_embeds, encoder_attention_mask + + # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit.QwenImageEditPipeline.encode_prompt + def encode_prompt( + self, + prompt: Union[str, List[str]], + image: Optional[torch.Tensor] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + prompt_embeds_mask: Optional[torch.Tensor] = None, + max_sequence_length: int = 1024, + ): + r""" + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + image (`torch.Tensor`, *optional*): + image to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) if prompt_embeds is None else prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds, prompt_embeds_mask = self._get_qwen_prompt_embeds(prompt, image, device) + + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + prompt_embeds_mask = prompt_embeds_mask.repeat(1, num_images_per_prompt, 1) + prompt_embeds_mask = prompt_embeds_mask.view(batch_size * num_images_per_prompt, seq_len) + + return prompt_embeds, prompt_embeds_mask + + # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit.QwenImageEditPipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + prompt_embeds_mask=None, + negative_prompt_embeds_mask=None, + callback_on_step_end_tensor_inputs=None, + max_sequence_length=None, + ): + if height % (self.vae_scale_factor * 2) != 0 or width % (self.vae_scale_factor * 2) != 0: + logger.warning( + f"`height` and `width` have to be divisible by {self.vae_scale_factor * 2} but are {height} and {width}. Dimensions will be resized accordingly" + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and prompt_embeds_mask is None: + raise ValueError( + "If `prompt_embeds` are provided, `prompt_embeds_mask` also have to be passed. Make sure to generate `prompt_embeds_mask` from the same text encoder that was used to generate `prompt_embeds`." + ) + if negative_prompt_embeds is not None and negative_prompt_embeds_mask is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_prompt_embeds_mask` also have to be passed. Make sure to generate `negative_prompt_embeds_mask` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + if max_sequence_length is not None and max_sequence_length > 1024: + raise ValueError(f"`max_sequence_length` cannot be greater than 1024 but is {max_sequence_length}") + + @staticmethod + # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._pack_latents + def _pack_latents(latents, batch_size, num_channels_latents, height, width): + latents = latents.view(batch_size, num_channels_latents, height // 2, 2, width // 2, 2) + latents = latents.permute(0, 2, 4, 1, 3, 5) + latents = latents.reshape(batch_size, (height // 2) * (width // 2), num_channels_latents * 4) + + return latents + + @staticmethod + # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage.QwenImagePipeline._unpack_latents + def _unpack_latents(latents, height, width, vae_scale_factor): + batch_size, num_patches, channels = latents.shape + + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (vae_scale_factor * 2)) + width = 2 * (int(width) // (vae_scale_factor * 2)) + + latents = latents.view(batch_size, height // 2, width // 2, channels // 4, 2, 2) + latents = latents.permute(0, 3, 1, 4, 2, 5) + + latents = latents.reshape(batch_size, channels // (2 * 2), 1, height, width) + + return latents + + # Copied from diffusers.pipelines.qwenimage.pipeline_qwenimage_edit.QwenImageEditPipeline._encode_vae_image + def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator): + if isinstance(generator, list): + image_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i], sample_mode="argmax") + for i in range(image.shape[0]) + ] + image_latents = torch.cat(image_latents, dim=0) + else: + image_latents = retrieve_latents(self.vae.encode(image), generator=generator, sample_mode="argmax") + latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, self.latent_channels, 1, 1, 1) + .to(image_latents.device, image_latents.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std) + .view(1, self.latent_channels, 1, 1, 1) + .to(image_latents.device, image_latents.dtype) + ) + image_latents = (image_latents - latents_mean) / latents_std + + return image_latents + + def prepare_latents( + self, + images, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + ): + # VAE applies 8x compression on images but we must also account for packing which requires + # latent height and width to be divisible by 2. + height = 2 * (int(height) // (self.vae_scale_factor * 2)) + width = 2 * (int(width) // (self.vae_scale_factor * 2)) + + shape = (batch_size, 1, num_channels_latents, height, width) + + image_latents = None + if images is not None: + if not isinstance(images, list): + images = [images] + all_image_latents = [] + for image in images: + image = image.to(device=device, dtype=dtype) + if image.shape[1] != self.latent_channels: + image_latents = self._encode_vae_image(image=image, generator=generator) + else: + image_latents = image + if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0: + # expand init_latents for batch_size + additional_image_per_prompt = batch_size // image_latents.shape[0] + image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0) + elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts." + ) + else: + image_latents = torch.cat([image_latents], dim=0) + + image_latent_height, image_latent_width = image_latents.shape[3:] + image_latents = self._pack_latents( + image_latents, batch_size, num_channels_latents, image_latent_height, image_latent_width + ) + all_image_latents.append(image_latents) + image_latents = torch.cat(all_image_latents, dim=1) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = self._pack_latents(latents, batch_size, num_channels_latents, height, width) + else: + latents = latents.to(device=device, dtype=dtype) + + return latents, image_latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def current_timestep(self): + return self._current_timestep + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image = None, + prompt: Union[str, List[str]] = None, + negative_prompt: Union[str, List[str]] = None, + true_cfg_scale: float = 4.0, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + sigmas: Optional[List[float]] = None, + guidance_scale: Optional[float] = None, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.Tensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + prompt_embeds_mask: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds_mask: Optional[torch.Tensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + comfyui_progressbar: bool = False, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + image (`torch.Tensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `true_cfg_scale` is + not greater than `1`). + true_cfg_scale (`float`, *optional*, defaults to 1.0): + true_cfg_scale (`float`, *optional*, defaults to 1.0): Guidance scale as defined in [Classifier-Free + Diffusion Guidance](https://huggingface.co/papers/2207.12598). `true_cfg_scale` is defined as `w` of + equation 2. of [Imagen Paper](https://huggingface.co/papers/2205.11487). Classifier-free guidance is + enabled by setting `true_cfg_scale > 1` and a provided `negative_prompt`. Higher guidance scale + encourages to generate images that are closely linked to the text `prompt`, usually at the expense of + lower image quality. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + guidance_scale (`float`, *optional*, defaults to None): + A guidance scale value for guidance distilled models. Unlike the traditional classifier-free guidance + where the guidance scale is applied during inference through noise prediction rescaling, guidance + distilled models take the guidance scale directly as an input parameter during forward pass. Guidance + scale is enabled by setting `guidance_scale > 1`. Higher guidance scale encourages to generate images + that are closely linked to the text `prompt`, usually at the expense of lower image quality. This + parameter in the pipeline is there to support future guidance-distilled models when they come up. It is + ignored when not using guidance distilled models. To enable traditional classifier-free guidance, + please pass `true_cfg_scale > 1.0` and `negative_prompt` (even an empty negative prompt like " " should + enable classifier-free guidance computations). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.Tensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will be generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.qwenimage.QwenImagePipelineOutput`] instead of a plain tuple. + attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int` defaults to 512): Maximum sequence length to use with the `prompt`. + + Examples: + + Returns: + [`~pipelines.qwenimage.QwenImagePipelineOutput`] or `tuple`: + [`~pipelines.qwenimage.QwenImagePipelineOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is a list with the generated images. + """ + image_size = image[-1].size if isinstance(image, list) else image.size + calculated_width, calculated_height = calculate_dimensions(1024 * 1024, image_size[0] / image_size[1]) + height = height or calculated_height + width = width or calculated_width + + multiple_of = self.vae_scale_factor * 2 + width = width // multiple_of * multiple_of + height = height // multiple_of * multiple_of + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + prompt_embeds_mask=prompt_embeds_mask, + negative_prompt_embeds_mask=negative_prompt_embeds_mask, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + max_sequence_length=max_sequence_length, + ) + + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._current_timestep = None + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + # 3. Preprocess image + if image is not None and not (isinstance(image, torch.Tensor) and image.size(1) == self.latent_channels): + if not isinstance(image, list): + image = [image] + condition_image_sizes = [] + condition_images = [] + vae_image_sizes = [] + vae_images = [] + for img in image: + image_width, image_height = img.size + condition_width, condition_height = calculate_dimensions( + CONDITION_IMAGE_SIZE, image_width / image_height + ) + vae_width, vae_height = calculate_dimensions(VAE_IMAGE_SIZE, image_width / image_height) + condition_image_sizes.append((condition_width, condition_height)) + vae_image_sizes.append((vae_width, vae_height)) + condition_images.append(self.image_processor.resize(img, condition_height, condition_width)) + vae_images.append(self.image_processor.preprocess(img, vae_height, vae_width).unsqueeze(2)) + + has_neg_prompt = negative_prompt is not None or ( + negative_prompt_embeds is not None and negative_prompt_embeds_mask is not None + ) + + if true_cfg_scale > 1 and not has_neg_prompt: + logger.warning( + f"true_cfg_scale is passed as {true_cfg_scale}, but classifier-free guidance is not enabled since no negative_prompt is provided." + ) + elif true_cfg_scale <= 1 and has_neg_prompt: + logger.warning( + " negative_prompt is passed but classifier-free guidance is not enabled since true_cfg_scale <= 1" + ) + + do_true_cfg = true_cfg_scale > 1 and has_neg_prompt + prompt_embeds, prompt_embeds_mask = self.encode_prompt( + image=condition_images, + prompt=prompt, + prompt_embeds=prompt_embeds, + prompt_embeds_mask=prompt_embeds_mask, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + if do_true_cfg: + negative_prompt_embeds, negative_prompt_embeds_mask = self.encode_prompt( + image=condition_images, + prompt=negative_prompt, + prompt_embeds=negative_prompt_embeds, + prompt_embeds_mask=negative_prompt_embeds_mask, + device=device, + num_images_per_prompt=num_images_per_prompt, + max_sequence_length=max_sequence_length, + ) + if comfyui_progressbar: + pbar.update(1) + + # 4. Prepare latent variables + num_channels_latents = self.transformer.config.in_channels // 4 + latents, image_latents = self.prepare_latents( + vae_images, + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + img_shapes = [ + [ + (1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2), + *[ + (1, vae_height // self.vae_scale_factor // 2, vae_width // self.vae_scale_factor // 2) + for vae_width, vae_height in vae_image_sizes + ], + ] + ] * batch_size + + # 5. Prepare timesteps + sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas + image_seq_len = latents.shape[1] + mu = calculate_shift( + image_seq_len, + self.scheduler.config.get("base_image_seq_len", 256), + self.scheduler.config.get("max_image_seq_len", 4096), + self.scheduler.config.get("base_shift", 0.5), + self.scheduler.config.get("max_shift", 1.15), + ) + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + sigmas=sigmas, + mu=mu, + ) + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self._num_timesteps = len(timesteps) + + # handle guidance + if self.transformer.config.guidance_embeds and guidance_scale is None: + raise ValueError("guidance_scale is required for guidance-distilled model.") + elif self.transformer.config.guidance_embeds: + guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32) + guidance = guidance.expand(latents.shape[0]) + elif not self.transformer.config.guidance_embeds and guidance_scale is not None: + logger.warning( + f"guidance_scale is passed as {guidance_scale}, but ignored since the model is not guidance-distilled." + ) + guidance = None + elif not self.transformer.config.guidance_embeds and guidance_scale is None: + guidance = None + + if self.attention_kwargs is None: + self._attention_kwargs = {} + + txt_seq_lens = prompt_embeds_mask.sum(dim=1).tolist() if prompt_embeds_mask is not None else None + negative_txt_seq_lens = ( + negative_prompt_embeds_mask.sum(dim=1).tolist() if negative_prompt_embeds_mask is not None else None + ) + if comfyui_progressbar: + pbar.update(1) + + # 6. Denoising loop + self.scheduler.set_begin_index(0) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + if self.interrupt: + continue + + if image_latents is not None: + latents_and_image_latents = torch.cat([latents, image_latents], dim=1) + else: + latents_and_image_latents = latents + + if do_true_cfg: + latent_model_input = torch.cat([latents_and_image_latents] * 2) + prompt_embeds_mask_input = [_negative_prompt_embeds_mask for _negative_prompt_embeds_mask in negative_prompt_embeds_mask] + [_prompt_embeds_mask for _prompt_embeds_mask in prompt_embeds_mask] + prompt_embeds_input = [_negative_prompt_embeds for _negative_prompt_embeds in negative_prompt_embeds] + [_prompt_embeds for _prompt_embeds in prompt_embeds] + img_shapes_input = img_shapes * 2 + txt_seq_lens_input = negative_txt_seq_lens + txt_seq_lens + else: + latent_model_input = latents_and_image_latents + prompt_embeds_mask_input = prompt_embeds_mask + prompt_embeds_input = prompt_embeds + img_shapes_input = img_shapes + txt_seq_lens_input = txt_seq_lens + + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # handle guidance + if self.transformer.config.guidance_embeds: + guidance = torch.full([1], guidance_scale, device=device, dtype=torch.float32) + guidance = guidance.expand(latent_model_input.shape[0]) + else: + guidance = None + + self._current_timestep = t + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]).to(latent_model_input.dtype) + + with torch.cuda.amp.autocast(dtype=latents.dtype), torch.cuda.device(device=latents.device): + noise_pred = self.transformer.forward_bs( + x=latent_model_input, + timestep=timestep / 1000, + guidance=guidance, + encoder_hidden_states_mask=prompt_embeds_mask_input, + encoder_hidden_states=prompt_embeds_input, + img_shapes=img_shapes_input, + txt_seq_lens=txt_seq_lens_input, + attention_kwargs=self.attention_kwargs, + return_dict=False, + ) + noise_pred = noise_pred[:, : latents.size(1)] + + if do_true_cfg: + neg_noise_pred, noise_pred = noise_pred.chunk(2) + comb_pred = neg_noise_pred + true_cfg_scale * (noise_pred - neg_noise_pred) + + cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True) + noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True) + noise_pred = comb_pred * (cond_norm / noise_norm) + + # compute the previous noisy sample x_t -> x_t-1 + latents_dtype = latents.dtype + latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0] + + if latents.dtype != latents_dtype: + if torch.backends.mps.is_available(): + # some platforms (eg. apple mps) misbehave due to a pytorch bug: https://github.com/pytorch/pytorch/pull/99272 + latents = latents.to(latents_dtype) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if XLA_AVAILABLE: + xm.mark_step() + + if comfyui_progressbar: + pbar.update(1) + + self._current_timestep = None + if output_type == "latent": + image = latents + else: + latents = self._unpack_latents(latents, height, width, self.vae_scale_factor) + latents = latents.to(self.vae.dtype) + latents_mean = ( + torch.tensor(self.vae.config.latents_mean) + .view(1, self.vae.config.z_dim, 1, 1, 1) + .to(latents.device, latents.dtype) + ) + latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, self.vae.config.z_dim, 1, 1, 1).to( + latents.device, latents.dtype + ) + latents = latents / latents_std + latents_mean + image = self.vae.decode(latents, return_dict=False)[0][:, :, 0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return QwenImagePipelineOutput(images=image) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan.py new file mode 100644 index 0000000000000000000000000000000000000000..f105c9a338bc608a571ba53a262bf667b538db32 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan.py @@ -0,0 +1,576 @@ +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor + +from ..models import (AutoencoderKLWan, AutoTokenizer, + WanT5EncoderModel, WanTransformer3DModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class WanPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = [] + model_cpu_offload_seq = "text_encoder->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: WanTransformer3DModel, + scheduler: FlowMatchEulerDiscreteScheduler, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, scheduler=scheduler + ) + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + comfyui_progressbar: bool = False, + shift: int = 5, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 1) + + # 5. Prepare latents + latent_channels = self.transformer.config.in_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + ) + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, (num_frames - 1) // self.vae.temporal_compression_ratio + 1, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = self.transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + ) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2.py new file mode 100644 index 0000000000000000000000000000000000000000..e96287a98f5b336b115ccf693e94b36501416a30 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2.py @@ -0,0 +1,591 @@ +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor + +from ..models import (AutoencoderKLWan, AutoTokenizer, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class Wan2_2Pipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = ["transformer_2"] + model_cpu_offload_seq = "text_encoder->transformer_2->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: Wan2_2Transformer3DModel, + transformer_2: Wan2_2Transformer3DModel = None, + scheduler: FlowMatchEulerDiscreteScheduler = None, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, + transformer_2=transformer_2, scheduler=scheduler + ) + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + boundary: float = 0.875, + comfyui_progressbar: bool = False, + shift: int = 5, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 1) + + # 5. Prepare latents + latent_channels = self.transformer.config.in_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + ) + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, (num_frames - 1) // self.vae.temporal_compression_ratio + 1, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + if self.transformer_2 is not None: + if t >= boundary * self.scheduler.config.num_train_timesteps: + local_transformer = self.transformer_2 + else: + local_transformer = self.transformer + else: + local_transformer = self.transformer + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = local_transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + ) + + # perform guidance + if do_classifier_free_guidance: + if self.transformer_2 is not None and (isinstance(self.guidance_scale, (list, tuple))): + sample_guide_scale = self.guidance_scale[1] if t >= self.transformer_2.config.boundary * self.scheduler.config.num_train_timesteps else self.guidance_scale[0] + else: + sample_guide_scale = self.guidance_scale + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + sample_guide_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_animate.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_animate.py new file mode 100644 index 0000000000000000000000000000000000000000..6a0df263374b2d6d15d57b46d8ab7a9ce2435d4d --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_animate.py @@ -0,0 +1,929 @@ +import inspect +import math +from copy import deepcopy +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import copy +import torch +import cv2 +import torch.nn.functional as F +from einops import rearrange +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from decord import VideoReader + +from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel_Animate) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class Wan2_2AnimatePipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = ["transformer_2", "clip_image_encoder"] + model_cpu_offload_seq = "text_encoder->clip_image_encoder->transformer_2->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: Wan2_2Transformer3DModel_Animate, + transformer_2: Wan2_2Transformer3DModel_Animate = None, + clip_image_encoder: CLIPModel = None, + scheduler: FlowMatchEulerDiscreteScheduler = None, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, + transformer_2=transformer_2, clip_image_encoder=clip_image_encoder, scheduler=scheduler + ) + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def padding_resize(self, img_ori, height=512, width=512, padding_color=(0, 0, 0), interpolation=cv2.INTER_LINEAR): + ori_height = img_ori.shape[0] + ori_width = img_ori.shape[1] + channel = img_ori.shape[2] + + img_pad = np.zeros((height, width, channel)) + if channel == 1: + img_pad[:, :, 0] = padding_color[0] + else: + img_pad[:, :, 0] = padding_color[0] + img_pad[:, :, 1] = padding_color[1] + img_pad[:, :, 2] = padding_color[2] + + if (ori_height / ori_width) > (height / width): + new_width = int(height / ori_height * ori_width) + img = cv2.resize(img_ori, (new_width, height), interpolation=interpolation) + padding = int((width - new_width) / 2) + if len(img.shape) == 2: + img = img[:, :, np.newaxis] + img_pad[:, padding: padding + new_width, :] = img + else: + new_height = int(width / ori_width * ori_height) + img = cv2.resize(img_ori, (width, new_height), interpolation=interpolation) + padding = int((height - new_height) / 2) + if len(img.shape) == 2: + img = img[:, :, np.newaxis] + img_pad[padding: padding + new_height, :, :] = img + + img_pad = np.uint8(img_pad) + + return img_pad + + def inputs_padding(self, x, target_len): + ndim = x.ndim + + if ndim == 4: + f = x.shape[0] + if target_len <= f: + return [deepcopy(x[i]) for i in range(target_len)] + + idx = 0 + flip = False + target_array = [] + while len(target_array) < target_len: + target_array.append(deepcopy(x[idx])) + if flip: + idx -= 1 + else: + idx += 1 + if idx == 0 or idx == f - 1: + flip = not flip + return target_array[:target_len] + + elif ndim == 5: + b, c, f, h, w = x.shape + + if target_len <= f: + return x[:, :, :target_len, :, :] + + indices = [] + idx = 0 + flip = False + while len(indices) < target_len: + indices.append(idx) + if flip: + idx -= 1 + else: + idx += 1 + if idx == 0 or idx == f - 1: + flip = not flip + indices = indices[:target_len] + + if isinstance(x, torch.Tensor): + indices_tensor = torch.tensor(indices, device=x.device, dtype=torch.long) + return x[:, :, indices_tensor, :, :] + else: + indices_array = np.array(indices) + return x[:, :, indices_array, :, :] + + else: + raise ValueError(f"Unsupported input dimension: {ndim}. Expected 4D or 5D.") + + def get_valid_len(self, real_len, clip_len=81, overlap=1): + real_clip_len = clip_len - overlap + last_clip_num = (real_len - overlap) % real_clip_len + if last_clip_num == 0: + extra = 0 + else: + extra = real_clip_len - last_clip_num + target_len = real_len + extra + return target_len + + def prepare_source(self, src_pose_path, src_face_path, src_ref_path): + pose_video_reader = VideoReader(src_pose_path) + pose_len = len(pose_video_reader) + pose_idxs = list(range(pose_len)) + pose_video = pose_video_reader.get_batch(pose_idxs).asnumpy() + + face_video_reader = VideoReader(src_face_path) + face_len = len(face_video_reader) + face_idxs = list(range(face_len)) + face_video = face_video_reader.get_batch(face_idxs).asnumpy() + height, width = pose_video[0].shape[:2] + + ref_image = cv2.imread(src_ref_path)[..., ::-1] + ref_image = self.padding_resize(ref_image, height=height, width=width) + return pose_video, face_video, ref_image + + def prepare_source_for_replace(self, src_bg_path, src_mask_path): + bg_video_reader = VideoReader(src_bg_path) + bg_len = len(bg_video_reader) + bg_idxs = list(range(bg_len)) + bg_video = bg_video_reader.get_batch(bg_idxs).asnumpy() + + mask_video_reader = VideoReader(src_mask_path) + mask_len = len(mask_video_reader) + mask_idxs = list(range(mask_len)) + mask_video = mask_video_reader.get_batch(mask_idxs).asnumpy() + mask_video = mask_video[:, :, :, 0] / 255 + return bg_video, mask_video + + def get_i2v_mask(self, lat_t, lat_h, lat_w, mask_len=1, mask_pixel_values=None, device="cuda"): + if mask_pixel_values is None: + msk = torch.zeros(1, (lat_t-1) * 4 + 1, lat_h, lat_w, device=device) + else: + msk = mask_pixel_values.clone() + msk[:, :mask_len] = 1 + msk = torch.concat([torch.repeat_interleave(msk[:, 0:1], repeats=4, dim=1), msk[:, 1:]], dim=1) + msk = msk.view(1, msk.shape[1] // 4, 4, lat_h, lat_w) + msk = msk.transpose(1, 2) + return msk + + def prepare_control_latents( + self, control, control_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the control to latents shape as we concatenate the control to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if control is not None: + control = control.to(device=device, dtype=dtype) + bs = 1 + new_control = [] + for i in range(0, control.shape[0], bs): + control_bs = control[i : i + bs] + control_bs = self.vae.encode(control_bs)[0] + control_bs = control_bs.mode() + new_control.append(control_bs) + control = torch.cat(new_control, dim = 0) + + if control_image is not None: + control_image = control_image.to(device=device, dtype=dtype) + bs = 1 + new_control_pixel_values = [] + for i in range(0, control_image.shape[0], bs): + control_pixel_values_bs = control_image[i : i + bs] + control_pixel_values_bs = self.vae.encode(control_pixel_values_bs)[0] + control_pixel_values_bs = control_pixel_values_bs.mode() + new_control_pixel_values.append(control_pixel_values_bs) + control_image_latents = torch.cat(new_control_pixel_values, dim = 0) + else: + control_image_latents = None + + return control, control_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + # frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + clip_len=77, + num_frames: int = 49, + num_inference_steps: int = 50, + pose_video = None, + face_video = None, + ref_image = None, + bg_video = None, + mask_video = None, + replace_flag = True, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + boundary: float = 0.875, + comfyui_progressbar: bool = False, + shift: int = 5, + refert_num = 1, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 1) + + # 4. Prepare latents + if pose_video is not None: + video_length = pose_video.shape[2] + pose_video = self.image_processor.preprocess(rearrange(pose_video, "b c f h w -> (b f) c h w"), height=height, width=width) + pose_video = pose_video.to(dtype=torch.float32) + pose_video = rearrange(pose_video, "(b f) c h w -> b c f h w", f=video_length) + else: + pose_video = None + + if face_video is not None: + video_length = face_video.shape[2] + face_video = self.image_processor.preprocess(rearrange(face_video, "b c f h w -> (b f) c h w")) + face_video = face_video.to(dtype=torch.float32) + face_video = rearrange(face_video, "(b f) c h w -> b c f h w", f=video_length) + else: + face_video = None + + real_frame_len = pose_video.size()[2] + target_len = self.get_valid_len(real_frame_len, clip_len, overlap=refert_num) + print('real frames: {} target frames: {}'.format(real_frame_len, target_len)) + pose_video = self.inputs_padding(pose_video, target_len).to(device, weight_dtype) + face_video = self.inputs_padding(face_video, target_len).to(device, weight_dtype) + ref_image = self.padding_resize(np.array(ref_image), height=height, width=width) + ref_image = torch.tensor(ref_image / 127.5 - 1).unsqueeze(0).permute([3, 0, 1, 2]).unsqueeze(0).to(device, weight_dtype) + + if replace_flag: + if bg_video is not None: + video_length = bg_video.shape[2] + bg_video = self.image_processor.preprocess(rearrange(bg_video, "b c f h w -> (b f) c h w"), height=height, width=width) + bg_video = bg_video.to(dtype=torch.float32) + bg_video = rearrange(bg_video, "(b f) c h w -> b c f h w", f=video_length) + else: + bg_video = None + bg_video = self.inputs_padding(bg_video, target_len).to(device, weight_dtype) + mask_video = self.inputs_padding(mask_video, target_len).to(device, weight_dtype) + + if comfyui_progressbar: + pbar.update(1) + + # 5. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, (num_frames - 1) // self.vae.temporal_compression_ratio + 1, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + + # 6. Denoising loop + start = 0 + end = clip_len + all_out_frames = [] + copy_timesteps = copy.deepcopy(timesteps) + copy_latents = copy.deepcopy(latents) + bs = pose_video.size()[0] + while True: + if start + refert_num >= pose_video.size()[2]: + break + + # Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, copy_timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, copy_timesteps) + self._num_timesteps = len(timesteps) + + latent_channels = self.transformer.config.in_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + copy_latents, + ) + + if start == 0: + mask_reft_len = 0 + else: + mask_reft_len = refert_num + + conditioning_pixel_values = pose_video[:, :, start:end] + face_pixel_values = face_video[:, :, start:end] + ref_pixel_values = ref_image.clone().detach() + if start > 0: + refer_t_pixel_values = out_frames[:, :, -refert_num:].clone().detach() + refer_t_pixel_values = (refer_t_pixel_values - 0.5) / 0.5 + else: + refer_t_pixel_values = torch.zeros(bs, 3, refert_num, height, width) + refer_t_pixel_values = refer_t_pixel_values.to(device=device, dtype=weight_dtype) + + pose_latents, ref_latents = self.prepare_control_latents( + conditioning_pixel_values, + ref_pixel_values, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance + ) + + mask_ref = self.get_i2v_mask(1, target_shape[-1], target_shape[-2], 1, device=device) + y_ref = torch.concat([mask_ref, ref_latents], dim=1).to(device=device, dtype=weight_dtype) + if mask_reft_len > 0: + if replace_flag: + # Image.fromarray(np.array((refer_t_pixel_values[0, :, 0].permute(1,2,0) * 0.5 + 0.5).float().cpu().numpy() *255, np.uint8)).save("1.jpg") + bg_pixel_values = bg_video[:, :, start:end] + y_reft = self.vae.encode( + torch.concat( + [ + refer_t_pixel_values[:, :, :mask_reft_len], + bg_pixel_values[:, :, mask_reft_len:] + ], dim=2 + ).to(device=device, dtype=weight_dtype) + )[0].mode() + + mask_pixel_values = 1 - mask_video[:, :, start:end] + mask_pixel_values = rearrange(mask_pixel_values, "b c t h w -> (b t) c h w") + mask_pixel_values = F.interpolate(mask_pixel_values, size=(target_shape[-1], target_shape[-2]), mode='nearest') + mask_pixel_values = rearrange(mask_pixel_values, "(b t) c h w -> b c t h w", b = bs)[:, 0] + msk_reft = self.get_i2v_mask( + int((clip_len - 1) // self.vae.temporal_compression_ratio + 1), target_shape[-1], target_shape[-2], mask_reft_len, mask_pixel_values=mask_pixel_values, device=device + ) + else: + refer_t_pixel_values = rearrange(refer_t_pixel_values[:, :, :mask_reft_len], "b c t h w -> (b t) c h w") + refer_t_pixel_values = F.interpolate(refer_t_pixel_values, size=(height, width), mode="bicubic") + refer_t_pixel_values = rearrange(refer_t_pixel_values, "(b t) c h w -> b c t h w", b = bs) + + y_reft = self.vae.encode( + torch.concat( + [ + refer_t_pixel_values, + torch.zeros(bs, 3, clip_len - mask_reft_len, height, width).to(device=device, dtype=weight_dtype), + ], dim=2, + ).to(device=device, dtype=weight_dtype) + )[0].mode() + msk_reft = self.get_i2v_mask( + int((clip_len - 1) // self.vae.temporal_compression_ratio + 1), target_shape[-1], target_shape[-2], mask_reft_len, device=device + ) + else: + if replace_flag: + bg_pixel_values = bg_video[:, :, start:end] + y_reft = self.vae.encode( + bg_pixel_values.to(device=device, dtype=weight_dtype) + )[0].mode() + + mask_pixel_values = 1 - mask_video[:, :, start:end] + mask_pixel_values = rearrange(mask_pixel_values, "b c t h w -> (b t) c h w") + mask_pixel_values = F.interpolate(mask_pixel_values, size=(target_shape[-1], target_shape[-2]), mode='nearest') + mask_pixel_values = rearrange(mask_pixel_values, "(b t) c h w -> b c t h w", b = bs)[:, 0] + msk_reft = self.get_i2v_mask( + int((clip_len - 1) // self.vae.temporal_compression_ratio + 1), target_shape[-1], target_shape[-2], mask_reft_len, mask_pixel_values=mask_pixel_values, device=device + ) + else: + y_reft = self.vae.encode( + torch.zeros(1, 3, clip_len - mask_reft_len, height, width).to(device=device, dtype=weight_dtype) + )[0].mode() + msk_reft = self.get_i2v_mask( + int((clip_len - 1) // self.vae.temporal_compression_ratio + 1), target_shape[-1], target_shape[-2], mask_reft_len, device=device + ) + + y_reft = torch.concat([msk_reft, y_reft], dim=1).to(device=device, dtype=weight_dtype) + y = torch.concat([y_ref, y_reft], dim=2) + + clip_context = self.clip_image_encoder([ref_pixel_values[0, :, :, :]]).to(device=device, dtype=weight_dtype) + + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + y_in = torch.cat([y] * 2) if do_classifier_free_guidance else y + clip_context_input = ( + torch.cat([clip_context] * 2) if do_classifier_free_guidance else clip_context + ) + pose_latents_input = ( + torch.cat([pose_latents] * 2) if do_classifier_free_guidance else pose_latents + ) + face_pixel_values_input = ( + torch.cat([torch.ones_like(face_pixel_values) * -1] + [face_pixel_values]) if do_classifier_free_guidance else face_pixel_values + ) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + if self.transformer_2 is not None: + if t >= boundary * self.scheduler.config.num_train_timesteps: + local_transformer = self.transformer_2 + else: + local_transformer = self.transformer + else: + local_transformer = self.transformer + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = local_transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + y=y_in, + clip_fea=clip_context_input, + pose_latents=pose_latents_input, + face_pixel_values=face_pixel_values_input, + ) + + # Perform guidance + if do_classifier_free_guidance: + if self.transformer_2 is not None and (isinstance(self.guidance_scale, (list, tuple))): + sample_guide_scale = self.guidance_scale[1] if t >= self.transformer_2.config.boundary * self.scheduler.config.num_train_timesteps else self.guidance_scale[0] + else: + sample_guide_scale = self.guidance_scale + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + sample_guide_scale * (noise_pred_text - noise_pred_uncond) + + # Compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + out_frames = self.decode_latents(latents[:, :, 1:]) + if start != 0: + out_frames = out_frames[:, :, refert_num:] + all_out_frames.append(out_frames.cpu()) + start += clip_len - refert_num + end += clip_len - refert_num + + videos = torch.cat(all_out_frames, dim=2)[:, :, :real_frame_len] + + # Offload all models + self.maybe_free_model_hooks() + + return WanPipelineOutput(videos=videos.float().cpu()) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_fun_control.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_fun_control.py new file mode 100644 index 0000000000000000000000000000000000000000..5b923191c43165bdf825b4eb0cc2cbb08ffab6b5 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_fun_control.py @@ -0,0 +1,903 @@ +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange +from PIL import Image +from transformers import T5Tokenizer + +from ..models import (AutoencoderKLWan, AutoTokenizer, + Wan2_2Transformer3DModel, WanT5EncoderModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class Wan2_2FunControlPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = ["transformer_2"] + model_cpu_offload_seq = "text_encoder->transformer_2->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: Wan2_2Transformer3DModel, + transformer_2: Wan2_2Transformer3DModel = None, + scheduler: FlowMatchEulerDiscreteScheduler = None, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, + transformer_2=transformer_2, scheduler=scheduler + ) + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance, noise_aug_strength + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if mask is not None: + mask = mask.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask = [] + for i in range(0, mask.shape[0], bs): + mask_bs = mask[i : i + bs] + mask_bs = self.vae.encode(mask_bs)[0] + mask_bs = mask_bs.mode() + new_mask.append(mask_bs) + mask = torch.cat(new_mask, dim = 0) + # mask = mask * self.vae.config.scaling_factor + + if masked_image is not None: + masked_image = masked_image.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask_pixel_values = [] + for i in range(0, masked_image.shape[0], bs): + mask_pixel_values_bs = masked_image[i : i + bs] + mask_pixel_values_bs = self.vae.encode(mask_pixel_values_bs)[0] + mask_pixel_values_bs = mask_pixel_values_bs.mode() + new_mask_pixel_values.append(mask_pixel_values_bs) + masked_image_latents = torch.cat(new_mask_pixel_values, dim = 0) + # masked_image_latents = masked_image_latents * self.vae.config.scaling_factor + else: + masked_image_latents = None + + return mask, masked_image_latents + + def prepare_control_latents( + self, control, control_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the control to latents shape as we concatenate the control to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if control is not None: + control = control.to(device=device, dtype=dtype) + bs = 1 + new_control = [] + for i in range(0, control.shape[0], bs): + control_bs = control[i : i + bs] + control_bs = self.vae.encode(control_bs)[0] + control_bs = control_bs.mode() + new_control.append(control_bs) + control = torch.cat(new_control, dim = 0) + + if control_image is not None: + control_image = control_image.to(device=device, dtype=dtype) + bs = 1 + new_control_pixel_values = [] + for i in range(0, control_image.shape[0], bs): + control_pixel_values_bs = control_image[i : i + bs] + control_pixel_values_bs = self.vae.encode(control_pixel_values_bs)[0] + control_pixel_values_bs = control_pixel_values_bs.mode() + new_control_pixel_values.append(control_pixel_values_bs) + control_image_latents = torch.cat(new_control_pixel_values, dim = 0) + else: + control_image_latents = None + + return control, control_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + video: Union[torch.FloatTensor] = None, + mask_video: Union[torch.FloatTensor] = None, + control_video: Union[torch.FloatTensor] = None, + control_camera_video: Union[torch.FloatTensor] = None, + start_image: Union[torch.FloatTensor] = None, + ref_image: Union[torch.FloatTensor] = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + boundary: float = 0.875, + comfyui_progressbar: bool = False, + shift: int = 5, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + # 5. Prepare latents. + if video is not None: + video_length = video.shape[2] + init_video = self.image_processor.preprocess(rearrange(video, "b c f h w -> (b f) c h w"), height=height, width=width) + init_video = init_video.to(dtype=torch.float32) + init_video = rearrange(init_video, "(b f) c h w -> b c f h w", f=video_length) + else: + init_video = None + + latent_channels = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + ) + if comfyui_progressbar: + pbar.update(1) + + # Prepare mask latent variables + if init_video is not None: + if (mask_video == 255).all(): + mask_latents = torch.tile( + torch.zeros_like(latents)[:, :1].to(device, weight_dtype), [1, 4, 1, 1, 1] + ) + masked_video_latents = torch.zeros_like(latents).to(device, weight_dtype) + if self.vae.spatial_compression_ratio >= 16: + mask = torch.ones_like(latents).to(device, weight_dtype)[:, :1].to(device, weight_dtype) + else: + bs, _, video_length, height, width = video.size() + mask_condition = self.mask_processor.preprocess(rearrange(mask_video, "b c f h w -> (b f) c h w"), height=height, width=width) + mask_condition = mask_condition.to(dtype=torch.float32) + mask_condition = rearrange(mask_condition, "(b f) c h w -> b c f h w", f=video_length) + + masked_video = init_video * (torch.tile(mask_condition, [1, 3, 1, 1, 1]) < 0.5) + _, masked_video_latents = self.prepare_mask_latents( + None, + masked_video, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance, + noise_aug_strength=None, + ) + + mask_condition = torch.concat( + [ + torch.repeat_interleave(mask_condition[:, :, 0:1], repeats=4, dim=2), + mask_condition[:, :, 1:] + ], dim=2 + ) + mask_condition = mask_condition.view(bs, mask_condition.shape[2] // 4, 4, height, width) + mask_condition = mask_condition.transpose(1, 2) + mask_latents = resize_mask(1 - mask_condition, masked_video_latents, True).to(device, weight_dtype) + + if self.vae.spatial_compression_ratio >= 16: + mask = F.interpolate(mask_condition[:, :1], size=latents.size()[-3:], mode='trilinear', align_corners=True).to(device, weight_dtype) + if not mask[:, :, 0, :, :].any(): + mask[:, :, 1:, :, :] = 1 + latents = (1 - mask) * masked_video_latents + mask * latents + + # Prepare mask latent variables + if control_camera_video is not None: + control_latents = None + # Rearrange dimensions + # Concatenate and transpose dimensions + control_camera_latents = torch.concat( + [ + torch.repeat_interleave(control_camera_video[:, :, 0:1], repeats=4, dim=2), + control_camera_video[:, :, 1:] + ], dim=2 + ).transpose(1, 2) + + # Reshape, transpose, and view into desired shape + b, f, c, h, w = control_camera_latents.shape + control_camera_latents = control_camera_latents.contiguous().view(b, f // 4, 4, c, h, w).transpose(2, 3) + control_camera_latents = control_camera_latents.contiguous().view(b, f // 4, c * 4, h, w).transpose(1, 2) + elif control_video is not None: + video_length = control_video.shape[2] + control_video = self.image_processor.preprocess(rearrange(control_video, "b c f h w -> (b f) c h w"), height=height, width=width) + control_video = control_video.to(dtype=torch.float32) + control_video = rearrange(control_video, "(b f) c h w -> b c f h w", f=video_length) + control_video_latents = self.prepare_control_latents( + None, + control_video, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance + )[1] + control_camera_latents = None + else: + control_video_latents = torch.zeros_like(latents).to(device, weight_dtype) + control_camera_latents = None + + if start_image is not None: + video_length = start_image.shape[2] + start_image = self.image_processor.preprocess(rearrange(start_image, "b c f h w -> (b f) c h w"), height=height, width=width) + start_image = start_image.to(dtype=torch.float32) + start_image = rearrange(start_image, "(b f) c h w -> b c f h w", f=video_length) + + start_image_latentes = self.prepare_control_latents( + None, + start_image, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance + )[1] + + start_image_latentes_conv_in = torch.zeros_like(latents) + if latents.size()[2] != 1: + start_image_latentes_conv_in[:, :, :1] = start_image_latentes + else: + start_image_latentes_conv_in = torch.zeros_like(latents) + + if self.transformer.config.get("add_ref_conv", False): + if ref_image is not None: + video_length = ref_image.shape[2] + ref_image = self.image_processor.preprocess(rearrange(ref_image, "b c f h w -> (b f) c h w"), height=height, width=width) + ref_image = ref_image.to(dtype=torch.float32) + ref_image = rearrange(ref_image, "(b f) c h w -> b c f h w", f=video_length) + + ref_image_latentes = self.prepare_control_latents( + None, + ref_image, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance + )[1] + ref_image_latentes = ref_image_latentes[:, :, 0] + else: + ref_image_latentes = torch.zeros_like(latents)[:, :, 0] + else: + if ref_image is not None: + raise ValueError("The add_ref_conv is False, but ref_image is not None") + else: + ref_image_latentes = None + + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, (num_frames - 1) // self.vae.temporal_compression_ratio + 1, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # Prepare mask latent variables + if control_camera_video is not None: + control_latents_input = None + control_camera_latents_input = ( + torch.cat([control_camera_latents] * 2) if do_classifier_free_guidance else control_camera_latents + ).to(device, weight_dtype) + else: + control_latents_input = ( + torch.cat([control_video_latents] * 2) if do_classifier_free_guidance else control_video_latents + ).to(device, weight_dtype) + control_camera_latents_input = None + + if init_video is not None: + mask_input = torch.cat([mask_latents] * 2) if do_classifier_free_guidance else mask_latents + masked_video_latents_input = ( + torch.cat([masked_video_latents] * 2) if do_classifier_free_guidance else masked_video_latents + ) + y = torch.cat([mask_input, masked_video_latents_input], dim=1).to(device, weight_dtype) + control_latents_input = y if control_latents_input is None else \ + torch.cat([control_latents_input, y], dim = 1) + else: + start_image_latentes_conv_in_input = ( + torch.cat([start_image_latentes_conv_in] * 2) if do_classifier_free_guidance else start_image_latentes_conv_in + ).to(device, weight_dtype) + control_latents_input = start_image_latentes_conv_in_input if control_latents_input is None else \ + torch.cat([control_latents_input, start_image_latentes_conv_in_input], dim = 1) + + if ref_image_latentes is not None: + full_ref = ( + torch.cat([ref_image_latentes] * 2) if do_classifier_free_guidance else ref_image_latentes + ).to(device, weight_dtype) + else: + full_ref = None + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + if self.vae.spatial_compression_ratio >= 16 and init_video is not None: + temp_ts = ((mask[0][0][:, ::2, ::2]) * t).flatten() + temp_ts = torch.cat([ + temp_ts, + temp_ts.new_ones(seq_len - temp_ts.size(0)) * t + ]) + temp_ts = temp_ts.unsqueeze(0) + timestep = temp_ts.expand(latent_model_input.shape[0], temp_ts.size(1)) + else: + timestep = t.expand(latent_model_input.shape[0]) + + if self.transformer_2 is not None: + if t >= boundary * self.scheduler.config.num_train_timesteps: + local_transformer = self.transformer_2 + else: + local_transformer = self.transformer + else: + local_transformer = self.transformer + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = local_transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + y=control_latents_input, + y_camera=control_camera_latents_input, + full_ref=full_ref, + ) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if self.vae.spatial_compression_ratio >= 16 and not mask[:, :, 0, :, :].any(): + latents = (1 - mask) * masked_video_latents + mask * latents + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_fun_inpaint.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_fun_inpaint.py new file mode 100644 index 0000000000000000000000000000000000000000..8eaf68e67d624b4d84e0cbfe62ef81836ee83d18 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_fun_inpaint.py @@ -0,0 +1,752 @@ +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange +from PIL import Image +from transformers import T5Tokenizer + +from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class Wan2_2FunInpaintPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = ["transformer_2"] + model_cpu_offload_seq = "text_encoder->transformer_2->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: Wan2_2Transformer3DModel, + transformer_2: Wan2_2Transformer3DModel = None, + scheduler: FlowMatchEulerDiscreteScheduler = None, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, + transformer_2=transformer_2, scheduler=scheduler + ) + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance, noise_aug_strength + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if mask is not None: + mask = mask.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask = [] + for i in range(0, mask.shape[0], bs): + mask_bs = mask[i : i + bs] + mask_bs = self.vae.encode(mask_bs)[0] + mask_bs = mask_bs.mode() + new_mask.append(mask_bs) + mask = torch.cat(new_mask, dim = 0) + # mask = mask * self.vae.config.scaling_factor + + if masked_image is not None: + masked_image = masked_image.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask_pixel_values = [] + for i in range(0, masked_image.shape[0], bs): + mask_pixel_values_bs = masked_image[i : i + bs] + mask_pixel_values_bs = self.vae.encode(mask_pixel_values_bs)[0] + mask_pixel_values_bs = mask_pixel_values_bs.mode() + new_mask_pixel_values.append(mask_pixel_values_bs) + masked_image_latents = torch.cat(new_mask_pixel_values, dim = 0) + # masked_image_latents = masked_image_latents * self.vae.config.scaling_factor + else: + masked_image_latents = None + + return mask, masked_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + video: Union[torch.FloatTensor] = None, + mask_video: Union[torch.FloatTensor] = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + boundary: float = 0.875, + comfyui_progressbar: bool = False, + shift: int = 5, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + # 5. Prepare latents. + if video is not None: + video_length = video.shape[2] + init_video = self.image_processor.preprocess(rearrange(video, "b c f h w -> (b f) c h w"), height=height, width=width) + init_video = init_video.to(dtype=torch.float32) + init_video = rearrange(init_video, "(b f) c h w -> b c f h w", f=video_length) + else: + init_video = None + + latent_channels = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + ) + if comfyui_progressbar: + pbar.update(1) + + # Prepare mask latent variables + if init_video is not None: + if (mask_video == 255).all(): + mask_latents = torch.tile( + torch.zeros_like(latents)[:, :1].to(device, weight_dtype), [1, 4, 1, 1, 1] + ) + masked_video_latents = torch.zeros_like(latents).to(device, weight_dtype) + if self.vae.spatial_compression_ratio >= 16: + mask = torch.ones_like(latents).to(device, weight_dtype)[:, :1].to(device, weight_dtype) + else: + bs, _, video_length, height, width = video.size() + mask_condition = self.mask_processor.preprocess(rearrange(mask_video, "b c f h w -> (b f) c h w"), height=height, width=width) + mask_condition = mask_condition.to(dtype=torch.float32) + mask_condition = rearrange(mask_condition, "(b f) c h w -> b c f h w", f=video_length) + + masked_video = init_video * (torch.tile(mask_condition, [1, 3, 1, 1, 1]) < 0.5) + _, masked_video_latents = self.prepare_mask_latents( + None, + masked_video, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance, + noise_aug_strength=None, + ) + + mask_condition = torch.concat( + [ + torch.repeat_interleave(mask_condition[:, :, 0:1], repeats=4, dim=2), + mask_condition[:, :, 1:] + ], dim=2 + ) + mask_condition = mask_condition.view(bs, mask_condition.shape[2] // 4, 4, height, width) + mask_condition = mask_condition.transpose(1, 2) + mask_latents = resize_mask(1 - mask_condition, masked_video_latents, True).to(device, weight_dtype) + + if self.vae.spatial_compression_ratio >= 16: + mask = F.interpolate(mask_condition[:, :1], size=latents.size()[-3:], mode='trilinear', align_corners=True).to(device, weight_dtype) + if not mask[:, :, 0, :, :].any(): + mask[:, :, 1:, :, :] = 1 + latents = (1 - mask) * masked_video_latents + mask * latents + + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, (num_frames - 1) // self.vae.temporal_compression_ratio + 1, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + if init_video is not None: + mask_input = torch.cat([mask_latents] * 2) if do_classifier_free_guidance else mask_latents + masked_video_latents_input = ( + torch.cat([masked_video_latents] * 2) if do_classifier_free_guidance else masked_video_latents + ) + y = torch.cat([mask_input, masked_video_latents_input], dim=1).to(device, weight_dtype) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + if self.vae.spatial_compression_ratio >= 16 and init_video is not None: + temp_ts = ((mask[0][0][:, ::2, ::2]) * t).flatten() + temp_ts = torch.cat([ + temp_ts, + temp_ts.new_ones(seq_len - temp_ts.size(0)) * t + ]) + temp_ts = temp_ts.unsqueeze(0) + timestep = temp_ts.expand(latent_model_input.shape[0], temp_ts.size(1)) + else: + timestep = t.expand(latent_model_input.shape[0]) + + if self.transformer_2 is not None: + if t >= boundary * self.scheduler.config.num_train_timesteps: + local_transformer = self.transformer_2 + else: + local_transformer = self.transformer + else: + local_transformer = self.transformer + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = local_transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + y=y, + ) + + # perform guidance + if do_classifier_free_guidance: + if self.transformer_2 is not None and (isinstance(self.guidance_scale, (list, tuple))): + sample_guide_scale = self.guidance_scale[1] if t >= self.transformer_2.config.boundary * self.scheduler.config.num_train_timesteps else self.guidance_scale[0] + else: + sample_guide_scale = self.guidance_scale + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + sample_guide_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if self.vae.spatial_compression_ratio >= 16 and not mask[:, :, 0, :, :].any(): + latents = (1 - mask) * masked_video_latents + mask * latents + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_s2v.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_s2v.py new file mode 100644 index 0000000000000000000000000000000000000000..eb421af38e26b8bed8903bdf9e4642f23c194506 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_s2v.py @@ -0,0 +1,815 @@ +import inspect +import math +import copy +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange +from PIL import Image +from torchvision import transforms +from transformers import T5Tokenizer + +from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + Wan2_2Transformer3DModel_S2V, WanAudioEncoder, + WanT5EncoderModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class Wan2_2S2VPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = ["transformer_2", "audio_encoder"] + model_cpu_offload_seq = "text_encoder->transformer_2->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + audio_encoder: WanAudioEncoder, + vae: AutoencoderKLWan, + transformer: Wan2_2Transformer3DModel_S2V, + transformer_2: Wan2_2Transformer3DModel_S2V = None, + scheduler: FlowMatchEulerDiscreteScheduler = None, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, + transformer_2=transformer_2, scheduler=scheduler, audio_encoder=audio_encoder + ) + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + self.motion_frames = 73 + self.audio_sample_m = 0 + self.drop_first_motion = True + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def encode_audio_embeddings(self, audio_path, num_frames, fps, weight_dtype, device): + z = self.audio_encoder.extract_audio_feat( + audio_path, return_all_layers=True) + audio_embed_bucket, num_repeat = self.audio_encoder.get_audio_embed_bucket_fps( + z, fps=fps, batch_frames=num_frames, m=self.audio_sample_m) + audio_embed_bucket = audio_embed_bucket.to(device, + weight_dtype) + audio_embed_bucket = audio_embed_bucket.unsqueeze(0) + if len(audio_embed_bucket.shape) == 3: + audio_embed_bucket = audio_embed_bucket.permute(0, 2, 1) + elif len(audio_embed_bucket.shape) == 4: + audio_embed_bucket = audio_embed_bucket.permute(0, 2, 3, 1) + return audio_embed_bucket, num_repeat + + def encode_pose_latents(self, pose_video, num_repeat, num_frames, size, fps, weight_dtype, device): + height, width = size + if not pose_video is None: + padding_frame_num = num_repeat * num_frames - pose_video.shape[2] + pose_video = torch.cat( + [ + pose_video, + -torch.ones([1, 3, padding_frame_num, height, width]) + ], + dim=2 + ) + + cond_tensors = torch.chunk(pose_video, num_repeat, dim=2) + else: + cond_tensors = [-torch.ones([1, 3, num_frames, height, width])] + + pose_latents = [] + for r in range(len(cond_tensors)): + cond = cond_tensors[r] + cond = torch.cat([cond[:, :, 0:1].repeat(1, 1, 1, 1, 1), cond], + dim=2) + cond_lat = self.vae.encode(cond.to(dtype=weight_dtype, device=device))[0].mode()[:, :, 1:] + pose_latents.append(cond_lat) + return pose_latents + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None, num_length_latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1 if num_length_latents is None else num_length_latents, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_control_latents( + self, control, control_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the control to latents shape as we concatenate the control to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if control is not None: + control = control.to(device=device, dtype=dtype) + bs = 1 + new_control = [] + for i in range(0, control.shape[0], bs): + control_bs = control[i : i + bs] + control_bs = self.vae.encode(control_bs)[0] + control_bs = control_bs.mode() + new_control.append(control_bs) + control = torch.cat(new_control, dim = 0) + + if control_image is not None: + control_image = control_image.to(device=device, dtype=dtype) + bs = 1 + new_control_pixel_values = [] + for i in range(0, control_image.shape[0], bs): + control_pixel_values_bs = control_image[i : i + bs] + control_pixel_values_bs = self.vae.encode(control_pixel_values_bs)[0] + control_pixel_values_bs = control_pixel_values_bs.mode() + new_control_pixel_values.append(control_pixel_values_bs) + control_image_latents = torch.cat(new_control_pixel_values, dim = 0) + else: + control_image_latents = None + + return control, control_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + # frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + ref_image: Union[torch.FloatTensor] = None, + audio_path = None, + pose_video = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + boundary: float = 0.875, + comfyui_progressbar: bool = False, + shift: int = 5, + fps: int = 16, + init_first_frame: bool = False, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + lat_motion_frames = (self.motion_frames + 3) // 4 + lat_target_frames = (num_frames + 3 + self.motion_frames) // 4 - lat_motion_frames + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + # 5. Prepare latents. + latent_channels = self.vae.config.latent_channels + if comfyui_progressbar: + pbar.update(1) + + video_length = ref_image.shape[2] + ref_image = self.image_processor.preprocess(rearrange(ref_image, "b c f h w -> (b f) c h w"), height=height, width=width) + ref_image = ref_image.to(dtype=torch.float32) + ref_image = rearrange(ref_image, "(b f) c h w -> b c f h w", f=video_length) + + ref_image_latentes = self.prepare_control_latents( + None, + ref_image, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance + )[1] + ref_image_latentes = ref_image_latentes[:, :, :1] + + # Extract audio emb + audio_emb, num_repeat = self.encode_audio_embeddings( + audio_path, num_frames=num_frames, fps=fps, weight_dtype=weight_dtype, device=device + ) + + # Encode the motion latents + motion_latents = torch.zeros( + [1, 3, self.motion_frames, height, width], + dtype=weight_dtype, + device=device + ) + videos_last_frames = motion_latents.detach() + drop_first_motion = self.drop_first_motion + if init_first_frame: + drop_first_motion = False + motion_latents[:, :, -6:] = ref_image + motion_latents = self.vae.encode(motion_latents)[0].mode() + + # Get pose cond input if need + if pose_video is not None: + video_length = pose_video.shape[2] + pose_video = self.image_processor.preprocess(rearrange(pose_video, "b c f h w -> (b f) c h w"), height=height, width=width) + pose_video = pose_video.to(dtype=torch.float32) + pose_video = rearrange(pose_video, "(b f) c h w -> b c f h w", f=video_length) + pose_latents = self.encode_pose_latents( + pose_video=pose_video, + num_repeat=num_repeat, + num_frames=num_frames, + size=(height, width), + fps=fps, + weight_dtype=weight_dtype, + device=device + ) + + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + videos = [] + copy_timesteps = copy.deepcopy(timesteps) + copy_latents = copy.deepcopy(latents) + for r in range(num_repeat): + # Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, copy_timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, copy_timesteps) + self._num_timesteps = len(timesteps) + + target_shape = (self.vae.latent_channels, lat_target_frames, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + copy_latents, + num_length_latents=target_shape[1] + ) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + with torch.no_grad(): + left_idx = r * num_frames + right_idx = r * num_frames + num_frames + cond_latents = pose_latents[r] if pose_video is not None else pose_latents[0] * 0 + cond_latents = cond_latents.to(dtype=weight_dtype, device=device) + audio_input = audio_emb[..., left_idx:right_idx] + + pose_latents_input = torch.cat([cond_latents] * 2) if do_classifier_free_guidance else cond_latents + motion_latents_input = torch.cat([motion_latents] * 2) if do_classifier_free_guidance else motion_latents + audio_emb_input = torch.cat([audio_input * 0] + [audio_input]) if do_classifier_free_guidance else audio_input + ref_image_latentes_input = torch.cat([ref_image_latentes] * 2) if do_classifier_free_guidance else ref_image_latentes + motion_frames=[[self.motion_frames, (self.motion_frames + 3) // 4]] * 2 if do_classifier_free_guidance else [[self.motion_frames, (self.motion_frames + 3) // 4]] + timestep = t.expand(latent_model_input.shape[0]) + + if self.transformer_2 is not None: + if t >= boundary * self.scheduler.config.num_train_timesteps: + local_transformer = self.transformer_2 + else: + local_transformer = self.transformer + else: + local_transformer = self.transformer + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = local_transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + cond_states=pose_latents_input, + motion_latents=motion_latents_input, + ref_latents=ref_image_latentes_input, + audio_input=audio_emb_input, + motion_frames=motion_frames, + drop_motion_frames=drop_first_motion and r == 0, + ) + # perform guidance + if do_classifier_free_guidance: + if self.transformer_2 is not None and (isinstance(self.guidance_scale, (list, tuple))): + sample_guide_scale = self.guidance_scale[1] if t >= self.transformer_2.config.boundary * self.scheduler.config.num_train_timesteps else self.guidance_scale[0] + else: + sample_guide_scale = self.guidance_scale + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + sample_guide_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if not (drop_first_motion and r == 0): + decode_latents = torch.cat([motion_latents, latents], dim=2) + else: + decode_latents = torch.cat([ref_image_latentes, latents], dim=2) + + image = self.vae.decode(decode_latents).sample + image = image[:, :, -(num_frames):] + if (drop_first_motion and r == 0): + image = image[:, :, 3:] + + overlap_frames_num = min(self.motion_frames, image.shape[2]) + videos_last_frames = torch.cat( + [ + videos_last_frames[:, :, overlap_frames_num:], + image[:, :, -overlap_frames_num:] + ], + dim=2 + ).to(dtype=motion_latents.dtype, device=motion_latents.device) + motion_latents = self.vae.encode(videos_last_frames)[0].mode() + videos.append(image) + + videos = torch.cat(videos, dim=2) + videos = (videos / 2 + 0.5).clamp(0, 1) + + # Offload all models + self.maybe_free_model_hooks() + + return WanPipelineOutput(videos=videos.float().cpu()) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_ti2v.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_ti2v.py new file mode 100644 index 0000000000000000000000000000000000000000..12b3bd8cf0aa36a16b059d3eb55c2b8fcad61762 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_ti2v.py @@ -0,0 +1,732 @@ +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange +from PIL import Image +from transformers import T5Tokenizer + +from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class Wan2_2TI2VPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = ["transformer_2"] + model_cpu_offload_seq = "text_encoder->transformer_2->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: Wan2_2Transformer3DModel, + transformer_2: Wan2_2Transformer3DModel = None, + scheduler: FlowMatchEulerDiscreteScheduler = None, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, + transformer_2=transformer_2, scheduler=scheduler + ) + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance, noise_aug_strength + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if mask is not None: + mask = mask.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask = [] + for i in range(0, mask.shape[0], bs): + mask_bs = mask[i : i + bs] + mask_bs = self.vae.encode(mask_bs)[0] + mask_bs = mask_bs.mode() + new_mask.append(mask_bs) + mask = torch.cat(new_mask, dim = 0) + # mask = mask * self.vae.config.scaling_factor + + if masked_image is not None: + masked_image = masked_image.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask_pixel_values = [] + for i in range(0, masked_image.shape[0], bs): + mask_pixel_values_bs = masked_image[i : i + bs] + mask_pixel_values_bs = self.vae.encode(mask_pixel_values_bs)[0] + mask_pixel_values_bs = mask_pixel_values_bs.mode() + new_mask_pixel_values.append(mask_pixel_values_bs) + masked_image_latents = torch.cat(new_mask_pixel_values, dim = 0) + # masked_image_latents = masked_image_latents * self.vae.config.scaling_factor + else: + masked_image_latents = None + + return mask, masked_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + video: Union[torch.FloatTensor] = None, + mask_video: Union[torch.FloatTensor] = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + boundary: float = 0.875, + comfyui_progressbar: bool = False, + shift: int = 5, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + # 5. Prepare latents. + if video is not None: + video_length = video.shape[2] + init_video = self.image_processor.preprocess(rearrange(video, "b c f h w -> (b f) c h w"), height=height, width=width) + init_video = init_video.to(dtype=torch.float32) + init_video = rearrange(init_video, "(b f) c h w -> b c f h w", f=video_length) + else: + init_video = None + + latent_channels = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + ) + if comfyui_progressbar: + pbar.update(1) + + # Prepare mask latent variables + if init_video is not None and not (mask_video == 255).all(): + bs, _, video_length, height, width = video.size() + mask_condition = self.mask_processor.preprocess(rearrange(mask_video, "b c f h w -> (b f) c h w"), height=height, width=width) + mask_condition = mask_condition.to(dtype=torch.float32) + mask_condition = rearrange(mask_condition, "(b f) c h w -> b c f h w", f=video_length) + + masked_video = init_video * (torch.tile(mask_condition, [1, 3, 1, 1, 1]) < 0.5) + _, masked_video_latents = self.prepare_mask_latents( + None, + masked_video, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance, + noise_aug_strength=None, + ) + + mask_condition = torch.concat( + [ + torch.repeat_interleave(mask_condition[:, :, 0:1], repeats=4, dim=2), + mask_condition[:, :, 1:] + ], dim=2 + ) + mask_condition = mask_condition.view(bs, mask_condition.shape[2] // 4, 4, height, width) + mask_condition = mask_condition.transpose(1, 2) + + mask = F.interpolate(mask_condition[:, :1], size=latents.size()[-3:], mode='trilinear', align_corners=True).to(device, weight_dtype) + latents = (1 - mask) * masked_video_latents + mask * latents + else: + init_video = None + + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, (num_frames - 1) // self.vae.temporal_compression_ratio + 1, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + if init_video is not None: + temp_ts = ((mask[0][0][:, ::2, ::2]) * t).flatten() + temp_ts = torch.cat([ + temp_ts, + temp_ts.new_ones(seq_len - temp_ts.size(0)) * t + ]) + temp_ts = temp_ts.unsqueeze(0) + timestep = temp_ts.expand(latent_model_input.shape[0], temp_ts.size(1)) + else: + timestep = t.expand(latent_model_input.shape[0]) + + if self.transformer_2 is not None: + if t >= boundary * self.scheduler.config.num_train_timesteps: + local_transformer = self.transformer_2 + else: + local_transformer = self.transformer + else: + local_transformer = self.transformer + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = local_transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + ) + + # perform guidance + if do_classifier_free_guidance: + if self.transformer_2 is not None and (isinstance(self.guidance_scale, (list, tuple))): + sample_guide_scale = self.guidance_scale[1] if t >= self.transformer_2.config.boundary * self.scheduler.config.num_train_timesteps else self.guidance_scale[0] + else: + sample_guide_scale = self.guidance_scale + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + sample_guide_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + if init_video is not None: + latents = (1 - mask) * masked_video_latents + mask * latents + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_vace_fun.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_vace_fun.py new file mode 100644 index 0000000000000000000000000000000000000000..c8b0d9932c5797af2e20081e37b26fae0c599486 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan2_2_vace_fun.py @@ -0,0 +1,801 @@ +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange +from PIL import Image +from transformers import T5Tokenizer + +from ..models import (AutoencoderKLWan, AutoTokenizer, + WanT5EncoderModel, VaceWanTransformer3DModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class Wan2_2VaceFunPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = ["transformer_2"] + model_cpu_offload_seq = "text_encoder->transformer_2->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: VaceWanTransformer3DModel, + transformer_2: VaceWanTransformer3DModel = None, + scheduler: FlowMatchEulerDiscreteScheduler = None, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, + transformer_2=transformer_2, scheduler=scheduler + ) + + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None, num_length_latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1 if num_length_latents is None else num_length_latents, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def vace_encode_frames(self, frames, ref_images, masks=None, vae=None): + vae = self.vae if vae is None else vae + weight_dtype = frames.dtype + if ref_images is None: + ref_images = [None] * len(frames) + else: + assert len(frames) == len(ref_images) + + if masks is None: + latents = vae.encode(frames)[0].mode() + else: + masks = [torch.where(m > 0.5, 1.0, 0.0).to(weight_dtype) for m in masks] + inactive = [i * (1 - m) + 0 * m for i, m in zip(frames, masks)] + reactive = [i * m + 0 * (1 - m) for i, m in zip(frames, masks)] + inactive = vae.encode(inactive)[0].mode() + reactive = vae.encode(reactive)[0].mode() + latents = [torch.cat((u, c), dim=0) for u, c in zip(inactive, reactive)] + + cat_latents = [] + for latent, refs in zip(latents, ref_images): + if refs is not None: + if masks is None: + ref_latent = vae.encode(refs)[0].mode() + else: + ref_latent = vae.encode(refs)[0].mode() + ref_latent = [torch.cat((u, torch.zeros_like(u)), dim=0) for u in ref_latent] + assert all([x.shape[1] == 1 for x in ref_latent]) + latent = torch.cat([*ref_latent, latent], dim=1) + cat_latents.append(latent) + return cat_latents + + def vace_encode_masks(self, masks, ref_images=None, vae_stride=[4, 8, 8]): + if ref_images is None: + ref_images = [None] * len(masks) + else: + assert len(masks) == len(ref_images) + + result_masks = [] + for mask, refs in zip(masks, ref_images): + c, depth, height, width = mask.shape + new_depth = int((depth + 3) // vae_stride[0]) + height = 2 * (int(height) // (vae_stride[1] * 2)) + width = 2 * (int(width) // (vae_stride[2] * 2)) + + # reshape + mask = mask[0, :, :, :] + mask = mask.view( + depth, height, vae_stride[1], width, vae_stride[1] + ) # depth, height, 8, width, 8 + mask = mask.permute(2, 4, 0, 1, 3) # 8, 8, depth, height, width + mask = mask.reshape( + vae_stride[1] * vae_stride[2], depth, height, width + ) # 8*8, depth, height, width + + # interpolation + mask = F.interpolate(mask.unsqueeze(0), size=(new_depth, height, width), mode='nearest-exact').squeeze(0) + + if refs is not None: + length = len(refs) + mask_pad = torch.zeros_like(mask[:, :length, :, :]) + mask = torch.cat((mask_pad, mask), dim=1) + result_masks.append(mask) + return result_masks + + def vace_latent(self, z, m): + return [torch.cat([zz, mm], dim=0) for zz, mm in zip(z, m)] + + def prepare_control_latents( + self, control, control_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the control to latents shape as we concatenate the control to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if control is not None: + control = control.to(device=device, dtype=dtype) + bs = 1 + new_control = [] + for i in range(0, control.shape[0], bs): + control_bs = control[i : i + bs] + control_bs = self.vae.encode(control_bs)[0] + control_bs = control_bs.mode() + new_control.append(control_bs) + control = torch.cat(new_control, dim = 0) + + if control_image is not None: + control_image = control_image.to(device=device, dtype=dtype) + bs = 1 + new_control_pixel_values = [] + for i in range(0, control_image.shape[0], bs): + control_pixel_values_bs = control_image[i : i + bs] + control_pixel_values_bs = self.vae.encode(control_pixel_values_bs)[0] + control_pixel_values_bs = control_pixel_values_bs.mode() + new_control_pixel_values.append(control_pixel_values_bs) + control_image_latents = torch.cat(new_control_pixel_values, dim = 0) + else: + control_image_latents = None + + return control, control_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + video: Union[torch.FloatTensor] = None, + mask_video: Union[torch.FloatTensor] = None, + control_video: Union[torch.FloatTensor] = None, + subject_ref_images: Union[torch.FloatTensor] = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + boundary: float = 0.875, + comfyui_progressbar: bool = False, + shift: int = 5, + vace_context_scale: float = 1.0, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + latent_channels = self.vae.config.latent_channels + + if comfyui_progressbar: + pbar.update(1) + + # Prepare mask latent variables + if mask_video is not None: + bs, _, video_length, height, width = video.size() + mask_condition = self.mask_processor.preprocess(rearrange(mask_video, "b c f h w -> (b f) c h w"), height=height, width=width) + mask_condition = mask_condition.to(dtype=torch.float32) + mask_condition = rearrange(mask_condition, "(b f) c h w -> b c f h w", f=video_length) + mask_condition = torch.tile(mask_condition, [1, 3, 1, 1, 1]).to(dtype=weight_dtype, device=device) + + + if control_video is not None: + video_length = control_video.shape[2] + control_video = self.image_processor.preprocess(rearrange(control_video, "b c f h w -> (b f) c h w"), height=height, width=width) + control_video = control_video.to(dtype=torch.float32) + input_video = rearrange(control_video, "(b f) c h w -> b c f h w", f=video_length) + + input_video = input_video.to(dtype=weight_dtype, device=device) + + elif video is not None: + video_length = video.shape[2] + init_video = self.image_processor.preprocess(rearrange(video, "b c f h w -> (b f) c h w"), height=height, width=width) + init_video = init_video.to(dtype=torch.float32) + init_video = rearrange(init_video, "(b f) c h w -> b c f h w", f=video_length).to(dtype=weight_dtype, device=device) + + input_video = init_video * (mask_condition < 0.5) + input_video = input_video.to(dtype=weight_dtype, device=device) + + if subject_ref_images is not None: + video_length = subject_ref_images.shape[2] + subject_ref_images = self.image_processor.preprocess(rearrange(subject_ref_images, "b c f h w -> (b f) c h w"), height=height, width=width) + subject_ref_images = subject_ref_images.to(dtype=torch.float32) + subject_ref_images = rearrange(subject_ref_images, "(b f) c h w -> b c f h w", f=video_length) + subject_ref_images = subject_ref_images.to(dtype=weight_dtype, device=device) + + bs, c, f, h, w = subject_ref_images.size() + new_subject_ref_images = [] + for i in range(bs): + new_subject_ref_images.append([]) + for j in range(f): + new_subject_ref_images[i].append(subject_ref_images[i, :, j:j+1]) + subject_ref_images = new_subject_ref_images + + vace_latents = self.vace_encode_frames(input_video, subject_ref_images, masks=mask_condition, vae=self.vae) + mask_latents = self.vace_encode_masks(mask_condition, subject_ref_images, vae_stride=[4, self.vae.spatial_compression_ratio, self.vae.spatial_compression_ratio]) + vace_context = self.vace_latent(vace_latents, mask_latents) + + # 5. Prepare latents. + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + num_length_latents=vace_latents[0].size(1) + ) + + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, vace_latents[0].size(1), vace_latents[0].size(2), vace_latents[0].size(3)) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + vace_context_input = torch.stack(vace_context * 2) if do_classifier_free_guidance else vace_context + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + if self.transformer_2 is not None: + if t >= boundary * self.scheduler.config.num_train_timesteps: + local_transformer = self.transformer_2 + else: + local_transformer = self.transformer + else: + local_transformer = self.transformer + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = local_transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + vace_context=vace_context_input, + seq_len=seq_len, + vace_context_scale=vace_context_scale, + ) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if subject_ref_images is not None: + len_subject_ref_images = len(subject_ref_images[0]) + latents = latents[:, :, len_subject_ref_images:, :, :] + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan_fun_control.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan_fun_control.py new file mode 100644 index 0000000000000000000000000000000000000000..4d4ec75e77a434f031b081d39eb4358dc59025a8 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan_fun_control.py @@ -0,0 +1,799 @@ +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange +from PIL import Image +from transformers import T5Tokenizer + +from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, WanTransformer3DModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class WanFunControlPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = [] + model_cpu_offload_seq = "text_encoder->clip_image_encoder->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: WanTransformer3DModel, + clip_image_encoder: CLIPModel, + scheduler: FlowMatchEulerDiscreteScheduler, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, clip_image_encoder=clip_image_encoder, scheduler=scheduler + ) + + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_control_latents( + self, control, control_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the control to latents shape as we concatenate the control to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if control is not None: + control = control.to(device=device, dtype=dtype) + bs = 1 + new_control = [] + for i in range(0, control.shape[0], bs): + control_bs = control[i : i + bs] + control_bs = self.vae.encode(control_bs)[0] + control_bs = control_bs.mode() + new_control.append(control_bs) + control = torch.cat(new_control, dim = 0) + + if control_image is not None: + control_image = control_image.to(device=device, dtype=dtype) + bs = 1 + new_control_pixel_values = [] + for i in range(0, control_image.shape[0], bs): + control_pixel_values_bs = control_image[i : i + bs] + control_pixel_values_bs = self.vae.encode(control_pixel_values_bs)[0] + control_pixel_values_bs = control_pixel_values_bs.mode() + new_control_pixel_values.append(control_pixel_values_bs) + control_image_latents = torch.cat(new_control_pixel_values, dim = 0) + else: + control_image_latents = None + + return control, control_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + control_video: Union[torch.FloatTensor] = None, + control_camera_video: Union[torch.FloatTensor] = None, + start_image: Union[torch.FloatTensor] = None, + ref_image: Union[torch.FloatTensor] = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + clip_image: Image = None, + max_sequence_length: int = 512, + comfyui_progressbar: bool = False, + shift: int = 5, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + # 5. Prepare latents. + latent_channels = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + ) + if comfyui_progressbar: + pbar.update(1) + + # Prepare mask latent variables + if control_camera_video is not None: + control_latents = None + # Rearrange dimensions + # Concatenate and transpose dimensions + control_camera_latents = torch.concat( + [ + torch.repeat_interleave(control_camera_video[:, :, 0:1], repeats=4, dim=2), + control_camera_video[:, :, 1:] + ], dim=2 + ).transpose(1, 2) + + # Reshape, transpose, and view into desired shape + b, f, c, h, w = control_camera_latents.shape + control_camera_latents = control_camera_latents.contiguous().view(b, f // 4, 4, c, h, w).transpose(2, 3) + control_camera_latents = control_camera_latents.contiguous().view(b, f // 4, c * 4, h, w).transpose(1, 2) + elif control_video is not None: + video_length = control_video.shape[2] + control_video = self.image_processor.preprocess(rearrange(control_video, "b c f h w -> (b f) c h w"), height=height, width=width) + control_video = control_video.to(dtype=torch.float32) + control_video = rearrange(control_video, "(b f) c h w -> b c f h w", f=video_length) + control_video_latents = self.prepare_control_latents( + None, + control_video, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance + )[1] + control_camera_latents = None + else: + control_video_latents = torch.zeros_like(latents).to(device, weight_dtype) + control_camera_latents = None + + if start_image is not None: + video_length = start_image.shape[2] + start_image = self.image_processor.preprocess(rearrange(start_image, "b c f h w -> (b f) c h w"), height=height, width=width) + start_image = start_image.to(dtype=torch.float32) + start_image = rearrange(start_image, "(b f) c h w -> b c f h w", f=video_length) + + start_image_latentes = self.prepare_control_latents( + None, + start_image, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance + )[1] + + start_image_latentes_conv_in = torch.zeros_like(latents) + if latents.size()[2] != 1: + start_image_latentes_conv_in[:, :, :1] = start_image_latentes + else: + start_image_latentes_conv_in = torch.zeros_like(latents) + + # Prepare clip latent variables + if clip_image is not None: + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(device, weight_dtype) + clip_context = self.clip_image_encoder([clip_image[:, None, :, :]]) + else: + clip_image = Image.new("RGB", (512, 512), color=(0, 0, 0)) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(device, weight_dtype) + clip_context = self.clip_image_encoder([clip_image[:, None, :, :]]) + clip_context = torch.zeros_like(clip_context) + + if self.transformer.config.get("add_ref_conv", False): + if ref_image is not None: + video_length = ref_image.shape[2] + ref_image = self.image_processor.preprocess(rearrange(ref_image, "b c f h w -> (b f) c h w"), height=height, width=width) + ref_image = ref_image.to(dtype=torch.float32) + ref_image = rearrange(ref_image, "(b f) c h w -> b c f h w", f=video_length) + + ref_image_latentes = self.prepare_control_latents( + None, + ref_image, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance + )[1] + ref_image_latentes = ref_image_latentes[:, :, 0] + else: + ref_image_latentes = torch.zeros_like(latents)[:, :, 0] + else: + if ref_image is not None: + raise ValueError("The add_ref_conv is False, but ref_image is not None") + else: + ref_image_latentes = None + + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, (num_frames - 1) // self.vae.temporal_compression_ratio + 1, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # Prepare mask latent variables + if control_camera_video is not None: + control_latents_input = None + control_camera_latents_input = ( + torch.cat([control_camera_latents] * 2) if do_classifier_free_guidance else control_camera_latents + ).to(device, weight_dtype) + else: + control_latents_input = ( + torch.cat([control_video_latents] * 2) if do_classifier_free_guidance else control_video_latents + ).to(device, weight_dtype) + control_camera_latents_input = None + + start_image_latentes_conv_in_input = ( + torch.cat([start_image_latentes_conv_in] * 2) if do_classifier_free_guidance else start_image_latentes_conv_in + ).to(device, weight_dtype) + control_latents_input = start_image_latentes_conv_in_input if control_latents_input is None else \ + torch.cat([control_latents_input, start_image_latentes_conv_in_input], dim = 1) + + clip_context_input = ( + torch.cat([clip_context] * 2) if do_classifier_free_guidance else clip_context + ) + + if ref_image_latentes is not None: + full_ref = ( + torch.cat([ref_image_latentes] * 2) if do_classifier_free_guidance else ref_image_latentes + ).to(device, weight_dtype) + else: + full_ref = None + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = self.transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + y=control_latents_input, + y_camera=control_camera_latents_input, + full_ref=full_ref, + clip_fea=clip_context_input, + ) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan_fun_inpaint.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan_fun_inpaint.py new file mode 100644 index 0000000000000000000000000000000000000000..35f3b96def177e1ce851d369d28353a19aefb3db --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan_fun_inpaint.py @@ -0,0 +1,734 @@ +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange +from PIL import Image +from transformers import T5Tokenizer + +from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, WanTransformer3DModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class WanFunInpaintPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = [] + model_cpu_offload_seq = "text_encoder->clip_image_encoder->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: WanTransformer3DModel, + clip_image_encoder: CLIPModel, + scheduler: FlowMatchEulerDiscreteScheduler, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, clip_image_encoder=clip_image_encoder, scheduler=scheduler + ) + + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance, noise_aug_strength + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if mask is not None: + mask = mask.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask = [] + for i in range(0, mask.shape[0], bs): + mask_bs = mask[i : i + bs] + mask_bs = self.vae.encode(mask_bs)[0] + mask_bs = mask_bs.mode() + new_mask.append(mask_bs) + mask = torch.cat(new_mask, dim = 0) + # mask = mask * self.vae.config.scaling_factor + + if masked_image is not None: + masked_image = masked_image.to(device=device, dtype=self.vae.dtype) + bs = 1 + new_mask_pixel_values = [] + for i in range(0, masked_image.shape[0], bs): + mask_pixel_values_bs = masked_image[i : i + bs] + mask_pixel_values_bs = self.vae.encode(mask_pixel_values_bs)[0] + mask_pixel_values_bs = mask_pixel_values_bs.mode() + new_mask_pixel_values.append(mask_pixel_values_bs) + masked_image_latents = torch.cat(new_mask_pixel_values, dim = 0) + # masked_image_latents = masked_image_latents * self.vae.config.scaling_factor + else: + masked_image_latents = None + + return mask, masked_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + video: Union[torch.FloatTensor] = None, + mask_video: Union[torch.FloatTensor] = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + clip_image: Image = None, + max_sequence_length: int = 512, + comfyui_progressbar: bool = False, + shift: int = 5, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + # 5. Prepare latents. + if video is not None: + video_length = video.shape[2] + init_video = self.image_processor.preprocess(rearrange(video, "b c f h w -> (b f) c h w"), height=height, width=width) + init_video = init_video.to(dtype=torch.float32) + init_video = rearrange(init_video, "(b f) c h w -> b c f h w", f=video_length) + else: + init_video = None + + latent_channels = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + ) + if comfyui_progressbar: + pbar.update(1) + + # Prepare mask latent variables + if init_video is not None: + if (mask_video == 255).all(): + mask_latents = torch.tile( + torch.zeros_like(latents)[:, :1].to(device, weight_dtype), [1, 4, 1, 1, 1] + ) + masked_video_latents = torch.zeros_like(latents).to(device, weight_dtype) + else: + bs, _, video_length, height, width = video.size() + mask_condition = self.mask_processor.preprocess(rearrange(mask_video, "b c f h w -> (b f) c h w"), height=height, width=width) + mask_condition = mask_condition.to(dtype=torch.float32) + mask_condition = rearrange(mask_condition, "(b f) c h w -> b c f h w", f=video_length) + + masked_video = init_video * (torch.tile(mask_condition, [1, 3, 1, 1, 1]) < 0.5) + _, masked_video_latents = self.prepare_mask_latents( + None, + masked_video, + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance, + noise_aug_strength=None, + ) + + mask_condition = torch.concat( + [ + torch.repeat_interleave(mask_condition[:, :, 0:1], repeats=4, dim=2), + mask_condition[:, :, 1:] + ], dim=2 + ) + mask_condition = mask_condition.view(bs, mask_condition.shape[2] // 4, 4, height, width) + mask_condition = mask_condition.transpose(1, 2) + mask_latents = resize_mask(1 - mask_condition, masked_video_latents, True).to(device, weight_dtype) + + # Prepare clip latent variables + if clip_image is not None: + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(device, weight_dtype) + clip_context = self.clip_image_encoder([clip_image[:, None, :, :]]) + else: + clip_image = Image.new("RGB", (512, 512), color=(0, 0, 0)) + clip_image = TF.to_tensor(clip_image).sub_(0.5).div_(0.5).to(device, weight_dtype) + clip_context = self.clip_image_encoder([clip_image[:, None, :, :]]) + clip_context = torch.zeros_like(clip_context) + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, (num_frames - 1) // self.vae.temporal_compression_ratio + 1, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + if init_video is not None: + mask_input = torch.cat([mask_latents] * 2) if do_classifier_free_guidance else mask_latents + masked_video_latents_input = ( + torch.cat([masked_video_latents] * 2) if do_classifier_free_guidance else masked_video_latents + ) + y = torch.cat([mask_input, masked_video_latents_input], dim=1).to(device, weight_dtype) + + clip_context_input = ( + torch.cat([clip_context] * 2) if do_classifier_free_guidance else clip_context + ) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = self.transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + y=y, + clip_fea=clip_context_input, + ) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan_phantom.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan_phantom.py new file mode 100644 index 0000000000000000000000000000000000000000..fd993b001a6c2d3f4b3ed9f71c5df2202d523b14 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan_phantom.py @@ -0,0 +1,695 @@ +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange +from PIL import Image +from transformers import T5Tokenizer + +from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, WanTransformer3DModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class WanFunPhantomPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = [] + model_cpu_offload_seq = "text_encoder->clip_image_encoder->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: WanTransformer3DModel, + scheduler: FlowMatchEulerDiscreteScheduler, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, scheduler=scheduler + ) + + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_control_latents( + self, control, control_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the control to latents shape as we concatenate the control to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if control is not None: + control = control.to(device=device, dtype=dtype) + bs = 1 + new_control = [] + for i in range(0, control.shape[0], bs): + control_bs = control[i : i + bs] + control_bs = self.vae.encode(control_bs)[0] + control_bs = control_bs.mode() + new_control.append(control_bs) + control = torch.cat(new_control, dim = 0) + + if control_image is not None: + control_image = control_image.to(device=device, dtype=dtype) + bs = 1 + new_control_pixel_values = [] + for i in range(0, control_image.shape[0], bs): + control_pixel_values_bs = control_image[i : i + bs] + control_pixel_values_bs = self.vae.encode(control_pixel_values_bs)[0] + control_pixel_values_bs = control_pixel_values_bs.mode() + new_control_pixel_values.append(control_pixel_values_bs) + control_image_latents = torch.cat(new_control_pixel_values, dim = 0) + else: + control_image_latents = None + + return control, control_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + subject_ref_images: Union[torch.FloatTensor] = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + comfyui_progressbar: bool = False, + shift: int = 5, + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + # 5. Prepare latents. + latent_channels = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + ) + if comfyui_progressbar: + pbar.update(1) + + if subject_ref_images is not None: + video_length = subject_ref_images.shape[2] + subject_ref_images = self.image_processor.preprocess(rearrange(subject_ref_images, "b c f h w -> (b f) c h w"), height=height, width=width) + subject_ref_images = subject_ref_images.to(dtype=torch.float32) + subject_ref_images = rearrange(subject_ref_images, "(b f) c h w -> b c f h w", f=video_length) + + subject_ref_images_latentes = torch.cat( + [ + self.prepare_control_latents( + None, + subject_ref_images[:, :, i:i+1], + batch_size, + height, + width, + weight_dtype, + device, + generator, + do_classifier_free_guidance + )[1] for i in range(video_length) + ], dim = 2 + ) + + if comfyui_progressbar: + pbar.update(1) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, (num_frames - 1) // self.vae.temporal_compression_ratio + 1, width // self.vae.spatial_compression_ratio, height // self.vae.spatial_compression_ratio) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + if subject_ref_images is not None: + subject_ref = ( + torch.cat( + [torch.zeros_like(subject_ref_images_latentes), subject_ref_images_latentes] + ) if do_classifier_free_guidance else subject_ref_images_latentes + ).to(device, weight_dtype) + else: + subject_ref = None + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = self.transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + seq_len=seq_len, + subject_ref=subject_ref, + ) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_wan_vace.py b/VideoX-Fun/videox_fun/pipeline/pipeline_wan_vace.py new file mode 100644 index 0000000000000000000000000000000000000000..7c0ded274bb5ebe29fb7840a23a210b2917ec071 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_wan_vace.py @@ -0,0 +1,787 @@ +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +import torchvision.transforms.functional as TF +from diffusers import FlowMatchEulerDiscreteScheduler +from diffusers.callbacks import MultiPipelineCallbacks, PipelineCallback +from diffusers.image_processor import VaeImageProcessor +from diffusers.models.embeddings import get_1d_rotary_pos_embed +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import BaseOutput, logging, replace_example_docstring +from diffusers.utils.torch_utils import randn_tensor +from diffusers.video_processor import VideoProcessor +from einops import rearrange +from PIL import Image +from transformers import T5Tokenizer + +from ..models import (AutoencoderKLWan, AutoTokenizer, + WanT5EncoderModel, VaceWanTransformer3DModel) +from ..utils.fm_solvers import (FlowDPMSolverMultistepScheduler, + get_sampling_sigmas) +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + pass + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +def resize_mask(mask, latent, process_first_frame_only=True): + latent_size = latent.size() + batch_size, channels, num_frames, height, width = mask.shape + + if process_first_frame_only: + target_size = list(latent_size[2:]) + target_size[0] = 1 + first_frame_resized = F.interpolate( + mask[:, :, 0:1, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + + target_size = list(latent_size[2:]) + target_size[0] = target_size[0] - 1 + if target_size[0] != 0: + remaining_frames_resized = F.interpolate( + mask[:, :, 1:, :, :], + size=target_size, + mode='trilinear', + align_corners=False + ) + resized_mask = torch.cat([first_frame_resized, remaining_frames_resized], dim=2) + else: + resized_mask = first_frame_resized + else: + target_size = list(latent_size[2:]) + resized_mask = F.interpolate( + mask, + size=target_size, + mode='trilinear', + align_corners=False + ) + return resized_mask + + +@dataclass +class WanPipelineOutput(BaseOutput): + r""" + Output class for CogVideo pipelines. + + Args: + video (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing + denoised PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)`. + """ + + videos: torch.Tensor + + +class WanVacePipeline(DiffusionPipeline): + r""" + Pipeline for text-to-video generation using Wan. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + """ + + _optional_components = [] + model_cpu_offload_seq = "text_encoder->transformer->vae" + + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + ] + + def __init__( + self, + tokenizer: AutoTokenizer, + text_encoder: WanT5EncoderModel, + vae: AutoencoderKLWan, + transformer: VaceWanTransformer3DModel, + scheduler: FlowMatchEulerDiscreteScheduler, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, scheduler=scheduler + ) + + self.video_processor = VideoProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae.spatial_compression_ratio) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae.spatial_compression_ratio, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def _get_t5_prompt_embeds( + self, + prompt: Union[str, List[str]] = None, + num_videos_per_prompt: int = 1, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + device = device or self._execution_device + dtype = dtype or self.text_encoder.dtype + + prompt = [prompt] if isinstance(prompt, str) else prompt + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + prompt_attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_sequence_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because `max_sequence_length` is set to " + f" {max_sequence_length} tokens: {removed_text}" + ) + + seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long() + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask.to(device))[0] + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + _, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return [u[:v] for u, v in zip(prompt_embeds, seq_lens)] + + def encode_prompt( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + do_classifier_free_guidance: bool = True, + num_videos_per_prompt: int = 1, + prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + max_sequence_length: int = 512, + device: Optional[torch.device] = None, + dtype: Optional[torch.dtype] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + Whether to use classifier free guidance or not. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + Number of videos that should be generated per prompt. torch device to place the resulting embeddings on + prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.Tensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + device: (`torch.device`, *optional*): + torch device + dtype: (`torch.dtype`, *optional*): + torch dtype + """ + device = device or self._execution_device + + prompt = [prompt] if isinstance(prompt, str) else prompt + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + prompt_embeds = self._get_t5_prompt_embeds( + prompt=prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + if do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + + negative_prompt_embeds = self._get_t5_prompt_embeds( + prompt=negative_prompt, + num_videos_per_prompt=num_videos_per_prompt, + max_sequence_length=max_sequence_length, + device=device, + dtype=dtype, + ) + + return prompt_embeds, negative_prompt_embeds + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None, num_length_latents=None + ): + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + shape = ( + batch_size, + num_channels_latents, + (num_frames - 1) // self.vae.temporal_compression_ratio + 1 if num_length_latents is None else num_length_latents, + height // self.vae.spatial_compression_ratio, + width // self.vae.spatial_compression_ratio, + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + if hasattr(self.scheduler, "init_noise_sigma"): + latents = latents * self.scheduler.init_noise_sigma + return latents + + def vace_encode_frames(self, frames, ref_images, masks=None, vae=None): + vae = self.vae if vae is None else vae + weight_dtype = frames.dtype + if ref_images is None: + ref_images = [None] * len(frames) + else: + assert len(frames) == len(ref_images) + + if masks is None: + latents = vae.encode(frames)[0].mode() + else: + masks = [torch.where(m > 0.5, 1.0, 0.0).to(weight_dtype) for m in masks] + inactive = [i * (1 - m) + 0 * m for i, m in zip(frames, masks)] + reactive = [i * m + 0 * (1 - m) for i, m in zip(frames, masks)] + inactive = vae.encode(inactive)[0].mode() + reactive = vae.encode(reactive)[0].mode() + latents = [torch.cat((u, c), dim=0) for u, c in zip(inactive, reactive)] + + cat_latents = [] + for latent, refs in zip(latents, ref_images): + if refs is not None: + if masks is None: + ref_latent = vae.encode(refs)[0].mode() + else: + ref_latent = vae.encode(refs)[0].mode() + ref_latent = [torch.cat((u, torch.zeros_like(u)), dim=0) for u in ref_latent] + assert all([x.shape[1] == 1 for x in ref_latent]) + latent = torch.cat([*ref_latent, latent], dim=1) + cat_latents.append(latent) + return cat_latents + + def vace_encode_masks(self, masks, ref_images=None, vae_stride=[4, 8, 8]): + if ref_images is None: + ref_images = [None] * len(masks) + else: + assert len(masks) == len(ref_images) + + result_masks = [] + for mask, refs in zip(masks, ref_images): + c, depth, height, width = mask.shape + new_depth = int((depth + 3) // vae_stride[0]) + height = 2 * (int(height) // (vae_stride[1] * 2)) + width = 2 * (int(width) // (vae_stride[2] * 2)) + + # reshape + mask = mask[0, :, :, :] + mask = mask.view( + depth, height, vae_stride[1], width, vae_stride[1] + ) # depth, height, 8, width, 8 + mask = mask.permute(2, 4, 0, 1, 3) # 8, 8, depth, height, width + mask = mask.reshape( + vae_stride[1] * vae_stride[2], depth, height, width + ) # 8*8, depth, height, width + + # interpolation + mask = F.interpolate(mask.unsqueeze(0), size=(new_depth, height, width), mode='nearest-exact').squeeze(0) + + if refs is not None: + length = len(refs) + mask_pad = torch.zeros_like(mask[:, :length, :, :]) + mask = torch.cat((mask_pad, mask), dim=1) + result_masks.append(mask) + return result_masks + + def vace_latent(self, z, m): + return [torch.cat([zz, mm], dim=0) for zz, mm in zip(z, m)] + + def prepare_control_latents( + self, control, control_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the control to latents shape as we concatenate the control to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + + if control is not None: + control = control.to(device=device, dtype=dtype) + bs = 1 + new_control = [] + for i in range(0, control.shape[0], bs): + control_bs = control[i : i + bs] + control_bs = self.vae.encode(control_bs)[0] + control_bs = control_bs.mode() + new_control.append(control_bs) + control = torch.cat(new_control, dim = 0) + + if control_image is not None: + control_image = control_image.to(device=device, dtype=dtype) + bs = 1 + new_control_pixel_values = [] + for i in range(0, control_image.shape[0], bs): + control_pixel_values_bs = control_image[i : i + bs] + control_pixel_values_bs = self.vae.encode(control_pixel_values_bs)[0] + control_pixel_values_bs = control_pixel_values_bs.mode() + new_control_pixel_values.append(control_pixel_values_bs) + control_image_latents = torch.cat(new_control_pixel_values, dim = 0) + else: + control_image_latents = None + + return control, control_image_latents + + def decode_latents(self, latents: torch.Tensor) -> torch.Tensor: + frames = self.vae.decode(latents.to(self.vae.dtype)).sample + frames = (frames / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloa16 + frames = frames.cpu().float().numpy() + return frames + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.latte.pipeline_latte.LattePipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def attention_kwargs(self): + return self._attention_kwargs + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 480, + width: int = 720, + video: Union[torch.FloatTensor] = None, + mask_video: Union[torch.FloatTensor] = None, + control_video: Union[torch.FloatTensor] = None, + subject_ref_images: Union[torch.FloatTensor] = None, + num_frames: int = 49, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 6, + num_videos_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: str = "numpy", + return_dict: bool = False, + callback_on_step_end: Optional[ + Union[Callable[[int, int, Dict], None], PipelineCallback, MultiPipelineCallbacks] + ] = None, + attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + comfyui_progressbar: bool = False, + shift: int = 5, + vace_context_scale: float = 1.0 + ) -> Union[WanPipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + Args: + + Examples: + + Returns: + + """ + + if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)): + callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_on_step_end_tensor_inputs, + prompt_embeds, + negative_prompt_embeds, + ) + self._guidance_scale = guidance_scale + self._attention_kwargs = attention_kwargs + self._interrupt = False + + # 2. Default call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + weight_dtype = self.text_encoder.dtype + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + negative_prompt, + do_classifier_free_guidance, + num_videos_per_prompt=num_videos_per_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + device=device, + ) + if do_classifier_free_guidance: + in_prompt_embeds = negative_prompt_embeds + prompt_embeds + else: + in_prompt_embeds = prompt_embeds + + # 4. Prepare timesteps + if isinstance(self.scheduler, FlowMatchEulerDiscreteScheduler): + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps, mu=1) + elif isinstance(self.scheduler, FlowUniPCMultistepScheduler): + self.scheduler.set_timesteps(num_inference_steps, device=device, shift=shift) + timesteps = self.scheduler.timesteps + elif isinstance(self.scheduler, FlowDPMSolverMultistepScheduler): + sampling_sigmas = get_sampling_sigmas(num_inference_steps, shift) + timesteps, _ = retrieve_timesteps( + self.scheduler, + device=device, + sigmas=sampling_sigmas) + else: + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + if comfyui_progressbar: + from comfy.utils import ProgressBar + pbar = ProgressBar(num_inference_steps + 2) + + latent_channels = self.vae.config.latent_channels + + if comfyui_progressbar: + pbar.update(1) + + # Prepare mask latent variables + if mask_video is not None: + bs, _, video_length, height, width = video.size() + mask_condition = self.mask_processor.preprocess(rearrange(mask_video, "b c f h w -> (b f) c h w"), height=height, width=width) + mask_condition = mask_condition.to(dtype=torch.float32) + mask_condition = rearrange(mask_condition, "(b f) c h w -> b c f h w", f=video_length) + mask_condition = torch.tile(mask_condition, [1, 3, 1, 1, 1]).to(dtype=weight_dtype, device=device) + + + if control_video is not None: + video_length = control_video.shape[2] + control_video = self.image_processor.preprocess(rearrange(control_video, "b c f h w -> (b f) c h w"), height=height, width=width) + control_video = control_video.to(dtype=torch.float32) + input_video = rearrange(control_video, "(b f) c h w -> b c f h w", f=video_length) + + input_video = input_video.to(dtype=weight_dtype, device=device) + + elif video is not None: + video_length = video.shape[2] + init_video = self.image_processor.preprocess(rearrange(video, "b c f h w -> (b f) c h w"), height=height, width=width) + init_video = init_video.to(dtype=torch.float32) + init_video = rearrange(init_video, "(b f) c h w -> b c f h w", f=video_length).to(dtype=weight_dtype, device=device) + + input_video = init_video * (mask_condition < 0.5) + input_video = input_video.to(dtype=weight_dtype, device=device) + + if subject_ref_images is not None: + video_length = subject_ref_images.shape[2] + subject_ref_images = self.image_processor.preprocess(rearrange(subject_ref_images, "b c f h w -> (b f) c h w"), height=height, width=width) + subject_ref_images = subject_ref_images.to(dtype=torch.float32) + subject_ref_images = rearrange(subject_ref_images, "(b f) c h w -> b c f h w", f=video_length) + subject_ref_images = subject_ref_images.to(dtype=weight_dtype, device=device) + + bs, c, f, h, w = subject_ref_images.size() + new_subject_ref_images = [] + for i in range(bs): + new_subject_ref_images.append([]) + for j in range(f): + new_subject_ref_images[i].append(subject_ref_images[i, :, j:j+1]) + subject_ref_images = new_subject_ref_images + + vace_latents = self.vace_encode_frames(input_video, subject_ref_images, masks=mask_condition, vae=self.vae) + mask_latents = self.vace_encode_masks(mask_condition, subject_ref_images) + vace_context = self.vace_latent(vace_latents, mask_latents) + + # 5. Prepare latents. + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + latent_channels, + num_frames, + height, + width, + weight_dtype, + device, + generator, + latents, + num_length_latents=vace_latents[0].size(1) + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + target_shape = (self.vae.latent_channels, vace_latents[0].size(1), vace_latents[0].size(2), vace_latents[0].size(3)) + seq_len = math.ceil((target_shape[2] * target_shape[3]) / (self.transformer.config.patch_size[1] * self.transformer.config.patch_size[2]) * target_shape[1]) + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self.transformer.num_inference_steps = num_inference_steps + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + self.transformer.current_steps = i + + if self.interrupt: + continue + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + if hasattr(self.scheduler, "scale_model_input"): + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + vace_context_input = torch.stack(vace_context * 2) if do_classifier_free_guidance else vace_context + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latent_model_input.shape[0]) + + # predict noise model_output + with torch.cuda.amp.autocast(dtype=weight_dtype), torch.cuda.device(device=device): + noise_pred = self.transformer( + x=latent_model_input, + context=in_prompt_embeds, + t=timestep, + vace_context=vace_context_input, + seq_len=seq_len, + vace_context_scale=vace_context_scale + ) + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if comfyui_progressbar: + pbar.update(1) + + if subject_ref_images is not None: + len_subject_ref_images = len(subject_ref_images[0]) + latents = latents[:, :, len_subject_ref_images:, :, :] + + if output_type == "numpy": + video = self.decode_latents(latents) + elif not output_type == "latent": + video = self.decode_latents(latents) + video = self.video_processor.postprocess_video(video=video, output_type=output_type) + else: + video = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + video = torch.from_numpy(video) + + return WanPipelineOutput(videos=video) diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_z_image.py b/VideoX-Fun/videox_fun/pipeline/pipeline_z_image.py new file mode 100644 index 0000000000000000000000000000000000000000..1b59b88e7843b94658d21102a05cf6114f2f7573 --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_z_image.py @@ -0,0 +1,613 @@ +# Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import numpy as np +import PIL +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import FromSingleFileMixin +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import (BaseOutput, is_torch_xla_available, logging, + replace_example_docstring) +from diffusers.utils.torch_utils import randn_tensor +from transformers import AutoTokenizer, PreTrainedModel + +from ..models import AutoencoderKL, ZImageTransformer2DModel + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import ZImagePipeline + + >>> pipe = ZImagePipeline.from_pretrained("Z-a-o/Z-Image-Turbo", torch_dtype=torch.bfloat16) + >>> pipe.to("cuda") + + >>> # Optionally, set the attention backend to flash-attn 2 or 3, default is SDPA in PyTorch. + >>> # (1) Use flash attention 2 + >>> # pipe.transformer.set_attention_backend("flash") + >>> # (2) Use flash attention 3 + >>> # pipe.transformer.set_attention_backend("_flash_3") + + >>> prompt = "一幅为名为“造相「Z-IMAGE-TURBO」”的项目设计的创意海报。画面巧妙地将文字概念视觉化：一辆复古蒸汽小火车化身为巨大的拉链头，正拉开厚厚的冬日积雪，展露出一个生机盎然的春天。" + >>> image = pipe( + diffusers. prompt, + diffusers. height=1024, + diffusers. width=1024, + diffusers. num_inference_steps=9, + diffusers. guidance_scale=0.0, + diffusers. generator=torch.Generator("cuda").manual_seed(42), + diffusers. ).images[0] + >>> image.save("zimage.png") + ``` +""" + + +# Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class ZImagePipelineOutput(BaseOutput): + """ + Output class for Z-Image image generation pipelines. + + Args: + images (`List[PIL.Image.Image]` or `torch.Tensor` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array or torch tensor of shape `(batch_size, + height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion + pipeline. Torch tensors can represent either the denoised images or the intermediate latents ready to be + passed to the decoder. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +class ZImagePipeline(DiffusionPipeline, FromSingleFileMixin): + model_cpu_offload_seq = "text_encoder->transformer->vae" + _optional_components = [] + _callback_tensor_inputs = ["latents", "prompt_embeds"] + + def __init__( + self, + scheduler: FlowMatchEulerDiscreteScheduler, + vae: AutoencoderKL, + text_encoder: PreTrainedModel, + tokenizer: AutoTokenizer, + transformer: ZImageTransformer2DModel, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + transformer=transformer, + ) + self.vae_scale_factor = ( + 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8 + ) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + + def encode_prompt( + self, + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + max_sequence_length: int = 512, + ): + prompt = [prompt] if isinstance(prompt, str) else prompt + prompt_embeds = self._encode_prompt( + prompt=prompt, + device=device, + prompt_embeds=prompt_embeds, + max_sequence_length=max_sequence_length, + ) + + if do_classifier_free_guidance: + if negative_prompt is None: + negative_prompt = ["" for _ in prompt] + else: + negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + assert len(prompt) == len(negative_prompt) + negative_prompt_embeds = self._encode_prompt( + prompt=negative_prompt, + device=device, + prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + ) + else: + negative_prompt_embeds = [] + return prompt_embeds, negative_prompt_embeds + + def _encode_prompt( + self, + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + max_sequence_length: int = 512, + ) -> List[torch.FloatTensor]: + device = device or self._execution_device + + if prompt_embeds is not None: + return prompt_embeds + + if isinstance(prompt, str): + prompt = [prompt] + + for i, prompt_item in enumerate(prompt): + messages = [ + {"role": "user", "content": prompt_item}, + ] + prompt_item = self.tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + prompt[i] = prompt_item + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + + prompt_embeds = self.text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] + + embeddings_list = [] + + for i in range(len(prompt_embeds)): + embeddings_list.append(prompt_embeds[i][prompt_masks[i]]) + + return embeddings_list + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + ): + height = 2 * (int(height) // (self.vae_scale_factor * 2)) + width = 2 * (int(width) // (self.vae_scale_factor * 2)) + + shape = (batch_size, num_channels_latents, height, width) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + return latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def joint_attention_kwargs(self): + return self._joint_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + sigmas: Optional[List[float]] = None, + guidance_scale: float = 5.0, + cfg_normalization: bool = False, + cfg_truncation: float = 1.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + negative_prompt_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + height (`int`, *optional*, defaults to 1024): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 1024): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + cfg_normalization (`bool`, *optional*, defaults to False): + Whether to apply configuration normalization. + cfg_truncation (`float`, *optional*, defaults to 1.0): + The truncation value for configuration. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will be generated by sampling using the supplied random `generator`. + prompt_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.ZImagePipelineOutput`] instead of a plain + tuple. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int`, *optional*, defaults to 512): + Maximum sequence length to use with the `prompt`. + + Examples: + + Returns: + [`~pipelines.z_image.ZImagePipelineOutput`] or `tuple`: [`~pipelines.z_image.ZImagePipelineOutput`] if + `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the + generated images. + """ + height = height or 1024 + width = width or 1024 + + vae_scale = self.vae_scale_factor * 2 + if height % vae_scale != 0: + raise ValueError( + f"Height must be divisible by {vae_scale} (got {height}). " + f"Please adjust the height to a multiple of {vae_scale}." + ) + if width % vae_scale != 0: + raise ValueError( + f"Width must be divisible by {vae_scale} (got {width}). " + f"Please adjust the width to a multiple of {vae_scale}." + ) + + device = self._execution_device + + self._guidance_scale = guidance_scale + self._joint_attention_kwargs = joint_attention_kwargs + self._interrupt = False + self._cfg_normalization = cfg_normalization + self._cfg_truncation = cfg_truncation + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = len(prompt_embeds) + + # If prompt_embeds is provided and prompt is None, skip encoding + if prompt_embeds is not None and prompt is None: + if self.do_classifier_free_guidance and negative_prompt_embeds is None: + raise ValueError( + "When `prompt_embeds` is provided without `prompt`, " + "`negative_prompt_embeds` must also be provided for classifier-free guidance." + ) + else: + ( + prompt_embeds, + negative_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + negative_prompt=negative_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + device=device, + max_sequence_length=max_sequence_length, + ) + + # 4. Prepare latent variables + num_channels_latents = self.transformer.in_channels + + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + torch.float32, + device, + generator, + latents, + ) + + # Repeat prompt_embeds for num_images_per_prompt + if num_images_per_prompt > 1: + prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)] + if self.do_classifier_free_guidance and negative_prompt_embeds: + negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)] + + actual_batch_size = batch_size * num_images_per_prompt + image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2) + + # 5. Prepare timesteps + mu = calculate_shift( + image_seq_len, + self.scheduler.config.get("base_image_seq_len", 256), + self.scheduler.config.get("max_image_seq_len", 4096), + self.scheduler.config.get("base_shift", 0.5), + self.scheduler.config.get("max_shift", 1.15), + ) + self.scheduler.sigma_min = 0.0 + scheduler_kwargs = {"mu": mu} + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + sigmas=sigmas, + **scheduler_kwargs, + ) + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self._num_timesteps = len(timesteps) + + # 6. Denoising loop + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latents.shape[0]) + timestep = (1000 - timestep) / 1000 + # Normalized time for time-aware config (0 at start, 1 at end) + t_norm = timestep[0].item() + + # Handle cfg truncation + current_guidance_scale = self.guidance_scale + if ( + self.do_classifier_free_guidance + and self._cfg_truncation is not None + and float(self._cfg_truncation) <= 1 + ): + if t_norm > self._cfg_truncation: + current_guidance_scale = 0.0 + + # Run CFG only if configured AND scale is non-zero + apply_cfg = self.do_classifier_free_guidance and current_guidance_scale > 0 + + if apply_cfg: + latents_typed = latents.to(self.transformer.dtype) + latent_model_input = latents_typed.repeat(2, 1, 1, 1) + prompt_embeds_model_input = prompt_embeds + negative_prompt_embeds + timestep_model_input = timestep.repeat(2) + else: + latent_model_input = latents.to(self.transformer.dtype) + prompt_embeds_model_input = prompt_embeds + timestep_model_input = timestep + + latent_model_input = latent_model_input.unsqueeze(2) + latent_model_input_list = list(latent_model_input.unbind(dim=0)) + + model_out_list = self.transformer( + latent_model_input_list, + timestep_model_input, + prompt_embeds_model_input, + )[0] + + if apply_cfg: + # Perform CFG + pos_out = model_out_list[:actual_batch_size] + neg_out = model_out_list[actual_batch_size:] + + noise_pred = [] + for j in range(actual_batch_size): + pos = pos_out[j].float() + neg = neg_out[j].float() + + pred = pos + current_guidance_scale * (pos - neg) + + # Renormalization + if self._cfg_normalization and float(self._cfg_normalization) > 0.0: + ori_pos_norm = torch.linalg.vector_norm(pos) + new_pos_norm = torch.linalg.vector_norm(pred) + max_new_norm = ori_pos_norm * float(self._cfg_normalization) + if new_pos_norm > max_new_norm: + pred = pred * (max_new_norm / new_pos_norm) + + noise_pred.append(pred) + + noise_pred = torch.stack(noise_pred, dim=0) + else: + noise_pred = torch.stack([t.float() for t in model_out_list], dim=0) + + noise_pred = noise_pred.squeeze(2) + noise_pred = -noise_pred + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0] + assert latents.dtype == torch.float32 + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if output_type == "latent": + image = latents + + else: + latents = latents.to(self.vae.dtype) + latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor + + image = self.vae.decode(latents, return_dict=False)[0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ZImagePipelineOutput(images=image) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/pipeline/pipeline_z_image_control.py b/VideoX-Fun/videox_fun/pipeline/pipeline_z_image_control.py new file mode 100644 index 0000000000000000000000000000000000000000..7d94e4f425f93487c43b30d589c208da02d9536a --- /dev/null +++ b/VideoX-Fun/videox_fun/pipeline/pipeline_z_image_control.py @@ -0,0 +1,688 @@ +# Copyright 2025 Alibaba Z-Image Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import numpy as np +import PIL +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import FromSingleFileMixin +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.schedulers import FlowMatchEulerDiscreteScheduler +from diffusers.utils import (BaseOutput, is_torch_xla_available, logging, + replace_example_docstring) +from diffusers.utils.torch_utils import randn_tensor +from transformers import AutoTokenizer, PreTrainedModel + +from ..models import AutoencoderKL, ZImageTransformer2DModel + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import ZImagePipeline + + >>> pipe = ZImagePipeline.from_pretrained("Z-a-o/Z-Image-Turbo", torch_dtype=torch.bfloat16) + >>> pipe.to("cuda") + + >>> # Optionally, set the attention backend to flash-attn 2 or 3, default is SDPA in PyTorch. + >>> # (1) Use flash attention 2 + >>> # pipe.transformer.set_attention_backend("flash") + >>> # (2) Use flash attention 3 + >>> # pipe.transformer.set_attention_backend("_flash_3") + + >>> prompt = "一幅为名为“造相「Z-IMAGE-TURBO」”的项目设计的创意海报。画面巧妙地将文字概念视觉化：一辆复古蒸汽小火车化身为巨大的拉链头，正拉开厚厚的冬日积雪，展露出一个生机盎然的春天。" + >>> image = pipe( + diffusers. prompt, + diffusers. height=1024, + diffusers. width=1024, + diffusers. num_inference_steps=9, + diffusers. guidance_scale=0.0, + diffusers. generator=torch.Generator("cuda").manual_seed(42), + diffusers. ).images[0] + >>> image.save("zimage.png") + ``` +""" + +import onnx +import onnxruntime as ort + + +def ort_inference(onnx_model_path, inputs): + inputs_onnx = {k: v.cpu().to(dtype=torch.float32).numpy() for k, v in inputs.items()} + session = ort.InferenceSession(onnx_model_path) + + ort_outputs = session.run(None, inputs_onnx) + return ort_outputs + + +# trans_session = ort.InferenceSession("/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-models/z_image_control_transformer_slim.onnx") + +# Copied from diffusers.pipelines.flux.pipeline_flux.calculate_shift +def calculate_shift( + image_seq_len, + base_seq_len: int = 256, + max_seq_len: int = 4096, + base_shift: float = 0.5, + max_shift: float = 1.15, +): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + sigmas: Optional[List[float]] = None, + **kwargs, +): + r""" + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, `timesteps` + must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to override the timestep spacing strategy of the scheduler. If `timesteps` is passed, + `num_inference_steps` and `sigmas` must be `None`. + sigmas (`List[float]`, *optional*): + Custom sigmas used to override the timestep spacing strategy of the scheduler. If `sigmas` is passed, + `num_inference_steps` and `timesteps` must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None and sigmas is not None: + raise ValueError("Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values") + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class ZImagePipelineOutput(BaseOutput): + """ + Output class for Z-Image image generation pipelines. + + Args: + images (`List[PIL.Image.Image]` or `torch.Tensor` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array or torch tensor of shape `(batch_size, + height, width, num_channels)`. PIL images or numpy array present the denoised images of the diffusion + pipeline. Torch tensors can represent either the denoised images or the intermediate latents ready to be + passed to the decoder. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +class ZImageControlPipeline(DiffusionPipeline, FromSingleFileMixin): + model_cpu_offload_seq = "text_encoder->transformer->vae" + _optional_components = [] + _callback_tensor_inputs = ["latents", "prompt_embeds"] + + def __init__( + self, + scheduler: FlowMatchEulerDiscreteScheduler, + vae: AutoencoderKL, + text_encoder: PreTrainedModel, + tokenizer: AutoTokenizer, + transformer: ZImageTransformer2DModel, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + transformer=transformer, + ) + self.vae_scale_factor = ( + 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8 + ) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor * 2) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + def encode_prompt( + self, + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + max_sequence_length: int = 512, + ): + prompt = [prompt] if isinstance(prompt, str) else prompt + prompt_embeds = self._encode_prompt( + prompt=prompt, + device=device, + prompt_embeds=prompt_embeds, + max_sequence_length=max_sequence_length, + ) + + if do_classifier_free_guidance: + if negative_prompt is None: + negative_prompt = ["" for _ in prompt] + else: + negative_prompt = [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + assert len(prompt) == len(negative_prompt) + negative_prompt_embeds = self._encode_prompt( + prompt=negative_prompt, + device=device, + prompt_embeds=negative_prompt_embeds, + max_sequence_length=max_sequence_length, + ) + else: + negative_prompt_embeds = [] + return prompt_embeds, negative_prompt_embeds + + def _encode_prompt( + self, + prompt: Union[str, List[str]], + device: Optional[torch.device] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + max_sequence_length: int = 512, + ) -> List[torch.FloatTensor]: + device = device or self._execution_device + + if prompt_embeds is not None: + return prompt_embeds + + if isinstance(prompt, str): + prompt = [prompt] + + for i, prompt_item in enumerate(prompt): + messages = [ + {"role": "user", "content": prompt_item}, + ] + prompt_item = self.tokenizer.apply_chat_template( + messages, + tokenize=False, + add_generation_prompt=True, + enable_thinking=True, + ) + prompt[i] = prompt_item + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_sequence_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids.to(device) + prompt_masks = text_inputs.attention_mask.to(device).bool() + + prompt_embeds = self.text_encoder( + input_ids=text_input_ids, + attention_mask=prompt_masks, + output_hidden_states=True, + ).hidden_states[-2] # 固定为 torch.Size([1, 512, 2560] + + embeddings_list = [] + + for i in range(len(prompt_embeds)): # TODO: 对于 ONNX 模型不支持动态 shape + # embeddings_list.append(prompt_embeds[i][prompt_masks[i]]) + embeddings_list.append(prompt_embeds[i]) + + return embeddings_list + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + ): + height = 2 * (int(height) // (self.vae_scale_factor * 2)) + width = 2 * (int(width) // (self.vae_scale_factor * 2)) + + shape = (batch_size, num_channels_latents, height, width) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + return latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def joint_attention_kwargs(self): + return self._joint_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + + control_image: Union[torch.FloatTensor] = None, + control_context_scale: float = 1.0, + + num_inference_steps: int = 50, + sigmas: Optional[List[float]] = None, + guidance_scale: float = 5.0, + cfg_normalization: bool = False, + cfg_truncation: float = 1.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[List[torch.FloatTensor]] = None, + negative_prompt_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + joint_attention_kwargs: Optional[Dict[str, Any]] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + max_sequence_length: int = 512, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + height (`int`, *optional*, defaults to 1024): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 1024): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + sigmas (`List[float]`, *optional*): + Custom sigmas to use for the denoising process with schedulers which support a `sigmas` argument in + their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is passed + will be used. + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + cfg_normalization (`bool`, *optional*, defaults to False): + Whether to apply configuration normalization. + cfg_truncation (`float`, *optional*, defaults to 1.0): + The truncation value for configuration. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will be generated by sampling using the supplied random `generator`. + prompt_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.ZImagePipelineOutput`] instead of a plain + tuple. + joint_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + max_sequence_length (`int`, *optional*, defaults to 512): + Maximum sequence length to use with the `prompt`. + + Examples: + + Returns: + [`~pipelines.z_image.ZImagePipelineOutput`] or `tuple`: [`~pipelines.z_image.ZImagePipelineOutput`] if + `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the + generated images. + """ + height = height or 1024 + width = width or 1024 + + vae_scale = self.vae_scale_factor * 2 + if height % vae_scale != 0: + raise ValueError( + f"Height must be divisible by {vae_scale} (got {height}). " + f"Please adjust the height to a multiple of {vae_scale}." + ) + if width % vae_scale != 0: + raise ValueError( + f"Width must be divisible by {vae_scale} (got {width}). " + f"Please adjust the width to a multiple of {vae_scale}." + ) + + self._guidance_scale = guidance_scale + self._joint_attention_kwargs = joint_attention_kwargs + self._interrupt = False + self._cfg_normalization = cfg_normalization + self._cfg_truncation = cfg_truncation + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = len(prompt_embeds) + + device = self._execution_device + + weight_dtype = self.text_encoder.dtype + num_channels_latents = self.transformer.in_channels + + if control_image is not None: + control_image = self.image_processor.preprocess(control_image, height=height, width=width) + control_image = control_image.to(dtype=weight_dtype, device=device) + control_latents = self.vae.encode(control_image)[0].mode() + # control_latents = ort_inference( + # "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-models/vae_encoder_slim.onnx", + # {"x": control_image}, + # )[0] + # control_latents = torch.from_numpy(control_latents).to(device=device, dtype=weight_dtype) + control_latents = (control_latents - self.vae.config.shift_factor) * self.vae.config.scaling_factor + else: + control_latents = torch.zeros_like(inpaint_latent) + + control_context = control_latents.unsqueeze(2) + + # If prompt_embeds is provided and prompt is None, skip encoding + if prompt_embeds is not None and prompt is None: + if self.do_classifier_free_guidance and negative_prompt_embeds is None: + raise ValueError( + "When `prompt_embeds` is provided without `prompt`, " + "`negative_prompt_embeds` must also be provided for classifier-free guidance." + ) + else: + ( + prompt_embeds, + negative_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + negative_prompt=negative_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + device=device, + max_sequence_length=max_sequence_length, + ) + + # 4. Prepare latent variables + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + torch.float32, + device, + generator, + latents, + ) + + # Repeat prompt_embeds for num_images_per_prompt + if num_images_per_prompt > 1: + prompt_embeds = [pe for pe in prompt_embeds for _ in range(num_images_per_prompt)] + if self.do_classifier_free_guidance and negative_prompt_embeds: + negative_prompt_embeds = [npe for npe in negative_prompt_embeds for _ in range(num_images_per_prompt)] + + actual_batch_size = batch_size * num_images_per_prompt + image_seq_len = (latents.shape[2] // 2) * (latents.shape[3] // 2) + + # 5. Prepare timesteps + mu = calculate_shift( + image_seq_len, + self.scheduler.config.get("base_image_seq_len", 256), + self.scheduler.config.get("max_image_seq_len", 4096), + self.scheduler.config.get("base_shift", 0.5), + self.scheduler.config.get("max_shift", 1.15), + ) + self.scheduler.sigma_min = 0.0 + scheduler_kwargs = {"mu": mu} + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + sigmas=sigmas, + **scheduler_kwargs, + ) + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + self._num_timesteps = len(timesteps) + + # 6. Denoising loop + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = t.expand(latents.shape[0]) + timestep = (1000 - timestep) / 1000 + # Normalized time for time-aware config (0 at start, 1 at end) + t_norm = timestep[0].item() + + # Handle cfg truncation + current_guidance_scale = self.guidance_scale + if ( + self.do_classifier_free_guidance + and self._cfg_truncation is not None + and float(self._cfg_truncation) <= 1 + ): + if t_norm > self._cfg_truncation: + current_guidance_scale = 0.0 + + # Run CFG only if configured AND scale is non-zero + apply_cfg = self.do_classifier_free_guidance and current_guidance_scale > 0 + + if apply_cfg: + latents_typed = latents.to(self.transformer.dtype) + latent_model_input = latents_typed.repeat(2, 1, 1, 1) + prompt_embeds_model_input = prompt_embeds + negative_prompt_embeds + timestep_model_input = timestep.repeat(2) + else: + latent_model_input = latents.to(self.transformer.dtype) + prompt_embeds_model_input = prompt_embeds + timestep_model_input = timestep + + latent_model_input = latent_model_input.unsqueeze(2) + latent_model_input_list = list(latent_model_input.unbind(dim=0)) + + # import pdb; pdb.set_trace() + model_out_list = self.transformer( + latent_model_input_list, + timestep_model_input, + prompt_embeds_model_input, + control_context=control_context, + control_context_scale=control_context_scale, + )[0] + # import pdb; pdb.set_trace() + # model_out_list = trans_session.run( + # None, + # { + # "latent_model_input": latent_model_input_list[0].cpu().to(dtype=torch.float32).numpy(), + # "timestep": timestep_model_input.cpu().to(dtype=torch.float32).numpy(), + # "prompt_embeds": prompt_embeds_model_input[0].cpu().to(dtype=torch.float32).numpy(), + # "control_context": control_context.cpu().to(dtype=torch.float32).numpy(), + # # "control_context_scale": np.array(control_context_scale).astype(np.float32), + # }, + # )[0] + # model_out_list = torch.from_numpy(model_out_list).to(device="cuda:0", dtype=torch.bfloat16) + # import pdb; pdb.set_trace() + """ + (Pdb) latent_model_input_list[0].shape + torch.Size([16, 1, 216, 124]) + (Pdb) timestep_model_input + tensor([0.], device='cuda:0') + (Pdb) prompt_embeds_model_input[0].shape + torch.Size([165, 2560]) + (Pdb) control_context.shape + torch.Size([1, 16, 1, 216, 124]) + (Pdb) control_context_scale + 0.75 + (Pdb) model_out_list[0].shape + torch.Size([16, 1, 216, 124]) + """ + # import pdb; pdb.set_trace() + + if apply_cfg: + # Perform CFG + pos_out = model_out_list[:actual_batch_size] + neg_out = model_out_list[actual_batch_size:] + + noise_pred = [] + for j in range(actual_batch_size): + pos = pos_out[j].float() + neg = neg_out[j].float() + + pred = pos + current_guidance_scale * (pos - neg) + + # Renormalization + if self._cfg_normalization and float(self._cfg_normalization) > 0.0: + ori_pos_norm = torch.linalg.vector_norm(pos) + new_pos_norm = torch.linalg.vector_norm(pred) + max_new_norm = ori_pos_norm * float(self._cfg_normalization) + if new_pos_norm > max_new_norm: + pred = pred * (max_new_norm / new_pos_norm) + + noise_pred.append(pred) + + noise_pred = torch.stack(noise_pred, dim=0) + else: + noise_pred = torch.stack([t.float() for t in model_out_list], dim=0) + + noise_pred = noise_pred.squeeze(2) + noise_pred = -noise_pred + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred.to(torch.float32), t, latents, return_dict=False)[0] + assert latents.dtype == torch.float32 + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if output_type == "latent": + image = latents + + else: + latents = latents.to(self.vae.dtype) + latents = (latents / self.vae.config.scaling_factor) + self.vae.config.shift_factor + + image = self.vae.decode(latents, return_dict=False)[0] + # image = ort_inference( + # "/data/tmp/yongqiang/nfs/Z-Image-Turbo.axera/python/VideoX-Fun/onnx-models/vae_decoder_slim.onnx", + # {"latent": latents}, + # )[0] + # image = torch.from_numpy(image).to(device=device, dtype=self.vae.dtype) + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + # import pdb; pdb.set_trace() + if not return_dict: + return (image,) + + return ZImagePipelineOutput(images=image) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/reward/MPS/README.md b/VideoX-Fun/videox_fun/reward/MPS/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d66d2ee73284e44bca36c5ba86dbc5911d64defa --- /dev/null +++ b/VideoX-Fun/videox_fun/reward/MPS/README.md @@ -0,0 +1 @@ +This folder is modified from the official [MPS](https://github.com/Kwai-Kolors/MPS/tree/main) repository. \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/reward/MPS/trainer/models/base_model.py b/VideoX-Fun/videox_fun/reward/MPS/trainer/models/base_model.py new file mode 100644 index 0000000000000000000000000000000000000000..df7907f388d3b2b4658e595db4189a80ab5acd77 --- /dev/null +++ b/VideoX-Fun/videox_fun/reward/MPS/trainer/models/base_model.py @@ -0,0 +1,7 @@ +from dataclasses import dataclass + + + +@dataclass +class BaseModelConfig: + pass \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/reward/MPS/trainer/models/clip_model.py b/VideoX-Fun/videox_fun/reward/MPS/trainer/models/clip_model.py new file mode 100644 index 0000000000000000000000000000000000000000..003bb5d583189980c056376025579e6c06cd16f9 --- /dev/null +++ b/VideoX-Fun/videox_fun/reward/MPS/trainer/models/clip_model.py @@ -0,0 +1,154 @@ +from dataclasses import dataclass +from transformers import CLIPModel as HFCLIPModel +from transformers import AutoTokenizer + +from torch import nn, einsum + +# Modified: import +# from trainer.models.base_model import BaseModelConfig +from .base_model import BaseModelConfig + +from transformers import CLIPConfig +from typing import Any, Optional, Tuple, Union +import torch + +# Modified: import +# from trainer.models.cross_modeling import Cross_model +from .cross_modeling import Cross_model + +import gc + +class XCLIPModel(HFCLIPModel): + def __init__(self, config: CLIPConfig): + super().__init__(config) + + def get_text_features( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> torch.FloatTensor: + + # Use CLIP model's config for some fields (if specified) instead of those of vision & text components. + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + text_outputs = self.text_model( + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + # pooled_output = text_outputs[1] + # text_features = self.text_projection(pooled_output) + last_hidden_state = text_outputs[0] + text_features = self.text_projection(last_hidden_state) + + pooled_output = text_outputs[1] + text_features_EOS = self.text_projection(pooled_output) + + + # del last_hidden_state, text_outputs + # gc.collect() + + return text_features, text_features_EOS + + def get_image_features( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> torch.FloatTensor: + + # Use CLIP model's config for some fields (if specified) instead of those of vision & text components. + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + vision_outputs = self.vision_model( + pixel_values=pixel_values, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + # pooled_output = vision_outputs[1] # pooled_output + # image_features = self.visual_projection(pooled_output) + last_hidden_state = vision_outputs[0] + image_features = self.visual_projection(last_hidden_state) + + return image_features + + + +@dataclass +class ClipModelConfig(BaseModelConfig): + _target_: str = "trainer.models.clip_model.CLIPModel" + pretrained_model_name_or_path: str ="openai/clip-vit-base-patch32" + + +class CLIPModel(nn.Module): + def __init__(self, config): + super().__init__() + # Modified: We convert the original ckpt (contains the entire model) to a `state_dict`. + # self.model = XCLIPModel.from_pretrained(ckpt) + self.model = XCLIPModel(config) + self.cross_model = Cross_model(dim=1024, layer_num=4, heads=16) + + def get_text_features(self, *args, **kwargs): + return self.model.get_text_features(*args, **kwargs) + + def get_image_features(self, *args, **kwargs): + return self.model.get_image_features(*args, **kwargs) + + def forward(self, text_inputs=None, image_inputs=None, condition_inputs=None): + outputs = () + + text_f, text_EOS = self.model.get_text_features(text_inputs) # B*77*1024 + outputs += text_EOS, + + image_f = self.model.get_image_features(image_inputs.half()) # 2B*257*1024 + # [B, 77, 1024] + condition_f, _ = self.model.get_text_features(condition_inputs) # B*5*1024 + + sim_text_condition = einsum('b i d, b j d -> b j i', text_f, condition_f) + sim_text_condition = torch.max(sim_text_condition, dim=1, keepdim=True)[0] + sim_text_condition = sim_text_condition / sim_text_condition.max() + mask = torch.where(sim_text_condition > 0.01, 0, float('-inf')) # B*1*77 + + # Modified: Support both torch.float16 and torch.bfloat16 + # mask = mask.repeat(1,image_f.shape[1],1) # B*257*77 + model_dtype = next(self.cross_model.parameters()).dtype + mask = mask.repeat(1,image_f.shape[1],1).to(model_dtype) # B*257*77 + # bc = int(image_f.shape[0]/2) + + # Modified: The original input consists of a (batch of) text and two (batches of) images, + # primarily used to compute which (batch of) image is more consistent with the text. + # The modified input consists of a (batch of) text and a (batch of) images. + # sim0 = self.cross_model(image_f[:bc,:,:], text_f,mask.half()) + # sim1 = self.cross_model(image_f[bc:,:,:], text_f,mask.half()) + # outputs += sim0[:,0,:], + # outputs += sim1[:,0,:], + sim = self.cross_model(image_f, text_f,mask) + outputs += sim[:,0,:], + + return outputs + + @property + def logit_scale(self): + return self.model.logit_scale + + def save(self, path): + self.model.save_pretrained(path) diff --git a/VideoX-Fun/videox_fun/reward/MPS/trainer/models/cross_modeling.py b/VideoX-Fun/videox_fun/reward/MPS/trainer/models/cross_modeling.py new file mode 100644 index 0000000000000000000000000000000000000000..31dcee5bb3b6ecebe993f9b8c75b23f32c3ad03e --- /dev/null +++ b/VideoX-Fun/videox_fun/reward/MPS/trainer/models/cross_modeling.py @@ -0,0 +1,291 @@ +import torch +from torch import einsum, nn +import torch.nn.functional as F +from einops import rearrange, repeat + +# helper functions + +def exists(val): + return val is not None + +def default(val, d): + return val if exists(val) else d + +# normalization +# they use layernorm without bias, something that pytorch does not offer + + +class LayerNorm(nn.Module): + def __init__(self, dim): + super().__init__() + self.weight = nn.Parameter(torch.ones(dim)) + self.register_buffer("bias", torch.zeros(dim)) + + def forward(self, x): + return F.layer_norm(x, x.shape[-1:], self.weight, self.bias) + +# residual + + +class Residual(nn.Module): + def __init__(self, fn): + super().__init__() + self.fn = fn + + def forward(self, x, *args, **kwargs): + return self.fn(x, *args, **kwargs) + x + + +# rotary positional embedding +# https://arxiv.org/abs/2104.09864 + + +class RotaryEmbedding(nn.Module): + def __init__(self, dim): + super().__init__() + inv_freq = 1.0 / (10000 ** (torch.arange(0, dim, 2).float() / dim)) + self.register_buffer("inv_freq", inv_freq) + + def forward(self, max_seq_len, *, device): + seq = torch.arange(max_seq_len, device=device, dtype=self.inv_freq.dtype) + freqs = einsum("i , j -> i j", seq, self.inv_freq) + return torch.cat((freqs, freqs), dim=-1) + + +def rotate_half(x): + x = rearrange(x, "... (j d) -> ... j d", j=2) + x1, x2 = x.unbind(dim=-2) + return torch.cat((-x2, x1), dim=-1) + + +def apply_rotary_pos_emb(pos, t): + return (t * pos.cos()) + (rotate_half(t) * pos.sin()) + + +# classic Noam Shazeer paper, except here they use SwiGLU instead of the more popular GEGLU for gating the feedforward +# https://arxiv.org/abs/2002.05202 + + +class SwiGLU(nn.Module): + def forward(self, x): + x, gate = x.chunk(2, dim=-1) + return F.silu(gate) * x + + +# parallel attention and feedforward with residual +# discovered by Wang et al + EleutherAI from GPT-J fame + +class ParallelTransformerBlock(nn.Module): + def __init__(self, dim, dim_head=64, heads=8, ff_mult=4): + super().__init__() + self.norm = LayerNorm(dim) + + attn_inner_dim = dim_head * heads + ff_inner_dim = dim * ff_mult + self.fused_dims = (attn_inner_dim, dim_head, dim_head, (ff_inner_dim * 2)) + + self.heads = heads + self.scale = dim_head**-0.5 + self.rotary_emb = RotaryEmbedding(dim_head) + + self.fused_attn_ff_proj = nn.Linear(dim, sum(self.fused_dims), bias=False) + self.attn_out = nn.Linear(attn_inner_dim, dim, bias=False) + + self.ff_out = nn.Sequential( + SwiGLU(), + nn.Linear(ff_inner_dim, dim, bias=False) + ) + + self.register_buffer("pos_emb", None, persistent=False) + + + def get_rotary_embedding(self, n, device): + if self.pos_emb is not None and self.pos_emb.shape[-2] >= n: + return self.pos_emb[:n] + + pos_emb = self.rotary_emb(n, device=device) + self.register_buffer("pos_emb", pos_emb, persistent=False) + return pos_emb + + def forward(self, x, attn_mask=None): + """ + einstein notation + b - batch + h - heads + n, i, j - sequence length (base sequence length, source, target) + d - feature dimension + """ + + n, device, h = x.shape[1], x.device, self.heads + + # pre layernorm + + x = self.norm(x) + + # attention queries, keys, values, and feedforward inner + + q, k, v, ff = self.fused_attn_ff_proj(x).split(self.fused_dims, dim=-1) + + # split heads + # they use multi-query single-key-value attention, yet another Noam Shazeer paper + # they found no performance loss past a certain scale, and more efficient decoding obviously + # https://arxiv.org/abs/1911.02150 + + q = rearrange(q, "b n (h d) -> b h n d", h=h) + + # rotary embeddings + + positions = self.get_rotary_embedding(n, device) + q, k = map(lambda t: apply_rotary_pos_emb(positions, t), (q, k)) + + # scale + + q = q * self.scale + + # similarity + + sim = einsum("b h i d, b j d -> b h i j", q, k) + + + # extra attention mask - for masking out attention from text CLS token to padding + + if exists(attn_mask): + attn_mask = rearrange(attn_mask, 'b i j -> b 1 i j') + sim = sim.masked_fill(~attn_mask, -torch.finfo(sim.dtype).max) + + # attention + + sim = sim - sim.amax(dim=-1, keepdim=True).detach() + attn = sim.softmax(dim=-1) + + # aggregate values + + out = einsum("b h i j, b j d -> b h i d", attn, v) + + # merge heads + + out = rearrange(out, "b h n d -> b n (h d)") + return self.attn_out(out) + self.ff_out(ff) + +# cross attention - using multi-query + one-headed key / values as in PaLM w/ optional parallel feedforward + +class CrossAttention(nn.Module): + def __init__( + self, + dim, + *, + context_dim=None, + dim_head=64, + heads=12, + parallel_ff=False, + ff_mult=4, + norm_context=False + ): + super().__init__() + self.heads = heads + self.scale = dim_head ** -0.5 + inner_dim = heads * dim_head + context_dim = default(context_dim, dim) + + self.norm = LayerNorm(dim) + self.context_norm = LayerNorm(context_dim) if norm_context else nn.Identity() + + self.to_q = nn.Linear(dim, inner_dim, bias=False) + self.to_kv = nn.Linear(context_dim, dim_head * 2, bias=False) + self.to_out = nn.Linear(inner_dim, dim, bias=False) + + # whether to have parallel feedforward + + ff_inner_dim = ff_mult * dim + + self.ff = nn.Sequential( + nn.Linear(dim, ff_inner_dim * 2, bias=False), + SwiGLU(), + nn.Linear(ff_inner_dim, dim, bias=False) + ) if parallel_ff else None + + def forward(self, x, context, mask): + """ + einstein notation + b - batch + h - heads + n, i, j - sequence length (base sequence length, source, target) + d - feature dimension + """ + + # pre-layernorm, for queries and context + + x = self.norm(x) + context = self.context_norm(context) + + # get queries + + q = self.to_q(x) + q = rearrange(q, 'b n (h d) -> b h n d', h = self.heads) + + # scale + + q = q * self.scale + + # get key / values + + k, v = self.to_kv(context).chunk(2, dim=-1) + + # query / key similarity + + sim = einsum('b h i d, b j d -> b h i j', q, k) + + # attention + mask = mask.unsqueeze(1).repeat(1,self.heads,1,1) + sim = sim + mask # context mask + sim = sim - sim.amax(dim=-1, keepdim=True) + attn = sim.softmax(dim=-1) + + # aggregate + + out = einsum('b h i j, b j d -> b h i d', attn, v) + + # merge and combine heads + + out = rearrange(out, 'b h n d -> b n (h d)') + out = self.to_out(out) + + # add parallel feedforward (for multimodal layers) + + if exists(self.ff): + out = out + self.ff(x) + + return out + + +class Cross_model(nn.Module): + def __init__( + self, + dim=512, + layer_num=4, + dim_head=64, + heads=8, + ff_mult=4 + ): + super().__init__() + + self.layers = nn.ModuleList([]) + + + for ind in range(layer_num): + self.layers.append(nn.ModuleList([ + Residual(CrossAttention(dim=dim, dim_head=dim_head, heads=heads, parallel_ff=True, ff_mult=ff_mult)), + Residual(ParallelTransformerBlock(dim=dim, dim_head=dim_head, heads=heads, ff_mult=ff_mult)) + ])) + + def forward( + self, + query_tokens, + context_tokens, + mask + ): + for cross_attn, self_attn_ff in self.layers: + query_tokens = cross_attn(query_tokens, context_tokens,mask) + query_tokens = self_attn_ff(query_tokens) + + return query_tokens \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/reward/aesthetic_predictor_v2_5/siglip_v2_5.py b/VideoX-Fun/videox_fun/reward/aesthetic_predictor_v2_5/siglip_v2_5.py new file mode 100644 index 0000000000000000000000000000000000000000..867f4295eb20ff2d1acff02c9303e9fd0a7c00de --- /dev/null +++ b/VideoX-Fun/videox_fun/reward/aesthetic_predictor_v2_5/siglip_v2_5.py @@ -0,0 +1,133 @@ +# Borrowed from https://github.com/discus0434/aesthetic-predictor-v2-5/blob/3125a9e/src/aesthetic_predictor_v2_5/siglip_v2_5.py +import os +from collections import OrderedDict +from os import PathLike +from typing import Final + +import torch +import torch.nn as nn +import torchvision.transforms as transforms +from transformers import ( + SiglipImageProcessor, + SiglipVisionConfig, + SiglipVisionModel, + logging, +) +from transformers.image_processing_utils import BatchFeature +from transformers.modeling_outputs import ImageClassifierOutputWithNoAttention + +logging.set_verbosity_error() + +URL: Final[str] = ( + "https://github.com/discus0434/aesthetic-predictor-v2-5/raw/main/models/aesthetic_predictor_v2_5.pth" +) + + +class AestheticPredictorV2_5Head(nn.Module): + def __init__(self, config: SiglipVisionConfig) -> None: + super().__init__() + self.scoring_head = nn.Sequential( + nn.Linear(config.hidden_size, 1024), + nn.Dropout(0.5), + nn.Linear(1024, 128), + nn.Dropout(0.5), + nn.Linear(128, 64), + nn.Dropout(0.5), + nn.Linear(64, 16), + nn.Dropout(0.2), + nn.Linear(16, 1), + ) + + def forward(self, image_embeds: torch.Tensor) -> torch.Tensor: + return self.scoring_head(image_embeds) + + +class AestheticPredictorV2_5Model(SiglipVisionModel): + PATCH_SIZE = 14 + + def __init__(self, config: SiglipVisionConfig, *args, **kwargs) -> None: + super().__init__(config, *args, **kwargs) + self.layers = AestheticPredictorV2_5Head(config) + self.post_init() + self.transforms = transforms.Compose([ + transforms.Resize((384, 384)), + transforms.ToTensor(), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + ]) + + def forward( + self, + pixel_values: torch.FloatTensor | None = None, + labels: torch.Tensor | None = None, + return_dict: bool | None = None, + ) -> tuple | ImageClassifierOutputWithNoAttention: + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + outputs = super().forward( + pixel_values=pixel_values, + return_dict=return_dict, + ) + image_embeds = outputs.pooler_output + image_embeds_norm = image_embeds / image_embeds.norm(dim=-1, keepdim=True) + prediction = self.layers(image_embeds_norm) + + loss = None + if labels is not None: + loss_fct = nn.MSELoss() + loss = loss_fct() + + if not return_dict: + return (loss, prediction, image_embeds) + + return ImageClassifierOutputWithNoAttention( + loss=loss, + logits=prediction, + hidden_states=image_embeds, + ) + + +class AestheticPredictorV2_5Processor(SiglipImageProcessor): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def __call__(self, *args, **kwargs) -> BatchFeature: + return super().__call__(*args, **kwargs) + + @classmethod + def from_pretrained( + self, + pretrained_model_name_or_path: str + | PathLike = "google/siglip-so400m-patch14-384", + *args, + **kwargs, + ) -> "AestheticPredictorV2_5Processor": + return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs) + + +def convert_v2_5_from_siglip( + predictor_name_or_path: str | PathLike | None = None, + encoder_model_name: str = "google/siglip-so400m-patch14-384", + *args, + **kwargs, +) -> tuple[AestheticPredictorV2_5Model, AestheticPredictorV2_5Processor]: + model = AestheticPredictorV2_5Model.from_pretrained( + encoder_model_name, *args, **kwargs + ) + + processor = AestheticPredictorV2_5Processor.from_pretrained( + encoder_model_name, *args, **kwargs + ) + + if predictor_name_or_path is None or not os.path.exists(predictor_name_or_path): + state_dict = torch.hub.load_state_dict_from_url(URL, map_location="cpu") + else: + state_dict = torch.load(predictor_name_or_path, map_location="cpu") + + assert isinstance(state_dict, OrderedDict) + + model.layers.load_state_dict(state_dict) + model.eval() + + return model, processor \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/reward/improved_aesthetic_predictor.py b/VideoX-Fun/videox_fun/reward/improved_aesthetic_predictor.py new file mode 100644 index 0000000000000000000000000000000000000000..43037b9fd5d80dc1402707c74f6b6a9584f26ce7 --- /dev/null +++ b/VideoX-Fun/videox_fun/reward/improved_aesthetic_predictor.py @@ -0,0 +1,49 @@ +import os + +import torch +import torch.nn as nn +from transformers import CLIPModel +from torchvision.datasets.utils import download_url + +URL = "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/Third_Party/sac%2Blogos%2Bava1-l14-linearMSE.pth" +FILENAME = "sac+logos+ava1-l14-linearMSE.pth" +MD5 = "b1047fd767a00134b8fd6529bf19521a" + + +class MLP(nn.Module): + def __init__(self): + super().__init__() + self.layers = nn.Sequential( + nn.Linear(768, 1024), + nn.Dropout(0.2), + nn.Linear(1024, 128), + nn.Dropout(0.2), + nn.Linear(128, 64), + nn.Dropout(0.1), + nn.Linear(64, 16), + nn.Linear(16, 1), + ) + + + def forward(self, embed): + return self.layers(embed) + + +class ImprovedAestheticPredictor(nn.Module): + def __init__(self, encoder_path="openai/clip-vit-large-patch14", predictor_path=None): + super().__init__() + self.encoder = CLIPModel.from_pretrained(encoder_path) + self.predictor = MLP() + if predictor_path is None or not os.path.exists(predictor_path): + download_url(URL, torch.hub.get_dir(), FILENAME, md5=MD5) + predictor_path = os.path.join(torch.hub.get_dir(), FILENAME) + state_dict = torch.load(predictor_path, map_location="cpu") + self.predictor.load_state_dict(state_dict) + self.eval() + + + def forward(self, pixel_values): + embed = self.encoder.get_image_features(pixel_values=pixel_values) + embed = embed / torch.linalg.vector_norm(embed, dim=-1, keepdim=True) + + return self.predictor(embed).squeeze(1) diff --git a/VideoX-Fun/videox_fun/reward/reward_fn.py b/VideoX-Fun/videox_fun/reward/reward_fn.py new file mode 100644 index 0000000000000000000000000000000000000000..6526919cbea00cb859a542d7758326c89d49ecf9 --- /dev/null +++ b/VideoX-Fun/videox_fun/reward/reward_fn.py @@ -0,0 +1,385 @@ +import os +from abc import ABC, abstractmethod + +import torch +import torchvision.transforms as transforms +from einops import rearrange +from torchvision.datasets.utils import download_url +from typing import Optional, Tuple + + +# All reward models. +__all__ = ["AestheticReward", "HPSReward", "PickScoreReward", "MPSReward"] + + +class BaseReward(ABC): + """An base class for reward models. A custom Reward class must implement two functions below. + """ + def __init__(self): + """Define your reward model and image transformations (optional) here. + """ + pass + + @abstractmethod + def __call__(self, batch_frames: torch.Tensor, batch_prompt: Optional[list[str]]=None) -> Tuple[torch.Tensor, torch.Tensor]: + """Given batch frames with shape `[B, C, T, H, W]` extracted from a list of videos and a list of prompts + (optional) correspondingly, return the loss and reward computed by your reward model (reduction by mean). + """ + pass + +class AestheticReward(BaseReward): + """Aesthetic Predictor [V2](https://github.com/christophschuhmann/improved-aesthetic-predictor) + and [V2.5](https://github.com/discus0434/aesthetic-predictor-v2-5) reward model. + """ + def __init__( + self, + encoder_path="openai/clip-vit-large-patch14", + predictor_path=None, + version="v2", + device="cpu", + dtype=torch.float16, + max_reward=10, + loss_scale=0.1, + ): + from .improved_aesthetic_predictor import ImprovedAestheticPredictor + from ..video_caption.utils.siglip_v2_5 import convert_v2_5_from_siglip + + self.encoder_path = encoder_path + self.predictor_path = predictor_path + self.version = version + self.device = device + self.dtype = dtype + self.max_reward = max_reward + self.loss_scale = loss_scale + + if self.version != "v2" and self.version != "v2.5": + raise ValueError("Only v2 and v2.5 are supported.") + if self.version == "v2": + assert "clip-vit-large-patch14" in encoder_path.lower() + self.model = ImprovedAestheticPredictor(encoder_path=self.encoder_path, predictor_path=self.predictor_path) + # https://huggingface.co/openai/clip-vit-large-patch14/blob/main/preprocessor_config.json + # TODO: [transforms.Resize(224), transforms.CenterCrop(224)] for any aspect ratio. + self.transform = transforms.Compose([ + transforms.Resize((224, 224), interpolation=transforms.InterpolationMode.BICUBIC), + transforms.Normalize(mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711]), + ]) + elif self.version == "v2.5": + assert "siglip-so400m-patch14-384" in encoder_path.lower() + self.model, _ = convert_v2_5_from_siglip(encoder_model_name=self.encoder_path) + # https://huggingface.co/google/siglip-so400m-patch14-384/blob/main/preprocessor_config.json + self.transform = transforms.Compose([ + transforms.Resize((384, 384), interpolation=transforms.InterpolationMode.BICUBIC), + transforms.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]), + ]) + + self.model.to(device=self.device, dtype=self.dtype) + self.model.requires_grad_(False) + + + def __call__(self, batch_frames: torch.Tensor, batch_prompt: Optional[list[str]]=None) -> Tuple[torch.Tensor, torch.Tensor]: + batch_frames = rearrange(batch_frames, "b c t h w -> t b c h w") + batch_loss, batch_reward = 0, 0 + for frames in batch_frames: + pixel_values = torch.stack([self.transform(frame) for frame in frames]) + pixel_values = pixel_values.to(self.device, dtype=self.dtype) + if self.version == "v2": + reward = self.model(pixel_values) + elif self.version == "v2.5": + reward = self.model(pixel_values).logits.squeeze() + # Convert reward to loss in [0, 1]. + if self.max_reward is None: + loss = (-1 * reward) * self.loss_scale + else: + loss = abs(reward - self.max_reward) * self.loss_scale + batch_loss, batch_reward = batch_loss + loss.mean(), batch_reward + reward.mean() + + return batch_loss / batch_frames.shape[0], batch_reward / batch_frames.shape[0] + + +class HPSReward(BaseReward): + """[HPS](https://github.com/tgxs002/HPSv2) v2 and v2.1 reward model. + """ + def __init__( + self, + model_path=None, + version="v2.0", + device="cpu", + dtype=torch.float16, + max_reward=1, + loss_scale=1, + ): + from hpsv2.src.open_clip import create_model_and_transforms, get_tokenizer + + self.model_path = model_path + self.version = version + self.device = device + self.dtype = dtype + self.max_reward = max_reward + self.loss_scale = loss_scale + + self.model, _, _ = create_model_and_transforms( + "ViT-H-14", + "laion2B-s32B-b79K", + precision=self.dtype, + device=self.device, + jit=False, + force_quick_gelu=False, + force_custom_text=False, + force_patch_dropout=False, + force_image_size=None, + pretrained_image=False, + image_mean=None, + image_std=None, + light_augmentation=True, + aug_cfg={}, + output_dict=True, + with_score_predictor=False, + with_region_predictor=False, + ) + self.tokenizer = get_tokenizer("ViT-H-14") + + # https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/blob/main/preprocessor_config.json + # TODO: [transforms.Resize(224), transforms.CenterCrop(224)] for any aspect ratio. + self.transform = transforms.Compose([ + transforms.Resize((224, 224), interpolation=transforms.InterpolationMode.BICUBIC), + transforms.Normalize(mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711]), + ]) + + if version == "v2.0": + url = "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/Third_Party/HPS_v2_compressed.pt" + filename = "HPS_v2_compressed.pt" + md5 = "fd9180de357abf01fdb4eaad64631db4" + elif version == "v2.1": + url = "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/Third_Party/HPS_v2.1_compressed.pt" + filename = "HPS_v2.1_compressed.pt" + md5 = "4067542e34ba2553a738c5ac6c1d75c0" + else: + raise ValueError("Only v2.0 and v2.1 are supported.") + if self.model_path is None or not os.path.exists(self.model_path): + download_url(url, torch.hub.get_dir(), md5=md5) + model_path = os.path.join(torch.hub.get_dir(), filename) + + state_dict = torch.load(model_path, map_location="cpu")["state_dict"] + self.model.load_state_dict(state_dict) + self.model.to(device=self.device, dtype=self.dtype) + self.model.requires_grad_(False) + self.model.eval() + + def __call__(self, batch_frames: torch.Tensor, batch_prompt: list[str]) -> Tuple[torch.Tensor, torch.Tensor]: + assert batch_frames.shape[0] == len(batch_prompt) + # Compute batch reward and loss in frame-wise. + batch_frames = rearrange(batch_frames, "b c t h w -> t b c h w") + batch_loss, batch_reward = 0, 0 + for frames in batch_frames: + image_inputs = torch.stack([self.transform(frame) for frame in frames]) + image_inputs = image_inputs.to(device=self.device, dtype=self.dtype) + text_inputs = self.tokenizer(batch_prompt).to(device=self.device) + outputs = self.model(image_inputs, text_inputs) + + image_features, text_features = outputs["image_features"], outputs["text_features"] + logits = image_features @ text_features.T + reward = torch.diagonal(logits) + # Convert reward to loss in [0, 1]. + if self.max_reward is None: + loss = (-1 * reward) * self.loss_scale + else: + loss = abs(reward - self.max_reward) * self.loss_scale + + batch_loss, batch_reward = batch_loss + loss.mean(), batch_reward + reward.mean() + + return batch_loss / batch_frames.shape[0], batch_reward / batch_frames.shape[0] + + +class PickScoreReward(BaseReward): + """[PickScore](https://github.com/yuvalkirstain/PickScore) reward model. + """ + def __init__( + self, + model_path="yuvalkirstain/PickScore_v1", + device="cpu", + dtype=torch.float16, + max_reward=1, + loss_scale=1, + ): + from transformers import AutoProcessor, AutoModel + + self.model_path = model_path + self.device = device + self.dtype = dtype + self.max_reward = max_reward + self.loss_scale = loss_scale + + # https://huggingface.co/yuvalkirstain/PickScore_v1/blob/main/preprocessor_config.json + self.transform = transforms.Compose([ + transforms.Resize(224, interpolation=transforms.InterpolationMode.BICUBIC), + transforms.CenterCrop(224), + transforms.Normalize(mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711]), + ]) + self.processor = AutoProcessor.from_pretrained("laion/CLIP-ViT-H-14-laion2B-s32B-b79K", torch_dtype=self.dtype) + self.model = AutoModel.from_pretrained(model_path, torch_dtype=self.dtype).eval().to(device) + self.model.requires_grad_(False) + self.model.eval() + + def __call__(self, batch_frames: torch.Tensor, batch_prompt: list[str]) -> Tuple[torch.Tensor, torch.Tensor]: + assert batch_frames.shape[0] == len(batch_prompt) + # Compute batch reward and loss in frame-wise. + batch_frames = rearrange(batch_frames, "b c t h w -> t b c h w") + batch_loss, batch_reward = 0, 0 + for frames in batch_frames: + image_inputs = torch.stack([self.transform(frame) for frame in frames]) + image_inputs = image_inputs.to(device=self.device, dtype=self.dtype) + text_inputs = self.processor( + text=batch_prompt, + padding=True, + truncation=True, + max_length=77, + return_tensors="pt", + ).to(self.device) + image_features = self.model.get_image_features(pixel_values=image_inputs) + text_features = self.model.get_text_features(**text_inputs) + image_features = image_features / torch.norm(image_features, dim=-1, keepdim=True) + text_features = text_features / torch.norm(text_features, dim=-1, keepdim=True) + + logits = image_features @ text_features.T + reward = torch.diagonal(logits) + # Convert reward to loss in [0, 1]. + if self.max_reward is None: + loss = (-1 * reward) * self.loss_scale + else: + loss = abs(reward - self.max_reward) * self.loss_scale + + batch_loss, batch_reward = batch_loss + loss.mean(), batch_reward + reward.mean() + + return batch_loss / batch_frames.shape[0], batch_reward / batch_frames.shape[0] + + +class MPSReward(BaseReward): + """[MPS](https://github.com/Kwai-Kolors/MPS) reward model. + """ + def __init__( + self, + model_path=None, + device="cpu", + dtype=torch.float16, + max_reward=1, + loss_scale=1, + ): + from transformers import AutoTokenizer, AutoConfig + from .MPS.trainer.models.clip_model import CLIPModel + + self.model_path = model_path + self.device = device + self.dtype = dtype + self.condition = "light, color, clarity, tone, style, ambiance, artistry, shape, face, hair, hands, limbs, structure, instance, texture, quantity, attributes, position, number, location, word, things." + self.max_reward = max_reward + self.loss_scale = loss_scale + + processor_name_or_path = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K" + # https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K/blob/main/preprocessor_config.json + # TODO: [transforms.Resize(224), transforms.CenterCrop(224)] for any aspect ratio. + self.transform = transforms.Compose([ + transforms.Resize((224, 224), interpolation=transforms.InterpolationMode.BICUBIC), + transforms.Normalize(mean=[0.48145466, 0.4578275, 0.40821073], std=[0.26862954, 0.26130258, 0.27577711]), + ]) + + # We convert the original [ckpt](http://drive.google.com/file/d/17qrK_aJkVNM75ZEvMEePpLj6L867MLkN/view?usp=sharing) + # (contains the entire model) to a `state_dict`. + url = "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/Third_Party/MPS_overall.pth" + filename = "MPS_overall.pth" + md5 = "1491cbbbd20565747fe07e7572e2ac56" + if self.model_path is None or not os.path.exists(self.model_path): + download_url(url, torch.hub.get_dir(), md5=md5) + model_path = os.path.join(torch.hub.get_dir(), filename) + + self.tokenizer = AutoTokenizer.from_pretrained(processor_name_or_path, trust_remote_code=True) + config = AutoConfig.from_pretrained(processor_name_or_path) + self.model = CLIPModel(config) + state_dict = torch.load(model_path, map_location="cpu") + self.model.load_state_dict(state_dict, strict=False) + self.model.to(device=self.device, dtype=self.dtype) + self.model.requires_grad_(False) + self.model.eval() + + def _tokenize(self, caption): + input_ids = self.tokenizer( + caption, + max_length=self.tokenizer.model_max_length, + padding="max_length", + truncation=True, + return_tensors="pt" + ).input_ids + + return input_ids + + def __call__( + self, + batch_frames: torch.Tensor, + batch_prompt: list[str], + batch_condition: Optional[list[str]] = None + ) -> Tuple[torch.Tensor, torch.Tensor]: + if batch_condition is None: + batch_condition = [self.condition] * len(batch_prompt) + batch_frames = rearrange(batch_frames, "b c t h w -> t b c h w") + batch_loss, batch_reward = 0, 0 + for frames in batch_frames: + image_inputs = torch.stack([self.transform(frame) for frame in frames]) + image_inputs = image_inputs.to(device=self.device, dtype=self.dtype) + text_inputs = self._tokenize(batch_prompt).to(self.device) + condition_inputs = self._tokenize(batch_condition).to(device=self.device) + text_features, image_features = self.model(text_inputs, image_inputs, condition_inputs) + + text_features = text_features / text_features.norm(dim=-1, keepdim=True) + image_features = image_features / image_features.norm(dim=-1, keepdim=True) + # reward = self.model.logit_scale.exp() * torch.diag(torch.einsum('bd,cd->bc', text_features, image_features)) + logits = image_features @ text_features.T + reward = torch.diagonal(logits) + # Convert reward to loss in [0, 1]. + if self.max_reward is None: + loss = (-1 * reward) * self.loss_scale + else: + loss = abs(reward - self.max_reward) * self.loss_scale + + batch_loss, batch_reward = batch_loss + loss.mean(), batch_reward + reward.mean() + + return batch_loss / batch_frames.shape[0], batch_reward / batch_frames.shape[0] + + +if __name__ == "__main__": + import numpy as np + from decord import VideoReader + + video_path_list = ["your_video_path_1.mp4", "your_video_path_2.mp4"] + prompt_list = ["your_prompt_1", "your_prompt_2"] + num_sampled_frames = 8 + + to_tensor = transforms.ToTensor() + + sampled_frames_list = [] + for video_path in video_path_list: + vr = VideoReader(video_path) + sampled_frame_indices = np.linspace(0, len(vr), num_sampled_frames, endpoint=False, dtype=int) + sampled_frames = vr.get_batch(sampled_frame_indices).asnumpy() + sampled_frames = torch.stack([to_tensor(frame) for frame in sampled_frames]) + sampled_frames_list.append(sampled_frames) + sampled_frames = torch.stack(sampled_frames_list) + sampled_frames = rearrange(sampled_frames, "b t c h w -> b c t h w") + + aesthetic_reward_v2 = AestheticReward(device="cuda", dtype=torch.bfloat16) + print(f"aesthetic_reward_v2: {aesthetic_reward_v2(sampled_frames)}") + + aesthetic_reward_v2_5 = AestheticReward( + encoder_path="google/siglip-so400m-patch14-384", version="v2.5", device="cuda", dtype=torch.bfloat16 + ) + print(f"aesthetic_reward_v2_5: {aesthetic_reward_v2_5(sampled_frames)}") + + hps_reward_v2 = HPSReward(device="cuda", dtype=torch.bfloat16) + print(f"hps_reward_v2: {hps_reward_v2(sampled_frames, prompt_list)}") + + hps_reward_v2_1 = HPSReward(version="v2.1", device="cuda", dtype=torch.bfloat16) + print(f"hps_reward_v2_1: {hps_reward_v2_1(sampled_frames, prompt_list)}") + + pick_score = PickScoreReward(device="cuda", dtype=torch.bfloat16) + print(f"pick_score_reward: {pick_score(sampled_frames, prompt_list)}") + + mps_score = MPSReward(device="cuda", dtype=torch.bfloat16) + print(f"mps_reward: {mps_score(sampled_frames, prompt_list)}") \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/ui/cogvideox_fun_ui.py b/VideoX-Fun/videox_fun/ui/cogvideox_fun_ui.py new file mode 100644 index 0000000000000000000000000000000000000000..748ca9605e95f31dcf6aecc769ac7c8ce37c7fd5 --- /dev/null +++ b/VideoX-Fun/videox_fun/ui/cogvideox_fun_ui.py @@ -0,0 +1,722 @@ +"""Modified from https://github.com/guoyww/AnimateDiff/blob/main/app.py +""" +import os +import random + +import cv2 +import gradio as gr +import numpy as np +import torch +from PIL import Image +from safetensors import safe_open + +from ..data.bucket_sampler import ASPECT_RATIO_512, get_closest_ratio +from ..models import (AutoencoderKLCogVideoX, CogVideoXTransformer3DModel, + T5EncoderModel, T5Tokenizer) +from ..pipeline import (CogVideoXFunControlPipeline, + CogVideoXFunInpaintPipeline, CogVideoXFunPipeline) +from ..utils.fp8_optimization import (convert_model_weight_to_float8, replace_parameters_by_name, + convert_weight_dtype_wrapper) +from ..utils.lora_utils import merge_lora, unmerge_lora +from ..utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, timer, + get_video_to_video_latent, save_videos_grid) +from .controller import (Fun_Controller, Fun_Controller_Client, + all_cheduler_dict, css, ddpm_scheduler_dict, + flow_scheduler_dict, gradio_version, + gradio_version_is_above_4) +from .ui import (create_cfg_and_seedbox, + create_fake_finetune_models_checkpoints, + create_fake_height_width, create_fake_model_checkpoints, + create_fake_model_type, create_finetune_models_checkpoints, + create_generation_method, + create_generation_methods_and_video_length, + create_height_width, create_model_checkpoints, + create_model_type, create_prompts, create_samplers, + create_ui_outputs) +from ..dist import set_multi_gpus_devices, shard_model + + +class CogVideoXFunController(Fun_Controller): + def update_diffusion_transformer(self, diffusion_transformer_dropdown): + print(f"Update diffusion transformer: {diffusion_transformer_dropdown}") + self.diffusion_transformer_dropdown = diffusion_transformer_dropdown + if diffusion_transformer_dropdown == "none": + return gr.update() + self.vae = AutoencoderKLCogVideoX.from_pretrained( + diffusion_transformer_dropdown, + subfolder="vae", + ).to(self.weight_dtype) + + # Get Transformer + self.transformer = CogVideoXTransformer3DModel.from_pretrained( + diffusion_transformer_dropdown, + subfolder="transformer", + low_cpu_mem_usage=True, + ).to(self.weight_dtype) + + # Get tokenizer and text_encoder + tokenizer = T5Tokenizer.from_pretrained( + diffusion_transformer_dropdown, subfolder="tokenizer" + ) + text_encoder = T5EncoderModel.from_pretrained( + diffusion_transformer_dropdown, subfolder="text_encoder", torch_dtype=self.weight_dtype + ) + + # Get pipeline + if self.model_type == "Inpaint": + if self.transformer.config.in_channels != self.vae.config.latent_channels: + self.pipeline = CogVideoXFunInpaintPipeline( + tokenizer=tokenizer, + text_encoder=text_encoder, + vae=self.vae, + transformer=self.transformer, + scheduler=self.scheduler_dict[list(self.scheduler_dict.keys())[0]].from_pretrained(diffusion_transformer_dropdown, subfolder="scheduler"), + ) + else: + self.pipeline = CogVideoXFunPipeline( + tokenizer=tokenizer, + text_encoder=text_encoder, + vae=self.vae, + transformer=self.transformer, + scheduler=self.scheduler_dict[list(self.scheduler_dict.keys())[0]].from_pretrained(diffusion_transformer_dropdown, subfolder="scheduler"), + ) + else: + self.pipeline = CogVideoXFunControlPipeline( + diffusion_transformer_dropdown, + vae=self.vae, + transformer=self.transformer, + scheduler=self.scheduler_dict[list(self.scheduler_dict.keys())[0]].from_pretrained(diffusion_transformer_dropdown, subfolder="scheduler"), + torch_dtype=self.weight_dtype + ) + + if self.ulysses_degree > 1 or self.ring_degree > 1: + from functools import partial + self.transformer.enable_multi_gpus_inference() + if self.fsdp_dit: + shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) + self.pipeline.transformer = shard_fn(self.pipeline.transformer) + print("Add FSDP DIT") + if self.fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) + self.pipeline.text_encoder = shard_fn(self.pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + + if self.compile_dit: + for i in range(len(self.pipeline.transformer.transformer_blocks)): + self.pipeline.transformer.transformer_blocks[i] = torch.compile(self.pipeline.transformer.transformer_blocks[i]) + print("Add Compile") + + if self.GPU_memory_mode == "sequential_cpu_offload": + self.pipeline.enable_sequential_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(self.pipeline.transformer, exclude_module_name=[], device=self.device) + convert_weight_dtype_wrapper(self.pipeline.transformer, self.weight_dtype) + self.pipeline.enable_model_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_cpu_offload": + self.pipeline.enable_model_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(self.pipeline.transformer, exclude_module_name=[], device=self.device) + convert_weight_dtype_wrapper(self.pipeline.transformer, self.weight_dtype) + self.pipeline.to(self.device) + else: + self.pipeline.to(self.device) + print("Update diffusion transformer done") + return gr.update() + + @timer + def generate( + self, + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image = None, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + base_model_2_dropdown=None, + lora_model_2_dropdown=None, + fps = None, + is_api = False, + ): + self.clear_cache() + + print(f"Input checking.") + _, comment = self.input_check( + resize_method, generation_method, start_image, end_image, validation_video,control_video, is_api + ) + print(f"Input checking down") + if comment != "OK": + return "", comment + is_image = True if generation_method == "Image Generation" else False + + if self.base_model_path != base_model_dropdown: + self.update_base_model(base_model_dropdown) + + if self.lora_model_path != lora_model_dropdown: + self.update_lora_model(lora_model_dropdown) + + print(f"Load scheduler.") + self.pipeline.scheduler = self.scheduler_dict[sampler_dropdown].from_config(self.pipeline.scheduler.config) + print(f"Load scheduler down.") + + if resize_method == "Resize according to Reference": + print(f"Calculate height and width according to Reference.") + height_slider, width_slider = self.get_height_width_from_reference( + base_resolution, start_image, validation_video, control_video, + ) + + if self.lora_model_path != "none": + print(f"Merge Lora.") + self.pipeline = merge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + print(f"Merge Lora done.") + + if fps is None: + fps = 8 + + print(f"Generate seed.") + if int(seed_textbox) != -1 and seed_textbox != "": torch.manual_seed(int(seed_textbox)) + else: seed_textbox = np.random.randint(0, 1e10) + generator = torch.Generator(device=self.device).manual_seed(int(seed_textbox)) + print(f"Generate seed done.") + + try: + print(f"Generation.") + if self.model_type == "Inpaint": + if self.transformer.config.in_channels != self.vae.config.latent_channels: + if generation_method == "Long Video Generation": + if validation_video is not None: + raise gr.Error(f"Video to Video is not Support Long Video Generation now.") + init_frames = 0 + last_frames = init_frames + partial_video_length + while init_frames < length_slider: + if last_frames >= length_slider: + _partial_video_length = length_slider - init_frames + _partial_video_length = int((_partial_video_length - 1) // self.vae.config.temporal_compression_ratio * self.vae.config.temporal_compression_ratio) + 1 + + if _partial_video_length <= 0: + break + else: + _partial_video_length = partial_video_length + + if last_frames >= length_slider: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=_partial_video_length, sample_size=(height_slider, width_slider)) + else: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, None, video_length=_partial_video_length, sample_size=(height_slider, width_slider)) + + with torch.no_grad(): + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = _partial_video_length, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + strength = 1, + ).videos + + if init_frames != 0: + mix_ratio = torch.from_numpy( + np.array([float(_index) / float(overlap_video_length) for _index in range(overlap_video_length)], np.float32) + ).unsqueeze(0).unsqueeze(0).unsqueeze(-1).unsqueeze(-1) + + new_sample[:, :, -overlap_video_length:] = new_sample[:, :, -overlap_video_length:] * (1 - mix_ratio) + \ + sample[:, :, :overlap_video_length] * mix_ratio + new_sample = torch.cat([new_sample, sample[:, :, overlap_video_length:]], dim = 2) + + sample = new_sample + else: + new_sample = sample + + if last_frames >= length_slider: + break + + start_image = [ + Image.fromarray( + (sample[0, :, _index].transpose(0, 1).transpose(1, 2) * 255).numpy().astype(np.uint8) + ) for _index in range(-overlap_video_length, 0) + ] + + init_frames = init_frames + _partial_video_length - overlap_video_length + last_frames = init_frames + _partial_video_length + else: + if validation_video is not None: + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(validation_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), validation_video_mask=validation_video_mask, fps=fps) + strength = denoise_strength + else: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) + strength = 1 + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + strength = strength, + ).videos + else: + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator + ).videos + else: + input_video, input_video_mask, ref_image, clip_image = get_video_to_video_latent(control_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), fps=fps) + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + + control_video = input_video, + ).videos + except Exception as e: + self.auto_model_clear_cache(self.pipeline.transformer) + self.auto_model_clear_cache(self.pipeline.text_encoder) + self.auto_model_clear_cache(self.pipeline.vae) + self.clear_cache() + + print(f"Error. error information is {str(e)}") + if self.lora_model_path != "none": + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + if is_api: + return "", f"Error. error information is {str(e)}" + else: + return gr.update(), gr.update(), f"Error. error information is {str(e)}" + + self.clear_cache() + # lora part + if self.lora_model_path != "none": + print(f"Unmerge Lora.") + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + print(f"Unmerge Lora done.") + + print(f"Saving outputs.") + save_sample_path = self.save_outputs( + is_image, length_slider, sample, fps=fps + ) + print(f"Saving outputs done.") + + if is_image or length_slider == 1: + if is_api: + return save_sample_path, "Success" + else: + if gradio_version_is_above_4: + return gr.Image(value=save_sample_path, visible=True), gr.Video(value=None, visible=False), "Success" + else: + return gr.Image.update(value=save_sample_path, visible=True), gr.Video.update(value=None, visible=False), "Success" + else: + if is_api: + return save_sample_path, "Success" + else: + if gradio_version_is_above_4: + return gr.Image(visible=False, value=None), gr.Video(value=save_sample_path, visible=True), "Success" + else: + return gr.Image.update(visible=False, value=None), gr.Video.update(value=save_sample_path, visible=True), "Success" + +CogVideoXFunController_Host = CogVideoXFunController +CogVideoXFunController_Client = Fun_Controller_Client + +def ui(GPU_memory_mode, scheduler_dict, compile_dit, weight_dtype, savedir_sample=None): + controller = CogVideoXFunController( + GPU_memory_mode, scheduler_dict, model_name=None, model_type="Inpaint", + compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # CogVideoX-Fun: + + A CogVideoX with more flexible generation conditions, capable of producing videos of different resolutions, around 6 seconds, and fps 8 (frames 1 to 49), as well as image generated videos. + + [Github](https://github.com/aigc-apps/CogVideoX-Fun/) + """ + ) + with gr.Column(variant="panel"): + model_type = create_model_type(visible=True) + diffusion_transformer_dropdown, diffusion_transformer_refresh_button = \ + create_model_checkpoints(controller, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider, personalized_refresh_button = \ + create_finetune_models_checkpoints(controller, visible=True) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts() + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller) + + resize_method, width_slider, height_slider, base_resolution = create_height_width( + default_height = 384, default_width = 672, maximum_height = 1344, + maximum_width = 1344, + ) + gr.Markdown( + """ + V1.0 and V1.1 support up to 49 frames of video generation, while V1.5 supports up to 85 frames. + (V1.0和V1.1支持最大49帧视频生成，V1.5支持最大85帧视频生成。) + """ + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation", "Long Video Generation"], + default_video_length=49, + maximum_video_length=85, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video to Video (视频到视频)", "Video Control (视频控制)"], prompt_textbox + ) + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + model_type.change( + fn=controller.update_model_type, + inputs=[model_type], + outputs=[] + ) + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return [gr.update(visible=True, maximum=85, value=49, interactive=True), gr.update(visible=False), gr.update(visible=False)] + elif generation_method == "Image Generation": + return [gr.update(minimum=1, maximum=1, value=1, interactive=False), gr.update(visible=False), gr.update(visible=False)] + else: + return [gr.update(visible=True, maximum=1344), gr.update(visible=True), gr.update(visible=True)] + generation_method.change( + upload_generation_method, generation_method, [length_slider, overlap_video_length, partial_video_length] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller + +def ui_host(GPU_memory_mode, scheduler_dict, model_name, model_type, compile_dit, weight_dtype, savedir_sample=None): + controller = CogVideoXFunController_Host( + GPU_memory_mode, scheduler_dict, model_name=model_name, model_type=model_type, + compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # CogVideoX-Fun + + A CogVideoX with more flexible generation conditions, capable of producing videos of different resolutions, around 6 seconds, and fps 8 (frames 1 to 49), as well as image generated videos. + + [Github](https://github.com/aigc-apps/CogVideoX-Fun/) + """ + ) + with gr.Column(variant="panel"): + model_type = create_fake_model_type(visible=False) + diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider = create_fake_finetune_models_checkpoints(visible=True) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts() + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller) + + resize_method, width_slider, height_slider, base_resolution = create_height_width( + default_height = 384, default_width = 672, maximum_height = 1344, + maximum_width = 1344, + ) + gr.Markdown( + """ + V1.0 and V1.1 support up to 49 frames of video generation, while V1.5 supports up to 85 frames. + (V1.0和V1.1支持最大49帧视频生成，V1.5支持最大85帧视频生成。) + """ + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=49, + maximum_video_length=85, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video to Video (视频到视频)", "Video Control (视频控制)"], prompt_textbox + ) + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return gr.update(visible=True, minimum=8, maximum=85, value=49, interactive=True) + elif generation_method == "Image Generation": + return gr.update(minimum=1, maximum=1, value=1, interactive=False) + generation_method.change( + upload_generation_method, generation_method, [length_slider] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller + +def ui_client(scheduler_dict, model_name, savedir_sample=None): + controller = CogVideoXFunController_Client(scheduler_dict, savedir_sample) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # CogVideoX-Fun + + A CogVideoX with more flexible generation conditions, capable of producing videos of different resolutions, around 6 seconds, and fps 8 (frames 1 to 49), as well as image generated videos. + + [Github](https://github.com/aigc-apps/CogVideoX-Fun/) + """ + ) + with gr.Column(variant="panel"): + diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider = create_fake_finetune_models_checkpoints(visible=True) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts() + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller, maximum_step=50) + + resize_method, width_slider, height_slider, base_resolution = create_fake_height_width( + default_height = 384, default_width = 672, maximum_height = 1344, + maximum_width = 1344, + ) + gr.Markdown( + """ + V1.0 and V1.1 support up to 49 frames of video generation, while V1.5 supports up to 85 frames. + (V1.0和V1.1支持最大49帧视频生成，V1.5支持最大85帧视频生成。) + """ + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=49, + maximum_video_length=85, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video to Video (视频到视频)"], prompt_textbox + ) + + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return gr.update(visible=True, minimum=5, maximum=85, value=49, interactive=True) + elif generation_method == "Image Generation": + return gr.update(minimum=1, maximum=1, value=1, interactive=False) + generation_method.change( + upload_generation_method, generation_method, [length_slider] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(), gr.update()] + source_method.change( + upload_source_method, source_method, [image_to_video_col, video_to_video_col, start_image, end_image, validation_video, validation_video_mask] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + denoise_strength, + seed_textbox, + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/ui/controller.py b/VideoX-Fun/videox_fun/ui/controller.py new file mode 100644 index 0000000000000000000000000000000000000000..31a5e9d0e76ecc32e4d92273ae7fb55a874c0dfe --- /dev/null +++ b/VideoX-Fun/videox_fun/ui/controller.py @@ -0,0 +1,514 @@ +"""Modified from https://github.com/guoyww/AnimateDiff/blob/main/app.py +""" +import base64 +import gc +import json +import os +import hashlib +import random +from datetime import datetime +from glob import glob + +import cv2 +import gradio as gr +import numpy as np +import pkg_resources +import requests +import torch +from diffusers import (CogVideoXDDIMScheduler, DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, + FlowMatchEulerDiscreteScheduler, PNDMScheduler) +from omegaconf import OmegaConf +from PIL import Image +from safetensors import safe_open + +from ..data.bucket_sampler import ASPECT_RATIO_512, get_closest_ratio +from ..utils.utils import save_videos_grid +from ..utils.fm_solvers import FlowDPMSolverMultistepScheduler +from ..utils.fm_solvers_unipc import FlowUniPCMultistepScheduler +from ..dist import set_multi_gpus_devices + +gradio_version = pkg_resources.get_distribution("gradio").version +gradio_version_is_above_4 = True if int(gradio_version.split('.')[0]) >= 4 else False + +css = """ +.toolbutton { + margin-buttom: 0em 0em 0em 0em; + max-width: 2.5em; + min-width: 2.5em !important; + height: 2.5em; +} +""" + +ddpm_scheduler_dict = { + "Euler": EulerDiscreteScheduler, + "Euler A": EulerAncestralDiscreteScheduler, + "DPM++": DPMSolverMultistepScheduler, + "PNDM": PNDMScheduler, + "DDIM": DDIMScheduler, + "DDIM_Origin": DDIMScheduler, + "DDIM_Cog": CogVideoXDDIMScheduler, +} +flow_scheduler_dict = { + "Flow": FlowMatchEulerDiscreteScheduler, + "Flow_Unipc": FlowUniPCMultistepScheduler, + "Flow_DPM++": FlowDPMSolverMultistepScheduler, +} +all_cheduler_dict = {**ddpm_scheduler_dict, **flow_scheduler_dict} + +class Fun_Controller: + def __init__( + self, GPU_memory_mode, scheduler_dict, model_name=None, model_type="Inpaint", + config_path=None, ulysses_degree=1, ring_degree=1, + fsdp_dit=False, fsdp_text_encoder=False, compile_dit=False, + weight_dtype=None, savedir_sample=None, + ): + # config dirs + self.basedir = os.getcwd() + self.config_dir = os.path.join(self.basedir, "config") + self.diffusion_transformer_dir = os.path.join(self.basedir, "models", "Diffusion_Transformer") + self.motion_module_dir = os.path.join(self.basedir, "models", "Motion_Module") + self.personalized_model_dir = os.path.join(self.basedir, "models", "Personalized_Model") + if savedir_sample is None: + self.savedir_sample = os.path.join(self.basedir, "samples", datetime.now().strftime("Gradio-%Y-%m-%dT%H-%M-%S")) + else: + self.savedir_sample = savedir_sample + os.makedirs(self.savedir_sample, exist_ok=True) + + self.GPU_memory_mode = GPU_memory_mode + self.model_name = model_name + self.diffusion_transformer_dropdown = model_name + self.scheduler_dict = scheduler_dict + self.model_type = model_type + if config_path is not None: + self.config_path = os.path.realpath(config_path) + self.config = OmegaConf.load(config_path) + else: + self.config_path = None + self.ulysses_degree = ulysses_degree + self.ring_degree = ring_degree + self.fsdp_dit = fsdp_dit + self.fsdp_text_encoder = fsdp_text_encoder + self.compile_dit = compile_dit + self.weight_dtype = weight_dtype + self.device = set_multi_gpus_devices(self.ulysses_degree, self.ring_degree) + + self.diffusion_transformer_list = [] + self.motion_module_list = [] + self.personalized_model_list = [] + self.config_list = [] + + # config models + self.tokenizer = None + self.text_encoder = None + self.vae = None + self.transformer = None + self.transformer_2 = None + self.pipeline = None + self.base_model_path = "none" + self.base_model_2_path = "none" + self.lora_model_path = "none" + self.lora_model_2_path = "none" + + self.refresh_config() + self.refresh_diffusion_transformer() + self.refresh_personalized_model() + if model_name != None: + self.update_diffusion_transformer(model_name) + + def refresh_config(self): + config_list = [] + for root, dirs, files in os.walk(self.config_dir): + for file in files: + if file.endswith(('.yaml', '.yml')): + full_path = os.path.join(root, file) + config_list.append(full_path) + self.config_list = config_list + + def refresh_diffusion_transformer(self): + self.diffusion_transformer_list = sorted(glob(os.path.join(self.diffusion_transformer_dir, "*/"))) + + def refresh_personalized_model(self): + personalized_model_list = sorted(glob(os.path.join(self.personalized_model_dir, "*.safetensors"))) + self.personalized_model_list = [os.path.basename(p) for p in personalized_model_list] + + def update_model_type(self, model_type): + self.model_type = model_type + + def update_config(self, config_dropdown): + self.config_path = config_dropdown + self.config = OmegaConf.load(config_dropdown) + print(f"Update config: {config_dropdown}") + + def update_diffusion_transformer(self, diffusion_transformer_dropdown): + pass + + def update_base_model(self, base_model_dropdown, is_checkpoint_2=False): + if not is_checkpoint_2: + self.base_model_path = base_model_dropdown + else: + self.base_model_2_path = base_model_dropdown + print(f"Update base model: {base_model_dropdown}") + if base_model_dropdown == "none": + return gr.update() + if self.transformer is None and not is_checkpoint_2: + gr.Info(f"Please select a pretrained model path.") + print(f"Please select a pretrained model path.") + return gr.update(value=None) + elif self.transformer_2 is None and is_checkpoint_2: + gr.Info(f"Please select a pretrained model path.") + print(f"Please select a pretrained model path.") + return gr.update(value=None) + else: + base_model_dropdown = os.path.join(self.personalized_model_dir, base_model_dropdown) + base_model_state_dict = {} + with safe_open(base_model_dropdown, framework="pt", device="cpu") as f: + for key in f.keys(): + base_model_state_dict[key] = f.get_tensor(key) + if not is_checkpoint_2: + self.transformer.load_state_dict(base_model_state_dict, strict=False) + else: + self.transformer_2.load_state_dict(base_model_state_dict, strict=False) + print("Update base model done") + return gr.update() + + def update_lora_model(self, lora_model_dropdown, is_checkpoint_2=False): + print(f"Update lora model: {lora_model_dropdown}") + if lora_model_dropdown == "none": + self.lora_model_path = "none" + return gr.update() + lora_model_dropdown = os.path.join(self.personalized_model_dir, lora_model_dropdown) + if not is_checkpoint_2: + self.lora_model_path = lora_model_dropdown + else: + self.lora_model_2_path = lora_model_dropdown + return gr.update() + + def clear_cache(self,): + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + + def auto_model_clear_cache(self, model): + origin_device = model.device + model = model.to("cpu") + gc.collect() + torch.cuda.empty_cache() + torch.cuda.ipc_collect() + model = model.to(origin_device) + + def input_check(self, + resize_method, + generation_method, + start_image, + end_image, + validation_video, + control_video, + is_api = False, + ): + if self.transformer is None: + if is_api: + return "", f"Please select a pretrained model path." + else: + raise gr.Error(f"Please select a pretrained model path.") + + if control_video is not None and self.model_type == "Inpaint": + if is_api: + return "", f"If specifying the control video, please set the model_type == \"Control\". " + else: + raise gr.Error(f"If specifying the control video, please set the model_type == \"Control\". ") + + if control_video is None and self.model_type == "Control": + if is_api: + return "", f"If set the model_type == \"Control\", please specifying the control video. " + else: + raise gr.Error(f"If set the model_type == \"Control\", please specifying the control video. ") + + if resize_method == "Resize according to Reference": + if start_image is None and validation_video is None and control_video is None: + if is_api: + return "", f"Please upload an image when using \"Resize according to Reference\"." + else: + raise gr.Error(f"Please upload an image when using \"Resize according to Reference\".") + + if self.transformer.config.in_channels == self.vae.config.latent_channels and start_image is not None: + if is_api: + return "", f"Please select an image to video pretrained model while using image to video." + else: + raise gr.Error(f"Please select an image to video pretrained model while using image to video.") + + if self.transformer.config.in_channels == self.vae.config.latent_channels and generation_method == "Long Video Generation": + if is_api: + return "", f"Please select an image to video pretrained model while using long video generation." + else: + raise gr.Error(f"Please select an image to video pretrained model while using long video generation.") + + if start_image is None and end_image is not None: + if is_api: + return "", f"If specifying the ending image of the video, please specify a starting image of the video." + else: + raise gr.Error(f"If specifying the ending image of the video, please specify a starting image of the video.") + return "", "OK" + + def get_height_width_from_reference( + self, + base_resolution, + start_image, + validation_video, + control_video, + ): + spatial_compression_ratio = self.vae.config.spatial_compression_ratio if hasattr(self.vae.config, "spatial_compression_ratio") else 8 + aspect_ratio_sample_size = {key : [x / 512 * base_resolution for x in ASPECT_RATIO_512[key]] for key in ASPECT_RATIO_512.keys()} + if self.model_type == "Inpaint": + if validation_video is not None: + original_width, original_height = Image.fromarray(cv2.VideoCapture(validation_video).read()[1]).size + else: + original_width, original_height = start_image[0].size if type(start_image) is list else Image.open(start_image).size + else: + original_width, original_height = Image.fromarray(cv2.VideoCapture(control_video).read()[1]).size + closest_size, closest_ratio = get_closest_ratio(original_height, original_width, ratios=aspect_ratio_sample_size) + height_slider, width_slider = [int(x / spatial_compression_ratio / 2) * spatial_compression_ratio * 2 for x in closest_size] + return height_slider, width_slider + + def save_outputs(self, is_image, length_slider, sample, fps): + def save_results(): + if not os.path.exists(self.savedir_sample): + os.makedirs(self.savedir_sample, exist_ok=True) + index = len([path for path in os.listdir(self.savedir_sample)]) + 1 + prefix = str(index).zfill(8) + + md5_hash = hashlib.md5(sample.cpu().numpy().tobytes()).hexdigest() + + if is_image or length_slider == 1: + save_sample_path = os.path.join(self.savedir_sample, prefix + f"-{md5_hash}.png") + print(f"Saving to {save_sample_path}") + image = sample[0, :, 0] + image = image.transpose(0, 1).transpose(1, 2) + image = (image * 255).numpy().astype(np.uint8) + image = Image.fromarray(image) + image.save(save_sample_path) + + else: + save_sample_path = os.path.join(self.savedir_sample, prefix + f"-{md5_hash}.mp4") + print(f"Saving to {save_sample_path}") + save_videos_grid(sample, save_sample_path, fps=fps) + return save_sample_path + + if self.ulysses_degree * self.ring_degree > 1: + import torch.distributed as dist + if dist.get_rank() == 0: + save_sample_path = save_results() + else: + save_sample_path = None + else: + save_sample_path = save_results() + return save_sample_path + + def generate( + self, + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + is_api = False, + ): + pass + +def post_to_host( + diffusion_transformer_dropdown, + base_model_dropdown, lora_model_dropdown, lora_alpha_slider, + prompt_textbox, negative_prompt_textbox, + sampler_dropdown, sample_step_slider, resize_method, width_slider, height_slider, + base_resolution, generation_method, length_slider, cfg_scale_slider, + start_image, end_image, validation_video, validation_video_mask, denoise_strength, seed_textbox, + ref_image = None, enable_teacache = None, teacache_threshold = None, num_skip_start_steps = None, + teacache_offload = None, cfg_skip_ratio = None,enable_riflex = None, riflex_k = None, +): + if start_image is not None: + with open(start_image, 'rb') as file: + file_content = file.read() + start_image_encoded_content = base64.b64encode(file_content) + start_image = start_image_encoded_content.decode('utf-8') + + if end_image is not None: + with open(end_image, 'rb') as file: + file_content = file.read() + end_image_encoded_content = base64.b64encode(file_content) + end_image = end_image_encoded_content.decode('utf-8') + + if validation_video is not None: + with open(validation_video, 'rb') as file: + file_content = file.read() + validation_video_encoded_content = base64.b64encode(file_content) + validation_video = validation_video_encoded_content.decode('utf-8') + + if validation_video_mask is not None: + with open(validation_video_mask, 'rb') as file: + file_content = file.read() + validation_video_mask_encoded_content = base64.b64encode(file_content) + validation_video_mask = validation_video_mask_encoded_content.decode('utf-8') + + if ref_image is not None: + with open(ref_image, 'rb') as file: + file_content = file.read() + ref_image_encoded_content = base64.b64encode(file_content) + ref_image = ref_image_encoded_content.decode('utf-8') + + datas = { + "base_model_path": base_model_dropdown, + "lora_model_path": lora_model_dropdown, + "lora_alpha_slider": lora_alpha_slider, + "prompt_textbox": prompt_textbox, + "negative_prompt_textbox": negative_prompt_textbox, + "sampler_dropdown": sampler_dropdown, + "sample_step_slider": sample_step_slider, + "resize_method": resize_method, + "width_slider": width_slider, + "height_slider": height_slider, + "base_resolution": base_resolution, + "generation_method": generation_method, + "length_slider": length_slider, + "cfg_scale_slider": cfg_scale_slider, + "start_image": start_image, + "end_image": end_image, + "validation_video": validation_video, + "validation_video_mask": validation_video_mask, + "denoise_strength": denoise_strength, + "seed_textbox": seed_textbox, + + "ref_image": ref_image, + "enable_teacache": enable_teacache, + "teacache_threshold": teacache_threshold, + "num_skip_start_steps": num_skip_start_steps, + "teacache_offload": teacache_offload, + "cfg_skip_ratio": cfg_skip_ratio, + "enable_riflex": enable_riflex, + "riflex_k": riflex_k, + } + + session = requests.session() + session.headers.update({"Authorization": os.environ.get("EAS_TOKEN")}) + + response = session.post(url=f'{os.environ.get("EAS_URL")}/videox_fun/infer_forward', json=datas, timeout=300) + + outputs = response.json() + return outputs + + +class Fun_Controller_Client: + def __init__(self, scheduler_dict, savedir_sample): + self.basedir = os.getcwd() + if savedir_sample is None: + self.savedir_sample = os.path.join(self.basedir, "samples", datetime.now().strftime("Gradio-%Y-%m-%dT%H-%M-%S")) + else: + self.savedir_sample = savedir_sample + os.makedirs(self.savedir_sample, exist_ok=True) + + self.scheduler_dict = scheduler_dict + + def generate( + self, + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + denoise_strength, + seed_textbox, + ref_image = None, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + ): + is_image = True if generation_method == "Image Generation" else False + + outputs = post_to_host( + diffusion_transformer_dropdown, + base_model_dropdown, lora_model_dropdown, lora_alpha_slider, + prompt_textbox, negative_prompt_textbox, + sampler_dropdown, sample_step_slider, resize_method, width_slider, height_slider, + base_resolution, generation_method, length_slider, cfg_scale_slider, + start_image, end_image, validation_video, validation_video_mask, denoise_strength, + seed_textbox, ref_image = ref_image, enable_teacache = enable_teacache, teacache_threshold = teacache_threshold, + num_skip_start_steps = num_skip_start_steps, teacache_offload = teacache_offload, + cfg_skip_ratio = cfg_skip_ratio, enable_riflex = enable_riflex, riflex_k = riflex_k, + ) + + try: + base64_encoding = outputs["base64_encoding"] + except: + return gr.Image(visible=False, value=None), gr.Video(None, visible=True), outputs["message"] + + decoded_data = base64.b64decode(base64_encoding) + + if not os.path.exists(self.savedir_sample): + os.makedirs(self.savedir_sample, exist_ok=True) + md5_hash = hashlib.md5(decoded_data).hexdigest() + + index = len([path for path in os.listdir(self.savedir_sample)]) + 1 + prefix = str(index).zfill(8) + + if is_image or length_slider == 1: + save_sample_path = os.path.join(self.savedir_sample, prefix + f"-{md5_hash}.png") + print(f"Saving to {save_sample_path}") + with open(save_sample_path, "wb") as file: + file.write(decoded_data) + if gradio_version_is_above_4: + return gr.Image(value=save_sample_path, visible=True), gr.Video(value=None, visible=False), "Success" + else: + return gr.Image.update(value=save_sample_path, visible=True), gr.Video.update(value=None, visible=False), "Success" + else: + save_sample_path = os.path.join(self.savedir_sample, prefix + f"-{md5_hash}.mp4") + print(f"Saving to {save_sample_path}") + with open(save_sample_path, "wb") as file: + file.write(decoded_data) + if gradio_version_is_above_4: + return gr.Image(visible=False, value=None), gr.Video(value=save_sample_path, visible=True), "Success" + else: + return gr.Image.update(visible=False, value=None), gr.Video.update(value=save_sample_path, visible=True), "Success" diff --git a/VideoX-Fun/videox_fun/ui/ui.py b/VideoX-Fun/videox_fun/ui/ui.py new file mode 100644 index 0000000000000000000000000000000000000000..ee7d02458458bdd287a8fbbe812f663d94b2f308 --- /dev/null +++ b/VideoX-Fun/videox_fun/ui/ui.py @@ -0,0 +1,358 @@ +import random + +import gradio as gr + + +def create_model_type(visible): + gr.Markdown( + """ + ### Model Type (模型的种类，正常模型还是控制模型). + """, + visible=visible, + ) + with gr.Row(): + model_type = gr.Dropdown( + label="The model type of the model (模型的种类，正常模型还是控制模型)", + choices=["Inpaint", "Control"], + value="Inpaint", + visible=visible, + interactive=True, + ) + return model_type + +def create_fake_model_type(visible): + gr.Markdown( + """ + ### Model Type (模型的种类，正常模型还是控制模型). + """, + visible=visible, + ) + with gr.Row(): + model_type = gr.Dropdown( + label="The model type of the model (模型的种类，正常模型还是控制模型)", + choices=["Inpaint", "Control"], + value="Inpaint", + interactive=False, + visible=visible, + ) + return model_type + +def create_model_checkpoints(controller, visible): + gr.Markdown( + """ + ### Model checkpoints (模型路径). + """ + ) + with gr.Row(visible=visible): + diffusion_transformer_dropdown = gr.Dropdown( + label="Pretrained Model Path (预训练模型路径)", + choices=controller.diffusion_transformer_list, + value="none", + interactive=True, + ) + diffusion_transformer_dropdown.change( + fn=controller.update_diffusion_transformer, + inputs=[diffusion_transformer_dropdown], + outputs=[diffusion_transformer_dropdown] + ) + + diffusion_transformer_refresh_button = gr.Button(value="\U0001F503", elem_classes="toolbutton") + def refresh_diffusion_transformer(): + controller.refresh_diffusion_transformer() + return gr.update(choices=controller.diffusion_transformer_list) + diffusion_transformer_refresh_button.click(fn=refresh_diffusion_transformer, inputs=[], outputs=[diffusion_transformer_dropdown]) + + return diffusion_transformer_dropdown, diffusion_transformer_refresh_button + +def create_fake_model_checkpoints(model_name, visible): + gr.Markdown( + """ + ### Model checkpoints (模型路径). + """ + ) + with gr.Row(visible=visible): + diffusion_transformer_dropdown = gr.Dropdown( + label="Pretrained Model Path (预训练模型路径)", + choices=[model_name], + value=model_name, + interactive=False, + ) + return diffusion_transformer_dropdown + +def create_finetune_models_checkpoints(controller, visible, add_checkpoint_2=False): + with gr.Row(visible=visible): + base_model_dropdown = gr.Dropdown( + label="Select base Dreambooth model (选择基模型[非必需])", + choices=["none"] + controller.personalized_model_list, + value="none", + interactive=True, + ) + if add_checkpoint_2: + base_model_2_dropdown = gr.Dropdown( + label="Select base Dreambooth model (选择第二个基模型[非必需])", + choices=["none"] + controller.personalized_model_list, + value="none", + interactive=True, + ) + + lora_model_dropdown = gr.Dropdown( + label="Select LoRA model (选择LoRA模型[非必需])", + choices=["none"] + controller.personalized_model_list, + value="none", + interactive=True, + ) + if add_checkpoint_2: + lora_model_2_dropdown = gr.Dropdown( + label="Select LoRA model (选择LoRA模型[非必需])", + choices=["none"] + controller.personalized_model_list, + value="none", + interactive=True, + ) + + lora_alpha_slider = gr.Slider(label="LoRA alpha (LoRA权重)", value=0.55, minimum=0, maximum=2, interactive=True) + + personalized_refresh_button = gr.Button(value="\U0001F503", elem_classes="toolbutton") + def update_personalized_model(): + controller.refresh_personalized_model() + return [ + gr.update(choices=controller.personalized_model_list), + gr.update(choices=["none"] + controller.personalized_model_list) + ] + personalized_refresh_button.click(fn=update_personalized_model, inputs=[], outputs=[base_model_dropdown, lora_model_dropdown]) + + if not add_checkpoint_2: + return base_model_dropdown, lora_model_dropdown, lora_alpha_slider, personalized_refresh_button + else: + return [base_model_dropdown, base_model_2_dropdown], [lora_model_dropdown, lora_model_2_dropdown], \ + lora_alpha_slider, personalized_refresh_button + +def create_fake_finetune_models_checkpoints(visible): + with gr.Row(): + base_model_dropdown = gr.Dropdown( + label="Select base Dreambooth model (选择基模型[非必需])", + choices=["none"], + value="none", + interactive=False, + visible=False + ) + with gr.Column(visible=False): + gr.Markdown( + """ + ### Minimalism is an example portrait of Lora, triggered by specific prompt words. More details can be found on [Wiki](https://github.com/aigc-apps/CogVideoX-Fun/wiki/Training-Lora). + """ + ) + with gr.Row(): + lora_model_dropdown = gr.Dropdown( + label="Select LoRA model", + choices=["none"], + value="none", + interactive=True, + ) + + lora_alpha_slider = gr.Slider(label="LoRA alpha (LoRA权重)", value=0.55, minimum=0, maximum=2, interactive=True) + + return base_model_dropdown, lora_model_dropdown, lora_alpha_slider + +def create_teacache_params( + enable_teacache = True, + teacache_threshold = 0.10, + num_skip_start_steps = 1, + teacache_offload = False, +): + enable_teacache = gr.Checkbox(label="Enable TeaCache", value=enable_teacache) + teacache_threshold = gr.Slider(0.00, 0.25, value=teacache_threshold, step=0.01, label="TeaCache Threshold") + num_skip_start_steps = gr.Slider(0, 10, value=num_skip_start_steps, step=5, label="Number of Skip Start Steps") + teacache_offload = gr.Checkbox(label="Offload TeaCache to CPU", value=teacache_offload) + return enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload + +def create_cfg_skip_params( + cfg_skip_ratio = 0 +): + cfg_skip_ratio = gr.Slider(0.00, 0.50, value=cfg_skip_ratio, step=0.01, label="CFG Skip Ratio") + return cfg_skip_ratio + +def create_cfg_riflex_k( + enable_riflex = False, + riflex_k = 6 +): + enable_riflex = gr.Checkbox(label="Enable Riflex", value=enable_riflex) + riflex_k = gr.Slider(0, 10, value=riflex_k, step=1, label="Riflex Intrinsic Frequency Index") + return enable_riflex, riflex_k + +def create_prompts( + prompt="A young woman with beautiful and clear eyes and blonde hair standing and white dress in a forest wearing a crown. She seems to be lost in thought, and the camera focuses on her face. The video is of high quality, and the view is very clear. High quality, masterpiece, best quality, highres, ultra-detailed, fantastic.", + negative_prompt="The video is not of a high quality, it has a low resolution. Watermark present in each frame. The background is solid. Strange body and strange trajectory. Distortion. " +): + gr.Markdown( + """ + ### Configs for Generation (生成参数配置). + """ + ) + + prompt_textbox = gr.Textbox(label="Prompt (正向提示词)", lines=2, value=prompt) + negative_prompt_textbox = gr.Textbox(label="Negative prompt (负向提示词)", lines=2, value=negative_prompt) + return prompt_textbox, negative_prompt_textbox + +def create_samplers(controller, maximum_step=100): + with gr.Row(): + sampler_dropdown = gr.Dropdown(label="Sampling method (采样器种类)", choices=list(controller.scheduler_dict.keys()), value=list(controller.scheduler_dict.keys())[0]) + sample_step_slider = gr.Slider(label="Sampling steps (生成步数)", value=50, minimum=10, maximum=maximum_step, step=1) + + return sampler_dropdown, sample_step_slider + +def create_height_width(default_height, default_width, maximum_height, maximum_width): + resize_method = gr.Radio( + ["Generate by", "Resize according to Reference"], + value="Generate by", + show_label=False, + ) + width_slider = gr.Slider(label="Width (视频宽度)", value=default_width, minimum=128, maximum=maximum_width, step=16) + height_slider = gr.Slider(label="Height (视频高度)", value=default_height, minimum=128, maximum=maximum_height, step=16) + base_resolution = gr.Radio(label="Base Resolution of Pretrained Models", value=512, choices=[512, 640, 768, 896, 960, 1024], visible=False) + + return resize_method, width_slider, height_slider, base_resolution + +def create_fake_height_width(default_height, default_width, maximum_height, maximum_width): + resize_method = gr.Radio( + ["Generate by", "Resize according to Reference"], + value="Generate by", + show_label=False, + ) + width_slider = gr.Slider(label="Width (视频宽度)", value=default_width, minimum=128, maximum=maximum_width, step=16, interactive=False) + height_slider = gr.Slider(label="Height (视频高度)", value=default_height, minimum=128, maximum=maximum_height, step=16, interactive=False) + base_resolution = gr.Radio(label="Base Resolution of Pretrained Models", value=512, choices=[512, 640, 768, 896, 960, 1024], interactive=False, visible=False) + + return resize_method, width_slider, height_slider, base_resolution + +def create_generation_methods_and_video_length( + generation_method_options, + default_video_length, + maximum_video_length +): + with gr.Group(): + generation_method = gr.Radio( + generation_method_options, + value="Video Generation", + show_label=False, + ) + with gr.Row(): + length_slider = gr.Slider(label="Animation length (视频帧数)", value=default_video_length, minimum=1, maximum=maximum_video_length, step=4) + overlap_video_length = gr.Slider(label="Overlap length (视频续写的重叠帧数)", value=4, minimum=1, maximum=4, step=1, visible=False) + partial_video_length = gr.Slider(label="Partial video generation length (每个部分的视频生成帧数)", value=25, minimum=5, maximum=maximum_video_length, step=4, visible=False) + + return generation_method, length_slider, overlap_video_length, partial_video_length + +def create_generation_method(source_method_options, prompt_textbox, support_end_image=True, support_ref_image=False): + source_method = gr.Radio( + source_method_options, + value="Text to Video (文本到视频)", + show_label=False, + ) + with gr.Column(visible = False) as image_to_video_col: + start_image = gr.Image( + label="The image at the beginning of the video (图片到视频的开始图片)", show_label=True, + elem_id="i2v_start", sources="upload", type="filepath", + ) + + template_gallery_path = ["asset/1.png", "asset/2.png", "asset/3.png", "asset/4.png", "asset/5.png"] + def select_template(evt: gr.SelectData): + text = { + "asset/1.png": "A brown dog is shaking its head and sitting on a light colored sofa in a comfortable room. Behind the dog, there is a framed painting on the shelf surrounded by pink flowers. The soft and warm lighting in the room creates a comfortable atmosphere.", + "asset/2.png": "A sailboat navigates through moderately rough seas, with waves and ocean spray visible. The sailboat features a white hull and sails, accompanied by an orange sail catching the wind. The sky above shows dramatic, cloudy formations with a sunset or sunrise backdrop, casting warm colors across the scene. The water reflects the golden light, enhancing the visual contrast between the dark ocean and the bright horizon. The camera captures the scene with a dynamic and immersive angle, showcasing the movement of the boat and the energy of the ocean.", + "asset/3.png": "A stunningly beautiful woman with flowing long hair stands gracefully, her elegant dress rippling and billowing in the gentle wind. Petals falling off. Her serene expression and the natural movement of her attire create an enchanting and captivating scene, full of ethereal charm.", + "asset/4.png": "An astronaut, clad in a full space suit with a helmet, plays an electric guitar while floating in a cosmic environment filled with glowing particles and rocky textures. The scene is illuminated by a warm light source, creating dramatic shadows and contrasts. The background features a complex geometry, similar to a space station or an alien landscape, indicating a futuristic or otherworldly setting.", + "asset/5.png": "Fireworks light up the evening sky over a sprawling cityscape with gothic-style buildings featuring pointed towers and clock faces. The city is lit by both artificial lights from the buildings and the colorful bursts of the fireworks. The scene is viewed from an elevated angle, showcasing a vibrant urban environment set against a backdrop of a dramatic, partially cloudy sky at dusk.", + }[template_gallery_path[evt.index]] + return template_gallery_path[evt.index], text + + template_gallery = gr.Gallery( + template_gallery_path, + columns=5, rows=1, + height=140, + allow_preview=False, + container=False, + label="Template Examples", + ) + template_gallery.select(select_template, None, [start_image, prompt_textbox]) + + with gr.Accordion("The image at the ending of the video (图片到视频的结束图片[非必需, Optional])", open=False, visible=support_end_image): + end_image = gr.Image(label="The image at the ending of the video (图片到视频的结束图片[非必需, Optional])", show_label=False, elem_id="i2v_end", sources="upload", type="filepath") + + with gr.Column(visible = False) as video_to_video_col: + with gr.Row(): + validation_video = gr.Video( + label="The video to convert (视频转视频的参考视频)", show_label=True, + elem_id="v2v", sources="upload", + ) + with gr.Accordion("The mask of the video to inpaint (视频重新绘制的mask[非必需, Optional])", open=False): + gr.Markdown( + """ + - Please set a larger denoise_strength when using validation_video_mask, such as 1.00 instead of 0.70 + (请设置更大的denoise_strength，当使用validation_video_mask的时候，比如1而不是0.70) + """ + ) + validation_video_mask = gr.Image( + label="The mask of the video to inpaint (视频重新绘制的mask[非必需, Optional])", + show_label=False, elem_id="v2v_mask", sources="upload", type="filepath" + ) + denoise_strength = gr.Slider(label="Denoise strength (重绘系数)", value=0.70, minimum=0.10, maximum=1.00, step=0.01) + + with gr.Column(visible = False) as control_video_col: + gr.Markdown( + """ + Demo pose control video can be downloaded here [URL](https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/cogvideox_fun/asset/v1.1/pose.mp4). + """ + ) + control_video = gr.Video( + label="The control video (用于提供控制信号的video)", show_label=True, + elem_id="v2v_control", sources="upload", + ) + ref_image = gr.Image( + label="The reference image for control video (控制视频的参考图片)", show_label=True, + elem_id="ref_image", sources="upload", type="filepath", visible=support_ref_image + ) + return image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image + +def create_cfg_and_seedbox(gradio_version_is_above_4): + cfg_scale_slider = gr.Slider(label="CFG Scale (引导系数)", value=6.0, minimum=0, maximum=20) + + with gr.Row(): + seed_textbox = gr.Textbox(label="Seed (随机种子)", value=43) + seed_button = gr.Button(value="\U0001F3B2", elem_classes="toolbutton") + seed_button.click( + fn=lambda: gr.Textbox(value=random.randint(1, 1e8)) if gradio_version_is_above_4 else gr.Textbox.update(value=random.randint(1, 1e8)), + inputs=[], + outputs=[seed_textbox] + ) + return cfg_scale_slider, seed_textbox, seed_button + +def create_ui_outputs(): + with gr.Column(): + result_image = gr.Image(label="Generated Image (生成图片)", interactive=False, visible=False) + result_video = gr.Video(label="Generated Animation (生成视频)", interactive=False) + infer_progress = gr.Textbox( + label="Generation Info (生成信息)", + value="No task currently", + interactive=False + ) + return result_image, result_video, infer_progress + +def create_config(controller): + gr.Markdown( + """ + ### Config Path (配置文件路径) + """ + ) + with gr.Row(): + config_dropdown = gr.Dropdown( + label="Config Path (配置文件路径)", + choices=controller.config_list, + value=controller.config_path, + interactive=True, + ) + config_refresh_button = gr.Button(value="\U0001F503", elem_classes="toolbutton") + def refresh_config(): + controller.refresh_config() + return gr.update(choices=controller.config_list) + config_refresh_button.click(fn=refresh_config, inputs=[], outputs=[config_dropdown]) + return config_dropdown, config_refresh_button \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/ui/wan2_2_fun_ui.py b/VideoX-Fun/videox_fun/ui/wan2_2_fun_ui.py new file mode 100644 index 0000000000000000000000000000000000000000..0e4a07aa63aed9d987ad91487d8fdbdb00359bdd --- /dev/null +++ b/VideoX-Fun/videox_fun/ui/wan2_2_fun_ui.py @@ -0,0 +1,803 @@ +"""Modified from https://github.com/guoyww/AnimateDiff/blob/main/app.py +""" +import os +import random + +import cv2 +import gradio as gr +import numpy as np +import torch +from omegaconf import OmegaConf +from PIL import Image +from safetensors import safe_open + +from ..data.bucket_sampler import ASPECT_RATIO_512, get_closest_ratio +from ..dist import set_multi_gpus_devices, shard_model +from ..models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, + CLIPModel, Wan2_2Transformer3DModel, WanT5EncoderModel) +from ..models.cache_utils import get_teacache_coefficients +from ..pipeline import Wan2_2FunControlPipeline, Wan2_2FunPipeline, Wan2_2FunInpaintPipeline +from ..utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from ..utils.lora_utils import merge_lora, unmerge_lora +from ..utils.utils import (filter_kwargs, get_image_latent, + get_image_to_video_latent, + get_video_to_video_latent, save_videos_grid, timer) +from .controller import (Fun_Controller, Fun_Controller_Client, + all_cheduler_dict, css, ddpm_scheduler_dict, + flow_scheduler_dict, gradio_version, + gradio_version_is_above_4) +from .ui import (create_cfg_and_seedbox, create_cfg_riflex_k, + create_cfg_skip_params, create_config, + create_fake_finetune_models_checkpoints, + create_fake_height_width, create_fake_model_checkpoints, + create_fake_model_type, create_finetune_models_checkpoints, + create_generation_method, + create_generation_methods_and_video_length, + create_height_width, create_model_checkpoints, + create_model_type, create_prompts, create_samplers, + create_teacache_params, create_ui_outputs) + + +class Wan2_2_Fun_Controller(Fun_Controller): + def update_diffusion_transformer(self, diffusion_transformer_dropdown): + print(f"Update diffusion transformer: {diffusion_transformer_dropdown}") + self.model_name = diffusion_transformer_dropdown + self.diffusion_transformer_dropdown = diffusion_transformer_dropdown + if diffusion_transformer_dropdown == "none": + return gr.update() + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[self.config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + self.vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(self.config['vae_kwargs']), + ).to(self.weight_dtype) + + # Get Transformer + self.transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(self.config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=self.weight_dtype, + ) + if self.config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + self.transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(self.config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=self.weight_dtype, + ) + else: + self.transformer_2 = None + + # Get Tokenizer + self.tokenizer = AutoTokenizer.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + # Get Text encoder + self.text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(self.config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=self.weight_dtype, + ) + self.text_encoder = self.text_encoder.eval() + + Chosen_Scheduler = self.scheduler_dict[list(self.scheduler_dict.keys())[0]] + self.scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(self.config['scheduler_kwargs'])) + ) + + # Get pipeline + if self.model_type == "Inpaint": + if self.transformer.config.in_channels != self.vae.config.latent_channels: + self.pipeline = Wan2_2FunInpaintPipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + transformer_2=self.transformer_2, + scheduler=self.scheduler, + ) + else: + self.pipeline = Wan2_2FunPipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + transformer_2=self.transformer_2, + scheduler=self.scheduler, + ) + else: + self.pipeline = Wan2_2FunControlPipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + transformer_2=self.transformer_2, + scheduler=self.scheduler, + ) + + if self.ulysses_degree > 1 or self.ring_degree > 1: + from functools import partial + self.transformer.enable_multi_gpus_inference() + if self.transformer_2 is not None: + self.transformer_2.enable_multi_gpus_inference() + if self.fsdp_dit: + shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) + self.pipeline.transformer = shard_fn(self.pipeline.transformer) + if self.transformer_2 is not None: + self.pipeline.transformer_2 = shard_fn(self.pipeline.transformer_2) + print("Add FSDP DIT") + if self.fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) + self.pipeline.text_encoder = shard_fn(self.pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + + if self.compile_dit: + for i in range(len(self.pipeline.transformer.blocks)): + self.pipeline.transformer.blocks[i] = torch.compile(self.pipeline.transformer.blocks[i]) + if self.transformer_2 is not None: + for i in range(len(self.pipeline.transformer_2.blocks)): + self.pipeline.transformer_2.blocks[i] = torch.compile(self.pipeline.transformer_2.blocks[i]) + print("Add Compile") + + if self.GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(self.transformer, ["modulation",], device=self.device) + self.transformer.freqs = self.transformer.freqs.to(device=self.device) + if self.transformer_2 is not None: + replace_parameters_by_name(self.transformer_2, ["modulation",], device=self.device) + self.transformer_2.freqs = self.transformer_2.freqs.to(device=self.device) + self.pipeline.enable_sequential_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(self.transformer, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer, self.weight_dtype) + if self.transformer_2 is not None: + convert_model_weight_to_float8(self.transformer_2, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer_2, self.weight_dtype) + self.pipeline.enable_model_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_cpu_offload": + self.pipeline.enable_model_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(self.transformer, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer, self.weight_dtype) + if self.transformer_2 is not None: + convert_model_weight_to_float8(self.transformer_2, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer_2, self.weight_dtype) + self.pipeline.to(self.device) + else: + self.pipeline.to(self.device) + print("Update diffusion transformer done") + return gr.update() + + @timer + def generate( + self, + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image = None, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + base_model_2_dropdown=None, + lora_model_2_dropdown=None, + fps = None, + is_api = False, + ): + self.clear_cache() + + print(f"Input checking.") + _, comment = self.input_check( + resize_method, generation_method, start_image, end_image, validation_video,control_video, is_api + ) + print(f"Input checking down") + if comment != "OK": + return "", comment + is_image = True if generation_method == "Image Generation" else False + + if self.base_model_path != base_model_dropdown: + self.update_base_model(base_model_dropdown) + if self.base_model_2_path != base_model_2_dropdown: + self.update_lora_model(base_model_2_dropdown, is_checkpoint_2=True) + + if self.lora_model_path != lora_model_dropdown: + self.update_lora_model(lora_model_dropdown) + if self.lora_model_2_path != lora_model_2_dropdown: + self.update_lora_model(lora_model_2_dropdown, is_checkpoint_2=True) + + print(f"Load scheduler.") + scheduler_config = self.pipeline.scheduler.config + if sampler_dropdown == "Flow_Unipc" or sampler_dropdown == "Flow_DPM++": + scheduler_config['shift'] = 1 + self.pipeline.scheduler = self.scheduler_dict[sampler_dropdown].from_config(scheduler_config) + print(f"Load scheduler down.") + + if resize_method == "Resize according to Reference": + print(f"Calculate height and width according to Reference.") + height_slider, width_slider = self.get_height_width_from_reference( + base_resolution, start_image, validation_video, control_video, + ) + + if self.lora_model_path != "none": + print(f"Merge Lora.") + self.pipeline = merge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + if self.transformer_2 is not None: + self.pipeline = merge_lora(self.pipeline, self.lora_model_2_path, multiplier=lora_alpha_slider, sub_transformer_name="transformer_2") + print(f"Merge Lora done.") + + coefficients = get_teacache_coefficients(self.diffusion_transformer_dropdown) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + self.pipeline.transformer.enable_teacache( + coefficients, sample_step_slider, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if self.transformer_2 is not None: + self.pipeline.transformer_2.share_teacache(self.pipeline.transformer) + else: + print(f"Disable TeaCache.") + self.pipeline.transformer.disable_teacache() + if self.transformer_2 is not None: + self.pipeline.transformer_2.disable_teacache() + + if cfg_skip_ratio is not None and cfg_skip_ratio >= 0: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + self.pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, sample_step_slider) + if self.transformer_2 is not None: + self.pipeline.transformer_2.share_cfg_skip(self.pipeline.transformer) + + print(f"Generate seed.") + if int(seed_textbox) != -1 and seed_textbox != "": torch.manual_seed(int(seed_textbox)) + else: seed_textbox = np.random.randint(0, 1e10) + generator = torch.Generator(device=self.device).manual_seed(int(seed_textbox)) + print(f"Generate seed done.") + + if fps is None: + fps = 16 + boundary = self.config['transformer_additional_kwargs'].get('boundary', 0.875) + + if enable_riflex: + print(f"Enable riflex") + latent_frames = (int(length_slider) - 1) // self.vae.config.temporal_compression_ratio + 1 + self.pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames if not is_image else 1) + if self.transformer_2 is not None: + self.pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames if not is_image else 1) + + try: + print(f"Generation.") + if self.model_type == "Inpaint": + if self.transformer.config.in_channels != self.vae.config.latent_channels: + if validation_video is not None: + input_video, input_video_mask, _, clip_image = get_video_to_video_latent(validation_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), validation_video_mask=validation_video_mask, fps=fps) + else: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + boundary = boundary + ).videos + else: + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + boundary = boundary + ).videos + else: + inpaint_video, inpaint_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, video_length=length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=(height_slider, width_slider)) + + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=length_slider if not is_image else 1, sample_size=(height_slider, width_slider), fps=fps, ref_image=None) + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + + video = inpaint_video, + mask_video = inpaint_video_mask, + control_video = input_video, + ref_image = ref_image, + boundary = boundary, + ).videos + print(f"Generation done.") + except Exception as e: + self.auto_model_clear_cache(self.pipeline.transformer) + self.auto_model_clear_cache(self.pipeline.text_encoder) + self.auto_model_clear_cache(self.pipeline.vae) + self.clear_cache() + + print(f"Error. error information is {str(e)}") + if self.lora_model_path != "none": + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + if is_api: + return "", f"Error. error information is {str(e)}" + else: + return gr.update(), gr.update(), f"Error. error information is {str(e)}" + + self.clear_cache() + # lora part + if self.lora_model_path != "none": + print(f"Unmerge Lora.") + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + print(f"Unmerge Lora done.") + + print(f"Saving outputs.") + save_sample_path = self.save_outputs( + is_image, length_slider, sample, fps=fps + ) + print(f"Saving outputs done.") + + if is_image or length_slider == 1: + if is_api: + return save_sample_path, "Success" + else: + if gradio_version_is_above_4: + return gr.Image(value=save_sample_path, visible=True), gr.Video(value=None, visible=False), "Success" + else: + return gr.Image.update(value=save_sample_path, visible=True), gr.Video.update(value=None, visible=False), "Success" + else: + if is_api: + return save_sample_path, "Success" + else: + if gradio_version_is_above_4: + return gr.Image(visible=False, value=None), gr.Video(value=save_sample_path, visible=True), "Success" + else: + return gr.Image.update(visible=False, value=None), gr.Video.update(value=save_sample_path, visible=True), "Success" + +Wan2_2_Fun_Controller_Host = Wan2_2_Fun_Controller +Wan2_2_Fun_Controller_Client = Fun_Controller_Client + +def ui(GPU_memory_mode, scheduler_dict, config_path, compile_dit, weight_dtype, savedir_sample=None): + controller = Wan2_2_Fun_Controller( + GPU_memory_mode, scheduler_dict, model_name=None, model_type="Inpaint", + config_path=config_path, compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan2.2-Fun: + + A Wan with more flexible generation conditions, capable of producing videos of different resolutions, around 5 seconds, and fps 16 (frames 1 to 81), as well as image generated videos. + + [Github](https://github.com/aigc-apps/VideoX-Fun/) + """ + ) + with gr.Column(variant="panel"): + config_dropdown, config_refresh_button = create_config(controller) + model_type = create_model_type(visible=True) + diffusion_transformer_dropdown, diffusion_transformer_refresh_button = \ + create_model_checkpoints(controller, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider, personalized_refresh_button = \ + create_finetune_models_checkpoints(controller, visible=True, add_checkpoint_2=True) + base_model_dropdown, base_model_2_dropdown = base_model_dropdown + lora_model_dropdown, lora_model_2_dropdown = lora_model_dropdown + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller) + + resize_method, width_slider, height_slider, base_resolution = create_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video Control (视频控制)"], prompt_textbox, support_ref_image=True + ) + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + config_dropdown.change( + fn=controller.update_config, + inputs=[config_dropdown], + outputs=[] + ) + + model_type.change( + fn=controller.update_model_type, + inputs=[model_type], + outputs=[] + ) + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return [gr.update(visible=True, maximum=161, value=81, interactive=True), gr.update(visible=False), gr.update(visible=False)] + elif generation_method == "Image Generation": + return [gr.update(minimum=1, maximum=1, value=1, interactive=False), gr.update(visible=False), gr.update(visible=False)] + else: + return [gr.update(visible=True, maximum=1344), gr.update(visible=True), gr.update(visible=True)] + generation_method.change( + upload_generation_method, generation_method, [length_slider, overlap_video_length, partial_video_length] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + base_model_2_dropdown, + lora_model_2_dropdown + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller + +def ui_host(GPU_memory_mode, scheduler_dict, model_name, model_type, config_path, compile_dit, weight_dtype, savedir_sample=None): + controller = Wan2_2_Fun_Controller_Host( + GPU_memory_mode, scheduler_dict, model_name=model_name, model_type=model_type, + config_path=config_path, compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan2.2-Fun: + + A Wan with more flexible generation conditions, capable of producing videos of different resolutions, around 5 seconds, and fps 16 (frames 1 to 81), as well as image generated videos. + + [Github](https://github.com/aigc-apps/VideoX-Fun/) + """ + ) + with gr.Column(variant="panel"): + model_type = create_fake_model_type(visible=False) + diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider = \ + create_fake_finetune_models_checkpoints(visible=True, add_checkpoint_2=True) + base_model_dropdown, base_model_2_dropdown = base_model_dropdown + lora_model_dropdown, lora_model_2_dropdown = lora_model_dropdown + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller) + + resize_method, width_slider, height_slider, base_resolution = create_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video Control (视频控制)"], prompt_textbox, support_ref_image=True + ) + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return gr.update(visible=True, minimum=1, maximum=161, value=81, interactive=True) + elif generation_method == "Image Generation": + return gr.update(minimum=1, maximum=1, value=1, interactive=False) + generation_method.change( + upload_generation_method, generation_method, [length_slider] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + base_model_2_dropdown, + lora_model_2_dropdown + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller + +def ui_client(scheduler_dict, model_name, savedir_sample=None): + controller = Wan2_2_Fun_Controller_Client(scheduler_dict, savedir_sample) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan2.2-Fun: + + A Wan with more flexible generation conditions, capable of producing videos of different resolutions, around 5 seconds, and fps 16 (frames 1 to 81), as well as image generated videos. + + [Github](https://github.com/aigc-apps/VideoX-Fun/) + """ + ) + with gr.Column(variant="panel"): + diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider = \ + create_fake_finetune_models_checkpoints(visible=True, add_checkpoint_2=True) + base_model_dropdown, base_model_2_dropdown = base_model_dropdown + lora_model_dropdown, lora_model_2_dropdown = lora_model_dropdown + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller, maximum_step=50) + + resize_method, width_slider, height_slider, base_resolution = create_fake_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox + ) + + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return gr.update(visible=True, minimum=5, maximum=161, value=49, interactive=True) + elif generation_method == "Image Generation": + return gr.update(minimum=1, maximum=1, value=1, interactive=False) + generation_method.change( + upload_generation_method, generation_method, [length_slider] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(), gr.update()] + source_method.change( + upload_source_method, source_method, [image_to_video_col, video_to_video_col, start_image, end_image, validation_video, validation_video_mask] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + base_model_2_dropdown, + lora_model_2_dropdown + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/ui/wan2_2_ui.py b/VideoX-Fun/videox_fun/ui/wan2_2_ui.py new file mode 100644 index 0000000000000000000000000000000000000000..4fcf81c7960a2cdd2e1b1f5c8605fea8a58dfc9d --- /dev/null +++ b/VideoX-Fun/videox_fun/ui/wan2_2_ui.py @@ -0,0 +1,797 @@ +"""Modified from https://github.com/guoyww/AnimateDiff/blob/main/app.py +""" +import os +import random + +import cv2 +import gradio as gr +import numpy as np +import torch +from omegaconf import OmegaConf +from PIL import Image +from safetensors import safe_open + +from ..data.bucket_sampler import ASPECT_RATIO_512, get_closest_ratio +from ..models import (AutoencoderKLWan, AutoencoderKLWan3_8, AutoTokenizer, CLIPModel, + WanT5EncoderModel, Wan2_2Transformer3DModel) +from ..models.cache_utils import get_teacache_coefficients +from ..pipeline import Wan2_2I2VPipeline, Wan2_2Pipeline, Wan2_2TI2VPipeline +from ..utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from ..utils.lora_utils import merge_lora, unmerge_lora +from ..utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, timer, + get_video_to_video_latent, save_videos_grid) +from .controller import (Fun_Controller, Fun_Controller_Client, + all_cheduler_dict, css, ddpm_scheduler_dict, + flow_scheduler_dict, gradio_version, + gradio_version_is_above_4) +from .ui import (create_cfg_and_seedbox, create_cfg_riflex_k, + create_cfg_skip_params, + create_fake_finetune_models_checkpoints, + create_fake_height_width, create_fake_model_checkpoints, + create_fake_model_type, create_finetune_models_checkpoints, + create_generation_method, + create_generation_methods_and_video_length, + create_height_width, create_model_checkpoints, + create_model_type, create_prompts, create_samplers, + create_teacache_params, create_ui_outputs, create_config) +from ..dist import set_multi_gpus_devices, shard_model + + +class Wan2_2_Controller(Fun_Controller): + def update_diffusion_transformer(self, diffusion_transformer_dropdown): + print(f"Update diffusion transformer: {diffusion_transformer_dropdown}") + self.model_name = diffusion_transformer_dropdown + self.diffusion_transformer_dropdown = diffusion_transformer_dropdown + if diffusion_transformer_dropdown == "none": + return gr.update() + Chosen_AutoencoderKL = { + "AutoencoderKLWan": AutoencoderKLWan, + "AutoencoderKLWan3_8": AutoencoderKLWan3_8 + }[self.config['vae_kwargs'].get('vae_type', 'AutoencoderKLWan')] + self.vae = Chosen_AutoencoderKL.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(self.config['vae_kwargs']), + ).to(self.weight_dtype) + + # Get Transformer + self.transformer = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['transformer_additional_kwargs'].get('transformer_low_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(self.config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=self.weight_dtype, + ) + if self.config['transformer_additional_kwargs'].get('transformer_combination_type', 'single') == "moe": + self.transformer_2 = Wan2_2Transformer3DModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['transformer_additional_kwargs'].get('transformer_high_noise_model_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(self.config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=self.weight_dtype, + ) + else: + self.transformer_2 = None + + # Get Tokenizer + self.tokenizer = AutoTokenizer.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + # Get Text encoder + self.text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(self.config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=self.weight_dtype, + ) + self.text_encoder = self.text_encoder.eval() + + Chosen_Scheduler = self.scheduler_dict[list(self.scheduler_dict.keys())[0]] + self.scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(self.config['scheduler_kwargs'])) + ) + + # Get pipeline + if self.model_type == "Inpaint": + if "wan_civitai_5b" in self.config_path: + self.pipeline = Wan2_2TI2VPipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + transformer_2=self.transformer_2, + scheduler=self.scheduler, + ) + else: + if self.transformer.config.in_channels != self.vae.config.latent_channels: + self.pipeline = Wan2_2I2VPipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + transformer_2=self.transformer_2, + scheduler=self.scheduler, + ) + else: + self.pipeline = Wan2_2Pipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + transformer_2=self.transformer_2, + scheduler=self.scheduler, + ) + else: + raise ValueError("Not support now") + + if self.ulysses_degree > 1 or self.ring_degree > 1: + from functools import partial + self.transformer.enable_multi_gpus_inference() + if self.transformer_2 is not None: + self.transformer_2.enable_multi_gpus_inference() + if self.fsdp_dit: + shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) + self.pipeline.transformer = shard_fn(self.pipeline.transformer) + if self.transformer_2 is not None: + self.pipeline.transformer_2 = shard_fn(self.pipeline.transformer_2) + print("Add FSDP DIT") + if self.fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) + self.pipeline.text_encoder = shard_fn(self.pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + + if self.compile_dit: + for i in range(len(self.pipeline.transformer.blocks)): + self.pipeline.transformer.blocks[i] = torch.compile(self.pipeline.transformer.blocks[i]) + if self.transformer_2 is not None: + for i in range(len(self.pipeline.transformer_2.blocks)): + self.pipeline.transformer_2.blocks[i] = torch.compile(self.pipeline.transformer_2.blocks[i]) + print("Add Compile") + + if self.GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(self.transformer, ["modulation",], device=self.device) + self.transformer.freqs = self.transformer.freqs.to(device=self.device) + if self.transformer_2 is not None: + replace_parameters_by_name(self.transformer_2, ["modulation",], device=self.device) + self.transformer_2.freqs = self.transformer_2.freqs.to(device=self.device) + self.pipeline.enable_sequential_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(self.transformer, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer, self.weight_dtype) + if self.transformer_2 is not None: + convert_model_weight_to_float8(self.transformer_2, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer_2, self.weight_dtype) + self.pipeline.enable_model_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_cpu_offload": + self.pipeline.enable_model_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(self.transformer, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer, self.weight_dtype) + if self.transformer_2 is not None: + convert_model_weight_to_float8(self.transformer_2, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer_2, self.weight_dtype) + self.pipeline.to(self.device) + else: + self.pipeline.to(self.device) + print("Update diffusion transformer done") + return gr.update() + + @timer + def generate( + self, + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image = None, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + base_model_2_dropdown=None, + lora_model_2_dropdown=None, + fps = None, + is_api = False, + ): + self.clear_cache() + + print(f"Input checking.") + _, comment = self.input_check( + resize_method, generation_method, start_image, end_image, validation_video,control_video, is_api + ) + print(f"Input checking down") + if comment != "OK": + return "", comment + is_image = True if generation_method == "Image Generation" else False + + if self.base_model_path != base_model_dropdown: + self.update_base_model(base_model_dropdown) + if self.base_model_2_path != base_model_2_dropdown: + self.update_lora_model(base_model_2_dropdown, is_checkpoint_2=True) + + if self.lora_model_path != lora_model_dropdown: + self.update_lora_model(lora_model_dropdown) + if self.lora_model_2_path != lora_model_2_dropdown: + self.update_lora_model(lora_model_2_dropdown, is_checkpoint_2=True) + + print(f"Load scheduler.") + scheduler_config = self.pipeline.scheduler.config + if sampler_dropdown == "Flow_Unipc" or sampler_dropdown == "Flow_DPM++": + scheduler_config['shift'] = 1 + self.pipeline.scheduler = self.scheduler_dict[sampler_dropdown].from_config(scheduler_config) + print(f"Load scheduler down.") + + if resize_method == "Resize according to Reference": + print(f"Calculate height and width according to Reference.") + height_slider, width_slider = self.get_height_width_from_reference( + base_resolution, start_image, validation_video, control_video, + ) + + if self.lora_model_path != "none": + print(f"Merge Lora.") + self.pipeline = merge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + if self.transformer_2 is not None: + self.pipeline = merge_lora(self.pipeline, self.lora_model_2_path, multiplier=lora_alpha_slider, sub_transformer_name="transformer_2") + print(f"Merge Lora done.") + + coefficients = get_teacache_coefficients(self.diffusion_transformer_dropdown) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + self.pipeline.transformer.enable_teacache( + coefficients, sample_step_slider, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + if self.transformer_2 is not None: + self.pipeline.transformer_2.share_teacache(self.pipeline.transformer) + else: + print(f"Disable TeaCache.") + self.pipeline.transformer.disable_teacache() + if self.transformer_2 is not None: + self.pipeline.transformer_2.disable_teacache() + + if cfg_skip_ratio is not None and cfg_skip_ratio >= 0: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + self.pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, sample_step_slider) + if self.transformer_2 is not None: + self.pipeline.transformer_2.share_cfg_skip(self.pipeline.transformer) + + print(f"Generate seed.") + if int(seed_textbox) != -1 and seed_textbox != "": torch.manual_seed(int(seed_textbox)) + else: seed_textbox = np.random.randint(0, 1e10) + generator = torch.Generator(device=self.device).manual_seed(int(seed_textbox)) + print(f"Generate seed done.") + + if fps is None: + fps = 16 + boundary = self.config['transformer_additional_kwargs'].get('boundary', 0.875) + + if enable_riflex: + print(f"Enable riflex") + latent_frames = (int(length_slider) - 1) // self.vae.config.temporal_compression_ratio + 1 + self.pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames if not is_image else 1) + if self.transformer_2 is not None: + self.pipeline.transformer_2.enable_riflex(k = riflex_k, L_test = latent_frames if not is_image else 1) + + try: + print(f"Generation.") + if self.model_type == "Inpaint": + if self.transformer.config.in_channels != self.vae.config.latent_channels: + if validation_video is not None: + input_video, input_video_mask, _, clip_image = get_video_to_video_latent(validation_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), validation_video_mask=validation_video_mask, fps=fps) + else: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + boundary = boundary + ).videos + else: + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + boundary = boundary + ).videos + else: + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=(height_slider, width_slider)) + + if start_image is not None: + start_image = get_image_latent(start_image, sample_size=(height_slider, width_slider)) + + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=length_slider if not is_image else 1, sample_size=(height_slider, width_slider), fps=fps, ref_image=None) + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + + control_video = input_video, + ref_image = ref_image, + start_image = start_image, + boundary = boundary + ).videos + print(f"Generation done.") + except Exception as e: + self.auto_model_clear_cache(self.pipeline.transformer) + self.auto_model_clear_cache(self.pipeline.text_encoder) + self.auto_model_clear_cache(self.pipeline.vae) + self.clear_cache() + + print(f"Error. error information is {str(e)}") + if self.lora_model_path != "none": + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + if self.transformer_2 is not None: + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_2_path, multiplier=lora_alpha_slider, sub_transformer_name="transformer_2") + if is_api: + return "", f"Error. error information is {str(e)}" + else: + return gr.update(), gr.update(), f"Error. error information is {str(e)}" + + self.clear_cache() + # lora part + if self.lora_model_path != "none": + print(f"Unmerge Lora.") + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + if self.transformer_2 is not None: + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_2_path, multiplier=lora_alpha_slider, sub_transformer_name="transformer_2") + print(f"Unmerge Lora done.") + + print(f"Saving outputs.") + save_sample_path = self.save_outputs( + is_image, length_slider, sample, fps=fps + ) + print(f"Saving outputs done.") + + if is_image or length_slider == 1: + if is_api: + return save_sample_path, "Success" + else: + if gradio_version_is_above_4: + return gr.Image(value=save_sample_path, visible=True), gr.Video(value=None, visible=False), "Success" + else: + return gr.Image.update(value=save_sample_path, visible=True), gr.Video.update(value=None, visible=False), "Success" + else: + if is_api: + return save_sample_path, "Success" + else: + if gradio_version_is_above_4: + return gr.Image(visible=False, value=None), gr.Video(value=save_sample_path, visible=True), "Success" + else: + return gr.Image.update(visible=False, value=None), gr.Video.update(value=save_sample_path, visible=True), "Success" + +Wan2_2_Controller_Host = Wan2_2_Controller +Wan2_2_Controller_Client = Fun_Controller_Client + +def ui(GPU_memory_mode, scheduler_dict, config_path, compile_dit, weight_dtype, savedir_sample=None): + controller = Wan2_2_Controller( + GPU_memory_mode, scheduler_dict, model_name=None, model_type="Inpaint", + config_path=config_path, compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan2_2: + """ + ) + with gr.Column(variant="panel"): + config_dropdown, config_refresh_button = create_config(controller) + model_type = create_model_type(visible=False) + diffusion_transformer_dropdown, diffusion_transformer_refresh_button = \ + create_model_checkpoints(controller, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider, personalized_refresh_button = \ + create_finetune_models_checkpoints(controller, visible=True, add_checkpoint_2=True) + base_model_dropdown, base_model_2_dropdown = base_model_dropdown + lora_model_dropdown, lora_model_2_dropdown = lora_model_dropdown + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller) + + resize_method, width_slider, height_slider, base_resolution = create_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox, support_end_image=False + ) + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + config_dropdown.change( + fn=controller.update_config, + inputs=[config_dropdown], + outputs=[] + ) + + model_type.change( + fn=controller.update_model_type, + inputs=[model_type], + outputs=[] + ) + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return [gr.update(visible=True, maximum=161, value=81, interactive=True), gr.update(visible=False), gr.update(visible=False)] + elif generation_method == "Image Generation": + return [gr.update(minimum=1, maximum=1, value=1, interactive=False), gr.update(visible=False), gr.update(visible=False)] + else: + return [gr.update(visible=True, maximum=1344), gr.update(visible=True), gr.update(visible=True)] + generation_method.change( + upload_generation_method, generation_method, [length_slider, overlap_video_length, partial_video_length] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + base_model_2_dropdown, + lora_model_2_dropdown + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller + +def ui_host(GPU_memory_mode, scheduler_dict, model_name, model_type, config_path, compile_dit, weight_dtype, savedir_sample=None): + controller = Wan2_2_Controller_Host( + GPU_memory_mode, scheduler_dict, model_name=model_name, model_type=model_type, + config_path=config_path, compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan2_2: + """ + ) + with gr.Column(variant="panel"): + model_type = create_fake_model_type(visible=False) + diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider = \ + create_fake_finetune_models_checkpoints(visible=True, add_checkpoint_2=True) + base_model_dropdown, base_model_2_dropdown = base_model_dropdown + lora_model_dropdown, lora_model_2_dropdown = lora_model_dropdown + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller) + + resize_method, width_slider, height_slider, base_resolution = create_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox + ) + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return gr.update(visible=True, minimum=1, maximum=161, value=81, interactive=True) + elif generation_method == "Image Generation": + return gr.update(minimum=1, maximum=1, value=1, interactive=False) + generation_method.change( + upload_generation_method, generation_method, [length_slider] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + base_model_2_dropdown, + lora_model_2_dropdown + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller + +def ui_client(scheduler_dict, model_name, savedir_sample=None): + controller = Wan2_2_Controller_Client(scheduler_dict, savedir_sample) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan2_2: + """ + ) + with gr.Column(variant="panel"): + diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider = \ + create_fake_finetune_models_checkpoints(visible=True, add_checkpoint_2=True) + base_model_dropdown, base_model_2_dropdown = base_model_dropdown + lora_model_dropdown, lora_model_2_dropdown = lora_model_dropdown + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller, maximum_step=50) + + resize_method, width_slider, height_slider, base_resolution = create_fake_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox + ) + + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return gr.update(visible=True, minimum=5, maximum=161, value=49, interactive=True) + elif generation_method == "Image Generation": + return gr.update(minimum=1, maximum=1, value=1, interactive=False) + generation_method.change( + upload_generation_method, generation_method, [length_slider] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(), gr.update()] + source_method.change( + upload_source_method, source_method, [image_to_video_col, video_to_video_col, start_image, end_image, validation_video, validation_video_mask] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + base_model_2_dropdown, + lora_model_2_dropdown + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/ui/wan_fun_ui.py b/VideoX-Fun/videox_fun/ui/wan_fun_ui.py new file mode 100644 index 0000000000000000000000000000000000000000..315bde87c347e46a3e9a2cb674cec1d153e1bc04 --- /dev/null +++ b/VideoX-Fun/videox_fun/ui/wan_fun_ui.py @@ -0,0 +1,752 @@ +"""Modified from https://github.com/guoyww/AnimateDiff/blob/main/app.py +""" +import os +import random + +import cv2 +import gradio as gr +import numpy as np +import torch +from omegaconf import OmegaConf +from PIL import Image +from safetensors import safe_open + +from ..data.bucket_sampler import ASPECT_RATIO_512, get_closest_ratio +from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, WanTransformer3DModel) +from ..models.cache_utils import get_teacache_coefficients +from ..pipeline import (WanFunControlPipeline, WanFunInpaintPipeline, + WanFunPipeline) +from ..utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from ..utils.lora_utils import merge_lora, unmerge_lora +from ..utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, timer, + get_video_to_video_latent, save_videos_grid) +from .controller import (Fun_Controller, Fun_Controller_Client, + all_cheduler_dict, css, ddpm_scheduler_dict, + flow_scheduler_dict, gradio_version, + gradio_version_is_above_4) +from .ui import (create_cfg_and_seedbox, create_cfg_riflex_k, + create_cfg_skip_params, + create_fake_finetune_models_checkpoints, + create_fake_height_width, create_fake_model_checkpoints, + create_fake_model_type, create_finetune_models_checkpoints, + create_generation_method, + create_generation_methods_and_video_length, + create_height_width, create_model_checkpoints, + create_model_type, create_prompts, create_samplers, + create_teacache_params, create_ui_outputs) +from ..dist import set_multi_gpus_devices, shard_model + + +class Wan_Fun_Controller(Fun_Controller): + def update_diffusion_transformer(self, diffusion_transformer_dropdown): + print(f"Update diffusion transformer: {diffusion_transformer_dropdown}") + self.model_name = diffusion_transformer_dropdown + self.diffusion_transformer_dropdown = diffusion_transformer_dropdown + if diffusion_transformer_dropdown == "none": + return gr.update() + self.vae = AutoencoderKLWan.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(self.config['vae_kwargs']), + ).to(self.weight_dtype) + + # Get Transformer + self.transformer = WanTransformer3DModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(self.config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=self.weight_dtype, + ) + + # Get Tokenizer + self.tokenizer = AutoTokenizer.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + # Get Text encoder + self.text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(self.config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=self.weight_dtype, + ) + self.text_encoder = self.text_encoder.eval() + + if self.transformer.config.in_channels != self.vae.config.latent_channels: + # Get Clip Image Encoder + self.clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ).to(self.weight_dtype) + self.clip_image_encoder = self.clip_image_encoder.eval() + else: + self.clip_image_encoder = None + + Chosen_Scheduler = self.scheduler_dict[list(self.scheduler_dict.keys())[0]] + self.scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(self.config['scheduler_kwargs'])) + ) + + # Get pipeline + if self.model_type == "Inpaint": + if self.transformer.config.in_channels != self.vae.config.latent_channels: + self.pipeline = WanFunInpaintPipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + scheduler=self.scheduler, + clip_image_encoder=self.clip_image_encoder, + ) + else: + self.pipeline = WanFunPipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + scheduler=self.scheduler, + ) + else: + self.pipeline = WanFunControlPipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + scheduler=self.scheduler, + clip_image_encoder=self.clip_image_encoder, + ) + + if self.ulysses_degree > 1 or self.ring_degree > 1: + from functools import partial + self.transformer.enable_multi_gpus_inference() + if self.fsdp_dit: + shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) + self.pipeline.transformer = shard_fn(self.pipeline.transformer) + print("Add FSDP DIT") + if self.fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) + self.pipeline.text_encoder = shard_fn(self.pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + + if self.compile_dit: + for i in range(len(self.pipeline.transformer.blocks)): + self.pipeline.transformer.blocks[i] = torch.compile(self.pipeline.transformer.blocks[i]) + print("Add Compile") + + if self.GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(self.transformer, ["modulation",], device=self.device) + self.transformer.freqs = self.transformer.freqs.to(device=self.device) + self.pipeline.enable_sequential_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(self.transformer, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer, self.weight_dtype) + self.pipeline.enable_model_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_cpu_offload": + self.pipeline.enable_model_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(self.transformer, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer, self.weight_dtype) + self.pipeline.to(self.device) + else: + self.pipeline.to(self.device) + print("Update diffusion transformer done") + return gr.update() + + @timer + def generate( + self, + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image = None, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + base_model_2_dropdown=None, + lora_model_2_dropdown=None, + fps = None, + is_api = False, + ): + self.clear_cache() + + print(f"Input checking.") + _, comment = self.input_check( + resize_method, generation_method, start_image, end_image, validation_video,control_video, is_api + ) + print(f"Input checking down") + if comment != "OK": + return "", comment + is_image = True if generation_method == "Image Generation" else False + + if self.base_model_path != base_model_dropdown: + self.update_base_model(base_model_dropdown) + + if self.lora_model_path != lora_model_dropdown: + self.update_lora_model(lora_model_dropdown) + + print(f"Load scheduler.") + scheduler_config = self.pipeline.scheduler.config + if sampler_dropdown == "Flow_Unipc" or sampler_dropdown == "Flow_DPM++": + scheduler_config['shift'] = 1 + self.pipeline.scheduler = self.scheduler_dict[sampler_dropdown].from_config(scheduler_config) + print(f"Load scheduler down.") + + if resize_method == "Resize according to Reference": + print(f"Calculate height and width according to Reference.") + height_slider, width_slider = self.get_height_width_from_reference( + base_resolution, start_image, validation_video, control_video, + ) + + if self.lora_model_path != "none": + print(f"Merge Lora.") + self.pipeline = merge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + print(f"Merge Lora done.") + + coefficients = get_teacache_coefficients(self.diffusion_transformer_dropdown) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + self.pipeline.transformer.enable_teacache( + coefficients, sample_step_slider, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + else: + print(f"Disable TeaCache.") + self.pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None and cfg_skip_ratio >= 0: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + self.pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, sample_step_slider) + + print(f"Generate seed.") + if int(seed_textbox) != -1 and seed_textbox != "": torch.manual_seed(int(seed_textbox)) + else: seed_textbox = np.random.randint(0, 1e10) + generator = torch.Generator(device=self.device).manual_seed(int(seed_textbox)) + print(f"Generate seed done.") + + if fps is None: + fps = 16 + + if enable_riflex: + print(f"Enable riflex") + latent_frames = (int(length_slider) - 1) // self.vae.config.temporal_compression_ratio + 1 + self.pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames if not is_image else 1) + + try: + print(f"Generation.") + if self.model_type == "Inpaint": + if self.transformer.config.in_channels != self.vae.config.latent_channels: + if validation_video is not None: + input_video, input_video_mask, _, clip_image = get_video_to_video_latent(validation_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), validation_video_mask=validation_video_mask, fps=fps) + else: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + ).videos + else: + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + ).videos + else: + if ref_image is not None: + clip_image = Image.open(ref_image).convert("RGB") + elif start_image is not None: + clip_image = Image.open(start_image).convert("RGB") + else: + clip_image = None + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=(height_slider, width_slider)) + + if start_image is not None: + start_image = get_image_latent(start_image, sample_size=(height_slider, width_slider)) + + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=length_slider if not is_image else 1, sample_size=(height_slider, width_slider), fps=fps, ref_image=None) + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + + control_video = input_video, + ref_image = ref_image, + start_image = start_image, + clip_image = clip_image, + ).videos + print(f"Generation done.") + except Exception as e: + self.auto_model_clear_cache(self.pipeline.transformer) + self.auto_model_clear_cache(self.pipeline.text_encoder) + self.auto_model_clear_cache(self.pipeline.vae) + self.clear_cache() + + print(f"Error. error information is {str(e)}") + if self.lora_model_path != "none": + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + if is_api: + return "", f"Error. error information is {str(e)}" + else: + return gr.update(), gr.update(), f"Error. error information is {str(e)}" + + self.clear_cache() + # lora part + if self.lora_model_path != "none": + print(f"Unmerge Lora.") + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + print(f"Unmerge Lora done.") + + print(f"Saving outputs.") + save_sample_path = self.save_outputs( + is_image, length_slider, sample, fps=fps + ) + print(f"Saving outputs done.") + + if is_image or length_slider == 1: + if is_api: + return save_sample_path, "Success" + else: + if gradio_version_is_above_4: + return gr.Image(value=save_sample_path, visible=True), gr.Video(value=None, visible=False), "Success" + else: + return gr.Image.update(value=save_sample_path, visible=True), gr.Video.update(value=None, visible=False), "Success" + else: + if is_api: + return save_sample_path, "Success" + else: + if gradio_version_is_above_4: + return gr.Image(visible=False, value=None), gr.Video(value=save_sample_path, visible=True), "Success" + else: + return gr.Image.update(visible=False, value=None), gr.Video.update(value=save_sample_path, visible=True), "Success" + +Wan_Fun_Controller_Host = Wan_Fun_Controller +Wan_Fun_Controller_Client = Fun_Controller_Client + +def ui(GPU_memory_mode, scheduler_dict, config_path, compile_dit, weight_dtype, savedir_sample=None): + controller = Wan_Fun_Controller( + GPU_memory_mode, scheduler_dict, model_name=None, model_type="Inpaint", + config_path=config_path, compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan-Fun: + + A Wan with more flexible generation conditions, capable of producing videos of different resolutions, around 5 seconds, and fps 16 (frames 1 to 81), as well as image generated videos. + + [Github](https://github.com/aigc-apps/CogVideoX-Fun/) + """ + ) + with gr.Column(variant="panel"): + model_type = create_model_type(visible=True) + diffusion_transformer_dropdown, diffusion_transformer_refresh_button = \ + create_model_checkpoints(controller, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider, personalized_refresh_button = \ + create_finetune_models_checkpoints(controller, visible=True) + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller) + + resize_method, width_slider, height_slider, base_resolution = create_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video Control (视频控制)"], prompt_textbox, support_ref_image=True + ) + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + model_type.change( + fn=controller.update_model_type, + inputs=[model_type], + outputs=[] + ) + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return [gr.update(visible=True, maximum=161, value=81, interactive=True), gr.update(visible=False), gr.update(visible=False)] + elif generation_method == "Image Generation": + return [gr.update(minimum=1, maximum=1, value=1, interactive=False), gr.update(visible=False), gr.update(visible=False)] + else: + return [gr.update(visible=True, maximum=1344), gr.update(visible=True), gr.update(visible=True)] + generation_method.change( + upload_generation_method, generation_method, [length_slider, overlap_video_length, partial_video_length] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller + +def ui_host(GPU_memory_mode, scheduler_dict, model_name, model_type, config_path, compile_dit, weight_dtype, savedir_sample=None): + controller = Wan_Fun_Controller_Host( + GPU_memory_mode, scheduler_dict, model_name=model_name, model_type=model_type, + config_path=config_path, compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan-Fun: + + A Wan with more flexible generation conditions, capable of producing videos of different resolutions, around 5 seconds, and fps 16 (frames 1 to 81), as well as image generated videos. + + [Github](https://github.com/aigc-apps/CogVideoX-Fun/) + """ + ) + with gr.Column(variant="panel"): + model_type = create_fake_model_type(visible=False) + diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider = create_fake_finetune_models_checkpoints(visible=True) + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller) + + resize_method, width_slider, height_slider, base_resolution = create_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)", "Video Control (视频控制)"], prompt_textbox, support_ref_image=True + ) + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return gr.update(visible=True, minimum=1, maximum=161, value=81, interactive=True) + elif generation_method == "Image Generation": + return gr.update(minimum=1, maximum=1, value=1, interactive=False) + generation_method.change( + upload_generation_method, generation_method, [length_slider] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller + +def ui_client(scheduler_dict, model_name, savedir_sample=None): + controller = Wan_Fun_Controller_Client(scheduler_dict, savedir_sample) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan-Fun: + + A Wan with more flexible generation conditions, capable of producing videos of different resolutions, around 5 seconds, and fps 16 (frames 1 to 81), as well as image generated videos. + + [Github](https://github.com/aigc-apps/CogVideoX-Fun/) + """ + ) + with gr.Column(variant="panel"): + diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider = create_fake_finetune_models_checkpoints(visible=True) + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller, maximum_step=50) + + resize_method, width_slider, height_slider, base_resolution = create_fake_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox + ) + + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return gr.update(visible=True, minimum=5, maximum=161, value=49, interactive=True) + elif generation_method == "Image Generation": + return gr.update(minimum=1, maximum=1, value=1, interactive=False) + generation_method.change( + upload_generation_method, generation_method, [length_slider] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(), gr.update()] + source_method.change( + upload_source_method, source_method, [image_to_video_col, video_to_video_col, start_image, end_image, validation_video, validation_video_mask] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/ui/wan_ui.py b/VideoX-Fun/videox_fun/ui/wan_ui.py new file mode 100644 index 0000000000000000000000000000000000000000..5e1e89d06af066dc777f75791d7ccf1142d1a019 --- /dev/null +++ b/VideoX-Fun/videox_fun/ui/wan_ui.py @@ -0,0 +1,732 @@ +"""Modified from https://github.com/guoyww/AnimateDiff/blob/main/app.py +""" +import os +import random + +import cv2 +import gradio as gr +import numpy as np +import torch +from omegaconf import OmegaConf +from PIL import Image +from safetensors import safe_open + +from ..data.bucket_sampler import ASPECT_RATIO_512, get_closest_ratio +from ..models import (AutoencoderKLWan, AutoTokenizer, CLIPModel, + WanT5EncoderModel, WanTransformer3DModel) +from ..models.cache_utils import get_teacache_coefficients +from ..pipeline import WanI2VPipeline, WanPipeline +from ..utils.fp8_optimization import (convert_model_weight_to_float8, + convert_weight_dtype_wrapper, + replace_parameters_by_name) +from ..utils.lora_utils import merge_lora, unmerge_lora +from ..utils.utils import (filter_kwargs, get_image_to_video_latent, get_image_latent, timer, + get_video_to_video_latent, save_videos_grid) +from .controller import (Fun_Controller, Fun_Controller_Client, + all_cheduler_dict, css, ddpm_scheduler_dict, + flow_scheduler_dict, gradio_version, + gradio_version_is_above_4) +from .ui import (create_cfg_and_seedbox, create_cfg_riflex_k, + create_cfg_skip_params, + create_fake_finetune_models_checkpoints, + create_fake_height_width, create_fake_model_checkpoints, + create_fake_model_type, create_finetune_models_checkpoints, + create_generation_method, + create_generation_methods_and_video_length, + create_height_width, create_model_checkpoints, + create_model_type, create_prompts, create_samplers, + create_teacache_params, create_ui_outputs) +from ..dist import set_multi_gpus_devices, shard_model + + +class Wan_Controller(Fun_Controller): + def update_diffusion_transformer(self, diffusion_transformer_dropdown): + print(f"Update diffusion transformer: {diffusion_transformer_dropdown}") + self.model_name = diffusion_transformer_dropdown + self.diffusion_transformer_dropdown = diffusion_transformer_dropdown + if diffusion_transformer_dropdown == "none": + return gr.update() + self.vae = AutoencoderKLWan.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['vae_kwargs'].get('vae_subpath', 'vae')), + additional_kwargs=OmegaConf.to_container(self.config['vae_kwargs']), + ).to(self.weight_dtype) + + # Get Transformer + self.transformer = WanTransformer3DModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['transformer_additional_kwargs'].get('transformer_subpath', 'transformer')), + transformer_additional_kwargs=OmegaConf.to_container(self.config['transformer_additional_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=self.weight_dtype, + ) + + # Get Tokenizer + self.tokenizer = AutoTokenizer.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['text_encoder_kwargs'].get('tokenizer_subpath', 'tokenizer')), + ) + + # Get Text encoder + self.text_encoder = WanT5EncoderModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['text_encoder_kwargs'].get('text_encoder_subpath', 'text_encoder')), + additional_kwargs=OmegaConf.to_container(self.config['text_encoder_kwargs']), + low_cpu_mem_usage=True, + torch_dtype=self.weight_dtype, + ) + self.text_encoder = self.text_encoder.eval() + + if self.transformer.config.in_channels != self.vae.config.latent_channels: + # Get Clip Image Encoder + self.clip_image_encoder = CLIPModel.from_pretrained( + os.path.join(diffusion_transformer_dropdown, self.config['image_encoder_kwargs'].get('image_encoder_subpath', 'image_encoder')), + ).to(self.weight_dtype) + self.clip_image_encoder = self.clip_image_encoder.eval() + else: + self.clip_image_encoder = None + + Chosen_Scheduler = self.scheduler_dict[list(self.scheduler_dict.keys())[0]] + self.scheduler = Chosen_Scheduler( + **filter_kwargs(Chosen_Scheduler, OmegaConf.to_container(self.config['scheduler_kwargs'])) + ) + + # Get pipeline + if self.model_type == "Inpaint": + if self.transformer.config.in_channels != self.vae.config.latent_channels: + self.pipeline = WanI2VPipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + scheduler=self.scheduler, + clip_image_encoder=self.clip_image_encoder, + ) + else: + self.pipeline = WanPipeline( + vae=self.vae, + tokenizer=self.tokenizer, + text_encoder=self.text_encoder, + transformer=self.transformer, + scheduler=self.scheduler, + ) + else: + raise ValueError("Not support now") + + if self.ulysses_degree > 1 or self.ring_degree > 1: + from functools import partial + self.transformer.enable_multi_gpus_inference() + if self.fsdp_dit: + shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) + self.pipeline.transformer = shard_fn(self.pipeline.transformer) + print("Add FSDP DIT") + if self.fsdp_text_encoder: + shard_fn = partial(shard_model, device_id=self.device, param_dtype=self.weight_dtype) + self.pipeline.text_encoder = shard_fn(self.pipeline.text_encoder) + print("Add FSDP TEXT ENCODER") + + if self.compile_dit: + for i in range(len(self.pipeline.transformer.blocks)): + self.pipeline.transformer.blocks[i] = torch.compile(self.pipeline.transformer.blocks[i]) + print("Add Compile") + + if self.GPU_memory_mode == "sequential_cpu_offload": + replace_parameters_by_name(self.transformer, ["modulation",], device=self.device) + self.transformer.freqs = self.transformer.freqs.to(device=self.device) + self.pipeline.enable_sequential_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_cpu_offload_and_qfloat8": + convert_model_weight_to_float8(self.transformer, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer, self.weight_dtype) + self.pipeline.enable_model_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_cpu_offload": + self.pipeline.enable_model_cpu_offload(device=self.device) + elif self.GPU_memory_mode == "model_full_load_and_qfloat8": + convert_model_weight_to_float8(self.transformer, exclude_module_name=["modulation",], device=self.device) + convert_weight_dtype_wrapper(self.transformer, self.weight_dtype) + self.pipeline.to(self.device) + else: + self.pipeline.to(self.device) + print("Update diffusion transformer done") + return gr.update() + + @timer + def generate( + self, + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image = None, + enable_teacache = None, + teacache_threshold = None, + num_skip_start_steps = None, + teacache_offload = None, + cfg_skip_ratio = None, + enable_riflex = None, + riflex_k = None, + base_model_2_dropdown=None, + lora_model_2_dropdown=None, + fps = None, + is_api = False, + ): + self.clear_cache() + + print(f"Input checking.") + _, comment = self.input_check( + resize_method, generation_method, start_image, end_image, validation_video,control_video, is_api + ) + print(f"Input checking down") + if comment != "OK": + return "", comment + is_image = True if generation_method == "Image Generation" else False + + if self.base_model_path != base_model_dropdown: + self.update_base_model(base_model_dropdown) + + if self.lora_model_path != lora_model_dropdown: + self.update_lora_model(lora_model_dropdown) + + print(f"Load scheduler.") + scheduler_config = self.pipeline.scheduler.config + if sampler_dropdown == "Flow_Unipc" or sampler_dropdown == "Flow_DPM++": + scheduler_config['shift'] = 1 + self.pipeline.scheduler = self.scheduler_dict[sampler_dropdown].from_config(scheduler_config) + print(f"Load scheduler down.") + + if resize_method == "Resize according to Reference": + print(f"Calculate height and width according to Reference.") + height_slider, width_slider = self.get_height_width_from_reference( + base_resolution, start_image, validation_video, control_video, + ) + + if self.lora_model_path != "none": + print(f"Merge Lora.") + self.pipeline = merge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + print(f"Merge Lora done.") + + coefficients = get_teacache_coefficients(self.diffusion_transformer_dropdown) if enable_teacache else None + if coefficients is not None: + print(f"Enable TeaCache with threshold {teacache_threshold} and skip the first {num_skip_start_steps} steps.") + self.pipeline.transformer.enable_teacache( + coefficients, sample_step_slider, teacache_threshold, num_skip_start_steps=num_skip_start_steps, offload=teacache_offload + ) + else: + print(f"Disable TeaCache.") + self.pipeline.transformer.disable_teacache() + + if cfg_skip_ratio is not None and cfg_skip_ratio >= 0: + print(f"Enable cfg_skip_ratio {cfg_skip_ratio}.") + self.pipeline.transformer.enable_cfg_skip(cfg_skip_ratio, sample_step_slider) + + print(f"Generate seed.") + if int(seed_textbox) != -1 and seed_textbox != "": torch.manual_seed(int(seed_textbox)) + else: seed_textbox = np.random.randint(0, 1e10) + generator = torch.Generator(device=self.device).manual_seed(int(seed_textbox)) + print(f"Generate seed done.") + + if fps is None: + fps = 16 + + if enable_riflex: + print(f"Enable riflex") + latent_frames = (int(length_slider) - 1) // self.vae.config.temporal_compression_ratio + 1 + self.pipeline.transformer.enable_riflex(k = riflex_k, L_test = latent_frames if not is_image else 1) + + try: + print(f"Generation.") + if self.model_type == "Inpaint": + if self.transformer.config.in_channels != self.vae.config.latent_channels: + if validation_video is not None: + input_video, input_video_mask, _, clip_image = get_video_to_video_latent(validation_video, length_slider if not is_image else 1, sample_size=(height_slider, width_slider), validation_video_mask=validation_video_mask, fps=fps) + else: + input_video, input_video_mask, clip_image = get_image_to_video_latent(start_image, end_image, length_slider if not is_image else 1, sample_size=(height_slider, width_slider)) + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + + video = input_video, + mask_video = input_video_mask, + clip_image = clip_image, + ).videos + else: + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + ).videos + else: + if ref_image is not None: + clip_image = Image.open(ref_image).convert("RGB") + elif start_image is not None: + clip_image = Image.open(start_image).convert("RGB") + else: + clip_image = None + + if ref_image is not None: + ref_image = get_image_latent(ref_image, sample_size=(height_slider, width_slider)) + + if start_image is not None: + start_image = get_image_latent(start_image, sample_size=(height_slider, width_slider)) + + input_video, input_video_mask, _, _ = get_video_to_video_latent(control_video, video_length=length_slider if not is_image else 1, sample_size=(height_slider, width_slider), fps=fps, ref_image=None) + + sample = self.pipeline( + prompt_textbox, + negative_prompt = negative_prompt_textbox, + num_inference_steps = sample_step_slider, + guidance_scale = cfg_scale_slider, + width = width_slider, + height = height_slider, + num_frames = length_slider if not is_image else 1, + generator = generator, + + control_video = input_video, + ref_image = ref_image, + start_image = start_image, + clip_image = clip_image, + ).videos + print(f"Generation done.") + except Exception as e: + self.auto_model_clear_cache(self.pipeline.transformer) + self.auto_model_clear_cache(self.pipeline.text_encoder) + self.auto_model_clear_cache(self.pipeline.vae) + self.clear_cache() + + print(f"Error. error information is {str(e)}") + if self.lora_model_path != "none": + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + if is_api: + return "", f"Error. error information is {str(e)}" + else: + return gr.update(), gr.update(), f"Error. error information is {str(e)}" + + self.clear_cache() + # lora part + if self.lora_model_path != "none": + print(f"Unmerge Lora.") + self.pipeline = unmerge_lora(self.pipeline, self.lora_model_path, multiplier=lora_alpha_slider) + print(f"Unmerge Lora done.") + + print(f"Saving outputs.") + save_sample_path = self.save_outputs( + is_image, length_slider, sample, fps=fps + ) + print(f"Saving outputs done.") + + if is_image or length_slider == 1: + if is_api: + return save_sample_path, "Success" + else: + if gradio_version_is_above_4: + return gr.Image(value=save_sample_path, visible=True), gr.Video(value=None, visible=False), "Success" + else: + return gr.Image.update(value=save_sample_path, visible=True), gr.Video.update(value=None, visible=False), "Success" + else: + if is_api: + return save_sample_path, "Success" + else: + if gradio_version_is_above_4: + return gr.Image(visible=False, value=None), gr.Video(value=save_sample_path, visible=True), "Success" + else: + return gr.Image.update(visible=False, value=None), gr.Video.update(value=save_sample_path, visible=True), "Success" + +Wan_Controller_Host = Wan_Controller +Wan_Controller_Client = Fun_Controller_Client + +def ui(GPU_memory_mode, scheduler_dict, config_path, compile_dit, weight_dtype, savedir_sample=None): + controller = Wan_Controller( + GPU_memory_mode, scheduler_dict, model_name=None, model_type="Inpaint", + config_path=config_path, compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan: + """ + ) + with gr.Column(variant="panel"): + model_type = create_model_type(visible=False) + diffusion_transformer_dropdown, diffusion_transformer_refresh_button = \ + create_model_checkpoints(controller, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider, personalized_refresh_button = \ + create_finetune_models_checkpoints(controller, visible=True) + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller) + + resize_method, width_slider, height_slider, base_resolution = create_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox, support_end_image=False + ) + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + model_type.change( + fn=controller.update_model_type, + inputs=[model_type], + outputs=[] + ) + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return [gr.update(visible=True, maximum=161, value=81, interactive=True), gr.update(visible=False), gr.update(visible=False)] + elif generation_method == "Image Generation": + return [gr.update(minimum=1, maximum=1, value=1, interactive=False), gr.update(visible=False), gr.update(visible=False)] + else: + return [gr.update(visible=True, maximum=1344), gr.update(visible=True), gr.update(visible=True)] + generation_method.change( + upload_generation_method, generation_method, [length_slider, overlap_video_length, partial_video_length] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller + +def ui_host(GPU_memory_mode, scheduler_dict, model_name, model_type, config_path, compile_dit, weight_dtype, savedir_sample=None): + controller = Wan_Controller_Host( + GPU_memory_mode, scheduler_dict, model_name=model_name, model_type=model_type, + config_path=config_path, compile_dit=compile_dit, + weight_dtype=weight_dtype, savedir_sample=savedir_sample, + ) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan: + """ + ) + with gr.Column(variant="panel"): + model_type = create_fake_model_type(visible=False) + diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider = create_fake_finetune_models_checkpoints(visible=True) + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller) + + resize_method, width_slider, height_slider, base_resolution = create_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox + ) + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return gr.update(visible=True, minimum=1, maximum=161, value=81, interactive=True) + elif generation_method == "Image Generation": + return gr.update(minimum=1, maximum=1, value=1, interactive=False) + generation_method.change( + upload_generation_method, generation_method, [length_slider] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Video to Video (视频到视频)": + return [gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(), gr.update(), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update()] + source_method.change( + upload_source_method, source_method, [ + image_to_video_col, video_to_video_col, control_video_col, start_image, end_image, + validation_video, validation_video_mask, control_video + ] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + overlap_video_length, + partial_video_length, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + control_video, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller + +def ui_client(scheduler_dict, model_name, savedir_sample=None): + controller = Wan_Controller_Client(scheduler_dict, savedir_sample) + + with gr.Blocks(css=css) as demo: + gr.Markdown( + """ + # Wan: + """ + ) + with gr.Column(variant="panel"): + diffusion_transformer_dropdown = create_fake_model_checkpoints(model_name, visible=True) + base_model_dropdown, lora_model_dropdown, lora_alpha_slider = create_fake_finetune_models_checkpoints(visible=True) + + with gr.Row(): + enable_teacache, teacache_threshold, num_skip_start_steps, teacache_offload = \ + create_teacache_params(True, 0.10, 1, False) + cfg_skip_ratio = create_cfg_skip_params(0) + enable_riflex, riflex_k = create_cfg_riflex_k(False, 6) + + with gr.Column(variant="panel"): + prompt_textbox, negative_prompt_textbox = create_prompts(negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走") + + with gr.Row(): + with gr.Column(): + sampler_dropdown, sample_step_slider = create_samplers(controller, maximum_step=50) + + resize_method, width_slider, height_slider, base_resolution = create_fake_height_width( + default_height = 480, default_width = 832, maximum_height = 1344, + maximum_width = 1344, + ) + generation_method, length_slider, overlap_video_length, partial_video_length = \ + create_generation_methods_and_video_length( + ["Video Generation", "Image Generation"], + default_video_length=81, + maximum_video_length=161, + ) + image_to_video_col, video_to_video_col, control_video_col, source_method, start_image, template_gallery, end_image, validation_video, validation_video_mask, denoise_strength, control_video, ref_image = create_generation_method( + ["Text to Video (文本到视频)", "Image to Video (图片到视频)"], prompt_textbox + ) + + cfg_scale_slider, seed_textbox, seed_button = create_cfg_and_seedbox(gradio_version_is_above_4) + + generate_button = gr.Button(value="Generate (生成)", variant='primary') + + result_image, result_video, infer_progress = create_ui_outputs() + + def upload_generation_method(generation_method): + if generation_method == "Video Generation": + return gr.update(visible=True, minimum=5, maximum=161, value=49, interactive=True) + elif generation_method == "Image Generation": + return gr.update(minimum=1, maximum=1, value=1, interactive=False) + generation_method.change( + upload_generation_method, generation_method, [length_slider] + ) + + def upload_source_method(source_method): + if source_method == "Text to Video (文本到视频)": + return [gr.update(visible=False), gr.update(visible=False), gr.update(value=None), gr.update(value=None), gr.update(value=None), gr.update(value=None)] + elif source_method == "Image to Video (图片到视频)": + return [gr.update(visible=True), gr.update(visible=False), gr.update(), gr.update(), gr.update(value=None), gr.update(value=None)] + else: + return [gr.update(visible=False), gr.update(visible=True), gr.update(value=None), gr.update(value=None), gr.update(), gr.update()] + source_method.change( + upload_source_method, source_method, [image_to_video_col, video_to_video_col, start_image, end_image, validation_video, validation_video_mask] + ) + + def upload_resize_method(resize_method): + if resize_method == "Generate by": + return [gr.update(visible=True), gr.update(visible=True), gr.update(visible=False)] + else: + return [gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)] + resize_method.change( + upload_resize_method, resize_method, [width_slider, height_slider, base_resolution] + ) + + generate_button.click( + fn=controller.generate, + inputs=[ + diffusion_transformer_dropdown, + base_model_dropdown, + lora_model_dropdown, + lora_alpha_slider, + prompt_textbox, + negative_prompt_textbox, + sampler_dropdown, + sample_step_slider, + resize_method, + width_slider, + height_slider, + base_resolution, + generation_method, + length_slider, + cfg_scale_slider, + start_image, + end_image, + validation_video, + validation_video_mask, + denoise_strength, + seed_textbox, + ref_image, + enable_teacache, + teacache_threshold, + num_skip_start_steps, + teacache_offload, + cfg_skip_ratio, + enable_riflex, + riflex_k, + ], + outputs=[result_image, result_video, infer_progress] + ) + return demo, controller \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/utils/ac_handle.py b/VideoX-Fun/videox_fun/utils/ac_handle.py new file mode 100644 index 0000000000000000000000000000000000000000..91df98a1e53a79ac18a0b9579656854e0f4f6042 --- /dev/null +++ b/VideoX-Fun/videox_fun/utils/ac_handle.py @@ -0,0 +1,64 @@ +from functools import partial + +from torch.distributed.algorithms._checkpoint.checkpoint_wrapper import ( + CheckpointImpl, + apply_activation_checkpointing, + checkpoint_wrapper, +) + + +non_reentrant_wrapper = partial( + checkpoint_wrapper, + checkpoint_impl=CheckpointImpl.NO_REENTRANT, +) + + +def apply_checkpointing(model, block, p): + """ + Apply selective activation checkpointing. + + Selectivity is defined as a percentage p, which means we apply ac + on p of the total blocks. p is a floating number in the range of + [0, 1]. + + Some examples: + p = 0: no ac for all blocks. same as `fsdp_activation_checkpointing=False` + p = 1: apply ac on every block. i.e. "full ac". + p = 1/2: [ac, no-ac, ac, no-ac, ...] + p = 1/3: [no-ac, ac, no-ac, no-ac, ac, no-ac, ...] + p = 2/3: [ac, no-ac, ac, ac, no-ac, ac, ...] + Since blocks are homogeneous, we make ac blocks evenly spaced among + all blocks. + + Implementation: + For a given ac ratio p, we should essentially apply ac on every "1/p" + blocks. The first ac block can be as early as the 0th block, or as + late as the "1/p"th block, and we pick the middle one: (0.5p)th block. + Therefore, we are essentially to apply ac on: + (0.5/p)th block, (1.5/p)th block, (2.5/p)th block, etc., and of course, + with these values rounding to integers. + Since ac is applied recursively, we can simply use the following math + in the code to apply ac on corresponding blocks. + """ + block_idx = 0 + cut_off = 1 / 2 + # when passing p as a fraction number (e.g. 1/3), it will be interpreted + # as a string in argv, thus we need eval("1/3") here for fractions. + p = eval(p) if isinstance(p, str) else p + + def selective_checkpointing(submodule): + nonlocal block_idx + nonlocal cut_off + + if isinstance(submodule, block): + block_idx += 1 + if block_idx * p >= cut_off: + cut_off += 1 + return True + return False + + apply_activation_checkpointing( + model, + checkpoint_wrapper_fn=non_reentrant_wrapper, + check_fn=selective_checkpointing, + ) diff --git a/VideoX-Fun/videox_fun/utils/cfg_optimization.py b/VideoX-Fun/videox_fun/utils/cfg_optimization.py new file mode 100644 index 0000000000000000000000000000000000000000..344a2ef87e0f8d6e3a7a57cfe9719ba1784f7608 --- /dev/null +++ b/VideoX-Fun/videox_fun/utils/cfg_optimization.py @@ -0,0 +1,39 @@ +import numpy as np +import torch + + +def cfg_skip(): + def decorator(func): + def wrapper(self, x, *args, **kwargs): + bs = len(x) + if bs >= 2 and self.cfg_skip_ratio is not None and self.current_steps >= self.num_inference_steps * (1 - self.cfg_skip_ratio): + bs_half = int(bs // 2) + + new_x = x[bs_half:] + + new_args = [] + for arg in args: + if isinstance(arg, (torch.Tensor, list, tuple, np.ndarray)): + new_args.append(arg[bs_half:]) + else: + new_args.append(arg) + + new_kwargs = {} + for key, content in kwargs.items(): + if isinstance(content, (torch.Tensor, list, tuple, np.ndarray)): + new_kwargs[key] = content[bs_half:] + else: + new_kwargs[key] = content + else: + new_x = x + new_args = args + new_kwargs = kwargs + + result = func(self, new_x, *new_args, **new_kwargs) + + if bs >= 2 and self.cfg_skip_ratio is not None and self.current_steps >= self.num_inference_steps * (1 - self.cfg_skip_ratio): + result = torch.cat([result, result], dim=0) + + return result + return wrapper + return decorator \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/utils/discrete_sampler.py b/VideoX-Fun/videox_fun/utils/discrete_sampler.py new file mode 100644 index 0000000000000000000000000000000000000000..40b8316f1ed9dac69071dc8f10d012b5d97c5c90 --- /dev/null +++ b/VideoX-Fun/videox_fun/utils/discrete_sampler.py @@ -0,0 +1,52 @@ +"""Modified from https://github.com/THUDM/CogVideo/blob/3710a612d8760f5cdb1741befeebb65b9e0f2fe0/sat/sgm/modules/diffusionmodules/sigma_sampling.py +""" +import torch + +class DiscreteSampling: + def __init__(self, num_idx, uniform_sampling=False, start_num_idx=0, sp_size=1): + self.num_idx = num_idx + self.start_num_idx = start_num_idx + self.uniform_sampling = uniform_sampling + self.is_distributed = torch.distributed.is_available() and torch.distributed.is_initialized() + + if self.is_distributed and self.uniform_sampling: + world_size = torch.distributed.get_world_size() + self.rank = torch.distributed.get_rank() + + i = 1 + while True: + if world_size % i != 0 or num_idx % (world_size // i) != 0: + i += 1 + else: + if i >= sp_size: + self.group_num = world_size // i + elif sp_size > world_size: + self.group_num = 1 + else: + self.group_num = world_size // sp_size + break + assert self.group_num > 0 + assert world_size % self.group_num == 0 + # the number of rank in one group + self.group_width = world_size // self.group_num + self.sigma_interval = self.num_idx // self.group_num + print('rank=%d world_size=%d group_num=%d group_width=%d sigma_interval=%s' % ( + self.rank, world_size, self.group_num, + self.group_width, self.sigma_interval)) + + def __call__(self, n_samples, generator=None, device=None): + if self.is_distributed and self.uniform_sampling: + group_index = self.rank // self.group_width + idx = torch.randint( + self.start_num_idx + group_index * self.sigma_interval, + self.start_num_idx + (group_index + 1) * self.sigma_interval, + (n_samples,), + generator=generator, device=device, + ) + print('proc[%d] idx=%s' % (self.rank, idx)) + else: + idx = torch.randint( + self.start_num_idx, self.start_num_idx + self.num_idx, (n_samples,), + generator=generator, device=device, + ) + return idx \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/utils/fm_solvers.py b/VideoX-Fun/videox_fun/utils/fm_solvers.py new file mode 100644 index 0000000000000000000000000000000000000000..2d516ca97c98cd8b5e0703a531577ca553d22a22 --- /dev/null +++ b/VideoX-Fun/videox_fun/utils/fm_solvers.py @@ -0,0 +1,857 @@ +# Copied from https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py +# Convert dpm solver for flow matching +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. + +import inspect +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import (KarrasDiffusionSchedulers, + SchedulerMixin, + SchedulerOutput) +from diffusers.utils import deprecate, is_scipy_available +from diffusers.utils.torch_utils import randn_tensor + +if is_scipy_available(): + pass + + +def get_sampling_sigmas(sampling_steps, shift): + sigma = np.linspace(1, 0, sampling_steps + 1)[:sampling_steps] + sigma = (shift * sigma / (1 + (shift - 1) * sigma)) + + return sigma + + +def retrieve_timesteps( + scheduler, + num_inference_steps=None, + device=None, + timesteps=None, + sigmas=None, + **kwargs, +): + if timesteps is not None and sigmas is not None: + raise ValueError( + "Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values" + ) + if timesteps is not None: + accepts_timesteps = "timesteps" in set( + inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + elif sigmas is not None: + accept_sigmas = "sigmas" in set( + inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accept_sigmas: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" sigmas schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class FlowDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): + """ + `FlowDPMSolverMultistepScheduler` is a fast dedicated high-order solver for diffusion ODEs. + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. This determines the resolution of the diffusion process. + solver_order (`int`, defaults to 2): + The DPMSolver order which can be `1`, `2`, or `3`. It is recommended to use `solver_order=2` for guided + sampling, and `solver_order=3` for unconditional sampling. This affects the number of model outputs stored + and used in multistep updates. + prediction_type (`str`, defaults to "flow_prediction"): + Prediction type of the scheduler function; must be `flow_prediction` for this scheduler, which predicts + the flow of the diffusion process. + shift (`float`, *optional*, defaults to 1.0): + A factor used to adjust the sigmas in the noise schedule. It modifies the step sizes during the sampling + process. + use_dynamic_shifting (`bool`, defaults to `False`): + Whether to apply dynamic shifting to the timesteps based on image resolution. If `True`, the shifting is + applied on the fly. + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This method adjusts the predicted sample to prevent + saturation and improve photorealism. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and + `algorithm_type="dpmsolver++"`. + algorithm_type (`str`, defaults to `dpmsolver++`): + Algorithm type for the solver; can be `dpmsolver`, `dpmsolver++`, `sde-dpmsolver` or `sde-dpmsolver++`. The + `dpmsolver` type implements the algorithms in the [DPMSolver](https://huggingface.co/papers/2206.00927) + paper, and the `dpmsolver++` type implements the algorithms in the + [DPMSolver++](https://huggingface.co/papers/2211.01095) paper. It is recommended to use `dpmsolver++` or + `sde-dpmsolver++` with `solver_order=2` for guided sampling like in Stable Diffusion. + solver_type (`str`, defaults to `midpoint`): + Solver type for the second-order solver; can be `midpoint` or `heun`. The solver type slightly affects the + sample quality, especially for a small number of steps. It is recommended to use `midpoint` solvers. + lower_order_final (`bool`, defaults to `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + euler_at_final (`bool`, defaults to `False`): + Whether to use Euler's method in the final step. It is a trade-off between numerical stability and detail + richness. This can stabilize the sampling of the SDE variant of DPMSolver for small number of inference + steps, but sometimes may result in blurring. + final_sigmas_type (`str`, *optional*, defaults to "zero"): + The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final + sigma is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0. + lambda_min_clipped (`float`, defaults to `-inf`): + Clipping threshold for the minimum value of `lambda(t)` for numerical stability. This is critical for the + cosine (`squaredcos_cap_v2`) noise schedule. + variance_type (`str`, *optional*): + Set to "learned" or "learned_range" for diffusion models that predict variance. If set, the model's output + contains the predicted Gaussian variance. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + solver_order: int = 2, + prediction_type: str = "flow_prediction", + shift: Optional[float] = 1.0, + use_dynamic_shifting=False, + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + algorithm_type: str = "dpmsolver++", + solver_type: str = "midpoint", + lower_order_final: bool = True, + euler_at_final: bool = False, + final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min" + lambda_min_clipped: float = -float("inf"), + variance_type: Optional[str] = None, + invert_sigmas: bool = False, + ): + if algorithm_type in ["dpmsolver", "sde-dpmsolver"]: + deprecation_message = f"algorithm_type {algorithm_type} is deprecated and will be removed in a future version. Choose from `dpmsolver++` or `sde-dpmsolver++` instead" + deprecate("algorithm_types dpmsolver and sde-dpmsolver", "1.0.0", + deprecation_message) + + # settings for DPM-Solver + if algorithm_type not in [ + "dpmsolver", "dpmsolver++", "sde-dpmsolver", "sde-dpmsolver++" + ]: + if algorithm_type == "deis": + self.register_to_config(algorithm_type="dpmsolver++") + else: + raise NotImplementedError( + f"{algorithm_type} is not implemented for {self.__class__}") + + if solver_type not in ["midpoint", "heun"]: + if solver_type in ["logrho", "bh1", "bh2"]: + self.register_to_config(solver_type="midpoint") + else: + raise NotImplementedError( + f"{solver_type} is not implemented for {self.__class__}") + + if algorithm_type not in ["dpmsolver++", "sde-dpmsolver++" + ] and final_sigmas_type == "zero": + raise ValueError( + f"`final_sigmas_type` {final_sigmas_type} is not supported for `algorithm_type` {algorithm_type}. Please choose `sigma_min` instead." + ) + + # setable values + self.num_inference_steps = None + alphas = np.linspace(1, 1 / num_train_timesteps, + num_train_timesteps)[::-1].copy() + sigmas = 1.0 - alphas + sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32) + + if not use_dynamic_shifting: + # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution + sigmas = shift * sigmas / (1 + + (shift - 1) * sigmas) # pyright: ignore + + self.sigmas = sigmas + self.timesteps = sigmas * num_train_timesteps + + self.model_outputs = [None] * solver_order + self.lower_order_nums = 0 + self._step_index = None + self._begin_index = None + + # self.sigmas = self.sigmas.to( + # "cpu") # to avoid too much CPU/GPU communication + self.sigma_min = self.sigmas[-1].item() + self.sigma_max = self.sigmas[0].item() + + @property + def step_index(self): + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + # Modified from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler.set_timesteps + def set_timesteps( + self, + num_inference_steps: Union[int, None] = None, + device: Union[str, torch.device] = None, + sigmas: Optional[List[float]] = None, + mu: Optional[Union[float, None]] = None, + shift: Optional[Union[float, None]] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + Args: + num_inference_steps (`int`): + Total number of the spacing of the time steps. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + + if self.config.use_dynamic_shifting and mu is None: + raise ValueError( + " you have to pass a value for `mu` when `use_dynamic_shifting` is set to be `True`" + ) + + if sigmas is None: + sigmas = np.linspace(self.sigma_max, self.sigma_min, + num_inference_steps + + 1).copy()[:-1] # pyright: ignore + + if self.config.use_dynamic_shifting: + sigmas = self.time_shift(mu, 1.0, sigmas) # pyright: ignore + else: + if shift is None: + shift = self.config.shift + sigmas = shift * sigmas / (1 + + (shift - 1) * sigmas) # pyright: ignore + + if self.config.final_sigmas_type == "sigma_min": + sigma_last = ((1 - self.alphas_cumprod[0]) / + self.alphas_cumprod[0])**0.5 + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + + timesteps = sigmas * self.config.num_train_timesteps + sigmas = np.concatenate([sigmas, [sigma_last] + ]).astype(np.float32) # pyright: ignore + + self.sigmas = torch.from_numpy(sigmas) + self.timesteps = torch.from_numpy(timesteps).to( + device=device, dtype=torch.int64) + + self.num_inference_steps = len(timesteps) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + + self._step_index = None + self._begin_index = None + # self.sigmas = self.sigmas.to( + # "cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float( + ) # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile( + abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze( + 1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp( + sample, -s, s + ) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma): + return sigma * self.config.num_train_timesteps + + def _sigma_to_alpha_sigma_t(self, sigma): + return 1 - sigma, sigma + + # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.set_timesteps + def time_shift(self, mu: float, sigma: float, t: torch.Tensor): + return math.exp(mu) / (math.exp(mu) + (1 / t - 1)**sigma) + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.convert_model_output + def convert_model_output( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + **kwargs, + ) -> torch.Tensor: + """ + Convert the model output to the corresponding type the DPMSolver/DPMSolver++ algorithm needs. DPM-Solver is + designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to discretize an + integral of the data prediction model. + + The algorithm and model type are decoupled. You can use either DPMSolver or DPMSolver++ for both noise + prediction and data prediction models. + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + Returns: + `torch.Tensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError( + "missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + # DPM-Solver++ needs to solve an integral of the data prediction model. + if self.config.algorithm_type in ["dpmsolver++", "sde-dpmsolver++"]: + if self.config.prediction_type == "flow_prediction": + sigma_t = self.sigmas[self.step_index] + x0_pred = sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`," + " `v_prediction`, or `flow_prediction` for the FlowDPMSolverMultistepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + + # DPM-Solver needs to solve an integral of the noise prediction model. + elif self.config.algorithm_type in ["dpmsolver", "sde-dpmsolver"]: + if self.config.prediction_type == "flow_prediction": + sigma_t = self.sigmas[self.step_index] + epsilon = sample - (1 - sigma_t) * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`," + " `v_prediction` or `flow_prediction` for the FlowDPMSolverMultistepScheduler." + ) + + if self.config.thresholding: + sigma_t = self.sigmas[self.step_index] + x0_pred = sample - sigma_t * model_output + x0_pred = self._threshold_sample(x0_pred) + epsilon = model_output + x0_pred + + return epsilon + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.dpm_solver_first_order_update + def dpm_solver_first_order_update( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + noise: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """ + One step for the first-order DPMSolver (equivalent to DDIM). + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + Returns: + `torch.Tensor`: + The sample tensor at the previous timestep. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop( + "prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError( + " missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s = self.sigmas[self.step_index + 1], self.sigmas[ + self.step_index] # pyright: ignore + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s, sigma_s = self._sigma_to_alpha_sigma_t(sigma_s) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s = torch.log(alpha_s) - torch.log(sigma_s) + + h = lambda_t - lambda_s + if self.config.algorithm_type == "dpmsolver++": + x_t = (sigma_t / + sigma_s) * sample - (alpha_t * + (torch.exp(-h) - 1.0)) * model_output + elif self.config.algorithm_type == "dpmsolver": + x_t = (alpha_t / + alpha_s) * sample - (sigma_t * + (torch.exp(h) - 1.0)) * model_output + elif self.config.algorithm_type == "sde-dpmsolver++": + assert noise is not None + x_t = ((sigma_t / sigma_s * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * model_output + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise) + elif self.config.algorithm_type == "sde-dpmsolver": + assert noise is not None + x_t = ((alpha_t / alpha_s) * sample - 2.0 * + (sigma_t * (torch.exp(h) - 1.0)) * model_output + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise) + return x_t # pyright: ignore + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.multistep_dpm_solver_second_order_update + def multistep_dpm_solver_second_order_update( + self, + model_output_list: List[torch.Tensor], + *args, + sample: torch.Tensor = None, + noise: Optional[torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + """ + One step for the second-order multistep DPMSolver. + Args: + model_output_list (`List[torch.Tensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + Returns: + `torch.Tensor`: + The sample tensor at the previous timestep. + """ + timestep_list = args[0] if len(args) > 0 else kwargs.pop( + "timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop( + "prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError( + " missing `sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1 = ( + self.sigmas[self.step_index + 1], # pyright: ignore + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], # pyright: ignore + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + + m0, m1 = model_output_list[-1], model_output_list[-2] + + h, h_0 = lambda_t - lambda_s0, lambda_s0 - lambda_s1 + r0 = h_0 / h + D0, D1 = m0, (1.0 / r0) * (m0 - m1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2211.01095 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ((sigma_t / sigma_s0) * sample - + (alpha_t * (torch.exp(-h) - 1.0)) * D0 - 0.5 * + (alpha_t * (torch.exp(-h) - 1.0)) * D1) + elif self.config.solver_type == "heun": + x_t = ((sigma_t / sigma_s0) * sample - + (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ((alpha_t / alpha_s0) * sample - + (sigma_t * (torch.exp(h) - 1.0)) * D0 - 0.5 * + (sigma_t * (torch.exp(h) - 1.0)) * D1) + elif self.config.solver_type == "heun": + x_t = ((alpha_t / alpha_s0) * sample - + (sigma_t * (torch.exp(h) - 1.0)) * D0 - + (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1) + elif self.config.algorithm_type == "sde-dpmsolver++": + assert noise is not None + if self.config.solver_type == "midpoint": + x_t = ((sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + 0.5 * + (alpha_t * (1 - torch.exp(-2.0 * h))) * D1 + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise) + elif self.config.solver_type == "heun": + x_t = ((sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + (alpha_t * ((1.0 - torch.exp(-2.0 * h)) / + (-2.0 * h) + 1.0)) * D1 + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise) + elif self.config.algorithm_type == "sde-dpmsolver": + assert noise is not None + if self.config.solver_type == "midpoint": + x_t = ((alpha_t / alpha_s0) * sample - 2.0 * + (sigma_t * (torch.exp(h) - 1.0)) * D0 - + (sigma_t * (torch.exp(h) - 1.0)) * D1 + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise) + elif self.config.solver_type == "heun": + x_t = ((alpha_t / alpha_s0) * sample - 2.0 * + (sigma_t * (torch.exp(h) - 1.0)) * D0 - 2.0 * + (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise) + return x_t # pyright: ignore + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.multistep_dpm_solver_third_order_update + def multistep_dpm_solver_third_order_update( + self, + model_output_list: List[torch.Tensor], + *args, + sample: torch.Tensor = None, + **kwargs, + ) -> torch.Tensor: + """ + One step for the third-order multistep DPMSolver. + Args: + model_output_list (`List[torch.Tensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.Tensor`): + A current instance of a sample created by diffusion process. + Returns: + `torch.Tensor`: + The sample tensor at the previous timestep. + """ + + timestep_list = args[0] if len(args) > 0 else kwargs.pop( + "timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop( + "prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError( + " missing`sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1, sigma_s2 = ( + self.sigmas[self.step_index + 1], # pyright: ignore + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], # pyright: ignore + self.sigmas[self.step_index - 2], # pyright: ignore + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + alpha_s2, sigma_s2 = self._sigma_to_alpha_sigma_t(sigma_s2) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + lambda_s2 = torch.log(alpha_s2) - torch.log(sigma_s2) + + m0, m1, m2 = model_output_list[-1], model_output_list[ + -2], model_output_list[-3] + + h, h_0, h_1 = lambda_t - lambda_s0, lambda_s0 - lambda_s1, lambda_s1 - lambda_s2 + r0, r1 = h_0 / h, h_1 / h + D0 = m0 + D1_0, D1_1 = (1.0 / r0) * (m0 - m1), (1.0 / r1) * (m1 - m2) + D1 = D1_0 + (r0 / (r0 + r1)) * (D1_0 - D1_1) + D2 = (1.0 / (r0 + r1)) * (D1_0 - D1_1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ((sigma_t / sigma_s0) * sample - + (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 - + (alpha_t * ((torch.exp(-h) - 1.0 + h) / h**2 - 0.5)) * D2) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ((alpha_t / alpha_s0) * sample - (sigma_t * + (torch.exp(h) - 1.0)) * D0 - + (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 - + (sigma_t * ((torch.exp(h) - 1.0 - h) / h**2 - 0.5)) * D2) + return x_t # pyright: ignore + + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + def _init_step_index(self, timestep): + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + # Modified from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.step + def step( + self, + model_output: torch.Tensor, + timestep: Union[int, torch.Tensor], + sample: torch.Tensor, + generator=None, + variance_noise: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep DPMSolver. + Args: + model_output (`torch.Tensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + variance_noise (`torch.Tensor`): + Alternative to generating noise with `generator` by directly providing the noise for the variance + itself. Useful for methods such as [`LEdits++`]. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Improve numerical stability for small number of steps + lower_order_final = (self.step_index == len(self.timesteps) - 1) and ( + self.config.euler_at_final or + (self.config.lower_order_final and len(self.timesteps) < 15) or + self.config.final_sigmas_type == "zero") + lower_order_second = ((self.step_index == len(self.timesteps) - 2) and + self.config.lower_order_final and + len(self.timesteps) < 15) + + model_output = self.convert_model_output(model_output, sample=sample) + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.model_outputs[-1] = model_output + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + if self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++" + ] and variance_noise is None: + noise = randn_tensor( + model_output.shape, + generator=generator, + device=model_output.device, + dtype=torch.float32) + elif self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]: + noise = variance_noise.to( + device=model_output.device, + dtype=torch.float32) # pyright: ignore + else: + noise = None + + if self.config.solver_order == 1 or self.lower_order_nums < 1 or lower_order_final: + prev_sample = self.dpm_solver_first_order_update( + model_output, sample=sample, noise=noise) + elif self.config.solver_order == 2 or self.lower_order_nums < 2 or lower_order_second: + prev_sample = self.multistep_dpm_solver_second_order_update( + self.model_outputs, sample=sample, noise=noise) + else: + prev_sample = self.multistep_dpm_solver_third_order_update( + self.model_outputs, sample=sample) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # Cast sample back to expected dtype + prev_sample = prev_sample.to(model_output.dtype) + + # upon completion increase step index by one + self._step_index += 1 # pyright: ignore + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.scale_model_input + def scale_model_input(self, sample: torch.Tensor, *args, + **kwargs) -> torch.Tensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + Args: + sample (`torch.Tensor`): + The input sample. + Returns: + `torch.Tensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.scale_model_input + def add_noise( + self, + original_samples: torch.Tensor, + noise: torch.Tensor, + timesteps: torch.IntTensor, + ) -> torch.Tensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to( + device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point( + timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to( + original_samples.device, dtype=torch.float32) + timesteps = timesteps.to( + original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [ + self.index_for_timestep(t, schedule_timesteps) + for t in timesteps + ] + elif self.step_index is not None: + # add_noise is called after first denoising step (for inpainting) + step_indices = [self.step_index] * timesteps.shape[0] + else: + # add noise is called before first denoising step to create initial latent(img2img) + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/utils/fm_solvers_unipc.py b/VideoX-Fun/videox_fun/utils/fm_solvers_unipc.py new file mode 100644 index 0000000000000000000000000000000000000000..b8b347b7eca72e961adc2f1fdbbef33ceb1eb009 --- /dev/null +++ b/VideoX-Fun/videox_fun/utils/fm_solvers_unipc.py @@ -0,0 +1,800 @@ +# Copied from https://github.com/huggingface/diffusers/blob/v0.31.0/src/diffusers/schedulers/scheduling_unipc_multistep.py +# Convert unipc for flow matching +# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved. + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import (KarrasDiffusionSchedulers, + SchedulerMixin, + SchedulerOutput) +from diffusers.utils import deprecate, is_scipy_available + +if is_scipy_available(): + import scipy.stats + + +class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin): + """ + `UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + solver_order (`int`, default `2`): + The UniPC order which can be any positive integer. The effective order of accuracy is `solver_order + 1` + due to the UniC. It is recommended to use `solver_order=2` for guided sampling, and `solver_order=3` for + unconditional sampling. + prediction_type (`str`, defaults to "flow_prediction"): + Prediction type of the scheduler function; must be `flow_prediction` for this scheduler, which predicts + the flow of the diffusion process. + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and `predict_x0=True`. + predict_x0 (`bool`, defaults to `True`): + Whether to use the updating algorithm on the predicted x0. + solver_type (`str`, default `bh2`): + Solver type for UniPC. It is recommended to use `bh1` for unconditional sampling when steps < 10, and `bh2` + otherwise. + lower_order_final (`bool`, default `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + disable_corrector (`list`, default `[]`): + Decides which step to disable the corrector to mitigate the misalignment between `epsilon_theta(x_t, c)` + and `epsilon_theta(x_t^c, c)` which can influence convergence for a large guidance scale. Corrector is + usually disabled during the first few steps. + solver_p (`SchedulerMixin`, default `None`): + Any other scheduler that if specified, the algorithm becomes `solver_p + UniC`. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + use_exponential_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use exponential sigmas for step sizes in the noise schedule during the sampling process. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + final_sigmas_type (`str`, defaults to `"zero"`): + The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final + sigma is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + solver_order: int = 2, + prediction_type: str = "flow_prediction", + shift: Optional[float] = 1.0, + use_dynamic_shifting=False, + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + predict_x0: bool = True, + solver_type: str = "bh2", + lower_order_final: bool = True, + disable_corrector: List[int] = [], + solver_p: SchedulerMixin = None, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min" + ): + + if solver_type not in ["bh1", "bh2"]: + if solver_type in ["midpoint", "heun", "logrho"]: + self.register_to_config(solver_type="bh2") + else: + raise NotImplementedError( + f"{solver_type} is not implemented for {self.__class__}") + + self.predict_x0 = predict_x0 + # setable values + self.num_inference_steps = None + alphas = np.linspace(1, 1 / num_train_timesteps, + num_train_timesteps)[::-1].copy() + sigmas = 1.0 - alphas + sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32) + + if not use_dynamic_shifting: + # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution + sigmas = shift * sigmas / (1 + + (shift - 1) * sigmas) # pyright: ignore + + self.sigmas = sigmas + self.timesteps = sigmas * num_train_timesteps + + self.model_outputs = [None] * solver_order + self.timestep_list = [None] * solver_order + self.lower_order_nums = 0 + self.disable_corrector = disable_corrector + self.solver_p = solver_p + self.last_sample = None + self._step_index = None + self._begin_index = None + + self.sigmas = self.sigmas.to( + "cpu") # to avoid too much CPU/GPU communication + self.sigma_min = self.sigmas[-1].item() + self.sigma_max = self.sigmas[0].item() + + @property + def step_index(self): + """ + The index counter for current timestep. It will increase 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + # Modified from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler.set_timesteps + def set_timesteps( + self, + num_inference_steps: Union[int, None] = None, + device: Union[str, torch.device] = None, + sigmas: Optional[List[float]] = None, + mu: Optional[Union[float, None]] = None, + shift: Optional[Union[float, None]] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + Args: + num_inference_steps (`int`): + Total number of the spacing of the time steps. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + + if self.config.use_dynamic_shifting and mu is None: + raise ValueError( + " you have to pass a value for `mu` when `use_dynamic_shifting` is set to be `True`" + ) + + if sigmas is None: + sigmas = np.linspace(self.sigma_max, self.sigma_min, + num_inference_steps + + 1).copy()[:-1] # pyright: ignore + + if self.config.use_dynamic_shifting: + sigmas = self.time_shift(mu, 1.0, sigmas) # pyright: ignore + else: + if shift is None: + shift = self.config.shift + sigmas = shift * sigmas / (1 + + (shift - 1) * sigmas) # pyright: ignore + + if self.config.final_sigmas_type == "sigma_min": + sigma_last = ((1 - self.alphas_cumprod[0]) / + self.alphas_cumprod[0])**0.5 + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + + timesteps = sigmas * self.config.num_train_timesteps + sigmas = np.concatenate([sigmas, [sigma_last] + ]).astype(np.float32) # pyright: ignore + + self.sigmas = torch.from_numpy(sigmas) + self.timesteps = torch.from_numpy(timesteps).to( + device=device, dtype=torch.int64) + + self.num_inference_steps = len(timesteps) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + self.last_sample = None + if self.solver_p: + self.solver_p.set_timesteps(self.num_inference_steps, device=device) + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to( + "cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float( + ) # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile( + abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze( + 1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp( + sample, -s, s + ) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma): + return sigma * self.config.num_train_timesteps + + def _sigma_to_alpha_sigma_t(self, sigma): + return 1 - sigma, sigma + + # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.set_timesteps + def time_shift(self, mu: float, sigma: float, t: torch.Tensor): + return math.exp(mu) / (math.exp(mu) + (1 / t - 1)**sigma) + + def convert_model_output( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + **kwargs, + ) -> torch.Tensor: + r""" + Convert the model output to the corresponding type the UniPC algorithm needs. + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.Tensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError( + "missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + + if self.predict_x0: + if self.config.prediction_type == "flow_prediction": + sigma_t = self.sigmas[self.step_index] + x0_pred = sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`," + " `v_prediction` or `flow_prediction` for the UniPCMultistepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + else: + if self.config.prediction_type == "flow_prediction": + sigma_t = self.sigmas[self.step_index] + epsilon = sample - (1 - sigma_t) * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`," + " `v_prediction` or `flow_prediction` for the UniPCMultistepScheduler." + ) + + if self.config.thresholding: + sigma_t = self.sigmas[self.step_index] + x0_pred = sample - sigma_t * model_output + x0_pred = self._threshold_sample(x0_pred) + epsilon = model_output + x0_pred + + return epsilon + + def multistep_uni_p_bh_update( + self, + model_output: torch.Tensor, + *args, + sample: torch.Tensor = None, + order: int = None, # pyright: ignore + **kwargs, + ) -> torch.Tensor: + """ + One step for the UniP (B(h) version). Alternatively, `self.solver_p` is used if is specified. + + Args: + model_output (`torch.Tensor`): + The direct output from the learned diffusion model at the current timestep. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + order (`int`): + The order of UniP at this timestep (corresponds to the *p* in UniPC-p). + + Returns: + `torch.Tensor`: + The sample tensor at the previous timestep. + """ + prev_timestep = args[0] if len(args) > 0 else kwargs.pop( + "prev_timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError( + " missing `sample` as a required keyward argument") + if order is None: + if len(args) > 2: + order = args[2] + else: + raise ValueError( + " missing `order` as a required keyward argument") + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + model_output_list = self.model_outputs + + s0 = self.timestep_list[-1] + m0 = model_output_list[-1] + x = sample + + if self.solver_p: + x_t = self.solver_p.step(model_output, s0, x).prev_sample + return x_t + + sigma_t, sigma_s0 = self.sigmas[self.step_index + 1], self.sigmas[ + self.step_index] # pyright: ignore + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + h = lambda_t - lambda_s0 + device = sample.device + + rks = [] + D1s = [] + for i in range(1, order): + si = self.step_index - i # pyright: ignore + mi = model_output_list[-(i + 1)] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h + rks.append(rk) + D1s.append((mi - m0) / rk) # pyright: ignore + + rks.append(1.0) + rks = torch.tensor(rks, device=device) + + R = [] + b = [] + + hh = -h if self.predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + + factorial_i = 1 + + if self.config.solver_type == "bh1": + B_h = hh + elif self.config.solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + R = torch.stack(R) + b = torch.tensor(b, device=device) + + if len(D1s) > 0: + D1s = torch.stack(D1s, dim=1) # (B, K) + # for order 2, we use a simplified version + if order == 2: + rhos_p = torch.tensor([0.5], dtype=x.dtype, device=device) + else: + rhos_p = torch.linalg.solve(R[:-1, :-1], + b[:-1]).to(device).to(x.dtype) + else: + D1s = None + + if self.predict_x0: + x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0 + if D1s is not None: + pred_res = torch.einsum("k,bkc...->bc...", rhos_p, + D1s) # pyright: ignore + else: + pred_res = 0 + x_t = x_t_ - alpha_t * B_h * pred_res + else: + x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0 + if D1s is not None: + pred_res = torch.einsum("k,bkc...->bc...", rhos_p, + D1s) # pyright: ignore + else: + pred_res = 0 + x_t = x_t_ - sigma_t * B_h * pred_res + + x_t = x_t.to(x.dtype) + return x_t + + def multistep_uni_c_bh_update( + self, + this_model_output: torch.Tensor, + *args, + last_sample: torch.Tensor = None, + this_sample: torch.Tensor = None, + order: int = None, # pyright: ignore + **kwargs, + ) -> torch.Tensor: + """ + One step for the UniC (B(h) version). + + Args: + this_model_output (`torch.Tensor`): + The model outputs at `x_t`. + this_timestep (`int`): + The current timestep `t`. + last_sample (`torch.Tensor`): + The generated sample before the last predictor `x_{t-1}`. + this_sample (`torch.Tensor`): + The generated sample after the last predictor `x_{t}`. + order (`int`): + The `p` of UniC-p at this step. The effective order of accuracy should be `order + 1`. + + Returns: + `torch.Tensor`: + The corrected sample tensor at the current timestep. + """ + this_timestep = args[0] if len(args) > 0 else kwargs.pop( + "this_timestep", None) + if last_sample is None: + if len(args) > 1: + last_sample = args[1] + else: + raise ValueError( + " missing`last_sample` as a required keyward argument") + if this_sample is None: + if len(args) > 2: + this_sample = args[2] + else: + raise ValueError( + " missing`this_sample` as a required keyward argument") + if order is None: + if len(args) > 3: + order = args[3] + else: + raise ValueError( + " missing`order` as a required keyward argument") + if this_timestep is not None: + deprecate( + "this_timestep", + "1.0.0", + "Passing `this_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + model_output_list = self.model_outputs + + m0 = model_output_list[-1] + x = last_sample + x_t = this_sample + model_t = this_model_output + + sigma_t, sigma_s0 = self.sigmas[self.step_index], self.sigmas[ + self.step_index - 1] # pyright: ignore + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + h = lambda_t - lambda_s0 + device = this_sample.device + + rks = [] + D1s = [] + for i in range(1, order): + si = self.step_index - (i + 1) # pyright: ignore + mi = model_output_list[-(i + 1)] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h + rks.append(rk) + D1s.append((mi - m0) / rk) # pyright: ignore + + rks.append(1.0) + rks = torch.tensor(rks, device=device) + + R = [] + b = [] + + hh = -h if self.predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + + factorial_i = 1 + + if self.config.solver_type == "bh1": + B_h = hh + elif self.config.solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + R = torch.stack(R) + b = torch.tensor(b, device=device) + + if len(D1s) > 0: + D1s = torch.stack(D1s, dim=1) + else: + D1s = None + + # for order 1, we use a simplified version + if order == 1: + rhos_c = torch.tensor([0.5], dtype=x.dtype, device=device) + else: + rhos_c = torch.linalg.solve(R, b).to(device).to(x.dtype) + + if self.predict_x0: + x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0 + if D1s is not None: + corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], D1s) + else: + corr_res = 0 + D1_t = model_t - m0 + x_t = x_t_ - alpha_t * B_h * (corr_res + rhos_c[-1] * D1_t) + else: + x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0 + if D1s is not None: + corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], D1s) + else: + corr_res = 0 + D1_t = model_t - m0 + x_t = x_t_ - sigma_t * B_h * (corr_res + rhos_c[-1] * D1_t) + x_t = x_t.to(x.dtype) + return x_t + + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index + def _init_step_index(self, timestep): + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step(self, + model_output: torch.Tensor, + timestep: Union[int, torch.Tensor], + sample: torch.Tensor, + return_dict: bool = True, + generator=None) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep UniPC. + + Args: + model_output (`torch.Tensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.Tensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + use_corrector = ( + self.step_index > 0 and + self.step_index - 1 not in self.disable_corrector and + self.last_sample is not None # pyright: ignore + ) + + model_output_convert = self.convert_model_output( + model_output, sample=sample) + if use_corrector: + sample = self.multistep_uni_c_bh_update( + this_model_output=model_output_convert, + last_sample=self.last_sample, + this_sample=sample, + order=self.this_order, + ) + + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.timestep_list[i] = self.timestep_list[i + 1] + + self.model_outputs[-1] = model_output_convert + self.timestep_list[-1] = timestep # pyright: ignore + + if self.config.lower_order_final: + this_order = min(self.config.solver_order, + len(self.timesteps) - + self.step_index) # pyright: ignore + else: + this_order = self.config.solver_order + + self.this_order = min(this_order, + self.lower_order_nums + 1) # warmup for multistep + assert self.this_order > 0 + + self.last_sample = sample + prev_sample = self.multistep_uni_p_bh_update( + model_output=model_output, # pass the original non-converted model output, in case solver-p is used + sample=sample, + order=self.this_order, + ) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # upon completion increase step index by one + self._step_index += 1 # pyright: ignore + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.Tensor, *args, + **kwargs) -> torch.Tensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.Tensor`): + The input sample. + + Returns: + `torch.Tensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise + def add_noise( + self, + original_samples: torch.Tensor, + noise: torch.Tensor, + timesteps: torch.IntTensor, + ) -> torch.Tensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to( + device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point( + timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to( + original_samples.device, dtype=torch.float32) + timesteps = timesteps.to( + original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [ + self.index_for_timestep(t, schedule_timesteps) + for t in timesteps + ] + elif self.step_index is not None: + # add_noise is called after first denoising step (for inpainting) + step_indices = [self.step_index] * timesteps.shape[0] + else: + # add noise is called before first denoising step to create initial latent(img2img) + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/utils/fp8_optimization.py b/VideoX-Fun/videox_fun/utils/fp8_optimization.py new file mode 100644 index 0000000000000000000000000000000000000000..b8b00387b1423f5fdbeaa57dccadb4a38e14576e --- /dev/null +++ b/VideoX-Fun/videox_fun/utils/fp8_optimization.py @@ -0,0 +1,64 @@ +"""Modified from https://github.com/kijai/ComfyUI-MochiWrapper +""" +import importlib.util + +import torch +import torch.nn as nn + +def replace_parameters_by_name(module, name_keywords, device): + from torch import nn + for name, param in list(module.named_parameters(recurse=False)): + if any(keyword in name for keyword in name_keywords): + if isinstance(param, nn.Parameter): + tensor = param.data + delattr(module, name) + setattr(module, name, tensor.to(device=device)) + for child_name, child_module in module.named_children(): + replace_parameters_by_name(child_module, name_keywords, device) + +def convert_model_weight_to_float8(model, exclude_module_name=['embed_tokens'], device=None): + for name, module in model.named_modules(): + flag = False + for _exclude_module_name in exclude_module_name: + if _exclude_module_name in name: + flag = True + if flag: + continue + for param_name, param in module.named_parameters(): + flag = False + for _exclude_module_name in exclude_module_name: + if _exclude_module_name in param_name: + flag = True + if flag: + continue + param.data = param.data.to(torch.float8_e4m3fn) + +def autocast_model_forward(cls, origin_dtype, *inputs, **kwargs): + weight_dtype = cls.weight.dtype + cls.to(origin_dtype) + + # Convert all inputs to the original dtype + inputs = [input.to(origin_dtype) for input in inputs] + out = cls.original_forward(*inputs, **kwargs) + + cls.to(weight_dtype) + return out + +def convert_weight_dtype_wrapper(module, origin_dtype): + for name, module in module.named_modules(): + if name == "" or "embed_tokens" in name: + continue + original_forward = module.forward + if hasattr(module, "weight") and module.weight is not None: + setattr(module, "original_forward", original_forward) + setattr( + module, + "forward", + lambda *inputs, m=module, **kwargs: autocast_model_forward(m, origin_dtype, *inputs, **kwargs) + ) + +def undo_convert_weight_dtype_wrapper(module): + for name, module in module.named_modules(): + if hasattr(module, "original_forward") and module.weight is not None: + setattr(module, "forward", module.original_forward) + delattr(module, "original_forward") \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/utils/lora_utils.py b/VideoX-Fun/videox_fun/utils/lora_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..d303cb47e7fc540feafcbc52fca807c85dc2f5e3 --- /dev/null +++ b/VideoX-Fun/videox_fun/utils/lora_utils.py @@ -0,0 +1,634 @@ +# LoRA network module +# reference: +# https://github.com/microsoft/LoRA/blob/main/loralib/layers.py +# https://github.com/cloneofsimo/lora/blob/master/lora_diffusion/lora.py +# https://github.com/bmaltais/kohya_ss + +import hashlib +import math +import os +from collections import defaultdict +from io import BytesIO +from typing import List, Optional, Type, Union + +import safetensors.torch +import torch +import torch.utils.checkpoint +from diffusers.models.lora import LoRACompatibleConv, LoRACompatibleLinear +from safetensors.torch import load_file +from transformers import T5EncoderModel + + +class LoRAModule(torch.nn.Module): + """ + replaces forward method of the original Linear, instead of replacing the original Linear module. + """ + + def __init__( + self, + lora_name, + org_module: torch.nn.Module, + multiplier=1.0, + lora_dim=4, + alpha=1, + dropout=None, + rank_dropout=None, + module_dropout=None, + ): + """if alpha == 0 or None, alpha is rank (no scaling).""" + super().__init__() + self.lora_name = lora_name + + if org_module.__class__.__name__ == "Conv2d": + in_dim = org_module.in_channels + out_dim = org_module.out_channels + else: + in_dim = org_module.in_features + out_dim = org_module.out_features + + self.lora_dim = lora_dim + if org_module.__class__.__name__ == "Conv2d": + kernel_size = org_module.kernel_size + stride = org_module.stride + padding = org_module.padding + self.lora_down = torch.nn.Conv2d(in_dim, self.lora_dim, kernel_size, stride, padding, bias=False) + self.lora_up = torch.nn.Conv2d(self.lora_dim, out_dim, (1, 1), (1, 1), bias=False) + else: + self.lora_down = torch.nn.Linear(in_dim, self.lora_dim, bias=False) + self.lora_up = torch.nn.Linear(self.lora_dim, out_dim, bias=False) + + if type(alpha) == torch.Tensor: + alpha = alpha.detach().float().numpy() # without casting, bf16 causes error + alpha = self.lora_dim if alpha is None or alpha == 0 else alpha + self.scale = alpha / self.lora_dim + self.register_buffer("alpha", torch.tensor(alpha)) + + # same as microsoft's + torch.nn.init.kaiming_uniform_(self.lora_down.weight, a=math.sqrt(5)) + torch.nn.init.zeros_(self.lora_up.weight) + + self.multiplier = multiplier + self.org_module = org_module # remove in applying + self.dropout = dropout + self.rank_dropout = rank_dropout + self.module_dropout = module_dropout + + def apply_to(self): + self.org_forward = self.org_module.forward + self.org_module.forward = self.forward + del self.org_module + + def forward(self, x, *args, **kwargs): + weight_dtype = x.dtype + org_forwarded = self.org_forward(x) + + # module dropout + if self.module_dropout is not None and self.training: + if torch.rand(1) < self.module_dropout: + return org_forwarded + + lx = self.lora_down(x.to(self.lora_down.weight.dtype)) + + # normal dropout + if self.dropout is not None and self.training: + lx = torch.nn.functional.dropout(lx, p=self.dropout) + + # rank dropout + if self.rank_dropout is not None and self.training: + mask = torch.rand((lx.size(0), self.lora_dim), device=lx.device) > self.rank_dropout + if len(lx.size()) == 3: + mask = mask.unsqueeze(1) # for Text Encoder + elif len(lx.size()) == 4: + mask = mask.unsqueeze(-1).unsqueeze(-1) # for Conv2d + lx = lx * mask + + # scaling for rank dropout: treat as if the rank is changed + scale = self.scale * (1.0 / (1.0 - self.rank_dropout)) # redundant for readability + else: + scale = self.scale + + lx = self.lora_up(lx) + + return org_forwarded.to(weight_dtype) + lx.to(weight_dtype) * self.multiplier * scale + +def addnet_hash_legacy(b): + """Old model hash used by sd-webui-additional-networks for .safetensors format files""" + m = hashlib.sha256() + + b.seek(0x100000) + m.update(b.read(0x10000)) + return m.hexdigest()[0:8] + +def addnet_hash_safetensors(b): + """New model hash used by sd-webui-additional-networks for .safetensors format files""" + hash_sha256 = hashlib.sha256() + blksize = 1024 * 1024 + + b.seek(0) + header = b.read(8) + n = int.from_bytes(header, "little") + + offset = n + 8 + b.seek(offset) + for chunk in iter(lambda: b.read(blksize), b""): + hash_sha256.update(chunk) + + return hash_sha256.hexdigest() + +def precalculate_safetensors_hashes(tensors, metadata): + """Precalculate the model hashes needed by sd-webui-additional-networks to + save time on indexing the model later.""" + + # Because writing user metadata to the file can change the result of + # sd_models.model_hash(), only retain the training metadata for purposes of + # calculating the hash, as they are meant to be immutable + metadata = {k: v for k, v in metadata.items() if k.startswith("ss_")} + + bytes = safetensors.torch.save(tensors, metadata) + b = BytesIO(bytes) + + model_hash = addnet_hash_safetensors(b) + legacy_hash = addnet_hash_legacy(b) + return model_hash, legacy_hash + +class LoRANetwork(torch.nn.Module): + TRANSFORMER_TARGET_REPLACE_MODULE = [ + "CogVideoXTransformer3DModel", "WanTransformer3DModel", \ + "Wan2_2Transformer3DModel", "FluxTransformer2DModel", "QwenImageTransformer2DModel", \ + "Wan2_2Transformer3DModel_Animate", "Wan2_2Transformer3DModel_S2V", "FantasyTalkingTransformer3DModel", \ + "HunyuanVideoTransformer3DModel", "Flux2Transformer2DModel", "ZImageTransformer2DModel", + ] + TEXT_ENCODER_TARGET_REPLACE_MODULE = ["T5LayerSelfAttention", "T5LayerFF", "BertEncoder", "T5SelfAttention", "T5CrossAttention"] + LORA_PREFIX_TRANSFORMER = "lora_unet" + LORA_PREFIX_TEXT_ENCODER = "lora_te" + def __init__( + self, + text_encoder: Union[List[T5EncoderModel], T5EncoderModel], + unet, + multiplier: float = 1.0, + lora_dim: int = 4, + alpha: float = 1, + dropout: Optional[float] = None, + module_class: Type[object] = LoRAModule, + skip_name: str = None, + target_name: str = None, + varbose: Optional[bool] = False, + ) -> None: + super().__init__() + self.multiplier = multiplier + + self.lora_dim = lora_dim + self.alpha = alpha + self.dropout = dropout + + print(f"create LoRA network. base dim (rank): {lora_dim}, alpha: {alpha}") + print(f"neuron dropout: p={self.dropout}") + + # create module instances + def create_modules( + is_unet: bool, + root_module: torch.nn.Module, + target_replace_modules: List[torch.nn.Module], + ) -> List[LoRAModule]: + prefix = ( + self.LORA_PREFIX_TRANSFORMER + if is_unet + else self.LORA_PREFIX_TEXT_ENCODER + ) + loras = [] + skipped = [] + for name, module in root_module.named_modules(): + if module.__class__.__name__ in target_replace_modules: + for child_name, child_module in module.named_modules(): + is_linear = child_module.__class__.__name__ == "Linear" or child_module.__class__.__name__ == "LoRACompatibleLinear" + is_conv2d = child_module.__class__.__name__ == "Conv2d" or child_module.__class__.__name__ == "LoRACompatibleConv" + is_conv2d_1x1 = is_conv2d and child_module.kernel_size == (1, 1) + + skip_names = skip_name.split(',') if skip_name is not None else [] + target_names = target_name.split(',') if target_name is not None else [] + + skip_names = [name.strip() for name in skip_names if name.strip()] + target_names = [name.strip() for name in target_names if name.strip()] + + if skip_names and any(skip_n in child_name for skip_n in skip_names): + continue + + if target_names and not any(target_n in child_name for target_n in target_names): + continue + + if is_linear or is_conv2d: + lora_name = prefix + "." + name + "." + child_name + lora_name = lora_name.replace(".", "_") + + dim = None + alpha = None + + if is_linear or is_conv2d_1x1: + dim = self.lora_dim + alpha = self.alpha + + if dim is None or dim == 0: + if is_linear or is_conv2d_1x1: + skipped.append(lora_name) + continue + + lora = module_class( + lora_name, + child_module, + self.multiplier, + dim, + alpha, + dropout=dropout, + ) + loras.append(lora) + return loras, skipped + + text_encoders = text_encoder if type(text_encoder) == list else [text_encoder] + + self.text_encoder_loras = [] + skipped_te = [] + for i, text_encoder in enumerate(text_encoders): + if text_encoder is not None: + text_encoder_loras, skipped = create_modules(False, text_encoder, LoRANetwork.TEXT_ENCODER_TARGET_REPLACE_MODULE) + self.text_encoder_loras.extend(text_encoder_loras) + skipped_te += skipped + print(f"create LoRA for Text Encoder: {len(self.text_encoder_loras)} modules.") + + self.unet_loras, skipped_un = create_modules(True, unet, LoRANetwork.TRANSFORMER_TARGET_REPLACE_MODULE) + print(f"create LoRA for U-Net: {len(self.unet_loras)} modules.") + + # assertion + names = set() + for lora in self.text_encoder_loras + self.unet_loras: + assert lora.lora_name not in names, f"duplicated lora name: {lora.lora_name}" + names.add(lora.lora_name) + + def apply_to(self, text_encoder, unet, apply_text_encoder=True, apply_unet=True): + if apply_text_encoder: + print("enable LoRA for text encoder") + else: + self.text_encoder_loras = [] + + if apply_unet: + print("enable LoRA for U-Net") + else: + self.unet_loras = [] + + for lora in self.text_encoder_loras + self.unet_loras: + lora.apply_to() + self.add_module(lora.lora_name, lora) + + def set_multiplier(self, multiplier): + self.multiplier = multiplier + for lora in self.text_encoder_loras + self.unet_loras: + lora.multiplier = self.multiplier + + def load_weights(self, file): + if os.path.splitext(file)[1] == ".safetensors": + from safetensors.torch import load_file + + weights_sd = load_file(file) + else: + weights_sd = torch.load(file, map_location="cpu") + info = self.load_state_dict(weights_sd, False) + return info + + def prepare_optimizer_params(self, text_encoder_lr, unet_lr, default_lr): + self.requires_grad_(True) + all_params = [] + + def enumerate_params(loras): + params = [] + for lora in loras: + params.extend(lora.parameters()) + return params + + if self.text_encoder_loras: + param_data = {"params": enumerate_params(self.text_encoder_loras)} + if text_encoder_lr is not None: + param_data["lr"] = text_encoder_lr + all_params.append(param_data) + + if self.unet_loras: + param_data = {"params": enumerate_params(self.unet_loras)} + if unet_lr is not None: + param_data["lr"] = unet_lr + all_params.append(param_data) + + return all_params + + def enable_gradient_checkpointing(self): + pass + + def get_trainable_params(self): + return self.parameters() + + def save_weights(self, file, dtype, metadata): + if metadata is not None and len(metadata) == 0: + metadata = None + + state_dict = self.state_dict() + + if dtype is not None: + for key in list(state_dict.keys()): + v = state_dict[key] + v = v.detach().clone().to("cpu").to(dtype) + state_dict[key] = v + + if os.path.splitext(file)[1] == ".safetensors": + from safetensors.torch import save_file + + # Precalculate model hashes to save time on indexing + if metadata is None: + metadata = {} + model_hash, legacy_hash = precalculate_safetensors_hashes(state_dict, metadata) + metadata["sshs_model_hash"] = model_hash + metadata["sshs_legacy_hash"] = legacy_hash + + save_file(state_dict, file, metadata) + else: + torch.save(state_dict, file) + +def create_network( + multiplier: float, + network_dim: Optional[int], + network_alpha: Optional[float], + text_encoder: Union[T5EncoderModel, List[T5EncoderModel]], + transformer, + neuron_dropout: Optional[float] = None, + skip_name: str = None, + target_name: str = None, + **kwargs, +): + if network_dim is None: + network_dim = 4 # default + if network_alpha is None: + network_alpha = 1.0 + + network = LoRANetwork( + text_encoder, + transformer, + multiplier=multiplier, + lora_dim=network_dim, + alpha=network_alpha, + dropout=neuron_dropout, + skip_name=skip_name, + target_name=target_name, + varbose=True, + ) + return network + +def merge_lora(pipeline, lora_path, multiplier, device='cpu', dtype=torch.float32, state_dict=None, transformer_only=False, sub_transformer_name="transformer"): + if lora_path is None: + return pipeline + + LORA_PREFIX_TRANSFORMER = "lora_unet" + LORA_PREFIX_TEXT_ENCODER = "lora_te" + if state_dict is None: + state_dict = load_file(lora_path) + else: + state_dict = state_dict + updates = defaultdict(dict) + for key, value in state_dict.items(): + if "lora_A" in key or "lora_B" in key: + key = "lora_unet__" + key + key = key.replace(".", "_") + if key.endswith("_lora_up_weight"): + key = key[:-15] + ".lora_up.weight" + if key.endswith("_lora_down_weight"): + key = key[:-17] + ".lora_down.weight" + if key.endswith("_lora_A_default_weight"): + key = key[:-21] + ".lora_A.weight" + if key.endswith("_lora_B_default_weight"): + key = key[:-21] + ".lora_B.weight" + if key.endswith("_lora_A_weight"): + key = key[:-14] + ".lora_A.weight" + if key.endswith("_lora_B_weight"): + key = key[:-14] + ".lora_B.weight" + if key.endswith("_alpha"): + key = key[:-6] + ".alpha" + key = key.replace(".lora_A.default.", ".lora_down.") + key = key.replace(".lora_B.default.", ".lora_up.") + key = key.replace(".lora_A.", ".lora_down.") + key = key.replace(".lora_B.", ".lora_up.") + layer, elem = key.split('.', 1) + updates[layer][elem] = value + + sequential_cpu_offload_flag = False + if pipeline.transformer.device == torch.device(type="meta"): + pipeline.remove_all_hooks() + sequential_cpu_offload_flag = True + offload_device = pipeline._offload_device + + for layer, elems in updates.items(): + + if "lora_te" in layer: + if transformer_only: + continue + else: + layer_infos = layer.split(LORA_PREFIX_TEXT_ENCODER + "_")[-1].split("_") + curr_layer = pipeline.text_encoder + else: + layer_infos = layer.split(LORA_PREFIX_TRANSFORMER + "_")[-1].split("_") + curr_layer = getattr(pipeline, sub_transformer_name) + + try: + curr_layer = curr_layer.__getattr__("_".join(layer_infos[1:])) + except Exception: + temp_name = layer_infos.pop(0) + try: + while len(layer_infos) > -1: + try: + curr_layer = curr_layer.__getattr__(temp_name + "_" + "_".join(layer_infos)) + break + except Exception: + try: + curr_layer = curr_layer.__getattr__(temp_name) + if len(layer_infos) > 0: + temp_name = layer_infos.pop(0) + elif len(layer_infos) == 0: + break + except Exception: + if len(layer_infos) == 0: + print(f'Error loading layer in front search: {layer}. Try it in back search.') + if len(temp_name) > 0: + temp_name += "_" + layer_infos.pop(0) + else: + temp_name = layer_infos.pop(0) + except Exception: + if "lora_te" in layer: + if transformer_only: + continue + else: + layer_infos = layer.split(LORA_PREFIX_TEXT_ENCODER + "_")[-1].split("_") + curr_layer = pipeline.text_encoder + else: + layer_infos = layer.split(LORA_PREFIX_TRANSFORMER + "_")[-1].split("_") + curr_layer = getattr(pipeline, sub_transformer_name) + + len_layer_infos = len(layer_infos) + start_index = 0 if len_layer_infos >= 1 and len(layer_infos[0]) > 0 else 1 + end_indx = len_layer_infos + + error_flag = False if len_layer_infos >= 1 else True + while start_index < len_layer_infos: + try: + if start_index >= end_indx: + print(f'Error loading layer in back search: {layer}') + error_flag = True + break + curr_layer = curr_layer.__getattr__("_".join(layer_infos[start_index:end_indx])) + start_index = end_indx + end_indx = len_layer_infos + except Exception: + end_indx -= 1 + if error_flag: + continue + + origin_dtype = curr_layer.weight.data.dtype + origin_device = curr_layer.weight.data.device + + curr_layer = curr_layer.to(device, dtype) + weight_up = elems['lora_up.weight'].to(device, dtype) + weight_down = elems['lora_down.weight'].to(device, dtype) + + if 'alpha' in elems.keys(): + alpha = elems['alpha'].item() / weight_up.shape[1] + else: + alpha = 1.0 + + if len(weight_up.shape) == 4: + curr_layer.weight.data += multiplier * alpha * torch.mm( + weight_up.squeeze(3).squeeze(2), weight_down.squeeze(3).squeeze(2) + ).unsqueeze(2).unsqueeze(3) + else: + curr_layer.weight.data += multiplier * alpha * torch.mm(weight_up, weight_down) + curr_layer = curr_layer.to(origin_device, origin_dtype) + + if sequential_cpu_offload_flag: + pipeline.enable_sequential_cpu_offload(device=offload_device) + return pipeline + +# TODO: Refactor with merge_lora. +def unmerge_lora(pipeline, lora_path, multiplier=1, device="cpu", dtype=torch.float32, sub_transformer_name="transformer"): + if lora_path is None: + return pipeline + + """Unmerge state_dict in LoRANetwork from the pipeline in diffusers.""" + LORA_PREFIX_UNET = "lora_unet" + LORA_PREFIX_TEXT_ENCODER = "lora_te" + state_dict = load_file(lora_path) + + updates = defaultdict(dict) + for key, value in state_dict.items(): + if "lora_A" in key or "lora_B" in key: + key = "lora_unet__" + key + key = key.replace(".", "_") + if key.endswith("_lora_up_weight"): + key = key[:-15] + ".lora_up.weight" + if key.endswith("_lora_down_weight"): + key = key[:-17] + ".lora_down.weight" + if key.endswith("_lora_A_default_weight"): + key = key[:-21] + ".lora_A.weight" + if key.endswith("_lora_B_default_weight"): + key = key[:-21] + ".lora_B.weight" + if key.endswith("_lora_A_weight"): + key = key[:-14] + ".lora_A.weight" + if key.endswith("_lora_B_weight"): + key = key[:-14] + ".lora_B.weight" + if key.endswith("_alpha"): + key = key[:-6] + ".alpha" + key = key.replace(".lora_A.default.", ".lora_down.") + key = key.replace(".lora_B.default.", ".lora_up.") + key = key.replace(".lora_A.", ".lora_down.") + key = key.replace(".lora_B.", ".lora_up.") + layer, elem = key.split('.', 1) + updates[layer][elem] = value + + sequential_cpu_offload_flag = False + if pipeline.transformer.device == torch.device(type="meta"): + pipeline.remove_all_hooks() + sequential_cpu_offload_flag = True + + for layer, elems in updates.items(): + + if "lora_te" in layer: + layer_infos = layer.split(LORA_PREFIX_TEXT_ENCODER + "_")[-1].split("_") + curr_layer = pipeline.text_encoder + else: + layer_infos = layer.split(LORA_PREFIX_UNET + "_")[-1].split("_") + curr_layer = getattr(pipeline, sub_transformer_name) + + try: + curr_layer = curr_layer.__getattr__("_".join(layer_infos[1:])) + except Exception: + temp_name = layer_infos.pop(0) + try: + while len(layer_infos) > -1: + try: + curr_layer = curr_layer.__getattr__(temp_name + "_" + "_".join(layer_infos)) + break + except Exception: + try: + curr_layer = curr_layer.__getattr__(temp_name) + if len(layer_infos) > 0: + temp_name = layer_infos.pop(0) + elif len(layer_infos) == 0: + break + except Exception: + if len(layer_infos) == 0: + print(f'Error loading layer in front search: {layer}. Try it in back search.') + if len(temp_name) > 0: + temp_name += "_" + layer_infos.pop(0) + else: + temp_name = layer_infos.pop(0) + except Exception: + if "lora_te" in layer: + layer_infos = layer.split(LORA_PREFIX_TEXT_ENCODER + "_")[-1].split("_") + curr_layer = pipeline.text_encoder + else: + layer_infos = layer.split(LORA_PREFIX_UNET + "_")[-1].split("_") + curr_layer = getattr(pipeline, sub_transformer_name) + len_layer_infos = len(layer_infos) + + start_index = 0 if len_layer_infos >= 1 and len(layer_infos[0]) > 0 else 1 + end_indx = len_layer_infos + + error_flag = False if len_layer_infos >= 1 else True + while start_index < len_layer_infos: + try: + if start_index >= end_indx: + print(f'Error loading layer in back search: {layer}') + error_flag = True + break + curr_layer = curr_layer.__getattr__("_".join(layer_infos[start_index:end_indx])) + start_index = end_indx + end_indx = len_layer_infos + except Exception: + end_indx -= 1 + if error_flag: + continue + + origin_dtype = curr_layer.weight.data.dtype + origin_device = curr_layer.weight.data.device + + curr_layer = curr_layer.to(device, dtype) + weight_up = elems['lora_up.weight'].to(device, dtype) + weight_down = elems['lora_down.weight'].to(device, dtype) + + if 'alpha' in elems.keys(): + alpha = elems['alpha'].item() / weight_up.shape[1] + else: + alpha = 1.0 + + if len(weight_up.shape) == 4: + curr_layer.weight.data -= multiplier * alpha * torch.mm( + weight_up.squeeze(3).squeeze(2), weight_down.squeeze(3).squeeze(2) + ).unsqueeze(2).unsqueeze(3) + else: + curr_layer.weight.data -= multiplier * alpha * torch.mm(weight_up, weight_down) + curr_layer = curr_layer.to(origin_device, origin_dtype) + + if sequential_cpu_offload_flag: + pipeline.enable_sequential_cpu_offload(device=device) + return pipeline diff --git a/VideoX-Fun/videox_fun/utils/utils.py b/VideoX-Fun/videox_fun/utils/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..730d5af33d7203a6a023d6855761fccc39134b25 --- /dev/null +++ b/VideoX-Fun/videox_fun/utils/utils.py @@ -0,0 +1,447 @@ +import gc +import inspect +import os +import shutil +import subprocess +import time + +import cv2 +import imageio +import numpy as np +import torch +import torchvision +from einops import rearrange +from PIL import Image + + +def filter_kwargs(cls, kwargs): + sig = inspect.signature(cls.__init__) + valid_params = set(sig.parameters.keys()) - {'self', 'cls'} + filtered_kwargs = {k: v for k, v in kwargs.items() if k in valid_params} + return filtered_kwargs + +def get_width_and_height_from_image_and_base_resolution(image, base_resolution): + target_pixels = int(base_resolution) * int(base_resolution) + original_width, original_height = Image.open(image).size + ratio = (target_pixels / (original_width * original_height)) ** 0.5 + width_slider = round(original_width * ratio) + height_slider = round(original_height * ratio) + return height_slider, width_slider + +def color_transfer(sc, dc): + """ + Transfer color distribution from of sc, referred to dc. + + Args: + sc (numpy.ndarray): input image to be transfered. + dc (numpy.ndarray): reference image + + Returns: + numpy.ndarray: Transferred color distribution on the sc. + """ + + def get_mean_and_std(img): + x_mean, x_std = cv2.meanStdDev(img) + x_mean = np.hstack(np.around(x_mean, 2)) + x_std = np.hstack(np.around(x_std, 2)) + return x_mean, x_std + + sc = cv2.cvtColor(sc, cv2.COLOR_RGB2LAB) + s_mean, s_std = get_mean_and_std(sc) + dc = cv2.cvtColor(dc, cv2.COLOR_RGB2LAB) + t_mean, t_std = get_mean_and_std(dc) + img_n = ((sc - s_mean) * (t_std / s_std)) + t_mean + np.putmask(img_n, img_n > 255, 255) + np.putmask(img_n, img_n < 0, 0) + dst = cv2.cvtColor(cv2.convertScaleAbs(img_n), cv2.COLOR_LAB2RGB) + return dst + +def save_videos_grid(videos: torch.Tensor, path: str, rescale=False, n_rows=6, fps=12, imageio_backend=True, color_transfer_post_process=False): + videos = rearrange(videos, "b c t h w -> t b c h w") + outputs = [] + for x in videos: + x = torchvision.utils.make_grid(x, nrow=n_rows) + x = x.transpose(0, 1).transpose(1, 2).squeeze(-1) + if rescale: + x = (x + 1.0) / 2.0 # -1,1 -> 0,1 + x = (x * 255).numpy().astype(np.uint8) + outputs.append(Image.fromarray(x)) + + if color_transfer_post_process: + for i in range(1, len(outputs)): + outputs[i] = Image.fromarray(color_transfer(np.uint8(outputs[i]), np.uint8(outputs[0]))) + + os.makedirs(os.path.dirname(path), exist_ok=True) + if imageio_backend: + if path.endswith("mp4"): + imageio.mimsave(path, outputs, fps=fps) + else: + imageio.mimsave(path, outputs, duration=(1000 * 1/fps)) + else: + if path.endswith("mp4"): + path = path.replace('.mp4', '.gif') + outputs[0].save(path, format='GIF', append_images=outputs, save_all=True, duration=100, loop=0) + +def merge_video_audio(video_path: str, audio_path: str): + """ + Merge the video and audio into a new video, with the duration set to the shorter of the two, + and overwrite the original video file. + + Parameters: + video_path (str): Path to the original video file + audio_path (str): Path to the audio file + """ + # check + if not os.path.exists(video_path): + raise FileNotFoundError(f"video file {video_path} does not exist") + if not os.path.exists(audio_path): + raise FileNotFoundError(f"audio file {audio_path} does not exist") + + base, ext = os.path.splitext(video_path) + temp_output = f"{base}_temp{ext}" + + try: + # create ffmpeg command + command = [ + 'ffmpeg', + '-y', # overwrite + '-i', + video_path, + '-i', + audio_path, + '-c:v', + 'copy', # copy video stream + '-c:a', + 'aac', # use AAC audio encoder + '-b:a', + '192k', # set audio bitrate (optional) + '-map', + '0:v:0', # select the first video stream + '-map', + '1:a:0', # select the first audio stream + '-shortest', # choose the shortest duration + temp_output + ] + + # execute the command + print("Start merging video and audio...") + result = subprocess.run( + command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + + # check result + if result.returncode != 0: + error_msg = f"FFmpeg execute failed: {result.stderr}" + print(error_msg) + raise RuntimeError(error_msg) + + shutil.move(temp_output, video_path) + print(f"Merge completed, saved to {video_path}") + + except Exception as e: + if os.path.exists(temp_output): + os.remove(temp_output) + print(f"merge_video_audio failed with error: {e}") + +def get_image_to_video_latent(validation_image_start, validation_image_end, video_length, sample_size): + if validation_image_start is not None and validation_image_end is not None: + if type(validation_image_start) is str and os.path.isfile(validation_image_start): + image_start = clip_image = Image.open(validation_image_start).convert("RGB") + image_start = image_start.resize([sample_size[1], sample_size[0]]) + clip_image = clip_image.resize([sample_size[1], sample_size[0]]) + else: + image_start = clip_image = validation_image_start + image_start = [_image_start.resize([sample_size[1], sample_size[0]]) for _image_start in image_start] + clip_image = [_clip_image.resize([sample_size[1], sample_size[0]]) for _clip_image in clip_image] + + if type(validation_image_end) is str and os.path.isfile(validation_image_end): + image_end = Image.open(validation_image_end).convert("RGB") + image_end = image_end.resize([sample_size[1], sample_size[0]]) + else: + image_end = validation_image_end + image_end = [_image_end.resize([sample_size[1], sample_size[0]]) for _image_end in image_end] + + if type(image_start) is list: + clip_image = clip_image[0] + start_video = torch.cat( + [torch.from_numpy(np.array(_image_start)).permute(2, 0, 1).unsqueeze(1).unsqueeze(0) for _image_start in image_start], + dim=2 + ) + input_video = torch.tile(start_video[:, :, :1], [1, 1, video_length, 1, 1]) + input_video[:, :, :len(image_start)] = start_video + + input_video_mask = torch.zeros_like(input_video[:, :1]) + input_video_mask[:, :, len(image_start):] = 255 + else: + input_video = torch.tile( + torch.from_numpy(np.array(image_start)).permute(2, 0, 1).unsqueeze(1).unsqueeze(0), + [1, 1, video_length, 1, 1] + ) + input_video_mask = torch.zeros_like(input_video[:, :1]) + input_video_mask[:, :, 1:] = 255 + + if type(image_end) is list: + image_end = [_image_end.resize(image_start[0].size if type(image_start) is list else image_start.size) for _image_end in image_end] + end_video = torch.cat( + [torch.from_numpy(np.array(_image_end)).permute(2, 0, 1).unsqueeze(1).unsqueeze(0) for _image_end in image_end], + dim=2 + ) + input_video[:, :, -len(end_video):] = end_video + + input_video_mask[:, :, -len(image_end):] = 0 + else: + image_end = image_end.resize(image_start[0].size if type(image_start) is list else image_start.size) + input_video[:, :, -1:] = torch.from_numpy(np.array(image_end)).permute(2, 0, 1).unsqueeze(1).unsqueeze(0) + input_video_mask[:, :, -1:] = 0 + + input_video = input_video / 255 + + elif validation_image_start is not None: + if type(validation_image_start) is str and os.path.isfile(validation_image_start): + image_start = clip_image = Image.open(validation_image_start).convert("RGB") + image_start = image_start.resize([sample_size[1], sample_size[0]]) + clip_image = clip_image.resize([sample_size[1], sample_size[0]]) + else: + image_start = clip_image = validation_image_start + image_start = [_image_start.resize([sample_size[1], sample_size[0]]) for _image_start in image_start] + clip_image = [_clip_image.resize([sample_size[1], sample_size[0]]) for _clip_image in clip_image] + image_end = None + + if type(image_start) is list: + clip_image = clip_image[0] + start_video = torch.cat( + [torch.from_numpy(np.array(_image_start)).permute(2, 0, 1).unsqueeze(1).unsqueeze(0) for _image_start in image_start], + dim=2 + ) + input_video = torch.tile(start_video[:, :, :1], [1, 1, video_length, 1, 1]) + input_video[:, :, :len(image_start)] = start_video + input_video = input_video / 255 + + input_video_mask = torch.zeros_like(input_video[:, :1]) + input_video_mask[:, :, len(image_start):] = 255 + else: + input_video = torch.tile( + torch.from_numpy(np.array(image_start)).permute(2, 0, 1).unsqueeze(1).unsqueeze(0), + [1, 1, video_length, 1, 1] + ) / 255 + input_video_mask = torch.zeros_like(input_video[:, :1]) + input_video_mask[:, :, 1:, ] = 255 + else: + image_start = None + image_end = None + input_video = torch.zeros([1, 3, video_length, sample_size[0], sample_size[1]]) + input_video_mask = torch.ones([1, 1, video_length, sample_size[0], sample_size[1]]) * 255 + clip_image = None + + del image_start + del image_end + gc.collect() + + return input_video, input_video_mask, clip_image + +def get_video_to_video_latent(input_video_path, video_length, sample_size, fps=None, validation_video_mask=None, ref_image=None): + if input_video_path is not None: + if isinstance(input_video_path, str): + cap = cv2.VideoCapture(input_video_path) + input_video = [] + + original_fps = cap.get(cv2.CAP_PROP_FPS) + frame_skip = 1 if fps is None else max(1,int(original_fps // fps)) + + frame_count = 0 + + while True: + ret, frame = cap.read() + if not ret: + break + + if frame_count % frame_skip == 0: + frame = cv2.resize(frame, (sample_size[1], sample_size[0])) + input_video.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)) + + frame_count += 1 + + cap.release() + else: + input_video = input_video_path + + input_video = torch.from_numpy(np.array(input_video))[:video_length] + input_video = input_video.permute([3, 0, 1, 2]).unsqueeze(0) / 255 + + if validation_video_mask is not None: + validation_video_mask = Image.open(validation_video_mask).convert('L').resize((sample_size[1], sample_size[0])) + input_video_mask = np.where(np.array(validation_video_mask) < 240, 0, 255) + + input_video_mask = torch.from_numpy(np.array(input_video_mask)).unsqueeze(0).unsqueeze(-1).permute([3, 0, 1, 2]).unsqueeze(0) + input_video_mask = torch.tile(input_video_mask, [1, 1, input_video.size()[2], 1, 1]) + input_video_mask = input_video_mask.to(input_video.device, input_video.dtype) + else: + input_video_mask = torch.zeros_like(input_video[:, :1]) + input_video_mask[:, :, :] = 255 + else: + input_video, input_video_mask = None, None + + if ref_image is not None: + if isinstance(ref_image, str): + clip_image = Image.open(ref_image).convert("RGB") + else: + clip_image = Image.fromarray(np.array(ref_image, np.uint8)) + else: + clip_image = None + + if ref_image is not None: + if isinstance(ref_image, str): + ref_image = Image.open(ref_image).convert("RGB") + ref_image = ref_image.resize((sample_size[1], sample_size[0])) + ref_image = torch.from_numpy(np.array(ref_image)) + ref_image = ref_image.unsqueeze(0).permute([3, 0, 1, 2]).unsqueeze(0) / 255 + else: + ref_image = torch.from_numpy(np.array(ref_image)) + ref_image = ref_image.unsqueeze(0).permute([3, 0, 1, 2]).unsqueeze(0) / 255 + return input_video, input_video_mask, ref_image, clip_image + +def get_image_latent(ref_image=None, sample_size=None, padding=False): + if ref_image is not None: + if isinstance(ref_image, str): + ref_image = Image.open(ref_image).convert("RGB") + if padding: + ref_image = padding_image(ref_image, sample_size[1], sample_size[0]) + ref_image = ref_image.resize((sample_size[1], sample_size[0])) + ref_image = torch.from_numpy(np.array(ref_image)) + ref_image = ref_image.unsqueeze(0).permute([3, 0, 1, 2]).unsqueeze(0) / 255 + elif isinstance(ref_image, Image.Image): + ref_image = ref_image.convert("RGB") + if padding: + ref_image = padding_image(ref_image, sample_size[1], sample_size[0]) + ref_image = ref_image.resize((sample_size[1], sample_size[0])) + ref_image = torch.from_numpy(np.array(ref_image)) + ref_image = ref_image.unsqueeze(0).permute([3, 0, 1, 2]).unsqueeze(0) / 255 + else: + ref_image = torch.from_numpy(np.array(ref_image)) + ref_image = ref_image.unsqueeze(0).permute([3, 0, 1, 2]).unsqueeze(0) / 255 + + return ref_image + +def get_image(ref_image=None): + if ref_image is not None: + if isinstance(ref_image, str): + ref_image = Image.open(ref_image).convert("RGB") + elif isinstance(ref_image, Image.Image): + ref_image = ref_image.convert("RGB") + + return ref_image + +def padding_image(images, new_width, new_height): + new_image = Image.new('RGB', (new_width, new_height), (255, 255, 255)) + + aspect_ratio = images.width / images.height + if new_width / new_height > 1: + if aspect_ratio > new_width / new_height: + new_img_width = new_width + new_img_height = int(new_img_width / aspect_ratio) + else: + new_img_height = new_height + new_img_width = int(new_img_height * aspect_ratio) + else: + if aspect_ratio > new_width / new_height: + new_img_width = new_width + new_img_height = int(new_img_width / aspect_ratio) + else: + new_img_height = new_height + new_img_width = int(new_img_height * aspect_ratio) + + resized_img = images.resize((new_img_width, new_img_height)) + + paste_x = (new_width - new_img_width) // 2 + paste_y = (new_height - new_img_height) // 2 + + new_image.paste(resized_img, (paste_x, paste_y)) + + return new_image + +def timer(func): + def wrapper(*args, **kwargs): + start_time = time.time() + result = func(*args, **kwargs) + end_time = time.time() + print(f"function {func.__name__} running for {end_time - start_time} seconds") + return result + return wrapper + +def timer_record(model_name=""): + def decorator(func): + def wrapper(*args, **kwargs): + torch.cuda.synchronize() + start_time = time.time() + result = func(*args, **kwargs) + torch.cuda.synchronize() + end_time = time.time() + import torch.distributed as dist + if dist.is_initialized(): + if dist.get_rank() == 0: + time_sum = end_time - start_time + print('# --------------------------------------------------------- #') + print(f'# {model_name} time: {time_sum}s') + print('# --------------------------------------------------------- #') + _write_to_excel(model_name, time_sum) + else: + time_sum = end_time - start_time + print('# --------------------------------------------------------- #') + print(f'# {model_name} time: {time_sum}s') + print('# --------------------------------------------------------- #') + _write_to_excel(model_name, time_sum) + return result + return wrapper + return decorator + +def _write_to_excel(model_name, time_sum): + import os + + import pandas as pd + + row_env = os.environ.get(f"{model_name}_EXCEL_ROW", "1") # 默认第1行 + col_env = os.environ.get(f"{model_name}_EXCEL_COL", "1") # 默认第A列 + file_path = os.environ.get("EXCEL_FILE", "timing_records.xlsx") # 默认文件名 + + try: + df = pd.read_excel(file_path, sheet_name="Sheet1", header=None) + except FileNotFoundError: + df = pd.DataFrame() + + row_idx = int(row_env) + col_idx = int(col_env) + + if row_idx >= len(df): + df = pd.concat([df, pd.DataFrame([ [None] * (len(df.columns) if not df.empty else 0) ] * (row_idx - len(df) + 1))], ignore_index=True) + + if col_idx >= len(df.columns): + df = pd.concat([df, pd.DataFrame(columns=range(len(df.columns), col_idx + 1))], axis=1) + + df.iloc[row_idx, col_idx] = time_sum + + df.to_excel(file_path, index=False, header=False, sheet_name="Sheet1") + +def get_autocast_dtype(): + try: + if not torch.cuda.is_available(): + print("CUDA not available, using float16 by default.") + return torch.float16 + + device = torch.cuda.current_device() + prop = torch.cuda.get_device_properties(device) + + print(f"GPU: {prop.name}, Compute Capability: {prop.major}.{prop.minor}") + + if prop.major >= 8: + if torch.cuda.is_bf16_supported(): + print("Using bfloat16.") + return torch.bfloat16 + else: + print("Compute capability >= 8.0 but bfloat16 not supported, falling back to float16.") + return torch.float16 + else: + print("GPU does not support bfloat16 natively, using float16.") + return torch.float16 + + except Exception as e: + print(f"Error detecting GPU capability: {e}, falling back to float16.") + return torch.float16 \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/README.md b/VideoX-Fun/videox_fun/video_caption/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d2aaefd6187805919338baa6ae9c9d799cce20a2 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/README.md @@ -0,0 +1,178 @@ +# Video Caption +English | [简体中文](./README_zh-CN.md) + +The folder contains codes for dataset preprocessing (i.e., video splitting, filtering, and recaptioning), and beautiful prompt used by EasyAnimate. +The entire process supports distributed parallel processing, capable of handling large-scale datasets. + +Meanwhile, we are collaborating with [Data-Juicer](https://github.com/modelscope/data-juicer/blob/main/docs/DJ_SORA.md), +allowing you to easily perform video data processing on [Aliyun PAI-DLC](https://help.aliyun.com/zh/pai/user-guide/video-preprocessing/). + +# Table of Content +- [Video Caption](#video-caption) +- [Table of Content](#table-of-content) + - [Quick Start](#quick-start) + - [Setup](#setup) + - [Data Preprocessing](#data-preprocessing) + - [Data Preparation](#data-preparation) + - [Video Splitting](#video-splitting) + - [Video Filtering](#video-filtering) + - [Video Recaptioning](#video-recaptioning) + - [Beautiful Prompt (For EasyAnimate Inference)](#beautiful-prompt-for-easyanimate-inference) + - [Batched Inference](#batched-inference) + - [OpenAI Server](#openai-server) + +## Quick Start + +### Setup +AliyunDSW or Docker is recommended to setup the environment, please refer to [Quick Start](../../README.md#quick-start). +You can also refer to the image build process in the [Dockerfile](../../Dockerfile.ds) to configure the conda environment and other dependencies locally. + +```shell +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:easyanimate + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:easyanimate + +# clone code +git clone https://github.com/aigc-apps/EasyAnimate.git + +# enter video_caption +cd EasyAnimate/easyanimate/video_caption +``` + +### Data Preprocessing +#### Data Preparation +Place the downloaded videos into a folder under [datasets](./datasets/) (preferably without nested structures, as the video names are used as unique IDs in subsequent processes). +Taking Panda-70M as an example, the entire dataset directory structure is shown as follows: +``` +📦 datasets/ +├── 📂 panda_70m/ +│ ├── 📂 videos/ +│ │ ├── 📂 data/ +│ │ │ └── 📄 --C66yU3LjM_2.mp4 +│ │ │ └── 📄 ... +``` + +#### Video Splitting +EasyAnimate utilizes [PySceneDetect](https://github.com/Breakthrough/PySceneDetect) to identify scene changes within the video +and performs video splitting via FFmpeg based on certain threshold values to ensure consistency of the video clip. +Video clips shorter than 3 seconds will be discarded, and those longer than 10 seconds will be splitted recursively. + +The entire workflow of video splitting is in the [stage_1_video_splitting.sh](./scripts/stage_1_video_splitting.sh). +After running +```shell +sh scripts/stage_1_video_splitting.sh +``` +the video clips are obtained in `easyanimate/video_caption/datasets/panda_70m/videos_clips/data/`. + +#### Video Filtering +Based on the videos obtained in the previous step, EasyAnimate provides a simple yet effective pipeline to filter out high-quality videos for recaptioning. +The overall process is as follows: + +- Scene transition filtering: Filter out videos with scene transition introduced by missing or superfluous splitting of PySceneDetect by calculating the semantic similarity +accoss the beginning frame, the last frame, and the keyframes via [CLIP](https://github.com/openai/CLIP) or [DINOv2](https://github.com/facebookresearch/dinov2). +- Aesthetic filtering: Filter out videos with poor content (blurry, dim, etc.) by calculating the average aesthetic score of uniformly sampled 4 frames via [aesthetic-predictor-v2-5](https://github.com/discus0434/aesthetic-predictor-v2-5). +- Text filtering: Use [EasyOCR](https://github.com/JaidedAI/EasyOCR) to calculate the text area proportion of the middle frame to filter out videos with a large area of text. +- Motion filtering: Calculate interframe optical flow differences to filter out videos that move too slowly or too quickly. + +The entire workflow of video filtering is in the [stage_2_video_filtering.sh](./scripts/stage_2_video_filtering.sh). +After running +```shell +sh scripts/stage_2_video_filtering.sh +``` +the semantic consistency score, aesthetic score, text score, and motion score of videos will be saved +in the corresponding meta files in the folder `easyanimate/video_caption/datasets/panda_70m/videos_clips/`. + +> [!NOTE] +> The computation of semantic consistency score depends on the [openai/clip-vit-large-patch14-336](https://huggingface.co/openai/clip-vit-large-patch14-336). +Meanwhile, the aesthetic score depends on the [google/siglip-so400m-patch14-384 model](https://huggingface.co/google/siglip-so400m-patch14-384). +Please run `HF_ENDPOINT=https://hf-mirror.com sh scripts/stage_2_video_filtering.sh` if you cannot access to huggingface.com. + + +#### Video Recaptioning +After obtaining the aboved high-quality filtered videos, EasyAnimate utilizes [InternVL2](https://internvl.readthedocs.io/en/latest/internvl2.0/introduction.html) to perform video recaptioning. +Subsequently, the recaptioning results are rewritten by LLMs to better meet with the requirements of video generation tasks. +Finally, an advanced [VideoCLIP-XL](https://arxiv.org/abs/2410.00741) model is used to filter out (video, long caption) pairs with poor alignment, resulting in the final training dataset. + +Please download the video caption model from [InternVL2](https://huggingface.co/collections/OpenGVLab/internvl-20-667d3961ab5eb12c7ed1463e) of the appropriate size based on the GPU memory of your machine. +For A100 with 40G VRAM, you can download [InternVL2-40B-AWQ](https://huggingface.co/OpenGVLab/InternVL2-40B-AWQ) by running +```shell +# Add HF_ENDPOINT=https://hf-mirror.com before the command if you cannot access to huggingface.com +huggingface-cli download OpenGVLab/InternVL2-40B-AWQ --local-dir-use-symlinks False --local-dir /PATH/TO/INTERNVL2_MODEL +``` + +Optionally, you can prepare local LLMs to rewrite the recaption results. +For example, you can download [Meta-Llama-3-8B-Instruct](https://huggingface.co/NousResearch/Meta-Llama-3-8B-Instruct) by running +```shell +# Add HF_ENDPOINT=https://hf-mirror.com before the command if you cannot access to huggingface.com +huggingface-cli download NousResearch/Meta-Llama-3-8B-Instruct --local-dir-use-symlinks False --local-dir /PATH/TO/REWRITE_MODEL +``` + +The entire workflow of video recaption is in the [stage_3_video_recaptioning.sh](./scripts/stage_3_video_recaptioning.sh). +After running +```shell +CAPTION_MODEL_PATH=/PATH/TO/INTERNVL2_MODEL REWRITE_MODEL_PATH=/PATH/TO/REWRITE_MODEL sh scripts/stage_3_video_recaptioning.sh +``` +the final train file is obtained in `easyanimate/video_caption/datasets/panda_70m/videos_clips/meta_train_info.json`. + + +### Beautiful Prompt (For EasyAnimate Inference) +Beautiful Prompt aims to rewrite and beautify the user-uploaded prompt via LLMs, mapping it to the style of EasyAnimate's training captions, +making it more suitable as the inference prompt and thus improving the quality of the generated videos. +We support batched inference with local LLMs or OpenAI compatible server based on [vLLM](https://github.com/vllm-project/vllm) for beautiful prompt. + +#### Batched Inference +1. Prepare original prompts in a jsonl file `easyanimate/video_caption/datasets/original_prompt.jsonl` with the following format: + ```json + {"prompt": "A stylish woman in a black leather jacket, red dress, and boots walks confidently down a damp Tokyo street."} + {"prompt": "An underwater world with realistic fish and other creatures of the sea."} + {"prompt": "a monarch butterfly perched on a tree trunk in the forest."} + {"prompt": "a child in a room with a bottle of wine and a lamp."} + {"prompt": "two men in suits walking down a hallway."} + ``` + +2. Then you can perform beautiful prompt by running + ```shell + # Meta-Llama-3-8B-Instruct is sufficient for this task. + # Download it from https://huggingface.co/NousResearch/Meta-Llama-3-8B-Instruct or https://www.modelscope.cn/models/LLM-Research/Meta-Llama-3-8B-Instruct to /path/to/your_llm + + python caption_rewrite.py \ + --video_metadata_path datasets/original_prompt.jsonl \ + --caption_column "prompt" \ + --beautiful_prompt_column "beautiful_prompt" \ + --batch_size 1 \ + --model_name /path/to/your_llm \ + --prompt prompt/beautiful_prompt.txt \ + --prefix '"detailed description": ' \ + --answer_template "your detailed description here" \ + --max_retry_count 10 \ + --saved_path datasets/beautiful_prompt.jsonl \ + --saved_freq 1 + ``` + +#### OpenAI Server ++ You can request OpenAI compatible server to perform beautiful prompt by running + ```shell + OPENAI_API_KEY="your_openai_api_key" OPENAI_BASE_URL="your_openai_base_url" python beautiful_prompt.py \ + --model "your_model_name" \ + --prompt "your_prompt" + ``` + ++ You can also deploy the OpenAI Compatible Server locally using vLLM. For example: + ```shell + # Meta-Llama-3-8B-Instruct is sufficient for this task. + # Download it from https://huggingface.co/NousResearch/Meta-Llama-3-8B-Instruct or https://www.modelscope.cn/models/LLM-Research/Meta-Llama-3-8B-Instruct to /path/to/your_llm + + # deploy the OpenAI compatible server + python -m vllm.entrypoints.openai.api_server serve /path/to/your_llm --dtype auto --api-key "your_api_key" + ``` + + Then you can perform beautiful prompt by running + ```shell + python -m beautiful_prompt.py \ + --model /path/to/your_llm \ + --prompt "your_prompt" \ + --base_url "http://localhost:8000/v1" \ + --api_key "your_api_key" + ``` diff --git a/VideoX-Fun/videox_fun/video_caption/README_zh-CN.md b/VideoX-Fun/videox_fun/video_caption/README_zh-CN.md new file mode 100644 index 0000000000000000000000000000000000000000..f7e8ace51e3ea49ab254c924a8f211345c904094 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/README_zh-CN.md @@ -0,0 +1,161 @@ +# 数据预处理 +[English](./README.md) | 简体中文 + +该文件夹包含 EasyAnimate 使用的数据集预处理（即视频切分、过滤和生成描述）和提示词美化的代码。整个过程支持分布式并行处理，能够处理大规模数据集。 + +此外，我们和 [Data-Juicer](https://github.com/modelscope/data-juicer/blob/main/docs/DJ_SORA.md) 合作，能让你在 [Aliyun PAI-DLC](https://help.aliyun.com/zh/pai/user-guide/video-preprocessing/) 轻松进行视频数据的处理。 + +# 目录 +- [数据预处理](#数据预处理) +- [目录](#目录) + - [快速开始](#快速开始) + - [安装](#安装) + - [数据集预处理](#数据集预处理) + - [数据准备](#数据准备) + - [视频切分](#视频切分) + - [视频过滤](#视频过滤) + - [视频描述](#视频描述) + - [提示词美化](#提示词美化) + - [批量推理](#批量推理) + - [OpenAI 服务器](#openai-服务器) + + +## 快速开始 +### 安装 +推荐使用阿里云 DSW 和 Docker 来安装环境，请参考 [快速开始](../../README_zh-CN.md#1-云使用-aliyundswdocker). 你也可以参考 [Dockerfile](../../Dockerfile.ds) 中的镜像构建流程在本地安装对应的 conda 环境和其余依赖。 + +```shell +# pull image +docker pull mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:easyanimate + +# enter image +docker run -it -p 7860:7860 --network host --gpus all --security-opt seccomp:unconfined --shm-size 200g mybigpai-public-registry.cn-beijing.cr.aliyuncs.com/easycv/torch_cuda:easyanimate + +# clone code +git clone https://github.com/aigc-apps/EasyAnimate.git + +# enter video_caption +cd EasyAnimate/easyanimate/video_caption +``` + +### 数据集预处理 +#### 数据准备 +将下载的视频准备到文件夹 [datasets](./datasets/)（最好不使用嵌套结构，因为视频名称在后续处理中用作唯一 ID）。以 Panda-70M 为例，完整的数据集目录结构如下所示： +``` +📦 datasets/ +├── 📂 panda_70m/ +│ ├── 📂 videos/ +│ │ ├── 📂 data/ +│ │ │ └── 📄 --C66yU3LjM_2.mp4 +│ │ │ └── 📄 ... +``` + +#### 视频切分 +EasyAnimate 使用 [PySceneDetect](https://github.com/Breakthrough/PySceneDetect) 来识别视频中的场景变化 +并根据某些阈值通过 FFmpeg 执行视频分割，以确保视频片段的一致性。 +短于 3 秒的视频片段将被丢弃，长于 10 秒的视频片段将被递归切分。 + +视频切分的完整流程在 [stage_1_video_splitting.sh](./scripts/stage_1_video_splitting.sh)。执行 +```shell +sh scripts/stage_1_video_splitting.sh +``` +后，切分后的视频位于 `easyanimate/video_caption/datasets/panda_70m/videos_clips/data/`。 + +#### 视频过滤 +基于上一步获得的视频，EasyAnimate 提供了一个简单而有效的流程来过滤出高质量的视频。总体流程如下： + +- 场景跳变过滤：通过 [CLIP](https://github.com/openai/CLIP) 或者 [DINOv2](https://github.com/facebookresearch/dinov2) 来计算关键帧和首尾帧的语义相似度，从而过滤掉由于 PySceneDetect 缺失或多余分割引入的场景跳变的视频。 +- 美学过滤：通过 [aesthetic-predictor-v2-5](https://github.com/discus0434/aesthetic-predictor-v2-5) 计算均匀采样的 4 帧视频的平均美学分数，从而筛选出内容不佳（模糊、昏暗等）的视频。 +- 文本过滤：使用 [EasyOCR](https://github.com/JaidedAI/EasyOCR) 计算中间帧的文本区域比例，过滤掉含有大面积文本的视频。 +- 运动过滤：计算帧间光流差，过滤掉移动太慢或太快的视频。 + +视频过滤的完整流程在 [stage_2_video_filtering.sh](./scripts/stage_2_video_filtering.sh)。执行 +```shell +sh scripts/stage_2_video_filtering.sh +``` +后，视频的美学得分、文本得分和运动得分对应的元文件保存在 `easyanimate/video_caption/datasets/panda_70m/videos_clips/`。 + +> [!NOTE] +> 美学得分的计算依赖于 [google/siglip-so400m-patch14-384 model](https://huggingface.co/google/siglip-so400m-patch14-384). +请执行 `HF_ENDPOINT=https://hf-mirror.com sh scripts/stage_2_video_filtering.sh` 如果你无法访问 huggingface.com. + +#### 视频描述 +在获得上述高质量的过滤视频后，EasyAnimate 利用 [InternVL2](https://internvl.readthedocs.io/en/latest/internvl2.0/introduction.html) 来生成视频描述。随后，使用 LLMs 对生成的视频描述进行重写，以更好地满足视频生成任务的要求。最后，使用自研的 [VideoCLIP-XL](https://arxiv.org/abs/2410.00741) 模型来过滤掉描述和视频内容不一致的数据，从而得到最终的训练数据集。 + +请根据机器的显存从 [InternVL2](https://huggingface.co/collections/OpenGVLab/internvl-20-667d3961ab5eb12c7ed1463e) 下载合适大小的模型。对于 A100 40G，你可以执行下面的命令来下载 [InternVL2-40B-AWQ](https://huggingface.co/OpenGVLab/InternVL2-40B-AWQ) +```shell +# Add HF_ENDPOINT=https://hf-mirror.com before the command if you cannot access to huggingface.com +huggingface-cli download OpenGVLab/InternVL2-40B-AWQ --local-dir-use-symlinks False --local-dir /PATH/TO/INTERNVL2_MODEL +``` + +你可以选择性地准备 LLMs 来改写上述视频描述的结果。例如，你执行下面的命令来下载 [Meta-Llama-3-8B-Instruct](https://huggingface.co/NousResearch/Meta-Llama-3-8B-Instruct) +```shell +# Add HF_ENDPOINT=https://hf-mirror.com before the command if you cannot access to huggingface.com +huggingface-cli download NousResearch/Meta-Llama-3-8B-Instruct --local-dir-use-symlinks False --local-dir /PATH/TO/REWRITE_MODEL +``` + +视频描述的完整流程在 [stage_3_video_recaptioning.sh](./scripts/stage_3_video_recaptioning.sh). +执行 +```shell +CAPTION_MODEL_PATH=/PATH/TO/INTERNVL2_MODEL REWRITE_MODEL_PATH=/PATH/TO/REWRITE_MODEL sh scripts/stage_3_video_recaptioning.sh +``` +后，最后的训练文件会保存在 `easyanimate/video_caption/datasets/panda_70m/videos_clips/meta_train_info.json`。 + +### 提示词美化 +提示词美化旨在通过 LLMs 重写和美化用户上传的提示，将其映射为 EasyAnimate 训练所使用的视频描述风格、 +使其更适合用作推理提示词，从而提高生成视频的质量。 + +基于 [vLLM](https://github.com/vllm-project/vllm)，我们支持使用本地 LLM 进行批量推理或请求 OpenAI 服务器的方式，以进行提示词美化。 + +#### 批量推理 +1. 将原始的提示词以下面的格式准备在文件 `easyanimate/video_caption/datasets/original_prompt.jsonl` 中： + ```json + {"prompt": "A stylish woman in a black leather jacket, red dress, and boots walks confidently down a damp Tokyo street."} + {"prompt": "An underwater world with realistic fish and other creatures of the sea."} + {"prompt": "a monarch butterfly perched on a tree trunk in the forest."} + {"prompt": "a child in a room with a bottle of wine and a lamp."} + {"prompt": "two men in suits walking down a hallway."} + ``` + +2. 随后你可以通过执行以下的命令进行提示词美化 + ```shell + # Meta-Llama-3-8B-Instruct is sufficient for this task. + # Download it from https://huggingface.co/NousResearch/Meta-Llama-3-8B-Instruct or https://www.modelscope.cn/models/LLM-Research/Meta-Llama-3-8B-Instruct to /path/to/your_llm + + python caption_rewrite.py \ + --video_metadata_path datasets/original_prompt.jsonl \ + --caption_column "prompt" \ + --beautiful_prompt_column "beautiful_prompt" \ + --batch_size 1 \ + --model_name /path/to/your_llm \ + --prompt prompt/beautiful_prompt.txt \ + --prefix '"detailed description": ' \ + --answer_template "your detailed description here" \ + --max_retry_count 10 \ + --saved_path datasets/beautiful_prompt.jsonl \ + --saved_freq 1 + ``` + +#### OpenAI 服务器 ++ 你可以通过请求 OpenAI 服务器的方式来进行提示词美化 + ```shell + OPENAI_API_KEY="your_openai_api_key" OPENAI_BASE_URL="your_openai_base_url" python beautiful_prompt.py \ + --model "your_model_name" \ + --prompt "your_prompt" + ``` + ++ 你也可以执行以下命令，通过 vLLM 将本地 LLMs 部署成兼容 OpenAI 的服务器 + ```shell + OPENAI_API_KEY="your_openai_api_key" OPENAI_BASE_URL="your_openai_base_url" python beautiful_prompt.py \ + --model "your_model_name" \ + --prompt "your_prompt" + ``` + + 然后再执行下面的命令来进行提示词美化 + ```shell + python -m beautiful_prompt.py \ + --model /path/to/your_llm \ + --prompt "your_prompt" \ + --base_url "http://localhost:8000/v1" \ + --api_key "your_api_key" + ``` \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/beautiful_prompt.py b/VideoX-Fun/videox_fun/video_caption/beautiful_prompt.py new file mode 100644 index 0000000000000000000000000000000000000000..2e206317de4fa82df5eb4c3671e7418d81a15659 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/beautiful_prompt.py @@ -0,0 +1,103 @@ +""" +This script (optional) can rewrite and beautify the user-uploaded prompt via LLMs, mapping it to the style of EasyAnimate's training captions, +making it more suitable as the inference prompt and thus improving the quality of the generated videos. + +Usage: ++ You can request OpenAI compatible server to perform beautiful prompt by running +```shell +export OPENAI_API_KEY="your_openai_api_key" OPENAI_BASE_URL="your_openai_base_url" python beautiful_prompt.py \ + --model "your_model_name" \ + --prompt "your_prompt" +``` ++ You can also deploy the OpenAI Compatible Server locally using vLLM. For example: +```shell +# Meta-Llama-3-8B-Instruct is sufficient for this task. +# Download it from https://huggingface.co/NousResearch/Meta-Llama-3-8B-Instruct or https://www.modelscope.cn/models/LLM-Research/Meta-Llama-3-8B-Instruct to /path/to/your_llm + +# deploy the OpenAI compatible server +python -m vllm.entrypoints.openai.api_server serve /path/to/your_llm --dtype auto --api-key "your_api_key" +``` + +Then you can perform beautiful prompt by running +```shell +python -m beautiful_prompt.py \ + --model /path/to/your_llm \ + --prompt "your_prompt" \ + --base_url "http://localhost:8000/v1" \ + --api_key "your_api_key" +``` +""" +import argparse +import os + +from openai import OpenAI + +from easyanimate.video_caption.caption_rewrite import extract_output + + +def parse_args(): + parser = argparse.ArgumentParser(description="Beautiful prompt.") + parser.add_argument("--model", type=str, required=True, help="The OpenAI model or the path to your local LLM.") + parser.add_argument("--prompt", type=str, required=True, help="The user-uploaded prompt.") + parser.add_argument( + "--template", + type=str, + default="easyanimate/video_caption/prompt/beautiful_prompt.txt", + help="A string or a txt file contains the template for beautiful prompt." + ) + parser.add_argument( + "--max_retry_nums", + type=int, + default=5, + help="Maximum number of retries to obtain an output that meets the JSON format." + ) + parser.add_argument( + "--base_url", + type=str, + default=None, + help="OpenAI API server url. If it is None, the OPENAI_BASE_URL from the environment variables will be used.", + ) + parser.add_argument( + "--api_key", + type=str, + default=None, + help="OpenAI API key. If it is None, the OPENAI_API_KEY from the environment variables will be used.", + ) + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + client = OpenAI( + base_url=os.getenv("OPENAI_BASE_URL", args.base_url), + api_key=os.environ.get("OPENAI_API_KEY", args.api_key), + ) + if args.template.endswith(".txt") and os.path.exists(args.template): + with open(args.template, "r") as f: + args.template = "".join(f.readlines()) + # print(f"Beautiful prompt template: {args.template}") + + for _ in range(args.max_retry_nums): + completion = client.chat.completions.create( + model=args.model, + messages=[ + # {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": args.template + "\n" + str(args.prompt)} + ], + temperature=0.7, + top_p=1, + max_tokens=1024, + ) + + output = completion.choices[0].message.content + output = extract_output(output, prefix='"detailed description": ') + if output is not None: + break + print(f"Beautiful prompt: {output}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/caption_rewrite.py b/VideoX-Fun/videox_fun/video_caption/caption_rewrite.py new file mode 100644 index 0000000000000000000000000000000000000000..dfa0a959fc247db015daa9c430ede54edf0fcd8b --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/caption_rewrite.py @@ -0,0 +1,298 @@ +import argparse +import os +import re +from copy import deepcopy + +import pandas as pd +import torch +from natsort import index_natsorted +from tqdm import tqdm +from transformers import AutoTokenizer +from vllm import LLM, SamplingParams + +from utils.logger import logger + + +def extract_output(s, prefix='"rewritten description": '): + """Customize the function according to the prompt.""" + # Since some LLMs struggles to output strictly formatted JSON strings as specified by the prompt, + # thus manually parse the output string `{"rewritten description": "your rewritten description here"}`. + match = re.search(r"{(.+?)}", s, re.DOTALL) + if not match: + logger.warning(f"{s} is not in the json format. Return None.") + return None + output = match.group(1).strip() + if output.startswith(prefix): + output = output[len(prefix) :] + if output[0] == '"' and output[-1] == '"': + return output[1:-1] + else: + logger.warning(f"{output} does not start and end with the double quote. Return None.") + return None + else: + logger.warning(f"{output} does not start with {prefix}. Return None.") + return None + +"""The file unifies the following two tasks: +1. Caption Rewrite: rewrite the video recaption results by LLMs. +2. Beautiful Prompt: rewrite and beautify the user-uploaded prompt via LLMs. + +For the caption rewrite task, the input video_metadata_path should have the following format: +```jsonl +{"video_path_column": "1.mp4", "caption_column": "a man is running in the street."} +... +{"video_path_column": "100.mp4", "caption_column": "a dog is chasing a cat."} +``` +The video_path_column in the argparse must be specified. + +For the beautiful prompt task, the input video_metadata_path should have the following format: +```jsonl +{"caption_column": "a man is running in the street."} +... +{"caption_column": "a dog is chasing a cat."} +``` +The beautiful_prompt_column in the argparse must be specified for the saving purpose. +""" + +def parse_args(): + parser = argparse.ArgumentParser(description="Rewrite the video caption by LLMs.") + parser.add_argument( + "--video_metadata_path", type=str, required=True, help="The path to the video dataset metadata (csv/jsonl)." + ) + parser.add_argument( + "--video_path_column", + type=str, + default=None, + help=( + "The column contains the video path (an absolute path or a relative path w.r.t the video_folder)." + "It is conflicted with the beautiful_prompt_column." + ), + ) + parser.add_argument( + "--caption_column", + type=str, + default="caption", + help="The column contains the video caption.", + ) + parser.add_argument( + "--beautiful_prompt_column", + type=str, + default=None, + help="The column name for the beautiful prompt column. It is conflicted with the video_path_column.", + ) + parser.add_argument( + "--batch_size", + type=int, + default=128, + required=False, + help="The batch size for vllm inference. Adjust according to the number of GPUs to maximize inference throughput.", + ) + parser.add_argument( + "--model_name", + type=str, + default="NousResearch/Meta-Llama-3-8B-Instruct", + ) + parser.add_argument( + "--prompt", + type=str, + required=True, + help="A string or a txt file contains the prompt.", + ) + parser.add_argument( + "--prefix", + type=str, + required=True, + help="The prefix to extract the output from LLMs.", + ) + parser.add_argument( + "--answer_template", + type=str, + default="", + help="The anwer template in the prompt. If specified, rewritten results same as the answer template will be removed.", + ) + parser.add_argument( + "--max_retry_count", + type=int, + default=1, + help="The maximum retry count to ensure outputs with the valid format from LLMs.", + ) + parser.add_argument("--saved_path", type=str, required=True, help="The save path to the output results (csv/jsonl).") + parser.add_argument("--saved_freq", type=int, default=1, help="The frequency to save the output results.") + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + if args.video_metadata_path.endswith(".csv"): + video_metadata_df = pd.read_csv(args.video_metadata_path) + elif args.video_metadata_path.endswith(".jsonl"): + video_metadata_df = pd.read_json(args.video_metadata_path, lines=True) + elif args.video_metadata_path.endswith(".json"): + video_metadata_df = pd.read_json(args.video_metadata_path) + else: + raise ValueError(f"The {args.video_metadata_path} must end with .csv, .jsonl or .json.") + + saved_suffix = os.path.splitext(args.saved_path)[1] + if saved_suffix not in set([".csv", ".jsonl", ".json"]): + raise ValueError(f"The saved_path must end with .csv, .jsonl or .json.") + + if args.video_path_column is None and args.beautiful_prompt_column is None: + raise ValueError("Either video_path_column or beautiful_prompt_column should be specified in the arguments.") + if args.video_path_column is not None and args.beautiful_prompt_column is not None: + raise ValueError( + "Both video_path_column and beautiful_prompt_column can not be specified in the arguments at the same time." + ) + + if os.path.exists(args.saved_path): + if args.saved_path.endswith(".csv"): + saved_metadata_df = pd.read_csv(args.saved_path) + elif args.saved_path.endswith(".jsonl"): + saved_metadata_df = pd.read_json(args.saved_path, lines=True) + + if args.video_path_column is not None: + # Filter out the unprocessed video-caption pairs by setting the indicator=True. + merged_df = video_metadata_df.merge(saved_metadata_df, on=args.video_path_column, how="outer", indicator=True) + video_metadata_df = merged_df[merged_df["_merge"] == "left_only"] + # Sorting to guarantee the same result for each process. + video_metadata_df = video_metadata_df.iloc[index_natsorted(video_metadata_df[args.video_path_column])] + video_metadata_df = video_metadata_df.reset_index(drop=True) + if args.beautiful_prompt_column is not None: + # Filter out the unprocessed caption-beautifil_prompt pairs by setting the indicator=True. + merged_df = video_metadata_df.merge(saved_metadata_df, on=args.caption_column, how="outer", indicator=True) + video_metadata_df = merged_df[merged_df["_merge"] == "left_only"] + # Sorting to guarantee the same result for each process. + video_metadata_df = video_metadata_df.iloc[index_natsorted(video_metadata_df[args.caption_column])] + video_metadata_df = video_metadata_df.reset_index(drop=True) + logger.info( + f"Resume from {args.saved_path}: {len(saved_metadata_df)} processed and {len(video_metadata_df)} to be processed." + ) + + if args.prompt.endswith(".txt") and os.path.exists(args.prompt): + with open(args.prompt, "r") as f: + args.prompt = "".join(f.readlines()) + logger.info(f"Prompt: {args.prompt}") + + if args.max_retry_count < 1: + raise ValueError(f"The max_retry_count {args.max_retry_count} must be greater than 0.") + + if args.video_path_column is not None: + video_path_list = video_metadata_df[args.video_path_column].tolist() + if args.caption_column in video_metadata_df.columns: + sampled_frame_caption_list = video_metadata_df[args.caption_column].tolist() + else: + # When two columns with the same name, the dataframe merge operation on will distinguish them by adding 'x' and 'y'. + sampled_frame_caption_list = video_metadata_df[args.caption_column + "_x"].tolist() + + CUDA_VISIBLE_DEVICES = os.getenv("CUDA_VISIBLE_DEVICES", None) + tensor_parallel_size = torch.cuda.device_count() if CUDA_VISIBLE_DEVICES is None else len(CUDA_VISIBLE_DEVICES.split(",")) + logger.info(f"Automatically set tensor_parallel_size={tensor_parallel_size} based on the available devices.") + + llm = LLM(model=args.model_name, trust_remote_code=True, tensor_parallel_size=tensor_parallel_size) + if "Meta-Llama-3" in args.model_name: + if "Meta-Llama-3-70B" in args.model_name: + # Llama-3-70B should use the tokenizer from Llama-3-8B + # https://github.com/vllm-project/vllm/issues/4180#issuecomment-2068292942 + tokenizer = AutoTokenizer.from_pretrained("NousResearch/Meta-Llama-3-8B-Instruct") + else: + tokenizer = AutoTokenizer.from_pretrained(args.model_name) + stop_token_ids = [tokenizer.eos_token_id, tokenizer.convert_tokens_to_ids("<|eot_id|>")] + sampling_params = SamplingParams(temperature=0.7, top_p=1, max_tokens=1024, stop_token_ids=stop_token_ids) + else: + tokenizer = AutoTokenizer.from_pretrained(args.model_name) + sampling_params = SamplingParams(temperature=0.7, top_p=1, max_tokens=1024) + + if args.video_path_column is not None: + result_dict = {args.video_path_column: [], args.caption_column: []} + if args.beautiful_prompt_column is not None: + result_dict = {args.caption_column: [], args.beautiful_prompt_column: []} + + for i in tqdm(range(0, len(sampled_frame_caption_list), args.batch_size)): + if args.video_path_column is not None: + batch_video_path = video_path_list[i : i + args.batch_size] + batch_caption = sampled_frame_caption_list[i : i + args.batch_size] + batch_prompt = [] + for caption in batch_caption: + # batch_prompt.append("user:" + args.prompt + str(caption) + "\n assistant:") + messages = [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": args.prompt + "\n" + str(caption)}, + ] + text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + batch_prompt.append(text) + + cur_retry_count = 0 + while cur_retry_count < args.max_retry_count: + if len(batch_prompt) == 0: + break + + batch_result = [] + batch_output = llm.generate(batch_prompt, sampling_params) + batch_output = [output.outputs[0].text.rstrip() for output in batch_output] + if args.prefix is not None: + batch_output = [extract_output(output, args.prefix) for output in batch_output] + + if args.video_path_column is not None: + retry_batch_video_path, retry_batch_prompt = [], [] + for (video_path, prompt, output) in zip(batch_video_path, batch_prompt, batch_output): + # Filter out data that does not meet the output format to retry. + if output is not None and output != args.answer_template: + batch_result.append((video_path, output)) + else: + retry_batch_video_path.append(video_path) + retry_batch_prompt.append(prompt) + if len(batch_result) != 0: + batch_video_path, batch_output = zip(*batch_result) + result_dict[args.video_path_column].extend(deepcopy(batch_video_path)) + result_dict[args.caption_column].extend(deepcopy(batch_output)) + + batch_video_path, batch_prompt = retry_batch_video_path, retry_batch_prompt + if args.beautiful_prompt_column is not None: + retry_batch_caption, retry_batch_prompt = [], [] + for (caption, prompt, output) in zip(batch_caption, batch_prompt, batch_output): + # Filter out data that does not meet the output format to retry. + if output is not None and output != args.answer_template: + batch_result.append((caption, output)) + else: + retry_batch_caption.append(caption) + retry_batch_prompt.append(prompt) + if len(batch_result) != 0: + batch_caption, batch_output = zip(*batch_result) + result_dict[args.caption_column].extend(deepcopy(batch_caption)) + result_dict[args.beautiful_prompt_column].extend(deepcopy(batch_output)) + + batch_caption, batch_prompt = retry_batch_caption, retry_batch_prompt + + cur_retry_count += 1 + logger.info( + f"Current retry count/Maximum retry count: {cur_retry_count}/{args.max_retry_count}.: " + f"Retrying {len(batch_prompt)} prompts with invalid output format." + ) + + # Save the metadata every args.saved_freq. + if (i // args.batch_size) % args.saved_freq == 0 or (i + 1) * args.batch_size >= len(sampled_frame_caption_list): + if len(result_dict[args.caption_column]) > 0: + result_df = pd.DataFrame(result_dict) + # Append is not supported (oss). + if args.saved_path.endswith(".csv"): + if os.path.exists(args.saved_path): + saved_df = pd.read_csv(args.saved_path) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df.to_csv(args.saved_path, index=False) + elif args.saved_path.endswith(".jsonl"): + if os.path.exists(args.saved_path): + saved_df = pd.read_json(args.saved_path, orient="records", lines=True) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df.to_json(args.saved_path, orient="records", lines=True, force_ascii=False) + logger.info(f"Save result to {args.saved_path}.") + + result_dict = {args.caption_column: []} + if args.video_path_column is not None: + result_dict = {args.video_path_column: [], args.caption_column: []} + if args.beautiful_prompt_column is not None: + result_dict = {args.caption_column: [], args.beautiful_prompt_column: []} + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/videox_fun/video_caption/compute_motion_score.py b/VideoX-Fun/videox_fun/video_caption/compute_motion_score.py new file mode 100644 index 0000000000000000000000000000000000000000..b54f7b3d89ec92397c559890fa881a455b264a92 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/compute_motion_score.py @@ -0,0 +1,208 @@ +import argparse +import gc +import os +from contextlib import contextmanager + +import cv2 +import numpy as np +import pandas as pd +from joblib import Parallel, delayed +from natsort import natsorted +from tqdm import tqdm + +from utils.filter import filter +from utils.logger import logger + + +@contextmanager +def VideoCapture(video_path): + cap = cv2.VideoCapture(video_path) + try: + yield cap + finally: + cap.release() + del cap + gc.collect() + + +def compute_motion_score(video_path): + video_motion_scores = [] + sampling_fps = 2 + + try: + with VideoCapture(video_path) as cap: + fps = cap.get(cv2.CAP_PROP_FPS) + valid_fps = min(max(sampling_fps, 1), fps) + frame_interval = int(fps / valid_fps) + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + + # if cannot get the second frame, use the last one + frame_interval = min(frame_interval, total_frames - 1) + + prev_frame = None + frame_count = -1 + while cap.isOpened(): + ret, frame = cap.read() + frame_count += 1 + + if not ret: + break + + # skip middle frames + if frame_count % frame_interval != 0: + continue + + gray_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) + if prev_frame is None: + prev_frame = gray_frame + continue + + flow = cv2.calcOpticalFlowFarneback( + prev_frame, + gray_frame, + None, + pyr_scale=0.5, + levels=3, + winsize=15, + iterations=3, + poly_n=5, + poly_sigma=1.2, + flags=0, + ) + mag, _ = cv2.cartToPolar(flow[..., 0], flow[..., 1]) + frame_motion_score = np.mean(mag) + video_motion_scores.append(frame_motion_score) + prev_frame = gray_frame + + motion_score_result = { + "video_path": video_path, + "motion_score": round(float(np.mean(video_motion_scores)), 5), + } + return motion_score_result + + except Exception as e: + print(f"Compute motion score for video {video_path} with error: {e}.") + + +def parse_args(): + parser = argparse.ArgumentParser(description="Compute the motion score of the videos.") + parser.add_argument("--video_folder", type=str, default="", help="The video folder.") + parser.add_argument( + "--video_metadata_path", type=str, default=None, help="The path to the video dataset metadata (csv/jsonl)." + ) + parser.add_argument( + "--video_path_column", + type=str, + default="video_path", + help="The column contains the video path (an absolute path or a relative path w.r.t the video_folder).", + ) + parser.add_argument("--saved_path", type=str, required=True, help="The save path to the output results (csv/jsonl).") + parser.add_argument("--saved_freq", type=int, default=1, help="The frequency to save the output results.") + parser.add_argument("--n_jobs", type=int, default=1, help="The number of concurrent processes.") + + parser.add_argument("--basic_metadata_path", type=str, default=None, help="The path to the basic metadata (csv/jsonl).") + parser.add_argument("--min_resolution", type=float, default=0, help="The resolution threshold.") + parser.add_argument("--min_duration", type=float, default=-1, help="The minimum duration.") + parser.add_argument("--max_duration", type=float, default=-1, help="The maximum duration.") + parser.add_argument( + "--aesthetic_score_metadata_path", type=str, default=None, help="The path to the video quality metadata (csv/jsonl)." + ) + parser.add_argument("--min_aesthetic_score", type=float, default=4.0, help="The aesthetic score threshold.") + parser.add_argument( + "--aesthetic_score_siglip_metadata_path", type=str, default=None, help="The path to the video quality metadata (csv/jsonl)." + ) + parser.add_argument("--min_aesthetic_score_siglip", type=float, default=4.0, help="The aesthetic score (SigLIP) threshold.") + parser.add_argument( + "--text_score_metadata_path", type=str, default=None, help="The path to the video text score metadata (csv/jsonl)." + ) + parser.add_argument("--min_text_score", type=float, default=0.02, help="The text threshold.") + parser.add_argument( + "--semantic_consistency_score_metadata_path", + nargs="+", + type=str, + default=None, + help="The path to the semantic consistency metadata (csv/jsonl)." + ) + parser.add_argument( + "--min_semantic_consistency_score", type=float, default=0.80, help="The semantic consistency score threshold." + ) + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + if args.video_metadata_path.endswith(".csv"): + video_metadata_df = pd.read_csv(args.video_metadata_path) + elif args.video_metadata_path.endswith(".jsonl"): + video_metadata_df = pd.read_json(args.video_metadata_path, lines=True) + else: + raise ValueError("The video_metadata_path must end with .csv or .jsonl.") + video_path_list = video_metadata_df[args.video_path_column].tolist() + + if not (args.saved_path.endswith(".csv") or args.saved_path.endswith(".jsonl")): + raise ValueError("The saved_path must end with .csv or .jsonl.") + + if os.path.exists(args.saved_path): + if args.saved_path.endswith(".csv"): + saved_metadata_df = pd.read_csv(args.saved_path) + elif args.saved_path.endswith(".jsonl"): + saved_metadata_df = pd.read_json(args.saved_path, lines=True) + saved_video_path_list = saved_metadata_df[args.video_path_column].tolist() + video_path_list = list(set(video_path_list).difference(set(saved_video_path_list))) + logger.info(f"Resume from {args.saved_path}: {len(saved_video_path_list)} processed and {len(video_path_list)} to be processed.") + + video_path_list = filter( + video_path_list, + basic_metadata_path=args.basic_metadata_path, + min_resolution=args.min_resolution, + min_duration=args.min_duration, + max_duration=args.max_duration, + aesthetic_score_metadata_path=args.aesthetic_score_metadata_path, + min_aesthetic_score=args.min_aesthetic_score, + aesthetic_score_siglip_metadata_path=args.aesthetic_score_siglip_metadata_path, + min_aesthetic_score_siglip=args.min_aesthetic_score_siglip, + text_score_metadata_path=args.text_score_metadata_path, + min_text_score=args.min_text_score, + semantic_consistency_score_metadata_path=args.semantic_consistency_score_metadata_path, + min_semantic_consistency_score=args.min_semantic_consistency_score, + video_path_column=args.video_path_column + ) + video_path_list = [os.path.join(args.video_folder, video_path) for video_path in video_path_list] + # Sorting to guarantee the same result for each process. + video_path_list = natsorted(video_path_list) + logger.info(f"{len(video_path_list)} videos are to be processed.") + + for i in tqdm(range(0, len(video_path_list), args.saved_freq)): + # Get motion score result for each video asynchronously. + motion_score_result_list = Parallel(n_jobs=args.n_jobs)( + delayed(compute_motion_score)(video_path) for video_path in tqdm(video_path_list[i: i + args.saved_freq]) + ) + result_list = [] + for motion_score_result in motion_score_result_list: + if motion_score_result is not None: + video_path = motion_score_result["video_path"] + if args.video_folder != "": + video_path = os.path.relpath(video_path, args.video_folder) + result_list.append({args.video_path_column: video_path, "motion_score": motion_score_result["motion_score"]}) + if len(result_list) == 0: + continue + + result_df = pd.DataFrame(result_list) + # Append is not supported (oss). + if args.saved_path.endswith(".csv"): + if os.path.exists(args.saved_path): + saved_df = pd.read_csv(args.saved_path) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df.to_csv(args.saved_path, index=False) + elif args.saved_path.endswith(".jsonl"): + if os.path.exists(args.saved_path): + saved_df = pd.read_json(args.saved_path, orient="records", lines=True) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df.to_json(args.saved_path, orient="records", lines=True, force_ascii=False) + logger.info(f"Save result to {args.saved_path}.") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/compute_semantic_consistency.py b/VideoX-Fun/videox_fun/video_caption/compute_semantic_consistency.py new file mode 100644 index 0000000000000000000000000000000000000000..993bdb1cd498bdabd4f4f35f008e9b342eb0d6d3 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/compute_semantic_consistency.py @@ -0,0 +1,235 @@ +import argparse +import os + +import numpy as np +import pandas as pd +import torch +from accelerate import PartialState +from accelerate.utils import gather_object +from natsort import natsorted +from torch.utils.data import DataLoader +from tqdm import tqdm +from transformers import AutoImageProcessor, AutoModel + +from utils.filter import filter +from utils.logger import logger +from utils.video_dataset import VideoDataset, collate_fn +from utils.video_utils import ALL_FRAME_SAMPLE_METHODS + + +ALL_MODEL_NAME = [ + "dinov2-small", + "dinov2-base", + "dinov2-large", + "clip-vit-large-patch14", + "clip-vit-base-patch32", + "clip-vit-large-patch14-336", +] + + +def init_model(model_name, device): + processor = AutoImageProcessor.from_pretrained(model_name) + model = AutoModel.from_pretrained(model_name).to(device) + return processor, model + + +def compute_adjacent_similarity(frame_features): + frame_features /= frame_features.norm(dim=-1, keepdim=True) + roll_frame_features = torch.roll(frame_features, shifts=-1, dims=0) + similarity_matrix = frame_features.squeeze(dim=1).cpu().numpy() @ roll_frame_features.squeeze(dim=1).cpu().numpy().T + + return np.diag(similarity_matrix).tolist()[:-1] + + +def parse_args(): + parser = argparse.ArgumentParser(description="Compute the semantic consistency score across frames.") + parser.add_argument( + "--video_metadata_path", type=str, required=True, help="The path to the video dataset metadata (csv/jsonl)." + ) + parser.add_argument( + "--video_path_column", + type=str, + default="video_path", + help="The column contains the video path (an absolute path or a relative path w.r.t the video_folder).", + ) + parser.add_argument("--video_folder", type=str, default="", help="The video folder.") + parser.add_argument( + "--model_path", type=str, default="openai/clip-vit-large-patch14-336", help="The path to the DINO/CLIP model." + ) + parser.add_argument("--frame_sample_method", type=str, choices=ALL_FRAME_SAMPLE_METHODS, default="keyframe+first") + parser.add_argument("--num_sampled_frames", type=int, default=1, help="The number of sampled frames.") + parser.add_argument("--sample_stride", type=int, default=None, help="The stride between two sampled frames.") + parser.add_argument("--batch_size", type=int, default=1, help="The batch size for the video dataset.") + parser.add_argument("--num_workers", type=int, default=1, help="The number of workers for the video dataset.") + parser.add_argument("--saved_path", type=str, required=True, help="The save path to the output results (csv/jsonl).") + parser.add_argument("--saved_freq", type=int, default=1, help="The frequency to save the output results.") + + parser.add_argument("--basic_metadata_path", type=str, default=None, help="The path to the basic metadata (csv/jsonl).") + parser.add_argument("--min_resolution", type=float, default=0, help="The resolution threshold.") + parser.add_argument("--min_duration", type=float, default=-1, help="The minimum duration.") + parser.add_argument("--max_duration", type=float, default=-1, help="The maximum duration.") + parser.add_argument( + "--aesthetic_score_metadata_path", type=str, default=None, help="The path to the video quality metadata (csv/jsonl)." + ) + parser.add_argument("--min_aesthetic_score", type=float, default=4.0, help="The aesthetic score threshold.") + parser.add_argument( + "--aesthetic_score_siglip_metadata_path", type=str, default=None, help="The path to the video quality metadata (csv/jsonl)." + ) + parser.add_argument("--min_aesthetic_score_siglip", type=float, default=4.0, help="The aesthetic score (SigLIP) threshold.") + parser.add_argument( + "--text_score_metadata_path", type=str, default=None, help="The path to the video text score metadata (csv/jsonl)." + ) + parser.add_argument("--min_text_score", type=float, default=0.02, help="The text threshold.") + parser.add_argument( + "--motion_score_metadata_path", type=str, default=None, help="The path to the video motion score metadata (csv/jsonl)." + ) + parser.add_argument("--min_motion_score", type=float, default=2, help="The motion threshold.") + parser.add_argument("--max_motion_score", type=float, default=999999, help="The maximum motion threshold.") + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + if args.video_metadata_path.endswith(".csv"): + video_metadata_df = pd.read_csv(args.video_metadata_path) + elif args.video_metadata_path.endswith(".jsonl"): + video_metadata_df = pd.read_json(args.video_metadata_path, lines=True) + else: + raise ValueError("The video_metadata_path must end with .csv or .jsonl.") + video_path_list = video_metadata_df[args.video_path_column].tolist() + + if not (args.saved_path.endswith(".csv") or args.saved_path.endswith(".jsonl")): + raise ValueError("The saved_path must end with .csv or .jsonl.") + + if os.path.exists(args.saved_path): + if args.saved_path.endswith(".csv"): + saved_metadata_df = pd.read_csv(args.saved_path) + elif args.saved_path.endswith(".jsonl"): + saved_metadata_df = pd.read_json(args.saved_path, lines=True) + saved_video_path_list = saved_metadata_df[args.video_path_column].tolist() + video_path_list = list(set(video_path_list).difference(set(saved_video_path_list))) + logger.info(f"Resume from {args.saved_path}: {len(saved_video_path_list)} processed and {len(video_path_list)} to be processed.") + + video_path_list = filter( + video_path_list, + basic_metadata_path=args.basic_metadata_path, + min_resolution=args.min_resolution, + min_duration=args.min_duration, + max_duration=args.max_duration, + aesthetic_score_metadata_path=args.aesthetic_score_metadata_path, + min_aesthetic_score=args.min_aesthetic_score, + aesthetic_score_siglip_metadata_path=args.aesthetic_score_siglip_metadata_path, + min_aesthetic_score_siglip=args.min_aesthetic_score_siglip, + text_score_metadata_path=args.text_score_metadata_path, + min_text_score=args.min_text_score, + motion_score_metadata_path=args.motion_score_metadata_path, + min_motion_score=args.min_motion_score, + max_motion_score=args.max_motion_score, + video_path_column=args.video_path_column + ) + # Sorting to guarantee the same result for each process. + video_path_list = natsorted(video_path_list) + + if not any(name in args.model_path for name in ALL_MODEL_NAME): + raise ValueError(f"The model_path should be among the following list: {ALL_MODEL_NAME}.") + + state = PartialState() + if state.is_main_process: + # Check if the model is downloaded in the main process. + processor, model = init_model(args.model_path, "cpu") + state.wait_for_everyone() + processor, model = init_model(args.model_path, state.device) + + index = len(video_path_list) - len(video_path_list) % state.num_processes + # Avoid the NCCL timeout in the final gather operation. + logger.warning( + f"Drop the last {len(video_path_list) % state.num_processes} videos " + "to ensure each process handles the same number of videos." + ) + video_path_list = video_path_list[:index] + logger.info(f"{len(video_path_list)} videos are to be processed.") + + result_dict = { + args.video_path_column: [], + "similarity_cross_frame": [], + "similarity_mean": [], + "sample_frame_idx": [], + } + with state.split_between_processes(video_path_list) as splitted_video_path_list: + video_dataset = VideoDataset( + dataset_inputs={args.video_path_column: splitted_video_path_list}, + video_folder=args.video_folder, + video_path_column=args.video_path_column, + sample_method=args.frame_sample_method, + num_sampled_frames=args.num_sampled_frames, + sample_stride=args.sample_stride, + ) + video_loader = DataLoader(video_dataset, batch_size=args.batch_size, num_workers=args.num_workers, collate_fn=collate_fn) + + for idx, batch in enumerate(tqdm(video_loader)): + if len(batch) > 0: + batch_video_path = [] + batch_frame = [] + batch_sampled_frame_idx = [] + # At least two frames are required to calculate cross-frame semantic consistency. + for path, frame, frame_idx in zip(batch["path"], batch["sampled_frame"], batch["sampled_frame_idx"]): + if len(frame) > 1: + batch_video_path.append(path) + batch_frame.append(frame) + batch_sampled_frame_idx.append(frame_idx) + else: + logger.warning(f"Skip {path} because it only has {len(frame)} frames.") + + frame_num_list = [len(video_frames) for video_frames in batch_frame] + # [B, T, H, W, C] => [(B * T), H, W, C] + reshaped_batch_frame = [frame for video_frames in batch_frame for frame in video_frames] + with torch.no_grad(): + inputs = processor(images=reshaped_batch_frame, return_tensors="pt").to(state.device) + if "dino" in args.model_path.lower(): + frame_features = model(**inputs).last_hidden_state.mean(dim=1) + else: # CLIP + frame_features = model.get_image_features(**inputs) + + # Each video may have a different number of sampled frames. + # Map the flattened frame features back to their original shape. + batch_frame_features = torch.split(frame_features, frame_num_list) + batch_simi_cross_frame = [compute_adjacent_similarity(frame_features) for frame_features in batch_frame_features] + batch_similarity_mean = [ + sum(simi_cross_frame) / len(simi_cross_frame) for simi_cross_frame in batch_simi_cross_frame + ] + + if args.video_folder == "": + saved_video_path_list = batch_video_path + else: + saved_video_path_list = [os.path.relpath(video_path, args.video_folder) for video_path in batch_video_path] + result_dict[args.video_path_column].extend(saved_video_path_list) + result_dict["similarity_cross_frame"].extend(batch_simi_cross_frame) + result_dict["similarity_mean"].extend(batch_similarity_mean) + result_dict["sample_frame_idx"].extend(batch_sampled_frame_idx) + + # Save the metadata in the main process every saved_freq. + if (idx % args.saved_freq) == 0 or idx == len(video_loader) - 1: + state.wait_for_everyone() + gathered_result_dict = {k: gather_object(v) for k, v in result_dict.items()} + if state.is_main_process and len(gathered_result_dict[args.video_path_column]) != 0: + result_df = pd.DataFrame(gathered_result_dict) + # Append is not supported (oss). + if args.saved_path.endswith(".csv"): + if os.path.exists(args.saved_path): + saved_df = pd.read_csv(args.saved_path) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df.to_csv(args.saved_path, index=False) + elif args.saved_path.endswith(".jsonl"): + if os.path.exists(args.saved_path): + saved_df = pd.read_json(args.saved_path, orient="records", lines=True) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df.to_json(args.saved_path, orient="records", lines=True, force_ascii=False) + logger.info(f"Save result to {args.saved_path}.") + for k in result_dict.keys(): + result_dict[k] = [] + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/videox_fun/video_caption/compute_text_score.py b/VideoX-Fun/videox_fun/video_caption/compute_text_score.py new file mode 100644 index 0000000000000000000000000000000000000000..3acebe122b00ad734ccbc3c253f89245df633373 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/compute_text_score.py @@ -0,0 +1,242 @@ +import argparse +import os + +import easyocr +import numpy as np +import pandas as pd +from accelerate import PartialState +from accelerate.utils import gather_object +from natsort import natsorted +from tqdm import tqdm +from torchvision.datasets.utils import download_url + +from utils.filter import filter +from utils.logger import logger +from utils.video_utils import extract_frames + + +def init_ocr_reader(root: str = "~/.cache/easyocr", device: str = "gpu"): + root = os.path.expanduser(root) + if not os.path.exists(root): + os.makedirs(root) + download_url( + "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/video_caption/easyocr/craft_mlt_25k.pth", + root, + filename="craft_mlt_25k.pth", + md5="2f8227d2def4037cdb3b34389dcf9ec1", + ) + ocr_reader = easyocr.Reader( + lang_list=["en", "ch_sim"], + gpu=device, + recognizer=False, + verbose=False, + model_storage_directory=root, + ) + + return ocr_reader + + +def triangle_area(p1, p2, p3): + """Compute the triangle area according to its coordinates. + """ + x1, y1 = p1 + x2, y2 = p2 + x3, y3 = p3 + tri_area = 0.5 * np.abs(x1 * y2 + x2 * y3 + x3 * y1 - x2 * y1 - x3 * y2 - x1 * y3) + return tri_area + + +def compute_text_score(video_path, ocr_reader, sample_method="mid", num_sampled_frames=1): + _, images = extract_frames(video_path, sample_method=sample_method, num_sampled_frames=num_sampled_frames) + images = [np.array(image) for image in images] + + frame_ocr_area_ratios = [] + for image in images: + # horizontal detected results and free-form detected + horizontal_list, free_list = ocr_reader.detect(np.asarray(image)) + width, height = image.shape[0], image.shape[1] + + total_area = width * height + # rectangles + rect_area = 0 + for xmin, xmax, ymin, ymax in horizontal_list[0]: + if xmax < xmin or ymax < ymin: + continue + rect_area += (xmax - xmin) * (ymax - ymin) + # free-form + quad_area = 0 + try: + for points in free_list[0]: + triangle1 = points[:3] + quad_area += triangle_area(*triangle1) + triangle2 = points[3:] + [points[0]] + quad_area += triangle_area(*triangle2) + except: + quad_area = 0 + text_area = rect_area + quad_area + + frame_ocr_area_ratios.append(text_area / total_area) + + text_score = round(np.mean(frame_ocr_area_ratios), 5) + + return text_score + + +def parse_args(): + parser = argparse.ArgumentParser(description="Compute the text score of the middle frame in the videos.") + parser.add_argument("--video_folder", type=str, default="", help="The video folder.") + parser.add_argument( + "--video_metadata_path", type=str, default=None, help="The path to the video dataset metadata (csv/jsonl)." + ) + parser.add_argument( + "--video_path_column", + type=str, + default="video_path", + help="The column contains the video path (an absolute path or a relative path w.r.t the video_folder).", + ) + parser.add_argument( + "--frame_sample_method", + type=str, + default="mid", + ) + parser.add_argument( + "--num_sampled_frames", + type=int, + default=1, + help="num_sampled_frames", + ) + parser.add_argument("--saved_path", type=str, required=True, help="The save path to the output results (csv/jsonl).") + parser.add_argument("--saved_freq", type=int, default=1, help="The frequency to save the output results.") + + parser.add_argument("--basic_metadata_path", type=str, default=None, help="The path to the basic metadata (csv/jsonl).") + parser.add_argument("--min_resolution", type=float, default=0, help="The resolution threshold.") + parser.add_argument("--min_duration", type=float, default=-1, help="The minimum duration.") + parser.add_argument("--max_duration", type=float, default=-1, help="The maximum duration.") + parser.add_argument( + "--aesthetic_score_metadata_path", type=str, default=None, help="The path to the video quality metadata (csv/jsonl)." + ) + parser.add_argument("--min_aesthetic_score", type=float, default=4.0, help="The aesthetic score threshold.") + parser.add_argument( + "--aesthetic_score_siglip_metadata_path", type=str, default=None, help="The path to the video quality metadata (csv/jsonl)." + ) + parser.add_argument("--min_aesthetic_score_siglip", type=float, default=4.0, help="The aesthetic score (SigLIP) threshold.") + parser.add_argument( + "--motion_score_metadata_path", type=str, default=None, help="The path to the video motion score metadata (csv/jsonl)." + ) + parser.add_argument("--min_motion_score", type=float, default=2, help="The minimum motion threshold.") + parser.add_argument("--max_motion_score", type=float, default=999999, help="The maximum motion threshold.") + parser.add_argument( + "--semantic_consistency_score_metadata_path", + nargs="+", + type=str, + default=None, + help="The path to the semantic consistency metadata (csv/jsonl)." + ) + parser.add_argument( + "--min_semantic_consistency_score", type=float, default=0.80, help="The semantic consistency score threshold." + ) + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + if args.video_metadata_path.endswith(".csv"): + video_metadata_df = pd.read_csv(args.video_metadata_path) + elif args.video_metadata_path.endswith(".jsonl"): + video_metadata_df = pd.read_json(args.video_metadata_path, lines=True) + else: + raise ValueError("The video_metadata_path must end with .csv or .jsonl.") + video_path_list = video_metadata_df[args.video_path_column].tolist() + + if not (args.saved_path.endswith(".csv") or args.saved_path.endswith(".jsonl")): + raise ValueError("The saved_path must end with .csv or .jsonl.") + + if os.path.exists(args.saved_path): + if args.saved_path.endswith(".csv"): + saved_metadata_df = pd.read_csv(args.saved_path) + elif args.saved_path.endswith(".jsonl"): + saved_metadata_df = pd.read_json(args.saved_path, lines=True) + saved_video_path_list = saved_metadata_df[args.video_path_column].tolist() + video_path_list = list(set(video_path_list).difference(set(saved_video_path_list))) + logger.info(f"Resume from {args.saved_path}: {len(saved_video_path_list)} processed and {len(video_path_list)} to be processed.") + + video_path_list = filter( + video_path_list, + basic_metadata_path=args.basic_metadata_path, + min_resolution=args.min_resolution, + min_duration=args.min_duration, + max_duration=args.max_duration, + aesthetic_score_metadata_path=args.aesthetic_score_metadata_path, + min_aesthetic_score=args.min_aesthetic_score, + aesthetic_score_siglip_metadata_path=args.aesthetic_score_siglip_metadata_path, + min_aesthetic_score_siglip=args.min_aesthetic_score_siglip, + motion_score_metadata_path=args.motion_score_metadata_path, + min_motion_score=args.min_motion_score, + max_motion_score=args.max_motion_score, + semantic_consistency_score_metadata_path=args.semantic_consistency_score_metadata_path, + min_semantic_consistency_score=args.min_semantic_consistency_score, + video_path_column=args.video_path_column + ) + video_path_list = [os.path.join(args.video_folder, video_path) for video_path in video_path_list] + # Sorting to guarantee the same result for each process. + video_path_list = natsorted(video_path_list) + + state = PartialState() + if state.is_main_process: + # Check if the model is downloaded in the main process. + ocr_reader = init_ocr_reader(device="cpu") + state.wait_for_everyone() + ocr_reader = init_ocr_reader(device=state.device) + + index = len(video_path_list) - len(video_path_list) % state.num_processes + # Avoid the NCCL timeout in the final gather operation. + logger.info( + f"Drop the last {len(video_path_list) % state.num_processes} videos to " + "ensure each process handles the same number of videos." + ) + video_path_list = video_path_list[:index] + logger.info(f"{len(video_path_list)} videos are to be processed.") + + result_list = [] + with state.split_between_processes(video_path_list) as splitted_video_path_list: + for i, video_path in enumerate(tqdm(splitted_video_path_list)): + try: + text_score = compute_text_score( + video_path, + ocr_reader, + sample_method=args.frame_sample_method, + num_sampled_frames=args.num_sampled_frames, + ) + video_meta_info = {} + if args.video_folder == "": + video_meta_info[args.video_path_column] = video_path + else: + video_meta_info[args.video_path_column] = os.path.relpath(video_path, args.video_folder) + video_meta_info["text_score"] = text_score + result_list.append(video_meta_info) + except Exception as e: + logger.warning(f"Compute text score for video {video_path} with error: {e}.") + if i % args.saved_freq == 0 or i == len(splitted_video_path_list) - 1: + state.wait_for_everyone() + gathered_result_list = gather_object(result_list) + if state.is_main_process and len(gathered_result_list) != 0: + result_df = pd.DataFrame(gathered_result_list) + # Append is not supported (oss). + if args.saved_path.endswith(".csv"): + if os.path.exists(args.saved_path): + saved_df = pd.read_csv(args.saved_path) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df.to_csv(args.saved_path, index=False) + elif args.saved_path.endswith(".jsonl"): + if os.path.exists(args.saved_path): + saved_df = pd.read_json(args.saved_path, orient="records", lines=True) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df.to_json(args.saved_path, orient="records", lines=True, force_ascii=False) + logger.info(f"Save result to {args.saved_path}.") + result_list = [] + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/compute_video_quality.py b/VideoX-Fun/videox_fun/video_caption/compute_video_quality.py new file mode 100644 index 0000000000000000000000000000000000000000..313c84c80c98e968908fb28b67df64993ea9b427 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/compute_video_quality.py @@ -0,0 +1,219 @@ +import argparse +import os + +import pandas as pd +from accelerate import PartialState +from accelerate.utils import gather_object +from natsort import index_natsorted +from tqdm import tqdm +from torch.utils.data import DataLoader + +import utils.image_evaluator as image_evaluator +import utils.video_evaluator as video_evaluator +from utils.filter import filter +from utils.logger import logger +from utils.video_dataset import VideoDataset, collate_fn + + +def parse_args(): + parser = argparse.ArgumentParser(description="Compute scores of uniform sampled frames from videos.") + parser.add_argument( + "--video_metadata_path", type=str, default=None, help="The path to the video dataset metadata (csv/jsonl)." + ) + parser.add_argument( + "--video_path_column", + type=str, + default="video_path", + help="The column contains the video path (an absolute path or a relative path w.r.t the video_folder).", + ) + parser.add_argument("--video_folder", type=str, default="", help="The video folder.") + parser.add_argument("--caption_column", type=str, default=None, help="The column contains the caption.") + parser.add_argument( + "--frame_sample_method", + type=str, + choices=["mid", "uniform", "image"], + default="uniform", + ) + parser.add_argument("--num_sampled_frames", type=int, default=8, help="The number of sampled frames.") + parser.add_argument("--metrics", nargs="+", type=str, required=True, help="The evaluation metric(s) for generated images.") + parser.add_argument("--batch_size", type=int, default=1, help="The batch size for the video dataset.") + parser.add_argument("--num_workers", type=int, default=1, help="The number of workers for the video dataset.") + parser.add_argument("--saved_path", type=str, required=True, help="The save path to the output results (csv/jsonl).") + parser.add_argument("--saved_freq", type=int, default=1, help="The frequency to save the output results.") + + parser.add_argument("--basic_metadata_path", type=str, default=None, help="The path to the basic metadata (csv/jsonl).") + parser.add_argument("--min_resolution", type=float, default=0, help="The resolution threshold.") + parser.add_argument("--min_duration", type=float, default=-1, help="The minimum duration.") + parser.add_argument("--max_duration", type=float, default=-1, help="The maximum duration.") + parser.add_argument( + "--text_score_metadata_path", type=str, default=None, help="The path to the video text score metadata (csv/jsonl)." + ) + parser.add_argument("--min_text_score", type=float, default=0.02, help="The text threshold.") + parser.add_argument( + "--motion_score_metadata_path", type=str, default=None, help="The path to the video motion score metadata (csv/jsonl)." + ) + parser.add_argument("--min_motion_score", type=float, default=2, help="The minimum motion threshold.") + parser.add_argument("--max_motion_score", type=float, default=999999, help="The maximum motion threshold.") + parser.add_argument( + "--semantic_consistency_score_metadata_path", + nargs="+", + type=str, + default=None, + help="The path to the semantic consistency metadata (csv/jsonl)." + ) + parser.add_argument( + "--min_semantic_consistency_score", type=float, default=0.80, help="The semantic consistency score threshold." + ) + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + if args.video_metadata_path.endswith(".csv"): + video_metadata_df = pd.read_csv(args.video_metadata_path) + elif args.video_metadata_path.endswith(".jsonl"): + video_metadata_df = pd.read_json(args.video_metadata_path, lines=True) + else: + raise ValueError("The video_metadata_path must end with .csv or .jsonl.") + + if not (args.saved_path.endswith(".csv") or args.saved_path.endswith(".jsonl")): + raise ValueError("The saved_path must end with .csv or .jsonl.") + + if os.path.exists(args.saved_path): + if args.saved_path.endswith(".csv"): + saved_metadata_df = pd.read_csv(args.saved_path) + elif args.saved_path.endswith(".jsonl"): + saved_metadata_df = pd.read_json(args.saved_path, lines=True) + + # Filter out the unprocessed video-caption pairs by setting the indicator=True. + merged_df = video_metadata_df.merge(saved_metadata_df, on=args.video_path_column, how="outer", indicator=True) + video_metadata_df = merged_df[merged_df["_merge"] == "left_only"] + # Sorting to guarantee the same result for each process. + video_metadata_df = video_metadata_df.iloc[index_natsorted(video_metadata_df[args.video_path_column])].reset_index(drop=True) + if args.caption_column is None: + video_metadata_df = video_metadata_df[[args.video_path_column]] + else: + video_metadata_df = video_metadata_df[[args.video_path_column, args.caption_column + "_x"]] + video_metadata_df.rename(columns={args.caption_column + "_x": args.caption_column}, inplace=True) + logger.info(f"Resume from {args.saved_path}: {len(saved_metadata_df)} processed and {len(video_metadata_df)} to be processed.") + + video_path_list = video_metadata_df[args.video_path_column].tolist() + video_path_list = filter( + video_path_list, + basic_metadata_path=args.basic_metadata_path, + min_resolution=args.min_resolution, + min_duration=args.min_duration, + max_duration=args.max_duration, + text_score_metadata_path=args.text_score_metadata_path, + min_text_score=args.min_text_score, + motion_score_metadata_path=args.motion_score_metadata_path, + min_motion_score=args.min_motion_score, + max_motion_score=args.max_motion_score, + semantic_consistency_score_metadata_path=args.semantic_consistency_score_metadata_path, + min_semantic_consistency_score=args.min_semantic_consistency_score, + video_path_column=args.video_path_column + ) + video_metadata_df = video_metadata_df[video_metadata_df[args.video_path_column].isin(video_path_list)] + + state = PartialState() + metric_fns = [] + for metric in args.metrics: + if hasattr(image_evaluator, metric): # frame-wise + if state.is_main_process: + logger.info("Initializing frame-wise evaluator metrics...") + # Check if the model is downloaded in the main process. + getattr(image_evaluator, metric)(device="cpu") + state.wait_for_everyone() + metric_fns.append(getattr(image_evaluator, metric)(device=state.device)) + else: # video-wise + if state.is_main_process: + logger.info("Initializing video-wise evaluator metrics...") + # Check if the model is downloaded in the main process. + getattr(video_evaluator, metric)(device="cpu") + state.wait_for_everyone() + metric_fns.append(getattr(video_evaluator, metric)(device=state.device)) + + result_dict = {args.video_path_column: [], "sample_frame_idx": []} + for metric in metric_fns: + result_dict[str(metric)] = [] + if args.caption_column is not None: + result_dict[args.caption_column] = [] + + if args.frame_sample_method == "image": + logger.warning("Set args.num_sampled_frames to 1 since args.frame_sample_method is image.") + args.num_sampled_frames = 1 + + index = len(video_metadata_df) - len(video_metadata_df) % state.num_processes + # Avoid the NCCL timeout in the final gather operation. + logger.info( + f"Drop the last {len(video_metadata_df) % state.num_processes} videos " + "to ensure each process handles the same number of videos." + ) + video_metadata_df = video_metadata_df.iloc[:index] + logger.info(f"{len(video_metadata_df)} videos are to be processed.") + + video_metadata_list = video_metadata_df.to_dict(orient='list') + with state.split_between_processes(video_metadata_list) as splitted_video_metadata: + video_dataset = VideoDataset( + dataset_inputs=splitted_video_metadata, + video_folder=args.video_folder, + video_path_column=args.video_path_column, + text_column=args.caption_column, + sample_method=args.frame_sample_method, + num_sampled_frames=args.num_sampled_frames + ) + video_loader = DataLoader(video_dataset, batch_size=args.batch_size, num_workers=args.num_workers, collate_fn=collate_fn) + + for idx, batch in enumerate(tqdm(video_loader)): + if len(batch) > 0: + batch_video_path = batch["path"] + result_dict["sample_frame_idx"].extend(batch["sampled_frame_idx"]) + batch_frame = batch["sampled_frame"] # [batch_size, num_sampled_frames, H, W, C] + batch_caption = None + if args.caption_column is not None: + batch_caption = batch["text"] + result_dict["caption"].extend(batch_caption) + # Compute the quality. + for i, metric in enumerate(args.metrics): + quality_scores = metric_fns[i](batch_frame, batch_caption) + if isinstance(quality_scores[0], list): # frame-wise + quality_scores = [ + [round(score, 5) for score in inner_list] + for inner_list in quality_scores + ] + else: # video-wise + quality_scores = [round(score, 5) for score in quality_scores] + result_dict[str(metric_fns[i])].extend(quality_scores) + + if args.video_folder == "": + saved_video_path_list = batch_video_path + else: + saved_video_path_list = [os.path.relpath(video_path, args.video_folder) for video_path in batch_video_path] + result_dict[args.video_path_column].extend(saved_video_path_list) + + # Save the metadata in the main process every saved_freq. + if (idx % args.saved_freq) == 0 or idx == len(video_loader) - 1: + state.wait_for_everyone() + gathered_result_dict = {k: gather_object(v) for k, v in result_dict.items()} + if state.is_main_process and len(gathered_result_dict[args.video_path_column]) != 0: + result_df = pd.DataFrame(gathered_result_dict) + # Append is not supported (oss). + if args.saved_path.endswith(".csv"): + if os.path.exists(args.saved_path): + saved_df = pd.read_csv(args.saved_path) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df.to_csv(args.saved_path, index=False) + elif args.saved_path.endswith(".jsonl"): + if os.path.exists(args.saved_path): + saved_df = pd.read_json(args.saved_path, orient="records", lines=True) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df.to_json(args.saved_path, orient="records", lines=True, force_ascii=False) + logger.info(f"Save result to {args.saved_path}.") + for k in result_dict.keys(): + result_dict[k] = [] + +if __name__ == "__main__": + main() diff --git a/VideoX-Fun/videox_fun/video_caption/cutscene_detect.py b/VideoX-Fun/videox_fun/video_caption/cutscene_detect.py new file mode 100644 index 0000000000000000000000000000000000000000..4e451101adf8cc2de2b54899e5caffa08301a424 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/cutscene_detect.py @@ -0,0 +1,96 @@ +import argparse +import os +from copy import deepcopy +from multiprocessing import Pool +from pathlib import Path + +import pandas as pd +from scenedetect import SceneManager, open_video +from scenedetect.detectors import ContentDetector +from tqdm import tqdm +from utils.logger import logger + + +def cutscene_detection_star(args): + return cutscene_detection(*args) + + +def cutscene_detection(video_path, video_folder, saved_path, cutscene_threshold=27, min_scene_len=15): + try: + if os.path.exists(saved_path): + logger.info(f"{video_path} has been processed.") + return + # Use PyAV as the backend to avoid (to some exent) containing the last frame of the previous scene. + # https://github.com/Breakthrough/PySceneDetect/issues/279#issuecomment-2152596761. + video = open_video(os.path.join(video_folder, video_path), backend="pyav") + frame_rate, frame_size = video.frame_rate, video.frame_size + duration = deepcopy(video.duration) + + frame_points, frame_timecode = [], {} + scene_manager = SceneManager() + scene_manager.add_detector( + # [ContentDetector, ThresholdDetector, AdaptiveDetector] + ContentDetector(threshold=cutscene_threshold, min_scene_len=min_scene_len) + ) + scene_manager.detect_scenes(video, show_progress=False) + scene_list = scene_manager.get_scene_list() + for scene in scene_list: + for frame_time_code in scene: + frame_index = frame_time_code.get_frames() + if frame_index not in frame_points: + frame_points.append(frame_index) + frame_timecode[frame_index] = frame_time_code + + del video, scene_manager + + frame_points = sorted(frame_points) + output_scene_list = [] + for idx in range(len(frame_points) - 1): + output_scene_list.append((frame_timecode[frame_points[idx]], frame_timecode[frame_points[idx+1]])) + + timecode_list = [(frame_timecode_tuple[0].get_timecode(), frame_timecode_tuple[1].get_timecode()) for frame_timecode_tuple in output_scene_list] + meta_scene = [{ + "video_path": video_path, + "timecode_list": timecode_list, + "fram_rate": frame_rate, + "frame_size": frame_size, + "duration": str(duration) # __repr__ + }] + if not os.path.exists(Path(saved_path).parent): + os.makedirs(Path(saved_path).parent, exist_ok=True) + pd.DataFrame(meta_scene).to_json(saved_path, orient="records", lines=True) + except Exception as e: + logger.warning(f"Cutscene detection with {video_path} failed. Error is: {e}.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Cutscene Detection") + parser.add_argument( + "--video_metadata_path", type=str, required=True, help="The path to the video dataset metadata (csv/jsonl)." + ) + parser.add_argument( + "--video_path_column", + type=str, + default="video_path", + help="The column contains the video path (an absolute path or a relative path w.r.t the video_folder).", + ) + parser.add_argument("--video_folder", type=str, default="", help="The video folder.") + parser.add_argument("--saved_folder", type=str, required=True, help="The save path to the output results (csv/jsonl).") + parser.add_argument("--cutscene_threshold", type=int, default=27, help="The threshold of ContentDetector.") + parser.add_argument("--n_jobs", type=int, default=1, help="The number of processes.") + + args = parser.parse_args() + + metadata_df = pd.read_json(args.video_metadata_path, lines=True) + video_path_list = metadata_df[args.video_path_column].tolist() + + # The glob can be slow when there are many small jsonl files. + saved_path_list = [os.path.join(args.saved_folder, Path(video_path).with_suffix(".jsonl")) for video_path in video_path_list] + args_list = [ + (video_path, args.video_folder, saved_path, args.cutscene_threshold) + for video_path, saved_path in zip(video_path_list, saved_path_list) + ] + # Since the length of the video is not uniform, the gather operation is not performed. + # We need to run easyanimate/video_caption/utils/gather_jsonl.py after the program finised. + with Pool(args.n_jobs) as pool: + results = list(tqdm(pool.imap(cutscene_detection_star, args_list), total=len(video_path_list))) diff --git a/VideoX-Fun/videox_fun/video_caption/filter_meta_train.py b/VideoX-Fun/videox_fun/video_caption/filter_meta_train.py new file mode 100644 index 0000000000000000000000000000000000000000..5d22bf990f08a8f58c95c78ab7a67be6a0c60c1c --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/filter_meta_train.py @@ -0,0 +1,93 @@ +import argparse +import os + +import pandas as pd +from natsort import natsorted +from utils.filter import filter +from utils.logger import logger + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument( + "--caption_metadata_path", type=str, default=None, help="The path to the video quality metadata (csv/jsonl)." + ) + parser.add_argument( + "--video_path_column", + type=str, + default="video_path", + help="The column contains the video path (an absolute path or a relative path w.r.t the video_folder).", + ) + parser.add_argument( + "--caption_column", + type=str, + default="caption", + help="The column contains the caption.", + ) + parser.add_argument("--video_folder", type=str, default="", help="The video folder.") + parser.add_argument( + "--basic_metadata_path", type=str, default=None, help="The path to the basic metadata (csv/jsonl)." + ) + parser.add_argument("--min_resolution", type=float, default=720*1280, help="The resolution threshold.") + parser.add_argument("--min_duration", type=float, default=-1, help="The minimum duration.") + parser.add_argument("--max_duration", type=float, default=-1, help="The maximum duration.") + parser.add_argument( + "--aesthetic_score_metadata_path", type=str, default=None, help="The path to the video quality metadata (csv/jsonl)." + ) + parser.add_argument("--min_aesthetic_score", type=float, default=4.0, help="The aesthetic score threshold.") + parser.add_argument( + "--aesthetic_score_siglip_metadata_path", type=str, default=None, help="The path to the video quality (SigLIP) metadata (csv/jsonl)." + ) + parser.add_argument("--min_aesthetic_score_siglip", type=float, default=4.0, help="The aesthetic score (SigLIP) threshold.") + parser.add_argument( + "--text_score_metadata_path", type=str, default=None, help="The path to the video text score metadata (csv/jsonl)." + ) + parser.add_argument("--min_text_score", type=float, default=0.02, help="The text threshold.") + parser.add_argument( + "--motion_score_metadata_path", type=str, default=None, help="The path to the video motion score metadata (csv/jsonl)." + ) + parser.add_argument("--min_motion_score", type=float, default=2, help="The motion threshold.") + parser.add_argument( + "--videoclipxl_score_metadata_path", type=str, default=None, help="The path to the video-caption VideoCLIPXL score metadata (csv/jsonl)." + ) + parser.add_argument("--min_videoclipxl_score", type=float, default=0.20, help="The VideoCLIPXL score threshold.") + parser.add_argument("--saved_path", type=str, required=True) + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + raw_caption_df = pd.read_json(args.caption_metadata_path, lines=True) + video_path_list = raw_caption_df[args.video_path_column].to_list() + filtered_video_path_list = filter( + video_path_list, + basic_metadata_path=args.basic_metadata_path, + min_resolution=args.min_resolution, + min_duration=args.min_duration, + max_duration=args.max_duration, + aesthetic_score_metadata_path=args.aesthetic_score_metadata_path, + min_aesthetic_score=args.min_aesthetic_score, + aesthetic_score_siglip_metadata_path=args.aesthetic_score_siglip_metadata_path, + min_aesthetic_score_siglip=args.min_aesthetic_score_siglip, + text_score_metadata_path=args.text_score_metadata_path, + min_text_score=args.min_text_score, + motion_score_metadata_path=args.motion_score_metadata_path, + min_motion_score=args.min_motion_score, + videoclipxl_score_metadata_path=args.videoclipxl_score_metadata_path, + min_videoclipxl_score=args.min_videoclipxl_score, + video_path_column=args.video_path_column + ) + filtered_video_path_list = natsorted(filtered_video_path_list) + filtered_caption_df = raw_caption_df[raw_caption_df[args.video_path_column].isin(filtered_video_path_list)] + train_df = filtered_caption_df.rename(columns={args.video_path_column: "file_path", args.caption_column: "text"}) + train_df["file_path"] = train_df["file_path"].map(lambda x: os.path.join(args.video_folder, x)) + train_df["type"] = "video" + train_df.to_json(args.saved_path, orient="records", force_ascii=False, indent=2) + logger.info(f"The final train file with {len(train_df)} videos are saved to {args.saved_path}.") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/internvl2_video_recaptioning.py b/VideoX-Fun/videox_fun/video_caption/internvl2_video_recaptioning.py new file mode 100644 index 0000000000000000000000000000000000000000..9db87617111853e4fb10b4ddaec4a21e0722474e --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/internvl2_video_recaptioning.py @@ -0,0 +1,265 @@ +import argparse +import os + +import pandas as pd +import torch +from natsort import natsorted +from transformers import AutoTokenizer +from tqdm import tqdm +from torch.utils.data import DataLoader +from vllm import LLM, SamplingParams + +from utils.filter import filter +from utils.logger import logger +from utils.video_dataset import VideoDataset, collate_fn + + +def recaption_batch_video(llm, batch_video_frames, prompt, sampling_params): + inputs = [ + { + "prompt": prompt, + "multi_modal_data": { + "image": video_frames + }, + } + for video_frames in batch_video_frames + ] + + outputs = llm.generate(inputs, sampling_params=sampling_params) + + batch_output = [] + for o in outputs: + generated_text = o.outputs[0].text + batch_output.append(generated_text) + + return batch_output + +def parse_args(): + parser = argparse.ArgumentParser(description="Recaption videos with InternVL2.") + parser.add_argument( + "--video_metadata_path", + type=str, + default=None, + help="The path to the video dataset metadata (csv/jsonl).", + ) + parser.add_argument( + "--video_path_column", + type=str, + default="video_path", + help="The column contains the video path (an absolute path or a relative path w.r.t the video_folder).", + ) + parser.add_argument( + "--caption_column", + type=str, + default="caption", + help="The column contains the caption.", + ) + parser.add_argument( + "--video_folder", type=str, default="", help="The video folder." + ) + parser.add_argument( + "--batch_size", + type=int, + default=1, + required=False, + help="The batch size for vllm inference. Adjust according to the number of GPUs to maximize inference throughput.", + ) + parser.add_argument( + "--num_workers", + type=int, + default=4, + required=False, + help="The number of workers for the video dataset.", + ) + parser.add_argument("--input_prompt", type=str, default="Describe this video in detail. Don\'t repeat.") + parser.add_argument( + "--model_path", type=str, default="OpenGVLab/InternVL2-40B-AWQ" + ) + parser.add_argument( + "--frame_sample_method", + type=str, + choices=["mid", "uniform", "image"], + default="uniform", + ) + parser.add_argument("--num_sampled_frames", type=int, default=8) + parser.add_argument( + "--saved_path", + type=str, + required=True, + help="The save path to the output results (csv/jsonl).", + ) + parser.add_argument( + "--saved_freq", + type=int, + default=1, + help="The frequency to save the output results.", + ) + + parser.add_argument( + "--basic_metadata_path", type=str, default=None, help="The path to the basic metadata (csv/jsonl)." + ) + parser.add_argument("--min_resolution", type=float, default=0, help="The resolution threshold.") + parser.add_argument("--min_duration", type=float, default=-1, help="The minimum duration.") + parser.add_argument("--max_duration", type=float, default=-1, help="The maximum duration.") + parser.add_argument( + "--aesthetic_score_metadata_path", type=str, default=None, help="The path to the video quality metadata (csv/jsonl)." + ) + parser.add_argument("--min_aesthetic_score", type=float, default=4.0, help="The aesthetic score threshold.") + parser.add_argument( + "--aesthetic_score_siglip_metadata_path", type=str, default=None, help="The path to the video quality metadata (csv/jsonl)." + ) + parser.add_argument("--min_aesthetic_score_siglip", type=float, default=4.0, help="The aesthetic score (SigLIP) threshold.") + parser.add_argument( + "--text_score_metadata_path", type=str, default=None, help="The path to the video text score metadata (csv/jsonl)." + ) + parser.add_argument("--min_text_score", type=float, default=0.02, help="The text threshold.") + parser.add_argument( + "--motion_score_metadata_path", type=str, default=None, help="The path to the video motion score metadata (csv/jsonl)." + ) + parser.add_argument("--min_motion_score", type=float, default=2, help="The motion threshold.") + parser.add_argument("--max_motion_score", type=float, default=999999, help="The maximum motion threshold.") + parser.add_argument( + "--semantic_consistency_score_metadata_path", + nargs="+", + type=str, + default=None, + help="The path to the semantic consistency metadata (csv/jsonl)." + ) + parser.add_argument( + "--min_semantic_consistency_score", type=float, default=0.80, help="The semantic consistency score threshold." + ) + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + if args.video_metadata_path.endswith(".csv"): + video_metadata_df = pd.read_csv(args.video_metadata_path) + elif args.video_metadata_path.endswith(".jsonl"): + video_metadata_df = pd.read_json(args.video_metadata_path, lines=True) + else: + raise ValueError("The video_metadata_path must end with .csv or .jsonl.") + video_path_list = video_metadata_df[args.video_path_column].tolist() + + if not (args.saved_path.endswith(".csv") or args.saved_path.endswith(".jsonl")): + raise ValueError("The saved_path must end with .csv or .jsonl.") + + if os.path.exists(args.saved_path): + if args.saved_path.endswith(".csv"): + saved_metadata_df = pd.read_csv(args.saved_path) + elif args.saved_path.endswith(".jsonl"): + saved_metadata_df = pd.read_json(args.saved_path, lines=True) + saved_video_path_list = saved_metadata_df[args.video_path_column].tolist() + video_path_list = list(set(video_path_list).difference(set(saved_video_path_list))) + logger.info( + f"Resume from {args.saved_path}: {len(saved_video_path_list)} processed and {len(video_path_list)} to be processed." + ) + + video_path_list = filter( + video_path_list, + basic_metadata_path=args.basic_metadata_path, + min_resolution=args.min_resolution, + min_duration=args.min_duration, + max_duration=args.max_duration, + aesthetic_score_metadata_path=args.aesthetic_score_metadata_path, + min_aesthetic_score=args.min_aesthetic_score, + aesthetic_score_siglip_metadata_path=args.aesthetic_score_siglip_metadata_path, + min_aesthetic_score_siglip=args.min_aesthetic_score_siglip, + text_score_metadata_path=args.text_score_metadata_path, + min_text_score=args.min_text_score, + motion_score_metadata_path=args.motion_score_metadata_path, + min_motion_score=args.min_motion_score, + semantic_consistency_score_metadata_path=args.semantic_consistency_score_metadata_path, + min_semantic_consistency_score=args.min_semantic_consistency_score, + video_path_column=args.video_path_column + ) + # Sorting to guarantee the same result for each process. + video_path_list = natsorted(video_path_list) + + video_dataset = VideoDataset( + dataset_inputs={args.video_path_column: video_path_list}, + video_path_column=args.video_path_column, + video_folder=args.video_folder, + sample_method=args.frame_sample_method, + num_sampled_frames=args.num_sampled_frames + ) + video_loader = DataLoader(video_dataset, batch_size=args.batch_size, num_workers=args.num_workers, collate_fn=collate_fn) + + # Initialize the vllm inference pipeline. + CUDA_VISIBLE_DEVICES = os.getenv("CUDA_VISIBLE_DEVICES", None) + tensor_parallel_size = torch.cuda.device_count() if CUDA_VISIBLE_DEVICES is None else len(CUDA_VISIBLE_DEVICES.split(",")) + logger.info(f"Automatically set tensor_parallel_size={tensor_parallel_size} based on the available devices.") + + max_dynamic_patch = 1 + if args.frame_sample_method == "image": + max_dynamic_patch = 12 + quantization = None + if "awq" in args.model_path.lower(): + quantization="awq" + llm = LLM( + model=args.model_path, + trust_remote_code=True, + max_model_len=8192, + limit_mm_per_prompt={"image": args.num_sampled_frames}, + gpu_memory_utilization=0.9, + tensor_parallel_size=tensor_parallel_size, + quantization=quantization, + dtype="float16", + mm_processor_kwargs={"max_dynamic_patch": max_dynamic_patch} + ) + tokenizer = AutoTokenizer.from_pretrained(args.model_path, trust_remote_code=True) + + if args.frame_sample_method == "image": + placeholders = "\n" + else: + placeholders = "".join(f"Frame{i}: \n" for i in range(1, args.num_sampled_frames + 1)) + messages = [{"role": "user", "content": f"{placeholders}{args.input_prompt}"}] + prompt = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) + + # Stop tokens for InternVL + # models variants may have different stop tokens + # please refer to the model card for the correct "stop words": + # https://huggingface.co/OpenGVLab/InternVL2-2B#service + stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"] + stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens] + sampling_params = SamplingParams(temperature=0, max_tokens=512, stop_token_ids=stop_token_ids) + + result_dict = {args.video_path_column: [], args.caption_column: []} + for idx, batch in enumerate(tqdm(video_loader)): + if len(batch) > 0: + batch_video_path = batch["path"] + batch_frame = batch["sampled_frame"] # [batch_size, num_sampled_frames, H, W, C] + batch_caption = recaption_batch_video(llm, batch_frame, prompt, sampling_params) + + if args.video_folder == "": + saved_video_path_list = batch_video_path + else: + saved_video_path_list = [os.path.relpath(video_path, args.video_folder) for video_path in batch_video_path] + result_dict[args.video_path_column].extend(saved_video_path_list) + result_dict["caption"].extend(batch_caption) + + if idx % args.saved_freq == 0 or idx == len(video_loader) - 1: + result_df = pd.DataFrame(result_dict) + + # Append is not supported (oss). + if args.saved_path.endswith(".csv"): + if os.path.exists(args.saved_path): + saved_df = pd.read_csv(args.saved_path) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df = result_df.iloc[natsorted(result_df.index, key=lambda x: result_df.loc[x, args.video_path_column])] + result_df.to_csv(args.saved_path, index=False) + elif args.saved_path.endswith(".jsonl"): + if os.path.exists(args.saved_path): + saved_df = pd.read_json(args.saved_path, orient="records", lines=True) + result_df = pd.concat([saved_df, result_df], ignore_index=True) + result_df = result_df.iloc[natsorted(result_df.index, key=lambda x: result_df.loc[x, args.video_path_column])] + result_df.to_json(args.saved_path, orient="records", lines=True, force_ascii=False) + logger.info(f"Save result to {args.saved_path}.") + result_dict = {args.video_path_column: [], args.caption_column: []} + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/package_patches/easyocr_detection_patched.py b/VideoX-Fun/videox_fun/video_caption/package_patches/easyocr_detection_patched.py new file mode 100644 index 0000000000000000000000000000000000000000..bbb49d660aa264c205610cc72b9d1f04ce66b4e6 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/package_patches/easyocr_detection_patched.py @@ -0,0 +1,116 @@ +"""Modified from https://github.com/JaidedAI/EasyOCR/blob/803b907/easyocr/detection.py. +1. Disable DataParallel. +""" +from collections import OrderedDict + +import cv2 +import numpy as np +import torch +import torch.backends.cudnn as cudnn +from PIL import Image +from torch.autograd import Variable + +from .craft import CRAFT +from .craft_utils import adjustResultCoordinates, getDetBoxes +from .imgproc import normalizeMeanVariance, resize_aspect_ratio + + +def copyStateDict(state_dict): + if list(state_dict.keys())[0].startswith("module"): + start_idx = 1 + else: + start_idx = 0 + new_state_dict = OrderedDict() + for k, v in state_dict.items(): + name = ".".join(k.split(".")[start_idx:]) + new_state_dict[name] = v + return new_state_dict + +def test_net(canvas_size, mag_ratio, net, image, text_threshold, link_threshold, low_text, poly, device, estimate_num_chars=False): + if isinstance(image, np.ndarray) and len(image.shape) == 4: # image is batch of np arrays + image_arrs = image + else: # image is single numpy array + image_arrs = [image] + + img_resized_list = [] + # resize + for img in image_arrs: + img_resized, target_ratio, size_heatmap = resize_aspect_ratio(img, canvas_size, + interpolation=cv2.INTER_LINEAR, + mag_ratio=mag_ratio) + img_resized_list.append(img_resized) + ratio_h = ratio_w = 1 / target_ratio + # preprocessing + x = [np.transpose(normalizeMeanVariance(n_img), (2, 0, 1)) + for n_img in img_resized_list] + x = torch.from_numpy(np.array(x)) + x = x.to(device) + + # forward pass + with torch.no_grad(): + y, feature = net(x) + + boxes_list, polys_list = [], [] + for out in y: + # make score and link map + score_text = out[:, :, 0].cpu().data.numpy() + score_link = out[:, :, 1].cpu().data.numpy() + + # Post-processing + boxes, polys, mapper = getDetBoxes( + score_text, score_link, text_threshold, link_threshold, low_text, poly, estimate_num_chars) + + # coordinate adjustment + boxes = adjustResultCoordinates(boxes, ratio_w, ratio_h) + polys = adjustResultCoordinates(polys, ratio_w, ratio_h) + if estimate_num_chars: + boxes = list(boxes) + polys = list(polys) + for k in range(len(polys)): + if estimate_num_chars: + boxes[k] = (boxes[k], mapper[k]) + if polys[k] is None: + polys[k] = boxes[k] + boxes_list.append(boxes) + polys_list.append(polys) + + return boxes_list, polys_list + +def get_detector(trained_model, device='cpu', quantize=True, cudnn_benchmark=False): + net = CRAFT() + + if device == 'cpu': + net.load_state_dict(copyStateDict(torch.load(trained_model, map_location=device))) + if quantize: + try: + torch.quantization.quantize_dynamic(net, dtype=torch.qint8, inplace=True) + except: + pass + else: + net.load_state_dict(copyStateDict(torch.load(trained_model, map_location=device))) + # net = torch.nn.DataParallel(net).to(device) + net = net.to(device) + cudnn.benchmark = cudnn_benchmark + + net.eval() + return net + +def get_textbox(detector, image, canvas_size, mag_ratio, text_threshold, link_threshold, low_text, poly, device, optimal_num_chars=None, **kwargs): + result = [] + estimate_num_chars = optimal_num_chars is not None + bboxes_list, polys_list = test_net(canvas_size, mag_ratio, detector, + image, text_threshold, + link_threshold, low_text, poly, + device, estimate_num_chars) + if estimate_num_chars: + polys_list = [[p for p, _ in sorted(polys, key=lambda x: abs(optimal_num_chars - x[1]))] + for polys in polys_list] + + for polys in polys_list: + single_img_result = [] + for i, box in enumerate(polys): + poly = np.array(box).astype(np.int32).reshape((-1)) + single_img_result.append(poly) + result.append(single_img_result) + + return result diff --git a/VideoX-Fun/videox_fun/video_caption/prompt/beautiful_prompt.txt b/VideoX-Fun/videox_fun/video_caption/prompt/beautiful_prompt.txt new file mode 100644 index 0000000000000000000000000000000000000000..88bd6e3ff7bc440fc2e882652b6fae3379f9a00e --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/prompt/beautiful_prompt.txt @@ -0,0 +1,9 @@ +I will upload some brief prompt words to be used for AI-generated videos. Please expand these brief prompt words into a more detailed description to enhance the quality of the generated videos. The detailed description should include the main subject (person, object, animal, or none) actions and their attributes or status sequence, the background (the objects, location, weather, and time), the view shot and camera movement. +The final detailed description must not exceed 200 words. Output with the following json format: +{"detailed description": "your detailed description here"} + +Here is an example: +brief prompt words: "A stylish woman in a black leather jacket, red dress, and boots walks confidently down a damp Tokyo street." +{"detailed description": "A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage. She wears a black leather jacket, a long red dress, and black boots, and carries a black purse. She wears sunglasses and red lipstick. She walks confidently and casually. The street is damp and reflective, creating a mirror effect of the colorful lights. Many pedestrians walk about."} + +Here are the brief prompt words: \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/prompt/rewrite.txt b/VideoX-Fun/videox_fun/video_caption/prompt/rewrite.txt new file mode 100644 index 0000000000000000000000000000000000000000..5d42e1023a295aec414a9acdb38edde76f1a1420 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/prompt/rewrite.txt @@ -0,0 +1,11 @@ +Please rewrite the video description to be useful for AI to re-generate the video, according to the following requirements +1. Do not start with something similar to 'The video/scene/frame shows' or "In this video/scene/frame". +2. Remove the subjective content deviates from describing the visual content of the video. For instance, a sentence like "It gives a feeling of ease and tranquility and makes people feel comfortable" is considered subjective. +3. Remove the non-existent description that does not in the visual content of the video, For instance, a sentence like "There is no visible detail that could be used to identify the individual beyond what is shown." is considered as the non-existent description. +4. The rewritten description should include the main subject (person, object, animal, or none) actions and their attributes or status sequence, the background (the objects, location, weather, and time). +5. If the original description includes the view shot, camera movement and the video style, the rewritten description should also include them. If not, there is no need to invent them on your own. +6. Here are some examples of good descriptions: 1) A stylish woman walks down a Tokyo street filled with warm glowing neon and animated city signage. She wears a black leather jacket, a long red dress, and black boots, and carries a black purse. She wears sunglasses and red lipstick. She walks confidently and casually. The street is damp and reflective, creating a mirror effect of the colorful lights. Many pedestrians walk about. 2) A large orange octopus is seen resting on the bottom of the ocean floor, blending in with the sandy and rocky terrain. Its tentacles are spread out around its body, and its eyes are closed. The octopus is unaware of a king crab that is crawling towards it from behind a rock, its claws raised and ready to attack. The crab is brown and spiny, with long legs and antennae. The scene is captured from a wide angle, showing the vastness and depth of the ocean. The water is clear and blue, with rays of sunlight filtering through. The shot is sharp and crisp, with a high dynamic range. The octopus and the crab are in focus, while the background is slightly blurred, creating a depth of field effect. +7. Output with the following json format: +{"rewritten description": "your rewritten description here"} + +Here is the video description: \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/requirements.txt b/VideoX-Fun/videox_fun/video_caption/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..36ae8e56b4adefc1075081a7444bd3f21367ec95 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/requirements.txt @@ -0,0 +1,7 @@ +pandas>=2.0.0 +easyocr==1.7.1 +git+https://github.com/openai/CLIP.git +natsort +joblib +scenedetect +av \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/scripts/stage_1_video_splitting.sh b/VideoX-Fun/videox_fun/video_caption/scripts/stage_1_video_splitting.sh new file mode 100644 index 0000000000000000000000000000000000000000..bf57dfbeaae8c35947fdfecb429aab586c6016cd --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/scripts/stage_1_video_splitting.sh @@ -0,0 +1,39 @@ +VIDEO_FOLDER="datasets/panda_70m/videos/data/" +META_FILE_PATH="datasets/panda_70m/videos/meta_file_info.jsonl" +SCENE_FOLDER="datasets/panda_70m/videos/meta_scene_info/" +SCENE_SAVED_PATH="datasets/panda_70m/videos/meta_scene_info.jsonl" +OUTPUT_FOLDER="datasets/panda_70m/videos_clips/data/" +RESOLUTION_THRESHOLD=$((512*512)) + +# Set the duration range of video clips. +export MIN_SECONDS=3 +export MAX_SECONDS=10 + +# Save all video names in a video folder as a meta file. +python -m utils.get_meta_file \ + --video_folder $VIDEO_FOLDER \ + --saved_path $META_FILE_PATH + +# Perform scene detection on the video dataset. +# Adjust the n_jobs parameter based on the actual number of CPU cores in the machine. +python cutscene_detect.py \ + --video_metadata_path $META_FILE_PATH \ + --video_folder $VIDEO_FOLDER \ + --saved_folder $SCENE_FOLDER \ + --n_jobs 32 + +# Gather all scene jsonl files to a single scene jsonl file. +# Adjust the n_jobs parameter based on the actual I/O speed in the machine. +python -m utils.gather_jsonl \ + --meta_folder $SCENE_FOLDER \ + --meta_file_path $SCENE_SAVED_PATH \ + --n_jobs 64 + +# Perform video splitting filtered by the RESOLUTION_THRESHOLD. +# It consumes more CPU computing resources compared to the above operations. +python video_splitting.py \ + --video_metadata_path $SCENE_SAVED_PATH \ + --video_folder $VIDEO_FOLDER \ + --output_folder $OUTPUT_FOLDER \ + --n_jobs 16 \ + --resolution_threshold $RESOLUTION_THRESHOLD \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/scripts/stage_2_video_filtering.sh b/VideoX-Fun/videox_fun/video_caption/scripts/stage_2_video_filtering.sh new file mode 100644 index 0000000000000000000000000000000000000000..d0115c62dfb2ed59edb8e3ad447966b7d1ba7c40 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/scripts/stage_2_video_filtering.sh @@ -0,0 +1,67 @@ +META_FILE_PATH="datasets/panda_70m/videos_clips/data/meta_file_info.jsonl" +VIDEO_FOLDER="datasets/panda_70m/videos_clips/data/" +CLIP_OR_DINO_MODEL_PATH="openai/clip-vit-large-patch14-336" +SEMANTIC_CONSISTENCY_SAVED_PATH="datasets/panda_70m/videos_clips/meta_consistency_info.jsonl" +MIN_SEMANTIC_CONSISTENCY_SCORE=0.8 +QUALITY_SAVED_PATH="datasets/panda_70m/videos_clips/meta_quality_info_siglip.jsonl" +MIN_AESTHETIC_SCORE_SIGLIP=4.0 +TEXT_SAVED_PATH="datasets/panda_70m/videos_clips/meta_text_info.jsonl" +MIN_TEXT_SCORE=0.02 +MOTION_SAVED_PATH="datasets/panda_70m/videos_clips/meta_motion_info.jsonl" + +python -m utils.get_meta_file \ + --video_folder $VIDEO_FOLDER \ + --saved_path $META_FILE_PATH + +# Get the semantic consistency score of all video clips. +# Adjust the num_workers and batch size parameter based on the machine's computing resources to achieve maximum GPU utilization. +accelerate launch compute_semantic_consistency.py \ + --video_metadata_path $META_FILE_PATH \ + --video_folder $VIDEO_FOLDER \ + --model_path $CLIP_OR_DINO_MODEL_PATH \ + --frame_sample_method keyframe+first \ + --batch_size 16 \ + --num_workers 4 \ + --saved_freq 10 \ + --saved_path $SEMANTIC_CONSISTENCY_SAVED_PATH + +# Get the aesthetic score (SigLIP) of all videos filtered by the semantic consistency score. +# Adjust the num_workers and batch size parameter based on the machine's computing resources to achieve maximum GPU utilization. +accelerate launch compute_video_quality.py \ + --video_metadata_path $META_FILE_PATH \ + --video_folder $VIDEO_FOLDER \ + --metrics AestheticScoreSigLIP \ + --frame_sample_method uniform \ + --num_sampled_frames 4 \ + --batch_size 16 \ + --num_workers 4 \ + --saved_freq 10 \ + --saved_path $QUALITY_SAVED_PATH \ + --semantic_consistency_score_metadata_path $SEMANTIC_CONSISTENCY_SAVED_PATH \ + --min_semantic_consistency_score $MIN_SEMANTIC_CONSISTENCY_SCORE + +# Get the text score of all videos filtered by the semantic consistency score and video quality score. +accelerate launch compute_text_score.py \ + --video_metadata_path $META_FILE_PATH \ + --video_folder $VIDEO_FOLDER \ + --saved_freq 10 \ + --saved_path $TEXT_SAVED_PATH \ + --semantic_consistency_score_metadata_path $SEMANTIC_CONSISTENCY_SAVED_PATH \ + --min_semantic_consistency_score $MIN_SEMANTIC_CONSISTENCY_SCORE \ + --aesthetic_score_siglip_metadata_path $QUALITY_SAVED_PATH \ + --min_aesthetic_score_siglip $MIN_AESTHETIC_SCORE_SIGLIP + +# Get the motion score of all videos filtered by the semantic consistency score, video quality score and text score. +# Adjust the n_jobs parameter based on the actual number of CPU cores in the machine. +python compute_motion_score.py \ + --video_metadata_path $META_FILE_PATH \ + --video_folder $VIDEO_FOLDER \ + --saved_freq 10 \ + --saved_path $MOTION_SAVED_PATH \ + --n_jobs 8 \ + --semantic_consistency_score_metadata_path $SEMANTIC_CONSISTENCY_SAVED_PATH \ + --min_semantic_consistency_score $MIN_SEMANTIC_CONSISTENCY_SCORE \ + --aesthetic_score_siglip_metadata_path $QUALITY_SAVED_PATH \ + --min_aesthetic_score_siglip $MIN_AESTHETIC_SCORE_SIGLIP \ + --text_score_metadata_path $TEXT_SAVED_PATH \ + --min_text_score $MIN_TEXT_SCORE \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/scripts/stage_3_video_recaptioning.sh b/VideoX-Fun/videox_fun/video_caption/scripts/stage_3_video_recaptioning.sh new file mode 100644 index 0000000000000000000000000000000000000000..d0b3d0325b9c373c9fae1cca5b1840d4245cc99f --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/scripts/stage_3_video_recaptioning.sh @@ -0,0 +1,78 @@ +META_FILE_PATH="datasets/panda_70m/videos_clips/data/meta_file_info.jsonl" +VIDEO_FOLDER="datasets/panda_70m/videos_clips/data/" +SEMANTIC_CONSISTENCY_SAVED_PATH="datasets/panda_70m/videos_clips/meta_consistency_info.jsonl" +MIN_SEMANTIC_CONSISTENCY_SCORE=0.8 +QUALITY_SAVED_PATH="datasets/panda_70m/videos_clips/meta_quality_info_siglip.jsonl" +MIN_AESTHETIC_SCORE_SIGLIP=4.0 +TEXT_SAVED_PATH="datasets/panda_70m/videos_clips/meta_text_info.jsonl" +MIN_TEXT_SCORE=0.02 +MOTION_SAVED_PATH="datasets/panda_70m/videos_clips/meta_motion_info.jsonl" +MIN_MOTION_SCORE=2 +MAX_MOTION_SCORE=20 +VIDEO_CAPTION_SAVED_PATH="datasets/panda_70m/meta_caption_info.jsonl" +REWRITTEN_VIDEO_CAPTION_SAVED_PATH="datasets/panda_70m/meta_caption_info_rewritten.jsonl" +VIDEOCLIPXL_SCORE_SAVED_PATH="datasets/panda_70m/meta_caption_info_rewritten_videoclipxl.jsonl" +MIN_VIDEOCLIPXL_SCORE=0.20 +TRAIN_SAVED_PATH="datasets/panda_70m/train_panda_70m.json" +# Manually download OpenGVLab/InternVL2-40B-AWQ to CAPTION_MODEL_PATH. +# You can also download OpenGVLab/InternVL2-2B-AWQ InternVL2-8B-AWQ InternVL2-26B-AWQ or InternVL2-Llama3-76B-AWQ +# This a trade-off between recaption quality and speed. +CAPTION_MODEL_PATH="OpenGVLab/InternVL2-40B-AWQ" +# Manually download meta-llama/Meta-Llama-3.1-70B-Instruct to REWRITE_MODEL_PATH. +# You can also download meta-llama/Meta-Llama-3.1-8B-Instruct Meta-Llama-3-8B-Instruct Meta-Llama-3-70B-Instruct. +REWRITE_MODEL_PATH="meta-llama/Meta-Llama-3.1-70B-Instruct" + +# Use InternVL2-AWQ to perform recaptioning. +# Adjust the num_workers and batch size parameter based on the machine's computing resources to achieve maximum GPU utilization. +python3 internvl2_video_recaptioning.py \ + --video_metadata_path ${META_FILE_PATH} \ + --video_folder ${VIDEO_FOLDER} \ + --model_path ${CAPTION_MODEL_PATH} \ + --saved_path $VIDEO_CAPTION_SAVED_PATH \ + --saved_freq 1 \ + --num_workers 4 \ + --batch_size 128 \ + --semantic_consistency_score_metadata_path $SEMANTIC_CONSISTENCY_SAVED_PATH \ + --min_semantic_consistency_score $MIN_SEMANTIC_CONSISTENCY_SCORE \ + --aesthetic_score_siglip_metadata_path $QUALITY_SAVED_PATH \ + --min_aesthetic_score_siglip $MIN_AESTHETIC_SCORE_SIGLIP \ + --text_score_metadata_path $TEXT_SAVED_PATH \ + --min_text_score $MIN_TEXT_SCORE \ + --motion_score_metadata_path $MOTION_SAVED_PATH \ + --min_motion_score $MIN_MOTION_SCORE \ + --max_motion_score $MAX_MOTION_SCORE + +# Rewrite video captions (optional). +python caption_rewrite.py \ + --video_metadata_path $VIDEO_CAPTION_SAVED_PATH \ + --video_path_column "video_path" \ + --batch_size 4096 \ + --model_name $REWRITE_MODEL_PATH \ + --prompt prompt/rewrite.txt \ + --prefix '"rewritten description": ' \ + --answer_template 'your rewritten description here' \ + --max_retry_count 10 \ + --saved_path $REWRITTEN_VIDEO_CAPTION_SAVED_PATH \ + --saved_freq 1 + +# Compute caption-video alignment (optional). +# Adjust the num_workers and batch size parameter based on the machine's computing resources to achieve maximum GPU utilization. +accelerate launch compute_video_quality.py \ + --video_metadata_path $REWRITTEN_VIDEO_CAPTION_SAVED_PATH \ + --caption_column caption \ + --video_folder $VIDEO_FOLDER \ + --frame_sample_method uniform \ + --num_sampled_frames 8 \ + --metrics VideoCLIPXLScore \ + --num_workers 4 \ + --batch_size 16 \ + --saved_path $VIDEOCLIPXL_SCORE_SAVED_PATH \ + --saved_freq 10 + +# Get the final train file. +python filter_meta_train.py \ + --caption_metadata_path $REWRITTEN_VIDEO_CAPTION_SAVED_PATH \ + --video_folder $VIDEO_FOLDER \ + --videoclipxl_score_metadata_path $VIDEOCLIPXL_SCORE_SAVED_PATH \ + --min_videoclipxl_score $MIN_VIDEOCLIPXL_SCORE \ + --saved_path $TRAIN_SAVED_PATH \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/filter.py b/VideoX-Fun/videox_fun/video_caption/utils/filter.py new file mode 100644 index 0000000000000000000000000000000000000000..f7a0c5997f34b6b5fc4502702237df1712a05885 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/filter.py @@ -0,0 +1,182 @@ +import ast +from typing import Optional + +import pandas as pd + +from .logger import logger + + +# Ensure each item in the video_path_list matches the paths in the video_path column of the metadata. +def filter( + video_path_list: list[str], + basic_metadata_path: Optional[str] = None, + min_resolution: float = 720*1280, + min_duration: float = -1, + max_duration: float = -1, + aesthetic_score_metadata_path: Optional[str] = None, + min_aesthetic_score: float = 4, + aesthetic_score_siglip_metadata_path: Optional[str] = None, + min_aesthetic_score_siglip: float = 4, + text_score_metadata_path: Optional[str] = None, + min_text_score: float = 0.02, + motion_score_metadata_path: Optional[str] = None, + min_motion_score: float = 2, + max_motion_score: float = 999999, + videoclipxl_score_metadata_path: Optional[str] = None, + min_videoclipxl_score: float = 0.20, + semantic_consistency_score_metadata_path: Optional[list[str]] = None, + min_semantic_consistency_score: float = 0.80, + video_path_column: str = "video_path" +): + if basic_metadata_path is not None: + if basic_metadata_path.endswith(".csv"): + basic_df = pd.read_csv(basic_metadata_path) + elif basic_metadata_path.endswith(".jsonl"): + basic_df = pd.read_json(basic_metadata_path, lines=True) + + basic_df["resolution"] = basic_df["frame_size"].apply(lambda x: x[0] * x[1]) + filtered_basic_df = basic_df[basic_df["resolution"] < min_resolution] + filtered_video_path_list = filtered_basic_df[video_path_column].tolist() + + video_path_list = list(set(video_path_list).difference(set(filtered_video_path_list))) + logger.info( + f"Load {basic_metadata_path} ({len(basic_df)}) and filter {len(filtered_video_path_list)} videos " + f"with resolution less than {min_resolution}." + ) + + if min_duration != -1: + filtered_basic_df = basic_df[basic_df["duration"] < min_duration] + filtered_video_path_list = filtered_basic_df[video_path_column].tolist() + + video_path_list = list(set(video_path_list).difference(set(filtered_video_path_list))) + logger.info( + f"Load {basic_metadata_path} and filter {len(filtered_video_path_list)} videos " + f"with duration less than {min_duration}." + ) + + if max_duration != -1: + filtered_basic_df = basic_df[basic_df["duration"] > max_duration] + filtered_video_path_list = filtered_basic_df[video_path_column].tolist() + + video_path_list = list(set(video_path_list).difference(set(filtered_video_path_list))) + logger.info( + f"Load {basic_metadata_path} and filter {len(filtered_video_path_list)} videos " + f"with duration greater than {max_duration}." + ) + + if aesthetic_score_metadata_path is not None: + if aesthetic_score_metadata_path.endswith(".csv"): + aesthetic_score_df = pd.read_csv(aesthetic_score_metadata_path) + elif aesthetic_score_metadata_path.endswith(".jsonl"): + aesthetic_score_df = pd.read_json(aesthetic_score_metadata_path, lines=True) + + # In pandas, csv will save lists as strings, whereas jsonl will not. + aesthetic_score_df["aesthetic_score"] = aesthetic_score_df["aesthetic_score"].apply( + lambda x: ast.literal_eval(x) if isinstance(x, str) else x + ) + aesthetic_score_df["aesthetic_score_mean"] = aesthetic_score_df["aesthetic_score"].apply(lambda x: sum(x) / len(x)) + filtered_aesthetic_score_df = aesthetic_score_df[aesthetic_score_df["aesthetic_score_mean"] < min_aesthetic_score] + filtered_video_path_list = filtered_aesthetic_score_df[video_path_column].tolist() + + video_path_list = list(set(video_path_list).difference(set(filtered_video_path_list))) + logger.info( + f"Load {aesthetic_score_metadata_path} ({len(aesthetic_score_df)}) and filter {len(filtered_video_path_list)} videos " + f"with aesthetic score less than {min_aesthetic_score}." + ) + + if aesthetic_score_siglip_metadata_path is not None: + if aesthetic_score_siglip_metadata_path.endswith(".csv"): + aesthetic_score_siglip_df = pd.read_csv(aesthetic_score_siglip_metadata_path) + elif aesthetic_score_siglip_metadata_path.endswith(".jsonl"): + aesthetic_score_siglip_df = pd.read_json(aesthetic_score_siglip_metadata_path, lines=True) + + # In pandas, csv will save lists as strings, whereas jsonl will not. + aesthetic_score_siglip_df["aesthetic_score_siglip"] = aesthetic_score_siglip_df["aesthetic_score_siglip"].apply( + lambda x: ast.literal_eval(x) if isinstance(x, str) else x + ) + aesthetic_score_siglip_df["aesthetic_score_siglip_mean"] = aesthetic_score_siglip_df["aesthetic_score_siglip"].apply( + lambda x: sum(x) / len(x) + ) + filtered_aesthetic_score_siglip_df = aesthetic_score_siglip_df[ + aesthetic_score_siglip_df["aesthetic_score_siglip_mean"] < min_aesthetic_score_siglip + ] + filtered_video_path_list = filtered_aesthetic_score_siglip_df[video_path_column].tolist() + + video_path_list = list(set(video_path_list).difference(set(filtered_video_path_list))) + logger.info( + f"Load {aesthetic_score_siglip_metadata_path} ({len(aesthetic_score_siglip_df)}) and filter {len(filtered_video_path_list)} videos " + f"with aesthetic score (SigLIP) less than {min_aesthetic_score_siglip}." + ) + + if text_score_metadata_path is not None: + if text_score_metadata_path.endswith(".csv"): + text_score_df = pd.read_csv(text_score_metadata_path) + elif text_score_metadata_path.endswith(".jsonl"): + text_score_df = pd.read_json(text_score_metadata_path, lines=True) + + filtered_text_score_df = text_score_df[text_score_df["text_score"] > min_text_score] + filtered_video_path_list = filtered_text_score_df[video_path_column].tolist() + + video_path_list = list(set(video_path_list).difference(set(filtered_video_path_list))) + logger.info( + f"Load {text_score_metadata_path} ({len(text_score_df)}) and filter {len(filtered_video_path_list)} videos " + f"with text score greater than {min_text_score}." + ) + + if motion_score_metadata_path is not None: + if motion_score_metadata_path.endswith(".csv"): + motion_score_df = pd.read_csv(motion_score_metadata_path) + elif motion_score_metadata_path.endswith(".jsonl"): + motion_score_df = pd.read_json(motion_score_metadata_path, lines=True) + + filtered_motion_score_df = motion_score_df[motion_score_df["motion_score"] < min_motion_score] + filtered_video_path_list = filtered_motion_score_df[video_path_column].tolist() + + video_path_list = list(set(video_path_list).difference(set(filtered_video_path_list))) + logger.info( + f"Load {motion_score_metadata_path} ({len(motion_score_df)}) and filter {len(filtered_video_path_list)} videos " + f"with motion score smaller than {min_motion_score}." + ) + + filtered_motion_score_df = motion_score_df[motion_score_df["motion_score"] > max_motion_score] + filtered_video_path_list = filtered_motion_score_df[video_path_column].tolist() + + video_path_list = list(set(video_path_list).difference(set(filtered_video_path_list))) + logger.info( + f"Load {motion_score_metadata_path} ({len(motion_score_df)}) and filter {len(filtered_video_path_list)} videos " + f"with motion score greater than {min_motion_score}." + ) + + if videoclipxl_score_metadata_path is not None: + if videoclipxl_score_metadata_path.endswith(".csv"): + videoclipxl_score_df = pd.read_csv(videoclipxl_score_metadata_path) + elif videoclipxl_score_metadata_path.endswith(".jsonl"): + videoclipxl_score_df = pd.read_json(videoclipxl_score_metadata_path, lines=True) + + filtered_videoclipxl_score_df = videoclipxl_score_df[videoclipxl_score_df["videoclipxl_score"] < min_videoclipxl_score] + filtered_video_path_list = filtered_videoclipxl_score_df[video_path_column].tolist() + + video_path_list = list(set(video_path_list).difference(set(filtered_video_path_list))) + logger.info( + f"Load {videoclipxl_score_metadata_path} ({len(videoclipxl_score_df)}) and " + f"filter {len(filtered_video_path_list)} videos with mixclip score smaller than {min_videoclipxl_score}." + ) + + if semantic_consistency_score_metadata_path is not None: + for f in semantic_consistency_score_metadata_path: + if f.endswith(".csv"): + semantic_consistency_score_df = pd.read_csv(f) + elif f.endswith(".jsonl"): + semantic_consistency_score_df = pd.read_json(f, lines=True) + filtered_semantic_consistency_score_df = semantic_consistency_score_df[ + semantic_consistency_score_df["similarity_cross_frame"].apply(lambda x: min(x) < min_semantic_consistency_score) + ] + filtered_video_path_list = filtered_semantic_consistency_score_df[video_path_column].tolist() + + video_path_list = list(set(video_path_list).difference(set(filtered_video_path_list))) + logger.info( + f"Load {f} ({len(semantic_consistency_score_df)}) and filter {len(filtered_video_path_list)} videos " + f"with the minimum semantic consistency score smaller than {min_semantic_consistency_score}." + ) + + return video_path_list diff --git a/VideoX-Fun/videox_fun/video_caption/utils/gather_jsonl.py b/VideoX-Fun/videox_fun/video_caption/utils/gather_jsonl.py new file mode 100644 index 0000000000000000000000000000000000000000..e7530537767dedcf576f8f82c75a3718c84fb0d3 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/gather_jsonl.py @@ -0,0 +1,63 @@ +import argparse +import json +import os +from multiprocessing import Manager, Pool +from pathlib import Path + +import pandas as pd +from natsort import index_natsorted + +from .get_meta_file import parallel_rglob +from .logger import logger + + +def process_file(file_path, shared_list): + with open(file_path, "r") as f: + for line in f: + data = json.loads(line) + shared_list.append(data) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Gather all jsonl files in a folder (meta_folder) to a single jsonl file (meta_file_path).") + parser.add_argument("--meta_folder", type=str, required=True) + parser.add_argument("--video_path_column", type=str, default="video_path") + parser.add_argument("--meta_file_path", type=str, required=True) + parser.add_argument("--n_jobs", type=int, default=1) + parser.add_argument("--recursive", action="store_true", help="Whether to search sub-folders recursively.") + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + if not os.path.exists(args.meta_folder): + raise ValueError(f"The meta_folder {args.meta_folder} does not exist.") + meta_folder = Path(args.meta_folder) + if args.recursive: + jsonl_files = [str(file) for file in parallel_rglob(meta_folder, f"*.jsonl", max_workers=args.n_jobs)] + else: + jsonl_files = [str(file) for file in meta_folder.glob(f"*.jsonl")] + + with Manager() as manager: + shared_list = manager.list() + with Pool(processes=args.n_jobs) as pool: + for file_path in jsonl_files: + pool.apply_async(process_file, args=(file_path, shared_list)) + pool.close() + pool.join() + + with open(args.meta_file_path, "w") as f: + for item in shared_list: + f.write(json.dumps(item) + '\n') + + df = pd.read_json(args.meta_file_path, lines=True) + df = df.iloc[index_natsorted(df[args.video_path_column])].reset_index(drop=True) + logger.info(f"Save the gathered single jsonl file to {args.meta_file_path}.") + df.to_json(args.meta_file_path, orient="records", lines=True, force_ascii=False) + + +if __name__ == '__main__': + main() diff --git a/VideoX-Fun/videox_fun/video_caption/utils/get_meta_file.py b/VideoX-Fun/videox_fun/video_caption/utils/get_meta_file.py new file mode 100644 index 0000000000000000000000000000000000000000..e013351cb76ff830b5774e0f9672e67d7a2a0e8b --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/get_meta_file.py @@ -0,0 +1,89 @@ +import argparse +import os + +import pandas as pd +from natsort import natsorted + +from .logger import logger + +ALL_VIDEO_EXT = set(["mp4", "webm", "mkv", "avi", "flv", "mov", "rmvb"]) +ALL_IMGAE_EXT = set(["png", "webp", "jpg", "jpeg", "bmp", "gif"]) + + +def get_relative_file_paths(directory, recursive=False, ext_set=None): + """Get the relative paths of subfiles (recursively) in the directory that match the extension set. + """ + if not recursive: + for entry in os.scandir(directory): + if entry.is_file(): + file_name = entry.name + if ext_set is not None: + ext = os.path.splitext(file_name)[1][1:].lower() + if ext in ext_set: + yield file_name + else: + yield file_name + else: + for root, _, files in os.walk(directory): + for file in files: + relative_path = os.path.relpath(os.path.join(root, file), directory) + if ext_set is not None: + ext = os.path.splitext(file)[1][1:].lower() + if ext in ext_set: + yield relative_path + else: + yield relative_path + + +def parse_args(): + parser = argparse.ArgumentParser(description="Compute scores of uniform sampled frames from videos.") + parser.add_argument( + "--image_path_column", + type=str, + default="image_path", + help="The column contains the image path (an absolute path or a relative path w.r.t the image_folder).", + ) + parser.add_argument("--image_folder", type=str, default=None, help="The video folder.") + parser.add_argument( + "--video_path_column", + type=str, + default="video_path", + help="The column contains the video path (an absolute path or a relative path w.r.t the video_folder).", + ) + parser.add_argument("--video_folder", type=str, default=None, help="The video folder.") + parser.add_argument("--saved_path", type=str, required=True, help="The save path to the output results (csv/jsonl).") + parser.add_argument("--recursive", action="store_true", help="Whether to search sub-folders recursively.") + + args = parser.parse_args() + return args + + +def main(): + args = parse_args() + + if args.video_folder is None and args.image_folder is None: + raise ValueError("Either video_folder or image_folder should be specified in the arguments.") + if args.video_folder is not None and args.image_folder is not None: + raise ValueError("Both video_folder and image_folder can not be specified in the arguments at the same time.") + if args.image_folder is None and not os.path.exists(args.video_folder): + raise ValueError(f"The video_folder {args.video_folder} does not exist.") + if args.video_folder is None and not os.path.exists(args.image_folder): + raise ValueError(f"The image_folder {args.image_folder} does not exist.") + + # Use the path name instead of the file name as video_path/image_path (unique ID). + if args.video_folder is not None: + video_path_list = list(get_relative_file_paths(args.video_folder, recursive=args.recursive, ext_set=ALL_VIDEO_EXT)) + video_path_list = natsorted(video_path_list) + meta_file_df = pd.DataFrame({args.video_path_column: video_path_list}) + + if args.image_folder is not None: + image_path_list = list(get_relative_file_paths(args.image_folder, recursive=args.recursive, ext_set=ALL_IMGAE_EXT)) + image_path_list = natsorted(image_path_list) + meta_file_df = pd.DataFrame({args.image_path_column: image_path_list}) + + logger.info(f"{len(meta_file_df)} files in total. Save the result to {args.saved_path}.") + meta_file_df.to_json(args.saved_path, orient="records", lines=True) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/image_evaluator.py b/VideoX-Fun/videox_fun/video_caption/utils/image_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..f402477813343a406ced6aa5bec661598a82fbd2 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/image_evaluator.py @@ -0,0 +1,249 @@ +import os +from typing import Union + +import clip +import torch +import torch.nn as nn +import torch.nn.functional as F +from PIL import Image +from torchvision.datasets.utils import download_url +from transformers import AutoModel, AutoProcessor + +from .siglip_v2_5 import convert_v2_5_from_siglip + +# All metrics. +__all__ = ["AestheticScore", "AestheticScoreSigLIP", "CLIPScore"] + +_MODELS = { + "CLIP_ViT-L/14": "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/video_caption/clip/ViT-L-14.pt", + "Aesthetics_V2": "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/video_caption/clip/sac%2Blogos%2Bava1-l14-linearMSE.pth", + "aesthetic_predictor_v2_5": "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/video_caption/clip/aesthetic_predictor_v2_5.pth", +} +_MD5 = { + "CLIP_ViT-L/14": "096db1af569b284eb76b3881534822d9", + "Aesthetics_V2": "b1047fd767a00134b8fd6529bf19521a", + "aesthetic_predictor_v2_5": "c46eb8c29f714c9231dc630b8226842a", +} + + +def get_list_depth(lst): + if isinstance(lst, list): + return 1 + max(get_list_depth(item) for item in lst) + else: + return 0 + + +def reshape_images(images: Union[list[list[Image.Image]], list[Image.Image]]): + # Check the input sanity. + depth = get_list_depth(images) + if depth == 1: # batch image input + if not isinstance(images[0], Image.Image): + raise ValueError("The item in 1D images should be Image.Image.") + num_sampled_frames = None + elif depth == 2: # batch video input + if not isinstance(images[0][0], Image.Image): + raise ValueError("The item in 2D images (videos) should be Image.Image.") + num_sampled_frames = len(images[0]) + if not all(len(video_frames) == num_sampled_frames for video_frames in images): + raise ValueError("All item in 2D images should be with the same length.") + # [batch_size, num_sampled_frames, H, W, C] => [batch_size * num_sampled_frames, H, W, C]. + reshaped_images = [] + for video_frames in images: + reshaped_images.extend([frame for frame in video_frames]) + images = reshaped_images + else: + raise ValueError("The input images should be in 1/2D list.") + + return images, num_sampled_frames + + +def reshape_scores(scores: list[float], num_sampled_frames: int) -> list[float]: + if isinstance(scores, list): + if num_sampled_frames is not None: # Batch video input + batch_size = len(scores) // num_sampled_frames + scores = [ + scores[i * num_sampled_frames:(i + 1) * num_sampled_frames] + for i in range(batch_size) + ] + return scores + else: + return [scores] + + +# if you changed the MLP architecture during training, change it also here: +class _MLP(nn.Module): + def __init__(self, input_size): + super().__init__() + self.input_size = input_size + self.layers = nn.Sequential( + nn.Linear(self.input_size, 1024), + # nn.ReLU(), + nn.Dropout(0.2), + nn.Linear(1024, 128), + # nn.ReLU(), + nn.Dropout(0.2), + nn.Linear(128, 64), + # nn.ReLU(), + nn.Dropout(0.1), + nn.Linear(64, 16), + # nn.ReLU(), + nn.Linear(16, 1), + ) + + def forward(self, x): + return self.layers(x) + + +class AestheticScore: + """Compute LAION Aesthetics Score V2 based on openai/clip. Note that the default + inference dtype with GPUs is fp16 in openai/clip. + + Ref: + 1. https://github.com/christophschuhmann/improved-aesthetic-predictor/blob/main/simple_inference.py. + 2. https://github.com/openai/CLIP/issues/30. + """ + + def __init__(self, root: str = "~/.cache/clip", device: str = "cpu"): + # The CLIP model is loaded in the evaluation mode. + self.root = os.path.expanduser(root) + if not os.path.exists(self.root): + os.makedirs(self.root) + filename = "ViT-L-14.pt" + download_url(_MODELS["CLIP_ViT-L/14"], self.root, filename=filename, md5=_MD5["CLIP_ViT-L/14"]) + self.clip_model, self.preprocess = clip.load(os.path.join(self.root, filename), device=device) + self.device = device + self._load_mlp() + + def _load_mlp(self): + filename = "sac+logos+ava1-l14-linearMSE.pth" + download_url(_MODELS["Aesthetics_V2"], self.root, filename=filename, md5=_MD5["Aesthetics_V2"]) + state_dict = torch.load(os.path.join(self.root, filename)) + self.mlp = _MLP(768) + self.mlp.load_state_dict(state_dict) + self.mlp.to(self.device) + self.mlp.eval() + + def __call__(self, images: Union[list[list[Image.Image]], list[Image.Image]], texts=None) -> list[float]: + images, num_sampled_frames = reshape_images(images) + + with torch.no_grad(): + images = torch.stack([self.preprocess(image) for image in images]).to(self.device) + image_embs = F.normalize(self.clip_model.encode_image(images)) + scores = self.mlp(image_embs.float()) # torch.float16 -> torch.float32, [N, 1] + + scores = scores.squeeze().tolist() # scalar or list + return reshape_scores(scores, num_sampled_frames) + + def __repr__(self) -> str: + return "aesthetic_score" + + +class AestheticScoreSigLIP: + """Compute Aesthetics Score V2.5 based on google/siglip-so400m-patch14-384. + + Ref: + 1. https://github.com/discus0434/aesthetic-predictor-v2-5. + 2. https://github.com/discus0434/aesthetic-predictor-v2-5/issues/2. + """ + + def __init__( + self, + root: str = "~/.cache/clip", + device: str = "cpu", + torch_dtype=torch.float16 + ): + self.root = os.path.expanduser(root) + if not os.path.exists(self.root): + os.makedirs(self.root) + filename = "aesthetic_predictor_v2_5.pth" + download_url(_MODELS["aesthetic_predictor_v2_5"], self.root, filename=filename, md5=_MD5["aesthetic_predictor_v2_5"]) + self.model, self.preprocessor = convert_v2_5_from_siglip( + predictor_name_or_path=os.path.join(self.root, filename), + low_cpu_mem_usage=True, + trust_remote_code=True, + ) + self.model = self.model.to(device=device, dtype=torch_dtype) + self.device = device + self.torch_dtype = torch_dtype + + def __call__(self, images: Union[list[list[Image.Image]], list[Image.Image]], texts=None) -> list[float]: + images, num_sampled_frames = reshape_images(images) + + pixel_values = self.preprocessor(images, return_tensors="pt").pixel_values + pixel_values = pixel_values.to(self.device, self.torch_dtype) + with torch.no_grad(): + scores = self.model(pixel_values).logits.squeeze().float().cpu().numpy() + + scores = scores.squeeze().tolist() # scalar or list + return reshape_scores(scores, num_sampled_frames) + + def __repr__(self) -> str: + return "aesthetic_score_siglip" + + +class CLIPScore: + """Compute CLIP scores for image-text pairs based on huggingface/transformers.""" + + def __init__( + self, + model_name_or_path: str = "openai/clip-vit-large-patch14", + torch_dtype=torch.float16, + device: str = "cpu", + ): + self.model = AutoModel.from_pretrained(model_name_or_path, torch_dtype=torch_dtype).eval().to(device) + self.processor = AutoProcessor.from_pretrained(model_name_or_path) + self.torch_dtype = torch_dtype + self.device = device + + def __call__(self, images: Union[list[list[Image.Image]], list[Image.Image]], texts: list[str]) -> list[float]: + assert len(images) == len(texts) + images, num_sampled_frames = reshape_images(images) + # Expand texts in the batch video input case. + if num_sampled_frames is not None: + texts = [[text] * num_sampled_frames for text in texts] + texts = [item for sublist in texts for item in sublist] + + image_inputs = self.processor(images=images, return_tensors="pt") # {"pixel_values": } + if self.torch_dtype == torch.float16: + image_inputs["pixel_values"] = image_inputs["pixel_values"].half() + text_inputs = self.processor(text=texts, return_tensors="pt", padding=True, truncation=True) # {"inputs_id": } + image_inputs, text_inputs = image_inputs.to(self.device), text_inputs.to(self.device) + with torch.no_grad(): + image_embs = F.normalize(self.model.get_image_features(**image_inputs)) + text_embs = F.normalize(self.model.get_text_features(**text_inputs)) + scores = text_embs @ image_embs.T # [N, N] + + scores = scores.squeeze().tolist() # scalar or list + return reshape_scores(scores, num_sampled_frames) + + def __repr__(self) -> str: + return "clip_score" + + +if __name__ == "__main__": + from torch.utils.data import DataLoader + from tqdm import tqdm + + from .video_dataset import VideoDataset, collate_fn + + aesthetic_score = AestheticScore(device="cuda") + aesthetic_score_siglip = AestheticScoreSigLIP(device="cuda") + # clip_score = CLIPScore(device="cuda") + + paths = ["your_image_path"] * 3 + # texts = ["a joker", "a woman", "a man"] + images = [Image.open(p).convert("RGB") for p in paths] + + print(aesthetic_score(images)) + # print(clip_score(images, texts)) + + test_dataset = VideoDataset( + dataset_inputs={"video_path": ["your_video_path"] * 3}, + sample_method="mid", + num_sampled_frames=2 + ) + test_loader = DataLoader(test_dataset, batch_size=1, num_workers=1, collate_fn=collate_fn) + + for idx, batch in enumerate(tqdm(test_loader)): + batch_frame = batch["sampled_frame"] + print(aesthetic_score_siglip(batch_frame)) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/logger.py b/VideoX-Fun/videox_fun/video_caption/utils/logger.py new file mode 100644 index 0000000000000000000000000000000000000000..754eaf6b379aa39e8b9469c95e17c8ec8128e30d --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/logger.py @@ -0,0 +1,36 @@ +# Borrowed from sd-webui-controlnet/scripts/logging.py +import copy +import logging +import sys + + +class ColoredFormatter(logging.Formatter): + COLORS = { + "DEBUG": "\033[0;36m", # CYAN + "INFO": "\033[0;32m", # GREEN + "WARNING": "\033[0;33m", # YELLOW + "ERROR": "\033[0;31m", # RED + "CRITICAL": "\033[0;37;41m", # WHITE ON RED + "RESET": "\033[0m", # RESET COLOR + } + + def format(self, record): + colored_record = copy.copy(record) + levelname = colored_record.levelname + seq = self.COLORS.get(levelname, self.COLORS["RESET"]) + colored_record.levelname = f"{seq}{levelname}{self.COLORS['RESET']}" + return super().format(colored_record) + + +# Create a new logger +logger = logging.getLogger("VideoCaption") +logger.propagate = False + +# Add handler if we don't have one. +if not logger.handlers: + handler = logging.StreamHandler(sys.stdout) + handler.setFormatter(ColoredFormatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")) + logger.addHandler(handler) + +# Configure logger +logger.setLevel("INFO") diff --git a/VideoX-Fun/videox_fun/video_caption/utils/longclip/README.md b/VideoX-Fun/videox_fun/video_caption/utils/longclip/README.md new file mode 100644 index 0000000000000000000000000000000000000000..9e99054e2097d5133812daf6af2b20e3673b2b88 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/longclip/README.md @@ -0,0 +1,19 @@ +# Long-CLIP +Codes in this directory are borrowed from https://github.com/beichenzbc/Long-CLIP/tree/4e6f5da/model. + +We only modify the following code in [model_longclip.py](model_longclip.py) from +```python +@property +def dtype(self): + return self.visual.conv1.weight.dtype +``` +to +```python +@property +def dtype(self): + # Fix: the VideoCLIP-XL inference. + if hasattr(self, "visual"): + return self.visual.conv1.weight.dtype + else: + return self.token_embedding.weight.dtype +``` \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/longclip/bpe_simple_vocab_16e6.txt.gz b/VideoX-Fun/videox_fun/video_caption/utils/longclip/bpe_simple_vocab_16e6.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..36a15856e00a06a9fbed8cdd34d2393fea4a3113 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/longclip/bpe_simple_vocab_16e6.txt.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a +size 1356917 diff --git a/VideoX-Fun/videox_fun/video_caption/utils/longclip/longclip.py b/VideoX-Fun/videox_fun/video_caption/utils/longclip/longclip.py new file mode 100644 index 0000000000000000000000000000000000000000..a2a5cc3e9f2bddf221ff414c73702f9c2a5a3af6 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/longclip/longclip.py @@ -0,0 +1,355 @@ +import hashlib +import os +import urllib +import warnings +from typing import Any, List, Union + +import torch +from PIL import Image +from pkg_resources import packaging +from torch import nn +from torchvision.transforms import (CenterCrop, Compose, Normalize, Resize, + ToTensor) +from tqdm import tqdm + +from .model_longclip import build_model +from .simple_tokenizer import SimpleTokenizer as _Tokenizer + +try: + from torchvision.transforms import InterpolationMode + BICUBIC = InterpolationMode.BICUBIC +except ImportError: + BICUBIC = Image.BICUBIC + + +if packaging.version.parse(torch.__version__) < packaging.version.parse("1.7.1"): + warnings.warn("PyTorch version 1.7.1 or higher is recommended") + + +__all__ = ["load", "tokenize"] +_tokenizer = _Tokenizer() + + +def _convert_image_to_rgb(image): + return image.convert("RGB") + + +def _transform(n_px): + return Compose([ + Resize(n_px, interpolation=BICUBIC), + CenterCrop(n_px), + _convert_image_to_rgb, + ToTensor(), + Normalize((0.48145466, 0.4578275, 0.40821073), (0.26862954, 0.26130258, 0.27577711)), + ]) + + + +def load(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", download_root: str = None): + """Load a long CLIP model + + Parameters + ---------- + name : str + A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict + + device : Union[str, torch.device] + The device to put the loaded model + + Returns + ------- + model : torch.nn.Module + The CLIP model + + preprocess : Callable[[PIL.Image], torch.Tensor] + A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input + """ + + model_path = name + + state_dict = torch.load(model_path, map_location="cpu") + + model = build_model(state_dict or model.state_dict(), load_from_clip = False).to(device) + + if str(device) == "cpu": + model.float() + + return model, _transform(model.visual.input_resolution) + + + + def _node_get(node: torch._C.Node, key: str): + """Gets attributes of a node which is polymorphic over return type. + + From https://github.com/pytorch/pytorch/pull/82628 + """ + sel = node.kindOf(key) + return getattr(node, sel)(key) + + def patch_device(module): + try: + graphs = [module.graph] if hasattr(module, "graph") else [] + except RuntimeError: + graphs = [] + + if hasattr(module, "forward1"): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes("prim::Constant"): + if "value" in node.attributeNames() and str(_node_get(node, "value")).startswith("cuda"): + node.copyAttributes(device_node) + + model.apply(patch_device) + patch_device(model.encode_image) + patch_device(model.encode_text) + + # patch dtype to float32 on CPU + if str(device) == "cpu": + float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[]) + float_input = list(float_holder.graph.findNode("aten::to").inputs())[1] + float_node = float_input.node() + + def patch_float(module): + try: + graphs = [module.graph] if hasattr(module, "graph") else [] + except RuntimeError: + graphs = [] + + if hasattr(module, "forward1"): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes("aten::to"): + inputs = list(node.inputs()) + for i in [1, 2]: # dtype can be the second or third argument to aten::to() + if _node_get(inputs[i].node(), "value") == 5: + inputs[i].node().copyAttributes(float_node) + + model.apply(patch_float) + patch_float(model.encode_image) + patch_float(model.encode_text) + + model.float() + + return model, _transform(model.input_resolution.item()) + + +def load_from_clip(name: str, device: Union[str, torch.device] = "cuda" if torch.cuda.is_available() else "cpu", jit: bool = False, download_root: str = None): + """Load from CLIP model for fine-tuning + + Parameters + ---------- + name : str + A model name listed by `clip.available_models()`, or the path to a model checkpoint containing the state_dict + + device : Union[str, torch.device] + The device to put the loaded model + + jit : bool + Whether to load the optimized JIT model or more hackable non-JIT model (default). + + download_root: str + path to download the model files; by default, it uses "~/.cache/clip" + + Returns + ------- + model : torch.nn.Module + The CLIP model + + preprocess : Callable[[PIL.Image], torch.Tensor] + A torchvision transform that converts a PIL image into a tensor that the returned model can take as its input + """ + + _MODELS = { + "RN50": "https://openaipublic.azureedge.net/clip/models/afeb0e10f9e5a86da6080e35cf09123aca3b358a0c3e3b6c78a7b63bc04b6762/RN50.pt", + "RN101": "https://openaipublic.azureedge.net/clip/models/8fa8567bab74a42d41c5915025a8e4538c3bdbe8804a470a72f30b0d94fab599/RN101.pt", + "RN50x4": "https://openaipublic.azureedge.net/clip/models/7e526bd135e493cef0776de27d5f42653e6b4c8bf9e0f653bb11773263205fdd/RN50x4.pt", + "RN50x16": "https://openaipublic.azureedge.net/clip/models/52378b407f34354e150460fe41077663dd5b39c54cd0bfd2b27167a4a06ec9aa/RN50x16.pt", + "RN50x64": "https://openaipublic.azureedge.net/clip/models/be1cfb55d75a9666199fb2206c106743da0f6468c9d327f3e0d0a543a9919d9c/RN50x64.pt", + "ViT-B/32": "https://openaipublic.azureedge.net/clip/models/40d365715913c9da98579312b702a82c18be219cc2a73407c4526f58eba950af/ViT-B-32.pt", + "ViT-B/16": "https://openaipublic.azureedge.net/clip/models/5806e77cd80f8b59890b7e101eabd078d9fb84e6937f9e85e4ecb61988df416f/ViT-B-16.pt", + "ViT-L/14": "https://openaipublic.azureedge.net/clip/models/b8cca3fd41ae0c99ba7e8951adf17d267cdb84cd88be6f7c2e0eca1737a03836/ViT-L-14.pt", + "ViT-L/14@336px": "https://openaipublic.azureedge.net/clip/models/3035c92b350959924f9f00213499208652fc7ea050643e8b385c2dac08641f02/ViT-L-14-336px.pt", + } + + def available_models() -> List[str]: + """Returns the names of available CLIP models""" + return list(_MODELS.keys()) + + def _download(url: str, root: str): + os.makedirs(root, exist_ok=True) + filename = os.path.basename(url) + + expected_sha256 = url.split("/")[-2] + download_target = os.path.join(root, filename) + + if os.path.exists(download_target) and not os.path.isfile(download_target): + raise RuntimeError(f"{download_target} exists and is not a regular file") + + if os.path.isfile(download_target): + if hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256: + return download_target + else: + warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file") + + with urllib.request.urlopen(url) as source, open(download_target, "wb") as output: + with tqdm(total=int(source.info().get("Content-Length")), ncols=80, unit='iB', unit_scale=True, unit_divisor=1024) as loop: + while True: + buffer = source.read(8192) + if not buffer: + break + + output.write(buffer) + loop.update(len(buffer)) + + if hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256: + raise RuntimeError("Model has been downloaded but the SHA256 checksum does not not match") + + return download_target + + if name in _MODELS: + model_path = _download(_MODELS[name], download_root or os.path.expanduser("~/.cache/clip")) + elif os.path.isfile(name): + model_path = name + else: + raise RuntimeError(f"Model {name} not found; available models = {available_models()}") + + with open(model_path, 'rb') as opened_file: + try: + # loading JIT archive + model = torch.jit.load(opened_file, map_location=device if jit else "cpu").eval() + state_dict = None + except RuntimeError: + # loading saved state dict + if jit: + warnings.warn(f"File {model_path} is not a JIT archive. Loading as a state dict instead") + jit = False + state_dict = torch.load(opened_file, map_location="cpu") + + model = build_model(state_dict or model.state_dict(), load_from_clip = True).to(device) + + positional_embedding_pre = model.positional_embedding.type(model.dtype) + + length, dim = positional_embedding_pre.shape + keep_len = 20 + posisitonal_embedding_new = torch.zeros([4*length-3*keep_len, dim], dtype=model.dtype) + for i in range(keep_len): + posisitonal_embedding_new[i] = positional_embedding_pre[i] + for i in range(length-1-keep_len): + posisitonal_embedding_new[4*i + keep_len] = positional_embedding_pre[i + keep_len] + posisitonal_embedding_new[4*i + 1 + keep_len] = 3*positional_embedding_pre[i + keep_len]/4 + 1*positional_embedding_pre[i+1+keep_len]/4 + posisitonal_embedding_new[4*i + 2+keep_len] = 2*positional_embedding_pre[i+keep_len]/4 + 2*positional_embedding_pre[i+1+keep_len]/4 + posisitonal_embedding_new[4*i + 3+keep_len] = 1*positional_embedding_pre[i+keep_len]/4 + 3*positional_embedding_pre[i+1+keep_len]/4 + + posisitonal_embedding_new[4*length -3*keep_len - 4] = positional_embedding_pre[length-1] + 0*(positional_embedding_pre[length-1] - positional_embedding_pre[length-2])/4 + posisitonal_embedding_new[4*length -3*keep_len - 3] = positional_embedding_pre[length-1] + 1*(positional_embedding_pre[length-1] - positional_embedding_pre[length-2])/4 + posisitonal_embedding_new[4*length -3*keep_len - 2] = positional_embedding_pre[length-1] + 2*(positional_embedding_pre[length-1] - positional_embedding_pre[length-2])/4 + posisitonal_embedding_new[4*length -3*keep_len - 1] = positional_embedding_pre[length-1] + 3*(positional_embedding_pre[length-1] - positional_embedding_pre[length-2])/4 + + positional_embedding_res = posisitonal_embedding_new.clone() + + model.positional_embedding = nn.Parameter(posisitonal_embedding_new, requires_grad=False) + model.positional_embedding_res = nn.Parameter(positional_embedding_res, requires_grad=True) + + if str(device) == "cpu": + model.float() + return model, _transform(model.visual.input_resolution) + + def _node_get(node: torch._C.Node, key: str): + """Gets attributes of a node which is polymorphic over return type. + + From https://github.com/pytorch/pytorch/pull/82628 + """ + sel = node.kindOf(key) + return getattr(node, sel)(key) + + def patch_device(module): + try: + graphs = [module.graph] if hasattr(module, "graph") else [] + except RuntimeError: + graphs = [] + + if hasattr(module, "forward1"): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes("prim::Constant"): + if "value" in node.attributeNames() and str(_node_get(node, "value")).startswith("cuda"): + node.copyAttributes(device_node) + + model.apply(patch_device) + patch_device(model.encode_image) + patch_device(model.encode_text) + + # patch dtype to float32 on CPU + if str(device) == "cpu": + float_holder = torch.jit.trace(lambda: torch.ones([]).float(), example_inputs=[]) + float_input = list(float_holder.graph.findNode("aten::to").inputs())[1] + float_node = float_input.node() + + def patch_float(module): + try: + graphs = [module.graph] if hasattr(module, "graph") else [] + except RuntimeError: + graphs = [] + + if hasattr(module, "forward1"): + graphs.append(module.forward1.graph) + + for graph in graphs: + for node in graph.findAllNodes("aten::to"): + inputs = list(node.inputs()) + for i in [1, 2]: # dtype can be the second or third argument to aten::to() + if _node_get(inputs[i].node(), "value") == 5: + inputs[i].node().copyAttributes(float_node) + + model.apply(patch_float) + patch_float(model.encode_image) + patch_float(model.encode_text) + + model.float() + + return model, _transform(model.input_resolution.item()) + +def tokenize(texts: Union[str, List[str]], context_length: int = 77*4-60, truncate: bool = False) -> Union[torch.IntTensor, torch.LongTensor]: + """ + Returns the tokenized representation of given input string(s) + + Parameters + ---------- + texts : Union[str, List[str]] + An input string or a list of input strings to tokenize + + context_length : int + The context length to use; all CLIP models use 77 as the context length + + truncate: bool + Whether to truncate the text in case its encoding is longer than the context length + + Returns + ------- + A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]. + We return LongTensor when torch version is <1.8.0, since older index_select requires indices to be long. + """ + if isinstance(texts, str): + texts = [texts] + + sot_token = _tokenizer.encoder["<|startoftext|>"] + eot_token = _tokenizer.encoder["<|endoftext|>"] + all_tokens = [[sot_token] + _tokenizer.encode(text) + [eot_token] for text in texts] + if packaging.version.parse(torch.__version__) < packaging.version.parse("1.8.0"): + result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) + else: + result = torch.zeros(len(all_tokens), context_length, dtype=torch.int) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + if truncate: + tokens = tokens[:context_length] + tokens[-1] = eot_token + else: + raise RuntimeError(f"Input {texts[i]} is too long for context length {context_length}") + result[i, :len(tokens)] = torch.tensor(tokens) + + return result diff --git a/VideoX-Fun/videox_fun/video_caption/utils/longclip/model_longclip.py b/VideoX-Fun/videox_fun/video_caption/utils/longclip/model_longclip.py new file mode 100644 index 0000000000000000000000000000000000000000..aefd521b9a45d12982d8727f03d25ae1c4cb39c7 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/longclip/model_longclip.py @@ -0,0 +1,471 @@ +from collections import OrderedDict +from typing import Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + + +class Bottleneck(nn.Module): + expansion = 4 + + def __init__(self, inplanes, planes, stride=1): + super().__init__() + + # all conv layers have stride 1. an avgpool is performed after the second convolution when stride > 1 + self.conv1 = nn.Conv2d(inplanes, planes, 1, bias=False) + self.bn1 = nn.BatchNorm2d(planes) + self.relu1 = nn.ReLU(inplace=True) + + self.conv2 = nn.Conv2d(planes, planes, 3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(planes) + self.relu2 = nn.ReLU(inplace=True) + + self.avgpool = nn.AvgPool2d(stride) if stride > 1 else nn.Identity() + + self.conv3 = nn.Conv2d(planes, planes * self.expansion, 1, bias=False) + self.bn3 = nn.BatchNorm2d(planes * self.expansion) + self.relu3 = nn.ReLU(inplace=True) + + self.downsample = None + self.stride = stride + + if stride > 1 or inplanes != planes * Bottleneck.expansion: + # downsampling layer is prepended with an avgpool, and the subsequent convolution has stride 1 + self.downsample = nn.Sequential(OrderedDict([ + ("-1", nn.AvgPool2d(stride)), + ("0", nn.Conv2d(inplanes, planes * self.expansion, 1, stride=1, bias=False)), + ("1", nn.BatchNorm2d(planes * self.expansion)) + ])) + + def forward(self, x: torch.Tensor): + identity = x + + out = self.relu1(self.bn1(self.conv1(x))) + out = self.relu2(self.bn2(self.conv2(out))) + out = self.avgpool(out) + out = self.bn3(self.conv3(out)) + + if self.downsample is not None: + identity = self.downsample(x) + + out += identity + out = self.relu3(out) + return out + + +class AttentionPool2d(nn.Module): + def __init__(self, spacial_dim: int, embed_dim: int, num_heads: int, output_dim: int = None): + super().__init__() + self.positional_embedding = nn.Parameter(torch.randn(spacial_dim ** 2 + 1, embed_dim) / embed_dim ** 0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim) + self.q_proj = nn.Linear(embed_dim, embed_dim) + self.v_proj = nn.Linear(embed_dim, embed_dim) + self.c_proj = nn.Linear(embed_dim, output_dim or embed_dim) + self.num_heads = num_heads + + def forward(self, x): + x = x.flatten(start_dim=2).permute(2, 0, 1) # NCHW -> (HW)NC + x = torch.cat([x.mean(dim=0, keepdim=True), x], dim=0) # (HW+1)NC + x = x + self.positional_embedding[:, None, :].to(x.dtype) # (HW+1)NC + x, _ = F.multi_head_attention_forward( + query=x[:1], key=x, value=x, + embed_dim_to_check=x.shape[-1], + num_heads=self.num_heads, + q_proj_weight=self.q_proj.weight, + k_proj_weight=self.k_proj.weight, + v_proj_weight=self.v_proj.weight, + in_proj_weight=None, + in_proj_bias=torch.cat([self.q_proj.bias, self.k_proj.bias, self.v_proj.bias]), + bias_k=None, + bias_v=None, + add_zero_attn=False, + dropout_p=0, + out_proj_weight=self.c_proj.weight, + out_proj_bias=self.c_proj.bias, + use_separate_proj_weight=True, + training=self.training, + need_weights=False + ) + return x.squeeze(0) + + +class ModifiedResNet(nn.Module): + """ + A ResNet class that is similar to torchvision's but contains the following changes: + - There are now 3 "stem" convolutions as opposed to 1, with an average pool instead of a max pool. + - Performs anti-aliasing strided convolutions, where an avgpool is prepended to convolutions with stride > 1 + - The final pooling layer is a QKV attention instead of an average pool + """ + + def __init__(self, layers, output_dim, heads, input_resolution=224, width=64): + super().__init__() + self.output_dim = output_dim + self.input_resolution = input_resolution + + # the 3-layer stem + self.conv1 = nn.Conv2d(3, width // 2, kernel_size=3, stride=2, padding=1, bias=False) + self.bn1 = nn.BatchNorm2d(width // 2) + self.relu1 = nn.ReLU(inplace=True) + self.conv2 = nn.Conv2d(width // 2, width // 2, kernel_size=3, padding=1, bias=False) + self.bn2 = nn.BatchNorm2d(width // 2) + self.relu2 = nn.ReLU(inplace=True) + self.conv3 = nn.Conv2d(width // 2, width, kernel_size=3, padding=1, bias=False) + self.bn3 = nn.BatchNorm2d(width) + self.relu3 = nn.ReLU(inplace=True) + self.avgpool = nn.AvgPool2d(2) + + # residual layers + self._inplanes = width # this is a *mutable* variable used during construction + self.layer1 = self._make_layer(width, layers[0]) + self.layer2 = self._make_layer(width * 2, layers[1], stride=2) + self.layer3 = self._make_layer(width * 4, layers[2], stride=2) + self.layer4 = self._make_layer(width * 8, layers[3], stride=2) + + embed_dim = width * 32 # the ResNet feature dimension + self.attnpool = AttentionPool2d(input_resolution // 32, embed_dim, heads, output_dim) + + def _make_layer(self, planes, blocks, stride=1): + layers = [Bottleneck(self._inplanes, planes, stride)] + + self._inplanes = planes * Bottleneck.expansion + for _ in range(1, blocks): + layers.append(Bottleneck(self._inplanes, planes)) + + return nn.Sequential(*layers) + + def forward(self, x): + def stem(x): + x = self.relu1(self.bn1(self.conv1(x))) + x = self.relu2(self.bn2(self.conv2(x))) + x = self.relu3(self.bn3(self.conv3(x))) + x = self.avgpool(x) + return x + + x = x.type(self.conv1.weight.dtype) + x = stem(x) + x = self.layer1(x) + x = self.layer2(x) + x = self.layer3(x) + x = self.layer4(x) + x = self.attnpool(x) + + return x + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential(OrderedDict([ + ("c_fc", nn.Linear(d_model, d_model * 4)), + ("gelu", QuickGELU()), + ("c_proj", nn.Linear(d_model * 4, d_model)) + ])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None + return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)]) + + def forward(self, x: torch.Tensor): + return self.resblocks(x) + + +class VisionTransformer(nn.Module): + def __init__(self, input_resolution: int, patch_size: int, width: int, layers: int, heads: int, output_dim: int): + super().__init__() + self.input_resolution = input_resolution + self.output_dim = output_dim + self.conv1 = nn.Conv2d(in_channels=3, out_channels=width, kernel_size=patch_size, stride=patch_size, bias=False) + + scale = width ** -0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width)) + self.ln_pre = LayerNorm(width) + + self.transformer = Transformer(width, layers, heads) + + self.ln_post = LayerNorm(width) + self.proj = nn.Parameter(scale * torch.randn(width, output_dim)) + + def forward(self, x: torch.Tensor): + x = self.conv1(x) # shape = [*, width, grid, grid] + x = x.reshape(x.shape[0], x.shape[1], -1) # shape = [*, width, grid ** 2] + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width] + x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1) # shape = [*, grid ** 2 + 1, width] + x = x + self.positional_embedding.to(x.dtype) + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + + x = self.ln_post(x[:, 0, :]) + + if self.proj is not None: + x = x @ self.proj + + return x + + +class CLIP(nn.Module): + def __init__(self, + embed_dim: int, + # vision + image_resolution: int, + vision_layers: Union[Tuple[int, int, int, int], int], + vision_width: int, + vision_patch_size: int, + # text + context_length: int, + vocab_size: int, + transformer_width: int, + transformer_heads: int, + transformer_layers: int, + load_from_clip: bool + ): + super().__init__() + + self.context_length = 248 + + if isinstance(vision_layers, (tuple, list)): + vision_heads = vision_width * 32 // 64 + self.visual = ModifiedResNet( + layers=vision_layers, + output_dim=embed_dim, + heads=vision_heads, + input_resolution=image_resolution, + width=vision_width + ) + else: + vision_heads = vision_width // 64 + self.visual = VisionTransformer( + input_resolution=image_resolution, + patch_size=vision_patch_size, + width=vision_width, + layers=vision_layers, + heads=vision_heads, + output_dim=embed_dim + ) + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask() + ) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + + if load_from_clip == False: + self.positional_embedding = nn.Parameter(torch.empty(248, transformer_width)) + self.positional_embedding_res = nn.Parameter(torch.empty(248, transformer_width)) + + else: + self.positional_embedding = nn.Parameter(torch.empty(77, transformer_width)) + + self.ln_final = LayerNorm(transformer_width) + + self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim)) + self.logit_scale = nn.Parameter(torch.ones([]) * np.log(1 / 0.07)) + + self.initialize_parameters() + self.mask1 = torch.zeros([248, 1]) + self.mask1[:20, :] = 1 + self.mask2 = torch.zeros([248, 1]) + self.mask2[20:, :] = 1 + + + def initialize_parameters(self): + nn.init.normal_(self.token_embedding.weight, std=0.02) + nn.init.normal_(self.positional_embedding, std=0.01) + + if isinstance(self.visual, ModifiedResNet): + if self.visual.attnpool is not None: + std = self.visual.attnpool.c_proj.in_features ** -0.5 + nn.init.normal_(self.visual.attnpool.q_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.k_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.v_proj.weight, std=std) + nn.init.normal_(self.visual.attnpool.c_proj.weight, std=std) + + for resnet_block in [self.visual.layer1, self.visual.layer2, self.visual.layer3, self.visual.layer4]: + for name, param in resnet_block.named_parameters(): + if name.endswith("bn3.weight"): + nn.init.zeros_(param) + + proj_std = (self.transformer.width ** -0.5) * ((2 * self.transformer.layers) ** -0.5) + attn_std = self.transformer.width ** -0.5 + fc_std = (2 * self.transformer.width) ** -0.5 + for block in self.transformer.resblocks: + nn.init.normal_(block.attn.in_proj_weight, std=attn_std) + nn.init.normal_(block.attn.out_proj.weight, std=proj_std) + nn.init.normal_(block.mlp.c_fc.weight, std=fc_std) + nn.init.normal_(block.mlp.c_proj.weight, std=proj_std) + + if self.text_projection is not None: + nn.init.normal_(self.text_projection, std=self.transformer.width ** -0.5) + + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(self.context_length, self.context_length) + mask.fill_(float("-inf")) + mask.triu_(1) # zero out the lower diagonal + return mask + + @property + def dtype(self): + # Fix: the mixclip inference. + if hasattr(self, "visual"): + return self.visual.conv1.weight.dtype + else: + return self.token_embedding.weight.dtype + + def encode_image(self, image): + return self.visual(image.type(self.dtype)) + + def encode_text(self, text): + x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model] + + x = x + (self.positional_embedding.to(x.device) * self.mask1.to(x.device)).type(self.dtype).to(x.device) + (self.positional_embedding_res.to(x.device) * self.mask2.to(x.device)).type(self.dtype).to(x.device) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x).type(self.dtype) + + # x.shape = [batch_size, n_ctx, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection + + return x + + def encode_text_full(self, text): + x = self.token_embedding(text).type(self.dtype) # [batch_size, n_ctx, d_model] + + x = x + (self.positional_embedding.to(x.device) * self.mask1.to(x.device)).type(self.dtype).to(x.device) + (self.positional_embedding_res.to(x.device) * self.mask2.to(x.device)).type(self.dtype).to(x.device) + + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x).type(self.dtype) + + # x.shape = [batch_size, n_ctx, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + #x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection + + return x + + + def forward(self, image, text): + image_features = self.encode_image(image) + text_features = self.encode_text(text) + + # normalized features + image_features = image_features / image_features.norm(dim=1, keepdim=True) + text_features = text_features / text_features.norm(dim=1, keepdim=True) + + # cosine similarity as logits + logit_scale = self.logit_scale.exp() + logits_per_image = logit_scale * image_features @ text_features.t() + logits_per_text = logits_per_image.t() + + # shape = [global_batch_size, global_batch_size] + return logits_per_image, logits_per_text + + +def convert_weights(model: nn.Module): + """Convert applicable model parameters to fp16""" + + def _convert_weights_to_fp16(l): + if isinstance(l, (nn.Conv1d, nn.Conv2d, nn.Linear)): + l.weight.data = l.weight.data.half() + if l.bias is not None: + l.bias.data = l.bias.data.half() + + if isinstance(l, nn.MultiheadAttention): + for attr in [*[f"{s}_proj_weight" for s in ["in", "q", "k", "v"]], "in_proj_bias", "bias_k", "bias_v"]: + tensor = getattr(l, attr) + if tensor is not None: + tensor.data = tensor.data.half() + + for name in ["text_projection", "proj"]: + if hasattr(l, name): + attr = getattr(l, name) + if attr is not None: + attr.data = attr.data.half() + + model.apply(_convert_weights_to_fp16) + + +def build_model(state_dict: dict, load_from_clip: bool): + vit = "visual.proj" in state_dict + + if vit: + vision_width = state_dict["visual.conv1.weight"].shape[0] + vision_layers = len([k for k in state_dict.keys() if k.startswith("visual.") and k.endswith(".attn.in_proj_weight")]) + vision_patch_size = state_dict["visual.conv1.weight"].shape[-1] + grid_size = round((state_dict["visual.positional_embedding"].shape[0] - 1) ** 0.5) + image_resolution = vision_patch_size * grid_size + else: + counts: list = [len(set(k.split(".")[2] for k in state_dict if k.startswith(f"visual.layer{b}"))) for b in [1, 2, 3, 4]] + vision_layers = tuple(counts) + vision_width = state_dict["visual.layer1.0.conv1.weight"].shape[0] + output_width = round((state_dict["visual.attnpool.positional_embedding"].shape[0] - 1) ** 0.5) + vision_patch_size = None + assert output_width ** 2 + 1 == state_dict["visual.attnpool.positional_embedding"].shape[0] + image_resolution = output_width * 32 + + embed_dim = state_dict["text_projection"].shape[1] + context_length = state_dict["positional_embedding"].shape[0] + vocab_size = state_dict["token_embedding.weight"].shape[0] + transformer_width = state_dict["ln_final.weight"].shape[0] + transformer_heads = transformer_width // 64 + transformer_layers = len(set(k.split(".")[2] for k in state_dict if k.startswith("transformer.resblocks"))) + + model = CLIP( + embed_dim, + image_resolution, vision_layers, vision_width, vision_patch_size, + context_length, vocab_size, transformer_width, transformer_heads, transformer_layers, load_from_clip + ) + + for key in ["input_resolution", "context_length", "vocab_size"]: + if key in state_dict: + del state_dict[key] + + convert_weights(model) + model.load_state_dict(state_dict) + return model.eval() diff --git a/VideoX-Fun/videox_fun/video_caption/utils/longclip/simple_tokenizer.py b/VideoX-Fun/videox_fun/video_caption/utils/longclip/simple_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..0a66286b7d5019c6e221932a813768038f839c91 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/longclip/simple_tokenizer.py @@ -0,0 +1,132 @@ +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re + + +@lru_cache() +def default_bpe(): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz") + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode("utf-8").split('\n') + merges = merges[1:49152-256-2+1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v+'' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'} + self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + ( token[-1] + '',) + pairs = get_pairs(word) + + if not pairs: + return token+'' + + while True: + bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word)-1 and word[i+1] == second: + new_word.append(first+second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('', ' ') + return text diff --git a/VideoX-Fun/videox_fun/video_caption/utils/siglip_v2_5.py b/VideoX-Fun/videox_fun/video_caption/utils/siglip_v2_5.py new file mode 100644 index 0000000000000000000000000000000000000000..56c0a16e5953587afee04c40c61a67c3ea89fd4a --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/siglip_v2_5.py @@ -0,0 +1,123 @@ +# Borrowed from https://github.com/discus0434/aesthetic-predictor-v2-5/blob/3125a9e/src/aesthetic_predictor_v2_5/siglip_v2_5.py. +import os +from collections import OrderedDict +from os import PathLike +from typing import Final + +import torch +import torch.nn as nn +from transformers import (SiglipImageProcessor, SiglipVisionConfig, + SiglipVisionModel, logging) +from transformers.image_processing_utils import BatchFeature +from transformers.modeling_outputs import ImageClassifierOutputWithNoAttention + +logging.set_verbosity_error() + +URL: Final[str] = ( + "https://github.com/discus0434/aesthetic-predictor-v2-5/raw/main/models/aesthetic_predictor_v2_5.pth" +) + + +class AestheticPredictorV2_5Head(nn.Module): + def __init__(self, config: SiglipVisionConfig) -> None: + super().__init__() + self.scoring_head = nn.Sequential( + nn.Linear(config.hidden_size, 1024), + nn.Dropout(0.5), + nn.Linear(1024, 128), + nn.Dropout(0.5), + nn.Linear(128, 64), + nn.Dropout(0.5), + nn.Linear(64, 16), + nn.Dropout(0.2), + nn.Linear(16, 1), + ) + + def forward(self, image_embeds: torch.Tensor) -> torch.Tensor: + return self.scoring_head(image_embeds) + + +class AestheticPredictorV2_5Model(SiglipVisionModel): + PATCH_SIZE = 14 + + def __init__(self, config: SiglipVisionConfig, *args, **kwargs) -> None: + super().__init__(config, *args, **kwargs) + self.layers = AestheticPredictorV2_5Head(config) + self.post_init() + + def forward( + self, + pixel_values: torch.FloatTensor | None = None, + labels: torch.Tensor | None = None, + return_dict: bool | None = None, + ) -> tuple | ImageClassifierOutputWithNoAttention: + return_dict = ( + return_dict if return_dict is not None else self.config.use_return_dict + ) + + outputs = super().forward( + pixel_values=pixel_values, + return_dict=return_dict, + ) + image_embeds = outputs.pooler_output + image_embeds_norm = image_embeds / image_embeds.norm(dim=-1, keepdim=True) + prediction = self.layers(image_embeds_norm) + + loss = None + if labels is not None: + loss_fct = nn.MSELoss() + loss = loss_fct() + + if not return_dict: + return (loss, prediction, image_embeds) + + return ImageClassifierOutputWithNoAttention( + loss=loss, + logits=prediction, + hidden_states=image_embeds, + ) + + +class AestheticPredictorV2_5Processor(SiglipImageProcessor): + def __init__(self, *args, **kwargs) -> None: + super().__init__(*args, **kwargs) + + def __call__(self, *args, **kwargs) -> BatchFeature: + return super().__call__(*args, **kwargs) + + @classmethod + def from_pretrained( + self, + pretrained_model_name_or_path: str + | PathLike = "google/siglip-so400m-patch14-384", + *args, + **kwargs, + ) -> "AestheticPredictorV2_5Processor": + return super().from_pretrained(pretrained_model_name_or_path, *args, **kwargs) + + +def convert_v2_5_from_siglip( + predictor_name_or_path: str | PathLike | None = None, + encoder_model_name: str = "google/siglip-so400m-patch14-384", + *args, + **kwargs, +) -> tuple[AestheticPredictorV2_5Model, AestheticPredictorV2_5Processor]: + model = AestheticPredictorV2_5Model.from_pretrained( + encoder_model_name, *args, **kwargs + ) + + processor = AestheticPredictorV2_5Processor.from_pretrained( + encoder_model_name, *args, **kwargs + ) + + if predictor_name_or_path is None or not os.path.exists(predictor_name_or_path): + state_dict = torch.hub.load_state_dict_from_url(URL, map_location="cpu") + else: + state_dict = torch.load(predictor_name_or_path, map_location="cpu") + + assert isinstance(state_dict, OrderedDict) + + model.layers.load_state_dict(state_dict) + model.eval() + + return model, processor \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/viclip/README.md b/VideoX-Fun/videox_fun/video_caption/utils/viclip/README.md new file mode 100644 index 0000000000000000000000000000000000000000..f5c9e2f1dccc77369adfc87569d967e172831ec2 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/viclip/README.md @@ -0,0 +1,2 @@ +# ViCLIP +Codes in this directory are borrowed from https://github.com/OpenGVLab/InternVideo/tree/73271ba/Data/InternVid/viclip. \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/viclip/bpe_simple_vocab_16e6.txt.gz b/VideoX-Fun/videox_fun/video_caption/utils/viclip/bpe_simple_vocab_16e6.txt.gz new file mode 100644 index 0000000000000000000000000000000000000000..36a15856e00a06a9fbed8cdd34d2393fea4a3113 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/viclip/bpe_simple_vocab_16e6.txt.gz @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:924691ac288e54409236115652ad4aa250f48203de50a9e4722a6ecd48d6804a +size 1356917 diff --git a/VideoX-Fun/videox_fun/video_caption/utils/viclip/simple_tokenizer.py b/VideoX-Fun/videox_fun/video_caption/utils/viclip/simple_tokenizer.py new file mode 100644 index 0000000000000000000000000000000000000000..744cba818a8ffaf53d4b557490843a33f43777c2 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/viclip/simple_tokenizer.py @@ -0,0 +1,135 @@ +import gzip +import html +import os +from functools import lru_cache + +import ftfy +import regex as re + + +@lru_cache() +def default_bpe(): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "bpe_simple_vocab_16e6.txt.gz") +# @lru_cache() +# def default_bpe(): +# return "bpe_simple_vocab_16e6.txt.gz" + + +@lru_cache() +def bytes_to_unicode(): + """ + Returns list of utf-8 byte and a corresponding list of unicode strings. + The reversible bpe codes work on unicode strings. + This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. + When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. + This is a signficant percentage of your normal, say, 32K bpe vocab. + To avoid that, we want lookup tables between utf-8 bytes and unicode strings. + And avoids mapping to whitespace/control characters the bpe code barfs on. + """ + bs = list(range(ord("!"), ord("~")+1))+list(range(ord("¡"), ord("¬")+1))+list(range(ord("®"), ord("ÿ")+1)) + cs = bs[:] + n = 0 + for b in range(2**8): + if b not in bs: + bs.append(b) + cs.append(2**8+n) + n += 1 + cs = [chr(n) for n in cs] + return dict(zip(bs, cs)) + + +def get_pairs(word): + """Return set of symbol pairs in a word. + Word is represented as tuple of symbols (symbols being variable-length strings). + """ + pairs = set() + prev_char = word[0] + for char in word[1:]: + pairs.add((prev_char, char)) + prev_char = char + return pairs + + +def basic_clean(text): + text = ftfy.fix_text(text) + text = html.unescape(html.unescape(text)) + return text.strip() + + +def whitespace_clean(text): + text = re.sub(r'\s+', ' ', text) + text = text.strip() + return text + + +class SimpleTokenizer(object): + def __init__(self, bpe_path: str = default_bpe()): + self.byte_encoder = bytes_to_unicode() + self.byte_decoder = {v: k for k, v in self.byte_encoder.items()} + merges = gzip.open(bpe_path).read().decode("utf-8").split('\n') + merges = merges[1:49152-256-2+1] + merges = [tuple(merge.split()) for merge in merges] + vocab = list(bytes_to_unicode().values()) + vocab = vocab + [v+'' for v in vocab] + for merge in merges: + vocab.append(''.join(merge)) + vocab.extend(['<|startoftext|>', '<|endoftext|>']) + self.encoder = dict(zip(vocab, range(len(vocab)))) + self.decoder = {v: k for k, v in self.encoder.items()} + self.bpe_ranks = dict(zip(merges, range(len(merges)))) + self.cache = {'<|startoftext|>': '<|startoftext|>', '<|endoftext|>': '<|endoftext|>'} + self.pat = re.compile(r"""<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+""", re.IGNORECASE) + + def bpe(self, token): + if token in self.cache: + return self.cache[token] + word = tuple(token[:-1]) + ( token[-1] + '',) + pairs = get_pairs(word) + + if not pairs: + return token+'' + + while True: + bigram = min(pairs, key = lambda pair: self.bpe_ranks.get(pair, float('inf'))) + if bigram not in self.bpe_ranks: + break + first, second = bigram + new_word = [] + i = 0 + while i < len(word): + try: + j = word.index(first, i) + new_word.extend(word[i:j]) + i = j + except: + new_word.extend(word[i:]) + break + + if word[i] == first and i < len(word)-1 and word[i+1] == second: + new_word.append(first+second) + i += 2 + else: + new_word.append(word[i]) + i += 1 + new_word = tuple(new_word) + word = new_word + if len(word) == 1: + break + else: + pairs = get_pairs(word) + word = ' '.join(word) + self.cache[token] = word + return word + + def encode(self, text): + bpe_tokens = [] + text = whitespace_clean(basic_clean(text)).lower() + for token in re.findall(self.pat, text): + token = ''.join(self.byte_encoder[b] for b in token.encode('utf-8')) + bpe_tokens.extend(self.encoder[bpe_token] for bpe_token in self.bpe(token).split(' ')) + return bpe_tokens + + def decode(self, tokens): + text = ''.join([self.decoder[token] for token in tokens]) + text = bytearray([self.byte_decoder[c] for c in text]).decode('utf-8', errors="replace").replace('', ' ') + return text \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/viclip/viclip.py b/VideoX-Fun/videox_fun/video_caption/utils/viclip/viclip.py new file mode 100644 index 0000000000000000000000000000000000000000..3721e8e829645289e0a4a7c01bfab68225bed257 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/viclip/viclip.py @@ -0,0 +1,262 @@ +import logging +import math +import os + +import torch +from einops import rearrange +from torch import nn + +# from .criterions import VTC_VTM_Loss +from .simple_tokenizer import SimpleTokenizer as _Tokenizer +from .viclip_text import clip_text_b16, clip_text_l14 +from .viclip_vision import clip_joint_b16, clip_joint_l14 + +logger = logging.getLogger(__name__) + + +class ViCLIP(nn.Module): + """docstring for ViCLIP""" + + def __init__(self, + tokenizer=None, + size='l', + pretrain=os.path.join(os.path.dirname(os.path.abspath(__file__)), "ViClip-InternVid-10M-FLT.pth"), + freeze_text=True): + super(ViCLIP, self).__init__() + if tokenizer: + self.tokenizer = tokenizer + else: + self.tokenizer = _Tokenizer() + self.max_txt_l = 32 + + if size.lower() == 'l': + self.vision_encoder_name = 'vit_l14' + elif size.lower() == 'b': + self.vision_encoder_name = 'vit_b16' + else: + raise NotImplementedError(f"Size {size} not implemented") + + self.vision_encoder_pretrained = False + self.inputs_image_res = 224 + self.vision_encoder_kernel_size = 1 + self.vision_encoder_center = True + self.video_input_num_frames = 8 + self.vision_encoder_drop_path_rate = 0.1 + self.vision_encoder_checkpoint_num = 24 + self.is_pretrain = pretrain + self.vision_width = 1024 + self.text_width = 768 + self.embed_dim = 768 + self.masking_prob = 0.9 + + if size.lower() == 'l': + self.text_encoder_name = 'vit_l14' + elif size.lower() == 'b': + self.text_encoder_name = 'vit_b16' + else: + raise NotImplementedError(f"Size {size} not implemented") + + self.text_encoder_pretrained = False#'bert-base-uncased' + self.text_encoder_d_model = 768 + + self.text_encoder_vocab_size = 49408 + + # create modules. + self.vision_encoder = self.build_vision_encoder() + self.text_encoder = self.build_text_encoder() + + self.temp = nn.parameter.Parameter(torch.ones([]) * 1 / 100.0) + self.temp_min = 1 / 100.0 + + if pretrain: + logger.info(f"Load pretrained weights from {pretrain}") + state_dict = torch.load(pretrain, map_location='cpu')['model'] + self.load_state_dict(state_dict) + + # Freeze weights + if freeze_text: + self.freeze_text() + + + def freeze_text(self): + """freeze text encoder""" + for p in self.text_encoder.parameters(): + p.requires_grad = False + + def no_weight_decay(self): + ret = {"temp"} + ret.update( + {"vision_encoder." + k for k in self.vision_encoder.no_weight_decay()} + ) + ret.update( + {"text_encoder." + k for k in self.text_encoder.no_weight_decay()} + ) + + return ret + + def forward(self, image, text, raw_text, idx, log_generation=None, return_sims=False): + """forward and calculate loss. + + Args: + image (torch.Tensor): The input images. Shape: [B,T,C,H,W]. + text (dict): TODO + idx (torch.Tensor): TODO + + Returns: TODO + + """ + self.clip_contrastive_temperature() + + vision_embeds = self.encode_vision(image) + text_embeds = self.encode_text(raw_text) + if return_sims: + sims = torch.nn.functional.normalize(vision_embeds, dim=-1) @ \ + torch.nn.functional.normalize(text_embeds, dim=-1).transpose(0, 1) + return sims + + # calculate loss + + ## VTC loss + loss_vtc = self.clip_loss.vtc_loss( + vision_embeds, text_embeds, idx, self.temp, all_gather=True + ) + + return dict( + loss_vtc=loss_vtc, + ) + + def encode_vision(self, image, test=False): + """encode image / videos as features. + + Args: + image (torch.Tensor): The input images. + test (bool): Whether testing. + + Returns: tuple. + - vision_embeds (torch.Tensor): The features of all patches. Shape: [B,T,L,C]. + - pooled_vision_embeds (torch.Tensor): The pooled features. Shape: [B,T,C]. + + """ + if image.ndim == 5: + image = image.permute(0, 2, 1, 3, 4).contiguous() + else: + image = image.unsqueeze(2) + + if not test and self.masking_prob > 0.0: + return self.vision_encoder( + image, masking_prob=self.masking_prob + ) + + return self.vision_encoder(image) + + def encode_text(self, text): + """encode text. + Args: + text (dict): The output of huggingface's `PreTrainedTokenizer`. contains keys: + - input_ids (torch.Tensor): Token ids to be fed to a model. Shape: [B,L]. + - attention_mask (torch.Tensor): The mask indicate padded tokens. Shape: [B,L]. 0 is padded token. + - other keys refer to "https://huggingface.co/docs/transformers/v4.21.2/en/main_classes/tokenizer#transformers.PreTrainedTokenizer.__call__". + Returns: tuple. + - text_embeds (torch.Tensor): The features of all tokens. Shape: [B,L,C]. + - pooled_text_embeds (torch.Tensor): The pooled features. Shape: [B,C]. + + """ + device = next(self.text_encoder.parameters()).device + text = self.text_encoder.tokenize( + text, context_length=self.max_txt_l + ).to(device) + text_embeds = self.text_encoder(text) + return text_embeds + + @torch.no_grad() + def clip_contrastive_temperature(self, min_val=0.001, max_val=0.5): + """Seems only used during pre-training""" + self.temp.clamp_(min=self.temp_min) + + def build_vision_encoder(self): + """build vision encoder + Returns: (vision_encoder, vision_layernorm). Each is a `nn.Module`. + + """ + encoder_name = self.vision_encoder_name + if encoder_name == "vit_l14": + vision_encoder = clip_joint_l14( + pretrained=self.vision_encoder_pretrained, + input_resolution=self.inputs_image_res, + kernel_size=self.vision_encoder_kernel_size, + center=self.vision_encoder_center, + num_frames=self.video_input_num_frames, + drop_path=self.vision_encoder_drop_path_rate, + checkpoint_num=self.vision_encoder_checkpoint_num, + ) + elif encoder_name == "vit_b16": + vision_encoder = clip_joint_b16( + pretrained=self.vision_encoder_pretrained, + input_resolution=self.inputs_image_res, + kernel_size=self.vision_encoder_kernel_size, + center=self.vision_encoder_center, + num_frames=self.video_input_num_frames, + drop_path=self.vision_encoder_drop_path_rate, + checkpoint_num=self.vision_encoder_checkpoint_num, + ) + else: + raise NotImplementedError(f"Not implemented: {encoder_name}") + + return vision_encoder + + def build_text_encoder(self): + """build text_encoder and possiblly video-to-text multimodal fusion encoder. + Returns: nn.Module. The text encoder + + """ + encoder_name = self.text_encoder_name + + if encoder_name == "vit_l14": + text_encoder = clip_text_l14( + pretrained=self.text_encoder_pretrained, + context_length=self.max_txt_l, + vocab_size=self.text_encoder_vocab_size, + checkpoint_num=0, + ) + elif encoder_name == "vit_b16": + text_encoder = clip_text_b16( + pretrained=self.text_encoder_pretrained, + context_length=self.max_txt_l, + vocab_size=self.text_encoder_vocab_size, + checkpoint_num=0, + ) + else: + raise NotImplementedError(f"Not implemented: {encoder_name}") + + return text_encoder + + def get_text_encoder(self): + """get text encoder, used for text and cross-modal encoding""" + encoder = self.text_encoder + return encoder.bert if hasattr(encoder, "bert") else encoder + + def get_text_features(self, input_text, tokenizer, text_feature_dict={}): + if input_text in text_feature_dict: + return text_feature_dict[input_text] + text_template= f"{input_text}" + with torch.no_grad(): + # text_token = tokenizer.encode(text_template).cuda() + text_features = self.encode_text(text_template).float() + text_features /= text_features.norm(dim=-1, keepdim=True) + text_feature_dict[input_text] = text_features + return text_features + + def get_vid_features(self, input_frames): + with torch.no_grad(): + clip_feat = self.encode_vision(input_frames,test=True).float() + clip_feat /= clip_feat.norm(dim=-1, keepdim=True) + return clip_feat + + def get_predict_label(self, clip_feature, text_feats_tensor, top=5): + label_probs = (100.0 * clip_feature @ text_feats_tensor.T).softmax(dim=-1) + top_probs, top_labels = label_probs.cpu().topk(top, dim=-1) + return top_probs, top_labels + + +if __name__ =="__main__": + tokenizer = _Tokenizer() \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/viclip/viclip_text.py b/VideoX-Fun/videox_fun/video_caption/utils/viclip/viclip_text.py new file mode 100644 index 0000000000000000000000000000000000000000..206fca1a73c7f01ade848a2b09098484a6b35646 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/viclip/viclip_text.py @@ -0,0 +1,298 @@ +import functools +import logging +import os +from collections import OrderedDict + +import numpy as np +import torch +import torch.nn.functional as F +import torch.utils.checkpoint as checkpoint +from pkg_resources import packaging +from torch import nn + +from .simple_tokenizer import SimpleTokenizer as _Tokenizer + +logger = logging.getLogger(__name__) + + +# On P1, model extracted from https://huggingface.co/laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K +MODEL_PATH = 'https://huggingface.co/laion' +_MODELS = { + "ViT-L/14": os.path.join(MODEL_PATH, "CLIP-ViT-L-14-DataComp.XL-s13B-b90K", "vit_l14_text.pth"), + "ViT-B/16": os.path.join(MODEL_PATH, "CLIP-ViT-B-16-DataComp.XL-s13B-b90K", "vit_b16_text.pth"), +} + + +class LayerNorm(nn.LayerNorm): + """Subclass torch's LayerNorm to handle fp16.""" + + def forward(self, x: torch.Tensor): + orig_type = x.dtype + ret = super().forward(x.type(torch.float32)) + return ret.type(orig_type) + + +class QuickGELU(nn.Module): + def forward(self, x: torch.Tensor): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + def __init__(self, d_model: int, n_head: int, attn_mask: torch.Tensor = None): + super().__init__() + + self.attn = nn.MultiheadAttention(d_model, n_head) + self.ln_1 = LayerNorm(d_model) + self.mlp = nn.Sequential(OrderedDict([ + ("c_fc", nn.Linear(d_model, d_model * 4)), + ("gelu", QuickGELU()), + ("c_proj", nn.Linear(d_model * 4, d_model)) + ])) + self.ln_2 = LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x: torch.Tensor): + self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None + return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x: torch.Tensor): + x = x + self.attention(self.ln_1(x)) + x = x + self.mlp(self.ln_2(x)) + return x + + +class Transformer(nn.Module): + def __init__(self, width: int, layers: int, heads: int, attn_mask: torch.Tensor = None, + checkpoint_num: int = 0): + super().__init__() + self.width = width + self.layers = layers + self.resblocks = nn.Sequential(*[ResidualAttentionBlock(width, heads, attn_mask) for _ in range(layers)]) + + self.checkpoint_num = checkpoint_num + + def forward(self, x: torch.Tensor): + if self.checkpoint_num > 0: + segments = min(self.checkpoint_num, len(self.resblocks)) + return checkpoint.checkpoint_sequential(self.resblocks, segments, x) + else: + return self.resblocks(x) + + +class CLIP_TEXT(nn.Module): + def __init__( + self, + embed_dim: int, + context_length: int, + vocab_size: int, + transformer_width: int, + transformer_heads: int, + transformer_layers: int, + checkpoint_num: int, + ): + super().__init__() + + self.context_length = context_length + self._tokenizer = _Tokenizer() + + self.transformer = Transformer( + width=transformer_width, + layers=transformer_layers, + heads=transformer_heads, + attn_mask=self.build_attention_mask(), + checkpoint_num=checkpoint_num, + ) + + self.vocab_size = vocab_size + self.token_embedding = nn.Embedding(vocab_size, transformer_width) + self.positional_embedding = nn.Parameter(torch.empty(self.context_length, transformer_width)) + self.ln_final = LayerNorm(transformer_width) + + self.text_projection = nn.Parameter(torch.empty(transformer_width, embed_dim)) + + def no_weight_decay(self): + return {'token_embedding', 'positional_embedding'} + + @functools.lru_cache(maxsize=None) + def build_attention_mask(self): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(self.context_length, self.context_length) + mask.fill_(float("-inf")) + mask.triu_(1) # zero out the lower diagonal + return mask + + def tokenize(self, texts, context_length=77, truncate=True): + """ + Returns the tokenized representation of given input string(s) + Parameters + ---------- + texts : Union[str, List[str]] + An input string or a list of input strings to tokenize + context_length : int + The context length to use; all CLIP models use 77 as the context length + truncate: bool + Whether to truncate the text in case its encoding is longer than the context length + Returns + ------- + A two-dimensional tensor containing the resulting tokens, shape = [number of input strings, context_length]. + We return LongTensor when torch version is <1.8.0, since older index_select requires indices to be long. + """ + if isinstance(texts, str): + texts = [texts] + + sot_token = self._tokenizer.encoder["<|startoftext|>"] + eot_token = self._tokenizer.encoder["<|endoftext|>"] + all_tokens = [[sot_token] + self._tokenizer.encode(text) + [eot_token] for text in texts] + if packaging.version.parse(torch.__version__) < packaging.version.parse("1.8.0"): + result = torch.zeros(len(all_tokens), context_length, dtype=torch.long) + else: + result = torch.zeros(len(all_tokens), context_length, dtype=torch.int) + + for i, tokens in enumerate(all_tokens): + if len(tokens) > context_length: + if truncate: + tokens = tokens[:context_length] + tokens[-1] = eot_token + else: + raise RuntimeError(f"Input {texts[i]} is too long for context length {context_length}") + result[i, :len(tokens)] = torch.tensor(tokens) + + return result + + def forward(self, text): + x = self.token_embedding(text) # [batch_size, n_ctx, d_model] + + x = x + self.positional_embedding + x = x.permute(1, 0, 2) # NLD -> LND + x = self.transformer(x) + x = x.permute(1, 0, 2) # LND -> NLD + x = self.ln_final(x) + + # x.shape = [batch_size, n_ctx, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + x = x[torch.arange(x.shape[0]), text.argmax(dim=-1)] @ self.text_projection + + return x + + +def clip_text_b16( + embed_dim=512, + context_length=77, + vocab_size=49408, + transformer_width=512, + transformer_heads=8, + transformer_layers=12, + checkpoint_num=0, + pretrained=True, +): + # raise NotImplementedError + model = CLIP_TEXT( + embed_dim, + context_length, + vocab_size, + transformer_width, + transformer_heads, + transformer_layers, + checkpoint_num, + ) + # pretrained = _MODELS["ViT-B/16"] + # logger.info(f"Load pretrained weights from {pretrained}") + # state_dict = torch.load(pretrained, map_location='cpu') + # model.load_state_dict(state_dict, strict=False) + # return model.eval() + if pretrained: + if isinstance(pretrained, str) and pretrained != "bert-base-uncased": + pretrained = _MODELS[pretrained] + else: + pretrained = _MODELS["ViT-B/16"] + logger.info(f"Load pretrained weights from {pretrained}") + state_dict = torch.load(pretrained, map_location='cpu') + if context_length != state_dict["positional_embedding"].size(0): + # assert context_length < state_dict["positional_embedding"].size(0), "Cannot increase context length." + print(f"Resize positional embedding from {state_dict['positional_embedding'].size(0)} to {context_length}") + if context_length < state_dict["positional_embedding"].size(0): + state_dict["positional_embedding"] = state_dict["positional_embedding"][:context_length] + else: + state_dict["positional_embedding"] = F.pad( + state_dict["positional_embedding"], + (0, 0, 0, context_length - state_dict["positional_embedding"].size(0)), + value=0, + ) + + message = model.load_state_dict(state_dict, strict=False) + print(f"Load pretrained weights from {pretrained}: {message}") + return model.eval() + + +def clip_text_l14( + embed_dim=768, + context_length=77, + vocab_size=49408, + transformer_width=768, + transformer_heads=12, + transformer_layers=12, + checkpoint_num=0, + pretrained=True, +): + model = CLIP_TEXT( + embed_dim, + context_length, + vocab_size, + transformer_width, + transformer_heads, + transformer_layers, + checkpoint_num, + ) + if pretrained: + if isinstance(pretrained, str) and pretrained != "bert-base-uncased": + pretrained = _MODELS[pretrained] + else: + pretrained = _MODELS["ViT-L/14"] + logger.info(f"Load pretrained weights from {pretrained}") + state_dict = torch.load(pretrained, map_location='cpu') + if context_length != state_dict["positional_embedding"].size(0): + # assert context_length < state_dict["positional_embedding"].size(0), "Cannot increase context length." + print(f"Resize positional embedding from {state_dict['positional_embedding'].size(0)} to {context_length}") + if context_length < state_dict["positional_embedding"].size(0): + state_dict["positional_embedding"] = state_dict["positional_embedding"][:context_length] + else: + state_dict["positional_embedding"] = F.pad( + state_dict["positional_embedding"], + (0, 0, 0, context_length - state_dict["positional_embedding"].size(0)), + value=0, + ) + + message = model.load_state_dict(state_dict, strict=False) + print(f"Load pretrained weights from {pretrained}: {message}") + return model.eval() + + +def clip_text_l14_336( + embed_dim=768, + context_length=77, + vocab_size=49408, + transformer_width=768, + transformer_heads=12, + transformer_layers=12, +): + raise NotImplementedError + model = CLIP_TEXT( + embed_dim, + context_length, + vocab_size, + transformer_width, + transformer_heads, + transformer_layers + ) + pretrained = _MODELS["ViT-L/14_336"] + logger.info(f"Load pretrained weights from {pretrained}") + state_dict = torch.load(pretrained, map_location='cpu') + model.load_state_dict(state_dict, strict=False) + return model.eval() + + +def build_clip(config): + model_cls = config.text_encoder.clip_teacher + model = eval(model_cls)() + return model \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/viclip/viclip_vision.py b/VideoX-Fun/videox_fun/video_caption/utils/viclip/viclip_vision.py new file mode 100644 index 0000000000000000000000000000000000000000..1b91be599d139e13ad686bc02a0ee508cf205a65 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/viclip/viclip_vision.py @@ -0,0 +1,361 @@ +#!/usr/bin/env python +import logging +import os +from collections import OrderedDict + +import torch +import torch.utils.checkpoint as checkpoint +from einops import rearrange +from timm.models.layers import DropPath +from timm.models.registry import register_model +from torch import nn + +# from models.utils import load_temp_embed_with_mismatch + +logger = logging.getLogger(__name__) + +def load_temp_embed_with_mismatch(temp_embed_old, temp_embed_new, add_zero=True): + """ + Add/Remove extra temporal_embeddings as needed. + https://arxiv.org/abs/2104.00650 shows adding zero paddings works. + + temp_embed_old: (1, num_frames_old, 1, d) + temp_embed_new: (1, num_frames_new, 1, d) + add_zero: bool, if True, add zero, else, interpolate trained embeddings. + """ + # TODO zero pad + num_frms_new = temp_embed_new.shape[1] + num_frms_old = temp_embed_old.shape[1] + logger.info(f"Load temporal_embeddings, lengths: {num_frms_old}-->{num_frms_new}") + if num_frms_new > num_frms_old: + if add_zero: + temp_embed_new[ + :, :num_frms_old + ] = temp_embed_old # untrained embeddings are zeros. + else: + temp_embed_new = interpolate_temporal_pos_embed(temp_embed_old, num_frms_new) + elif num_frms_new < num_frms_old: + temp_embed_new = temp_embed_old[:, :num_frms_new] + else: # = + temp_embed_new = temp_embed_old + return temp_embed_new + + +# On P1, model extracted from https://huggingface.co/laion/CLIP-ViT-L-14-DataComp.XL-s13B-b90K +MODEL_PATH = '' +_MODELS = { + "ViT-L/14": os.path.join(MODEL_PATH, "ViCLIP-L_InternVid-FLT-10M.pth"), + "ViT-B/16": os.path.join(MODEL_PATH, "ViCLIP-B-InternVid-FLT-10M.pth"), +} + + +class QuickGELU(nn.Module): + def forward(self, x): + return x * torch.sigmoid(1.702 * x) + + +class ResidualAttentionBlock(nn.Module): + def __init__(self, d_model, n_head, drop_path=0., attn_mask=None, dropout=0.): + super().__init__() + + self.drop_path1 = DropPath(drop_path) if drop_path > 0. else nn.Identity() + self.drop_path2 = DropPath(drop_path) if drop_path > 0. else nn.Identity() + # logger.info(f'Droppath: {drop_path}') + self.attn = nn.MultiheadAttention(d_model, n_head, dropout=dropout) + self.ln_1 = nn.LayerNorm(d_model) + self.mlp = nn.Sequential(OrderedDict([ + ("c_fc", nn.Linear(d_model, d_model * 4)), + ("gelu", QuickGELU()), + ("drop1", nn.Dropout(dropout)), + ("c_proj", nn.Linear(d_model * 4, d_model)), + ("drop2", nn.Dropout(dropout)), + ])) + self.ln_2 = nn.LayerNorm(d_model) + self.attn_mask = attn_mask + + def attention(self, x): + self.attn_mask = self.attn_mask.to(dtype=x.dtype, device=x.device) if self.attn_mask is not None else None + return self.attn(x, x, x, need_weights=False, attn_mask=self.attn_mask)[0] + + def forward(self, x): + x = x + self.drop_path1(self.attention(self.ln_1(x))) + x = x + self.drop_path2(self.mlp(self.ln_2(x))) + return x + + +class Transformer(nn.Module): + def __init__(self, width, layers, heads, drop_path=0., checkpoint_num=0, dropout=0.): + super().__init__() + dpr = [x.item() for x in torch.linspace(0, drop_path, layers)] + self.resblocks = nn.ModuleList() + for idx in range(layers): + self.resblocks.append(ResidualAttentionBlock(width, heads, drop_path=dpr[idx], dropout=dropout)) + self.checkpoint_num = checkpoint_num + + def forward(self, x): + for idx, blk in enumerate(self.resblocks): + if idx < self.checkpoint_num: + x = checkpoint.checkpoint(blk, x) + else: + x = blk(x) + return x + + +class VisionTransformer(nn.Module): + def __init__( + self, input_resolution, patch_size, width, layers, heads, output_dim=None, + kernel_size=1, num_frames=8, drop_path=0, checkpoint_num=0, dropout=0., + temp_embed=True, + ): + super().__init__() + self.output_dim = output_dim + self.conv1 = nn.Conv3d( + 3, width, + (kernel_size, patch_size, patch_size), + (kernel_size, patch_size, patch_size), + (0, 0, 0), bias=False + ) + + scale = width ** -0.5 + self.class_embedding = nn.Parameter(scale * torch.randn(width)) + self.positional_embedding = nn.Parameter(scale * torch.randn((input_resolution // patch_size) ** 2 + 1, width)) + self.ln_pre = nn.LayerNorm(width) + if temp_embed: + self.temporal_positional_embedding = nn.Parameter(torch.zeros(1, num_frames, width)) + + self.transformer = Transformer( + width, layers, heads, drop_path=drop_path, checkpoint_num=checkpoint_num, + dropout=dropout) + + self.ln_post = nn.LayerNorm(width) + if output_dim is not None: + self.proj = nn.Parameter(torch.empty(width, output_dim)) + else: + self.proj = None + + self.dropout = nn.Dropout(dropout) + + def get_num_layers(self): + return len(self.transformer.resblocks) + + @torch.jit.ignore + def no_weight_decay(self): + return {'positional_embedding', 'class_embedding', 'temporal_positional_embedding'} + + def mask_tokens(self, inputs, masking_prob=0.0): + B, L, _ = inputs.shape + + # This is different from text as we are masking a fix number of tokens + Lm = int(masking_prob * L) + masked_indices = torch.zeros(B, L) + indices = torch.argsort(torch.rand_like(masked_indices), dim=-1)[:, :Lm] + batch_indices = ( + torch.arange(masked_indices.shape[0]).unsqueeze(-1).expand_as(indices) + ) + masked_indices[batch_indices, indices] = 1 + + masked_indices = masked_indices.bool() + + return inputs[~masked_indices].reshape(B, -1, inputs.shape[-1]) + + def forward(self, x, masking_prob=0.0): + x = self.conv1(x) # shape = [*, width, grid, grid] + B, C, T, H, W = x.shape + x = x.permute(0, 2, 3, 4, 1).reshape(B * T, H * W, C) + + x = torch.cat([self.class_embedding.to(x.dtype) + torch.zeros(x.shape[0], 1, x.shape[-1], dtype=x.dtype, device=x.device), x], dim=1) # shape = [*, grid ** 2 + 1, width] + x = x + self.positional_embedding.to(x.dtype) + + # temporal pos + cls_tokens = x[:B, :1, :] + x = x[:, 1:] + x = rearrange(x, '(b t) n m -> (b n) t m', b=B, t=T) + if hasattr(self, 'temporal_positional_embedding'): + if x.size(1) == 1: + # This is a workaround for unused parameter issue + x = x + self.temporal_positional_embedding.mean(1) + else: + x = x + self.temporal_positional_embedding + x = rearrange(x, '(b n) t m -> b (n t) m', b=B, t=T) + + if masking_prob > 0.0: + x = self.mask_tokens(x, masking_prob) + + x = torch.cat((cls_tokens, x), dim=1) + + x = self.ln_pre(x) + + x = x.permute(1, 0, 2) #BND -> NBD + x = self.transformer(x) + + x = self.ln_post(x) + + if self.proj is not None: + x = self.dropout(x[0]) @ self.proj + else: + x = x.permute(1, 0, 2) #NBD -> BND + + return x + + +def inflate_weight(weight_2d, time_dim, center=True): + logger.info(f'Init center: {center}') + if center: + weight_3d = torch.zeros(*weight_2d.shape) + weight_3d = weight_3d.unsqueeze(2).repeat(1, 1, time_dim, 1, 1) + middle_idx = time_dim // 2 + weight_3d[:, :, middle_idx, :, :] = weight_2d + else: + weight_3d = weight_2d.unsqueeze(2).repeat(1, 1, time_dim, 1, 1) + weight_3d = weight_3d / time_dim + return weight_3d + + +def load_state_dict(model, state_dict, input_resolution=224, patch_size=16, center=True): + state_dict_3d = model.state_dict() + for k in state_dict.keys(): + if k in state_dict_3d.keys() and state_dict[k].shape != state_dict_3d[k].shape: + if len(state_dict_3d[k].shape) <= 2: + logger.info(f'Ignore: {k}') + continue + logger.info(f'Inflate: {k}, {state_dict[k].shape} => {state_dict_3d[k].shape}') + time_dim = state_dict_3d[k].shape[2] + state_dict[k] = inflate_weight(state_dict[k], time_dim, center=center) + + pos_embed_checkpoint = state_dict['positional_embedding'] + embedding_size = pos_embed_checkpoint.shape[-1] + num_patches = (input_resolution // patch_size) ** 2 + orig_size = int((pos_embed_checkpoint.shape[-2] - 1) ** 0.5) + new_size = int(num_patches ** 0.5) + if orig_size != new_size: + logger.info(f'Pos_emb from {orig_size} to {new_size}') + extra_tokens = pos_embed_checkpoint[:1] + pos_tokens = pos_embed_checkpoint[1:] + pos_tokens = pos_tokens.reshape(-1, orig_size, orig_size, embedding_size).permute(0, 3, 1, 2) + pos_tokens = torch.nn.functional.interpolate( + pos_tokens, size=(new_size, new_size), mode='bicubic', align_corners=False) + pos_tokens = pos_tokens.permute(0, 2, 3, 1).flatten(0, 2) + new_pos_embed = torch.cat((extra_tokens, pos_tokens), dim=0) + state_dict['positional_embedding'] = new_pos_embed + + message = model.load_state_dict(state_dict, strict=False) + logger.info(f"Load pretrained weights: {message}") + + +@register_model +def clip_joint_b16( + pretrained=False, input_resolution=224, kernel_size=1, + center=True, num_frames=8, drop_path=0., checkpoint_num=0, + dropout=0., +): + model = VisionTransformer( + input_resolution=input_resolution, patch_size=16, + width=768, layers=12, heads=12, output_dim=512, + kernel_size=kernel_size, num_frames=num_frames, + drop_path=drop_path, checkpoint_num=checkpoint_num, + dropout=dropout, + ) + # raise NotImplementedError + if pretrained: + if isinstance(pretrained, str): + model_name = pretrained + else: + model_name = "ViT-B/16" + + logger.info('load pretrained weights') + state_dict = torch.load(_MODELS[model_name], map_location='cpu') + load_state_dict(model, state_dict, input_resolution=input_resolution, patch_size=16, center=center) + return model.eval() + + +@register_model +def clip_joint_l14( + pretrained=False, input_resolution=224, kernel_size=1, + center=True, num_frames=8, drop_path=0., checkpoint_num=0, + dropout=0., +): + model = VisionTransformer( + input_resolution=input_resolution, patch_size=14, + width=1024, layers=24, heads=16, output_dim=768, + kernel_size=kernel_size, num_frames=num_frames, + drop_path=drop_path, checkpoint_num=checkpoint_num, + dropout=dropout, + ) + + if pretrained: + if isinstance(pretrained, str): + model_name = pretrained + else: + model_name = "ViT-L/14" + logger.info('load pretrained weights') + state_dict = torch.load(_MODELS[model_name], map_location='cpu') + load_state_dict(model, state_dict, input_resolution=input_resolution, patch_size=14, center=center) + return model.eval() + + +@register_model +def clip_joint_l14_336( + pretrained=True, input_resolution=336, kernel_size=1, + center=True, num_frames=8, drop_path=0. +): + raise NotImplementedError + model = VisionTransformer( + input_resolution=input_resolution, patch_size=14, + width=1024, layers=24, heads=16, output_dim=768, + kernel_size=kernel_size, num_frames=num_frames, + drop_path=drop_path, + ) + if pretrained: + logger.info('load pretrained weights') + state_dict = torch.load(_MODELS["ViT-L/14_336"], map_location='cpu') + load_state_dict(model, state_dict, input_resolution=input_resolution, patch_size=14, center=center) + return model.eval() + + +def interpolate_pos_embed_vit(state_dict, new_model): + key = "vision_encoder.temporal_positional_embedding" + if key in state_dict: + vision_temp_embed_new = new_model.state_dict()[key] + vision_temp_embed_new = vision_temp_embed_new.unsqueeze(2) # [1, n, d] -> [1, n, 1, d] + vision_temp_embed_old = state_dict[key] + vision_temp_embed_old = vision_temp_embed_old.unsqueeze(2) + + state_dict[key] = load_temp_embed_with_mismatch( + vision_temp_embed_old, vision_temp_embed_new, add_zero=False + ).squeeze(2) + + key = "text_encoder.positional_embedding" + if key in state_dict: + text_temp_embed_new = new_model.state_dict()[key] + text_temp_embed_new = text_temp_embed_new.unsqueeze(0).unsqueeze(2) # [n, d] -> [1, n, 1, d] + text_temp_embed_old = state_dict[key] + text_temp_embed_old = text_temp_embed_old.unsqueeze(0).unsqueeze(2) + + state_dict[key] = load_temp_embed_with_mismatch( + text_temp_embed_old, text_temp_embed_new, add_zero=False + ).squeeze(2).squeeze(0) + return state_dict + + +if __name__ == '__main__': + import time + + import numpy as np + from fvcore.nn import FlopCountAnalysis, flop_count_table + + seed = 4217 + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + num_frames = 8 + + # model = clip_joint_b16(pretrained=True, kernel_size=1, num_frames=8, num_classes=400, drop_path=0.1) + # logger.info(model) + model = clip_joint_l14(pretrained=False) + + flops = FlopCountAnalysis(model, torch.rand(1, 3, num_frames, 224, 224)) + s = time.time() + logger.info(flop_count_table(flops, max_depth=1)) + logger.info(time.time()-s) + # logger.info(model(torch.rand(1, 3, num_frames, 224, 224)).shape) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/video_dataset.py b/VideoX-Fun/videox_fun/video_caption/utils/video_dataset.py new file mode 100644 index 0000000000000000000000000000000000000000..e8c79842637f1aeadac8f698de3bda86f4597615 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/video_dataset.py @@ -0,0 +1,101 @@ +import os +from pathlib import Path +from typing import Optional + +from func_timeout import FunctionTimedOut, func_timeout +from PIL import Image +from torch.utils.data import DataLoader, Dataset + +from .logger import logger +from .video_utils import extract_frames + + +ALL_VIDEO_EXT = set([".mp4", ".webm", ".mkv", ".avi", ".flv", ".mov", ".ts"]) +VIDEO_READER_TIMEOUT = 300 + + +def collate_fn(batch): + batch = list(filter(lambda x: x is not None, batch)) + if len(batch) != 0: + return {k: [item[k] for item in batch] for k in batch[0].keys()} + return {} + + +class VideoDataset(Dataset): + def __init__( + self, + dataset_inputs: dict[str, list[str]], + video_folder: Optional[str] = None, + video_path_column: str = "video_path", + text_column: Optional[str] = None, + sample_method: str = "mid", + num_sampled_frames: int = 1, + sample_stride: Optional[int] = None + ): + length = len(dataset_inputs[list(dataset_inputs.keys())[0]]) + if not all(len(v) == length for v in dataset_inputs.values()): + raise ValueError("All values in the dataset_inputs must have the same length.") + + self.video_path_column = video_path_column + self.video_folder = video_folder + self.video_path_list = dataset_inputs[video_path_column] + if self.video_folder is not None: + self.video_path_list = [os.path.join(self.video_folder, video_path) for video_path in self.video_path_list] + self.text_column = text_column + self.text_list = dataset_inputs[self.text_column] if self.text_column is not None else None + + self.sample_method = sample_method + self.num_sampled_frames = num_sampled_frames + self.sample_stride = sample_stride + + def __getitem__(self, index): + video_path = self.video_path_list[index] + if self.sample_method == "image": + try: + sampled_frame_idx_list = None + with open(video_path, "rb") as f: + sampled_frame_list = [Image.open(f).convert("RGB")] + except Exception as e: + logger.warning(f"Failed to extract frames from video {video_path}. Error is {e}.") + return None + else: + # It is a trick to deal with decord hanging when reading some abnormal videos. + try: + sample_args = (video_path, self.sample_method, self.num_sampled_frames, self.sample_stride) + sampled_frame_idx_list, sampled_frame_list = func_timeout( + VIDEO_READER_TIMEOUT, extract_frames, args=sample_args + ) + except FunctionTimedOut: + logger.warning(f"Read {video_path} timeout.") + return None + except Exception as e: + logger.warning(f"Failed to extract frames from video {video_path}. Error is {e}.") + return None + + item = { + "path": video_path, + "sampled_frame_idx": sampled_frame_idx_list, + "sampled_frame": sampled_frame_list, + } + if self.text_list is not None: + item["text"] = self.text_list[index] + + return item + + def __len__(self): + return len(self.video_path_list) + + +if __name__ == "__main__": + video_folder = Path("your_video_folder") + video_path_list = [] + for ext in ALL_VIDEO_EXT: + video_path_list += [str(file.relative_to(video_folder)) for file in video_folder.glob(f"*.{ext}")] + + video_dataset = VideoDataset(dataset_inputs={"video_path": video_path_list}) + video_dataloader = DataLoader( + video_dataset, batch_size=16, num_workers=16, collate_fn=collate_fn + ) + for idx, batch in enumerate(video_dataloader): + if len(batch) != 0: + print(batch["video_path"], batch["sampled_frame_idx"], len(batch["video_path"])) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/video_evaluator.py b/VideoX-Fun/videox_fun/video_caption/utils/video_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..1b644907c639cab7c3829db84fea2569ffa91fbf --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/video_evaluator.py @@ -0,0 +1,120 @@ +import os +from typing import List + +import cv2 +import numpy as np +import torch +import torch.nn as nn +from PIL import Image +from torchvision.datasets.utils import download_url + +from .longclip import longclip +from .viclip import get_viclip +from .video_utils import extract_frames + +# All metrics. +__all__ = ["VideoCLIPXLScore"] + +_MODELS = { + "ViClip-InternVid-10M-FLT": "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/video_caption/clip/ViClip-InternVid-10M-FLT.pth", + "LongCLIP-L": "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/video_caption/clip/longclip-L.pt", + "VideoCLIP-XL-v2": "https://pai-aigc-photog.oss-cn-hangzhou.aliyuncs.com/easyanimate/video_caption/clip/VideoCLIP-XL-v2.bin", +} +_MD5 = { + "ViClip-InternVid-10M-FLT": "b1ebf538225438b3b75e477da7735cd0", + "LongCLIP-L": "5478b662f6f85ca0ebd4bb05f9b592f3", + "VideoCLIP-XL-v2": "cebda0bab14b677ec061a57e80791f35", +} + +def normalize( + data: np.array, + mean: list[float] = [0.485, 0.456, 0.406], + std: list[float] = [0.229, 0.224, 0.225] +): + v_mean = np.array(mean).reshape(1, 1, 3) + v_std = np.array(std).reshape(1, 1, 3) + + return (data / 255.0 - v_mean) / v_std + + +class VideoCLIPXL(nn.Module): + def __init__(self, root: str = "~/.cache/clip"): + super(VideoCLIPXL, self).__init__() + + self.root = os.path.expanduser(root) + if not os.path.exists(self.root): + os.makedirs(self.root) + + k = "LongCLIP-L" + filename = os.path.basename(_MODELS[k]) + download_url(_MODELS[k], self.root, filename=filename, md5=_MD5[k]) + self.model = longclip.load(os.path.join(self.root, filename), device="cpu")[0].float() + + k = "ViClip-InternVid-10M-FLT" + filename = os.path.basename(_MODELS[k]) + download_url(_MODELS[k], self.root, filename=filename, md5=_MD5[k]) + self.viclip_model = get_viclip("l", os.path.join(self.root, filename))["viclip"].float() + + # delete unused encoder + del self.model.visual + del self.viclip_model.text_encoder + + +class VideoCLIPXLScore(): + def __init__(self, root: str = "~/.cache/clip", device: str = "cpu"): + self.root = os.path.expanduser(root) + if not os.path.exists(self.root): + os.makedirs(self.root) + + k = "VideoCLIP-XL-v2" + filename = os.path.basename(_MODELS[k]) + download_url(_MODELS[k], self.root, filename=filename, md5=_MD5[k]) + self.model = VideoCLIPXL() + state_dict = torch.load(os.path.join(self.root, filename), map_location="cpu") + self.model.load_state_dict(state_dict) + self.model.to(device) + + self.device = device + + def __call__(self, videos: List[List[Image.Image]], texts: List[str]): + assert len(videos) == len(texts) + + # Use cv2.resize in accordance with the official demo. Resize and Normalize => B * [T, 224, 224, 3]. + videos = [[cv2.cvtColor(np.array(f), cv2.COLOR_RGB2BGR) for f in v] for v in videos] + resize_videos = [[cv2.resize(f, (224, 224)) for f in v] for v in videos] + resize_normalizied_videos = [normalize(np.stack(v)) for v in resize_videos] + + video_inputs = torch.stack([torch.from_numpy(v) for v in resize_normalizied_videos]) + video_inputs = video_inputs.float().permute(0, 1, 4, 2, 3).to(self.device, non_blocking=True) # BTCHW + + with torch.no_grad(): + vid_features = torch.stack( + [self.model.viclip_model.get_vid_features(x.unsqueeze(0)).float() for x in video_inputs] + ) + vid_features.squeeze_() + # vid_features = self.model.viclip_model.get_vid_features(video_inputs).float() + text_inputs = longclip.tokenize(texts, truncate=True).to(self.device) + text_features = self.model.model.encode_text(text_inputs) + text_features = text_features / text_features.norm(dim=1, keepdim=True) + scores = text_features @ vid_features.T + + return scores.tolist() if len(videos) == 1 else scores.diagonal().tolist() + + def __repr__(self): + return "videoclipxl_score" + + +if __name__ == "__main__": + videos = ["your_video_path"] * 3 + texts = [ + "a joker", + "glasses and flower", + "The video opens with a view of a white building with multiple windows, partially obscured by leafless tree branches. The scene transitions to a closer view of the same building, with the tree branches more prominent in the foreground. The focus then shifts to a street sign that reads 'Abesses' in bold, yellow letters against a green background. The sign is attached to a metal structure, possibly a tram or bus stop. The sign is illuminated by a light source above it, and the background reveals a glimpse of the building and tree branches from earlier shots. The colors are muted, with the yellow sign standing out against the grey and green hues." + ] + + video_clip_xl_score = VideoCLIPXLScore(device="cuda") + batch_frames = [] + for v in videos: + sampled_frames = extract_frames(v, sample_method="uniform", num_sampled_frames=8)[1] + batch_frames.append(sampled_frames) + print(video_clip_xl_score(batch_frames, texts)) \ No newline at end of file diff --git a/VideoX-Fun/videox_fun/video_caption/utils/video_utils.py b/VideoX-Fun/videox_fun/video_caption/utils/video_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..bc9cc6c2e56c34f10a8f1543e6cced76aa8dc6c8 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/utils/video_utils.py @@ -0,0 +1,106 @@ +import gc +import random +import shutil +import subprocess +from contextlib import contextmanager +from typing import List, Optional, Tuple + +import numpy as np +from decord import VideoReader +from PIL import Image + + +ALL_FRAME_SAMPLE_METHODS = [ + "mid", "uniform", "random", "stride", "first", "last", "keyframe", "keyframe+first", "keyframe+last" +] + + +@contextmanager +def video_reader(*args, **kwargs): + """A context manager to solve the memory leak of decord. + """ + vr = VideoReader(*args, **kwargs) + try: + yield vr + finally: + del vr + gc.collect() + + +def get_keyframe_index(video_path): + """Extract the frame index list of I-frames. In general, the first frame in a video should be the I-frame. + The extracted frame index is more accurate than the pts_time * avg_fps. + """ + assert shutil.which("ffprobe") is not None, f"Please install ffprobe and make sure it is in the system path." + + command = [ + "ffprobe", + "-v", "quiet", + "-select_streams", "v:0", + "-show_entries", "frame=pict_type", + "-of", "csv=p=0", + video_path + ] + result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + + keyframe_index_list = [] + frame_index = 0 + for line in result.stdout.split("\n"): + line = line.strip(",") + pict_type = line.strip() + if pict_type == "I": + keyframe_index_list.append(frame_index) + if pict_type == "I" or pict_type == "B" or pict_type == "P": + frame_index += 1 + + return keyframe_index_list, frame_index + +def extract_frames( + video_path: str, + sample_method: str = "mid", + num_sampled_frames: int = 1, + sample_stride: Optional[int] = None, + **kwargs +) -> Optional[Tuple[List[int], List[Image.Image]]]: + if num_sampled_frames < 1: + raise ValueError(f"The num_sampled_frames must be greater than 1.") + if sample_stride is not None and sample_stride < 1: + raise ValueError(f"The sample_stride must be greater than 1.") + if sample_stride is not None and sample_method not in ["random", "stride"]: + raise ValueError(f"The sample_method must be random or stride when sample_stride is specified.") + with video_reader(video_path, num_threads=2, **kwargs) as vr: + if sample_method == "mid": + sampled_frame_idx_list = [len(vr) // 2] + elif sample_method == "uniform": + sampled_frame_idx_list = np.linspace(0, len(vr), num_sampled_frames, endpoint=False, dtype=int) + elif sample_method == "random": + clip_length = min(len(vr), (num_sampled_frames - 1) * sample_stride + 1) + start_idx = random.randint(0, len(vr) - clip_length) + sampled_frame_idx_list = np.linspace(start_idx, start_idx + clip_length - 1, num_sampled_frames, dtype=int) + elif sample_method == "stride": + sampled_frame_idx_list = np.arange(0, len(vr), sample_stride) + elif sample_method == "first": + sampled_frame_idx_list = [0] + elif sample_method == "last": + sampled_frame_idx_list = [len(vr) - 1] + elif sample_method == "keyframe": + sampled_frame_idx_list, final_frame_index = get_keyframe_index(video_path) + elif sample_method == "keyframe+first": # keyframe + the first second + sampled_frame_idx_list, final_frame_index = get_keyframe_index(video_path) + if len(sampled_frame_idx_list) == 1 or sampled_frame_idx_list[1] > 1 * vr.get_avg_fps(): + if int(1 * vr.get_avg_fps()) > len(vr): + raise ValueError(f"The duration of {video_path} is less than 1s.") + sampled_frame_idx_list.insert(1, int(1 * vr.get_avg_fps())) + elif sample_method == "keyframe+last": # keyframe + the last frame + sampled_frame_idx_list, final_frame_index = get_keyframe_index(video_path) + if sampled_frame_idx_list[-1] != (len(vr) - 1): + sampled_frame_idx_list.append(len(vr) - 1) + else: + raise ValueError(f"The sample_method must be within {ALL_FRAME_SAMPLE_METHODS}.") + if "keyframe" in sample_method: + if final_frame_index != len(vr): + raise ValueError(f"The keyframe index list is not accurate. Please check the video {video_path}.") + sampled_frame_list = vr.get_batch(sampled_frame_idx_list).asnumpy() + sampled_frame_list = [Image.fromarray(frame) for frame in sampled_frame_list] + + return list(sampled_frame_idx_list), sampled_frame_list diff --git a/VideoX-Fun/videox_fun/video_caption/video_splitting.py b/VideoX-Fun/videox_fun/video_caption/video_splitting.py new file mode 100644 index 0000000000000000000000000000000000000000..12415223ae7dcbd66ffe900716547388d4044f80 --- /dev/null +++ b/VideoX-Fun/videox_fun/video_caption/video_splitting.py @@ -0,0 +1,172 @@ +import argparse +import os +import subprocess +from datetime import datetime, timedelta +from multiprocessing import Pool +from pathlib import Path + +import pandas as pd +from tqdm import tqdm +from utils.logger import logger + +MIN_SECONDS = int(os.getenv("MIN_SECONDS", 3)) +MAX_SECONDS = int(os.getenv("MAX_SECONDS", 10)) + + +def get_command(start_time, video_path, video_duration, output_path): + # Use FFmpeg to split the video. Re-encoding is needed to ensure the accuracy of the clip + # at the cost of consuming computational resources. + return [ + 'ffmpeg', + '-hide_banner', + '-loglevel', 'panic', + '-ss', str(start_time.time()), + '-i', video_path, + '-t', str(video_duration), + '-c:v', 'libx264', + '-preset', 'veryfast', + '-crf', '22', + '-c:a', 'aac', + '-sn', + output_path + ] + + +def clip_video_star(args): + return clip_video(*args) + + +def clip_video(video_path, timecode_list, output_folder, video_duration): + """Recursively clip the video within the range of [MIN_SECONDS, MAX_SECONDS], + according to the timecode obtained from easyanimate/video_caption/cutscene_detect.py. + """ + try: + os.makedirs(output_folder, exist_ok=True) + video_stem = Path(video_path).stem + + if len(timecode_list) == 0: # The video of a single scene. + splitted_timecode_list = [] + start_time = datetime.strptime("00:00:00.000", "%H:%M:%S.%f") + end_time = datetime.strptime(video_duration, "%H:%M:%S.%f") + cur_start = start_time + splitted_index = 0 + while cur_start < end_time: + cur_end = min(cur_start + timedelta(seconds=MAX_SECONDS), end_time) + cur_video_duration = (cur_end - cur_start).total_seconds() + if cur_video_duration < MIN_SECONDS: + cur_start = cur_end + splitted_index += 1 + continue + splitted_timecode_list.append([cur_start.strftime("%H:%M:%S.%f")[:-3], cur_end.strftime("%H:%M:%S.%f")[:-3]]) + output_path = os.path.join(output_folder, video_stem + f"_{splitted_index}.mp4") + if os.path.exists(output_path): + logger.info(f"The clipped video {output_path} exists.") + cur_start = cur_end + splitted_index += 1 + continue + else: + command = get_command(cur_start, video_path, cur_video_duration, output_path) + try: + subprocess.run(command, check=True) + except Exception as e: + logger.warning(f"Run {command} error: {e}.") + finally: + cur_start = cur_end + splitted_index += 1 + + for i, timecode in enumerate(timecode_list): # The video of multiple scenes. + start_time = datetime.strptime(timecode[0], "%H:%M:%S.%f") + end_time = datetime.strptime(timecode[1], "%H:%M:%S.%f") + video_duration = (end_time - start_time).total_seconds() + output_path = os.path.join(output_folder, video_stem + f"_{i}.mp4") + if os.path.exists(output_path): + logger.info(f"The clipped video {output_path} exists.") + continue + if video_duration < MIN_SECONDS: + continue + if video_duration > MAX_SECONDS: + splitted_timecode_list = [] + cur_start = start_time + splitted_index = 0 + while cur_start < end_time: + cur_end = min(cur_start + timedelta(seconds=MAX_SECONDS), end_time) + cur_video_duration = (cur_end - cur_start).total_seconds() + if cur_video_duration < MIN_SECONDS: + break + splitted_timecode_list.append([cur_start.strftime("%H:%M:%S.%f")[:-3], cur_end.strftime("%H:%M:%S.%f")[:-3]]) + splitted_output_path = os.path.join(output_folder, video_stem + f"_{i}_{splitted_index}.mp4") + if os.path.exists(splitted_output_path): + logger.info(f"The clipped video {splitted_output_path} exists.") + cur_start = cur_end + splitted_index += 1 + continue + else: + command = get_command(cur_start, video_path, cur_video_duration, splitted_output_path) + try: + subprocess.run(command, check=True) + except Exception as e: + logger.warning(f"Run {command} error: {e}.") + finally: + cur_start = cur_end + splitted_index += 1 + + continue + + # We found that the current scene detected by PySceneDetect includes a few frames from + # the next scene occasionally. Directly discard the last few frames of the current scene. + video_duration = video_duration - 0.5 + command = get_command(start_time, video_path, video_duration, output_path) + subprocess.run(command, check=True) + except Exception as e: + logger.warning(f"Clip video with {video_path}. Error is: {e}.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Video Splitting") + parser.add_argument( + "--video_metadata_path", type=str, default=None, help="The path to the video dataset metadata (csv/jsonl)." + ) + parser.add_argument( + "--video_path_column", + type=str, + default="video_path", + help="The column contains the video path (an absolute path or a relative path w.r.t the video_folder).", + ) + parser.add_argument("--video_folder", type=str, default="", help="The video folder.") + parser.add_argument("--output_folder", type=str, default="outputs") + parser.add_argument("--n_jobs", type=int, default=16) + + parser.add_argument("--resolution_threshold", type=float, default=0, help="The resolution threshold.") + + args = parser.parse_args() + + video_metadata_df = pd.read_json(args.video_metadata_path, lines=True) + num_videos = len(video_metadata_df) + video_metadata_df["resolution"] = video_metadata_df["frame_size"].apply(lambda x: x[0] * x[1]) + video_metadata_df = video_metadata_df[video_metadata_df["resolution"] >= args.resolution_threshold] + logger.info(f"Filter {num_videos - len(video_metadata_df)} videos with resolution smaller than {args.resolution_threshold}.") + video_path_list = video_metadata_df[args.video_path_column].to_list() + video_timecode_list = video_metadata_df["timecode_list"].to_list() + video_duration_list = video_metadata_df["duration"].to_list() + + if args.video_folder == "": + output_folder_list = [args.output_folder] * len(video_path_list) + video_name_list = [Path(video_path).name for video_path in video_path_list] + # We only check the unique video name with the absolute video path. + if len(video_name_list) != len(set(video_name_list)): + logger.error(f"The video path in {args.video_metadata_path} should has an unique video name.") + else: + output_folder_list = [os.path.join(args.output_folder, os.path.dirname(video_path)) for video_path in video_path_list] + video_path_list = [os.path.join(args.video_folder, video_path) for video_path in video_path_list] + + args_list = [ + (video_path, timecode_list, output_folder, video_duration) + for video_path, timecode_list, output_folder, video_duration in zip( + video_path_list, video_timecode_list, output_folder_list, video_duration_list + ) + ] + with Pool(args.n_jobs) as pool: + # results = list(tqdm(pool.imap(clip_video_star, args_list), total=len(video_path_list))) + results = pool.imap(clip_video_star, args_list) + for result in tqdm(results, total=len(video_path_list)): + pass \ No newline at end of file diff --git a/text_encoder_axmodel/qwen3_p128_l0_together.axmodel b/text_encoder_axmodel/qwen3_p128_l0_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..5e2df5d7c8c8e1a140e869b2f75990f2036d59cc --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l0_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:015b26cbbbcc59544d502f7f275029454085f3698e4ad7e2436486dcc7b3ad14 +size 115537447 diff --git a/text_encoder_axmodel/qwen3_p128_l10_together.axmodel b/text_encoder_axmodel/qwen3_p128_l10_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..90fbaeee8a483a0de0539fa6c40b014ecefe3bb9 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l10_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b2f2fcf3c191bf02af884ff78a01494e2a33987afe234a498e23a950259e6d2 +size 115537351 diff --git a/text_encoder_axmodel/qwen3_p128_l11_together.axmodel b/text_encoder_axmodel/qwen3_p128_l11_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..9c78cd0c600b7ef104ec25aed105cc9396dee7bc --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l11_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:db3c47e0999062f99cc9ad9967793c5fb3b2f229c0a0d184b5c1f0802903a8f5 +size 115537607 diff --git a/text_encoder_axmodel/qwen3_p128_l12_together.axmodel b/text_encoder_axmodel/qwen3_p128_l12_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..19c67976787420ae1b823d90dd42fdfbd1881134 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l12_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ad3cf494241357617379b34de312839905b124afdf9b843635207abf7955fd84 +size 115537383 diff --git a/text_encoder_axmodel/qwen3_p128_l13_together.axmodel b/text_encoder_axmodel/qwen3_p128_l13_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..02551c57aeb54a6fc8ebcb6427203b4765c2493b --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l13_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e8a9e1797c46316c05bfe0b3bdce0d360bc318073061a52ae66de2d2a7e86303 +size 115537479 diff --git a/text_encoder_axmodel/qwen3_p128_l14_together.axmodel b/text_encoder_axmodel/qwen3_p128_l14_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..6140582319c49113cadb927fd442f4b821abbb74 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l14_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:da738314ea4725b23d2282836af173ab94aac121c2efed3d1d18bb87d06d49d1 +size 115537351 diff --git a/text_encoder_axmodel/qwen3_p128_l15_together.axmodel b/text_encoder_axmodel/qwen3_p128_l15_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..39964df1fc7f447fd024b632d7392e3ffdeb3c47 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l15_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:65489d36d9940e37a490961f67849cf0ef421a2770ee29ed4bb9faed2477c645 +size 115537351 diff --git a/text_encoder_axmodel/qwen3_p128_l16_together.axmodel b/text_encoder_axmodel/qwen3_p128_l16_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..f3974d0c94bf395cafd5f4c31a8c488f3361931e --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l16_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7106c322af685e404eee9bdf9e02f30b6540350e3da45c6d5c7c084a7338e60c +size 115537351 diff --git a/text_encoder_axmodel/qwen3_p128_l17_together.axmodel b/text_encoder_axmodel/qwen3_p128_l17_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..a0f048edb1e065e214aed2708307c94cb16d4faf --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l17_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3c930e344e0c88a008d22865c7d6bcff04d37d487dcbe7b5607d30377982ceb4 +size 115537479 diff --git a/text_encoder_axmodel/qwen3_p128_l18_together.axmodel b/text_encoder_axmodel/qwen3_p128_l18_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..7bc81cf92fc377a8cbb75c3353870030ff5051ce --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l18_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:43e4d2b5bd7f2dd81538a57ad42f2d9bdad6a6a02ebad88bd67184795700557d +size 115537351 diff --git a/text_encoder_axmodel/qwen3_p128_l19_together.axmodel b/text_encoder_axmodel/qwen3_p128_l19_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..277537dfc665061dd74b56924eaaed7443863865 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l19_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36f63f8618a15950d4d36241d2d5a580d64247b13580c3a0b260b88a6f0a86a1 +size 115537351 diff --git a/text_encoder_axmodel/qwen3_p128_l1_together.axmodel b/text_encoder_axmodel/qwen3_p128_l1_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..7798276f419bc742db0398226e2559f5ab5b653b --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l1_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ffef0fe95513ec5e1a1c982fe0005be11f513e191a161761eaf74a46301b1462 +size 115543143 diff --git a/text_encoder_axmodel/qwen3_p128_l20_together.axmodel b/text_encoder_axmodel/qwen3_p128_l20_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..a57f1fb4b504cb81062877947def0085f0f0a734 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l20_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b133dba645ddadb0d9b7e0d465dd6892064542287f897de7061b5d4e054ca707 +size 115537351 diff --git a/text_encoder_axmodel/qwen3_p128_l21_together.axmodel b/text_encoder_axmodel/qwen3_p128_l21_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..3ec2728eef5a1b09b369e26cb653f0a333a9b37e --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l21_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edb50372189cfedc616024bfd1707b7c736e383a67a1ddea83d91377168093cf +size 115537543 diff --git a/text_encoder_axmodel/qwen3_p128_l22_together.axmodel b/text_encoder_axmodel/qwen3_p128_l22_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..1e1c7166f9ad9a9f046e2827277c59dbd3fefcd3 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l22_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:59d1e0c9ef05c27c505333430bd924aafc1370ceb3f3a41f00a94855544ec918 +size 115537799 diff --git a/text_encoder_axmodel/qwen3_p128_l23_together.axmodel b/text_encoder_axmodel/qwen3_p128_l23_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..3e542811db25bed82737c07f38e757288b6105ed --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l23_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:222a6aae70db8a2b599665df91cf62e75d83bb9d6f59494c88fa94605c250a56 +size 115537543 diff --git a/text_encoder_axmodel/qwen3_p128_l24_together.axmodel b/text_encoder_axmodel/qwen3_p128_l24_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..fed44e7679cfc8b44455e3177cb539cd1a6e8a3d --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l24_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:726e2ad2fd64be1e4fcdf6ee63dd2d849f98d9dd65183716736215ad622f2ea4 +size 115538311 diff --git a/text_encoder_axmodel/qwen3_p128_l25_together.axmodel b/text_encoder_axmodel/qwen3_p128_l25_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..de61278c16bc53cce4a96051d05eb5a77f538128 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l25_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c88da927270ec22f563e9c94012c2c16a64de480fd6e25c8919d7550dbe3b708 +size 115538151 diff --git a/text_encoder_axmodel/qwen3_p128_l26_together.axmodel b/text_encoder_axmodel/qwen3_p128_l26_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..8191d759b6745138dbdb623fbfacf7d85319f529 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l26_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33107e3b4334ed0a8baecbbd352cc70ce290fcbb649e162a7ae93bdafa42774d +size 115537863 diff --git a/text_encoder_axmodel/qwen3_p128_l27_together.axmodel b/text_encoder_axmodel/qwen3_p128_l27_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..eaea7c5aaffbb802021cdd994e31e58348c03adf --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l27_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42716b80a9248b5e37a7e1a55b51d95c044649c59fed7fbac4a4012ecb26a84f +size 115538375 diff --git a/text_encoder_axmodel/qwen3_p128_l28_together.axmodel b/text_encoder_axmodel/qwen3_p128_l28_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..30221c70703446c6f6ab9d650d40150b1d60d819 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l28_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a25389c51fd1f52d46f0475610262eb15eaf087096f39522dd27b9efffc00653 +size 115538599 diff --git a/text_encoder_axmodel/qwen3_p128_l29_together.axmodel b/text_encoder_axmodel/qwen3_p128_l29_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..cfa3914f0a16184308679c63297d12748ae1b0ab --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l29_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56417341b4f97ac3d4d71c92137c55fcb4502f23cf11e107716313c67bc1627b +size 115537895 diff --git a/text_encoder_axmodel/qwen3_p128_l2_together.axmodel b/text_encoder_axmodel/qwen3_p128_l2_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..ecf112ff98e7782c6fb8018e8b98b50eb80e46fc --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l2_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c4e90130a51bfb4f074921d0ff2a0496239feb40ba268a773c335e854ba56a45 +size 115541735 diff --git a/text_encoder_axmodel/qwen3_p128_l30_together.axmodel b/text_encoder_axmodel/qwen3_p128_l30_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..5bb5882eb6c0fa572987bf6864470119cfb768ac --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l30_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bb6dc8db5597bccdec7767fa71c99035e60d5a0ffa6fcaaa8fc5c9e970c39aff +size 115538023 diff --git a/text_encoder_axmodel/qwen3_p128_l31_together.axmodel b/text_encoder_axmodel/qwen3_p128_l31_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..f4f7f919c6cd42beaeb1b42c2231917d0e741e3d --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l31_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8be5e89099bd82cd6d64f832336124188de912a35dc3e4b9797c26dcd659733a +size 115538855 diff --git a/text_encoder_axmodel/qwen3_p128_l32_together.axmodel b/text_encoder_axmodel/qwen3_p128_l32_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..493243ce0a627abc41ee5bfe7f1d0d5cba8e3864 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l32_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cb4272b001ff95fc6acb364724ad3b77bce4e82ed229aa7fa55ffc44bae371f +size 115538119 diff --git a/text_encoder_axmodel/qwen3_p128_l33_together.axmodel b/text_encoder_axmodel/qwen3_p128_l33_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..b05ed32699f627feaa742fa2cc6f4b587c3f86c8 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l33_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a2d81ab8016d5a4e46a2bfa6b6dc95ca8eda70fdb81ca7ad3dccd02bed5fdc92 +size 115538055 diff --git a/text_encoder_axmodel/qwen3_p128_l34_together.axmodel b/text_encoder_axmodel/qwen3_p128_l34_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..8edba01c968f74b1b96dc9d01b20470fc72b6dda --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l34_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d8c7a53f2665ccb389048f9f4c488f14d402000d461421d81b46bbb21d443efd +size 115537799 diff --git a/text_encoder_axmodel/qwen3_p128_l35_together.axmodel b/text_encoder_axmodel/qwen3_p128_l35_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..e13edd8d7efcdaff025a37f62f48573e4e610e44 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l35_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e84f6127a4823c419650e539acdabea27cdf8736f20c1c63695c76e486585c6 +size 95402599 diff --git a/text_encoder_axmodel/qwen3_p128_l3_together.axmodel b/text_encoder_axmodel/qwen3_p128_l3_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..f95990b33c61b513bbdbdb12cc7311475add4618 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l3_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e1d47d2b1af69d8648d809adf3643f5fe3599f346f921e86e97a4edbbd058862 +size 115548359 diff --git a/text_encoder_axmodel/qwen3_p128_l4_together.axmodel b/text_encoder_axmodel/qwen3_p128_l4_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..0f74d16349eae1d1e31463ac45e7df3a4f3e5935 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l4_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:82ea99922331fe2e333acdbd93f44d3d35a0621fc7eacfc8f3b3f189606932a8 +size 115537351 diff --git a/text_encoder_axmodel/qwen3_p128_l5_together.axmodel b/text_encoder_axmodel/qwen3_p128_l5_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..04ed51c9eb57bc28a8ee8b0525f30860009b4fd5 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l5_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8d823b21b6a048e1f714bf769f6bbcc513563717192516c32fa529b58c3413a2 +size 115537383 diff --git a/text_encoder_axmodel/qwen3_p128_l6_together.axmodel b/text_encoder_axmodel/qwen3_p128_l6_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..80d75d1cbcd5c0a0f1a82d1154a8963d72950ea3 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l6_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63e6d100f322d74a74cec4951999dae8683e59b07a88409effa2bdbd61c83d44 +size 115540775 diff --git a/text_encoder_axmodel/qwen3_p128_l7_together.axmodel b/text_encoder_axmodel/qwen3_p128_l7_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..1c5bb090254520304aa002f7e95f49dad33738a2 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l7_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0337890a0ebdd65a6028df6038d6eb173630f0913d82e859d22f9f970bc681b4 +size 115540647 diff --git a/text_encoder_axmodel/qwen3_p128_l8_together.axmodel b/text_encoder_axmodel/qwen3_p128_l8_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..0f9c8bd3a88dada36d1a1943be35ace646cc7828 --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l8_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:45d1bc52508c5d5fd1789b7866dcaf2d5e511248c262f6f3223cc7e2360f9dc6 +size 115537351 diff --git a/text_encoder_axmodel/qwen3_p128_l9_together.axmodel b/text_encoder_axmodel/qwen3_p128_l9_together.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..fe8c31aafd11efdaf0246112adfba158df9ddc6e --- /dev/null +++ b/text_encoder_axmodel/qwen3_p128_l9_together.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:055979ef0a4293ed3bd29ab027cbe41454e119d61b14d68fdf4f4ee616953adc +size 115537575 diff --git a/text_encoder_axmodel/qwen3_post.axmodel b/text_encoder_axmodel/qwen3_post.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..9b4bdffdeeaed6a25256b25d6d6464c194c082b3 --- /dev/null +++ b/text_encoder_axmodel/qwen3_post.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d88728ff98d9d20633619d0d94a9959694cdd12e4400d1f48f9a3a0e0a903ecf +size 424638492 diff --git a/transformer_axmodel/auto_00_model_layers_29_Add_4_output_0_to_sample_auto.axmodel b/transformer_axmodel/auto_00_model_layers_29_Add_4_output_0_to_sample_auto.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..97c057ef6fe94dce6987c123c6b06ae504e745fe --- /dev/null +++ b/transformer_axmodel/auto_00_model_layers_29_Add_4_output_0_to_sample_auto.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e30218016cd3015f713e219e8ace67539c646b378ae5b79e9e450b0622b88ff5 +size 1467569 diff --git a/transformer_axmodel/cfg_00_timestep_to_model_t_embedder_mlp_mlp_2_Gemm_output_0_config.axmodel b/transformer_axmodel/cfg_00_timestep_to_model_t_embedder_mlp_mlp_2_Gemm_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..a3e960ff29508d424d9df394bf93320fb43462fd --- /dev/null +++ b/transformer_axmodel/cfg_00_timestep_to_model_t_embedder_mlp_mlp_2_Gemm_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:63b3caa2599f82c1069551e911efb327cb4a66b37f5566a491095f429f64c511 +size 599987 diff --git a/transformer_axmodel/cfg_01_prompt_embeds_to_model_Slice_1_output_0_config.axmodel b/transformer_axmodel/cfg_01_prompt_embeds_to_model_Slice_1_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..ccd1fb1b006c4126cacc6d6302f3dab112ac6ddb --- /dev/null +++ b/transformer_axmodel/cfg_01_prompt_embeds_to_model_Slice_1_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2c7cf54bc49b24637c8c60735dfd0944ec4dbde9ef9b677fde45344fcf88d572 +size 394749113 diff --git a/transformer_axmodel/cfg_02_latent_model_input_to_model_Slice_output_0_config.axmodel b/transformer_axmodel/cfg_02_latent_model_input_to_model_Slice_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..be439ffe8828a9c7f1a39d28bcb012c3df894647 --- /dev/null +++ b/transformer_axmodel/cfg_02_latent_model_input_to_model_Slice_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24938cd2848c5309660550ef307ce7ade2ef699d1e482ce03dd9d2e3667fdbd0 +size 413729938 diff --git a/transformer_axmodel/cfg_03_model_Slice_1_output_0_to_model_layers_0_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_03_model_Slice_1_output_0_to_model_layers_0_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..dc277538f8fb5ca67f050304817f8b5e17a99cf4 --- /dev/null +++ b/transformer_axmodel/cfg_03_model_Slice_1_output_0_to_model_layers_0_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4046d49ee08ac391f2d7826bfe94e6376aac788c2deb0d7a3c4010dad5572e49 +size 206688928 diff --git a/transformer_axmodel/cfg_04_model_layers_0_Add_4_output_0_to_model_layers_1_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_04_model_layers_0_Add_4_output_0_to_model_layers_1_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..955f787c3297abcc99c48757622648edc9b0009e --- /dev/null +++ b/transformer_axmodel/cfg_04_model_layers_0_Add_4_output_0_to_model_layers_1_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9eab0c6b21fa4c14c538b93a55863806ffbc37f71a2879413ca548483a254053 +size 206628750 diff --git a/transformer_axmodel/cfg_05_model_layers_1_Add_4_output_0_to_model_layers_2_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_05_model_layers_1_Add_4_output_0_to_model_layers_2_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..c3ff85c759e9574c8f47c442d900018ca5d52ddc --- /dev/null +++ b/transformer_axmodel/cfg_05_model_layers_1_Add_4_output_0_to_model_layers_2_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:00ed4555dc8d92e39c40854c15935d08bd326a350c933ff04c25781dbf53fbad +size 206628500 diff --git a/transformer_axmodel/cfg_06_model_layers_2_Add_4_output_0_to_model_layers_3_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_06_model_layers_2_Add_4_output_0_to_model_layers_3_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..609c049cd5aaeb0cb22371dbed5c3a68fdef8717 --- /dev/null +++ b/transformer_axmodel/cfg_06_model_layers_2_Add_4_output_0_to_model_layers_3_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3f424834ac1254b6789c39beb2e29360264b62a4400cb80d46981c2f7f108f11 +size 206628782 diff --git a/transformer_axmodel/cfg_07_model_layers_3_Add_4_output_0_to_model_layers_4_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_07_model_layers_3_Add_4_output_0_to_model_layers_4_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..e10b861aae3abd12fedef32295c7b4d6d6395a8a --- /dev/null +++ b/transformer_axmodel/cfg_07_model_layers_3_Add_4_output_0_to_model_layers_4_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:23dd63f9d15165c369f584d9282a86b5e877172d2846dc2c6b59d39eb8ebd809 +size 206628430 diff --git a/transformer_axmodel/cfg_08_model_layers_4_Add_4_output_0_to_model_layers_5_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_08_model_layers_4_Add_4_output_0_to_model_layers_5_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..e97365ddae5fea0d2c8e449ba4984390aa08564e --- /dev/null +++ b/transformer_axmodel/cfg_08_model_layers_4_Add_4_output_0_to_model_layers_5_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b6c28c3b05ed6abe0078f34e47be2662101e97eebdcde68a195384764c021259 +size 206628430 diff --git a/transformer_axmodel/cfg_09_model_layers_5_Add_4_output_0_to_model_layers_6_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_09_model_layers_5_Add_4_output_0_to_model_layers_6_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..d6fefb9e8a91b676046a7617809863f5cf548b57 --- /dev/null +++ b/transformer_axmodel/cfg_09_model_layers_5_Add_4_output_0_to_model_layers_6_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:caf0d1157a8ebdf700fed66c954b39b4deb6a77ad20dea9dc7d3f6964f3e12e8 +size 206791990 diff --git a/transformer_axmodel/cfg_10_model_layers_6_Add_4_output_0_to_model_layers_7_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_10_model_layers_6_Add_4_output_0_to_model_layers_7_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..217718e0b10a46a2e7424f57af919860b0d623e2 --- /dev/null +++ b/transformer_axmodel/cfg_10_model_layers_6_Add_4_output_0_to_model_layers_7_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5656739760428e392b990c7bdba495308cf7a1dea42b1491ec902c6057e08230 +size 206791894 diff --git a/transformer_axmodel/cfg_11_model_layers_7_Add_4_output_0_to_model_layers_8_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_11_model_layers_7_Add_4_output_0_to_model_layers_8_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..930347fb606855a253ef70fae58089de8c03583b --- /dev/null +++ b/transformer_axmodel/cfg_11_model_layers_7_Add_4_output_0_to_model_layers_8_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:333f38d67c51fbd8bc245e44e3356f83417771330012d9b4b9eed08dead52d12 +size 206790862 diff --git a/transformer_axmodel/cfg_12_model_layers_8_Add_4_output_0_to_model_layers_9_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_12_model_layers_8_Add_4_output_0_to_model_layers_9_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..e4767401007a3f83a85812e10befd569f212288c --- /dev/null +++ b/transformer_axmodel/cfg_12_model_layers_8_Add_4_output_0_to_model_layers_9_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0204743b711a5b08f7831ead79957c6a2ba6690eb3e01257b5863cca0a883c9f +size 206791638 diff --git a/transformer_axmodel/cfg_13_model_layers_9_Add_4_output_0_to_model_layers_10_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_13_model_layers_9_Add_4_output_0_to_model_layers_10_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..b2904fa780dcc69026c6121f3b9260268e0f3252 --- /dev/null +++ b/transformer_axmodel/cfg_13_model_layers_9_Add_4_output_0_to_model_layers_10_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:547744c6070588a8ab48f36c3d377cea11b60deac36367052cee5ed908483584 +size 206794768 diff --git a/transformer_axmodel/cfg_14_model_layers_10_Add_4_output_0_to_model_layers_11_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_14_model_layers_10_Add_4_output_0_to_model_layers_11_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..e31d4df3c5b3bccebdd7111303707c620fb9d040 --- /dev/null +++ b/transformer_axmodel/cfg_14_model_layers_10_Add_4_output_0_to_model_layers_11_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6d63eb7938fc5d3be9e607fd38090dfc01a71050553167cc10a6434023d48b8f +size 206791521 diff --git a/transformer_axmodel/cfg_15_model_layers_11_Add_4_output_0_to_model_layers_12_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_15_model_layers_11_Add_4_output_0_to_model_layers_12_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..a5dd94523433fa120348144983285b5a6e09bf6d --- /dev/null +++ b/transformer_axmodel/cfg_15_model_layers_11_Add_4_output_0_to_model_layers_12_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:195699f44c2d9d01289fc1ff56b4308c9d7d437fbcdb43f9a951c4953b5fc9c0 +size 206791545 diff --git a/transformer_axmodel/cfg_16_model_layers_12_Add_4_output_0_to_model_layers_13_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_16_model_layers_12_Add_4_output_0_to_model_layers_13_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..04a7eb614d8cce12f9ca2698158a8f6449f62816 --- /dev/null +++ b/transformer_axmodel/cfg_16_model_layers_12_Add_4_output_0_to_model_layers_13_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a368250f9b205e434832dfe527c00734f173517bbd1e85e8bb21db77f9c719fb +size 206794615 diff --git a/transformer_axmodel/cfg_17_model_layers_13_Add_4_output_0_to_model_layers_14_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_17_model_layers_13_Add_4_output_0_to_model_layers_14_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..9046a3661194ccc037edab5863cf315a0dd2a2af --- /dev/null +++ b/transformer_axmodel/cfg_17_model_layers_13_Add_4_output_0_to_model_layers_14_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4ef0fa1c1efc23d27b373341b37bc120f4fa57634931f850106bd7a14d9a8042 +size 206791457 diff --git a/transformer_axmodel/cfg_18_model_layers_14_Add_4_output_0_to_model_layers_15_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_18_model_layers_14_Add_4_output_0_to_model_layers_15_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..0f4b9992fa16d850e0e6071cee7e1af02adbd855 --- /dev/null +++ b/transformer_axmodel/cfg_18_model_layers_14_Add_4_output_0_to_model_layers_15_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1cd4e9378df6c7422e48fbb8cd288e377014970c5b20636a24f3045c8baf3cf4 +size 206794655 diff --git a/transformer_axmodel/cfg_19_model_layers_15_Add_4_output_0_to_model_layers_16_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_19_model_layers_15_Add_4_output_0_to_model_layers_16_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..e54fe1f98b2c2772e54bf0a269015eaff84553c7 --- /dev/null +++ b/transformer_axmodel/cfg_19_model_layers_15_Add_4_output_0_to_model_layers_16_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:025d4b08ebd6d3c08fec03874487ef15d409ebc44a0b5df4234349ad61d0a1d3 +size 206790305 diff --git a/transformer_axmodel/cfg_20_model_layers_16_Add_4_output_0_to_model_layers_17_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_20_model_layers_16_Add_4_output_0_to_model_layers_17_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..06c92ec180842c14c6b0fe6272812937d20d8fd8 --- /dev/null +++ b/transformer_axmodel/cfg_20_model_layers_16_Add_4_output_0_to_model_layers_17_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:409fed725e1e86abbaad76478d3efc934583d133760dddf01eb935551a2bf6f0 +size 206792887 diff --git a/transformer_axmodel/cfg_21_model_layers_17_Add_4_output_0_to_model_layers_18_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_21_model_layers_17_Add_4_output_0_to_model_layers_18_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..48e4755c0a64c8a4bc9661ac1f46555b9672dc8a --- /dev/null +++ b/transformer_axmodel/cfg_21_model_layers_17_Add_4_output_0_to_model_layers_18_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:30610e7306e69ec16bdeba9e361beb395ef4ec46c864764c1b02bb0da0226a53 +size 206792983 diff --git a/transformer_axmodel/cfg_22_model_layers_18_Add_4_output_0_to_model_layers_19_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_22_model_layers_18_Add_4_output_0_to_model_layers_19_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..8f824937991d08f67937a2184bc93f1549878a85 --- /dev/null +++ b/transformer_axmodel/cfg_22_model_layers_18_Add_4_output_0_to_model_layers_19_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b7d58cd848de4bb4a6ffd12e815c4700a2ea9f00cfaaf09f2b7cfb18e350c79 +size 206794577 diff --git a/transformer_axmodel/cfg_23_model_layers_19_Add_4_output_0_to_model_layers_20_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_23_model_layers_19_Add_4_output_0_to_model_layers_20_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..de0eb6757e2846246b3eee447117e503be61a4c9 --- /dev/null +++ b/transformer_axmodel/cfg_23_model_layers_19_Add_4_output_0_to_model_layers_20_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4cadbe7a49ac276a9da470640f8333d1ace969d6269932b018ae894639721968 +size 206792531 diff --git a/transformer_axmodel/cfg_24_model_layers_20_Add_4_output_0_to_model_layers_21_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_24_model_layers_20_Add_4_output_0_to_model_layers_21_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..e4ce10a2b55646f96af4350c91bfd985e708d068 --- /dev/null +++ b/transformer_axmodel/cfg_24_model_layers_20_Add_4_output_0_to_model_layers_21_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8fca01a83080b1bd529580956e3ee21e29b18fa135e7d603c752badbbf296c3b +size 206793675 diff --git a/transformer_axmodel/cfg_25_model_layers_21_Add_4_output_0_to_model_layers_22_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_25_model_layers_21_Add_4_output_0_to_model_layers_22_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..78d1616d6c091b169f0600275ecc94b8bc95bcca --- /dev/null +++ b/transformer_axmodel/cfg_25_model_layers_21_Add_4_output_0_to_model_layers_22_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:84724b42679ef092009b91120f311da271cbd1a62efafb7fa48c8aa690407f77 +size 206792979 diff --git a/transformer_axmodel/cfg_26_model_layers_22_Add_4_output_0_to_model_layers_23_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_26_model_layers_22_Add_4_output_0_to_model_layers_23_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..da86b57f6cd03d2b5ddc2d71ca7d2f140325830d --- /dev/null +++ b/transformer_axmodel/cfg_26_model_layers_22_Add_4_output_0_to_model_layers_23_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9b97034e89c0bc93dc9f22637dbb1158721795a4c7e0d439c58ed9c8ae1643a3 +size 206793457 diff --git a/transformer_axmodel/cfg_27_model_layers_23_Add_4_output_0_to_model_layers_24_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_27_model_layers_23_Add_4_output_0_to_model_layers_24_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..1e943354576d0699098a34da2608c6b36d3ec0ac --- /dev/null +++ b/transformer_axmodel/cfg_27_model_layers_23_Add_4_output_0_to_model_layers_24_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:36570e771d1763eb5b39ea9d6b39a28461eb1a8a3156321c801328f4b7067cfc +size 206791115 diff --git a/transformer_axmodel/cfg_28_model_layers_24_Add_4_output_0_to_model_layers_25_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_28_model_layers_24_Add_4_output_0_to_model_layers_25_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..ca01e7493edb89603c152e66907284e73beedb8c --- /dev/null +++ b/transformer_axmodel/cfg_28_model_layers_24_Add_4_output_0_to_model_layers_25_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e88dac81a2be4c746d150254e5fcab82f2862e7072ab35f7a9c69a7eb3d50f45 +size 206793011 diff --git a/transformer_axmodel/cfg_29_model_layers_25_Add_4_output_0_to_model_layers_26_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_29_model_layers_25_Add_4_output_0_to_model_layers_26_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..9a853e1963fb7d1a3ad11be010fcaa451a474cf1 --- /dev/null +++ b/transformer_axmodel/cfg_29_model_layers_25_Add_4_output_0_to_model_layers_26_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:50718b3b2a1f420f99478eda30b4cab306e8ecae799f147e25a3ba75665e6fd0 +size 206792971 diff --git a/transformer_axmodel/cfg_30_model_layers_26_Add_4_output_0_to_model_layers_27_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_30_model_layers_26_Add_4_output_0_to_model_layers_27_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..3ea63a6a4421fdaa57227df6500a6a87f05a7a80 --- /dev/null +++ b/transformer_axmodel/cfg_30_model_layers_26_Add_4_output_0_to_model_layers_27_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2b7a795213d3d309c2c8d1cd73bd78202e2abca5738241128df26ff11319240 +size 206792659 diff --git a/transformer_axmodel/cfg_31_model_layers_27_Add_4_output_0_to_model_layers_28_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_31_model_layers_27_Add_4_output_0_to_model_layers_28_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..50a64ac6e037b9e6e89b51659e70a64b86caf6ac --- /dev/null +++ b/transformer_axmodel/cfg_31_model_layers_27_Add_4_output_0_to_model_layers_28_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a494e8c857875ee0af328d7e146dd6c255d97aa0f7295afe495abccfcb59aad8 +size 206789581 diff --git a/transformer_axmodel/cfg_32_model_layers_28_Add_4_output_0_to_model_layers_29_Add_4_output_0_config.axmodel b/transformer_axmodel/cfg_32_model_layers_28_Add_4_output_0_to_model_layers_29_Add_4_output_0_config.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..60f271710faa851188b6d2409f99b222f3f63e56 --- /dev/null +++ b/transformer_axmodel/cfg_32_model_layers_28_Add_4_output_0_to_model_layers_29_Add_4_output_0_config.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3119c5b9dcb136a4919e4d5a9626c5ed76f57400a1fd1006af70fd4b11053b1 +size 206787475 diff --git a/transformer_onnx/auto_00_model_layers_29_Add_4_output_0_to_sample_auto.onnx b/transformer_onnx/auto_00_model_layers_29_Add_4_output_0_to_sample_auto.onnx new file mode 100644 index 0000000000000000000000000000000000000000..e35a86a5f26cc4a987a66e146079ec818f9a5ea0 --- /dev/null +++ b/transformer_onnx/auto_00_model_layers_29_Add_4_output_0_to_sample_auto.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e3a6aca4fa5180ca13a29c537fd49807db71329ede085e97e6b159b74044086f +size 4965844 diff --git a/transformer_onnx/cfg_00_timestep_to_model_t_embedder_mlp_mlp_2_Gemm_output_0_config.onnx b/transformer_onnx/cfg_00_timestep_to_model_t_embedder_mlp_mlp_2_Gemm_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..45c56876f02e0e631ba32a656cc6ef1bdb0b0f34 --- /dev/null +++ b/transformer_onnx/cfg_00_timestep_to_model_t_embedder_mlp_mlp_2_Gemm_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42262f3efb7c0b985ba87ad04bc12d5a829de527524561dca7e20f3b9f8a5adf +size 2105230 diff --git a/transformer_onnx/cfg_01_prompt_embeds_to_model_Slice_1_output_0_config.onnx b/transformer_onnx/cfg_01_prompt_embeds_to_model_Slice_1_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..cba7aed140ec936cde48e4d7a2cb165904fe8a71 --- /dev/null +++ b/transformer_onnx/cfg_01_prompt_embeds_to_model_Slice_1_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2caabd83421925eab599da4ce80a80017715b6eaa202785d7482f023afaf9081 +size 1457141784 diff --git a/transformer_onnx/cfg_02_latent_model_input_to_model_Slice_output_0_config.onnx b/transformer_onnx/cfg_02_latent_model_input_to_model_Slice_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..0059f585b747fdf11b6ab999cbb991ae4427dd61 --- /dev/null +++ b/transformer_onnx/cfg_02_latent_model_input_to_model_Slice_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81b378c69574fdf6fb9de3b8567446e9058d324e891c57b8563cc543063384ef +size 1464326700 diff --git a/transformer_onnx/cfg_03_model_Slice_1_output_0_to_model_layers_0_Add_4_output_0_config.onnx b/transformer_onnx/cfg_03_model_Slice_1_output_0_to_model_layers_0_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..3820a630a32fc35dd288fe099e6bae8ec2671cc3 --- /dev/null +++ b/transformer_onnx/cfg_03_model_Slice_1_output_0_to_model_layers_0_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f0af7a49135683aff2f51c8c893a0f4a01503980bccfcd89e0561a01aafbc9c3 +size 723984556 diff --git a/transformer_onnx/cfg_04_model_layers_0_Add_4_output_0_to_model_layers_1_Add_4_output_0_config.onnx b/transformer_onnx/cfg_04_model_layers_0_Add_4_output_0_to_model_layers_1_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..10a629e9a9ea8218d2a0f929324d90b595358948 --- /dev/null +++ b/transformer_onnx/cfg_04_model_layers_0_Add_4_output_0_to_model_layers_1_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d18eb368db7db6ea9fe77141190c045ef1ece78cf3a8a057746944add6a2516b +size 723984182 diff --git a/transformer_onnx/cfg_05_model_layers_1_Add_4_output_0_to_model_layers_2_Add_4_output_0_config.onnx b/transformer_onnx/cfg_05_model_layers_1_Add_4_output_0_to_model_layers_2_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..c21976bb7bb8a80167bd803578dfd6f6649a8d38 --- /dev/null +++ b/transformer_onnx/cfg_05_model_layers_1_Add_4_output_0_to_model_layers_2_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2062d9bb262b2b29cd1004e3c6b1683ae7758e54b088fad0ea9c191116ec1fa7 +size 723984182 diff --git a/transformer_onnx/cfg_06_model_layers_2_Add_4_output_0_to_model_layers_3_Add_4_output_0_config.onnx b/transformer_onnx/cfg_06_model_layers_2_Add_4_output_0_to_model_layers_3_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..9ecd65d3717588d8d3a61b7abf3309b78f3d18f0 --- /dev/null +++ b/transformer_onnx/cfg_06_model_layers_2_Add_4_output_0_to_model_layers_3_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56e08eeb1df6d167862b7d2ee95dc4ba7aa8ca8588562cae729f9171ac00c63c +size 723984182 diff --git a/transformer_onnx/cfg_07_model_layers_3_Add_4_output_0_to_model_layers_4_Add_4_output_0_config.onnx b/transformer_onnx/cfg_07_model_layers_3_Add_4_output_0_to_model_layers_4_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..0bccb4beb2f8b72269793dda9a389d3b4ea398cd --- /dev/null +++ b/transformer_onnx/cfg_07_model_layers_3_Add_4_output_0_to_model_layers_4_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42149f816c66f021ac8aa0454fb3d09a85b3ca42c76ad3b1c878ccb22933ef57 +size 723984182 diff --git a/transformer_onnx/cfg_08_model_layers_4_Add_4_output_0_to_model_layers_5_Add_4_output_0_config.onnx b/transformer_onnx/cfg_08_model_layers_4_Add_4_output_0_to_model_layers_5_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..7527f9959a88394f31263d6779fe67abaa7cf3ae --- /dev/null +++ b/transformer_onnx/cfg_08_model_layers_4_Add_4_output_0_to_model_layers_5_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0e80e59cb0f09007ec806767785b4558b6b46c1d295df346edfef5d9097649fe +size 723984182 diff --git a/transformer_onnx/cfg_09_model_layers_5_Add_4_output_0_to_model_layers_6_Add_4_output_0_config.onnx b/transformer_onnx/cfg_09_model_layers_5_Add_4_output_0_to_model_layers_6_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..2ff8c41a4a00467a337751d073dc741c9a6d6586 --- /dev/null +++ b/transformer_onnx/cfg_09_model_layers_5_Add_4_output_0_to_model_layers_6_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d4ec62a986d3e37841aec38d32ac3be502e69d1322635d5bc9dd02784aa7ff8f +size 723984182 diff --git a/transformer_onnx/cfg_10_model_layers_6_Add_4_output_0_to_model_layers_7_Add_4_output_0_config.onnx b/transformer_onnx/cfg_10_model_layers_6_Add_4_output_0_to_model_layers_7_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..b9523a1b8e4de12118c1dfcba9cdfc030c71d83b --- /dev/null +++ b/transformer_onnx/cfg_10_model_layers_6_Add_4_output_0_to_model_layers_7_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:576a6c5915e0d741b3e9bf3ebf057609c4ad6540f7598f5cf447598d9c49a767 +size 723984182 diff --git a/transformer_onnx/cfg_11_model_layers_7_Add_4_output_0_to_model_layers_8_Add_4_output_0_config.onnx b/transformer_onnx/cfg_11_model_layers_7_Add_4_output_0_to_model_layers_8_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..13b8483dcfe66105413c04e7529eb89103770b11 --- /dev/null +++ b/transformer_onnx/cfg_11_model_layers_7_Add_4_output_0_to_model_layers_8_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ecdcce6929d3d65b84a1a9b24662bfd68b38ee35e707fab5734d1511e463ed05 +size 723984182 diff --git a/transformer_onnx/cfg_12_model_layers_8_Add_4_output_0_to_model_layers_9_Add_4_output_0_config.onnx b/transformer_onnx/cfg_12_model_layers_8_Add_4_output_0_to_model_layers_9_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..626ebe36e27ad7cb60e1f0d6fd583474585471ce --- /dev/null +++ b/transformer_onnx/cfg_12_model_layers_8_Add_4_output_0_to_model_layers_9_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:919baa688635a219c951c7eb40a9c12748d685950ac551ea1018672b872df404 +size 723984182 diff --git a/transformer_onnx/cfg_13_model_layers_9_Add_4_output_0_to_model_layers_10_Add_4_output_0_config.onnx b/transformer_onnx/cfg_13_model_layers_9_Add_4_output_0_to_model_layers_10_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..f4847834a1ce10d726a1356a90e88568072ecdae --- /dev/null +++ b/transformer_onnx/cfg_13_model_layers_9_Add_4_output_0_to_model_layers_10_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:882baa78891c5e898bb385381b1c35c8a23bd1ac616646725722101fc1af7c90 +size 723984647 diff --git a/transformer_onnx/cfg_14_model_layers_10_Add_4_output_0_to_model_layers_11_Add_4_output_0_config.onnx b/transformer_onnx/cfg_14_model_layers_10_Add_4_output_0_to_model_layers_11_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..13a0b5336543b574003629e027192786c8eb82cb --- /dev/null +++ b/transformer_onnx/cfg_14_model_layers_10_Add_4_output_0_to_model_layers_11_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13e078b4f70b16489dfa2ec7abbdf1a78f7f111f254a0c0522e48f88213aa81e +size 723984652 diff --git a/transformer_onnx/cfg_15_model_layers_11_Add_4_output_0_to_model_layers_12_Add_4_output_0_config.onnx b/transformer_onnx/cfg_15_model_layers_11_Add_4_output_0_to_model_layers_12_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..38a0344bfe2e4b7db8d3e9bc0503c159ef27e216 --- /dev/null +++ b/transformer_onnx/cfg_15_model_layers_11_Add_4_output_0_to_model_layers_12_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f37a8f1147a8bd41417941d82022b3c8c0ee08340a79d7e37fbb51f35889b9a1 +size 723984652 diff --git a/transformer_onnx/cfg_16_model_layers_12_Add_4_output_0_to_model_layers_13_Add_4_output_0_config.onnx b/transformer_onnx/cfg_16_model_layers_12_Add_4_output_0_to_model_layers_13_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..1f6e1d54ac587dfb9b4ba4f7e6112a63957a8e6f --- /dev/null +++ b/transformer_onnx/cfg_16_model_layers_12_Add_4_output_0_to_model_layers_13_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e12af2dc2c6f4817b8e990a51c306904cbbe01c636f841d026908a8fadbfdceb +size 723984652 diff --git a/transformer_onnx/cfg_17_model_layers_13_Add_4_output_0_to_model_layers_14_Add_4_output_0_config.onnx b/transformer_onnx/cfg_17_model_layers_13_Add_4_output_0_to_model_layers_14_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..ef871e1173e11b72c0574583dde6d1f6dba0c143 --- /dev/null +++ b/transformer_onnx/cfg_17_model_layers_13_Add_4_output_0_to_model_layers_14_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc0b913780f6c8fbb21001df2fd367ace49a2244fc18ae8d172f9f501de7afd8 +size 723984652 diff --git a/transformer_onnx/cfg_18_model_layers_14_Add_4_output_0_to_model_layers_15_Add_4_output_0_config.onnx b/transformer_onnx/cfg_18_model_layers_14_Add_4_output_0_to_model_layers_15_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..33e2c264569a161f9aa8ae5af3a2f41088a99241 --- /dev/null +++ b/transformer_onnx/cfg_18_model_layers_14_Add_4_output_0_to_model_layers_15_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:428dae7a1e8344661b6e2c36446e38c3b2f05bc90b9411773c5abcb9dca5fac7 +size 723984652 diff --git a/transformer_onnx/cfg_19_model_layers_15_Add_4_output_0_to_model_layers_16_Add_4_output_0_config.onnx b/transformer_onnx/cfg_19_model_layers_15_Add_4_output_0_to_model_layers_16_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..73b718812e6802472c8cb6817ed13193741e6b88 --- /dev/null +++ b/transformer_onnx/cfg_19_model_layers_15_Add_4_output_0_to_model_layers_16_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:616563662bff3e3829c15c285f36bfc8e8484aa9d97e618134dd3e444f48554c +size 723984652 diff --git a/transformer_onnx/cfg_20_model_layers_16_Add_4_output_0_to_model_layers_17_Add_4_output_0_config.onnx b/transformer_onnx/cfg_20_model_layers_16_Add_4_output_0_to_model_layers_17_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..8654b822a1b7b731f056ff6094ad193d324c00bc --- /dev/null +++ b/transformer_onnx/cfg_20_model_layers_16_Add_4_output_0_to_model_layers_17_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a19d9fa435325302ef94dde8e0c2734448398efcbad99d4f33c50f6db77ac48e +size 723984652 diff --git a/transformer_onnx/cfg_21_model_layers_17_Add_4_output_0_to_model_layers_18_Add_4_output_0_config.onnx b/transformer_onnx/cfg_21_model_layers_17_Add_4_output_0_to_model_layers_18_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..b6934bd521f76a53e2b29f49fe7a4e15eeed9f56 --- /dev/null +++ b/transformer_onnx/cfg_21_model_layers_17_Add_4_output_0_to_model_layers_18_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9b6de460801e2e736a0d262b00532cdd167b2c35e80658c239faa384eecbadd +size 723984652 diff --git a/transformer_onnx/cfg_22_model_layers_18_Add_4_output_0_to_model_layers_19_Add_4_output_0_config.onnx b/transformer_onnx/cfg_22_model_layers_18_Add_4_output_0_to_model_layers_19_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..b5b58598053e7716be67a0807ec9a74b6d567f3c --- /dev/null +++ b/transformer_onnx/cfg_22_model_layers_18_Add_4_output_0_to_model_layers_19_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a25df910c614fcd15e478e5eee3543a7215631961686c86e3b7d82d3e07392a +size 723984652 diff --git a/transformer_onnx/cfg_23_model_layers_19_Add_4_output_0_to_model_layers_20_Add_4_output_0_config.onnx b/transformer_onnx/cfg_23_model_layers_19_Add_4_output_0_to_model_layers_20_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..e1f927fbc10ad5649732b4153a27ee633d91a55e --- /dev/null +++ b/transformer_onnx/cfg_23_model_layers_19_Add_4_output_0_to_model_layers_20_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7784212ba5a8d79ec763b5de851e4f905c9938b4860f4bc9e4e6ad38c4eb7b59 +size 723984652 diff --git a/transformer_onnx/cfg_24_model_layers_20_Add_4_output_0_to_model_layers_21_Add_4_output_0_config.onnx b/transformer_onnx/cfg_24_model_layers_20_Add_4_output_0_to_model_layers_21_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..96263fb7555bbd147ff128ed70493db85ff352a1 --- /dev/null +++ b/transformer_onnx/cfg_24_model_layers_20_Add_4_output_0_to_model_layers_21_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29c71e53f73e2dd24d3d95f284d19952e82e7332abeb07907d43e656e1c63257 +size 723984652 diff --git a/transformer_onnx/cfg_25_model_layers_21_Add_4_output_0_to_model_layers_22_Add_4_output_0_config.onnx b/transformer_onnx/cfg_25_model_layers_21_Add_4_output_0_to_model_layers_22_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..a2271b4e9fcdd45e7411a1a6aed22dc79302a394 --- /dev/null +++ b/transformer_onnx/cfg_25_model_layers_21_Add_4_output_0_to_model_layers_22_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5698919a13f4e69489b75d5caddf2599f5a110e2be28ece17c613ce337cb12ef +size 723984652 diff --git a/transformer_onnx/cfg_26_model_layers_22_Add_4_output_0_to_model_layers_23_Add_4_output_0_config.onnx b/transformer_onnx/cfg_26_model_layers_22_Add_4_output_0_to_model_layers_23_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..5b3a31e44061b3efdd1ef7e09cc6001daad380cc --- /dev/null +++ b/transformer_onnx/cfg_26_model_layers_22_Add_4_output_0_to_model_layers_23_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4b1b8c7428e660ea3549945ea0fbde65aabf4815b26a3514e921c9e246e0c21 +size 723984652 diff --git a/transformer_onnx/cfg_27_model_layers_23_Add_4_output_0_to_model_layers_24_Add_4_output_0_config.onnx b/transformer_onnx/cfg_27_model_layers_23_Add_4_output_0_to_model_layers_24_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..3008fe79817d77397ffa2eb98125f4a37da4ca33 --- /dev/null +++ b/transformer_onnx/cfg_27_model_layers_23_Add_4_output_0_to_model_layers_24_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f6dac61776e404c49867594e20ec6ee3eb8d4668887c0176661aef330b937775 +size 723984652 diff --git a/transformer_onnx/cfg_28_model_layers_24_Add_4_output_0_to_model_layers_25_Add_4_output_0_config.onnx b/transformer_onnx/cfg_28_model_layers_24_Add_4_output_0_to_model_layers_25_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..53e9a2fc5a7f2954ce020f54168d096aff539b9e --- /dev/null +++ b/transformer_onnx/cfg_28_model_layers_24_Add_4_output_0_to_model_layers_25_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b8958ce6434f86bef5ee8cda7d110f1a6683906c46dd3fff151284fc1537b340 +size 723984652 diff --git a/transformer_onnx/cfg_29_model_layers_25_Add_4_output_0_to_model_layers_26_Add_4_output_0_config.onnx b/transformer_onnx/cfg_29_model_layers_25_Add_4_output_0_to_model_layers_26_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..c8e2218255ecfa2bb8f29769ae36a388a6df8fcd --- /dev/null +++ b/transformer_onnx/cfg_29_model_layers_25_Add_4_output_0_to_model_layers_26_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0362d247265d331d1d5970eadc3cb99e91ae8ef0640ac6e09983ebec09f190f0 +size 723984652 diff --git a/transformer_onnx/cfg_30_model_layers_26_Add_4_output_0_to_model_layers_27_Add_4_output_0_config.onnx b/transformer_onnx/cfg_30_model_layers_26_Add_4_output_0_to_model_layers_27_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..d34f0ac9c18b4a72b21a4c41b4feeb1ae12804d3 --- /dev/null +++ b/transformer_onnx/cfg_30_model_layers_26_Add_4_output_0_to_model_layers_27_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81c05fd7fde897a8c9cd5fe6b54b9bfe1669938e751d3627817dde9a3f1885cc +size 723984652 diff --git a/transformer_onnx/cfg_31_model_layers_27_Add_4_output_0_to_model_layers_28_Add_4_output_0_config.onnx b/transformer_onnx/cfg_31_model_layers_27_Add_4_output_0_to_model_layers_28_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..e3fba60fa7b842a7b685696d9cbb29ed2aa95948 --- /dev/null +++ b/transformer_onnx/cfg_31_model_layers_27_Add_4_output_0_to_model_layers_28_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1be564ab67bb79281071a6914aed266a1aabcdf75806c768c0e1171b1d7f9c5a +size 723984652 diff --git a/transformer_onnx/cfg_32_model_layers_28_Add_4_output_0_to_model_layers_29_Add_4_output_0_config.onnx b/transformer_onnx/cfg_32_model_layers_28_Add_4_output_0_to_model_layers_29_Add_4_output_0_config.onnx new file mode 100644 index 0000000000000000000000000000000000000000..dcc2977edc7d01d2ca6347bed7e52318d9c3a410 --- /dev/null +++ b/transformer_onnx/cfg_32_model_layers_28_Add_4_output_0_to_model_layers_29_Add_4_output_0_config.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ede09f8a0dedd9ae9c98b9e9b365ff30f1fb52ec2ba05286f0749b09ad2ed02d +size 723984652 diff --git a/vae_model/vae_decoder.axmodel b/vae_model/vae_decoder.axmodel new file mode 100644 index 0000000000000000000000000000000000000000..1ea06e0a104479b4d88e16144ec011a533757ae9 --- /dev/null +++ b/vae_model/vae_decoder.axmodel @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0798299fccdffae63be19d73323279e0f4b83883f634faaa88a44a9fe09ccc19 +size 71404143 diff --git a/vae_model/vae_decoder_simp_slim.onnx b/vae_model/vae_decoder_simp_slim.onnx new file mode 100644 index 0000000000000000000000000000000000000000..9e5267fbed52aa4844170b51d3199c1714fad2df --- /dev/null +++ b/vae_model/vae_decoder_simp_slim.onnx @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cf1511e996fed688f16a7d6d5e8301c2aad01253e76b31766497f185a245a6cc +size 198281164